url去重

def filter_duplicate(self, url):
        """
        url去重
        """
        if self.filterDuplicate:
            if url in self.historys: 
                raise Exception('duplicate excepiton: %s is duplicate' % url)
            else:
                self.historys.add(url)
        else:
            pass