def process_item(self, item, spider):  # pylint: disable=unused-argument
        """
        Stops processing item if we've already seen this URL before.
        """
        url = self.clean_url(item["url"])

        if self.is_sequence_start_page(url):
            url = url.parent

        if url in self.urls_seen:
            raise DropItem(u"Dropping duplicate url {url}".format(url=item["url"]))
        else:
            self.urls_seen.add(url)
            return item
def process_item ( self , item , spider ) : url = self . clean_url ( item [ "url" ] ) if self . is_sequence_start_page ( url ) : url = url . parent if url in self . urls_seen : raise DropItem ( u"Dropping duplicate url {url}" . format ( url = item [ "url" ] ) ) else : self . urls_seen . add ( url ) return item
Stops processing item if we've already seen this URL before.