<s> from scrapy . exceptions import DropItem <EOL> from scrapy_sci . status import Status , Reader <EOL> class DuplicatesPipeline ( object ) : <EOL> def __init__ ( self ) : <EOL> self . ids_seen = set ( ) <EOL> status = Status ( ) <EOL> for classifier in status . classifiers . keys ( ) : <EOL> for rf in status . classifiers [ classifier ] [ '<STR_LIT>' ] : <EOL> json = Reader . read_reviewed ( rf ) <EOL> self . ids_seen . add ( json [ '<STR_LIT>' ] ) <EOL> def process_item ( self , item , spider ) : <EOL> if item [ '<STR_LIT>' ] in self . ids_seen : <EOL> raise DropItem ( "<STR_LIT>" % item ) <EOL> else : <EOL> self . ids_seen . add ( item [ '<STR_LIT>' ] ) <EOL> return item </s>
