<s> from __future__ import print_function <EOL> import os <EOL> import logging <EOL> from scrapy . utils . job import job_dir <EOL> from scrapy . utils . request import request_fingerprint <EOL> class BaseDupeFilter ( object ) : <EOL> @ classmethod <EOL> def from_settings ( cls , settings ) : <EOL> return cls ( ) <EOL> def request_seen ( self , request ) : <EOL> return False <EOL> def open ( self ) : <EOL> pass <EOL> def close ( self , reason ) : <EOL> pass <EOL> def log ( self , request , spider ) : <EOL> pass <EOL> class RFPDupeFilter ( BaseDupeFilter ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , path = None , debug = False ) : <EOL> self . file = None <EOL> self . fingerprints = set ( ) <EOL> self . logdupes = True <EOL> self . debug = debug <EOL> self . logger = logging . getLogger ( __name__ ) <EOL> if path : <EOL> self . file = open ( os . path . join ( path , '<STR_LIT>' ) , '<STR_LIT>' ) <EOL> self . file . seek ( <NUM_LIT:0> ) <EOL> self . fingerprints . update ( x . rstrip ( ) for x in self . file ) <EOL> @ classmethod <EOL> def from_settings ( cls , settings ) : <EOL> debug = settings . getbool ( '<STR_LIT>' ) <EOL> return cls ( job_dir ( settings ) , debug ) <EOL> def request_seen ( self , request ) : <EOL> fp = self . request_fingerprint ( request ) <EOL> if fp in self . fingerprints : <EOL> return True <EOL> self . fingerprints . add ( fp ) <EOL> if self . file : <EOL> self . file . write ( fp + os . linesep ) <EOL> def request_fingerprint ( self , request ) : <EOL> return request_fingerprint ( request ) <EOL> def close ( self , reason ) : <EOL> if self . file : <EOL> self . file . close ( ) <EOL> def log ( self , request , spider ) : <EOL> if self . debug : <EOL> msg = "<STR_LIT>" <EOL> self . logger . debug ( msg , { '<STR_LIT>' : request } , extra = { '<STR_LIT>' : spider } ) <EOL> elif self . logdupes : <EOL> msg = ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> self . logger . debug ( msg , { '<STR_LIT>' : request } , extra = { '<STR_LIT>' : spider } ) <EOL> self . logdupes = False <EOL> spider . crawler . stats . inc_value ( '<STR_LIT>' , spider = spider ) </s>
