<s> """<STR_LIT>""" <EOL> import logging <EOL> from scrapy . http import Request <EOL> logger = logging . getLogger ( __name__ ) <EOL> class DepthMiddleware ( object ) : <EOL> def __init__ ( self , maxdepth , stats = None , verbose_stats = False , prio = <NUM_LIT:1> ) : <EOL> self . maxdepth = maxdepth <EOL> self . stats = stats <EOL> self . verbose_stats = verbose_stats <EOL> self . prio = prio <EOL> @ classmethod <EOL> def from_crawler ( cls , crawler ) : <EOL> settings = crawler . settings <EOL> maxdepth = settings . getint ( '<STR_LIT>' ) <EOL> verbose = settings . getbool ( '<STR_LIT>' ) <EOL> prio = settings . getint ( '<STR_LIT>' ) <EOL> return cls ( maxdepth , crawler . stats , verbose , prio ) <EOL> def process_spider_output ( self , response , result , spider ) : <EOL> def _filter ( request ) : <EOL> if isinstance ( request , Request ) : <EOL> depth = response . meta [ '<STR_LIT>' ] + <NUM_LIT:1> <EOL> request . meta [ '<STR_LIT>' ] = depth <EOL> if self . prio : <EOL> request . priority -= depth * self . prio <EOL> if self . maxdepth and depth > self . maxdepth : <EOL> logger . debug ( <EOL> "<STR_LIT>" , <EOL> { '<STR_LIT>' : self . maxdepth , '<STR_LIT>' : request . url } , <EOL> extra = { '<STR_LIT>' : spider } <EOL> ) <EOL> return False <EOL> elif self . stats : <EOL> if self . verbose_stats : <EOL> self . stats . inc_value ( '<STR_LIT>' % depth , <EOL> spider = spider ) <EOL> self . stats . max_value ( '<STR_LIT>' , depth , <EOL> spider = spider ) <EOL> return True <EOL> if self . stats and '<STR_LIT>' not in response . meta : <EOL> response . meta [ '<STR_LIT>' ] = <NUM_LIT:0> <EOL> if self . verbose_stats : <EOL> self . stats . inc_value ( '<STR_LIT>' , spider = spider ) <EOL> return ( r for r in result or ( ) if _filter ( r ) ) </s>
