<s> import os <EOL> import json <EOL> import logging <EOL> from os . path import join , exists <EOL> from scrapy . utils . reqser import request_to_dict , request_from_dict <EOL> from scrapy . utils . misc import load_object <EOL> from scrapy . utils . job import job_dir <EOL> logger = logging . getLogger ( __name__ ) <EOL> class Scheduler ( object ) : <EOL> def __init__ ( self , dupefilter , jobdir = None , dqclass = None , mqclass = None , <EOL> logunser = False , stats = None , pqclass = None ) : <EOL> self . df = dupefilter <EOL> self . dqdir = self . _dqdir ( jobdir ) <EOL> self . pqclass = pqclass <EOL> self . dqclass = dqclass <EOL> self . mqclass = mqclass <EOL> self . logunser = logunser <EOL> self . stats = stats <EOL> @ classmethod <EOL> def from_crawler ( cls , crawler ) : <EOL> settings = crawler . settings <EOL> dupefilter_cls = load_object ( settings [ '<STR_LIT>' ] ) <EOL> dupefilter = dupefilter_cls . from_settings ( settings ) <EOL> pqclass = load_object ( settings [ '<STR_LIT>' ] ) <EOL> dqclass = load_object ( settings [ '<STR_LIT>' ] ) <EOL> mqclass = load_object ( settings [ '<STR_LIT>' ] ) <EOL> logunser = settings . getbool ( '<STR_LIT>' ) <EOL> return cls ( dupefilter , jobdir = job_dir ( settings ) , logunser = logunser , <EOL> stats = crawler . stats , pqclass = pqclass , dqclass = dqclass , mqclass = mqclass ) <EOL> def has_pending_requests ( self ) : <EOL> return len ( self ) > <NUM_LIT:0> <EOL> def open ( self , spider ) : <EOL> self . spider = spider <EOL> self . mqs = self . pqclass ( self . _newmq ) <EOL> self . dqs = self . _dq ( ) if self . dqdir else None <EOL> return self . df . open ( ) <EOL> def close ( self , reason ) : <EOL> if self . dqs : <EOL> prios = self . dqs . close ( ) <EOL> with open ( join ( self . dqdir , '<STR_LIT>' ) , '<STR_LIT:w>' ) as f : <EOL> json . dump ( prios , f ) <EOL> return self . df . close ( reason ) <EOL> def enqueue_request ( self , request ) : <EOL> if not request . dont_filter and self . df . request_seen ( request ) : <EOL> self . df . log ( request , self . spider ) <EOL> return False <EOL> dqok = self . _dqpush ( request ) <EOL> if dqok : <EOL> self . stats . inc_value ( '<STR_LIT>' , spider = self . spider ) <EOL> else : <EOL> self . _mqpush ( request ) <EOL> self . stats . inc_value ( '<STR_LIT>' , spider = self . spider ) <EOL> self . stats . inc_value ( '<STR_LIT>' , spider = self . spider ) <EOL> return True <EOL> def next_request ( self ) : <EOL> request = self . mqs . pop ( ) <EOL> if request : <EOL> self . stats . inc_value ( '<STR_LIT>' , spider = self . spider ) <EOL> else : <EOL> request = self . _dqpop ( ) <EOL> if request : <EOL> self . stats . inc_value ( '<STR_LIT>' , spider = self . spider ) <EOL> if request : <EOL> self . stats . inc_value ( '<STR_LIT>' , spider = self . spider ) <EOL> return request <EOL> def __len__ ( self ) : <EOL> return len ( self . dqs ) + len ( self . mqs ) if self . dqs else len ( self . mqs ) <EOL> def _dqpush ( self , request ) : <EOL> if self . dqs is None : <EOL> return <EOL> try : <EOL> reqd = request_to_dict ( request , self . spider ) <EOL> self . dqs . push ( reqd , - request . priority ) <EOL> except ValueError as e : <EOL> if self . logunser : <EOL> logger . error ( "<STR_LIT>" , <EOL> { '<STR_LIT>' : request , '<STR_LIT>' : e } , <EOL> exc_info = True , extra = { '<STR_LIT>' : self . spider } ) <EOL> return <EOL> else : <EOL> return True <EOL> def _mqpush ( self , request ) : <EOL> self . mqs . push ( request , - request . priority ) <EOL> def _dqpop ( self ) : <EOL> if self . dqs : <EOL> d = self . dqs . pop ( ) <EOL> if d : <EOL> return request_from_dict ( d , self . spider ) <EOL> def _newmq ( self , priority ) : <EOL> return self . mqclass ( ) <EOL> def _newdq ( self , priority ) : <EOL> return self . dqclass ( join ( self . dqdir , '<STR_LIT>' % priority ) ) <EOL> def _dq ( self ) : <EOL> activef = join ( self . dqdir , '<STR_LIT>' ) <EOL> if exists ( activef ) : <EOL> with open ( activef ) as f : <EOL> prios = json . load ( f ) <EOL> else : <EOL> prios = ( ) <EOL> q = self . pqclass ( self . _newdq , startprios = prios ) <EOL> if q : <EOL> logger . info ( "<STR_LIT>" , <EOL> { '<STR_LIT>' : len ( q ) } , extra = { '<STR_LIT>' : self . spider } ) <EOL> return q <EOL> def _dqdir ( self , jobdir ) : <EOL> if jobdir : <EOL> dqdir = join ( jobdir , '<STR_LIT>' ) <EOL> if not exists ( dqdir ) : <EOL> os . makedirs ( dqdir ) <EOL> return dqdir </s>
