<s> from __future__ import absolute_import <EOL> from datetime import datetime , timedelta <EOL> from grab . spider . error import SpiderMisuseError <EOL> from grab . base import copy_config <EOL> class BaseTask ( object ) : <EOL> pass <EOL> class Task ( BaseTask ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , name = None , url = None , grab = None , grab_config = None , <EOL> priority = None , priority_is_custom = True , <EOL> network_try_count = <NUM_LIT:0> , task_try_count = <NUM_LIT:1> , <EOL> disable_cache = False , refresh_cache = False , <EOL> valid_status = [ ] , use_proxylist = True , <EOL> cache_timeout = None , delay = <NUM_LIT:0> , <EOL> raw = False , callback = None , <EOL> fallback_name = None , <EOL> ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> if name == '<STR_LIT>' : <EOL> raise SpiderMisuseError ( '<STR_LIT>' ) <EOL> self . name = name <EOL> if url is None and grab is None and grab_config is None : <EOL> raise SpiderMisuseError ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> if url is not None and grab is not None : <EOL> raise SpiderMisuseError ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> if url is not None and grab_config is not None : <EOL> raise SpiderMisuseError ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> if grab is not None and grab_config is not None : <EOL> raise SpiderMisuseError ( <EOL> '<STR_LIT>' ) <EOL> if grab : <EOL> self . setup_grab_config ( grab . dump_config ( ) ) <EOL> elif grab_config : <EOL> self . setup_grab_config ( grab_config ) <EOL> else : <EOL> self . grab_config = None <EOL> self . url = url <EOL> self . process_delay_option ( delay ) <EOL> self . fallback_name = fallback_name <EOL> self . priority_is_custom = priority_is_custom <EOL> self . priority = priority <EOL> self . network_try_count = network_try_count <EOL> self . task_try_count = task_try_count <EOL> self . disable_cache = disable_cache <EOL> self . refresh_cache = refresh_cache <EOL> self . valid_status = valid_status <EOL> self . use_proxylist = use_proxylist <EOL> self . cache_timeout = cache_timeout <EOL> self . raw = raw <EOL> self . origin_task_generator = None <EOL> self . callback = callback <EOL> self . coroutines_stack = [ ] <EOL> for key , value in kwargs . items ( ) : <EOL> setattr ( self , key , value ) <EOL> def get ( self , key , default = None ) : <EOL> """<STR_LIT>""" <EOL> return getattr ( self , key , default ) <EOL> def process_delay_option ( self , delay ) : <EOL> if delay : <EOL> self . schedule_time = datetime . utcnow ( ) + timedelta ( seconds = delay ) <EOL> self . original_delay = delay <EOL> else : <EOL> self . schedule_time = None <EOL> self . original_delay = None <EOL> def setup_grab_config ( self , grab_config ) : <EOL> self . grab_config = copy_config ( grab_config ) <EOL> self . url = grab_config [ '<STR_LIT:url>' ] <EOL> def clone ( self , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> attr_copy = self . __dict__ . copy ( ) <EOL> if attr_copy . get ( '<STR_LIT>' ) is not None : <EOL> del attr_copy [ '<STR_LIT:url>' ] <EOL> task = Task ( ** attr_copy ) <EOL> if '<STR_LIT>' not in kwargs : <EOL> task . network_try_count = <NUM_LIT:0> <EOL> if '<STR_LIT>' not in kwargs : <EOL> task . task_try_count = self . task_try_count + <NUM_LIT:1> <EOL> if '<STR_LIT>' not in kwargs : <EOL> task . refresh_cache = False <EOL> if '<STR_LIT>' not in kwargs : <EOL> task . disable_cache = False <EOL> if kwargs . get ( '<STR_LIT:url>' ) is not None and kwargs . get ( '<STR_LIT>' ) is not None : <EOL> raise SpiderMisuseError ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> if ( kwargs . get ( '<STR_LIT:url>' ) is not None and <EOL> kwargs . get ( '<STR_LIT>' ) is not None ) : <EOL> raise SpiderMisuseError ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> if ( kwargs . get ( '<STR_LIT>' ) is not None and <EOL> kwargs . get ( '<STR_LIT>' ) is not None ) : <EOL> raise SpiderMisuseError ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> if kwargs . get ( '<STR_LIT>' ) : <EOL> task . setup_grab_config ( kwargs [ '<STR_LIT>' ] . dump_config ( ) ) <EOL> del kwargs [ '<STR_LIT>' ] <EOL> elif kwargs . get ( '<STR_LIT>' ) : <EOL> task . setup_grab_config ( kwargs [ '<STR_LIT>' ] ) <EOL> del kwargs [ '<STR_LIT>' ] <EOL> elif kwargs . get ( '<STR_LIT:url>' ) : <EOL> task . url = kwargs [ '<STR_LIT:url>' ] <EOL> if task . grab_config : <EOL> task . grab_config [ '<STR_LIT:url>' ] = kwargs [ '<STR_LIT:url>' ] <EOL> del kwargs [ '<STR_LIT:url>' ] <EOL> for key , value in kwargs . items ( ) : <EOL> setattr ( task , key , value ) <EOL> task . process_delay_option ( task . get ( '<STR_LIT>' , None ) ) <EOL> return task <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % self . url <EOL> def __lt__ ( self , other ) : <EOL> return self . priority < other . priority <EOL> def __eq__ ( self , other ) : <EOL> if not self . priority or not other . priority : <EOL> return True <EOL> else : <EOL> return self . priority == other . priority <EOL> def get_fallback_handler ( self , spider ) : <EOL> if self . fallback_name : <EOL> return getattr ( spider , self . fallback_name ) <EOL> elif self . name : <EOL> fb_name = '<STR_LIT>' % self . name <EOL> if hasattr ( spider , fb_name ) : <EOL> return getattr ( spider , fb_name ) <EOL> else : <EOL> return None <EOL> def inline_task ( f ) : <EOL> def wrap ( self , grab , task ) : <EOL> origin_task_generator = f ( self , grab , task ) <EOL> self . handler_for_inline_task ( None , origin_task_generator ) <EOL> return wrap </s>
