<s> from __future__ import unicode_literals <EOL> from builtins import str <EOL> import datetime , json , logging , os <EOL> from scrapy import signals <EOL> from scrapy . spiders import Spider <EOL> from scrapy . spiders import CrawlSpider <EOL> from scrapy . xlib . pydispatch import dispatcher <EOL> from scrapy . exceptions import CloseSpider <EOL> import django <EOL> django . setup ( ) <EOL> from django . core . exceptions import ObjectDoesNotExist <EOL> from django . db . models import Q <EOL> from dynamic_scraper . models import Log , LogMarker <EOL> class DjangoBaseSpider ( CrawlSpider ) : <EOL> name = None <EOL> action_successful = False <EOL> mandatory_vars = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , ] <EOL> allowed_domains = [ ] <EOL> start_urls = [ ] <EOL> conf = { <EOL> "<STR_LIT>" : False , <EOL> "<STR_LIT>" : '<STR_LIT>' , <EOL> "<STR_LIT>" : { } , <EOL> "<STR_LIT>" : '<STR_LIT>' , <EOL> "<STR_LIT>" : True , <EOL> "<STR_LIT>" : '<STR_LIT>' , <EOL> "<STR_LIT>" : <NUM_LIT> , <EOL> } <EOL> mp_request_kwargs = { } <EOL> dp_request_kwargs = { } <EOL> command = '<STR_LIT>' <EOL> command += '<STR_LIT>' <EOL> command += '<STR_LIT>' <EOL> command += '<STR_LIT>' <EOL> def __init__ ( self , * args , ** kwargs ) : <EOL> msg = "<STR_LIT>" . format ( s = os . environ . get ( "<STR_LIT>" ) ) <EOL> logging . info ( msg ) <EOL> super ( DjangoBaseSpider , self ) . __init__ ( None , ** kwargs ) <EOL> self . _check_mandatory_vars ( ) <EOL> def _set_ref_object ( self , ref_object_class , ** kwargs ) : <EOL> if not '<STR_LIT:id>' in kwargs : <EOL> msg = "<STR_LIT>" . format ( c = self . command ) <EOL> logging . error ( msg ) <EOL> raise CloseSpider ( msg ) <EOL> try : <EOL> self . ref_object = ref_object_class . objects . get ( pk = kwargs [ '<STR_LIT:id>' ] ) <EOL> except ObjectDoesNotExist : <EOL> msg = "<STR_LIT>" . format ( <EOL> id = kwargs [ '<STR_LIT:id>' ] , c = self . command ) <EOL> logging . error ( msg ) <EOL> raise CloseSpider ( msg ) <EOL> def _set_config ( self , log_msg , ** kwargs ) : <EOL> from scrapy . utils . project import get_project_settings <EOL> settings = get_project_settings ( ) <EOL> if '<STR_LIT>' in kwargs : <EOL> self . conf [ '<STR_LIT>' ] = kwargs [ '<STR_LIT>' ] <EOL> if len ( log_msg ) > <NUM_LIT:0> : <EOL> log_msg += "<STR_LIT:U+002CU+0020>" <EOL> log_msg += "<STR_LIT>" + self . conf [ '<STR_LIT>' ] <EOL> if '<STR_LIT>' in kwargs : <EOL> if kwargs [ '<STR_LIT>' ] == '<STR_LIT:yes>' : <EOL> self . conf [ '<STR_LIT>' ] = True <EOL> else : <EOL> self . conf [ '<STR_LIT>' ] = False <EOL> if len ( log_msg ) > <NUM_LIT:0> : <EOL> log_msg += "<STR_LIT:U+002CU+0020>" <EOL> log_msg += "<STR_LIT>" + str ( self . conf [ '<STR_LIT>' ] ) <EOL> self . conf [ '<STR_LIT>' ] = settings . get ( '<STR_LIT>' , self . conf [ '<STR_LIT>' ] ) <EOL> if '<STR_LIT>' not in self . conf [ '<STR_LIT>' ] : <EOL> self . conf [ '<STR_LIT>' ] [ '<STR_LIT>' ] = <NUM_LIT:0.5> <EOL> self . conf [ '<STR_LIT>' ] = settings . get ( '<STR_LIT>' , self . conf [ '<STR_LIT>' ] ) <EOL> if self . conf [ "<STR_LIT>" ] == '<STR_LIT>' : <EOL> msg = "<STR_LIT>" <EOL> logging . info ( msg ) <EOL> if settings . get ( '<STR_LIT>' ) and len ( settings . get ( '<STR_LIT>' ) ) > <NUM_LIT:0> : <EOL> msg = "<STR_LIT>" <EOL> msg += "<STR_LIT>" . format ( <EOL> size = next ( iter ( settings . get ( '<STR_LIT>' ) . keys ( ) ) ) ) <EOL> else : <EOL> msg = "<STR_LIT>" <EOL> logging . info ( msg ) <EOL> elif self . conf [ "<STR_LIT>" ] == '<STR_LIT>' : <EOL> msg = "<STR_LIT>" <EOL> logging . info ( msg ) <EOL> else : <EOL> msg = "<STR_LIT>" <EOL> logging . info ( msg ) <EOL> self . conf [ '<STR_LIT>' ] = settings . get ( '<STR_LIT>' , self . conf [ '<STR_LIT>' ] ) <EOL> self . conf [ '<STR_LIT>' ] = settings . get ( '<STR_LIT>' , self . conf [ '<STR_LIT>' ] ) <EOL> self . conf [ '<STR_LIT>' ] = settings . get ( '<STR_LIT>' , self . conf [ '<STR_LIT>' ] ) <EOL> self . log ( "<STR_LIT>" + log_msg , logging . INFO ) <EOL> dispatcher . connect ( self . spider_closed , signal = signals . spider_closed ) <EOL> def _check_mandatory_vars ( self ) : <EOL> if self . conf [ '<STR_LIT>' ] == '<STR_LIT>' : <EOL> if not getattr ( self , '<STR_LIT>' , None ) : <EOL> msg = "<STR_LIT>" <EOL> logging . error ( msg ) <EOL> raise CloseSpider ( msg ) <EOL> msg = "<STR_LIT>" + str ( self . scheduler_runtime ) + "<STR_LIT>" <EOL> self . log ( msg , logging . INFO ) <EOL> for var in self . mandatory_vars : <EOL> attr = getattr ( self , var , None ) <EOL> if not attr : <EOL> msg = "<STR_LIT>" . format ( a = var , c = self . command ) <EOL> logging . error ( msg ) <EOL> raise CloseSpider ( msg ) <EOL> if self . scraper . status == '<STR_LIT:P>' or self . scraper . status == '<STR_LIT:I>' : <EOL> msg = '<STR_LIT>' . format ( s = self . scraper . get_status_display ( ) ) <EOL> self . log ( msg , logging . WARNING ) <EOL> raise CloseSpider ( msg ) <EOL> def _set_request_kwargs ( self ) : <EOL> try : <EOL> self . scraper . get_main_page_rpt ( ) <EOL> except ObjectDoesNotExist : <EOL> raise CloseSpider ( "<STR_LIT>" ) <EOL> for rpt in self . scraper . requestpagetype_set . all ( ) : <EOL> if rpt . page_type == '<STR_LIT>' : <EOL> pt_dict = self . mp_request_kwargs <EOL> else : <EOL> self . dp_request_kwargs [ rpt . page_type ] = { } <EOL> pt_dict = self . dp_request_kwargs [ rpt . page_type ] <EOL> if rpt . headers != '<STR_LIT>' : <EOL> try : <EOL> headers = json . loads ( rpt . headers ) <EOL> except ValueError : <EOL> raise CloseSpider ( "<STR_LIT>" . format ( a = rpt . page_type ) ) <EOL> if not isinstance ( headers , dict ) : <EOL> raise CloseSpider ( "<STR_LIT>" . format ( a = rpt . page_type ) ) <EOL> pt_dict [ '<STR_LIT>' ] = headers <EOL> if rpt . body != '<STR_LIT>' : <EOL> pt_dict [ '<STR_LIT:body>' ] = rpt . body <EOL> if rpt . cookies != '<STR_LIT>' : <EOL> try : <EOL> cookies = json . loads ( rpt . cookies ) <EOL> except ValueError : <EOL> raise CloseSpider ( "<STR_LIT>" . format ( a = rpt . page_type ) ) <EOL> if not isinstance ( cookies , dict ) : <EOL> raise CloseSpider ( "<STR_LIT>" . format ( a = rpt . page_type ) ) <EOL> pt_dict [ '<STR_LIT>' ] = cookies <EOL> if rpt . meta != '<STR_LIT>' : <EOL> try : <EOL> meta = json . loads ( rpt . meta ) <EOL> except ValueError : <EOL> raise CloseSpider ( "<STR_LIT>" . format ( a = rpt . page_type ) ) <EOL> if not isinstance ( meta , dict ) : <EOL> raise CloseSpider ( "<STR_LIT>" . format ( a = rpt . page_type ) ) <EOL> pt_dict [ '<STR_LIT>' ] = meta <EOL> def _set_meta_splash_args ( self ) : <EOL> for rpt in self . scraper . requestpagetype_set . all ( ) : <EOL> if rpt . page_type == '<STR_LIT>' : <EOL> pt_dict = self . mp_request_kwargs <EOL> else : <EOL> pt_dict = self . dp_request_kwargs [ rpt . page_type ] <EOL> if rpt . content_type == '<STR_LIT:H>' and rpt . render_javascript : <EOL> if '<STR_LIT>' not in pt_dict : <EOL> pt_dict [ '<STR_LIT>' ] = { } <EOL> pt_dict [ '<STR_LIT>' ] [ '<STR_LIT>' ] = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT:args>' : self . conf [ '<STR_LIT>' ] . copy ( ) <EOL> } <EOL> def spider_closed ( self ) : <EOL> if self . conf [ '<STR_LIT>' ] == '<STR_LIT>' and self . conf [ '<STR_LIT>' ] : <EOL> time_delta , factor , num_crawls = self . scheduler . calc_next_action_time ( self . action_successful , self . scheduler_runtime . next_action_factor , self . scheduler_runtime . num_zero_actions ) <EOL> self . scheduler_runtime . next_action_time = datetime . datetime . now ( ) + time_delta <EOL> self . scheduler_runtime . next_action_factor = factor <EOL> self . scheduler_runtime . num_zero_actions = num_crawls <EOL> self . scheduler_runtime . save ( ) <EOL> msg = "<STR_LIT>" <EOL> msg += "<STR_LIT>" . format ( nat = str ( self . scheduler_runtime . next_action_time . strftime ( "<STR_LIT>" ) ) ) <EOL> msg += "<STR_LIT>" . format ( naf = str ( self . scheduler_runtime . next_action_factor ) ) <EOL> msg += "<STR_LIT>" . format ( za = str ( self . scheduler_runtime . num_zero_actions ) ) <EOL> self . log ( msg , logging . INFO ) <EOL> def log ( self , message , level = logging . DEBUG ) : <EOL> if self . conf [ '<STR_LIT>' ] == '<STR_LIT>' and self . conf [ '<STR_LIT>' ] : <EOL> if self . conf [ '<STR_LIT>' ] and level >= getattr ( logging , self . conf [ '<STR_LIT>' ] ) : <EOL> l = Log ( ) <EOL> l . message = message <EOL> l . ref_object = self . ref_object . __class__ . __name__ + "<STR_LIT>" + str ( self . ref_object . pk ) + "<STR_LIT:)>" <EOL> l . type = '<STR_LIT:None>' <EOL> l . level = int ( level ) <EOL> l . spider_name = self . name <EOL> l . scraper = self . scraper <EOL> lms = LogMarker . objects . filter ( <EOL> Q ( ref_object = l . ref_object ) | Q ( ref_object = '<STR_LIT>' ) , <EOL> Q ( spider_name = l . spider_name ) | Q ( spider_name = '<STR_LIT>' ) , <EOL> Q ( scraper = l . scraper ) | Q ( scraper__isnull = True ) , <EOL> ) <EOL> for lm in lms : <EOL> if lm . message_contains in l . message : <EOL> if lm . custom_type : <EOL> l . type = lm . custom_type <EOL> else : <EOL> l . type = lm . get_mark_with_type_display ( ) <EOL> l . save ( ) <EOL> if Log . objects . count ( ) > self . conf [ '<STR_LIT>' ] : <EOL> items = Log . objects . all ( ) [ self . conf [ '<STR_LIT>' ] : ] <EOL> for item in items : <EOL> item . delete ( ) <EOL> logging . log ( level , message ) </s>
