<s> from __future__ import unicode_literals <EOL> from builtins import str <EOL> import logging <EOL> from scrapy import signals <EOL> from scrapy . exceptions import CloseSpider <EOL> from scrapy . http import Request <EOL> from scrapy . xlib . pydispatch import dispatcher <EOL> from dynamic_scraper . spiders . django_base_spider import DjangoBaseSpider <EOL> from dynamic_scraper . models import Scraper <EOL> class CheckerTest ( DjangoBaseSpider ) : <EOL> name = '<STR_LIT>' <EOL> mandatory_vars = [ '<STR_LIT>' , '<STR_LIT>' , ] <EOL> command = '<STR_LIT>' <EOL> def __init__ ( self , * args , ** kwargs ) : <EOL> self . _set_ref_object ( Scraper , ** kwargs ) <EOL> self . scraper = self . ref_object <EOL> self . _set_config ( ** kwargs ) <EOL> if self . scraper . checker_set . count ( ) == <NUM_LIT:0> : <EOL> msg = "<STR_LIT>" <EOL> logging . error ( msg ) <EOL> raise CloseSpider ( msg ) <EOL> for checker in self . scraper . checker_set . all ( ) : <EOL> if checker . checker_type == '<STR_LIT:4>' : <EOL> if not checker . checker_ref_url : <EOL> msg = "<STR_LIT>" . format ( c = str ( checker ) , cr = self . command ) <EOL> logging . error ( msg ) <EOL> raise CloseSpider ( msg ) <EOL> if checker . checker_type == '<STR_LIT:X>' : <EOL> if not checker . checker_x_path or not checker . checker_ref_url : <EOL> msg = "<STR_LIT>" . format ( c = str ( checker ) , cr = self . command ) <EOL> logging . error ( msg ) <EOL> raise CloseSpider ( msg ) <EOL> self . _set_request_kwargs ( ) <EOL> self . _set_meta_splash_args ( ) <EOL> dispatcher . connect ( self . response_received , signal = signals . response_received ) <EOL> def _set_config ( self , ** kwargs ) : <EOL> log_msg = "<STR_LIT>" <EOL> super ( CheckerTest , self ) . _set_config ( log_msg , ** kwargs ) <EOL> def spider_closed ( self ) : <EOL> pass <EOL> def start_requests ( self ) : <EOL> for checker in self . scraper . checker_set . all ( ) : <EOL> url = checker . checker_ref_url <EOL> rpt = self . scraper . get_rpt_for_scraped_obj_attr ( checker . scraped_obj_attr ) <EOL> kwargs = self . dp_request_kwargs [ rpt . page_type ] . copy ( ) <EOL> if '<STR_LIT>' not in kwargs : <EOL> kwargs [ '<STR_LIT>' ] = { } <EOL> kwargs [ '<STR_LIT>' ] [ '<STR_LIT>' ] = checker <EOL> kwargs [ '<STR_LIT>' ] [ '<STR_LIT>' ] = rpt <EOL> self . _set_meta_splash_args ( ) <EOL> if url : <EOL> if rpt . request_type == '<STR_LIT:R>' : <EOL> yield Request ( url , callback = self . parse , method = rpt . method , dont_filter = True , ** kwargs ) <EOL> else : <EOL> yield FormRequest ( url , callback = self . parse , method = rpt . method , formdata = self . dp_form_data [ rpt . page_type ] , dont_filter = True , ** kwargs ) <EOL> def response_received ( self , ** kwargs ) : <EOL> checker = kwargs [ '<STR_LIT>' ] . request . meta [ '<STR_LIT>' ] <EOL> rpt = kwargs [ '<STR_LIT>' ] . request . meta [ '<STR_LIT>' ] <EOL> if kwargs [ '<STR_LIT>' ] . status == <NUM_LIT> : <EOL> if checker . checker_type == '<STR_LIT:4>' : <EOL> self . log ( "<STR_LIT>" . format ( c = str ( checker ) ) , logging . INFO ) <EOL> if checker . checker_type == '<STR_LIT:X>' : <EOL> self . log ( '<STR_LIT>' . format ( c = str ( checker ) ) , logging . WARNING ) <EOL> else : <EOL> if checker . checker_type == '<STR_LIT:4>' : <EOL> self . log ( '<STR_LIT>' . format ( c = str ( checker ) ) , logging . WARNING ) <EOL> def parse ( self , response ) : <EOL> checker = response . request . meta [ '<STR_LIT>' ] <EOL> rpt = response . request . meta [ '<STR_LIT>' ] <EOL> if checker . checker_type == '<STR_LIT:4>' : <EOL> return <EOL> try : <EOL> test_select = response . xpath ( checker . checker_x_path ) . extract ( ) <EOL> except ValueError : <EOL> self . log ( '<STR_LIT>' . format ( c = str ( checker ) ) , logging . ERROR ) <EOL> return <EOL> if len ( test_select ) == <NUM_LIT:0> : <EOL> self . log ( "<STR_LIT>" . format ( c = str ( checker ) ) , logging . ERROR ) <EOL> else : <EOL> if checker . checker_x_path_result == '<STR_LIT>' : <EOL> self . log ( "<STR_LIT>" . format ( c = str ( checker ) ) , logging . INFO ) <EOL> else : <EOL> if test_select [ <NUM_LIT:0> ] != checker . checker_x_path_result : <EOL> self . log ( "<STR_LIT>" . format ( c = str ( checker ) ) , logging . ERROR ) <EOL> else : <EOL> self . log ( "<STR_LIT>" . format ( c = str ( checker ) ) , logging . INFO ) </s>
