<s> from __future__ import unicode_literals <EOL> import os . path , sys <EOL> from scraper . models import Event <EOL> from scraper . scraper_test import EventSpider , ScraperTest <EOL> from dynamic_scraper . models import Checker , RequestPageType , ScraperElem , SchedulerRuntime <EOL> class ScraperJSONRunTest ( ScraperTest ) : <EOL> def setUpScraperJSONDefaultScraper ( self ) : <EOL> self . se_base . x_path = '<STR_LIT>' <EOL> self . se_base . save ( ) <EOL> self . se_title . x_path = '<STR_LIT:title>' <EOL> self . se_title . save ( ) <EOL> self . se_url . x_path = '<STR_LIT:url>' <EOL> self . se_url . save ( ) <EOL> self . se_desc . x_path = '<STR_LIT:description>' <EOL> self . se_desc . request_page_type = '<STR_LIT>' <EOL> self . se_desc . save ( ) <EOL> self . se_es_1 . x_path = '<STR_LIT:title>' <EOL> self . se_es_1 . save ( ) <EOL> self . rpt_mp . content_type = '<STR_LIT>' <EOL> self . rpt_mp . save ( ) <EOL> self . event_website . url = os . path . join ( self . SERVER_URL , '<STR_LIT>' ) <EOL> self . event_website . save ( ) <EOL> def extraSetUpHTMLChecker ( self ) : <EOL> self . checker = Checker ( ) <EOL> self . checker . scraped_obj_attr = self . soa_url <EOL> self . checker . scraper = self . scraper <EOL> self . checker . checker_type = '<STR_LIT:X>' <EOL> self . checker . checker_x_path = '<STR_LIT>' <EOL> self . checker . checker_x_path_result = '<STR_LIT>' <EOL> self . checker . checker_ref_url = '<STR_LIT>' <EOL> self . checker . save ( ) <EOL> scheduler_rt = SchedulerRuntime ( ) <EOL> scheduler_rt . save ( ) <EOL> self . event = Event ( title = '<STR_LIT>' , event_website = self . event_website , <EOL> description = '<STR_LIT>' , <EOL> url = '<STR_LIT>' , <EOL> checker_runtime = scheduler_rt ) <EOL> self . event . save ( ) <EOL> def extraSetUpJSONChecker ( self ) : <EOL> self . rpt_dp1 . content_type = '<STR_LIT>' <EOL> self . rpt_dp1 . save ( ) <EOL> self . checker = Checker ( ) <EOL> self . checker . scraped_obj_attr = self . soa_url <EOL> self . checker . scraper = self . scraper <EOL> self . checker . checker_type = '<STR_LIT:X>' <EOL> self . checker . checker_x_path = '<STR_LIT>' <EOL> self . checker . checker_x_path_result = '<STR_LIT>' <EOL> self . checker . checker_ref_url = '<STR_LIT>' <EOL> self . checker . save ( ) <EOL> scheduler_rt = SchedulerRuntime ( ) <EOL> scheduler_rt . save ( ) <EOL> self . event = Event ( title = '<STR_LIT>' , event_website = self . event_website , <EOL> description = '<STR_LIT>' , <EOL> url = '<STR_LIT>' , <EOL> checker_runtime = scheduler_rt ) <EOL> self . event . save ( ) <EOL> def test_num_scraped_objects ( self ) : <EOL> self . setUpScraperJSONDefaultScraper ( ) <EOL> self . run_event_spider ( <NUM_LIT:1> ) <EOL> self . assertEqual ( len ( Event . objects . all ( ) ) , <NUM_LIT:3> ) <EOL> def test_non_repetition ( self ) : <EOL> self . setUpScraperJSONDefaultScraper ( ) <EOL> self . run_event_spider ( <NUM_LIT:1> ) <EOL> self . assertEqual ( len ( Event . objects . filter ( title = '<STR_LIT>' ) ) , <NUM_LIT:1> ) <EOL> def test_non_data_mixing ( self ) : <EOL> self . setUpScraperJSONDefaultScraper ( ) <EOL> self . run_event_spider ( <NUM_LIT:1> ) <EOL> self . assertEqual ( Event . objects . get ( pk = <NUM_LIT:1> ) . title , '<STR_LIT>' ) <EOL> self . assertEqual ( Event . objects . get ( pk = <NUM_LIT:1> ) . description , '<STR_LIT>' ) <EOL> def test_static_processor_empty_x_path ( self ) : <EOL> self . setUpScraperJSONDefaultScraper ( ) <EOL> self . se_title . x_path = '<STR_LIT>' <EOL> self . se_title . processors = '<STR_LIT>' <EOL> self . se_title . proc_ctxt = "<STR_LIT>" <EOL> self . se_title . save ( ) <EOL> self . run_event_spider ( <NUM_LIT:1> ) <EOL> self . assertEqual ( len ( Event . objects . filter ( title = '<STR_LIT>' ) ) , <NUM_LIT:3> ) <EOL> def test_detail_page ( self ) : <EOL> self . setUpScraperJSONDefaultScraper ( ) <EOL> self . se_desc . x_path = '<STR_LIT>' <EOL> self . se_desc . request_page_type = '<STR_LIT>' <EOL> self . se_desc . save ( ) <EOL> self . run_event_spider ( <NUM_LIT:1> ) <EOL> self . assertEqual ( len ( Event . objects . filter ( description = '<STR_LIT>' ) ) , <NUM_LIT:1> ) <EOL> def test_detail_page_json ( self ) : <EOL> self . setUpScraperJSONDefaultScraper ( ) <EOL> self . rpt_dp1 . content_type = '<STR_LIT>' <EOL> self . rpt_dp1 . save ( ) <EOL> self . se_url . x_path = '<STR_LIT>' <EOL> self . se_url . save ( ) <EOL> self . se_desc . x_path = '<STR_LIT>' <EOL> self . se_desc . request_page_type = '<STR_LIT>' <EOL> self . se_desc . save ( ) <EOL> self . run_event_spider ( <NUM_LIT:1> ) <EOL> self . assertEqual ( len ( Event . objects . filter ( description = '<STR_LIT>' ) ) , <NUM_LIT:1> ) <EOL> def test_multiple_detail_pages ( self ) : <EOL> self . setUpScraperJSONDefaultScraper ( ) <EOL> self . se_desc . x_path = '<STR_LIT>' <EOL> self . se_desc . request_page_type = '<STR_LIT>' <EOL> self . se_desc . save ( ) <EOL> self . soa_url . id_field = False <EOL> self . soa_url . save_to_db = False <EOL> self . soa_url . save ( ) <EOL> self . soa_url2 . save_to_db = False <EOL> self . soa_url2 . save ( ) <EOL> self . rpt_dp2 = RequestPageType ( page_type = '<STR_LIT>' , scraper = self . scraper , scraped_obj_attr = self . soa_url2 , content_type = '<STR_LIT>' ) <EOL> self . rpt_dp2 . save ( ) <EOL> self . se_url2 = ScraperElem ( scraped_obj_attr = self . soa_url2 , scraper = self . scraper , <EOL> x_path = '<STR_LIT>' , request_page_type = '<STR_LIT>' ) <EOL> self . se_url2 . save ( ) <EOL> self . se_desc2 = ScraperElem ( scraped_obj_attr = self . soa_desc2 , scraper = self . scraper , <EOL> x_path = '<STR_LIT>' , request_page_type = '<STR_LIT>' , mandatory = False ) <EOL> self . se_desc2 . save ( ) <EOL> self . run_event_spider ( <NUM_LIT:1> ) <EOL> events = Event . objects . filter ( <EOL> title = '<STR_LIT>' , <EOL> description2 = '<STR_LIT>' , <EOL> ) <EOL> self . assertEqual ( len ( events ) , <NUM_LIT:1> ) <EOL> def test_checker_x_path_type_x_path_delete ( self ) : <EOL> self . setUpScraperJSONDefaultScraper ( ) <EOL> self . extraSetUpHTMLChecker ( ) <EOL> self . assertEqual ( len ( Event . objects . all ( ) ) , <NUM_LIT:1> ) <EOL> self . run_event_checker ( <NUM_LIT:1> ) <EOL> self . assertEqual ( len ( Event . objects . all ( ) ) , <NUM_LIT:0> ) <EOL> def test_checker_x_path_type_x_path_no_delete ( self ) : <EOL> self . setUpScraperJSONDefaultScraper ( ) <EOL> self . extraSetUpHTMLChecker ( ) <EOL> self . checker . checker_x_path = '<STR_LIT>' <EOL> self . checker . save ( ) <EOL> self . assertEqual ( len ( Event . objects . all ( ) ) , <NUM_LIT:1> ) <EOL> self . run_event_checker ( <NUM_LIT:1> ) <EOL> self . assertEqual ( len ( Event . objects . all ( ) ) , <NUM_LIT:1> ) <EOL> def test_json_checker_x_path_type_x_path_delete ( self ) : <EOL> self . setUpScraperJSONDefaultScraper ( ) <EOL> self . extraSetUpJSONChecker ( ) <EOL> self . assertEqual ( len ( Event . objects . all ( ) ) , <NUM_LIT:1> ) <EOL> self . run_event_checker ( <NUM_LIT:1> ) <EOL> self . assertEqual ( len ( Event . objects . all ( ) ) , <NUM_LIT:0> ) <EOL> def test_json_checker_x_path_type_x_path_no_delete ( self ) : <EOL> self . setUpScraperJSONDefaultScraper ( ) <EOL> self . extraSetUpJSONChecker ( ) <EOL> self . checker . checker_x_path = '<STR_LIT>' <EOL> self . checker . save ( ) <EOL> self . assertEqual ( len ( Event . objects . all ( ) ) , <NUM_LIT:1> ) <EOL> self . run_event_checker ( <NUM_LIT:1> ) <EOL> self . assertEqual ( len ( Event . objects . all ( ) ) , <NUM_LIT:1> ) </s>
