<s> from __future__ import unicode_literals <EOL> from builtins import str <EOL> import os . path , random , unittest <EOL> from scrapy . exceptions import CloseSpider <EOL> from scraper . models import Event <EOL> from scraper . scraper_test import EventChecker , ScraperTest <EOL> from dynamic_scraper . models import Checker , Log , SchedulerRuntime <EOL> PROJECT_ROOT = os . path . abspath ( os . path . dirname ( __file__ ) ) <EOL> class CheckerRunTest ( ScraperTest ) : <EOL> def setUp ( self ) : <EOL> super ( CheckerRunTest , self ) . setUp ( ) <EOL> self . checker = Checker ( ) <EOL> self . checker . scraped_obj_attr = self . soa_url <EOL> self . checker . scraper = self . scraper <EOL> self . checker . checker_type = '<STR_LIT:X>' <EOL> self . checker . checker_x_path = '<STR_LIT>' <EOL> self . checker . checker_x_path_result = '<STR_LIT>' <EOL> self . checker . checker_ref_url = '<STR_LIT>' <EOL> self . checker . save ( ) <EOL> scheduler_rt = SchedulerRuntime ( ) <EOL> scheduler_rt . save ( ) <EOL> self . event = Event ( title = '<STR_LIT>' , event_website = self . event_website , <EOL> description = '<STR_LIT>' , <EOL> url = '<STR_LIT>' , <EOL> checker_runtime = scheduler_rt ) <EOL> self . event . save ( ) <EOL> def setUpWithSecondChecker ( self ) : <EOL> self . checker2 = Checker ( ) <EOL> self . checker2 . scraped_obj_attr = self . soa_url <EOL> self . checker2 . scraper = self . scraper <EOL> self . checker2 . checker_type = '<STR_LIT:X>' <EOL> self . checker2 . checker_x_path = '<STR_LIT>' <EOL> self . checker2 . checker_x_path_result = '<STR_LIT>' <EOL> self . checker2 . checker_ref_url = '<STR_LIT>' <EOL> self . checker2 . save ( ) <EOL> def test_no_checker ( self ) : <EOL> self . checker . delete ( ) <EOL> self . assertRaises ( CloseSpider , self . run_event_checker , <NUM_LIT:1> ) <EOL> def test_x_path_type_keep ( self ) : <EOL> self . event . url = '<STR_LIT>' <EOL> self . event . save ( ) <EOL> self . run_event_checker ( <NUM_LIT:1> ) <EOL> self . assertEqual ( len ( Event . objects . all ( ) ) , <NUM_LIT:1> ) <EOL> def test_x_path_type_keep_double ( self ) : <EOL> self . setUpWithSecondChecker ( ) <EOL> self . event . url = '<STR_LIT>' <EOL> self . event . save ( ) <EOL> self . run_event_checker ( <NUM_LIT:1> ) <EOL> self . assertEqual ( len ( Event . objects . all ( ) ) , <NUM_LIT:1> ) <EOL> def test_x_path_type_blank_result_field_keep ( self ) : <EOL> self . scraper . checker_x_path_result = '<STR_LIT>' <EOL> self . event . url = '<STR_LIT>' <EOL> self . event . save ( ) <EOL> self . run_event_checker ( <NUM_LIT:1> ) <EOL> self . assertEqual ( len ( Event . objects . all ( ) ) , <NUM_LIT:1> ) <EOL> def test_x_path_type_404_delete ( self ) : <EOL> self . event . url = '<STR_LIT>' <EOL> self . event . save ( ) <EOL> self . run_event_checker ( <NUM_LIT:1> ) <EOL> self . assertEqual ( len ( Event . objects . all ( ) ) , <NUM_LIT:0> ) <EOL> def test_x_path_type_404_delete_with_zero_actions ( self ) : <EOL> self . event . url = '<STR_LIT>' <EOL> self . event . save ( ) <EOL> self . event . checker_runtime . num_zero_actions = <NUM_LIT:3> <EOL> self . event . checker_runtime . save ( ) <EOL> kwargs = { <EOL> '<STR_LIT:id>' : <NUM_LIT:1> , <EOL> '<STR_LIT>' : '<STR_LIT:yes>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } <EOL> checker = EventChecker ( ** kwargs ) <EOL> self . process . crawl ( checker , ** kwargs ) <EOL> self . process . start ( ) <EOL> self . assertEqual ( len ( Event . objects . all ( ) ) , <NUM_LIT:1> ) <EOL> def test_x_path_type_x_path_delete ( self ) : <EOL> self . event . url = '<STR_LIT>' <EOL> self . event . save ( ) <EOL> self . run_event_checker ( <NUM_LIT:1> ) <EOL> self . assertEqual ( len ( Event . objects . all ( ) ) , <NUM_LIT:0> ) <EOL> def test_x_path_type_x_path_first_delete_double ( self ) : <EOL> self . setUpWithSecondChecker ( ) <EOL> self . event . url = '<STR_LIT>' <EOL> self . event . save ( ) <EOL> self . run_event_checker ( <NUM_LIT:1> ) <EOL> self . assertEqual ( len ( Event . objects . all ( ) ) , <NUM_LIT:0> ) <EOL> def test_x_path_type_x_path_second_delete_double ( self ) : <EOL> self . setUpWithSecondChecker ( ) <EOL> self . checker . checker_x_path = '<STR_LIT>' <EOL> self . checker . save ( ) <EOL> self . event . url = '<STR_LIT>' <EOL> self . event . save ( ) <EOL> self . run_event_checker ( <NUM_LIT:1> ) <EOL> self . assertEqual ( len ( Event . objects . all ( ) ) , <NUM_LIT:0> ) <EOL> def test_x_path_type_blank_result_field_x_path_delete ( self ) : <EOL> self . scraper . checker_x_path_result = '<STR_LIT>' <EOL> self . event . url = '<STR_LIT>' <EOL> self . event . save ( ) <EOL> self . run_event_checker ( <NUM_LIT:1> ) <EOL> self . assertEqual ( len ( Event . objects . all ( ) ) , <NUM_LIT:0> ) <EOL> def _create_imgs_in_dirs ( self , img_dirs ) : <EOL> img_paths = [ ] <EOL> file_name = '<STR_LIT>' . format ( rnd = str ( random . randint ( <NUM_LIT:0> , <NUM_LIT> ) ) ) <EOL> self . event . description = file_name <EOL> self . event . save ( ) <EOL> for img_dir in img_dirs : <EOL> path = os . path . join ( self . PROJECT_ROOT , img_dir , file_name ) <EOL> if not os . path . exists ( os . path . dirname ( path ) ) : <EOL> os . makedirs ( os . path . dirname ( path ) ) <EOL> if not os . path . exists ( path ) : <EOL> file = open ( path , "<STR_LIT:w>" ) <EOL> file . write ( '<STR_LIT>' ) <EOL> file . close ( ) <EOL> img_paths . append ( path ) <EOL> return img_paths <EOL> def _run_img_test_with_dirs ( self , img_dirs ) : <EOL> img_paths = self . _create_imgs_in_dirs ( img_dirs ) <EOL> self . se_desc . mandatory = True <EOL> self . se_desc . save ( ) <EOL> self . soa_desc . attr_type = '<STR_LIT:I>' <EOL> self . soa_desc . save ( ) <EOL> self . event . url = '<STR_LIT>' <EOL> self . event . save ( ) <EOL> for path in img_paths : <EOL> self . assertTrue ( os . path . exists ( path ) ) <EOL> self . run_event_checker ( <NUM_LIT:1> ) <EOL> self . assertEqual ( len ( Event . objects . all ( ) ) , <NUM_LIT:0> ) <EOL> for path in img_paths : <EOL> self . assertFalse ( os . path . exists ( path ) ) <EOL> def test_delete_with_img_flat_no_thumbs ( self ) : <EOL> img_dirs = [ '<STR_LIT>' , ] <EOL> self . _run_img_test_with_dirs ( img_dirs ) <EOL> def test_delete_with_img_flat_with_thumbs ( self ) : <EOL> img_dirs = [ '<STR_LIT>' , ] <EOL> self . _run_img_test_with_dirs ( img_dirs ) <EOL> def test_delete_with_img_all_no_thumbs ( self ) : <EOL> img_dirs = [ '<STR_LIT>' , ] <EOL> self . _run_img_test_with_dirs ( img_dirs ) <EOL> def test_delete_with_img_all_with_thumbs ( self ) : <EOL> img_dirs = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , ] <EOL> self . _run_img_test_with_dirs ( img_dirs ) <EOL> def test_delete_with_img_thumbs_with_thumbs ( self ) : <EOL> img_dirs = [ '<STR_LIT>' , '<STR_LIT>' , ] <EOL> self . _run_img_test_with_dirs ( img_dirs ) <EOL> def test_404_type_404_delete ( self ) : <EOL> self . checker . checker_type = '<STR_LIT:4>' <EOL> self . checker . save ( ) <EOL> self . event . url = '<STR_LIT>' <EOL> self . event . save ( ) <EOL> self . run_event_checker ( <NUM_LIT:1> ) <EOL> self . assertEqual ( len ( Event . objects . all ( ) ) , <NUM_LIT:0> ) <EOL> def test_404_type_x_path_delete ( self ) : <EOL> self . checker . checker_type = '<STR_LIT:4>' <EOL> self . checker . save ( ) <EOL> self . run_event_checker ( <NUM_LIT:1> ) <EOL> self . assertEqual ( len ( Event . objects . all ( ) ) , <NUM_LIT:1> ) <EOL> @ unittest . skip ( "<STR_LIT>" ) <EOL> def test_checker_test_wrong_checker_config ( self ) : <EOL> self . checker . checker_ref_url = '<STR_LIT>' <EOL> self . checker . save ( ) <EOL> self . assertRaises ( CloseSpider , self . run_checker_test ( <NUM_LIT:1> ) ) </s>
