<s> from utils import percentage , datetime_to_unix <EOL> from models import Run , ErrorLog <EOL> from scraper . item_scraper . errors import ERROR_CATEGORIES <EOL> class SpiderLogAnalysis ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , working , times_run , offers_log , errors , <EOL> error_evolution ) : <EOL> self . working = working <EOL> self . times_run = times_run <EOL> self . offers_log = offers_log <EOL> self . errors = errors <EOL> self . error_evolution = error_evolution <EOL> def process_errors ( error_logs ) : <EOL> '''<STR_LIT>''' <EOL> total = len ( error_logs ) <EOL> if total == <NUM_LIT:0> : <EOL> errors = [ ( category , <NUM_LIT:0> ) for category in ERROR_CATEGORIES ] <EOL> return errors <EOL> def count_category ( cat ) : <EOL> """<STR_LIT>""" <EOL> return sum ( [ ( <NUM_LIT:1> if log . category == cat else <NUM_LIT:0> ) for log in error_logs ] ) <EOL> errors = [ ( category , percentage ( count_category ( category ) , total ) ) <EOL> for category in ERROR_CATEGORIES ] <EOL> return errors <EOL> def analyze ( sites , start_date , end_date ) : <EOL> data = { } <EOL> for site in sites : <EOL> runs = Run . objects . filter ( site = site , <EOL> start__gte = start_date , <EOL> end__lte = end_date ) <EOL> all_errors = ErrorLog . objects . filter ( site = site , <EOL> date_time__gte = start_date , <EOL> date_time__lte = end_date ) <EOL> period_error_distribution = process_errors ( all_errors ) <EOL> error_evolution = [ ] <EOL> for run in runs : <EOL> errors = ErrorLog . objects . filter ( site = site , <EOL> date_time__gte = run . start , <EOL> date_time__lte = run . end ) <EOL> run_time = datetime_to_unix ( run . start ) <EOL> processed_errors = process_errors ( errors ) <EOL> error_evolution += [ { <EOL> '<STR_LIT:date>' : run_time , <EOL> '<STR_LIT>' : processed_errors <EOL> } ] <EOL> working = False <EOL> if runs : <EOL> last_run = runs . order_by ( '<STR_LIT>' ) [ <NUM_LIT:0> ] <EOL> if last_run . scraped > <NUM_LIT:0> : <EOL> working = True <EOL> offers_log = [ [ datetime_to_unix ( run . start ) , run . scraped ] <EOL> for run in runs ] <EOL> analysis = SpiderLogAnalysis ( working = working , <EOL> times_run = len ( runs ) , <EOL> offers_log = offers_log , <EOL> errors = period_error_distribution , <EOL> error_evolution = error_evolution ) <EOL> data [ site . name ] = analysis <EOL> return data </s>
