<s> from __future__ import unicode_literals <EOL> from __future__ import division <EOL> import os <EOL> from calaccess_raw . management . commands import CalAccessCommand <EOL> from calaccess_raw import get_model_list <EOL> from django . conf import settings <EOL> from django . core . management import call_command <EOL> from django . db . models import Sum <EOL> from django . forms . models import model_to_dict <EOL> from django . contrib . humanize . templatetags . humanize import naturaltime <EOL> from clint . textui import progress <EOL> from csv import DictWriter <EOL> def calc_percent ( whole , total ) : <EOL> """<STR_LIT:U+0020>""" <EOL> try : <EOL> pct = whole / total <EOL> except ZeroDivisionError : <EOL> pct = <NUM_LIT:0> <EOL> else : <EOL> pct * <NUM_LIT:100> <EOL> return pct <EOL> class Command ( CalAccessCommand ) : <EOL> help = '<STR_LIT>' <EOL> def add_arguments ( self , parser ) : <EOL> """<STR_LIT>""" <EOL> super ( Command , self ) . add_arguments ( parser ) <EOL> def handle ( self , * args , ** options ) : <EOL> super ( Command , self ) . handle ( * args , ** options ) <EOL> self . missing_raw_files = [ ] <EOL> self . empty_raw_files = [ ] <EOL> self . unknown_raw_files = [ ] <EOL> try : <EOL> last_complete_download = self . command_logs . filter ( <EOL> command = '<STR_LIT>' , <EOL> finish_datetime__isnull = False <EOL> ) . order_by ( '<STR_LIT>' ) [ <NUM_LIT:0> ] <EOL> except IndexError : <EOL> self . failure ( "<STR_LIT>" ) <EOL> self . log ( "<STR_LIT>" ) <EOL> return <EOL> else : <EOL> self . raw_data_files = self . raw_data_files . filter ( <EOL> version = last_complete_download . version <EOL> ) <EOL> downloaded_release_datetime = last_complete_download . version . release_datetime <EOL> current_release_datetime = self . get_download_metadata ( ) [ '<STR_LIT>' ] <EOL> if current_release_datetime != downloaded_release_datetime : <EOL> since_new_release = naturaltime ( current_release_datetime ) <EOL> self . failure ( "<STR_LIT>" <EOL> "<STR_LIT>" . format ( since_new_release ) ) <EOL> self . log ( "<STR_LIT>" ) <EOL> self . log ( "<STR_LIT>" ) <EOL> if self . verbosity == <NUM_LIT:1> : <EOL> model_list = progress . bar ( get_model_list ( ) ) <EOL> else : <EOL> model_list = get_model_list ( ) <EOL> for model in model_list : <EOL> call_command ( <EOL> '<STR_LIT>' , <EOL> model . __name__ , <EOL> verbosity = self . verbosity <EOL> ) <EOL> self . num_download_files = self . raw_data_files . count ( ) <EOL> self . num_clean_files = self . raw_data_files . filter ( <EOL> clean_records_count__gt = <NUM_LIT:0> <EOL> ) . count ( ) <EOL> self . num_loaded_files = self . raw_data_files . filter ( <EOL> load_records_count__gt = <NUM_LIT:0> <EOL> ) . count ( ) <EOL> total_raw_records = self . sum_count_column ( '<STR_LIT>' ) <EOL> total_clean_records = self . sum_count_column ( '<STR_LIT>' ) <EOL> total_loaded_records = self . sum_count_column ( '<STR_LIT>' ) <EOL> self . log ( <EOL> '<STR_LIT>' . format ( <EOL> calc_percent ( total_clean_records , total_raw_records ) <EOL> ) <EOL> ) <EOL> self . log ( <EOL> '<STR_LIT>' . format ( <EOL> calc_percent ( total_loaded_records , total_raw_records ) <EOL> ) <EOL> ) <EOL> self . write_csv_results ( ) <EOL> self . duration ( ) <EOL> def sum_count_column ( self , column_name ) : <EOL> """<STR_LIT>""" <EOL> result = self . raw_data_files . aggregate ( <EOL> the_sum = Sum ( column_name ) <EOL> ) <EOL> return result [ '<STR_LIT>' ] <EOL> def write_csv_results ( self ) : <EOL> """<STR_LIT>""" <EOL> file_name = os . path . join ( <EOL> getattr ( settings , '<STR_LIT>' ) , <EOL> '<STR_LIT>' <EOL> ) <EOL> with open ( file_name , '<STR_LIT:w>' ) as f : <EOL> fieldnames = [ <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' <EOL> ] <EOL> writer = DictWriter ( f , fieldnames = fieldnames ) <EOL> writer . writeheader ( ) <EOL> for i in self . raw_data_files : <EOL> row = model_to_dict ( i ) <EOL> del row [ '<STR_LIT:id>' ] <EOL> del row [ '<STR_LIT:version>' ] <EOL> row . update ( { <EOL> '<STR_LIT>' : '<STR_LIT>' . format ( <EOL> calc_percent ( i . clean_records_count , i . download_records_count ) <EOL> ) <EOL> } ) <EOL> row . update ( { <EOL> '<STR_LIT>' : '<STR_LIT>' . format ( <EOL> calc_percent ( i . load_records_count , i . download_records_count ) <EOL> ) <EOL> } ) <EOL> writer . writerow ( row ) </s>
