<s> import codecs <EOL> import json <EOL> import re <EOL> from datetime import datetime , timedelta <EOL> from optparse import make_option <EOL> from os import path , unlink <EOL> from django . conf import settings <EOL> from django . core . management . base import BaseCommand , CommandError <EOL> import commonware . log <EOL> from olympia import amo <EOL> from olympia . addons . models import Addon <EOL> from olympia . stats . models import update_inc , UpdateCount <EOL> from . import get_date_from_file , save_stats_to_file <EOL> log = commonware . log . getLogger ( '<STR_LIT>' ) <EOL> LOCALE_REGEX = re . compile ( r"""<STR_LIT>""" , re . VERBOSE ) <EOL> VALID_STATUSES = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" ] <EOL> VALID_APP_GUIDS = amo . APP_GUIDS . keys ( ) <EOL> APPVERSION_REGEX = re . compile ( <EOL> r"""<STR_LIT>""" , re . VERBOSE ) <EOL> class Command ( BaseCommand ) : <EOL> """<STR_LIT>""" <EOL> help = __doc__ <EOL> option_list = BaseCommand . option_list + ( <EOL> make_option ( '<STR_LIT>' , action = '<STR_LIT:store>' , type = '<STR_LIT:string>' , <EOL> dest = '<STR_LIT:date>' , help = '<STR_LIT>' ) , <EOL> make_option ( '<STR_LIT>' , action = '<STR_LIT:store>' , type = '<STR_LIT:string>' , default = '<STR_LIT:\t>' , <EOL> dest = '<STR_LIT>' , help = '<STR_LIT>' ) , <EOL> ) <EOL> def handle ( self , * args , ** options ) : <EOL> start = datetime . now ( ) <EOL> day = options [ '<STR_LIT:date>' ] <EOL> if not day : <EOL> day = ( datetime . now ( ) - timedelta ( days = <NUM_LIT:1> ) ) . strftime ( '<STR_LIT>' ) <EOL> folder = args [ <NUM_LIT:0> ] if args else '<STR_LIT>' <EOL> folder = path . join ( settings . TMP_PATH , folder , day ) <EOL> sep = options [ '<STR_LIT>' ] <EOL> groups = ( '<STR_LIT:version>' , '<STR_LIT:status>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> group_filepaths = [ ] <EOL> for group in groups : <EOL> filepath = path . join ( folder , '<STR_LIT>' % group ) <EOL> if get_date_from_file ( filepath , sep ) != day : <EOL> raise CommandError ( '<STR_LIT>' % <EOL> filepath ) <EOL> group_filepaths . append ( ( group , filepath ) ) <EOL> UpdateCount . objects . filter ( date = day ) . delete ( ) <EOL> update_counts = { } <EOL> guids_to_addon = ( dict ( Addon . objects . exclude ( guid__isnull = True ) <EOL> . exclude ( type = amo . ADDON_PERSONA ) <EOL> . values_list ( '<STR_LIT>' , '<STR_LIT:id>' ) ) ) <EOL> index = - <NUM_LIT:1> <EOL> for group , filepath in group_filepaths : <EOL> with codecs . open ( filepath , encoding = '<STR_LIT:utf8>' ) as results_file : <EOL> for line in results_file : <EOL> index += <NUM_LIT:1> <EOL> if index and ( index % <NUM_LIT> ) == <NUM_LIT:0> : <EOL> log . info ( '<STR_LIT>' % index ) <EOL> splitted = line [ : - <NUM_LIT:1> ] . split ( sep ) <EOL> if ( ( group == '<STR_LIT>' and len ( splitted ) != <NUM_LIT:6> ) <EOL> or ( group != '<STR_LIT>' and len ( splitted ) != <NUM_LIT:5> ) ) : <EOL> log . debug ( '<STR_LIT>' % line ) <EOL> continue <EOL> if group == '<STR_LIT>' : <EOL> day , addon_guid , app_id , app_ver , count , update_type = splitted <EOL> else : <EOL> day , addon_guid , data , count , update_type = splitted <EOL> addon_guid = addon_guid . strip ( ) <EOL> if update_type : <EOL> update_type . strip ( ) <EOL> if update_type in [ '<STR_LIT:0>' , '<STR_LIT>' , '<STR_LIT:None>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' ] : <EOL> update_type = None <EOL> try : <EOL> count = int ( count ) <EOL> if update_type : <EOL> update_type = int ( update_type ) <EOL> except ValueError : <EOL> continue <EOL> if update_type and update_type & <NUM_LIT> != <NUM_LIT:16> : <EOL> log . debug ( "<STR_LIT>" % <EOL> update_type ) <EOL> continue <EOL> if addon_guid and addon_guid in guids_to_addon : <EOL> addon_id = guids_to_addon [ addon_guid ] <EOL> else : <EOL> log . debug ( u"<STR_LIT>" <EOL> . format ( guid = addon_guid . strip ( ) ) ) <EOL> continue <EOL> if addon_guid in update_counts : <EOL> uc = update_counts [ addon_guid ] <EOL> else : <EOL> uc = UpdateCount ( date = day , addon_id = addon_id , count = <NUM_LIT:0> ) <EOL> update_counts [ addon_guid ] = uc <EOL> if group == '<STR_LIT:version>' : <EOL> self . update_version ( uc , data , count ) <EOL> uc . count += count <EOL> elif group == '<STR_LIT:status>' : <EOL> self . update_status ( uc , data , count ) <EOL> elif group == '<STR_LIT>' : <EOL> self . update_app ( uc , app_id , app_ver , count ) <EOL> elif group == '<STR_LIT>' : <EOL> self . update_os ( uc , data , count ) <EOL> elif group == '<STR_LIT>' : <EOL> self . update_locale ( uc , data , count ) <EOL> for addon_guid , update_count in update_counts . iteritems ( ) : <EOL> self . trim_field ( update_count . locales ) <EOL> self . trim_field ( update_count . versions ) <EOL> UpdateCount . objects . bulk_create ( update_counts . values ( ) , <NUM_LIT:100> ) <EOL> for udate_count in update_counts . values ( ) : <EOL> save_stats_to_file ( update_count ) <EOL> log . info ( '<STR_LIT>' % ( index + <NUM_LIT:1> ) ) <EOL> log . debug ( '<STR_LIT>' % ( datetime . now ( ) - start ) ) <EOL> for _ , filepath in group_filepaths : <EOL> log . debug ( '<STR_LIT>' . format ( path = filepath ) ) <EOL> unlink ( filepath ) <EOL> def update_version ( self , update_count , version , count ) : <EOL> """<STR_LIT>""" <EOL> version = version [ : <NUM_LIT:32> ] <EOL> update_count . versions = update_inc ( update_count . versions , version , <EOL> count ) <EOL> def update_status ( self , update_count , status , count ) : <EOL> """<STR_LIT>""" <EOL> if status in VALID_STATUSES : <EOL> update_count . statuses = update_inc ( update_count . statuses , status , <EOL> count ) <EOL> def update_app ( self , update_count , app_id , app_ver , count ) : <EOL> """<STR_LIT>""" <EOL> if ( app_id not in VALID_APP_GUIDS or <EOL> not re . match ( APPVERSION_REGEX , app_ver ) ) : <EOL> return <EOL> if update_count . applications is None : <EOL> update_count . applications = { } <EOL> app = update_count . applications . get ( app_id , { } ) <EOL> update_count . applications . update ( <EOL> { app_id : update_inc ( app , app_ver , count ) } ) <EOL> def update_os ( self , update_count , os , count ) : <EOL> """<STR_LIT>""" <EOL> if os . lower ( ) in amo . PLATFORM_DICT : <EOL> update_count . oses = update_inc ( update_count . oses , os , count ) <EOL> def update_locale ( self , update_count , locale , count ) : <EOL> """<STR_LIT>""" <EOL> locale = locale . replace ( '<STR_LIT:_>' , '<STR_LIT:->' ) <EOL> if re . match ( LOCALE_REGEX , locale ) : <EOL> update_count . locales = update_inc ( update_count . locales , locale , <EOL> count ) <EOL> def trim_field ( self , field ) : <EOL> """<STR_LIT>""" <EOL> def fits ( field ) : <EOL> """<STR_LIT>""" <EOL> return len ( json . dumps ( field ) ) < ( <NUM_LIT:2> ** <NUM_LIT:16> ) <EOL> if fits ( field ) : <EOL> return <EOL> values = list ( reversed ( sorted ( field . items ( ) , key = lambda v : v [ <NUM_LIT:1> ] ) ) ) <EOL> while not fits ( field ) : <EOL> key , count = values . pop ( ) <EOL> del field [ key ] </s>
