<s> import collections <EOL> from django . conf import settings <EOL> from django . core . management import call_command <EOL> from olympia import amo <EOL> from olympia . applications . models import AppVersion <EOL> from olympia . lib . es . utils import create_index <EOL> from olympia . stats . models import ( <EOL> CollectionCount , DownloadCount , StatsSearchMixin , UpdateCount ) <EOL> CHUNK_SIZE = <NUM_LIT> <EOL> def es_dict ( items ) : <EOL> if not items : <EOL> return { } <EOL> if hasattr ( items , '<STR_LIT>' ) : <EOL> items = items . items ( ) <EOL> return [ { '<STR_LIT:k>' : key , '<STR_LIT:v>' : value } for key , value in items ] <EOL> """<STR_LIT>""" <EOL> def extract_update_count ( update , all_apps = None ) : <EOL> doc = { '<STR_LIT>' : update . addon_id , <EOL> '<STR_LIT:date>' : update . date , <EOL> '<STR_LIT:count>' : update . count , <EOL> '<STR_LIT:id>' : update . id , <EOL> '<STR_LIT>' : '<STR_LIT>' . format ( update . addon_id , update . date ) , <EOL> '<STR_LIT>' : es_dict ( update . versions ) , <EOL> '<STR_LIT>' : [ ] , <EOL> '<STR_LIT>' : [ ] , <EOL> '<STR_LIT>' : [ ] , <EOL> '<STR_LIT:status>' : [ ] } <EOL> if update . oses : <EOL> os = collections . defaultdict ( int ) <EOL> for key , count in update . oses . items ( ) : <EOL> platform = None <EOL> if unicode ( key ) . lower ( ) in amo . PLATFORM_DICT : <EOL> platform = amo . PLATFORM_DICT [ unicode ( key ) . lower ( ) ] <EOL> elif key in amo . PLATFORMS : <EOL> platform = amo . PLATFORMS [ key ] <EOL> if platform is not None : <EOL> os [ platform . name ] += count <EOL> doc [ '<STR_LIT>' ] = es_dict ( ( unicode ( k ) , v ) for k , v in os . items ( ) ) <EOL> if update . locales : <EOL> locales = collections . defaultdict ( int ) <EOL> for locale , count in update . locales . items ( ) : <EOL> try : <EOL> locales [ locale . lower ( ) ] += int ( count ) <EOL> except ValueError : <EOL> pass <EOL> doc [ '<STR_LIT>' ] = es_dict ( locales ) <EOL> if update . applications : <EOL> apps = collections . defaultdict ( dict ) <EOL> for guid , version_counts in update . applications . items ( ) : <EOL> if guid not in amo . APP_GUIDS : <EOL> continue <EOL> app = amo . APP_GUIDS [ guid ] <EOL> for version , count in version_counts . items ( ) : <EOL> try : <EOL> apps [ app . guid ] [ version ] = int ( count ) <EOL> except ValueError : <EOL> pass <EOL> doc [ '<STR_LIT>' ] = dict ( ( app , es_dict ( vals ) ) for app , vals in apps . items ( ) ) <EOL> if update . statuses : <EOL> doc [ '<STR_LIT:status>' ] = es_dict ( ( k , v ) for k , v in update . statuses . items ( ) <EOL> if k != '<STR_LIT:null>' ) <EOL> return doc <EOL> def extract_download_count ( dl ) : <EOL> return { '<STR_LIT>' : dl . addon_id , <EOL> '<STR_LIT:date>' : dl . date , <EOL> '<STR_LIT:count>' : dl . count , <EOL> '<STR_LIT>' : es_dict ( dl . sources ) if dl . sources else { } , <EOL> '<STR_LIT:id>' : dl . id , <EOL> '<STR_LIT>' : '<STR_LIT>' . format ( dl . addon_id , dl . date ) } <EOL> def extract_addon_collection ( collection_count , addon_collections , <EOL> collection_stats ) : <EOL> addon_collection_count = sum ( [ c . count for c in addon_collections ] ) <EOL> collection_stats = dict ( [ [ c . name , c . count ] for c in collection_stats ] ) <EOL> return { '<STR_LIT:date>' : collection_count . date , <EOL> '<STR_LIT:id>' : collection_count . collection_id , <EOL> '<STR_LIT>' : '<STR_LIT>' . format ( collection_count . collection_id , <EOL> collection_count . date ) , <EOL> '<STR_LIT:count>' : collection_count . count , <EOL> '<STR_LIT:data>' : es_dict ( { <EOL> '<STR_LIT>' : addon_collection_count , <EOL> '<STR_LIT>' : collection_stats . get ( '<STR_LIT>' , <NUM_LIT:0> ) , <EOL> '<STR_LIT>' : collection_stats . get ( '<STR_LIT>' , <NUM_LIT:0> ) , <EOL> '<STR_LIT>' : collection_stats . get ( '<STR_LIT>' , <NUM_LIT:0> ) , <EOL> } ) } <EOL> def extract_theme_user_count ( user_count ) : <EOL> return { '<STR_LIT>' : user_count . addon_id , <EOL> '<STR_LIT:date>' : user_count . date , <EOL> '<STR_LIT:count>' : user_count . count , <EOL> '<STR_LIT:id>' : user_count . id , <EOL> '<STR_LIT>' : '<STR_LIT>' . format ( user_count . addon_id , user_count . date ) } <EOL> def get_all_app_versions ( ) : <EOL> vals = AppVersion . objects . values_list ( '<STR_LIT>' , '<STR_LIT:version>' ) <EOL> rv = collections . defaultdict ( list ) <EOL> for app , version in vals : <EOL> rv [ app ] . append ( version ) <EOL> return dict ( rv ) <EOL> def get_alias ( ) : <EOL> return settings . ES_INDEXES . get ( StatsSearchMixin . ES_ALIAS_KEY ) <EOL> def create_new_index ( index_name = None , config = None ) : <EOL> if config is None : <EOL> config = { } <EOL> if index_name is None : <EOL> index_name = get_alias ( ) <EOL> config [ '<STR_LIT>' ] = get_mappings ( ) <EOL> create_index ( index_name , config ) <EOL> def reindex ( index_name ) : <EOL> call_command ( '<STR_LIT>' , index = index_name ) <EOL> def get_mappings ( ) : <EOL> mapping = { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT:id>' : { '<STR_LIT:type>' : '<STR_LIT>' } , <EOL> '<STR_LIT>' : { '<STR_LIT:type>' : '<STR_LIT:float>' , '<STR_LIT>' : <NUM_LIT:1.0> } , <EOL> '<STR_LIT:count>' : { '<STR_LIT:type>' : '<STR_LIT>' } , <EOL> '<STR_LIT:data>' : { <EOL> '<STR_LIT>' : '<STR_LIT:true>' , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT:v>' : { '<STR_LIT:type>' : '<STR_LIT>' } , <EOL> '<STR_LIT:k>' : { '<STR_LIT:type>' : '<STR_LIT:string>' } <EOL> } <EOL> } , <EOL> '<STR_LIT:date>' : { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT:type>' : '<STR_LIT:date>' <EOL> } <EOL> } <EOL> } <EOL> models = ( CollectionCount , DownloadCount , UpdateCount ) <EOL> return dict ( ( m . _meta . db_table , mapping ) for m in models ) </s>
