<s> import argparse <EOL> import datetime <EOL> import functools <EOL> import json <EOL> import sys <EOL> import os <EOL> sys . path . append ( os . environ . get ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> import usage_audit <EOL> from stacktach . models import InstanceUsage <EOL> from stacktach import datetime_to_decimal as dt <EOL> from stacktach import models <EOL> from stacktach . reconciler import Reconciler <EOL> from stacktach import stacklog <EOL> OLD_LAUNCHES_QUERY = """<STR_LIT>""" <EOL> OLD_RECONCILES_QUERY = """<STR_LIT>""" <EOL> DEFAULT_UMS_OFFSET = <NUM_LIT:4> * <NUM_LIT> * <NUM_LIT> <EOL> reconciler = None <EOL> def _get_new_launches ( beginning , ending ) : <EOL> filters = { <EOL> '<STR_LIT>' : beginning , <EOL> '<STR_LIT>' : ending , <EOL> } <EOL> return models . InstanceUsage . objects . filter ( ** filters ) <EOL> def _get_deletes ( beginning , ending ) : <EOL> filters = { <EOL> '<STR_LIT>' : beginning , <EOL> '<STR_LIT>' : ending , <EOL> } <EOL> return models . InstanceDeletes . objects . filter ( ** filters ) <EOL> def _get_exists ( beginning , ending ) : <EOL> filters = { <EOL> '<STR_LIT>' : beginning , <EOL> '<STR_LIT>' : beginning , <EOL> '<STR_LIT>' : ending , <EOL> } <EOL> return models . InstanceExists . objects . filter ( ** filters ) <EOL> def cell_and_compute ( instance , launched_at ) : <EOL> usage_list = InstanceUsage . find ( instance , launched_at ) <EOL> deployment = None <EOL> usage = None <EOL> if usage_list : <EOL> usage = usage_list [ <NUM_LIT:0> ] <EOL> deployment = usage . latest_deployment_for_request_id ( ) <EOL> cell = ( deployment and deployment . name ) or '<STR_LIT:->' <EOL> compute = ( usage and usage . host ( ) ) or '<STR_LIT:->' <EOL> return cell , compute <EOL> def _audit_launches_to_exists ( launches , exists , beginning ) : <EOL> fails = [ ] <EOL> for ( instance , launches ) in launches . items ( ) : <EOL> if instance in exists : <EOL> for expected in launches : <EOL> found = False <EOL> for actual in exists [ instance ] : <EOL> if int ( expected [ '<STR_LIT>' ] ) == int ( actual [ '<STR_LIT>' ] ) : <EOL> found = True <EOL> if not found : <EOL> rec = False <EOL> if reconciler : <EOL> args = ( expected [ '<STR_LIT:id>' ] , beginning ) <EOL> rec = reconciler . missing_exists_for_instance ( * args ) <EOL> launched_at = dt . dt_from_decimal ( expected [ '<STR_LIT>' ] ) <EOL> msg = "<STR_LIT>" <EOL> msg = msg % ( instance , launched_at ) <EOL> cell , compute = cell_and_compute ( instance , launched_at ) <EOL> fails . append ( [ '<STR_LIT>' , expected [ '<STR_LIT:id>' ] , msg , <EOL> '<STR_LIT:Y>' if rec else '<STR_LIT:N>' , cell , compute ] ) <EOL> else : <EOL> rec = False <EOL> if reconciler : <EOL> args = ( launches [ <NUM_LIT:0> ] [ '<STR_LIT:id>' ] , beginning ) <EOL> rec = reconciler . missing_exists_for_instance ( * args ) <EOL> msg = "<STR_LIT>" % instance <EOL> launched_at = dt . dt_from_decimal ( launches [ <NUM_LIT:0> ] [ '<STR_LIT>' ] ) <EOL> cell , compute = cell_and_compute ( instance , launched_at ) <EOL> fails . append ( [ '<STR_LIT:->' , msg , '<STR_LIT:Y>' if rec else '<STR_LIT:N>' , <EOL> cell , compute ] ) <EOL> return fails <EOL> def _launch_audit_for_period ( beginning , ending ) : <EOL> launches_dict = { } <EOL> new_launches = _get_new_launches ( beginning , ending ) <EOL> for launch in new_launches : <EOL> instance = launch . instance <EOL> l = { '<STR_LIT:id>' : launch . id , '<STR_LIT>' : launch . launched_at } <EOL> if instance in launches_dict : <EOL> launches_dict [ instance ] . append ( l ) <EOL> else : <EOL> launches_dict [ instance ] = [ l , ] <EOL> old_launches = models . InstanceUsage . objects . raw ( OLD_LAUNCHES_QUERY , <EOL> [ beginning , beginning , beginning ] ) <EOL> old_launches_dict = { } <EOL> for launch in old_launches : <EOL> instance = launch . instance <EOL> l = { '<STR_LIT:id>' : launch . id , '<STR_LIT>' : launch . launched_at } <EOL> if instance not in old_launches_dict or ( old_launches_dict [ instance ] [ '<STR_LIT>' ] < <EOL> launch . launched_at ) : <EOL> old_launches_dict [ instance ] = l <EOL> old_recs = models . InstanceReconcile . objects . raw ( OLD_RECONCILES_QUERY , <EOL> [ beginning , beginning , beginning ] ) <EOL> for rec in old_recs : <EOL> instance = rec . instance <EOL> l = { '<STR_LIT:id>' : rec . id , '<STR_LIT>' : rec . launched_at } <EOL> if instance not in old_launches_dict or ( old_launches_dict [ instance ] [ '<STR_LIT>' ] < <EOL> rec . launched_at ) : <EOL> old_launches_dict [ instance ] = l <EOL> for instance , launch in old_launches_dict . items ( ) : <EOL> if instance in launches_dict : <EOL> launches_dict [ instance ] . append ( launch ) <EOL> else : <EOL> launches_dict [ instance ] = [ launch , ] <EOL> exists_dict = { } <EOL> exists = _get_exists ( beginning , ending ) <EOL> for exist in exists : <EOL> instance = exist . instance <EOL> e = { '<STR_LIT:id>' : exist . id , <EOL> '<STR_LIT>' : exist . launched_at , <EOL> '<STR_LIT>' : exist . deleted_at } <EOL> if instance in exists_dict : <EOL> exists_dict [ instance ] . append ( e ) <EOL> else : <EOL> exists_dict [ instance ] = [ e , ] <EOL> launch_to_exists_fails = _audit_launches_to_exists ( launches_dict , <EOL> exists_dict , <EOL> beginning ) <EOL> return launch_to_exists_fails , new_launches . count ( ) , len ( old_launches_dict ) <EOL> def audit_for_period ( beginning , ending , ums = False , ums_offset = <NUM_LIT:0> ) : <EOL> beginning_decimal = dt . dt_to_decimal ( beginning ) <EOL> ending_decimal = dt . dt_to_decimal ( ending ) <EOL> if ums : <EOL> verifier_audit_func = functools . partial ( <EOL> usage_audit . _verifier_audit_for_day_ums , ums_offset = ums_offset <EOL> ) <EOL> else : <EOL> verifier_audit_func = usage_audit . _verifier_audit_for_day <EOL> ( verify_summary , <EOL> verify_detail ) = verifier_audit_func ( beginning_decimal , ending_decimal , <EOL> models . InstanceExists ) <EOL> detail , new_count , old_count = _launch_audit_for_period ( beginning_decimal , <EOL> ending_decimal ) <EOL> summary = { <EOL> '<STR_LIT>' : verify_summary , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : new_count , <EOL> '<STR_LIT>' : old_count , <EOL> '<STR_LIT>' : len ( detail ) <EOL> } , <EOL> } <EOL> details = { <EOL> '<STR_LIT>' : verify_detail , <EOL> '<STR_LIT>' : detail , <EOL> } <EOL> return summary , details <EOL> def store_results ( start , end , summary , details ) : <EOL> values = { <EOL> '<STR_LIT>' : make_json_report ( summary , details ) , <EOL> '<STR_LIT>' : dt . dt_to_decimal ( datetime . datetime . utcnow ( ) ) , <EOL> '<STR_LIT>' : start , <EOL> '<STR_LIT>' : end , <EOL> '<STR_LIT:version>' : <NUM_LIT:7> , <EOL> '<STR_LIT:name>' : '<STR_LIT>' <EOL> } <EOL> report = models . JsonReport ( ** values ) <EOL> report . save ( ) <EOL> def make_json_report ( summary , details ) : <EOL> report = { <EOL> '<STR_LIT>' : summary , <EOL> '<STR_LIT>' : [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' ] , <EOL> '<STR_LIT>' : details [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : [ '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> '<STR_LIT>' : details [ '<STR_LIT>' ] <EOL> } <EOL> return json . dumps ( report ) <EOL> def valid_datetime ( d ) : <EOL> try : <EOL> t = datetime . datetime . strptime ( d , "<STR_LIT>" ) <EOL> return t <EOL> except Exception , e : <EOL> raise argparse . ArgumentTypeError ( <EOL> "<STR_LIT>" % d ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> parser = argparse . ArgumentParser ( '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , <EOL> choices = [ '<STR_LIT>' , '<STR_LIT>' ] , default = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , <EOL> help = "<STR_LIT>" , <EOL> type = valid_datetime , default = None ) <EOL> parser . add_argument ( '<STR_LIT>' , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" , <EOL> type = bool , default = False ) <EOL> parser . add_argument ( '<STR_LIT>' , <EOL> help = "<STR_LIT>" , <EOL> type = bool , default = False ) <EOL> parser . add_argument ( '<STR_LIT>' , <EOL> help = "<STR_LIT>" , <EOL> type = str , <EOL> default = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" , <EOL> action = '<STR_LIT:store_true>' ) <EOL> parser . add_argument ( '<STR_LIT>' , <EOL> help = "<STR_LIT>" , <EOL> type = int , <EOL> default = DEFAULT_UMS_OFFSET ) <EOL> args = parser . parse_args ( ) <EOL> if args . ums and args . period_length != '<STR_LIT>' : <EOL> print "<STR_LIT>" <EOL> sys . exit ( <NUM_LIT:0> ) <EOL> stacklog . set_default_logger_name ( '<STR_LIT>' ) <EOL> parent_logger = stacklog . get_logger ( '<STR_LIT>' , is_parent = True ) <EOL> log_listener = stacklog . LogListener ( parent_logger ) <EOL> log_listener . start ( ) <EOL> if args . reconcile : <EOL> with open ( args . reconciler_config ) as f : <EOL> reconciler_config = json . load ( f ) <EOL> reconciler = Reconciler ( reconciler_config ) <EOL> if args . utcdatetime is not None : <EOL> time = args . utcdatetime <EOL> else : <EOL> time = datetime . datetime . utcnow ( ) <EOL> start , end = usage_audit . get_previous_period ( time , args . period_length ) <EOL> summary , details = audit_for_period ( start , end , ums = args . ums , <EOL> ums_offset = args . ums_offset ) <EOL> if not args . store : <EOL> print make_json_report ( summary , details ) <EOL> else : <EOL> store_results ( start , end , summary , details ) </s>
