<s> """<STR_LIT>""" <EOL> from __future__ import with_statement <EOL> import datetime <EOL> import logging <EOL> import os <EOL> import random <EOL> import re <EOL> import sys <EOL> import threading <EOL> import time <EOL> import warnings <EOL> from google . appengine . api import apiproxy_stub_map <EOL> from google . appengine . api import lib_config <EOL> from google . appengine . api import memcache <EOL> from google . appengine . api import quota <EOL> from google . appengine . api import users <EOL> from google . appengine . ext . appstats import datamodel_pb <EOL> from google . appengine . ext . appstats import formatting <EOL> def _to_micropennies_per_op ( pennies , per ) : <EOL> """<STR_LIT>""" <EOL> return ( pennies * <NUM_LIT> ) / per <EOL> class ConfigDefaults ( object ) : <EOL> """<STR_LIT>""" <EOL> DEBUG = False <EOL> DUMP_LEVEL = - <NUM_LIT:1> <EOL> SHELL_OK = os . getenv ( '<STR_LIT>' , '<STR_LIT>' ) . startswith ( '<STR_LIT>' ) <EOL> DEFAULT_SCRIPT = "<STR_LIT>" <EOL> KEY_DISTANCE = <NUM_LIT:100> <EOL> KEY_MODULUS = <NUM_LIT:1000> <EOL> KEY_NAMESPACE = '<STR_LIT>' <EOL> KEY_PREFIX = '<STR_LIT>' <EOL> KEY_TEMPLATE = '<STR_LIT>' <EOL> PART_SUFFIX = '<STR_LIT>' <EOL> FULL_SUFFIX = '<STR_LIT>' <EOL> LOCK_SUFFIX = '<STR_LIT>' <EOL> MAX_STACK = <NUM_LIT:10> <EOL> MAX_LOCALS = <NUM_LIT:10> <EOL> MAX_REPR = <NUM_LIT:100> <EOL> MAX_DEPTH = <NUM_LIT:10> <EOL> RE_STACK_BOTTOM = r'<STR_LIT>' <EOL> RE_STACK_SKIP = r'<STR_LIT>' <EOL> LOCK_TIMEOUT = <NUM_LIT:1> <EOL> TZOFFSET = <NUM_LIT:8> * <NUM_LIT> <EOL> stats_url = '<STR_LIT>' <EOL> RECORD_FRACTION = <NUM_LIT:1.0> <EOL> FILTER_LIST = [ ] <EOL> DATASTORE_DETAILS = False <EOL> CALC_RPC_COSTS = False <EOL> DATASTORE_READ_OP_COST = _to_micropennies_per_op ( <NUM_LIT:7> , <NUM_LIT> ) <EOL> DATASTORE_WRITE_OP_COST = _to_micropennies_per_op ( <NUM_LIT:10> , <NUM_LIT> ) <EOL> DATASTORE_SMALL_OP_COST = _to_micropennies_per_op ( <NUM_LIT:1> , <NUM_LIT> ) <EOL> MAIL_RECIPIENT_COST = _to_micropennies_per_op ( <NUM_LIT:1> , <NUM_LIT:1000> ) <EOL> CHANNEL_CREATE_COST = _to_micropennies_per_op ( <NUM_LIT:1> , <NUM_LIT:100> ) <EOL> XMPP_STANZA_COST = _to_micropennies_per_op ( <NUM_LIT:10> , <NUM_LIT> ) <EOL> def should_record ( env ) : <EOL> """<STR_LIT>""" <EOL> if config . FILTER_LIST : <EOL> if config . DEBUG : <EOL> logging . debug ( '<STR_LIT>' , config . FILTER_LIST ) <EOL> for filter_dict in config . FILTER_LIST : <EOL> for key , regex in filter_dict . iteritems ( ) : <EOL> negated = isinstance ( regex , str ) and regex . startswith ( '<STR_LIT:!>' ) <EOL> if negated : <EOL> regex = regex [ <NUM_LIT:1> : ] <EOL> value = env . get ( key , '<STR_LIT>' ) <EOL> if bool ( re . match ( regex , value ) ) == negated : <EOL> if config . DEBUG : <EOL> logging . debug ( '<STR_LIT>' , regex , key , value ) <EOL> break <EOL> else : <EOL> if config . DEBUG : <EOL> logging . debug ( '<STR_LIT>' , filter_dict ) <EOL> break <EOL> else : <EOL> if config . DEBUG : <EOL> logging . debug ( '<STR_LIT>' ) <EOL> return False <EOL> if config . RECORD_FRACTION >= <NUM_LIT:1.0> : <EOL> return True <EOL> return random . random ( ) < config . RECORD_FRACTION <EOL> def normalize_path ( path ) : <EOL> """<STR_LIT>""" <EOL> return path <EOL> def extract_key ( request ) : <EOL> """<STR_LIT>""" <EOL> key = config . normalize_path ( request . http_path ( ) ) <EOL> if request . http_method ( ) != '<STR_LIT:GET>' : <EOL> key = '<STR_LIT>' % ( request . http_method ( ) , key ) <EOL> return key <EOL> config = lib_config . register ( '<STR_LIT>' , ConfigDefaults . __dict__ ) <EOL> class Recorder ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , env ) : <EOL> """<STR_LIT>""" <EOL> self . env = dict ( kv for kv in env . iteritems ( ) if isinstance ( kv [ <NUM_LIT:1> ] , str ) ) <EOL> self . start_timestamp = time . time ( ) <EOL> self . http_status = <NUM_LIT:0> <EOL> self . end_timestamp = self . start_timestamp <EOL> self . traces = [ ] <EOL> self . pending = { } <EOL> self . overhead = ( time . time ( ) - self . start_timestamp ) <EOL> self . _lock = threading . Lock ( ) <EOL> def http_method ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . env . get ( '<STR_LIT>' , '<STR_LIT:GET>' ) <EOL> def http_path ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . env . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> def http_query ( self ) : <EOL> """<STR_LIT>""" <EOL> query_string = self . env . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> if query_string : <EOL> query_string = '<STR_LIT:?>' + query_string <EOL> return query_string <EOL> def record_custom_event ( self , label , data = None , start = None , end = None ) : <EOL> """<STR_LIT>""" <EOL> pre_now = time . time ( ) <EOL> sreq = format_value ( data ) <EOL> now = time . time ( ) <EOL> if end and not start : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> start = start or pre_now <EOL> end = end or pre_now <EOL> delta = int ( <NUM_LIT:1000> * ( start - self . start_timestamp ) ) <EOL> duration = int ( <NUM_LIT:1000> * ( end - start ) ) <EOL> trace = datamodel_pb . IndividualRpcStatsProto ( ) <EOL> self . get_call_stack ( trace ) <EOL> trace . set_service_call_name ( '<STR_LIT>' + label ) <EOL> trace . set_request_data_summary ( sreq ) <EOL> trace . set_start_offset_milliseconds ( delta ) <EOL> trace . set_duration_milliseconds ( duration ) <EOL> with self . _lock : <EOL> self . traces . append ( trace ) <EOL> self . overhead += ( now - pre_now ) <EOL> def record_datastore_details ( self , call , request , response , trace ) : <EOL> """<STR_LIT>""" <EOL> if call == '<STR_LIT>' : <EOL> self . record_put_details ( response , trace ) <EOL> elif call == '<STR_LIT>' : <EOL> self . record_delete_details ( response , trace ) <EOL> elif call == '<STR_LIT>' : <EOL> self . record_commit_details ( response , trace ) <EOL> elif call in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> self . record_query_details ( call , request , response , trace ) <EOL> elif call == '<STR_LIT>' : <EOL> self . record_get_details ( request , response , trace ) <EOL> elif call == '<STR_LIT>' : <EOL> self . record_allocate_ids_details ( trace ) <EOL> def record_put_details ( self , response , trace ) : <EOL> """<STR_LIT>""" <EOL> if config . DATASTORE_DETAILS : <EOL> details = trace . mutable_datastore_details ( ) <EOL> for key in response . key_list ( ) : <EOL> detail = details . add_keys_written ( ) <EOL> detail . CopyFrom ( key ) <EOL> if config . CALC_RPC_COSTS : <EOL> writes = response . cost ( ) . entity_writes ( ) + response . cost ( ) . index_writes ( ) <EOL> trace . set_call_cost_microdollars ( writes * config . DATASTORE_WRITE_OP_COST ) <EOL> _add_billed_op_to_trace ( trace , writes , <EOL> datamodel_pb . BilledOpProto . DATASTORE_WRITE ) <EOL> def record_delete_details ( self , response , trace ) : <EOL> """<STR_LIT>""" <EOL> if config . CALC_RPC_COSTS : <EOL> writes = response . cost ( ) . entity_writes ( ) + response . cost ( ) . index_writes ( ) <EOL> trace . set_call_cost_microdollars ( writes * config . DATASTORE_WRITE_OP_COST ) <EOL> _add_billed_op_to_trace ( trace , writes , <EOL> datamodel_pb . BilledOpProto . DATASTORE_WRITE ) <EOL> def record_commit_details ( self , response , trace ) : <EOL> """<STR_LIT>""" <EOL> if config . CALC_RPC_COSTS : <EOL> cost = response . cost ( ) <EOL> writes = ( cost . commitcost ( ) . requested_entity_puts ( ) + <EOL> cost . commitcost ( ) . requested_entity_deletes ( ) + <EOL> cost . index_writes ( ) ) <EOL> trace . set_call_cost_microdollars ( writes * config . DATASTORE_WRITE_OP_COST ) <EOL> _add_billed_op_to_trace ( trace , writes , <EOL> datamodel_pb . BilledOpProto . DATASTORE_WRITE ) <EOL> def record_get_details ( self , request , response , trace ) : <EOL> """<STR_LIT>""" <EOL> if config . DATASTORE_DETAILS : <EOL> details = trace . mutable_datastore_details ( ) <EOL> for key in request . key_list ( ) : <EOL> detail = details . add_keys_read ( ) <EOL> detail . CopyFrom ( key ) <EOL> for entity_present in response . entity_list ( ) : <EOL> details . add_get_successful_fetch ( entity_present . has_entity ( ) ) <EOL> if config . CALC_RPC_COSTS : <EOL> keys_to_read = len ( request . key_list ( ) ) <EOL> trace . set_call_cost_microdollars ( <EOL> keys_to_read * config . DATASTORE_READ_OP_COST ) <EOL> _add_billed_op_to_trace ( trace , keys_to_read , <EOL> datamodel_pb . BilledOpProto . DATASTORE_READ ) <EOL> def record_query_details ( self , call , request , response , trace ) : <EOL> """<STR_LIT>""" <EOL> details = trace . mutable_datastore_details ( ) <EOL> if not response . small_ops ( ) : <EOL> for entity in response . result_list ( ) : <EOL> detail = details . add_keys_read ( ) <EOL> detail . CopyFrom ( entity . key ( ) ) <EOL> if call == '<STR_LIT>' : <EOL> if config . DATASTORE_DETAILS : <EOL> if request . has_kind ( ) : <EOL> details . set_query_kind ( request . kind ( ) ) <EOL> if request . has_ancestor ( ) : <EOL> ancestor = details . mutable_query_ancestor ( ) <EOL> ancestor . CopyFrom ( request . ancestor ( ) ) <EOL> if response . has_cursor ( ) : <EOL> details . set_query_nextcursor ( response . cursor ( ) . cursor ( ) ) <EOL> baseline_reads = <NUM_LIT:1> <EOL> elif call == '<STR_LIT>' : <EOL> if config . DATASTORE_DETAILS : <EOL> details . set_query_thiscursor ( request . cursor ( ) . cursor ( ) ) <EOL> if response . has_cursor ( ) : <EOL> details . set_query_nextcursor ( response . cursor ( ) . cursor ( ) ) <EOL> baseline_reads = <NUM_LIT:0> <EOL> if config . CALC_RPC_COSTS : <EOL> num_results = len ( response . result_list ( ) ) + response . skipped_results ( ) <EOL> cost_micropennies = config . DATASTORE_READ_OP_COST * baseline_reads <EOL> if response . small_ops ( ) : <EOL> cost_micropennies += config . DATASTORE_SMALL_OP_COST * num_results <EOL> trace . set_call_cost_microdollars ( cost_micropennies ) <EOL> _add_billed_op_to_trace ( trace , baseline_reads , <EOL> datamodel_pb . BilledOpProto . DATASTORE_READ ) <EOL> _add_billed_op_to_trace ( trace , num_results , <EOL> datamodel_pb . BilledOpProto . DATASTORE_SMALL ) <EOL> else : <EOL> cost_micropennies += config . DATASTORE_READ_OP_COST * num_results <EOL> trace . set_call_cost_microdollars ( cost_micropennies ) <EOL> _add_billed_op_to_trace ( trace , num_results + baseline_reads , <EOL> datamodel_pb . BilledOpProto . DATASTORE_READ ) <EOL> def record_allocate_ids_details ( self , trace ) : <EOL> """<STR_LIT>""" <EOL> trace . set_call_cost_microdollars ( config . DATASTORE_SMALL_OP_COST ) <EOL> _add_billed_op_to_trace ( trace , <NUM_LIT:1> , <EOL> datamodel_pb . BilledOpProto . DATASTORE_SMALL ) <EOL> def record_xmpp_details ( self , call , request , trace ) : <EOL> """<STR_LIT>""" <EOL> stanzas = <NUM_LIT:0> <EOL> if call == '<STR_LIT>' : <EOL> stanzas = request . jid_size ( ) <EOL> elif call in ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) : <EOL> stanzas = <NUM_LIT:1> <EOL> trace . set_call_cost_microdollars ( stanzas * config . XMPP_STANZA_COST ) <EOL> _add_billed_op_to_trace ( trace , stanzas , <EOL> datamodel_pb . BilledOpProto . XMPP_STANZA ) <EOL> def record_channel_details ( self , call , trace ) : <EOL> """<STR_LIT>""" <EOL> if call == '<STR_LIT>' : <EOL> trace . set_call_cost_microdollars ( config . CHANNEL_CREATE_COST ) <EOL> _add_billed_op_to_trace ( trace , <NUM_LIT:1> , <EOL> datamodel_pb . BilledOpProto . CHANNEL_OPEN ) <EOL> def record_mail_details ( self , call , request , trace ) : <EOL> """<STR_LIT>""" <EOL> if call in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> num_recipients = ( request . to_size ( ) + request . cc_size ( ) + <EOL> request . bcc_size ( ) ) <EOL> trace . set_call_cost_microdollars ( <EOL> config . MAIL_RECIPIENT_COST * num_recipients ) <EOL> _add_billed_op_to_trace ( trace , num_recipients , <EOL> datamodel_pb . BilledOpProto . MAIL_RECIPIENT ) <EOL> def record_rpc_request ( self , service , call , request , response , rpc ) : <EOL> """<STR_LIT>""" <EOL> pre_now = time . time ( ) <EOL> sreq = format_value ( request ) <EOL> now = time . time ( ) <EOL> delta = int ( <NUM_LIT:1000> * ( now - self . start_timestamp ) ) <EOL> trace = datamodel_pb . IndividualRpcStatsProto ( ) <EOL> self . get_call_stack ( trace ) <EOL> trace . set_service_call_name ( '<STR_LIT>' % ( service , call ) ) <EOL> trace . set_request_data_summary ( sreq ) <EOL> trace . set_start_offset_milliseconds ( delta ) <EOL> with self . _lock : <EOL> if rpc is not None : <EOL> self . pending [ rpc ] = len ( self . traces ) <EOL> self . traces . append ( trace ) <EOL> self . overhead += ( now - pre_now ) <EOL> def record_rpc_response ( self , service , call , request , response , rpc ) : <EOL> """<STR_LIT>""" <EOL> now = time . time ( ) <EOL> key = '<STR_LIT>' % ( service , call ) <EOL> delta = int ( <NUM_LIT:1000> * ( now - self . start_timestamp ) ) <EOL> sresp = format_value ( response ) <EOL> if rpc is not None : <EOL> with self . _lock : <EOL> index = self . pending . get ( rpc ) <EOL> if index is not None : <EOL> del self . pending [ rpc ] <EOL> if <NUM_LIT:0> <= index < len ( self . traces ) : <EOL> trace = self . traces [ index ] <EOL> trace . set_response_data_summary ( sresp ) <EOL> duration = delta - trace . start_offset_milliseconds ( ) <EOL> trace . set_duration_milliseconds ( duration ) <EOL> if ( config . CALC_RPC_COSTS or <EOL> config . DATASTORE_DETAILS ) and service == '<STR_LIT>' : <EOL> self . record_datastore_details ( call , request , response , trace ) <EOL> elif config . CALC_RPC_COSTS and service == '<STR_LIT>' : <EOL> self . record_xmpp_details ( call , request , trace ) <EOL> elif config . CALC_RPC_COSTS and service == '<STR_LIT>' : <EOL> self . record_channel_details ( call , trace ) <EOL> elif config . CALC_RPC_COSTS and service == '<STR_LIT>' : <EOL> self . record_mail_details ( call , request , trace ) <EOL> self . overhead += ( time . time ( ) - now ) <EOL> return <EOL> else : <EOL> with self . _lock : <EOL> for trace in reversed ( self . traces ) : <EOL> if ( trace . service_call_name ( ) == key and <EOL> not trace . response_data_summary ( ) ) : <EOL> if config . DEBUG : <EOL> logging . debug ( '<STR_LIT>' ) <EOL> trace . set_response_data_summary ( sresp ) <EOL> duration = delta - trace . start_offset_milliseconds ( ) <EOL> trace . set_duration_milliseconds ( duration ) <EOL> self . overhead += ( time . time ( ) - now ) <EOL> return <EOL> logging . warn ( '<STR_LIT>' ) <EOL> trace = datamodel_pb . IndividualRpcStatsProto ( ) <EOL> self . get_call_stack ( trace ) <EOL> trace . set_service_call_name ( key ) <EOL> trace . set_request_data_summary ( sresp ) <EOL> trace . set_start_offset_milliseconds ( delta ) <EOL> with self . _lock : <EOL> self . traces . append ( trace ) <EOL> self . overhead += ( time . time ( ) - now ) <EOL> def record_http_status ( self , status ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> self . http_status = int ( status ) <EOL> except ( ValueError , TypeError ) : <EOL> self . http_status = <NUM_LIT:0> <EOL> self . end_timestamp = time . time ( ) <EOL> def save ( self ) : <EOL> """<STR_LIT>""" <EOL> t0 = time . time ( ) <EOL> with self . _lock : <EOL> num_pending = len ( self . pending ) <EOL> if num_pending : <EOL> logging . warn ( '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> num_pending ) <EOL> self . dump ( ) <EOL> try : <EOL> key , len_part , len_full = self . _save ( ) <EOL> except Exception : <EOL> logging . exception ( '<STR_LIT>' ) <EOL> return <EOL> t1 = time . time ( ) <EOL> link = '<STR_LIT>' % ( <EOL> self . env . get ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> config . stats_url , <EOL> int ( self . start_timestamp * <NUM_LIT:1000> ) ) <EOL> logging . info ( '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> key , len_part , len_full , self . overhead , t1 - t0 , link ) <EOL> def _save ( self ) : <EOL> """<STR_LIT>""" <EOL> part , full = self . get_both_protos_encoded ( ) <EOL> key = make_key ( self . start_timestamp ) <EOL> errors = memcache . set_multi ( { config . PART_SUFFIX : part , <EOL> config . FULL_SUFFIX : full } , <EOL> time = <NUM_LIT> * <NUM_LIT> , key_prefix = key , <EOL> namespace = config . KEY_NAMESPACE ) <EOL> if errors : <EOL> logging . warn ( '<STR_LIT>' , errors ) <EOL> return key , len ( part ) , len ( full ) <EOL> def get_both_protos_encoded ( self ) : <EOL> """<STR_LIT>""" <EOL> proto = self . get_summary_proto ( ) <EOL> part_encoded = proto . Encode ( ) <EOL> self . add_full_info_to_proto ( proto ) <EOL> full_encoded = proto . Encode ( ) <EOL> if len ( full_encoded ) <= memcache . MAX_VALUE_SIZE : <EOL> return part_encoded , full_encoded <EOL> if config . MAX_LOCALS > <NUM_LIT:0> : <EOL> for trace in proto . individual_stats_list ( ) : <EOL> for frame in trace . call_stack_list ( ) : <EOL> frame . clear_variables ( ) <EOL> full_encoded = proto . Encode ( ) <EOL> if len ( full_encoded ) <= memcache . MAX_VALUE_SIZE : <EOL> logging . info ( '<STR_LIT>' ) <EOL> return part_encoded , full_encoded <EOL> if config . MAX_STACK > <NUM_LIT:0> : <EOL> for trace in proto . individual_stats_list ( ) : <EOL> trace . clear_call_stack ( ) <EOL> full_encoded = proto . Encode ( ) <EOL> if len ( full_encoded ) <= memcache . MAX_VALUE_SIZE : <EOL> logging . info ( '<STR_LIT>' ) <EOL> return part_encoded , full_encoded <EOL> logging . info ( '<STR_LIT>' ) <EOL> del proto . individual_stats_list ( ) [ <NUM_LIT:100> : ] <EOL> full_encoded = proto . Encode ( ) <EOL> return part_encoded , full_encoded <EOL> def add_full_info_to_proto ( self , proto ) : <EOL> """<STR_LIT>""" <EOL> user_email = self . env . get ( '<STR_LIT>' ) <EOL> if user_email : <EOL> proto . set_user_email ( user_email ) <EOL> if self . env . get ( '<STR_LIT>' ) == '<STR_LIT:1>' : <EOL> proto . set_is_admin ( True ) <EOL> for key , value in sorted ( self . env . iteritems ( ) ) : <EOL> x = proto . add_cgi_env ( ) <EOL> x . set_key ( key ) <EOL> x . set_value ( value ) <EOL> with self . _lock : <EOL> proto . individual_stats_list ( ) . extend ( self . traces ) <EOL> def get_full_proto ( self ) : <EOL> """<STR_LIT>""" <EOL> proto = self . get_summary_proto ( ) <EOL> self . add_full_info_to_proto ( proto ) <EOL> return StatsProto ( proto ) <EOL> def get_summary_proto_encoded ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . get_summary_proto ( ) . Encode ( ) <EOL> def get_summary_proto ( self ) : <EOL> """<STR_LIT>""" <EOL> summary = datamodel_pb . RequestStatProto ( ) <EOL> summary . set_start_timestamp_milliseconds ( int ( self . start_timestamp * <NUM_LIT:1000> ) ) <EOL> method = self . http_method ( ) <EOL> if method != '<STR_LIT:GET>' : <EOL> summary . set_http_method ( method ) <EOL> path = self . http_path ( ) <EOL> if path != '<STR_LIT:/>' : <EOL> summary . set_http_path ( path ) <EOL> query = self . http_query ( ) <EOL> if query : <EOL> summary . set_http_query ( query ) <EOL> status = int ( self . http_status ) <EOL> if status != <NUM_LIT:200> : <EOL> summary . set_http_status ( status ) <EOL> duration = int ( <NUM_LIT:1000> * ( self . end_timestamp - self . start_timestamp ) ) <EOL> summary . set_duration_milliseconds ( duration ) <EOL> summary . set_overhead_walltime_milliseconds ( int ( self . overhead * <NUM_LIT:1000> ) ) <EOL> rpc_stats = self . get_rpcstats ( ) . items ( ) <EOL> rpc_stats . sort ( key = lambda x : ( - x [ <NUM_LIT:1> ] [ <NUM_LIT:0> ] , x [ <NUM_LIT:0> ] ) ) <EOL> for key , value in rpc_stats : <EOL> x = summary . add_rpc_stats ( ) <EOL> x . set_service_call_name ( key ) <EOL> x . set_total_amount_of_calls ( value [ <NUM_LIT:0> ] ) <EOL> x . set_total_cost_of_calls_microdollars ( value [ <NUM_LIT:1> ] ) <EOL> for billed_op in value [ <NUM_LIT:2> ] . itervalues ( ) : <EOL> x . total_billed_ops_list ( ) . append ( billed_op ) <EOL> return summary <EOL> def get_rpcstats ( self ) : <EOL> """<STR_LIT>""" <EOL> rpcstats = { } <EOL> with self . _lock : <EOL> values = [ [ trace . service_call_name ( ) , trace . call_cost_microdollars ( ) , <EOL> trace . billed_ops_list ( ) ] for trace in self . traces ] <EOL> for value in values : <EOL> if value [ <NUM_LIT:0> ] in rpcstats : <EOL> stats_for_rpc = rpcstats [ value [ <NUM_LIT:0> ] ] <EOL> stats_for_rpc [ <NUM_LIT:0> ] += <NUM_LIT:1> <EOL> stats_for_rpc [ <NUM_LIT:1> ] += value [ <NUM_LIT:1> ] <EOL> else : <EOL> rpcstats [ value [ <NUM_LIT:0> ] ] = [ <NUM_LIT:1> , value [ <NUM_LIT:1> ] , { } ] <EOL> _add_billed_ops_to_map ( rpcstats [ value [ <NUM_LIT:0> ] ] [ <NUM_LIT:2> ] , value [ <NUM_LIT:2> ] ) <EOL> return rpcstats <EOL> def get_total_api_mcycles ( self ) : <EOL> """<STR_LIT>""" <EOL> warnings . warn ( '<STR_LIT>' , <EOL> UserWarning , <EOL> stacklevel = <NUM_LIT:2> ) <EOL> return <NUM_LIT:0> <EOL> def dump ( self , level = None ) : <EOL> """<STR_LIT>""" <EOL> if level is None : <EOL> level = config . DUMP_LEVEL <EOL> if level < <NUM_LIT:0> : <EOL> return <EOL> logging . info ( '<STR_LIT>' , <EOL> format_time ( self . start_timestamp ) , <EOL> self . http_method ( ) , <EOL> self . http_path ( ) , <EOL> self . http_query ( ) , <EOL> self . http_status , <EOL> self . end_timestamp - self . start_timestamp ) <EOL> for key , value in sorted ( self . get_rpcstats ( ) . iteritems ( ) ) : <EOL> logging . info ( '<STR_LIT>' , key , value ) <EOL> if level <= <NUM_LIT:0> : <EOL> return <EOL> with self . _lock : <EOL> for trace in self . traces : <EOL> start = trace . start_offset_milliseconds ( ) <EOL> logging . info ( '<STR_LIT>' , <EOL> trace . start_offset_milliseconds ( ) , <EOL> trace . service_call_name ( ) , <EOL> trace . duration_milliseconds ( ) ) <EOL> logging . info ( '<STR_LIT>' , trace . request_data_summary ( ) ) <EOL> logging . info ( '<STR_LIT>' , trace . response_data_summary ( ) ) <EOL> if level <= <NUM_LIT:1> : <EOL> continue <EOL> for entry in trace . call_stack_list ( ) : <EOL> logging . info ( '<STR_LIT>' , <EOL> entry . class_or_file_name ( ) , <EOL> entry . line_number ( ) , <EOL> entry . function_name ( ) ) <EOL> for variable in entry . variables_list ( ) : <EOL> logging . info ( '<STR_LIT>' , variable . key ( ) , variable . value ( ) ) <EOL> def get_call_stack ( self , trace ) : <EOL> """<STR_LIT>""" <EOL> frame = sys . _getframe ( <NUM_LIT:0> ) <EOL> while frame is not None and trace . call_stack_size ( ) < config . MAX_STACK : <EOL> if not self . get_frame_summary ( frame , trace ) : <EOL> break <EOL> frame = frame . f_back <EOL> sys_path_entries = None <EOL> @ classmethod <EOL> def init_sys_path_entries ( cls ) : <EOL> """<STR_LIT>""" <EOL> cls . sys_path_entries = sorted ( enumerate ( sys . path ) , <EOL> key = lambda x : ( - len ( x [ <NUM_LIT:1> ] ) , x [ <NUM_LIT:0> ] ) ) <EOL> def get_frame_summary ( self , frame , trace ) : <EOL> """<STR_LIT>""" <EOL> if self . sys_path_entries is None : <EOL> self . init_sys_path_entries ( ) <EOL> filename = frame . f_code . co_filename <EOL> if filename and not ( filename . startswith ( '<STR_LIT:<>' ) and filename . endswith ( '<STR_LIT:>>' ) ) : <EOL> for i , entry in self . sys_path_entries : <EOL> if filename . startswith ( entry ) : <EOL> filename = '<STR_LIT>' % i + filename [ len ( entry ) : ] <EOL> break <EOL> funcname = frame . f_code . co_name <EOL> lineno = frame . f_lineno <EOL> code_key = '<STR_LIT>' % ( filename , funcname , lineno ) <EOL> if re . search ( config . RE_STACK_BOTTOM , code_key ) : <EOL> return False <EOL> if re . search ( config . RE_STACK_SKIP , code_key ) : <EOL> return True <EOL> entry = trace . add_call_stack ( ) <EOL> entry . set_class_or_file_name ( filename ) <EOL> entry . set_line_number ( lineno ) <EOL> entry . set_function_name ( funcname ) <EOL> if frame . f_globals is frame . f_locals : <EOL> return True <EOL> max_locals = config . MAX_LOCALS <EOL> if max_locals <= <NUM_LIT:0> : <EOL> return True <EOL> for name , value in sorted ( frame . f_locals . iteritems ( ) ) : <EOL> x = entry . add_variables ( ) <EOL> x . set_key ( name ) <EOL> x . set_value ( format_value ( value ) ) <EOL> max_locals -= <NUM_LIT:1> <EOL> if max_locals <= <NUM_LIT:0> : <EOL> break <EOL> return True <EOL> def mcycles_to_seconds ( mcycles ) : <EOL> """<STR_LIT>""" <EOL> if mcycles is None : <EOL> return <NUM_LIT:0> <EOL> return quota . megacycles_to_cpu_seconds ( mcycles ) <EOL> def mcycles_to_msecs ( mcycles ) : <EOL> """<STR_LIT>""" <EOL> return int ( mcycles_to_seconds ( mcycles ) * <NUM_LIT:1000> ) <EOL> def make_key ( timestamp ) : <EOL> """<STR_LIT>""" <EOL> distance = config . KEY_DISTANCE <EOL> modulus = config . KEY_MODULUS <EOL> tmpl = config . KEY_PREFIX + config . KEY_TEMPLATE <EOL> msecs = int ( timestamp * <NUM_LIT:1000> ) <EOL> index = ( ( msecs // distance ) % modulus ) * distance <EOL> return tmpl % index <EOL> def format_time ( timestamp ) : <EOL> """<STR_LIT>""" <EOL> timestamp = datetime . datetime . utcfromtimestamp ( timestamp ) <EOL> timestamp -= datetime . timedelta ( seconds = config . TZOFFSET ) <EOL> return timestamp . isoformat ( ) [ : - <NUM_LIT:3> ] . replace ( '<STR_LIT:T>' , '<STR_LIT:U+0020>' ) <EOL> def format_value ( val ) : <EOL> """<STR_LIT>""" <EOL> return formatting . _format_value ( val , config . MAX_REPR , config . MAX_DEPTH ) <EOL> def billed_ops_to_str ( billed_ops_list ) : <EOL> """<STR_LIT>""" <EOL> ops_as_strs = [ ] <EOL> for op in billed_ops_list : <EOL> op_name = datamodel_pb . BilledOpProto . BilledOp_Name ( op . op ( ) ) <EOL> ops_as_strs . append ( '<STR_LIT>' % ( op_name , op . num_ops ( ) ) ) <EOL> return '<STR_LIT:U+002CU+0020>' . join ( ops_as_strs ) <EOL> def total_billed_ops_to_str ( self ) : <EOL> """<STR_LIT>""" <EOL> return billed_ops_to_str ( self . total_billed_ops_list ( ) ) <EOL> def individual_billed_ops_to_str ( self ) : <EOL> """<STR_LIT>""" <EOL> return billed_ops_to_str ( self . billed_ops_list ( ) ) <EOL> class StatsProto ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , proto = None ) : <EOL> if not isinstance ( proto , datamodel_pb . RequestStatProto ) : <EOL> proto = datamodel_pb . RequestStatProto ( proto ) <EOL> self . _proto = proto <EOL> def __getattr__ ( self , key ) : <EOL> return getattr ( self . _proto , key ) <EOL> def start_time_formatted ( self ) : <EOL> """<STR_LIT>""" <EOL> return format_time ( self . start_timestamp_milliseconds ( ) * <NUM_LIT> ) <EOL> def api_milliseconds ( self ) : <EOL> """<STR_LIT>""" <EOL> warnings . warn ( '<STR_LIT>' , <EOL> UserWarning , <EOL> stacklevel = <NUM_LIT:2> ) <EOL> return <NUM_LIT:0> <EOL> def processor_mcycles ( self ) : <EOL> warnings . warn ( '<STR_LIT>' , <EOL> UserWarning , <EOL> stacklevel = <NUM_LIT:2> ) <EOL> return self . _proto . processor_mcycles ( ) <EOL> def processor_milliseconds ( self ) : <EOL> """<STR_LIT>""" <EOL> warnings . warn ( '<STR_LIT>' , <EOL> UserWarning , <EOL> stacklevel = <NUM_LIT:2> ) <EOL> return mcycles_to_msecs ( self . _proto . processor_mcycles ( ) ) <EOL> __combined_rpc_count = None <EOL> def combined_rpc_count ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . __combined_rpc_count is None : <EOL> self . __combined_rpc_count = sum ( x . total_amount_of_calls ( ) <EOL> for x in self . rpc_stats_list ( ) ) <EOL> return self . __combined_rpc_count <EOL> __combined_rpc_cost_micropennies = None <EOL> def combined_rpc_cost_micropennies ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . __combined_rpc_cost_micropennies is None : <EOL> self . __combined_rpc_cost_micropennies = ( <EOL> sum ( x . total_cost_of_calls_microdollars ( ) <EOL> for x in self . rpc_stats_list ( ) ) ) <EOL> return self . __combined_rpc_cost_micropennies <EOL> __combined_rpc_billed_ops = None <EOL> def combined_rpc_billed_ops ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . __combined_rpc_billed_ops is None : <EOL> combined_ops_dict = { } <EOL> for stats in self . rpc_stats_list ( ) : <EOL> _add_billed_ops_to_map ( combined_ops_dict , <EOL> stats . total_billed_ops_list ( ) ) <EOL> self . __combined_rpc_billed_ops = billed_ops_to_str ( <EOL> combined_ops_dict . itervalues ( ) ) <EOL> return self . __combined_rpc_billed_ops <EOL> def load_summary_protos ( java_application = False ) : <EOL> """<STR_LIT>""" <EOL> tmpl = config . KEY_PREFIX + config . KEY_TEMPLATE + config . PART_SUFFIX <EOL> if java_application : <EOL> tmpl = '<STR_LIT:">' + tmpl + '<STR_LIT:">' <EOL> keys = [ tmpl % i <EOL> for i in <EOL> range ( <NUM_LIT:0> , config . KEY_DISTANCE * config . KEY_MODULUS , <EOL> config . KEY_DISTANCE ) ] <EOL> results = memcache . get_multi ( keys , namespace = config . KEY_NAMESPACE ) <EOL> records = [ ] <EOL> for rec in results . itervalues ( ) : <EOL> try : <EOL> pb = StatsProto ( rec ) <EOL> except Exception , err : <EOL> logging . warn ( '<STR_LIT>' , err ) <EOL> else : <EOL> records . append ( pb ) <EOL> logging . info ( '<STR_LIT>' , <EOL> len ( results ) , len ( records ) ) <EOL> records . sort ( key = lambda pb : - pb . start_timestamp_milliseconds ( ) ) <EOL> return records <EOL> def load_full_proto ( timestamp , java_application = False ) : <EOL> """<STR_LIT>""" <EOL> full_key = make_key ( timestamp ) + config . FULL_SUFFIX <EOL> if java_application : <EOL> full_key = '<STR_LIT:">' + full_key + '<STR_LIT:">' <EOL> full_binary = memcache . get ( full_key , namespace = config . KEY_NAMESPACE ) <EOL> if full_binary is None : <EOL> logging . debug ( '<STR_LIT>' , full_key ) <EOL> return None <EOL> try : <EOL> full = StatsProto ( full_binary ) <EOL> except Exception , err : <EOL> logging . warn ( '<STR_LIT>' , full_key , err ) <EOL> return None <EOL> if full . start_timestamp_milliseconds ( ) != int ( timestamp * <NUM_LIT:1000> ) : <EOL> logging . debug ( '<STR_LIT>' , <EOL> int ( timestamp * <NUM_LIT:1000> ) , full . start_timestamp_milliseconds ( ) ) <EOL> return None <EOL> return full <EOL> class AppstatsDjangoMiddleware ( object ) : <EOL> """<STR_LIT>""" <EOL> def process_request ( self , request ) : <EOL> """<STR_LIT>""" <EOL> start_recording ( ) <EOL> def process_response ( self , request , response ) : <EOL> """<STR_LIT>""" <EOL> end_recording ( response . status_code ) <EOL> return response <EOL> AppStatsDjangoMiddleware = AppstatsDjangoMiddleware <EOL> def appstats_wsgi_middleware ( app ) : <EOL> """<STR_LIT>""" <EOL> def appstats_wsgi_wrapper ( environ , start_response ) : <EOL> """<STR_LIT>""" <EOL> start_recording ( environ ) <EOL> save_status = [ None ] <EOL> datamodel_pb . AggregateRpcStatsProto . total_billed_ops_str = ( <EOL> total_billed_ops_to_str ) <EOL> datamodel_pb . IndividualRpcStatsProto . billed_ops_str = ( <EOL> individual_billed_ops_to_str ) <EOL> def appstats_start_response ( status , headers , exc_info = None ) : <EOL> """<STR_LIT>""" <EOL> save_status . append ( status ) <EOL> return start_response ( status , headers , exc_info ) <EOL> try : <EOL> result = app ( environ , appstats_start_response ) <EOL> except Exception : <EOL> end_recording ( <NUM_LIT> ) <EOL> raise <EOL> if result is not None : <EOL> for value in result : <EOL> yield value <EOL> status = save_status [ - <NUM_LIT:1> ] <EOL> if status is not None : <EOL> status = status [ : <NUM_LIT:3> ] <EOL> end_recording ( status ) <EOL> return appstats_wsgi_wrapper <EOL> def _synchronized ( method ) : <EOL> """<STR_LIT>""" <EOL> def synchronized_wrapper ( self , * args ) : <EOL> with self . _lock : <EOL> return method ( self , * args ) <EOL> return synchronized_wrapper <EOL> class RequestLocalRecorderProxy ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> self . _recorders = { } <EOL> self . _lock = threading . RLock ( ) <EOL> @ _synchronized <EOL> def __getattr__ ( self , key ) : <EOL> if not self . has_recorder_for_current_request ( ) : <EOL> raise AttributeError ( '<STR_LIT>' ) <EOL> return getattr ( self . get_for_current_request ( ) , key ) <EOL> @ _synchronized <EOL> def has_recorder_for_current_request ( self ) : <EOL> """<STR_LIT>""" <EOL> return os . environ . get ( '<STR_LIT>' ) in self . _recorders <EOL> @ _synchronized <EOL> def set_for_current_request ( self , new_recorder ) : <EOL> """<STR_LIT>""" <EOL> self . _recorders [ os . environ . get ( '<STR_LIT>' ) ] = new_recorder <EOL> _set_global_recorder ( new_recorder ) <EOL> @ _synchronized <EOL> def get_for_current_request ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . _recorders . get ( os . environ . get ( '<STR_LIT>' ) ) <EOL> @ _synchronized <EOL> def clear_for_current_request ( self ) : <EOL> """<STR_LIT>""" <EOL> if os . environ . get ( '<STR_LIT>' ) in self . _recorders : <EOL> del self . _recorders [ os . environ . get ( '<STR_LIT>' ) ] <EOL> _clear_global_recorder ( ) <EOL> @ _synchronized <EOL> def _clear_all ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _recorders . clear ( ) <EOL> _clear_global_recorder ( ) <EOL> def _set_global_recorder ( new_recorder ) : <EOL> if os . environ . get ( '<STR_LIT>' ) != '<STR_LIT>' : <EOL> global recorder <EOL> recorder = new_recorder <EOL> def _clear_global_recorder ( ) : <EOL> _set_global_recorder ( None ) <EOL> recorder_proxy = RequestLocalRecorderProxy ( ) <EOL> if os . environ . get ( '<STR_LIT>' ) != '<STR_LIT>' : <EOL> recorder = None <EOL> def dont_record ( ) : <EOL> """<STR_LIT>""" <EOL> recorder_proxy . clear_for_current_request ( ) <EOL> def lock_key ( ) : <EOL> """<STR_LIT>""" <EOL> return config . KEY_PREFIX + config . LOCK_SUFFIX <EOL> def start_recording ( env = None ) : <EOL> """<STR_LIT>""" <EOL> recorder_proxy . clear_for_current_request ( ) <EOL> if env is None : <EOL> env = os . environ <EOL> if not config . should_record ( env ) : <EOL> return <EOL> if memcache . add ( lock_key ( ) , <NUM_LIT:0> , <EOL> time = config . LOCK_TIMEOUT , namespace = config . KEY_NAMESPACE ) : <EOL> recorder_proxy . set_for_current_request ( Recorder ( env ) ) <EOL> if config . DEBUG : <EOL> logging . debug ( '<STR_LIT>' ) <EOL> def end_recording ( status , firepython_set_extension_data = None ) : <EOL> """<STR_LIT>""" <EOL> if firepython_set_extension_data is not None : <EOL> warnings . warn ( '<STR_LIT>' ) <EOL> rec = recorder_proxy . get_for_current_request ( ) <EOL> recorder_proxy . clear_for_current_request ( ) <EOL> if config . DEBUG : <EOL> logging . debug ( '<STR_LIT>' ) <EOL> if rec is not None : <EOL> try : <EOL> rec . record_http_status ( status ) <EOL> rec . save ( ) <EOL> finally : <EOL> memcache . delete ( lock_key ( ) , namespace = config . KEY_NAMESPACE ) <EOL> def pre_call_hook ( service , call , request , response , rpc = None ) : <EOL> """<STR_LIT>""" <EOL> if recorder_proxy . has_recorder_for_current_request ( ) : <EOL> if config . DEBUG : <EOL> logging . debug ( '<STR_LIT>' , service , call ) <EOL> recorder_proxy . record_rpc_request ( service , call , request , response , rpc ) <EOL> def post_call_hook ( service , call , request , response , rpc = None , error = None ) : <EOL> """<STR_LIT>""" <EOL> if recorder_proxy . has_recorder_for_current_request ( ) : <EOL> if config . DEBUG : <EOL> logging . debug ( '<STR_LIT>' , service , call ) <EOL> recorder_proxy . record_rpc_response ( service , call , request , response , rpc ) <EOL> def _add_billed_ops_to_map ( billed_ops_dict , billed_ops_list ) : <EOL> """<STR_LIT>""" <EOL> for billed_op in billed_ops_list : <EOL> if billed_op . op ( ) not in billed_ops_dict : <EOL> update_me = datamodel_pb . BilledOpProto ( ) <EOL> update_me . set_op ( billed_op . op ( ) ) <EOL> update_me . set_num_ops ( <NUM_LIT:0> ) <EOL> billed_ops_dict [ billed_op . op ( ) ] = update_me <EOL> update_me = billed_ops_dict [ billed_op . op ( ) ] <EOL> update_me . set_num_ops ( update_me . num_ops ( ) + billed_op . num_ops ( ) ) <EOL> def _add_billed_op_to_trace ( trace , num_ops , op ) : <EOL> """<STR_LIT>""" <EOL> if num_ops : <EOL> billed_op = trace . add_billed_ops ( ) <EOL> billed_op . set_num_ops ( num_ops ) <EOL> billed_op . set_op ( op ) <EOL> apiproxy_stub_map . apiproxy . GetPreCallHooks ( ) . Append ( '<STR_LIT>' , pre_call_hook ) <EOL> apiproxy_stub_map . apiproxy . GetPostCallHooks ( ) . Append ( '<STR_LIT>' , post_call_hook ) </s>
