<s> """<STR_LIT>""" <EOL> import cgi <EOL> import cStringIO <EOL> import email . Utils <EOL> import logging <EOL> import mimetypes <EOL> import os <EOL> import re <EOL> import sys <EOL> import time <EOL> import traceback <EOL> from google . appengine . api import users <EOL> from google . appengine . ext import admin <EOL> from google . appengine . ext import webapp <EOL> from google . appengine . ext . appstats import datamodel_pb <EOL> from google . appengine . ext . appstats import recording <EOL> from google . appengine . ext . webapp import _template <EOL> from google . appengine . ext . webapp import util <EOL> DEBUG = recording . config . DEBUG <EOL> def _add_billed_ops_to_map ( billed_ops_map , billed_ops_list ) : <EOL> """<STR_LIT>""" <EOL> for billed_op in billed_ops_list : <EOL> if billed_op . op ( ) not in billed_ops_map : <EOL> update_me = datamodel_pb . BilledOpProto ( ) <EOL> update_me . set_op ( billed_op . op ( ) ) <EOL> update_me . set_num_ops ( <NUM_LIT:0> ) <EOL> billed_ops_map [ billed_op . op ( ) ] = update_me <EOL> update_me = billed_ops_map [ billed_op . op ( ) ] <EOL> update_me . set_num_ops ( update_me . num_ops ( ) + billed_op . num_ops ( ) ) <EOL> def _billed_ops_to_str ( billed_ops ) : <EOL> """<STR_LIT>""" <EOL> ops_as_strs = [ ] <EOL> for op in billed_ops : <EOL> op_name = datamodel_pb . BilledOpProto . BilledOp_Name ( op . op ( ) ) <EOL> ops_as_strs . append ( '<STR_LIT>' % ( op_name , op . num_ops ( ) ) ) <EOL> return '<STR_LIT:U+002CU+0020>' . join ( ops_as_strs ) <EOL> def _as_percentage_of ( cost_micropennies , total_cost_micropennies ) : <EOL> """<STR_LIT>""" <EOL> if total_cost_micropennies == <NUM_LIT:0> : <EOL> return <NUM_LIT:0> <EOL> return round ( ( float ( cost_micropennies ) / float ( total_cost_micropennies ) ) <EOL> * <NUM_LIT:100> , <NUM_LIT:1> ) <EOL> def render ( tmplname , data ) : <EOL> """<STR_LIT>""" <EOL> here = os . path . dirname ( __file__ ) <EOL> tmpl = os . path . join ( here , '<STR_LIT>' , tmplname ) <EOL> data [ '<STR_LIT>' ] = os . environ <EOL> data [ '<STR_LIT>' ] = recording . config . SHELL_OK <EOL> data [ '<STR_LIT>' ] = os . getenv ( '<STR_LIT>' ) <EOL> try : <EOL> return _template . render ( tmpl , data ) <EOL> except Exception , err : <EOL> logging . exception ( '<STR_LIT>' , tmpl ) <EOL> return '<STR_LIT>' % ( tmplname , err ) <EOL> class AllStatsInfo ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , calls , cost , billed_ops ) : <EOL> self . calls = calls <EOL> self . cost = cost <EOL> self . billed_ops = billed_ops <EOL> class PathStatsInfo ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , cost , billed_ops , num_requests , most_recent_requests ) : <EOL> self . cost = cost <EOL> self . billed_ops = billed_ops <EOL> self . num_requests = num_requests <EOL> self . most_recent_requests = most_recent_requests <EOL> class PivotInfo ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , name , calls , cost , billed_ops , cost_pct ) : <EOL> self . name = name <EOL> self . calls = calls <EOL> self . cost = cost <EOL> self . billed_ops = billed_ops <EOL> self . cost_pct = cost_pct <EOL> def to_list ( self ) : <EOL> """<STR_LIT>""" <EOL> return [ self . name , self . calls , self . cost , self . billed_ops , self . cost_pct ] <EOL> @ classmethod <EOL> def from_list ( cls , values ) : <EOL> return cls ( values [ <NUM_LIT:0> ] , values [ <NUM_LIT:1> ] , values [ <NUM_LIT:2> ] , values [ <NUM_LIT:3> ] , values [ <NUM_LIT:4> ] ) <EOL> class SummaryHandler ( webapp . RequestHandler ) : <EOL> """<STR_LIT>""" <EOL> def get ( self ) : <EOL> recording . dont_record ( ) <EOL> if not self . request . path . endswith ( '<STR_LIT:/>' ) : <EOL> self . redirect ( self . request . path + '<STR_LIT:/>' ) <EOL> return <EOL> summaries = recording . load_summary_protos ( ) <EOL> data = self . _get_summary_data ( summaries ) <EOL> self . response . out . write ( render ( '<STR_LIT>' , data ) ) <EOL> def _get_summary_data ( self , summaries ) : <EOL> """<STR_LIT>""" <EOL> allstats = { } <EOL> pathstats = { } <EOL> pivot_path_rpc = { } <EOL> pivot_rpc_path = { } <EOL> total_cost_micropennies = <NUM_LIT:0> <EOL> summaries = sorted ( summaries , <EOL> key = lambda x : ( - x . start_timestamp_milliseconds ( ) ) ) <EOL> for index , summary in enumerate ( summaries ) : <EOL> path_key = recording . config . extract_key ( summary ) <EOL> if path_key not in pathstats : <EOL> pathstats [ path_key ] = PathStatsInfo ( <NUM_LIT:0> , { } , <NUM_LIT:1> , [ index + <NUM_LIT:1> ] ) <EOL> else : <EOL> pathstats_info = pathstats [ path_key ] <EOL> pathstats_info . num_requests += <NUM_LIT:1> <EOL> if len ( pathstats_info . most_recent_requests ) > <NUM_LIT:10> : <EOL> if pathstats_info . most_recent_requests [ - <NUM_LIT:1> ] : <EOL> pathstats_info . most_recent_requests . append ( <NUM_LIT:0> ) <EOL> else : <EOL> pathstats_info . most_recent_requests . append ( index + <NUM_LIT:1> ) <EOL> if path_key not in pivot_path_rpc : <EOL> pivot_path_rpc [ path_key ] = { } <EOL> for x in summary . rpc_stats_list ( ) : <EOL> rpc_key = x . service_call_name ( ) <EOL> total_calls = x . total_amount_of_calls ( ) <EOL> cost_micropennies = x . total_cost_of_calls_microdollars ( ) <EOL> total_cost_micropennies += cost_micropennies <EOL> pathstats [ path_key ] . cost += cost_micropennies <EOL> _add_billed_ops_to_map ( pathstats [ path_key ] . billed_ops , <EOL> x . total_billed_ops_list ( ) ) <EOL> if rpc_key in allstats : <EOL> allstats [ rpc_key ] . calls += total_calls <EOL> allstats [ rpc_key ] . cost += cost_micropennies <EOL> else : <EOL> allstats [ rpc_key ] = AllStatsInfo ( total_calls , cost_micropennies , { } ) <EOL> _add_billed_ops_to_map ( <EOL> allstats [ rpc_key ] . billed_ops , x . total_billed_ops_list ( ) ) <EOL> if rpc_key not in pivot_path_rpc [ path_key ] : <EOL> pivot_path_rpc [ path_key ] [ rpc_key ] = PivotInfo ( rpc_key , <NUM_LIT:0> , <NUM_LIT:0> , { } , <NUM_LIT:0> ) <EOL> pivot_path_rpc [ path_key ] [ rpc_key ] . calls += total_calls <EOL> pivot_path_rpc [ path_key ] [ rpc_key ] . cost += cost_micropennies <EOL> _add_billed_ops_to_map ( pivot_path_rpc [ path_key ] [ rpc_key ] . billed_ops , <EOL> x . total_billed_ops_list ( ) ) <EOL> if rpc_key not in pivot_rpc_path : <EOL> pivot_rpc_path [ rpc_key ] = { } <EOL> if path_key not in pivot_rpc_path [ rpc_key ] : <EOL> pivot_rpc_path [ rpc_key ] [ path_key ] = PivotInfo ( path_key , <NUM_LIT:0> , <NUM_LIT:0> , { } , <NUM_LIT:0> ) <EOL> pivot_rpc_path [ rpc_key ] [ path_key ] . calls += total_calls <EOL> pivot_rpc_path [ rpc_key ] [ path_key ] . cost += cost_micropennies <EOL> _add_billed_ops_to_map ( pivot_rpc_path [ rpc_key ] [ path_key ] . billed_ops , <EOL> x . total_billed_ops_list ( ) ) <EOL> allstats_by_count = [ ] <EOL> for k , v in allstats . iteritems ( ) : <EOL> for path_vals in pivot_rpc_path [ k ] . itervalues ( ) : <EOL> path_vals . billed_ops = _billed_ops_to_str ( <EOL> path_vals . billed_ops . itervalues ( ) ) <EOL> path_vals . cost_pct = _as_percentage_of ( <EOL> path_vals . cost , total_cost_micropennies ) <EOL> pivot = sorted ( pivot_rpc_path [ k ] . itervalues ( ) , <EOL> key = lambda x : ( - x . calls , x . name ) ) <EOL> allstats_by_count . append ( ( <EOL> k , v . calls , v . cost , _billed_ops_to_str ( v . billed_ops . itervalues ( ) ) , <EOL> _as_percentage_of ( v . cost , total_cost_micropennies ) , <EOL> [ x . to_list ( ) for x in pivot ] ) ) <EOL> allstats_by_count . sort ( key = lambda x : ( - x [ <NUM_LIT:1> ] , x [ <NUM_LIT:0> ] ) ) <EOL> pathstats_by_count = [ ] <EOL> for path_key , pathstats_info in pathstats . iteritems ( ) : <EOL> rpc_count = <NUM_LIT:0> <EOL> for rpc_vals in pivot_path_rpc [ path_key ] . itervalues ( ) : <EOL> rpc_vals . billed_ops = _billed_ops_to_str ( <EOL> rpc_vals . billed_ops . itervalues ( ) ) <EOL> rpc_vals . cost_pct = _as_percentage_of ( <EOL> rpc_vals . cost , total_cost_micropennies ) <EOL> rpc_count += rpc_vals . calls <EOL> pivot = sorted ( pivot_path_rpc [ path_key ] . itervalues ( ) , <EOL> key = lambda x : ( - x . calls , x . name ) ) <EOL> pathstats_by_count . append ( ( <EOL> path_key , rpc_count , pathstats_info . cost , <EOL> _billed_ops_to_str ( pathstats_info . billed_ops . itervalues ( ) ) , <EOL> _as_percentage_of ( pathstats_info . cost , total_cost_micropennies ) , <EOL> pathstats_info . num_requests , <EOL> pathstats_info . most_recent_requests , <EOL> [ x . to_list ( ) for x in pivot ] ) ) <EOL> pathstats_by_count . sort ( key = lambda x : ( - x [ <NUM_LIT:1> ] , - x [ <NUM_LIT:5> ] , x [ <NUM_LIT:0> ] ) ) <EOL> return { '<STR_LIT>' : summaries , <EOL> '<STR_LIT>' : allstats_by_count , <EOL> '<STR_LIT>' : pathstats_by_count , <EOL> } <EOL> class DetailsHandler ( webapp . RequestHandler ) : <EOL> """<STR_LIT>""" <EOL> def get ( self ) : <EOL> recording . dont_record ( ) <EOL> time_key = self . request . get ( '<STR_LIT:time>' ) <EOL> timestamp = None <EOL> record = None <EOL> if time_key : <EOL> try : <EOL> timestamp = int ( time_key ) * <NUM_LIT> <EOL> except Exception : <EOL> pass <EOL> if timestamp : <EOL> record = recording . load_full_proto ( timestamp ) <EOL> render_record ( self . response , record , '<STR_LIT>' ) <EOL> def render_record ( response , record , file_url = None , extra_data = None ) : <EOL> """<STR_LIT>""" <EOL> data = { } <EOL> if extra_data is not None : <EOL> data . update ( extra_data ) <EOL> if record is None : <EOL> if extra_data is None : <EOL> response . set_status ( <NUM_LIT> ) <EOL> response . out . write ( render ( '<STR_LIT>' , data ) ) <EOL> return <EOL> data . update ( get_details_data ( record , file_url ) ) <EOL> response . out . write ( render ( '<STR_LIT>' , data ) ) <EOL> def get_details_data ( record , file_url = None ) : <EOL> """<STR_LIT>""" <EOL> rpcstats_map = { } <EOL> for rpc_stat in record . individual_stats_list ( ) : <EOL> key = rpc_stat . service_call_name ( ) <EOL> count , real , api , rpc_cost_micropennies , billed_ops = rpcstats_map . get ( <EOL> key , ( <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> , { } ) ) <EOL> count += <NUM_LIT:1> <EOL> real += rpc_stat . duration_milliseconds ( ) <EOL> api += rpc_stat . api_mcycles ( ) <EOL> rpc_cost_micropennies += rpc_stat . call_cost_microdollars ( ) <EOL> _add_billed_ops_to_map ( billed_ops , rpc_stat . billed_ops_list ( ) ) <EOL> rpcstats_map [ key ] = ( count , real , api , rpc_cost_micropennies , billed_ops ) <EOL> rpcstats_by_count = [ <EOL> ( name , count , real , recording . mcycles_to_msecs ( api ) , <EOL> rpc_cost_micropennies , _billed_ops_to_str ( billed_ops . itervalues ( ) ) ) <EOL> for name , ( count , real , api , rpc_cost_micropennies , billed_ops ) <EOL> in rpcstats_map . iteritems ( ) ] <EOL> rpcstats_by_count . sort ( key = lambda x : - x [ <NUM_LIT:1> ] ) <EOL> real_total = <NUM_LIT:0> <EOL> api_total_mcycles = <NUM_LIT:0> <EOL> for i , rpc_stat in enumerate ( record . individual_stats_list ( ) ) : <EOL> real_total += rpc_stat . duration_milliseconds ( ) <EOL> api_total_mcycles += rpc_stat . api_mcycles ( ) <EOL> api_total = recording . mcycles_to_msecs ( api_total_mcycles ) <EOL> return { '<STR_LIT>' : sys , <EOL> '<STR_LIT>' : record , <EOL> '<STR_LIT>' : rpcstats_by_count , <EOL> '<STR_LIT>' : real_total , <EOL> '<STR_LIT>' : api_total , <EOL> '<STR_LIT>' : file_url , <EOL> } <EOL> class ShellHandler ( webapp . RequestHandler ) : <EOL> """<STR_LIT>""" <EOL> def _check_access ( self ) : <EOL> if recording . config . SHELL_OK : <EOL> return True <EOL> self . response . set_status ( <NUM_LIT> ) <EOL> self . response . out . write ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> return False <EOL> def get ( self ) : <EOL> recording . dont_record ( ) <EOL> if not self . _check_access ( ) : <EOL> return <EOL> script = self . request . get ( '<STR_LIT>' , recording . config . DEFAULT_SCRIPT ) <EOL> extra_data = { '<STR_LIT>' : True , <EOL> '<STR_LIT>' : script , <EOL> '<STR_LIT>' : admin . get_xsrf_token ( ) , <EOL> } <EOL> render_record ( self . response , None , '<STR_LIT>' , extra_data ) <EOL> @ admin . xsrf_required <EOL> def post ( self ) : <EOL> recording . dont_record ( ) <EOL> if not self . _check_access ( ) : <EOL> return <EOL> recorder = recording . Recorder ( os . environ ) <EOL> recording . recorder_proxy . set_for_current_request ( recorder ) <EOL> script = self . request . get ( '<STR_LIT>' , '<STR_LIT>' ) . replace ( '<STR_LIT:\r\n>' , '<STR_LIT:\n>' ) <EOL> output , errors = self . execute_script ( script ) <EOL> recording . recorder_proxy . clear_for_current_request ( ) <EOL> recorder . record_http_status ( <NUM_LIT:0> ) <EOL> recorder . save ( ) <EOL> record = recorder . get_full_proto ( ) <EOL> extra_data = { '<STR_LIT>' : True , <EOL> '<STR_LIT>' : script , <EOL> '<STR_LIT>' : output , <EOL> '<STR_LIT>' : errors , <EOL> '<STR_LIT>' : int ( recorder . start_timestamp * <NUM_LIT:1000> ) , <EOL> '<STR_LIT>' : admin . get_xsrf_token ( ) , <EOL> } <EOL> render_record ( self . response , record , '<STR_LIT>' , extra_data ) <EOL> def execute_script ( self , script ) : <EOL> save_stdout = sys . stdout <EOL> save_stderr = sys . stderr <EOL> new_stdout = cStringIO . StringIO ( ) <EOL> new_stderr = cStringIO . StringIO ( ) <EOL> try : <EOL> sys . stdout = new_stdout <EOL> sys . stderr = new_stderr <EOL> exec ( script , { } ) <EOL> except BaseException : <EOL> traceback . print_exc ( ) <EOL> finally : <EOL> sys . stdout = save_stdout <EOL> sys . stderr = save_stderr <EOL> return new_stdout . getvalue ( ) , new_stderr . getvalue ( ) <EOL> class FileHandler ( webapp . RequestHandler ) : <EOL> """<STR_LIT>""" <EOL> def get ( self ) : <EOL> recording . dont_record ( ) <EOL> lineno = self . request . get ( '<STR_LIT:n>' ) <EOL> try : <EOL> lineno = int ( lineno ) <EOL> except : <EOL> lineno = <NUM_LIT:0> <EOL> filename = self . request . get ( '<STR_LIT:f>' ) or '<STR_LIT>' <EOL> orig_filename = filename <EOL> match = re . match ( '<STR_LIT>' , filename ) <EOL> if match : <EOL> index , tail = match . groups ( ) <EOL> index = int ( index ) <EOL> if index < len ( sys . path ) : <EOL> filename = sys . path [ index ] + tail <EOL> try : <EOL> fp = open ( filename ) <EOL> except IOError , err : <EOL> self . response . out . write ( '<STR_LIT>' % <EOL> cgi . escape ( str ( err ) ) ) <EOL> self . response . set_status ( <NUM_LIT> ) <EOL> else : <EOL> try : <EOL> data = { '<STR_LIT>' : fp , <EOL> '<STR_LIT:filename>' : filename , <EOL> '<STR_LIT>' : orig_filename , <EOL> '<STR_LIT>' : lineno , <EOL> } <EOL> self . response . out . write ( render ( '<STR_LIT>' , data ) ) <EOL> finally : <EOL> fp . close ( ) <EOL> class StaticHandler ( webapp . RequestHandler ) : <EOL> """<STR_LIT>""" <EOL> def get ( self ) : <EOL> recording . dont_record ( ) <EOL> here = os . path . dirname ( __file__ ) <EOL> fn = self . request . path <EOL> i = fn . rfind ( '<STR_LIT:/>' ) <EOL> fn = fn [ i + <NUM_LIT:1> : ] <EOL> fn = os . path . join ( here , '<STR_LIT>' , fn ) <EOL> ctype , encoding = mimetypes . guess_type ( fn ) <EOL> assert ctype and '<STR_LIT:/>' in ctype , repr ( ctype ) <EOL> expiry = <NUM_LIT> <EOL> expiration = email . Utils . formatdate ( time . time ( ) + expiry , usegmt = True ) <EOL> fp = open ( fn , '<STR_LIT:rb>' ) <EOL> try : <EOL> self . response . out . write ( fp . read ( ) ) <EOL> finally : <EOL> fp . close ( ) <EOL> self . response . headers [ '<STR_LIT>' ] = ctype <EOL> self . response . headers [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> self . response . headers [ '<STR_LIT>' ] = expiration <EOL> URLMAP = [ <EOL> ( '<STR_LIT>' , DetailsHandler ) , <EOL> ( '<STR_LIT>' , ShellHandler ) , <EOL> ( '<STR_LIT>' , FileHandler ) , <EOL> ( '<STR_LIT>' , StaticHandler ) , <EOL> ( '<STR_LIT>' , SummaryHandler ) , <EOL> ] <EOL> class AuthCheckMiddleware ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , application ) : <EOL> self . _application = application <EOL> def __call__ ( self , environ , start_response ) : <EOL> if not environ . get ( '<STR_LIT>' , '<STR_LIT>' ) . startswith ( '<STR_LIT>' ) : <EOL> if not users . is_current_user_admin ( ) : <EOL> if users . get_current_user ( ) is None : <EOL> start_response ( '<STR_LIT>' , <EOL> [ ( '<STR_LIT>' , <EOL> users . create_login_url ( os . getenv ( '<STR_LIT>' , '<STR_LIT>' ) ) ) ] ) <EOL> return [ ] <EOL> else : <EOL> start_response ( '<STR_LIT>' , [ ] ) <EOL> return [ '<STR_LIT>' ] <EOL> return self . _application ( environ , start_response ) <EOL> app = AuthCheckMiddleware ( webapp . WSGIApplication ( URLMAP , debug = DEBUG ) ) <EOL> def main ( ) : <EOL> """<STR_LIT>""" <EOL> util . run_bare_wsgi_app ( app ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> main ( ) </s>
