<s> if __name__ == '<STR_LIT:__main__>' : <EOL> import adddeps <EOL> import argparse <EOL> import csv <EOL> import hashlib <EOL> import itertools <EOL> import logging <EOL> import math <EOL> import os <EOL> import sqlalchemy . orm . exc <EOL> import subprocess <EOL> import sys <EOL> from collections import defaultdict <EOL> from fn import _ <EOL> from fn import Stream <EOL> from fn . iters import repeat <EOL> from pprint import pprint <EOL> import opentuner <EOL> from opentuner import resultsdb <EOL> from opentuner . resultsdb . models import * <EOL> log = logging . getLogger ( '<STR_LIT>' ) <EOL> argparser = argparse . ArgumentParser ( ) <EOL> argparser . add_argument ( '<STR_LIT>' ) <EOL> argparser . add_argument ( '<STR_LIT>' , action = '<STR_LIT:store_true>' , <EOL> help = "<STR_LIT>" ) <EOL> argparser . add_argument ( '<STR_LIT>' , action = '<STR_LIT:store_true>' , <EOL> help = '<STR_LIT>' ) <EOL> argparser . add_argument ( '<STR_LIT>' , type = float , default = <NUM_LIT:10> , <EOL> help = "<STR_LIT>" ) <EOL> argparser . add_argument ( '<STR_LIT>' , default = '<STR_LIT>' , <EOL> help = "<STR_LIT>" ) <EOL> argparser . add_argument ( '<STR_LIT>' , default = "<STR_LIT>" ) <EOL> argparser . add_argument ( '<STR_LIT>' , type = int , default = <NUM_LIT:1> , <EOL> help = "<STR_LIT>" ) <EOL> PCTSTEPS = map ( _ / <NUM_LIT> , xrange ( <NUM_LIT> ) ) <EOL> def mean ( vals ) : <EOL> n = <NUM_LIT:0.0> <EOL> d = <NUM_LIT:0.0> <EOL> for v in vals : <EOL> if v is not None : <EOL> n += v <EOL> d += <NUM_LIT:1.0> <EOL> if d == <NUM_LIT:0.0> : <EOL> return None <EOL> return n / d <EOL> def median ( vals ) : <EOL> vals = sorted ( vals ) <EOL> a = ( len ( vals ) - <NUM_LIT:1> ) / <NUM_LIT:2> <EOL> b = ( len ( vals ) ) / <NUM_LIT:2> <EOL> return ( vals [ a ] + vals [ b ] ) / <NUM_LIT> <EOL> def percentile ( vals , pct ) : <EOL> vals = sorted ( vals ) <EOL> pos = ( len ( vals ) - <NUM_LIT:1> ) * pct <EOL> a = int ( math . floor ( pos ) ) <EOL> b = min ( len ( vals ) - <NUM_LIT:1> , a + <NUM_LIT:1> ) <EOL> return ( <NUM_LIT:1.0> - ( pos - a ) ) * vals [ a ] + ( pos - a ) * vals [ b ] <EOL> def variance ( vals ) : <EOL> vals = filter ( lambda x : x is not None , vals ) <EOL> avg = mean ( vals ) <EOL> if avg is None : <EOL> return None <EOL> if avg in ( float ( '<STR_LIT>' ) , float ( '<STR_LIT>' ) ) : <EOL> return avg <EOL> return mean ( map ( ( _ - avg ) ** <NUM_LIT:2> , vals ) ) <EOL> def stddev ( vals ) : <EOL> var = variance ( vals ) <EOL> if var is None : <EOL> return None <EOL> return math . sqrt ( var ) <EOL> def hash_args ( x ) : <EOL> d = dict ( vars ( x ) ) <EOL> for k in ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) : <EOL> d [ k ] = None <EOL> return hashlib . sha256 ( str ( sorted ( d . items ( ) ) ) ) . hexdigest ( ) [ : <NUM_LIT:20> ] <EOL> def run_label ( tr , short = False ) : <EOL> techniques = '<STR_LIT:U+002C>' . join ( tr . args . technique ) <EOL> if not tr . name or tr . name == '<STR_LIT>' : <EOL> if short : <EOL> return techniques <EOL> else : <EOL> return "<STR_LIT>" % ( techniques , hash_args ( tr . args ) [ : <NUM_LIT:6> ] ) <EOL> else : <EOL> return tr . name <EOL> def run_dir ( base , tr ) : <EOL> return os . path . join ( base , <EOL> tr . program . project , <EOL> tr . program . name . split ( '<STR_LIT:/>' ) [ - <NUM_LIT:1> ] , <EOL> tr . program_version . version [ : <NUM_LIT:16> ] ) <EOL> class StatsMain ( object ) : <EOL> def __init__ ( self , args ) : <EOL> self . args = args <EOL> path = args . stats_input <EOL> self . dbs = list ( ) <EOL> for f in os . listdir ( path ) : <EOL> if '<STR_LIT>' in f : <EOL> continue <EOL> try : <EOL> e , sm = resultsdb . connect ( '<STR_LIT>' + os . path . join ( path , f ) ) <EOL> self . dbs . append ( sm ( ) ) <EOL> except : <EOL> log . error ( '<STR_LIT>' , <EOL> os . path . join ( path , f ) , <EOL> exc_info = True ) <EOL> def main ( self ) : <EOL> dir_label_runs = defaultdict ( lambda : defaultdict ( list ) ) <EOL> for session in self . dbs : <EOL> q = ( session . query ( resultsdb . models . TuningRun ) <EOL> . filter_by ( state = '<STR_LIT>' ) <EOL> . order_by ( '<STR_LIT:name>' ) ) <EOL> if self . args . label : <EOL> q = q . filter ( TuningRun . name . in_ ( <EOL> map ( str . strip , self . args . label . split ( '<STR_LIT:U+002C>' ) ) ) ) <EOL> for tr in q : <EOL> d = run_dir ( self . args . stats_dir , tr ) <EOL> d = os . path . normpath ( d ) <EOL> dir_label_runs [ d ] [ run_label ( tr ) ] . append ( ( tr , session ) ) <EOL> summary_report = defaultdict ( lambda : defaultdict ( list ) ) <EOL> for d , label_runs in dir_label_runs . iteritems ( ) : <EOL> if not os . path . isdir ( d ) : <EOL> os . makedirs ( d ) <EOL> session = label_runs . values ( ) [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] <EOL> objective = label_runs . values ( ) [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] . objective <EOL> all_run_ids = map ( _ [ <NUM_LIT:0> ] . id , itertools . chain ( * label_runs . values ( ) ) ) <EOL> q = ( session . query ( Result ) <EOL> . filter ( Result . tuning_run_id . in_ ( all_run_ids ) ) <EOL> . filter ( Result . time < float ( '<STR_LIT>' ) ) <EOL> . filter_by ( was_new_best = True , state = '<STR_LIT:OK>' ) ) <EOL> total = q . count ( ) <EOL> if total == <NUM_LIT:0> : <EOL> continue <EOL> q = objective . filter_acceptable ( q ) <EOL> acceptable = q . count ( ) <EOL> q = q . order_by ( * objective . result_order_by_terms ( ) ) <EOL> best = q . limit ( <NUM_LIT:1> ) . one ( ) <EOL> worst = q . offset ( acceptable - <NUM_LIT:1> ) . limit ( <NUM_LIT:1> ) . one ( ) <EOL> map ( len , label_runs . values ( ) ) <EOL> log . info ( "<STR_LIT>" <EOL> "<STR_LIT>" , <EOL> d , <EOL> best . time , <EOL> worst . time , <EOL> acceptable , <EOL> total , <EOL> len ( label_runs . values ( ) ) , <EOL> min ( map ( len , label_runs . values ( ) ) ) , <EOL> max ( map ( len , label_runs . values ( ) ) ) ) <EOL> for label , runs in sorted ( label_runs . items ( ) ) : <EOL> if len ( runs ) < self . args . min_runs : <EOL> print len ( runs ) , self . args . min_runs <EOL> continue <EOL> log . debug ( '<STR_LIT>' , d , label , len ( runs ) , runs [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] . args . technique ) <EOL> self . combined_stats_over_time ( d , label , runs , objective , worst , best ) <EOL> final_scores = list ( ) <EOL> for run , session in runs : <EOL> try : <EOL> final = ( session . query ( Result ) <EOL> . filter_by ( tuning_run = run , <EOL> configuration = run . final_config ) <EOL> . limit ( <NUM_LIT:1> ) <EOL> . one ( ) ) <EOL> except sqlalchemy . orm . exc . NoResultFound : <EOL> continue <EOL> final_scores . append ( objective . stats_quality_score ( final , worst , best ) ) <EOL> final_scores . sort ( ) <EOL> if final_scores : <EOL> norm = objective . stats_quality_score ( best , worst , best ) <EOL> if norm > <NUM_LIT> : <EOL> summary_report [ d ] [ run_label ( run , short = True ) ] = ( <EOL> percentile ( final_scores , <NUM_LIT:0.5> ) / norm , <EOL> percentile ( final_scores , <NUM_LIT:0.1> ) / norm , <EOL> percentile ( final_scores , <NUM_LIT> ) / norm , <EOL> ) <EOL> else : <EOL> summary_report [ d ] [ run_label ( run , short = True ) ] = ( <EOL> percentile ( final_scores , <NUM_LIT:0.5> ) + norm + <NUM_LIT:1.0> , <EOL> percentile ( final_scores , <NUM_LIT:0.1> ) + norm + <NUM_LIT:1.0> , <EOL> percentile ( final_scores , <NUM_LIT> ) + norm + <NUM_LIT:1.0> , <EOL> ) <EOL> with open ( self . args . stats_dir + "<STR_LIT>" , '<STR_LIT:w>' ) as o : <EOL> keys = sorted ( reduce ( set . union , <EOL> [ set ( x . keys ( ) ) for x in summary_report . values ( ) ] , <EOL> set ( ) ) ) <EOL> print >> o , '<STR_LIT>' , <EOL> for k in keys : <EOL> print >> o , k , <EOL> print >> o <EOL> for d , label_vals in sorted ( summary_report . items ( ) ) : <EOL> print >> o , d . split ( '<STR_LIT:/>' ) [ - <NUM_LIT:2> ] , <EOL> for k in keys : <EOL> if k in label_vals : <EOL> print >> o , '<STR_LIT:->' , label_vals [ k ] [ <NUM_LIT:0> ] , label_vals [ k ] [ <NUM_LIT:1> ] , label_vals [ k ] [ <NUM_LIT:2> ] , <EOL> else : <EOL> print >> o , '<STR_LIT:->' , '<STR_LIT:->' , '<STR_LIT:->' , '<STR_LIT:->' , <EOL> print >> o <EOL> if keys : <EOL> plotcmd = [ """<STR_LIT>""" , <EOL> """<STR_LIT>""" % keys [ <NUM_LIT:0> ] ] <EOL> for n , k in enumerate ( keys [ <NUM_LIT:1> : ] ) : <EOL> plotcmd . append ( """<STR_LIT>""" % ( <EOL> <NUM_LIT:4> * n + <NUM_LIT:7> , <EOL> <NUM_LIT:4> * n + <NUM_LIT:8> , <EOL> <NUM_LIT:4> * n + <NUM_LIT:9> , <EOL> k ) ) <EOL> self . gnuplot_summary_file ( self . args . stats_dir , '<STR_LIT>' , plotcmd ) <EOL> for d , label_runs in dir_label_runs . iteritems ( ) : <EOL> labels = [ k for k , v in label_runs . iteritems ( ) <EOL> if len ( v ) >= self . args . min_runs ] <EOL> self . gnuplot_file ( d , <EOL> "<STR_LIT>" , <EOL> [ '<STR_LIT>' % ( l , l ) for l in labels ] ) <EOL> self . gnuplot_file ( d , <EOL> "<STR_LIT>" , <EOL> [ '<STR_LIT>' % ( l , l ) for l in labels ] ) <EOL> self . gnuplot_file ( d , <EOL> "<STR_LIT>" , <EOL> [ '<STR_LIT>' % ( l , l ) for l in labels ] ) <EOL> self . gnuplot_file ( d , <EOL> "<STR_LIT>" , <EOL> [ '<STR_LIT>' % ( l , l ) for l in labels ] ) <EOL> print <EOL> print "<STR_LIT>" , d <EOL> pprint ( self . technique_scores ( d , labels , '<STR_LIT>' ) ) <EOL> def technique_scores ( self , directory , labels , ykey , xkey = '<STR_LIT>' , factor = <NUM_LIT> ) : <EOL> max_duration = None <EOL> min_value = float ( '<STR_LIT>' ) <EOL> for label in labels : <EOL> try : <EOL> dr = csv . DictReader ( open ( os . path . join ( directory , label + "<STR_LIT>" ) ) , delimiter = '<STR_LIT:U+0020>' , lineterminator = '<STR_LIT:\n>' ) <EOL> lastrow = list ( dr ) [ - <NUM_LIT:1> ] <EOL> max_duration = max ( max_duration , float ( lastrow [ xkey ] ) ) <EOL> min_value = min ( min_value , float ( lastrow [ ykey ] ) ) <EOL> except : <EOL> log . exception ( "<STR_LIT>" ) <EOL> scores = list ( ) <EOL> for label in labels : <EOL> try : <EOL> dr = csv . DictReader ( open ( os . path . join ( directory , label + "<STR_LIT>" ) ) , delimiter = '<STR_LIT:U+0020>' , lineterminator = '<STR_LIT:\n>' ) <EOL> score = <NUM_LIT:0.0> <EOL> lastsec = <NUM_LIT:0.0> <EOL> value = float ( '<STR_LIT>' ) <EOL> for row in dr : <EOL> duration = float ( row [ xkey ] ) - lastsec <EOL> lastsec = float ( row [ xkey ] ) <EOL> value = float ( row [ ykey ] ) <EOL> score += duration * ( value - min_value ) <EOL> score += ( factor * max_duration - lastsec ) * ( value - min_value ) <EOL> scores . append ( ( score , label ) ) <EOL> except : <EOL> log . exception ( "<STR_LIT>" ) <EOL> return sorted ( scores ) <EOL> def combined_stats_over_time ( self , <EOL> output_dir , <EOL> label , <EOL> runs , <EOL> objective , <EOL> worst , <EOL> best , <EOL> ) : <EOL> """<STR_LIT>""" <EOL> extract_fn = _ . result . time <EOL> combine_fn = min <EOL> no_data = <NUM_LIT> <EOL> log . debug ( "<STR_LIT>" , label , output_dir ) <EOL> by_run = [ self . stats_over_time ( session , run , extract_fn , combine_fn , no_data ) <EOL> for run , session in runs ] <EOL> max_len = max ( map ( len , by_run ) ) <EOL> by_run_streams = [ Stream ( ) << x << repeat ( x [ - <NUM_LIT:1> ] , max_len - len ( x ) ) <EOL> for x in by_run ] <EOL> by_quanta = zip ( * by_run_streams [ : ] ) <EOL> def data_file ( suffix , headers , value_function ) : <EOL> with open ( os . path . join ( output_dir , label + suffix ) , '<STR_LIT:w>' ) as fd : <EOL> out = csv . writer ( fd , delimiter = '<STR_LIT:U+0020>' , lineterminator = '<STR_LIT:\n>' ) <EOL> out . writerow ( [ '<STR_LIT>' ] + headers ) <EOL> for quanta , values in enumerate ( by_quanta ) : <EOL> sec = quanta * self . args . stats_quanta <EOL> out . writerow ( [ sec ] + value_function ( values ) ) <EOL> data_file ( '<STR_LIT>' , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> lambda values : [ mean ( values ) , stddev ( values ) ] ) <EOL> self . gnuplot_file ( output_dir , <EOL> label + '<STR_LIT>' , <EOL> [ '<STR_LIT:">' + label + '<STR_LIT>' ] ) <EOL> def extract_percentiles ( values ) : <EOL> values = sorted ( values ) <EOL> return ( [ values [ int ( round ( p * ( len ( values ) - <NUM_LIT:1> ) ) ) ] for p in PCTSTEPS ] <EOL> + [ mean ( values ) ] ) <EOL> data_file ( "<STR_LIT>" , PCTSTEPS + [ '<STR_LIT>' ] , extract_percentiles ) <EOL> self . gnuplot_file ( output_dir , <EOL> label + '<STR_LIT>' , <EOL> reversed ( [ <EOL> '<STR_LIT:">' + label + '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT:">' + label + '<STR_LIT>' , <EOL> ] ) ) <EOL> def gnuplot_file ( self , output_dir , prefix , plotcmd ) : <EOL> with open ( os . path . join ( output_dir , prefix + '<STR_LIT>' ) , '<STR_LIT:w>' ) as fd : <EOL> print >> fd , '<STR_LIT>' <EOL> print >> fd , '<STR_LIT>' % ( prefix + '<STR_LIT>' ) <EOL> print >> fd , '<STR_LIT>' <EOL> print >> fd , '<STR_LIT>' <EOL> print >> fd , '<STR_LIT>' , '<STR_LIT>' . join ( plotcmd ) <EOL> try : <EOL> subprocess . call ( [ '<STR_LIT>' , prefix + '<STR_LIT>' ] , cwd = output_dir , stdin = None ) <EOL> except OSError : <EOL> log . error ( "<STR_LIT>" ) <EOL> def gnuplot_summary_file ( self , output_dir , prefix , plotcmd ) : <EOL> with open ( os . path . join ( output_dir , prefix + '<STR_LIT>' ) , '<STR_LIT:w>' ) as fd : <EOL> print >> fd , '<STR_LIT>' <EOL> print >> fd , '<STR_LIT>' % ( prefix + '<STR_LIT>' ) <EOL> print >> fd , '''<STR_LIT>''' <EOL> print >> fd , '<STR_LIT>' , '<STR_LIT>' . join ( plotcmd ) <EOL> subprocess . call ( [ '<STR_LIT>' , prefix + '<STR_LIT>' ] , cwd = output_dir , stdin = None ) <EOL> def stats_over_time ( self , <EOL> session , <EOL> run , <EOL> extract_fn , <EOL> combine_fn , <EOL> no_data = None ) : <EOL> """<STR_LIT>""" <EOL> value_by_quanta = [ no_data ] <EOL> start_date = run . start_date <EOL> subq = ( session . query ( Result . id ) <EOL> . filter_by ( tuning_run = run , was_new_best = True , state = '<STR_LIT:OK>' ) ) <EOL> q = ( session . query ( DesiredResult ) <EOL> . join ( Result ) <EOL> . filter ( DesiredResult . state == '<STR_LIT>' , <EOL> DesiredResult . tuning_run == run , <EOL> DesiredResult . result_id . in_ ( subq . subquery ( ) ) ) <EOL> . order_by ( DesiredResult . request_date ) ) <EOL> first_id = None <EOL> for dr in q : <EOL> if first_id is None : <EOL> first_id = dr . id <EOL> td = ( dr . request_date - start_date ) <EOL> duration = td . seconds + ( td . days * <NUM_LIT> * <NUM_LIT> ) <EOL> if self . args . by_request_count : <EOL> quanta = dr . id - first_id <EOL> else : <EOL> quanta = int ( duration / self . args . stats_quanta ) <EOL> while len ( value_by_quanta ) <= quanta : <EOL> value_by_quanta . append ( value_by_quanta [ - <NUM_LIT:1> ] ) <EOL> if value_by_quanta [ - <NUM_LIT:1> ] is no_data : <EOL> value_by_quanta [ - <NUM_LIT:1> ] = extract_fn ( dr ) <EOL> else : <EOL> value_by_quanta [ - <NUM_LIT:1> ] = combine_fn ( value_by_quanta [ - <NUM_LIT:1> ] , extract_fn ( dr ) ) <EOL> return value_by_quanta <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> opentuner . tuningrunmain . init_logging ( ) <EOL> sys . exit ( StatsMain ( argparser . parse_args ( ) ) . main ( ) ) </s>
