<s> from __future__ import division <EOL> import os <EOL> import math <EOL> import time <EOL> from tempfile import mktemp <EOL> from base64 import b64encode <EOL> from collections import Counter , namedtuple <EOL> try : <EOL> import jinja2 <EOL> import pandas as pd <EOL> import matplotlib <EOL> matplotlib . use ( '<STR_LIT>' ) <EOL> import matplotlib . pyplot as plt <EOL> import numpy as np <EOL> low_filter = np . vectorize ( lambda x : x > <NUM_LIT:0> and x or <NUM_LIT:0> ) <EOL> import_error = None <EOL> except ImportError as e : <EOL> import_error = e <EOL> jinja2 = None <EOL> pd = None <EOL> plt = None <EOL> np = None <EOL> low_filter = None <EOL> from wlauto import Instrument , Parameter , File <EOL> from wlauto . exceptions import ConfigError , InstrumentError , DeviceError <EOL> from wlauto . instrumentation import instrument_is_installed <EOL> from wlauto . utils . types import caseless_string , list_or_caseless_string , list_of_ints <EOL> from wlauto . utils . misc import list_to_mask <EOL> FREQ_TABLE_FILE = '<STR_LIT>' <EOL> CPUS_TABLE_FILE = '<STR_LIT>' <EOL> MEASURED_CPUS_TABLE_FILE = '<STR_LIT>' <EOL> IDLE_TABLE_FILE = '<STR_LIT>' <EOL> REPORT_TEMPLATE_FILE = '<STR_LIT>' <EOL> EM_TEMPLATE_FILE = '<STR_LIT>' <EOL> IdlePowerState = namedtuple ( '<STR_LIT>' , [ '<STR_LIT>' ] ) <EOL> CapPowerState = namedtuple ( '<STR_LIT>' , [ '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> class EnergyModel ( object ) : <EOL> def __init__ ( self ) : <EOL> self . big_cluster_idle_states = [ ] <EOL> self . little_cluster_idle_states = [ ] <EOL> self . big_cluster_cap_states = [ ] <EOL> self . little_cluster_cap_states = [ ] <EOL> self . big_core_idle_states = [ ] <EOL> self . little_core_idle_states = [ ] <EOL> self . big_core_cap_states = [ ] <EOL> self . little_core_cap_states = [ ] <EOL> def add_cap_entry ( self , cluster , perf , clust_pow , core_pow ) : <EOL> if cluster == '<STR_LIT>' : <EOL> self . big_cluster_cap_states . append ( CapPowerState ( perf , clust_pow ) ) <EOL> self . big_core_cap_states . append ( CapPowerState ( perf , core_pow ) ) <EOL> elif cluster == '<STR_LIT>' : <EOL> self . little_cluster_cap_states . append ( CapPowerState ( perf , clust_pow ) ) <EOL> self . little_core_cap_states . append ( CapPowerState ( perf , core_pow ) ) <EOL> else : <EOL> raise ValueError ( '<STR_LIT>' . format ( cluster ) ) <EOL> def add_cluster_idle ( self , cluster , values ) : <EOL> for value in values : <EOL> if cluster == '<STR_LIT>' : <EOL> self . big_cluster_idle_states . append ( IdlePowerState ( value ) ) <EOL> elif cluster == '<STR_LIT>' : <EOL> self . little_cluster_idle_states . append ( IdlePowerState ( value ) ) <EOL> else : <EOL> raise ValueError ( '<STR_LIT>' . format ( cluster ) ) <EOL> def add_core_idle ( self , cluster , values ) : <EOL> for value in values : <EOL> if cluster == '<STR_LIT>' : <EOL> self . big_core_idle_states . append ( IdlePowerState ( value ) ) <EOL> elif cluster == '<STR_LIT>' : <EOL> self . little_core_idle_states . append ( IdlePowerState ( value ) ) <EOL> else : <EOL> raise ValueError ( '<STR_LIT>' . format ( cluster ) ) <EOL> class PowerPerformanceAnalysis ( object ) : <EOL> def __init__ ( self , data ) : <EOL> self . summary = { } <EOL> big_freqs = data [ data . cluster == '<STR_LIT>' ] . frequency . unique ( ) <EOL> little_freqs = data [ data . cluster == '<STR_LIT>' ] . frequency . unique ( ) <EOL> self . summary [ '<STR_LIT>' ] = max ( set ( big_freqs ) . intersection ( set ( little_freqs ) ) ) <EOL> big_sc = data [ ( data . cluster == '<STR_LIT>' ) & <EOL> ( data . frequency == self . summary [ '<STR_LIT>' ] ) & <EOL> ( data . cpus == <NUM_LIT:1> ) ] <EOL> little_sc = data [ ( data . cluster == '<STR_LIT>' ) & <EOL> ( data . frequency == self . summary [ '<STR_LIT>' ] ) & <EOL> ( data . cpus == <NUM_LIT:1> ) ] <EOL> self . summary [ '<STR_LIT>' ] = big_sc . performance . item ( ) / little_sc . performance . item ( ) <EOL> self . summary [ '<STR_LIT>' ] = big_sc . power . item ( ) / little_sc . power . item ( ) <EOL> self . summary [ '<STR_LIT>' ] = data [ data . cpus == <NUM_LIT:1> ] . performance . max ( ) <EOL> self . summary [ '<STR_LIT>' ] = data [ data . cpus == <NUM_LIT:1> ] . power . max ( ) <EOL> def build_energy_model ( freq_power_table , cpus_power , idle_power , first_cluster_idle_state ) : <EOL> em = EnergyModel ( ) <EOL> idle_power_sc = idle_power [ idle_power . cpus == <NUM_LIT:1> ] <EOL> perf_data = get_normalized_single_core_data ( freq_power_table ) <EOL> for cluster in [ '<STR_LIT>' , '<STR_LIT>' ] : <EOL> cluster_cpus_power = cpus_power [ cluster ] . dropna ( ) <EOL> cluster_power = cluster_cpus_power [ '<STR_LIT>' ] . apply ( int ) <EOL> core_power = ( cluster_cpus_power [ '<STR_LIT:1>' ] - cluster_power ) . apply ( int ) <EOL> performance = ( perf_data [ perf_data . cluster == cluster ] . performance_norm * <NUM_LIT> / <NUM_LIT:100> ) . apply ( int ) <EOL> for perf , clust_pow , core_pow in zip ( performance , cluster_power , core_power ) : <EOL> em . add_cap_entry ( cluster , perf , clust_pow , core_pow ) <EOL> all_idle_power = idle_power_sc [ idle_power_sc . cluster == cluster ] . power . values <EOL> idle_core_power = low_filter ( all_idle_power [ : first_cluster_idle_state ] - <EOL> all_idle_power [ first_cluster_idle_state - <NUM_LIT:1> ] ) <EOL> idle_cluster_power = low_filter ( all_idle_power [ first_cluster_idle_state - <NUM_LIT:1> : ] ) <EOL> em . add_cluster_idle ( cluster , idle_cluster_power ) <EOL> em . add_core_idle ( cluster , idle_core_power ) <EOL> return em <EOL> def generate_em_c_file ( em , big_core , little_core , em_template_file , outfile ) : <EOL> with open ( em_template_file ) as fh : <EOL> em_template = jinja2 . Template ( fh . read ( ) ) <EOL> em_text = em_template . render ( <EOL> big_core = big_core , <EOL> little_core = little_core , <EOL> em = em , <EOL> ) <EOL> with open ( outfile , '<STR_LIT:w>' ) as wfh : <EOL> wfh . write ( em_text ) <EOL> return em_text <EOL> def generate_report ( freq_power_table , measured_cpus_table , cpus_table , idle_power_table , <EOL> report_template_file , device_name , em_text , outfile ) : <EOL> cap_power_analysis = PowerPerformanceAnalysis ( freq_power_table ) <EOL> single_core_norm = get_normalized_single_core_data ( freq_power_table ) <EOL> cap_power_plot = get_cap_power_plot ( single_core_norm ) <EOL> idle_power_plot = get_idle_power_plot ( idle_power_table ) <EOL> fig , axes = plt . subplots ( <NUM_LIT:1> , <NUM_LIT:2> ) <EOL> fig . set_size_inches ( <NUM_LIT:16> , <NUM_LIT:8> ) <EOL> for i , cluster in enumerate ( reversed ( cpus_table . columns . levels [ <NUM_LIT:0> ] ) ) : <EOL> projected = cpus_table [ cluster ] . dropna ( subset = [ '<STR_LIT:1>' ] ) <EOL> plot_cpus_table ( projected , axes [ i ] , cluster ) <EOL> cpus_plot_data = get_figure_data ( fig ) <EOL> with open ( report_template_file ) as fh : <EOL> report_template = jinja2 . Template ( fh . read ( ) ) <EOL> html = report_template . render ( <EOL> device_name = device_name , <EOL> freq_power_table = freq_power_table . set_index ( [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) . to_html ( ) , <EOL> cap_power_analysis = cap_power_analysis , <EOL> cap_power_plot = get_figure_data ( cap_power_plot ) , <EOL> idle_power_table = idle_power_table . set_index ( [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:state>' ] ) . to_html ( ) , <EOL> idle_power_plot = get_figure_data ( idle_power_plot ) , <EOL> cpus_table = cpus_table . to_html ( ) , <EOL> cpus_plot = cpus_plot_data , <EOL> em_text = em_text , <EOL> ) <EOL> with open ( outfile , '<STR_LIT:w>' ) as wfh : <EOL> wfh . write ( html ) <EOL> return html <EOL> def wa_result_to_power_perf_table ( df , performance_metric , index ) : <EOL> table = df . pivot_table ( index = index + [ '<STR_LIT>' ] , <EOL> columns = '<STR_LIT>' , values = '<STR_LIT:value>' ) . reset_index ( ) <EOL> result_mean = table . groupby ( index ) . mean ( ) <EOL> result_std = table . groupby ( index ) . std ( ) <EOL> result_std . columns = [ c + '<STR_LIT>' for c in result_std . columns ] <EOL> result_count = table . groupby ( index ) . count ( ) <EOL> result_count . columns = [ c + '<STR_LIT>' for c in result_count . columns ] <EOL> count_sqrt = result_count . apply ( lambda x : x . apply ( math . sqrt ) ) <EOL> count_sqrt . columns = result_std . columns <EOL> result_error = <NUM_LIT> * result_std / count_sqrt <EOL> result_error . columns = [ c + '<STR_LIT>' for c in result_mean . columns ] <EOL> result = pd . concat ( [ result_mean , result_std , result_count , result_error ] , axis = <NUM_LIT:1> ) <EOL> del result [ '<STR_LIT>' ] <EOL> del result [ '<STR_LIT>' ] <EOL> del result [ '<STR_LIT>' ] <EOL> del result [ '<STR_LIT>' ] <EOL> updated_columns = [ ] <EOL> for column in result . columns : <EOL> if column == performance_metric : <EOL> updated_columns . append ( '<STR_LIT>' ) <EOL> elif column == performance_metric + '<STR_LIT>' : <EOL> updated_columns . append ( '<STR_LIT>' ) <EOL> elif column == performance_metric + '<STR_LIT>' : <EOL> updated_columns . append ( '<STR_LIT>' ) <EOL> else : <EOL> updated_columns . append ( column . replace ( '<STR_LIT:U+0020>' , '<STR_LIT:_>' ) ) <EOL> result . columns = updated_columns <EOL> result = result [ sorted ( result . columns ) ] <EOL> result . reset_index ( inplace = True ) <EOL> return result <EOL> def get_figure_data ( fig , fmt = '<STR_LIT>' ) : <EOL> tmp = mktemp ( ) <EOL> fig . savefig ( tmp , format = fmt , bbox_inches = '<STR_LIT>' ) <EOL> with open ( tmp , '<STR_LIT:rb>' ) as fh : <EOL> image_data = b64encode ( fh . read ( ) ) <EOL> os . remove ( tmp ) <EOL> return image_data <EOL> def get_normalized_single_core_data ( data ) : <EOL> finite_power = np . isfinite ( data . power ) <EOL> finite_perf = np . isfinite ( data . performance ) <EOL> data_single_core = data [ ( data . cpus == <NUM_LIT:1> ) & finite_perf & finite_power ] . copy ( ) <EOL> data_single_core [ '<STR_LIT>' ] = ( data_single_core . performance / <EOL> data_single_core . performance . max ( ) * <NUM_LIT:100> ) . apply ( int ) <EOL> data_single_core [ '<STR_LIT>' ] = ( data_single_core . power / <EOL> data_single_core . power . max ( ) * <NUM_LIT:100> ) . apply ( int ) <EOL> return data_single_core <EOL> def get_cap_power_plot ( data_single_core ) : <EOL> big_single_core = data_single_core [ ( data_single_core . cluster == '<STR_LIT>' ) & <EOL> ( data_single_core . cpus == <NUM_LIT:1> ) ] <EOL> little_single_core = data_single_core [ ( data_single_core . cluster == '<STR_LIT>' ) & <EOL> ( data_single_core . cpus == <NUM_LIT:1> ) ] <EOL> fig , axes = plt . subplots ( <NUM_LIT:1> , <NUM_LIT:1> , figsize = ( <NUM_LIT:12> , <NUM_LIT:8> ) ) <EOL> axes . plot ( big_single_core . performance_norm , <EOL> big_single_core . power_norm , <EOL> marker = '<STR_LIT:o>' ) <EOL> axes . plot ( little_single_core . performance_norm , <EOL> little_single_core . power_norm , <EOL> marker = '<STR_LIT:o>' ) <EOL> axes . set_xlim ( <NUM_LIT:0> , <NUM_LIT> ) <EOL> axes . set_ylim ( <NUM_LIT:0> , <NUM_LIT> ) <EOL> axes . set_xlabel ( '<STR_LIT>' ) <EOL> axes . set_ylabel ( '<STR_LIT>' ) <EOL> axes . grid ( ) <EOL> axes . legend ( [ '<STR_LIT>' , '<STR_LIT>' ] , loc = <NUM_LIT:0> ) <EOL> return fig <EOL> def get_idle_power_plot ( df ) : <EOL> fig , axes = plt . subplots ( <NUM_LIT:1> , <NUM_LIT:2> , figsize = ( <NUM_LIT:15> , <NUM_LIT:7> ) ) <EOL> for cluster , ax in zip ( [ '<STR_LIT>' , '<STR_LIT>' ] , axes ) : <EOL> data = df [ df . cluster == cluster ] . pivot_table ( index = [ '<STR_LIT:state>' ] , columns = '<STR_LIT>' , values = '<STR_LIT>' ) <EOL> err = df [ df . cluster == cluster ] . pivot_table ( index = [ '<STR_LIT:state>' ] , columns = '<STR_LIT>' , values = '<STR_LIT>' ) <EOL> data . plot ( kind = '<STR_LIT:bar>' , ax = ax , rot = <NUM_LIT:30> , yerr = err ) <EOL> ax . set_title ( '<STR_LIT>' . format ( cluster ) ) <EOL> ax . set_xlim ( - <NUM_LIT:1> , len ( data . columns ) - <NUM_LIT:0.5> ) <EOL> ax . set_ylabel ( '<STR_LIT>' ) <EOL> return fig <EOL> def fit_polynomial ( s , n ) : <EOL> coeffs = np . polyfit ( s . index , s . values , n ) <EOL> poly = np . poly1d ( coeffs ) <EOL> return poly ( s . index ) <EOL> def get_cpus_power_table ( data , index , opps , leak_factors ) : <EOL> power_table = data [ [ index , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ] . pivot_table ( index = index , <EOL> columns = [ '<STR_LIT>' , '<STR_LIT>' ] , <EOL> values = '<STR_LIT>' ) <EOL> bs_power_table = pd . DataFrame ( index = power_table . index , columns = power_table . columns ) <EOL> for cluster in power_table . columns . levels [ <NUM_LIT:0> ] : <EOL> power_table [ cluster , <NUM_LIT:0> ] = ( power_table [ cluster , <NUM_LIT:1> ] - <EOL> ( power_table [ cluster , <NUM_LIT:2> ] - <EOL> power_table [ cluster , <NUM_LIT:1> ] ) ) <EOL> bs_power_table . loc [ power_table [ cluster , <NUM_LIT:1> ] . notnull ( ) , ( cluster , <NUM_LIT:1> ) ] = fit_polynomial ( power_table [ cluster , <NUM_LIT:1> ] . dropna ( ) , <NUM_LIT:2> ) <EOL> bs_power_table . loc [ power_table [ cluster , <NUM_LIT:2> ] . notnull ( ) , ( cluster , <NUM_LIT:2> ) ] = fit_polynomial ( power_table [ cluster , <NUM_LIT:2> ] . dropna ( ) , <NUM_LIT:2> ) <EOL> if opps [ cluster ] is None : <EOL> bs_power_table . loc [ bs_power_table [ cluster , <NUM_LIT:1> ] . notnull ( ) , ( cluster , <NUM_LIT:0> ) ] = ( <NUM_LIT:2> * power_table [ cluster , <NUM_LIT:1> ] - power_table [ cluster , <NUM_LIT:2> ] ) . values <EOL> else : <EOL> voltages = opps [ cluster ] . set_index ( '<STR_LIT>' ) . sort_index ( ) <EOL> leakage = leak_factors [ cluster ] * <NUM_LIT:2> * voltages [ '<STR_LIT>' ] ** <NUM_LIT:3> / <NUM_LIT> ** <NUM_LIT:3> <EOL> leakage_delta = leakage - leakage [ leakage . index [ <NUM_LIT:0> ] ] <EOL> bs_power_table . loc [ : , ( cluster , <NUM_LIT:0> ) ] = ( <NUM_LIT:2> * bs_power_table [ cluster , <NUM_LIT:1> ] + leakage_delta - bs_power_table [ cluster , <NUM_LIT:2> ] ) <EOL> power_table = power_table [ sorted ( power_table . columns , <EOL> cmp = lambda x , y : cmp ( y [ <NUM_LIT:0> ] , x [ <NUM_LIT:0> ] ) or cmp ( x [ <NUM_LIT:1> ] , y [ <NUM_LIT:1> ] ) ) ] <EOL> bs_power_table = bs_power_table [ sorted ( bs_power_table . columns , <EOL> cmp = lambda x , y : cmp ( y [ <NUM_LIT:0> ] , x [ <NUM_LIT:0> ] ) or cmp ( x [ <NUM_LIT:1> ] , y [ <NUM_LIT:1> ] ) ) ] <EOL> old_levels = power_table . columns . levels <EOL> power_table . columns . set_levels ( [ old_levels [ <NUM_LIT:0> ] , list ( map ( str , old_levels [ <NUM_LIT:1> ] ) [ : - <NUM_LIT:1> ] ) + [ '<STR_LIT>' ] ] , <EOL> inplace = True ) <EOL> bs_power_table . columns . set_levels ( [ old_levels [ <NUM_LIT:0> ] , list ( map ( str , old_levels [ <NUM_LIT:1> ] ) [ : - <NUM_LIT:1> ] ) + [ '<STR_LIT>' ] ] , <EOL> inplace = True ) <EOL> return power_table , bs_power_table <EOL> def plot_cpus_table ( projected , ax , cluster ) : <EOL> projected . T . plot ( ax = ax , marker = '<STR_LIT:o>' ) <EOL> ax . set_title ( '<STR_LIT>' . format ( cluster ) ) <EOL> ax . set_xticklabels ( projected . columns ) <EOL> ax . set_xticks ( range ( <NUM_LIT:0> , <NUM_LIT:5> ) ) <EOL> ax . set_xlim ( - <NUM_LIT:0.5> , len ( projected . columns ) - <NUM_LIT:0.5> ) <EOL> ax . set_ylabel ( '<STR_LIT>' ) <EOL> ax . grid ( True ) <EOL> def opp_table ( d ) : <EOL> if d is None : <EOL> return None <EOL> return pd . DataFrame ( d . items ( ) , columns = [ '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> class EnergyModelInstrument ( Instrument ) : <EOL> name = '<STR_LIT>' <EOL> desicription = """<STR_LIT>""" <EOL> parameters = [ <EOL> Parameter ( '<STR_LIT>' , kind = caseless_string , <EOL> description = """<STR_LIT>""" ) , <EOL> Parameter ( '<STR_LIT>' , kind = caseless_string , <EOL> description = """<STR_LIT>""" ) , <EOL> Parameter ( '<STR_LIT>' , kind = caseless_string , mandatory = True , <EOL> description = """<STR_LIT>""" ) , <EOL> Parameter ( '<STR_LIT>' , kind = list_or_caseless_string , <EOL> description = """<STR_LIT>""" ) , <EOL> Parameter ( '<STR_LIT>' , kind = list_or_caseless_string , <EOL> description = """<STR_LIT>""" ) , <EOL> Parameter ( '<STR_LIT>' , kind = float , default = <NUM_LIT:1.0> , <EOL> description = """<STR_LIT>""" ) , <EOL> Parameter ( '<STR_LIT>' , kind = list_of_ints , <EOL> description = """<STR_LIT>""" ) , <EOL> Parameter ( '<STR_LIT>' , kind = list_of_ints , <EOL> description = """<STR_LIT>""" ) , <EOL> Parameter ( '<STR_LIT>' , kind = str , default = '<STR_LIT>' , <EOL> description = "<STR_LIT>" ) , <EOL> Parameter ( '<STR_LIT>' , kind = dict , default = { } , <EOL> description = "<STR_LIT>" ) , <EOL> Parameter ( '<STR_LIT>' , kind = int , default = - <NUM_LIT:1> , <EOL> description = '''<STR_LIT>''' ) , <EOL> Parameter ( '<STR_LIT>' , kind = bool , default = False , <EOL> description = '''<STR_LIT>''' ) , <EOL> Parameter ( '<STR_LIT>' , kind = int , default = <NUM_LIT:0> , <EOL> description = """<STR_LIT>""" ) , <EOL> Parameter ( '<STR_LIT>' , kind = opp_table , <EOL> description = """<STR_LIT>""" ) , <EOL> Parameter ( '<STR_LIT>' , kind = opp_table , <EOL> description = """<STR_LIT>""" ) , <EOL> Parameter ( '<STR_LIT>' , kind = int , default = <NUM_LIT> , <EOL> description = """<STR_LIT>""" ) , <EOL> Parameter ( '<STR_LIT>' , kind = int , default = <NUM_LIT> , <EOL> description = """<STR_LIT>""" ) , <EOL> ] <EOL> def validate ( self ) : <EOL> if import_error : <EOL> message = '<STR_LIT>' <EOL> raise InstrumentError ( message . format ( import_error . message ) ) <EOL> for capability in [ '<STR_LIT>' , '<STR_LIT>' ] : <EOL> if not self . device . has ( capability ) : <EOL> message = '<STR_LIT>' <EOL> raise ConfigError ( message . format ( capability ) ) <EOL> device_cores = set ( self . device . core_names ) <EOL> if ( self . power_metric and self . energy_metric ) or not ( self . power_metric or self . energy_metric ) : <EOL> raise ConfigError ( '<STR_LIT>' ) <EOL> if not device_cores : <EOL> raise ConfigError ( '<STR_LIT>' ) <EOL> elif len ( device_cores ) != <NUM_LIT:2> : <EOL> raise ConfigError ( '<STR_LIT>' ) <EOL> if self . big_core and self . big_core not in self . device . core_names : <EOL> raise ConfigError ( '<STR_LIT>' . format ( self . big_core , self . device . name ) ) <EOL> if not self . big_core : <EOL> self . big_core = self . device . core_names [ - <NUM_LIT:1> ] <EOL> if not self . device_name : <EOL> self . device_name = self . device . name <EOL> if self . num_of_freqs_to_thermal_adjust and not instrument_is_installed ( '<STR_LIT>' ) : <EOL> self . logger . warn ( '<STR_LIT>' ) <EOL> self . num_of_freqs_to_thermal_adjust = <NUM_LIT:0> <EOL> def initialize ( self , context ) : <EOL> self . number_of_cpus = { } <EOL> self . report_template_file = context . resolver . get ( File ( self , REPORT_TEMPLATE_FILE ) ) <EOL> self . em_template_file = context . resolver . get ( File ( self , EM_TEMPLATE_FILE ) ) <EOL> self . little_core = ( set ( self . device . core_names ) - set ( [ self . big_core ] ) ) . pop ( ) <EOL> self . perform_runtime_validation ( ) <EOL> self . enable_all_cores ( ) <EOL> self . configure_clusters ( ) <EOL> self . discover_idle_states ( ) <EOL> self . disable_thermal_management ( ) <EOL> self . initialize_job_queue ( context ) <EOL> self . initialize_result_tracking ( ) <EOL> def setup ( self , context ) : <EOL> if not context . spec . label . startswith ( '<STR_LIT>' ) : <EOL> return <EOL> for idle_state in self . get_device_idle_states ( self . measured_cluster ) : <EOL> if idle_state . index > context . spec . idle_state_index : <EOL> idle_state . disable = <NUM_LIT:1> <EOL> else : <EOL> idle_state . disable = <NUM_LIT:0> <EOL> def fast_start ( self , context ) : <EOL> self . start_time = time . time ( ) <EOL> def fast_stop ( self , context ) : <EOL> self . run_time = time . time ( ) - self . start_time <EOL> def on_iteration_start ( self , context ) : <EOL> self . setup_measurement ( context . spec . cluster ) <EOL> def thermal_correction ( self , context ) : <EOL> if not self . num_of_freqs_to_thermal_adjust or self . num_of_freqs_to_thermal_adjust > len ( self . big_frequencies ) : <EOL> return <NUM_LIT:0> <EOL> freqs = self . big_frequencies [ - self . num_of_freqs_to_thermal_adjust : ] <EOL> spec = context . result . spec <EOL> if spec . frequency not in freqs : <EOL> return <NUM_LIT:0> <EOL> data_path = os . path . join ( context . output_directory , '<STR_LIT>' , '<STR_LIT>' . format ( self . big_core ) ) <EOL> data = pd . read_csv ( data_path ) [ '<STR_LIT>' ] <EOL> return _adjust_for_thermal ( data , filt_method = lambda x : pd . rolling_median ( x , <NUM_LIT:1000> ) , thresh = <NUM_LIT> , window = <NUM_LIT> ) <EOL> def slow_update_result ( self , context ) : <EOL> spec = context . result . spec <EOL> cluster = spec . cluster <EOL> is_freq_iteration = spec . label . startswith ( '<STR_LIT>' ) <EOL> perf_metric = <NUM_LIT:0> <EOL> power_metric = <NUM_LIT:0> <EOL> thermal_adjusted_power = <NUM_LIT:0> <EOL> if is_freq_iteration and cluster == '<STR_LIT>' : <EOL> thermal_adjusted_power = self . thermal_correction ( context ) <EOL> for metric in context . result . metrics : <EOL> if metric . name == self . performance_metric : <EOL> perf_metric = metric . value <EOL> elif thermal_adjusted_power and metric . name in self . big_power_metrics : <EOL> power_metric += thermal_adjusted_power * self . power_scaling_factor <EOL> elif ( cluster == '<STR_LIT>' ) and metric . name in self . big_power_metrics : <EOL> power_metric += metric . value * self . power_scaling_factor <EOL> elif ( cluster == '<STR_LIT>' ) and metric . name in self . little_power_metrics : <EOL> power_metric += metric . value * self . power_scaling_factor <EOL> elif thermal_adjusted_power and metric . name in self . big_energy_metrics : <EOL> power_metric += thermal_adjusted_power / self . run_time * self . power_scaling_factor <EOL> elif ( cluster == '<STR_LIT>' ) and metric . name in self . big_energy_metrics : <EOL> power_metric += metric . value / self . run_time * self . power_scaling_factor <EOL> elif ( cluster == '<STR_LIT>' ) and metric . name in self . little_energy_metrics : <EOL> power_metric += metric . value / self . run_time * self . power_scaling_factor <EOL> if not ( power_metric and ( perf_metric or not is_freq_iteration ) ) : <EOL> message = '<STR_LIT>' <EOL> raise InstrumentError ( message . format ( context . result . spec . id , context . current_iteration ) ) <EOL> if is_freq_iteration : <EOL> index_matter = [ cluster , spec . num_cpus , <EOL> spec . frequency , context . result . iteration ] <EOL> data = self . freq_data <EOL> else : <EOL> index_matter = [ cluster , spec . num_cpus , <EOL> spec . idle_state_id , spec . idle_state_desc , context . result . iteration ] <EOL> data = self . idle_data <EOL> if self . no_hotplug : <EOL> power_metric = spec . num_cpus * ( power_metric / self . number_of_cpus [ cluster ] ) <EOL> data . append ( index_matter + [ '<STR_LIT>' , perf_metric ] ) <EOL> data . append ( index_matter + [ '<STR_LIT>' , power_metric ] ) <EOL> def before_overall_results_processing ( self , context ) : <EOL> if not self . idle_data or not self . freq_data : <EOL> self . logger . warning ( '<STR_LIT>' ) <EOL> return <EOL> output_directory = os . path . join ( context . output_directory , '<STR_LIT>' ) <EOL> os . makedirs ( output_directory ) <EOL> df = pd . DataFrame ( self . idle_data , columns = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT:state>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:value>' ] ) <EOL> idle_power_table = wa_result_to_power_perf_table ( df , '<STR_LIT>' , index = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:state>' ] ) <EOL> idle_output = os . path . join ( output_directory , IDLE_TABLE_FILE ) <EOL> with open ( idle_output , '<STR_LIT:w>' ) as wfh : <EOL> idle_power_table . to_csv ( wfh , index = False ) <EOL> context . add_artifact ( '<STR_LIT>' , idle_output , '<STR_LIT>' ) <EOL> df = pd . DataFrame ( self . freq_data , <EOL> columns = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:value>' ] ) <EOL> freq_power_table = wa_result_to_power_perf_table ( df , self . performance_metric , <EOL> index = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> freq_output = os . path . join ( output_directory , FREQ_TABLE_FILE ) <EOL> with open ( freq_output , '<STR_LIT:w>' ) as wfh : <EOL> freq_power_table . to_csv ( wfh , index = False ) <EOL> context . add_artifact ( '<STR_LIT>' , freq_output , '<STR_LIT>' ) <EOL> if self . big_opps is None or self . little_opps is None : <EOL> message = '<STR_LIT>' <EOL> self . logger . warning ( message ) <EOL> opps = { '<STR_LIT>' : self . big_opps , '<STR_LIT>' : self . little_opps } <EOL> leakages = { '<STR_LIT>' : self . big_leakage , '<STR_LIT>' : self . little_leakage } <EOL> try : <EOL> measured_cpus_table , cpus_table = get_cpus_power_table ( freq_power_table , '<STR_LIT>' , opps , leakages ) <EOL> except ( ValueError , KeyError , IndexError ) as e : <EOL> self . logger . error ( '<STR_LIT>' . format ( e ) ) <EOL> return <EOL> measured_cpus_output = os . path . join ( output_directory , MEASURED_CPUS_TABLE_FILE ) <EOL> with open ( measured_cpus_output , '<STR_LIT:w>' ) as wfh : <EOL> measured_cpus_table . to_csv ( wfh ) <EOL> context . add_artifact ( '<STR_LIT>' , measured_cpus_output , '<STR_LIT>' ) <EOL> cpus_output = os . path . join ( output_directory , CPUS_TABLE_FILE ) <EOL> with open ( cpus_output , '<STR_LIT:w>' ) as wfh : <EOL> cpus_table . to_csv ( wfh ) <EOL> context . add_artifact ( '<STR_LIT>' , cpus_output , '<STR_LIT>' ) <EOL> em = build_energy_model ( freq_power_table , cpus_table , idle_power_table , self . first_cluster_idle_state ) <EOL> em_file = os . path . join ( output_directory , '<STR_LIT>' . format ( self . device_name ) ) <EOL> em_text = generate_em_c_file ( em , self . big_core , self . little_core , <EOL> self . em_template_file , em_file ) <EOL> context . add_artifact ( '<STR_LIT>' , em_file , '<STR_LIT:data>' ) <EOL> report_file = os . path . join ( output_directory , '<STR_LIT>' ) <EOL> generate_report ( freq_power_table , measured_cpus_table , cpus_table , <EOL> idle_power_table , self . report_template_file , <EOL> self . device_name , em_text , report_file ) <EOL> context . add_artifact ( '<STR_LIT>' , report_file , '<STR_LIT>' ) <EOL> def initialize_result_tracking ( self ) : <EOL> self . freq_data = [ ] <EOL> self . idle_data = [ ] <EOL> self . big_power_metrics = [ ] <EOL> self . little_power_metrics = [ ] <EOL> self . big_energy_metrics = [ ] <EOL> self . little_energy_metrics = [ ] <EOL> if self . power_metric : <EOL> self . big_power_metrics = [ pm . format ( core = self . big_core ) for pm in self . power_metric ] <EOL> self . little_power_metrics = [ pm . format ( core = self . little_core ) for pm in self . power_metric ] <EOL> else : <EOL> self . big_energy_metrics = [ em . format ( core = self . big_core ) for em in self . energy_metric ] <EOL> self . little_energy_metrics = [ em . format ( core = self . little_core ) for em in self . energy_metric ] <EOL> def configure_clusters ( self ) : <EOL> self . measured_cores = None <EOL> self . measuring_cores = None <EOL> self . cpuset = self . device . get_cgroup_controller ( '<STR_LIT>' ) <EOL> self . cpuset . create_group ( '<STR_LIT>' , self . big_cpus , [ <NUM_LIT:0> ] ) <EOL> self . cpuset . create_group ( '<STR_LIT>' , self . little_cpus , [ <NUM_LIT:0> ] ) <EOL> for cluster in set ( self . device . core_clusters ) : <EOL> self . device . set_cluster_governor ( cluster , '<STR_LIT>' ) <EOL> def discover_idle_states ( self ) : <EOL> online_cpu = self . device . get_online_cpus ( self . big_core ) [ <NUM_LIT:0> ] <EOL> self . big_idle_states = self . device . get_cpuidle_states ( online_cpu ) <EOL> online_cpu = self . device . get_online_cpus ( self . little_core ) [ <NUM_LIT:0> ] <EOL> self . little_idle_states = self . device . get_cpuidle_states ( online_cpu ) <EOL> if not ( len ( self . big_idle_states ) >= <NUM_LIT:2> and len ( self . little_idle_states ) >= <NUM_LIT:2> ) : <EOL> raise DeviceError ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> def setup_measurement ( self , measured ) : <EOL> measuring = '<STR_LIT>' if measured == '<STR_LIT>' else '<STR_LIT>' <EOL> self . measured_cluster = measured <EOL> self . measuring_cluster = measuring <EOL> self . measured_cpus = self . big_cpus if measured == '<STR_LIT>' else self . little_cpus <EOL> self . measuring_cpus = self . little_cpus if measured == '<STR_LIT>' else self . big_cpus <EOL> self . reset ( ) <EOL> def reset ( self ) : <EOL> self . enable_all_cores ( ) <EOL> self . enable_all_idle_states ( ) <EOL> self . reset_cgroups ( ) <EOL> self . cpuset . move_all_tasks_to ( self . measuring_cluster ) <EOL> server_process = '<STR_LIT>' if self . device . platform == '<STR_LIT>' else '<STR_LIT>' <EOL> server_pids = self . device . get_pids_of ( server_process ) <EOL> children_ps = [ e for e in self . device . ps ( ) <EOL> if e . ppid in server_pids and e . name != '<STR_LIT>' ] <EOL> children_pids = [ e . pid for e in children_ps ] <EOL> pids_to_move = server_pids + children_pids <EOL> self . cpuset . root . add_tasks ( pids_to_move ) <EOL> for pid in pids_to_move : <EOL> try : <EOL> self . device . execute ( '<STR_LIT>' . format ( list_to_mask ( self . measuring_cpus ) , pid ) ) <EOL> except DeviceError : <EOL> pass <EOL> def enable_all_cores ( self ) : <EOL> counter = Counter ( self . device . core_names ) <EOL> for core , number in counter . iteritems ( ) : <EOL> self . device . set_number_of_online_cpus ( core , number ) <EOL> self . big_cpus = self . device . get_online_cpus ( self . big_core ) <EOL> self . little_cpus = self . device . get_online_cpus ( self . little_core ) <EOL> def enable_all_idle_states ( self ) : <EOL> for cpu in self . device . online_cpus : <EOL> for state in self . device . get_cpuidle_states ( cpu ) : <EOL> state . disable = <NUM_LIT:0> <EOL> def reset_cgroups ( self ) : <EOL> self . big_cpus = self . device . get_online_cpus ( self . big_core ) <EOL> self . little_cpus = self . device . get_online_cpus ( self . little_core ) <EOL> self . cpuset . big . set ( self . big_cpus , <NUM_LIT:0> ) <EOL> self . cpuset . little . set ( self . little_cpus , <NUM_LIT:0> ) <EOL> def perform_runtime_validation ( self ) : <EOL> if not self . device . is_rooted : <EOL> raise InstrumentError ( '<STR_LIT>' ) <EOL> if '<STR_LIT>' not in self . device . list_available_cluster_governors ( <NUM_LIT:0> ) : <EOL> raise InstrumentError ( '<STR_LIT>' ) <EOL> error_message = '<STR_LIT>' <EOL> available_frequencies = self . device . list_available_core_frequencies ( self . big_core ) <EOL> if self . big_frequencies : <EOL> for freq in self . big_frequencies : <EOL> if freq not in available_frequencies : <EOL> raise ConfigError ( error_message . format ( freq , self . big_core ) ) <EOL> else : <EOL> self . big_frequencies = available_frequencies <EOL> available_frequencies = self . device . list_available_core_frequencies ( self . little_core ) <EOL> if self . little_frequencies : <EOL> for freq in self . little_frequencies : <EOL> if freq not in available_frequencies : <EOL> raise ConfigError ( error_message . format ( freq , self . little_core ) ) <EOL> else : <EOL> self . little_frequencies = available_frequencies <EOL> def initialize_job_queue ( self , context ) : <EOL> old_specs = [ ] <EOL> for job in context . runner . job_queue : <EOL> if job . spec not in old_specs : <EOL> old_specs . append ( job . spec ) <EOL> new_specs = self . get_cluster_specs ( old_specs , '<STR_LIT>' , context ) <EOL> new_specs . extend ( self . get_cluster_specs ( old_specs , '<STR_LIT>' , context ) ) <EOL> context . config . workload_specs = new_specs <EOL> config_file = os . path . join ( context . host_working_directory , '<STR_LIT>' ) <EOL> with open ( config_file , '<STR_LIT:wb>' ) as wfh : <EOL> context . config . serialize ( wfh ) <EOL> context . runner . init_queue ( new_specs ) <EOL> def get_cluster_specs ( self , old_specs , cluster , context ) : <EOL> core = self . get_core_name ( cluster ) <EOL> self . number_of_cpus [ cluster ] = sum ( [ <NUM_LIT:1> for c in self . device . core_names if c == core ] ) <EOL> cluster_frequencies = self . get_frequencies_param ( cluster ) <EOL> if not cluster_frequencies : <EOL> raise InstrumentError ( '<STR_LIT>' . format ( core ) ) <EOL> min_frequency = min ( cluster_frequencies ) <EOL> idle_states = self . get_device_idle_states ( cluster ) <EOL> new_specs = [ ] <EOL> for state in idle_states : <EOL> for num_cpus in xrange ( <NUM_LIT:1> , self . number_of_cpus [ cluster ] + <NUM_LIT:1> ) : <EOL> spec = old_specs [ <NUM_LIT:0> ] . copy ( ) <EOL> spec . workload_name = self . idle_workload <EOL> spec . workload_parameters = self . idle_workload_params <EOL> spec . idle_state_id = state . id <EOL> spec . idle_state_desc = state . desc <EOL> spec . idle_state_index = state . index <EOL> if not self . no_hotplug : <EOL> spec . runtime_parameters [ '<STR_LIT>' . format ( core ) ] = num_cpus <EOL> spec . runtime_parameters [ '<STR_LIT>' . format ( core ) ] = min_frequency <EOL> spec . runtime_parameters [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> spec . cluster = cluster <EOL> spec . num_cpus = num_cpus <EOL> spec . id = '<STR_LIT>' . format ( cluster , state . id , num_cpus ) <EOL> spec . label = '<STR_LIT>' . format ( cluster ) <EOL> spec . number_of_iterations = old_specs [ <NUM_LIT:0> ] . number_of_iterations <EOL> spec . load ( self . device , context . config . ext_loader ) <EOL> spec . workload . init_resources ( context ) <EOL> spec . workload . validate ( ) <EOL> new_specs . append ( spec ) <EOL> for old_spec in old_specs : <EOL> if old_spec . workload_name not in [ '<STR_LIT>' , '<STR_LIT>' ] : <EOL> raise ConfigError ( '<STR_LIT>' ) <EOL> for freq in cluster_frequencies : <EOL> for num_cpus in xrange ( <NUM_LIT:1> , self . number_of_cpus [ cluster ] + <NUM_LIT:1> ) : <EOL> spec = old_spec . copy ( ) <EOL> spec . runtime_parameters [ '<STR_LIT>' . format ( core ) ] = freq <EOL> if not self . no_hotplug : <EOL> spec . runtime_parameters [ '<STR_LIT>' . format ( core ) ] = num_cpus <EOL> spec . runtime_parameters [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> spec . id = '<STR_LIT>' . format ( cluster , num_cpus , freq ) <EOL> spec . label = '<STR_LIT>' . format ( cluster , spec . label ) <EOL> spec . workload_parameters [ '<STR_LIT>' ] = list_to_mask ( self . get_cpus ( cluster ) ) <EOL> spec . workload_parameters [ '<STR_LIT>' ] = num_cpus <EOL> if old_spec . workload_name == '<STR_LIT>' : <EOL> spec . workload_parameters [ '<STR_LIT>' ] = <NUM_LIT> <EOL> spec . cluster = cluster <EOL> spec . num_cpus = num_cpus <EOL> spec . frequency = freq <EOL> spec . load ( self . device , context . config . ext_loader ) <EOL> spec . workload . init_resources ( context ) <EOL> spec . workload . validate ( ) <EOL> new_specs . append ( spec ) <EOL> return new_specs <EOL> def disable_thermal_management ( self ) : <EOL> if self . device . file_exists ( '<STR_LIT>' ) : <EOL> tzone_paths = self . device . execute ( '<STR_LIT>' ) <EOL> for tzpath in tzone_paths . strip ( ) . split ( ) : <EOL> mode_file = '<STR_LIT>' . format ( tzpath ) <EOL> if self . device . file_exists ( mode_file ) : <EOL> self . device . set_sysfile_value ( mode_file , '<STR_LIT>' ) <EOL> def get_device_idle_states ( self , cluster ) : <EOL> if cluster == '<STR_LIT>' : <EOL> online_cpus = self . device . get_online_cpus ( self . big_core ) <EOL> else : <EOL> online_cpus = self . device . get_online_cpus ( self . little_core ) <EOL> idle_states = [ ] <EOL> for cpu in online_cpus : <EOL> idle_states . extend ( self . device . get_cpuidle_states ( cpu ) ) <EOL> return idle_states <EOL> def get_core_name ( self , cluster ) : <EOL> if cluster == '<STR_LIT>' : <EOL> return self . big_core <EOL> else : <EOL> return self . little_core <EOL> def get_cpus ( self , cluster ) : <EOL> if cluster == '<STR_LIT>' : <EOL> return self . big_cpus <EOL> else : <EOL> return self . little_cpus <EOL> def get_frequencies_param ( self , cluster ) : <EOL> if cluster == '<STR_LIT>' : <EOL> return self . big_frequencies <EOL> else : <EOL> return self . little_frequencies <EOL> def _adjust_for_thermal ( data , filt_method = lambda x : x , thresh = <NUM_LIT> , window = <NUM_LIT> , tdiff_threshold = <NUM_LIT> ) : <EOL> n = filt_method ( data ) <EOL> n = n [ ~ np . isnan ( n ) ] <EOL> d = np . diff ( n ) <EOL> d = d [ ~ np . isnan ( d ) ] <EOL> dmin = min ( d ) <EOL> dmax = max ( d ) <EOL> index_up = np . max ( ( d > dmax * thresh ) . nonzero ( ) ) <EOL> index_down = np . min ( ( d < dmin * thresh ) . nonzero ( ) ) <EOL> low_average = np . average ( n [ index_up : index_up + window ] ) <EOL> high_average = np . average ( n [ index_down - window : index_down ] ) <EOL> if low_average > high_average or index_down - index_up < tdiff_threshold : <EOL> return <NUM_LIT:0> <EOL> else : <EOL> return low_average <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> import sys <EOL> indir , outdir = sys . argv [ <NUM_LIT:1> ] , sys . argv [ <NUM_LIT:2> ] <EOL> device_name = '<STR_LIT>' <EOL> big_core = '<STR_LIT>' <EOL> little_core = '<STR_LIT>' <EOL> first_cluster_idle_state = - <NUM_LIT:1> <EOL> this_dir = os . path . dirname ( __file__ ) <EOL> report_template_file = os . path . join ( this_dir , REPORT_TEMPLATE_FILE ) <EOL> em_template_file = os . path . join ( this_dir , EM_TEMPLATE_FILE ) <EOL> freq_power_table = pd . read_csv ( os . path . join ( indir , FREQ_TABLE_FILE ) ) <EOL> measured_cpus_table , cpus_table = pd . read_csv ( os . path . join ( indir , CPUS_TABLE_FILE ) , <EOL> header = range ( <NUM_LIT:2> ) , index_col = <NUM_LIT:0> ) <EOL> idle_power_table = pd . read_csv ( os . path . join ( indir , IDLE_TABLE_FILE ) ) <EOL> if not os . path . exists ( outdir ) : <EOL> os . makedirs ( outdir ) <EOL> report_file = os . path . join ( outdir , '<STR_LIT>' ) <EOL> em_file = os . path . join ( outdir , '<STR_LIT>' . format ( device_name ) ) <EOL> em = build_energy_model ( freq_power_table , cpus_table , <EOL> idle_power_table , first_cluster_idle_state ) <EOL> em_text = generate_em_c_file ( em , big_core , little_core , <EOL> em_template_file , em_file ) <EOL> generate_report ( freq_power_table , measured_cpus_table , cpus_table , <EOL> idle_power_table , report_template_file , device_name , <EOL> em_text , report_file ) </s>
