<s> from . import ureg , Q_ <EOL> from datetime import datetime <EOL> import pandas as pd <EOL> import numpy as np <EOL> from warnings import warn <EOL> import pytz <EOL> from eemeter . evaluation import Period <EOL> class ConsumptionData ( object ) : <EOL> """<STR_LIT>""" <EOL> UNITS = { <EOL> "<STR_LIT>" : { <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : <NUM_LIT:1.0> , <EOL> } , <EOL> "<STR_LIT>" : { <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : <NUM_LIT:1.0> , <EOL> } , <EOL> "<STR_LIT>" : { <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : <NUM_LIT:1.0> , <EOL> } , <EOL> "<STR_LIT>" : { <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : <NUM_LIT:1.0> , <EOL> } , <EOL> "<STR_LIT>" : { <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : <NUM_LIT:1.0> , <EOL> } , <EOL> "<STR_LIT>" : { <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : <NUM_LIT:1.0> , <EOL> } , <EOL> "<STR_LIT>" : { <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : <NUM_LIT:1.0> , <EOL> } , <EOL> "<STR_LIT>" : { <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : <NUM_LIT:1.0> , <EOL> } , <EOL> "<STR_LIT>" : { <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : <NUM_LIT:1.0> , <EOL> } , <EOL> "<STR_LIT>" : { <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : <NUM_LIT> , <EOL> } , <EOL> "<STR_LIT>" : { <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : <NUM_LIT> , <EOL> } , <EOL> "<STR_LIT>" : { <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : <NUM_LIT> , <EOL> } , <EOL> } <EOL> def __init__ ( self , records , fuel_type , unit_name , <EOL> record_type = "<STR_LIT>" , freq = None , pulse_value = None , name = None , <EOL> data = None , estimated = None ) : <EOL> if unit_name in self . UNITS : <EOL> self . unit_name = self . UNITS [ unit_name ] [ "<STR_LIT>" ] <EOL> multiplier = self . UNITS [ unit_name ] [ "<STR_LIT>" ] <EOL> else : <EOL> message = '<STR_LIT>' . format ( unit_name ) <EOL> raise ValueError ( message ) <EOL> if fuel_type not in [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" ] : <EOL> message = '<STR_LIT>' . format ( fuel_type ) <EOL> raise ValueError ( message ) <EOL> else : <EOL> self . fuel_type = fuel_type <EOL> self . freq = freq <EOL> self . pulse_value = pulse_value <EOL> self . name = name <EOL> if data is not None : <EOL> if not records is None : <EOL> message = "<STR_LIT>" <EOL> raise ValueError ( message ) <EOL> if estimated is None : <EOL> message = "<STR_LIT>" "<STR_LIT>" "<STR_LIT>" "<STR_LIT>" <EOL> raise ValueError ( message ) <EOL> self . data = data <EOL> self . estimated = estimated <EOL> if freq is None : <EOL> self . freq_timedelta = None <EOL> elif freq [ - <NUM_LIT:1> ] not in [ "<STR_LIT:D>" , "<STR_LIT:H>" , "<STR_LIT:T>" , "<STR_LIT:S>" ] : <EOL> message = '<STR_LIT>' . format ( freq ) <EOL> raise ValueError ( message ) <EOL> else : <EOL> try : <EOL> dummy_start_date = datetime ( <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT:1> , tzinfo = pytz . utc ) <EOL> dummy_date_range = pd . date_range ( dummy_start_date , <EOL> periods = <NUM_LIT:2> , freq = freq ) <EOL> freq_timedelta = dummy_date_range [ <NUM_LIT:1> ] - dummy_date_range [ <NUM_LIT:0> ] <EOL> except ValueError : <EOL> message = '<STR_LIT>' . format ( freq ) <EOL> raise ValueError ( message ) <EOL> self . freq_timedelta = freq_timedelta <EOL> return <EOL> if "<STR_LIT>" == record_type : <EOL> if freq is None or freq [ - <NUM_LIT:1> ] not in [ "<STR_LIT:D>" , "<STR_LIT:H>" , "<STR_LIT:T>" , "<STR_LIT:S>" ] : <EOL> message = '<STR_LIT>' . format ( freq ) <EOL> raise ValueError ( message ) <EOL> else : <EOL> try : <EOL> dummy_start_date = datetime ( <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT:1> , tzinfo = pytz . utc ) <EOL> dummy_date_range = pd . date_range ( dummy_start_date , <EOL> periods = <NUM_LIT:2> , freq = freq ) <EOL> freq_timedelta = dummy_date_range [ <NUM_LIT:1> ] - dummy_date_range [ <NUM_LIT:0> ] <EOL> except ValueError : <EOL> message = '<STR_LIT>' . format ( freq ) <EOL> raise ValueError ( message ) <EOL> self . freq_timedelta = freq_timedelta <EOL> self . data , self . estimated = self . _import_interval ( records ) <EOL> elif record_type in [ "<STR_LIT>" , "<STR_LIT>" ] : <EOL> self . freq_timedelta = None <EOL> self . data , self . estimated = self . _import_arbitrary ( records ) <EOL> elif record_type in [ "<STR_LIT>" , "<STR_LIT>" ] : <EOL> self . freq_timedelta = None <EOL> self . data , self . estimated = self . _import_arbitrary_start ( records ) <EOL> elif record_type in [ "<STR_LIT>" , "<STR_LIT>" ] : <EOL> self . freq_timedelta = None <EOL> self . data , self . estimated = self . _import_arbitrary_end ( records ) <EOL> elif "<STR_LIT>" == record_type : <EOL> if pulse_value is None or pulse_value <= <NUM_LIT:0> : <EOL> message = '<STR_LIT>' '<STR_LIT>' . format ( pulse_value ) <EOL> raise ValueError ( message ) <EOL> self . freq_timedelta = None <EOL> self . data , self . estimated = self . _import_pulse ( records ) <EOL> else : <EOL> message ( '<STR_LIT>' . format ( record_type ) ) <EOL> raise ValueError <EOL> self . data *= multiplier <EOL> def _import_interval ( self , records ) : <EOL> if records == [ ] : <EOL> return pd . Series ( [ ] ) , pd . Series ( [ ] ) <EOL> if records [ <NUM_LIT:0> ] . get ( "<STR_LIT:start>" ) is not None : <EOL> key = "<STR_LIT:start>" <EOL> elif records [ <NUM_LIT:0> ] . get ( "<STR_LIT:end>" ) is not None : <EOL> key = "<STR_LIT:end>" <EOL> else : <EOL> message = '<STR_LIT>' '<STR_LIT>' <EOL> raise ValueError ( message ) <EOL> try : <EOL> sorted_records = sorted ( records , key = lambda x : x [ key ] ) <EOL> except KeyError : <EOL> message = '<STR_LIT>' '<STR_LIT>' <EOL> raise ValueError ( message ) <EOL> start_limit = sorted_records [ <NUM_LIT:0> ] . get ( key ) <EOL> end_limit = sorted_records [ - <NUM_LIT:1> ] . get ( key ) <EOL> dt_index = pd . date_range ( start = start_limit , end = end_limit , <EOL> freq = self . freq ) <EOL> if key == "<STR_LIT:end>" : <EOL> dt_index -= self . freq_timedelta <EOL> data = pd . Series ( np . tile ( np . nan , dt_index . shape ) , <EOL> index = dt_index ) <EOL> estimated = pd . Series ( np . tile ( False , dt_index . shape ) , <EOL> index = dt_index ) <EOL> for record in sorted_records : <EOL> dt = record . get ( key ) <EOL> value = record . get ( "<STR_LIT:value>" ) <EOL> est = record . get ( "<STR_LIT>" ) <EOL> if dt is None : <EOL> message = '<STR_LIT>' '<STR_LIT>' <EOL> raise ValueError ( message ) <EOL> if key == "<STR_LIT:end>" : <EOL> dt -= self . freq_timedelta <EOL> try : <EOL> current_value = data [ dt ] <EOL> except KeyError : <EOL> current_value = None <EOL> if current_value is None : <EOL> message = "<STR_LIT>" "<STR_LIT>" . format ( dt , value ) <EOL> warn ( message ) <EOL> elif pd . isnull ( current_value ) : <EOL> data [ dt ] = value <EOL> if est : <EOL> estimated [ dt ] = True <EOL> else : <EOL> message = "<STR_LIT>" "<STR_LIT>" . format ( dt , value ) <EOL> warn ( message ) <EOL> return data , estimated <EOL> def _import_arbitrary ( self , records ) : <EOL> if records == [ ] : <EOL> return pd . Series ( [ ] ) , pd . Series ( [ ] ) <EOL> try : <EOL> sorted_records = sorted ( records , key = lambda x : x [ "<STR_LIT:start>" ] ) <EOL> except KeyError : <EOL> message = '<STR_LIT>' '<STR_LIT>' <EOL> raise ValueError ( message ) <EOL> start_datetimes = [ ] <EOL> values = [ ] <EOL> estimateds = [ ] <EOL> previous_end_datetime = None <EOL> for record in sorted_records : <EOL> start = record . get ( "<STR_LIT:start>" ) <EOL> end = record . get ( "<STR_LIT:end>" ) <EOL> value = record . get ( "<STR_LIT:value>" ) <EOL> estimated = record . get ( "<STR_LIT>" ) <EOL> if start is None or end is None : <EOL> message = '<STR_LIT>' '<STR_LIT>' <EOL> raise ValueError ( message ) <EOL> else : <EOL> if start >= end : <EOL> message = '<STR_LIT>' '<STR_LIT>' . format ( start , end ) <EOL> raise ValueError ( message ) <EOL> if previous_end_datetime is None or start == previous_end_datetime : <EOL> start_datetimes . append ( start ) <EOL> values . append ( value ) <EOL> previous_end_datetime = end <EOL> estimateds . append ( bool ( estimated ) ) <EOL> elif start < previous_end_datetime : <EOL> message = '<STR_LIT>' '<STR_LIT>' . format ( start , previous_end_datetime ) <EOL> warn ( message ) <EOL> else : <EOL> start_datetimes . append ( previous_end_datetime ) <EOL> values . append ( np . nan ) <EOL> estimateds . append ( False ) <EOL> start_datetimes . append ( start ) <EOL> values . append ( value ) <EOL> previous_end_datetime = end <EOL> estimateds . append ( bool ( estimated ) ) <EOL> start_datetimes . append ( previous_end_datetime ) <EOL> values . append ( np . nan ) <EOL> estimateds . append ( False ) <EOL> dt_index = pd . DatetimeIndex ( start_datetimes ) <EOL> data = pd . Series ( values , index = dt_index ) <EOL> estimated = pd . Series ( estimateds , index = dt_index ) <EOL> return data , estimated <EOL> def _import_arbitrary_start ( self , records ) : <EOL> if records == [ ] : <EOL> return pd . Series ( [ ] ) , pd . Series ( [ ] ) <EOL> try : <EOL> sorted_records = sorted ( records , key = lambda x : x [ "<STR_LIT:start>" ] ) <EOL> except KeyError : <EOL> message = '<STR_LIT>' <EOL> raise ValueError ( message ) <EOL> start_datetimes = [ ] <EOL> values = [ ] <EOL> estimateds = [ ] <EOL> for record in sorted_records : <EOL> start = record [ "<STR_LIT:start>" ] <EOL> value = record . get ( "<STR_LIT:value>" ) <EOL> estimated = record . get ( "<STR_LIT>" ) <EOL> start_datetimes . append ( start ) <EOL> values . append ( value ) <EOL> estimateds . append ( bool ( estimated ) ) <EOL> last_end = sorted_records [ - <NUM_LIT:1> ] . get ( "<STR_LIT:end>" ) <EOL> if last_end is None : <EOL> values [ - <NUM_LIT:1> ] = np . nan <EOL> else : <EOL> start_datetimes . append ( last_end ) <EOL> values . append ( np . nan ) <EOL> estimateds . append ( False ) <EOL> dt_index = pd . DatetimeIndex ( start_datetimes ) <EOL> data = pd . Series ( values , index = dt_index ) <EOL> estimated = pd . Series ( estimateds , index = dt_index ) <EOL> return data , estimated <EOL> def _import_arbitrary_end ( self , records ) : <EOL> if records == [ ] : <EOL> return pd . Series ( [ ] ) , pd . Series ( [ ] ) <EOL> try : <EOL> sorted_records = sorted ( records , key = lambda x : x [ "<STR_LIT:end>" ] ) <EOL> except KeyError : <EOL> message = '<STR_LIT>' <EOL> raise ValueError ( message ) <EOL> end_datetimes = [ r [ "<STR_LIT:end>" ] for r in sorted_records ] <EOL> values = [ r . get ( "<STR_LIT:value>" ) for r in sorted_records [ <NUM_LIT:1> : ] ] <EOL> estimateds = [ bool ( r . get ( "<STR_LIT>" ) ) for r in sorted_records [ <NUM_LIT:1> : ] ] <EOL> first_start = sorted_records [ <NUM_LIT:0> ] . get ( "<STR_LIT:start>" ) <EOL> if first_start is not None : <EOL> end_datetimes . insert ( <NUM_LIT:0> , first_start ) <EOL> values . insert ( <NUM_LIT:0> , sorted_records [ <NUM_LIT:0> ] . get ( "<STR_LIT:value>" ) ) <EOL> estimateds . insert ( <NUM_LIT:0> , bool ( sorted_records [ <NUM_LIT:0> ] . get ( "<STR_LIT>" ) ) ) <EOL> values . append ( np . nan ) <EOL> estimateds . append ( False ) <EOL> dt_index = pd . DatetimeIndex ( end_datetimes ) <EOL> data = pd . Series ( values , index = dt_index ) <EOL> estimated = pd . Series ( estimateds , index = dt_index ) <EOL> return data , estimated <EOL> def _import_pulse ( self , records ) : <EOL> if records == [ ] : <EOL> return pd . Series ( [ ] ) , pd . Series ( [ ] ) <EOL> try : <EOL> sorted_records = sorted ( records , key = lambda x : x [ "<STR_LIT>" ] ) <EOL> except KeyError : <EOL> message = '<STR_LIT>' <EOL> raise ValueError ( message ) <EOL> pulses = [ r . get ( "<STR_LIT>" ) for r in sorted_records ] <EOL> estimateds = [ bool ( r . get ( "<STR_LIT>" ) ) for r in sorted_records ] <EOL> values = np . tile ( float ( self . pulse_value ) , ( len ( pulses ) , ) ) <EOL> values [ <NUM_LIT:0> ] = np . nan <EOL> dt_index = pd . DatetimeIndex ( pulses ) <EOL> data = pd . Series ( values , index = dt_index ) <EOL> estimated = pd . Series ( estimateds , index = dt_index ) <EOL> return data , estimated <EOL> def to ( self , unit_name ) : <EOL> """<STR_LIT>""" <EOL> old_quantities = Q_ ( self . data . values , ureg [ self . unit_name ] ) <EOL> new_quantities = old_quantities . to ( unit_name ) <EOL> return new_quantities . magnitude <EOL> def periods ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . freq_timedelta is None : <EOL> periods = [ Period ( start , end ) for start , end in <EOL> zip ( self . data . index , self . data . index [ <NUM_LIT:1> : ] ) ] <EOL> return periods <EOL> else : <EOL> periods = [ Period ( dt , dt + self . freq_timedelta ) <EOL> for dt in self . data . index ] <EOL> return periods <EOL> def average_daily_consumptions ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . freq_timedelta is None : <EOL> avgs , n_days = [ ] , [ ] <EOL> for v , ns in zip ( self . data , np . diff ( self . data . index ) ) : <EOL> days = ns . astype ( '<STR_LIT:d>' ) / <NUM_LIT> <EOL> avgs . append ( v / days ) <EOL> n_days . append ( days ) <EOL> return np . array ( avgs ) , np . array ( n_days ) <EOL> else : <EOL> days = self . freq_timedelta . days + self . freq_timedelta . seconds / <NUM_LIT> <EOL> avgs , n_days = [ ] , [ ] <EOL> for v in self . data : <EOL> avgs . append ( v / days ) <EOL> n_days . append ( days ) <EOL> return np . array ( avgs ) , np . array ( n_days ) <EOL> def total_period ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . data . shape [ <NUM_LIT:0> ] < <NUM_LIT:1> : <EOL> return None <EOL> start_date = self . data . index [ <NUM_LIT:0> ] <EOL> end_date = self . data . index [ - <NUM_LIT:1> ] <EOL> if self . freq_timedelta is not None : <EOL> end_date += self . freq_timedelta <EOL> return Period ( start_date , end_date ) <EOL> def total_days ( self ) : <EOL> """<STR_LIT>""" <EOL> period = self . total_period ( ) <EOL> if period is None : <EOL> return <NUM_LIT:0> <EOL> else : <EOL> tdelta = period . timedelta <EOL> return tdelta . days + tdelta . seconds / <NUM_LIT> <EOL> def records ( self , record_type = "<STR_LIT>" ) : <EOL> """<STR_LIT>""" <EOL> records = [ ] <EOL> if record_type == "<STR_LIT>" : <EOL> for s , v , est in zip ( self . data . index , self . data , self . estimated ) : <EOL> records . append ( { <EOL> "<STR_LIT:start>" : s . to_datetime ( ) , <EOL> "<STR_LIT:value>" : v , <EOL> "<STR_LIT>" : bool ( est ) , <EOL> } ) <EOL> elif record_type in [ "<STR_LIT>" , "<STR_LIT>" ] : <EOL> for s , e , v , est in zip ( self . data . index , self . data . index [ <NUM_LIT:1> : ] , self . data , self . estimated ) : <EOL> records . append ( { <EOL> "<STR_LIT:start>" : s . to_datetime ( ) , <EOL> "<STR_LIT:end>" : e . to_datetime ( ) , <EOL> "<STR_LIT:value>" : v , <EOL> "<STR_LIT>" : bool ( est ) , <EOL> } ) <EOL> elif record_type in [ "<STR_LIT>" , "<STR_LIT>" ] : <EOL> for s , v , est in zip ( self . data . index , self . data , self . estimated ) : <EOL> records . append ( { <EOL> "<STR_LIT:start>" : s . to_datetime ( ) , <EOL> "<STR_LIT:value>" : v , <EOL> "<STR_LIT>" : bool ( est ) , <EOL> } ) <EOL> elif record_type in [ "<STR_LIT>" , "<STR_LIT>" ] : <EOL> records . append ( { <EOL> "<STR_LIT:end>" : self . data . index [ <NUM_LIT:0> ] . to_datetime ( ) , <EOL> "<STR_LIT:value>" : np . nan , <EOL> "<STR_LIT>" : False , <EOL> } ) <EOL> for e , v , est in zip ( self . data . index [ <NUM_LIT:1> : ] , self . data , self . estimated ) : <EOL> records . append ( { <EOL> "<STR_LIT:end>" : e . to_datetime ( ) , <EOL> "<STR_LIT:value>" : v , <EOL> "<STR_LIT>" : bool ( est ) , <EOL> } ) <EOL> elif record_type == "<STR_LIT>" : <EOL> for i in self . data . index : <EOL> records . append ( { <EOL> "<STR_LIT>" : i . to_datetime ( ) , <EOL> } ) <EOL> shape = ( self . data . values . shape [ <NUM_LIT:0> ] - <NUM_LIT:1> , ) <EOL> if len ( records ) > <NUM_LIT:1> and not all ( self . data . values [ <NUM_LIT:1> : ] == np . tile ( self . data . values [ <NUM_LIT:1> ] , shape ) ) : <EOL> message = '<STR_LIT>' '<STR_LIT>' . format ( self . data . values ) <EOL> warn ( message ) <EOL> else : <EOL> message = "<STR_LIT>" . format ( record_type ) <EOL> raise ValueError ( message ) <EOL> return records <EOL> def filter_by_period ( self , period ) : <EOL> """<STR_LIT>""" <EOL> filtered_data = None <EOL> filtered_estimated = None <EOL> if period . start is None and period . end is None : <EOL> filtered_data = self . data . copy ( ) <EOL> filtered_estimated = self . estimated . copy ( ) <EOL> elif period . start is None and period . end is not None : <EOL> filtered_data = self . data [ : period . end ] . copy ( ) <EOL> filtered_estimated = self . estimated [ : period . end ] . copy ( ) <EOL> elif period . start is not None and period . end is None : <EOL> filtered_data = self . data [ period . start : ] . copy ( ) <EOL> filtered_estimated = self . estimated [ period . start : ] . copy ( ) <EOL> else : <EOL> filtered_data = self . data [ period . start : period . end ] . copy ( ) <EOL> filtered_estimated = self . estimated [ period . start : period . end ] . copy ( ) <EOL> if self . freq is None and filtered_data . shape [ <NUM_LIT:0> ] > <NUM_LIT:0> : <EOL> filtered_data . iloc [ - <NUM_LIT:1> ] = np . nan <EOL> filtered_estimated . iloc [ - <NUM_LIT:1> ] = np . nan <EOL> filtered_consumption_data = ConsumptionData ( <EOL> records = None , <EOL> fuel_type = self . fuel_type , <EOL> unit_name = self . unit_name , <EOL> data = filtered_data , <EOL> estimated = filtered_estimated ) <EOL> return filtered_consumption_data <EOL> def json ( self ) : <EOL> return { <EOL> "<STR_LIT>" : self . fuel_type , <EOL> "<STR_LIT>" : self . unit_name , <EOL> "<STR_LIT>" : [ { <EOL> "<STR_LIT:start>" : r [ "<STR_LIT:start>" ] . isoformat ( ) , <EOL> "<STR_LIT:end>" : r [ "<STR_LIT:end>" ] . isoformat ( ) , <EOL> "<STR_LIT:value>" : r [ "<STR_LIT:value>" ] , <EOL> "<STR_LIT>" : r [ "<STR_LIT>" ] , <EOL> } for r in self . records ( ) ] , <EOL> } <EOL> def __repr__ ( self ) : <EOL> string = "<STR_LIT>" . format ( self . fuel_type , <EOL> self . unit_name ) <EOL> string += self . data . __repr__ ( ) <EOL> return string </s>
