<s> from eemeter . uploader import uploaders , constants <EOL> from eemeter . uploader . requester import Requester <EOL> from datetime import date , datetime <EOL> from eemeter . location import Location <EOL> from eemeter . evaluation import Period <EOL> from eemeter . consumption import ConsumptionData <EOL> from eemeter . project import Project <EOL> import pandas as pd <EOL> import pytz <EOL> import re <EOL> __all__ = [ <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> ] <EOL> def upload_dicts ( project_dict , consumption_dict , url , access_token , project_owner , verbose = True ) : <EOL> """<STR_LIT>""" <EOL> project_df , consumption_df = _dicts_to_dataframes ( project_dict , <EOL> consumption_dict ) <EOL> upload_dataframes ( project_df , consumption_df , url , access_token , project_owner , verbose ) <EOL> def upload_csvs ( project_csv_file , consumption_csv_file , url , access_token , project_owner , verbose = True ) : <EOL> """<STR_LIT>""" <EOL> project_df , consumption_df = _csvs_to_dataframes ( project_csv_file , <EOL> consumption_csv_file ) <EOL> upload_dataframes ( project_df , consumption_df , url , access_token , project_owner , verbose ) <EOL> def upload_dataframes2 ( project_df , consumption_df , url , access_token , project_owner , verbose = True ) : <EOL> """<STR_LIT>""" <EOL> requester = Requester ( url , access_token ) <EOL> project_attribute_key_uploader = uploaders . ProjectAttributeKeyUploader ( requester , verbose ) <EOL> project_uploader = uploaders . ProjectUploader ( requester , verbose ) <EOL> project_attribute_uploader = uploaders . ProjectAttributeUploader ( requester , verbose ) <EOL> consumption_metadata_uploader = uploaders . ConsumptionMetadataUploader ( requester , verbose ) <EOL> consumption_record_uploader = uploaders . ConsumptionRecordUploader ( requester , verbose ) <EOL> project_attribute_keys_data = _get_project_attribute_keys_data ( project_df ) <EOL> for data in project_attribute_keys_data : <EOL> response_data = project_attribute_key_uploader . sync ( data ) <EOL> data . update ( response_data ) <EOL> project_ids = { } <EOL> for project_data , project_attributes_data in _get_project_data ( project_df , project_attribute_keys_data ) : <EOL> project_data [ "<STR_LIT>" ] = project_owner <EOL> project_response_data = project_uploader . sync ( project_data ) <EOL> project_id = project_response_data [ "<STR_LIT>" ] <EOL> project_pk = project_response_data [ "<STR_LIT:id>" ] <EOL> project_ids [ project_pk ] = project_id <EOL> for project_attribute_data in project_attributes_data : <EOL> project_attribute_data [ "<STR_LIT>" ] = project_pk <EOL> project_attribute_data = project_attribute_uploader . sync ( project_attribute_data ) <EOL> for consumption_metadata_data , consumption_records_data in _get_consumption_data ( consumption_df ) : <EOL> project_id = consumption_metadata_data . pop ( "<STR_LIT>" ) <EOL> if project_id in project_ids : <EOL> consumption_metadata_data [ "<STR_LIT:id>" ] = project_ids [ project_id ] <EOL> else : <EOL> project_data = { <EOL> "<STR_LIT>" : project_id , <EOL> "<STR_LIT>" : project_owner , <EOL> } <EOL> project_response_data = project_uploader . sync ( project_data ) <EOL> consumption_metadata_data [ "<STR_LIT>" ] = project_response_data [ "<STR_LIT:id>" ] <EOL> consumption_metadata_response_data = consumption_metadata_uploader . sync ( consumption_metadata_data ) <EOL> consumption_metadata_id = consumption_metadata_response_data [ "<STR_LIT:id>" ] <EOL> for consumption_record_data in consumption_records_data : <EOL> consumption_record_data [ "<STR_LIT>" ] = consumption_metadata_id <EOL> consumption_records_response_data = consumption_record_uploader . bulk_sync ( consumption_records_data ) <EOL> def upload_dataframes ( project_df , consumption_df , url , access_token , project_owner , verbose = True ) : <EOL> """<STR_LIT>""" <EOL> requester = Requester ( url , access_token ) <EOL> project_attribute_key_uploader = uploaders . ProjectAttributeKeyUploader ( requester , verbose ) <EOL> project_uploader = uploaders . ProjectUploader ( requester , verbose ) <EOL> project_attribute_uploader = uploaders . ProjectAttributeUploader ( requester , verbose ) <EOL> consumption_metadata_uploader = uploaders . ConsumptionMetadataUploader ( requester , verbose ) <EOL> consumption_record_uploader = uploaders . ConsumptionRecordUploader ( requester , verbose ) <EOL> project_attribute_keys_data = _get_project_attribute_keys_data ( project_df ) <EOL> for data in project_attribute_keys_data : <EOL> response_data = project_attribute_key_uploader . sync ( data ) <EOL> data . update ( response_data ) <EOL> project_ids = { } <EOL> for project_data , project_attributes_data in _get_project_data ( project_df , project_attribute_keys_data ) : <EOL> project_data [ "<STR_LIT>" ] = project_owner <EOL> project_response_data = project_uploader . sync ( project_data ) <EOL> project_id = project_response_data [ "<STR_LIT>" ] <EOL> project_pk = project_response_data [ "<STR_LIT:id>" ] <EOL> project_ids [ project_pk ] = project_id <EOL> for project_attribute_data in project_attributes_data : <EOL> project_attribute_data [ "<STR_LIT>" ] = project_pk <EOL> project_attribute_data = project_attribute_uploader . sync ( project_attribute_data ) <EOL> for consumption_metadata_data , consumption_records_data in _get_consumption_data ( consumption_df ) : <EOL> project_id = consumption_metadata_data . pop ( "<STR_LIT>" ) <EOL> if project_id in project_ids : <EOL> consumption_metadata_data [ "<STR_LIT:id>" ] = project_ids [ project_id ] <EOL> else : <EOL> project_data = { <EOL> "<STR_LIT>" : project_id , <EOL> "<STR_LIT>" : project_owner , <EOL> } <EOL> project_response_data = project_uploader . sync ( project_data ) <EOL> consumption_metadata_data [ "<STR_LIT>" ] = project_response_data [ "<STR_LIT:id>" ] <EOL> consumption_metadata_response_data = consumption_metadata_uploader . sync ( consumption_metadata_data ) <EOL> consumption_metadata_id = consumption_metadata_response_data [ "<STR_LIT:id>" ] <EOL> for consumption_record_data in consumption_records_data : <EOL> consumption_record_data [ "<STR_LIT>" ] = consumption_metadata_id <EOL> consumption_records_response_data = consumption_record_uploader . sync ( consumption_records_data ) <EOL> def _dicts_to_dataframes ( project_dict , consumption_dict ) : <EOL> project_df = pd . DataFrame ( project_dict ) <EOL> consumption_df = pd . DataFrame ( consumption_dict ) <EOL> return project_df , consumption_df <EOL> def _csvs_to_dataframes ( project_csv , consumption_csv ) : <EOL> project_df = pd . read_csv ( project_csv ) <EOL> project_df . baseline_period_end = pd . to_datetime ( project_df . baseline_period_end ) <EOL> project_df . reporting_period_start = pd . to_datetime ( project_df . reporting_period_start ) <EOL> consumption_df = pd . read_csv ( consumption_csv ) <EOL> consumption_df . start = pd . to_datetime ( consumption_df . start ) <EOL> consumption_df . end = pd . to_datetime ( consumption_df . end ) <EOL> return project_df , consumption_df <EOL> def _get_project_attribute_keys_data ( project_df ) : <EOL> project_attribute_keys_data = [ ] <EOL> for column_name in project_df . columns : <EOL> if column_name in constants . STANDARD_PROJECT_DATA_COLUMN_NAMES : <EOL> continue <EOL> if column_name in constants . STANDARD_PROJECT_ATTRIBUTE_KEYS : <EOL> project_attribute_key = constants . STANDARD_PROJECT_ATTRIBUTE_KEYS [ column_name ] <EOL> project_attribute_key_data = { <EOL> "<STR_LIT:name>" : project_attribute_key [ "<STR_LIT:name>" ] , <EOL> "<STR_LIT>" : project_attribute_key [ "<STR_LIT>" ] , <EOL> "<STR_LIT>" : project_attribute_key [ "<STR_LIT>" ] , <EOL> } <EOL> else : <EOL> project_attribute_key_data = _infer_project_attribute_key_data ( <EOL> column_name , project_df [ column_name ] ) <EOL> project_attribute_keys_data . append ( project_attribute_key_data ) <EOL> return project_attribute_keys_data <EOL> def _infer_project_attribute_key_data ( column_name , column ) : <EOL> project_attribute_key_data = { <EOL> "<STR_LIT:name>" : column_name , <EOL> "<STR_LIT>" : _infer_display_name ( column_name ) , <EOL> "<STR_LIT>" : _infer_data_type ( column ) , <EOL> } <EOL> return project_attribute_key_data <EOL> def _infer_data_type ( column ) : <EOL> if column . shape [ <NUM_LIT:0> ] == <NUM_LIT:0> : <EOL> return None <EOL> if column . dtype == "<STR_LIT>" : <EOL> return "<STR_LIT>" <EOL> elif column . dtype == "<STR_LIT>" : <EOL> return "<STR_LIT>" <EOL> elif column . dtype == "<STR_LIT:bool>" : <EOL> return "<STR_LIT>" <EOL> elif column . dtype == "<STR_LIT:object>" : <EOL> try : <EOL> pd . to_datetime ( column [ : <NUM_LIT:10> ] ) <EOL> except ValueError : <EOL> return "<STR_LIT>" <EOL> try : <EOL> datetime . strptime ( column [ <NUM_LIT:0> ] , "<STR_LIT>" ) <EOL> return "<STR_LIT>" <EOL> except ValueError : <EOL> return "<STR_LIT>" <EOL> else : <EOL> return None <EOL> def _infer_display_name ( column_name ) : <EOL> s1 = re . sub ( '<STR_LIT>' , r'<STR_LIT>' , column_name ) <EOL> underscored = re . sub ( '<STR_LIT>' , r'<STR_LIT>' , s1 ) . lower ( ) <EOL> display_name = "<STR_LIT:U+0020>" . join ( [ str . capitalize ( w ) for w in underscored . split ( "<STR_LIT:_>" ) ] ) <EOL> return display_name <EOL> def _get_project_data ( project_df , project_attribute_keys_data ) : <EOL> for i , row in project_df . iterrows ( ) : <EOL> project_data = { <EOL> "<STR_LIT>" : row . project_id , <EOL> "<STR_LIT>" : row . zipcode , <EOL> "<STR_LIT>" : row . weather_station , <EOL> "<STR_LIT>" : row . latitude , <EOL> "<STR_LIT>" : row . longitude , <EOL> "<STR_LIT>" : None , <EOL> "<STR_LIT>" : pytz . UTC . localize ( row . baseline_period_end ) . strftime ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" : pytz . UTC . localize ( row . reporting_period_start ) . strftime ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" : None , <EOL> } <EOL> project_attributes_data = [ ] <EOL> for project_attribute_key_data in project_attribute_keys_data : <EOL> project_attribute_data = _get_project_attribute_data ( row , project_attribute_key_data ) <EOL> project_attributes_data . append ( project_attribute_data ) <EOL> yield project_data , project_attributes_data <EOL> def _get_project_attribute_data ( row , project_attribute_key_data ) : <EOL> name = project_attribute_key_data [ "<STR_LIT:name>" ] <EOL> data_type = project_attribute_key_data [ "<STR_LIT>" ] <EOL> project_attribute_data = { <EOL> "<STR_LIT:key>" : project_attribute_key_data [ "<STR_LIT:id>" ] , <EOL> } <EOL> if data_type == "<STR_LIT>" : <EOL> project_attribute_data [ "<STR_LIT>" ] = ( row [ name ] == "<STR_LIT:True>" ) <EOL> elif data_type == "<STR_LIT>" : <EOL> project_attribute_data [ "<STR_LIT>" ] = row [ name ] <EOL> elif data_type == "<STR_LIT>" : <EOL> project_attribute_data [ "<STR_LIT>" ] = row [ name ] <EOL> elif data_type == "<STR_LIT>" : <EOL> dt = datetime . strptime ( row [ name ] , "<STR_LIT>" ) <EOL> project_attribute_data [ "<STR_LIT>" ] = row [ name ] <EOL> elif data_type == "<STR_LIT>" : <EOL> project_attribute_data [ "<STR_LIT>" ] = float ( row [ name ] ) <EOL> elif data_type == "<STR_LIT>" : <EOL> project_attribute_data [ "<STR_LIT>" ] = int ( row [ name ] ) <EOL> else : <EOL> raise NotImplementedError <EOL> return project_attribute_data <EOL> def _get_consumption_data ( consumption_df ) : <EOL> for project_id , project_consumption in consumption_df . groupby ( "<STR_LIT>" ) : <EOL> for fuel_type , fuel_type_consumption in project_consumption . groupby ( "<STR_LIT>" ) : <EOL> unique_unit_names = fuel_type_consumption . unit_name . unique ( ) <EOL> assert unique_unit_names . shape [ <NUM_LIT:0> ] == <NUM_LIT:1> <EOL> consumption_metadata_data = { <EOL> "<STR_LIT>" : project_id , <EOL> "<STR_LIT>" : constants . FUEL_TYPES [ fuel_type ] , <EOL> "<STR_LIT>" : constants . ENERGY_UNIT [ unique_unit_names [ <NUM_LIT:0> ] ] , <EOL> } <EOL> consumption_records_data = _get_consumption_records_data ( <EOL> fuel_type_consumption ) <EOL> yield consumption_metadata_data , consumption_records_data <EOL> def _get_consumption_records_data ( consumption_df ) : <EOL> raw_consumption_records_data = _get_raw_consumption_records_data ( <EOL> consumption_df ) <EOL> consumption_records_data = _process_raw_consumption_records_data ( <EOL> raw_consumption_records_data ) <EOL> return consumption_records_data <EOL> def _get_raw_consumption_records_data ( consumption_df ) : <EOL> raw_consumption_records_data = [ ] <EOL> for i , row in consumption_df . iterrows ( ) : <EOL> consumption_record_data = { <EOL> "<STR_LIT:start>" : row . start , <EOL> "<STR_LIT:end>" : row . end , <EOL> "<STR_LIT:value>" : row . value , <EOL> "<STR_LIT>" : row . estimated , <EOL> } <EOL> raw_consumption_records_data . append ( consumption_record_data ) <EOL> return raw_consumption_records_data <EOL> def _process_raw_consumption_records_data ( records ) : <EOL> fuel_type , unit_name = "<STR_LIT>" , "<STR_LIT>" <EOL> consumption_data = ConsumptionData ( records , fuel_type , unit_name , <EOL> record_type = "<STR_LIT>" ) <EOL> consumption_records_data = [ ] <EOL> for ( d1 , value ) , ( d2 , estimated ) in zip ( consumption_data . data . iteritems ( ) , consumption_data . estimated . iteritems ( ) ) : <EOL> assert d1 == d2 <EOL> record = { <EOL> "<STR_LIT:start>" : pytz . UTC . localize ( d1 . to_datetime ( ) ) . strftime ( "<STR_LIT>" ) , <EOL> "<STR_LIT:value>" : value , <EOL> "<STR_LIT>" : bool ( estimated ) , <EOL> } <EOL> consumption_records_data . append ( record ) <EOL> return consumption_records_data </s>
