<s> """<STR_LIT>""" <EOL> import logging <EOL> import pprint <EOL> import json <EOL> import os <EOL> import requests <EOL> import csv <EOL> import datetime as dt <EOL> import time <EOL> from calendar import timegm <EOL> def upload_file ( upload_header , upload_filepath , main_url , upload_dataset_id , upload_datatype ) : <EOL> """<STR_LIT>""" <EOL> def _upload_file_to_aws ( aws_upload_details ) : <EOL> """<STR_LIT>""" <EOL> sig_uri = aws_upload_details [ '<STR_LIT>' ] <EOL> now = dt . datetime . utcnow ( ) <EOL> expires = now + dt . timedelta ( hours = <NUM_LIT:1> ) <EOL> now_ts = timegm ( now . timetuple ( ) ) <EOL> key = '<STR_LIT>' % ( filename , now_ts ) <EOL> payload = { } <EOL> payload [ '<STR_LIT>' ] = expires . isoformat ( ) + '<STR_LIT>' <EOL> payload [ '<STR_LIT>' ] = [ <EOL> { '<STR_LIT>' : aws_upload_details [ '<STR_LIT>' ] } , <EOL> { '<STR_LIT:Content-Type>' : '<STR_LIT>' } , <EOL> { '<STR_LIT>' : '<STR_LIT>' } , <EOL> { '<STR_LIT>' : '<STR_LIT>' } , <EOL> { '<STR_LIT:key>' : key } <EOL> ] <EOL> sig_result = requests . post ( main_url + sig_uri , <EOL> headers = upload_header , <EOL> data = json . dumps ( payload ) ) <EOL> if sig_result . status_code != <NUM_LIT:200> : <EOL> msg = "<STR_LIT>" <EOL> raise RuntimeError ( msg ) <EOL> else : <EOL> sig_result = sig_result . json ( ) <EOL> upload_url = "<STR_LIT>" % ( aws_upload_details [ '<STR_LIT>' ] ) <EOL> s3_payload = [ <EOL> ( '<STR_LIT:key>' , key ) , <EOL> ( '<STR_LIT>' , aws_upload_details [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT:Content-Type>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , sig_result [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , sig_result [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT:file>' , ( filename , open ( upload_filepath , '<STR_LIT:rb>' ) ) ) <EOL> ] <EOL> result = requests . post ( upload_url , <EOL> files = s3_payload ) <EOL> if result . status_code != <NUM_LIT:200> : <EOL> msg = "<STR_LIT>" % result . reason <EOL> raise RuntimeError ( msg ) <EOL> completion_uri = aws_upload_details [ '<STR_LIT>' ] <EOL> completion_payload = { <EOL> '<STR_LIT>' : upload_dataset_id , <EOL> '<STR_LIT:key>' : key , <EOL> '<STR_LIT>' : upload_datatype <EOL> } <EOL> return requests . get ( main_url + completion_uri , <EOL> headers = upload_header , <EOL> params = completion_payload ) <EOL> def _upload_file_to_file_system ( upload_details ) : <EOL> """<STR_LIT>""" <EOL> upload_url = "<STR_LIT>" % ( main_url , upload_details [ '<STR_LIT>' ] ) <EOL> fsysparams = { '<STR_LIT>' : upload_filepath , <EOL> '<STR_LIT>' : upload_dataset_id , <EOL> '<STR_LIT>' : upload_datatype } <EOL> return requests . post ( upload_url , <EOL> params = fsysparams , <EOL> files = { '<STR_LIT:filename>' : open ( upload_filepath , '<STR_LIT:rb>' ) } , <EOL> headers = upload_header ) <EOL> upload_details = requests . get ( main_url + '<STR_LIT>' , headers = upload_header ) <EOL> upload_details = upload_details . json ( ) <EOL> filename = os . path . basename ( upload_filepath ) <EOL> if upload_details [ '<STR_LIT>' ] == '<STR_LIT>' : <EOL> return _upload_file_to_aws ( upload_details ) <EOL> elif upload_details [ '<STR_LIT>' ] == '<STR_LIT>' : <EOL> return _upload_file_to_file_system ( upload_details ) <EOL> else : <EOL> raise RuntimeError ( "<STR_LIT>" % <EOL> upload_details [ '<STR_LIT>' ] ) <EOL> def check_status ( resultOut , partmsg , log , PIIDflag = None ) : <EOL> """<STR_LIT>""" <EOL> if resultOut . status_code in [ <NUM_LIT:200> , <NUM_LIT> , <NUM_LIT> ] : <EOL> if PIIDflag == '<STR_LIT>' : <EOL> msg = pprint . pformat ( resultOut . json ( ) , indent = <NUM_LIT:2> , width = <NUM_LIT> ) <EOL> else : <EOL> try : <EOL> if '<STR_LIT:status>' in resultOut . json ( ) . keys ( ) and resultOut . json ( ) [ '<STR_LIT:status>' ] == '<STR_LIT:error>' : <EOL> msg = resultOut . json ( ) [ '<STR_LIT:message>' ] <EOL> log . error ( partmsg + '<STR_LIT>' ) <EOL> log . debug ( msg ) <EOL> raise RuntimeError <EOL> elif '<STR_LIT:success>' in resultOut . json ( ) . keys ( ) and not resultOut . json ( ) [ '<STR_LIT:success>' ] : <EOL> msg = resultOut . json ( ) <EOL> log . error ( partmsg + '<STR_LIT>' ) <EOL> log . debug ( msg ) <EOL> raise RuntimeError <EOL> else : <EOL> if PIIDflag == '<STR_LIT>' : <EOL> msg = '<STR_LIT>' + str ( len ( resultOut . json ( ) [ '<STR_LIT>' ] [ <NUM_LIT:0> ] ) ) <EOL> elif PIIDflag == '<STR_LIT>' : <EOL> msg = '<STR_LIT>' + str ( len ( resultOut . json ( ) [ '<STR_LIT>' ] [ <NUM_LIT:0> ] ) ) <EOL> elif PIIDflag == '<STR_LIT>' : <EOL> msg = pprint . pformat ( resultOut . json ( ) [ '<STR_LIT>' ] , indent = <NUM_LIT:2> , width = <NUM_LIT> ) <EOL> elif PIIDflag == '<STR_LIT>' : <EOL> msg = "<STR_LIT>" + str ( resultOut . json ( ) [ '<STR_LIT>' ] ) + "<STR_LIT>" + str ( resultOut . json ( ) [ '<STR_LIT>' ] ) + "<STR_LIT>" + str ( resultOut . json ( ) [ '<STR_LIT>' ] ) <EOL> else : <EOL> msg = pprint . pformat ( resultOut . json ( ) , indent = <NUM_LIT:2> , width = <NUM_LIT> ) <EOL> except : <EOL> log . error ( partmsg , '<STR_LIT>' ) <EOL> log . debug ( '<STR_LIT>' ) <EOL> raise RuntimeError <EOL> log . info ( partmsg + '<STR_LIT>' ) <EOL> log . debug ( msg ) <EOL> else : <EOL> msg = resultOut . reason <EOL> log . error ( partmsg + '<STR_LIT>' ) <EOL> log . debug ( msg ) <EOL> raise RuntimeError <EOL> return <EOL> def check_progress ( mainURL , Header , progress_key ) : <EOL> """<STR_LIT>""" <EOL> time . sleep ( <NUM_LIT:5> ) <EOL> progressResult = requests . get ( mainURL + '<STR_LIT>' , <EOL> headers = Header , <EOL> data = json . dumps ( { '<STR_LIT>' : progress_key } ) ) <EOL> if progressResult . json ( ) [ '<STR_LIT>' ] == <NUM_LIT:100> : <EOL> return ( progressResult ) <EOL> else : <EOL> progressResult = check_progress ( mainURL , Header , progress_key ) <EOL> def read_map_file ( mapfilePath ) : <EOL> """<STR_LIT>""" <EOL> assert ( os . path . isfile ( mapfilePath ) ) , "<STR_LIT>" + mapfilePath <EOL> mapReader = csv . reader ( open ( mapfilePath , '<STR_LIT:r>' ) ) <EOL> mapReader . next ( ) <EOL> maplist = list ( ) <EOL> for rowitem in mapReader : <EOL> maplist . append ( rowitem ) <EOL> return maplist <EOL> def setup_logger ( filename ) : <EOL> """<STR_LIT>""" <EOL> logging . getLogger ( "<STR_LIT>" ) . setLevel ( logging . WARNING ) <EOL> logger = logging . getLogger ( ) <EOL> logger . setLevel ( logging . DEBUG ) <EOL> formatter = logging . Formatter ( '<STR_LIT>' ) <EOL> formatter_console = logging . Formatter ( '<STR_LIT>' ) <EOL> fh = logging . FileHandler ( filename , mode = '<STR_LIT:a>' ) <EOL> fh . setLevel ( logging . DEBUG ) <EOL> fh . setFormatter ( formatter ) <EOL> logger . addHandler ( fh ) <EOL> ch = logging . StreamHandler ( ) <EOL> ch . setLevel ( logging . INFO ) <EOL> ch . setFormatter ( formatter_console ) <EOL> logger . addHandler ( ch ) <EOL> return logger </s>
