<s> """<STR_LIT>""" <EOL> import requests <EOL> import simplejson as json <EOL> from datetime import datetime , timedelta <EOL> from calendar import timegm <EOL> import os <EOL> class APIClient ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , base_url , api_key = "<STR_LIT>" , email_address = "<STR_LIT>" ) : <EOL> """<STR_LIT>""" <EOL> self . base_url = base_url <EOL> self . email_address = email_address <EOL> self . no_org_uris = [ ] <EOL> self . org_id = None <EOL> if api_key and email_address : <EOL> self . authorization = "<STR_LIT>" % ( email_address , api_key ) <EOL> else : <EOL> self . authorization = None <EOL> self . schema = self . _request ( '<STR_LIT>' ) <EOL> self . schema_uri_by_name = { e [ '<STR_LIT:name>' ] : k <EOL> for k , e in self . schema . iteritems ( ) } <EOL> self . no_org_uris = [ self . schema_uri_by_name [ '<STR_LIT>' ] ] <EOL> for name in self . schema_uri_by_name : <EOL> if hasattr ( APIClient , name ) : <EOL> continue <EOL> def make_function ( name ) : <EOL> fn = lambda obj , ** args : obj . _request ( name = name , ** args ) <EOL> endpoint = self . _get_endpoint_by_name ( name ) <EOL> fn . __doc__ = endpoint [ '<STR_LIT:description>' ] <EOL> fn . __name__ = str ( endpoint [ '<STR_LIT:name>' ] ) <EOL> return fn <EOL> setattr ( APIClient , name , make_function ( name ) ) <EOL> def _get_endpoint_by_name ( self , name ) : <EOL> """<STR_LIT>""" <EOL> uri = self . schema_uri_by_name [ name ] <EOL> endpoint = self . schema [ uri ] <EOL> endpoint [ '<STR_LIT>' ] = uri <EOL> return endpoint <EOL> def get_schema ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . schema <EOL> def describe ( self , what ) : <EOL> """<STR_LIT>""" <EOL> if what in self . schema_uri_by_name : <EOL> endpoint = self . _get_endpoint_by_name ( what ) <EOL> uri = endpoint [ '<STR_LIT>' ] <EOL> elif what in self . schema : <EOL> uri = what <EOL> endpoint = self . schema [ what ] <EOL> else : <EOL> print "<STR_LIT>" % what <EOL> return <EOL> print endpoint [ '<STR_LIT:name>' ] <EOL> print uri <EOL> print endpoint [ '<STR_LIT:description>' ] <EOL> def set_organization_id ( self , org_id ) : <EOL> """<STR_LIT>""" <EOL> orgs = self . get_organizations ( ) <EOL> orgs_by_id = { org [ '<STR_LIT:id>' ] : org <EOL> for org in orgs } <EOL> if org_id not in orgs_by_id : <EOL> msg = "<STR_LIT>" % ( <EOL> self . email_address , org_id <EOL> ) <EOL> raise RuntimeError ( msg ) <EOL> self . org_id = org_id <EOL> def _request ( <EOL> self , uri = None , name = None , method = None , payload = None , headers = None , <EOL> params = None <EOL> ) : <EOL> """<STR_LIT>""" <EOL> if uri is None and name is not None : <EOL> uri = self . schema_uri_by_name [ name ] <EOL> if method is None and name . startswith ( '<STR_LIT>' ) : <EOL> method = '<STR_LIT>' <EOL> if method is None : <EOL> if payload is not None : <EOL> method = '<STR_LIT>' <EOL> else : <EOL> method = '<STR_LIT>' <EOL> method = method . lower ( ) <EOL> if headers is None : <EOL> headers = { } <EOL> if self . authorization : <EOL> headers [ '<STR_LIT>' ] = self . authorization <EOL> if params is None : <EOL> params = { } <EOL> if payload is None : <EOL> payload = { } <EOL> if self . org_id and uri not in self . no_org_uris : <EOL> if method == '<STR_LIT>' : <EOL> if '<STR_LIT>' not in params : <EOL> params [ '<STR_LIT>' ] = self . org_id <EOL> else : <EOL> if '<STR_LIT>' not in payload : <EOL> payload [ '<STR_LIT>' ] = self . org_id <EOL> if payload : <EOL> if not isinstance ( payload , basestring ) : <EOL> payload = json . dumps ( payload ) <EOL> fn = getattr ( requests , method ) <EOL> _url = self . base_url + uri <EOL> response = fn ( _url , data = payload , params = params , headers = headers ) <EOL> try : <EOL> return response . json ( ) <EOL> except json . JSONDecodeError : <EOL> return response <EOL> def get_organizations ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . _request ( name = '<STR_LIT>' ) [ '<STR_LIT>' ] <EOL> def create_dataset ( self , name ) : <EOL> """<STR_LIT>""" <EOL> payload = { '<STR_LIT:name>' : name } <EOL> return self . _request ( name = '<STR_LIT>' , <EOL> payload = payload ) <EOL> def upload_file ( <EOL> self , filepath , import_record_id , source_type <EOL> ) : <EOL> """<STR_LIT>""" <EOL> upload_details = self . _request ( name = '<STR_LIT>' ) <EOL> if upload_details [ '<STR_LIT>' ] == '<STR_LIT>' : <EOL> return self . _upload_file_to_s3 ( <EOL> filepath , upload_details , import_record_id , source_type <EOL> ) <EOL> elif upload_details [ '<STR_LIT>' ] == '<STR_LIT>' : <EOL> return self . _upload_file_to_filesystem ( <EOL> filepath , upload_details , import_record_id , source_type <EOL> ) <EOL> else : <EOL> raise RuntimeError ( "<STR_LIT>" % <EOL> upload_details [ '<STR_LIT>' ] ) <EOL> def _upload_file_to_filesystem ( <EOL> self , filepath , upload_details , import_record_id , source_type <EOL> ) : <EOL> """<STR_LIT>""" <EOL> filename = os . path . basename ( filepath ) <EOL> upload_url = "<STR_LIT>" % ( self . base_url , upload_details [ '<STR_LIT>' ] ) <EOL> params = { <EOL> '<STR_LIT>' : filename , <EOL> '<STR_LIT>' : import_record_id , <EOL> '<STR_LIT>' : source_type <EOL> } <EOL> headers = { '<STR_LIT>' : self . authorization } <EOL> return requests . post ( upload_url , <EOL> params = params , <EOL> files = { '<STR_LIT:filename>' : open ( filepath , '<STR_LIT:rb>' ) } , <EOL> headers = headers ) <EOL> def _upload_file_to_s3 ( <EOL> self , filepath , upload_details , import_record_id , source_type <EOL> ) : <EOL> """<STR_LIT>""" <EOL> filename = os . path . basename ( filepath ) <EOL> sig_uri = upload_details [ '<STR_LIT>' ] <EOL> payload = { } <EOL> now = datetime . utcnow ( ) <EOL> expires = now + timedelta ( hours = <NUM_LIT:1> ) <EOL> now_ts = timegm ( now . timetuple ( ) ) <EOL> key = '<STR_LIT>' % ( filename , now_ts ) <EOL> payload [ '<STR_LIT>' ] = expires . isoformat ( ) + '<STR_LIT>' <EOL> payload [ '<STR_LIT>' ] = [ <EOL> { '<STR_LIT>' : upload_details [ '<STR_LIT>' ] } , <EOL> { '<STR_LIT:Content-Type>' : '<STR_LIT>' } , <EOL> { '<STR_LIT>' : '<STR_LIT>' } , <EOL> { '<STR_LIT>' : '<STR_LIT>' } , <EOL> { '<STR_LIT:key>' : key } <EOL> ] <EOL> sig = self . _request ( sig_uri , payload = payload ) <EOL> upload_url = "<STR_LIT>" % ( <EOL> upload_details [ '<STR_LIT>' ] <EOL> ) <EOL> s3_payload = [ <EOL> ( '<STR_LIT:key>' , key ) , <EOL> ( '<STR_LIT>' , upload_details [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT:Content-Type>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , sig [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , sig [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT:file>' , open ( filepath , '<STR_LIT:rb>' ) ) <EOL> ] <EOL> res = requests . post ( upload_url , files = s3_payload ) <EOL> if res . status_code != <NUM_LIT:200> : <EOL> msg = "<STR_LIT>" % res . content <EOL> raise RuntimeError ( msg ) <EOL> completion_uri = upload_details [ '<STR_LIT>' ] <EOL> completion_payload = { <EOL> '<STR_LIT>' : import_record_id , <EOL> '<STR_LIT:key>' : key , <EOL> '<STR_LIT>' : source_type <EOL> } <EOL> return self . _request ( completion_uri , <EOL> params = completion_payload , <EOL> method = '<STR_LIT>' ) </s>
