<s> import csv <EOL> import sys <EOL> from apiclient . http import MediaFileUpload <EOL> import apiclient . errors <EOL> import urllib <EOL> import requests <EOL> import json <EOL> import pprint <EOL> import re <EOL> import logging <EOL> logger = logging . getLogger ( "<STR_LIT>" ) <EOL> import random <EOL> import os <EOL> import json <EOL> from gdcmdtools . base import GDBase <EOL> from gdcmdtools . perm import GDPerm <EOL> from gdcmdtools . auth import GDAuth <EOL> DICT_OF_CONVERTIBLE_FILE_TYPE = { <EOL> '<STR_LIT>' : [ <EOL> "<STR_LIT>" , <EOL> [ ] ] , <EOL> '<STR_LIT>' : [ <EOL> "<STR_LIT>" , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ] , <EOL> '<STR_LIT>' : [ <EOL> "<STR_LIT>" , <EOL> [ '<STR_LIT>' ] ] , <EOL> '<STR_LIT>' : [ <EOL> "<STR_LIT>" , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ] , <EOL> '<STR_LIT>' : [ <EOL> "<STR_LIT>" , <EOL> [ '<STR_LIT>' ] ] , <EOL> '<STR_LIT>' : [ <EOL> "<STR_LIT>" , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ] , <EOL> '<STR_LIT>' : [ <EOL> "<STR_LIT>" , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:html>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ] , <EOL> '<STR_LIT>' : [ <EOL> "<STR_LIT>" , <EOL> [ '<STR_LIT>' ] ] , <EOL> } <EOL> class GDPut : <EOL> def __init__ ( self , ** kwargs ) : <EOL> logger . debug ( kwargs ) <EOL> for key , value in kwargs . items ( ) : <EOL> setattr ( self , key , value ) <EOL> self . ft_headers = None <EOL> self . csv_latlng_suffix = "<STR_LIT>" % random . getrandbits ( <NUM_LIT:16> ) <EOL> auth = GDAuth ( ) <EOL> creds = auth . get_credentials ( ) <EOL> self . auth_user = creds . id_token . get ( "<STR_LIT:email>" , None ) <EOL> if creds is None : <EOL> raise Exception ( "<STR_LIT>" ) <EOL> self . http = auth . get_authorized_http ( ) <EOL> base = GDBase ( ) <EOL> self . file_id = None <EOL> if self . replace_id : <EOL> self . file_id = base . get_id_from_url ( self . replace_id ) <EOL> self . service = base . get_drive_service ( self . http ) <EOL> self . root = base . get_root ( ) <EOL> if self . target_type == "<STR_LIT>" : <EOL> self . ft_service = base . get_ft_service ( self . http ) <EOL> self . local_size = os . path . getsize ( self . source_file ) <EOL> def if_folder_writable ( self ) : <EOL> try : <EOL> permissions = self . service . permissions ( ) . list ( fileId = self . folder_id ) . execute ( ) <EOL> valid_roles = [ "<STR_LIT>" , "<STR_LIT>" ] <EOL> logger . debug ( pprint . pformat ( permissions ) ) <EOL> for p in permissions [ "<STR_LIT>" ] : <EOL> email = p . get ( "<STR_LIT>" , None ) . lower ( ) <EOL> role = p . get ( "<STR_LIT>" , None ) . lower ( ) <EOL> logger . debug ( "<STR_LIT>" % ( email , role ) ) <EOL> if ( email == self . auth_user ) and ( role in valid_roles ) : <EOL> return True <EOL> except : <EOL> return False <EOL> return False <EOL> def run ( self ) : <EOL> if self . folder_id : <EOL> if self . if_folder_writable ( ) == False : <EOL> raise Exception ( <EOL> "<STR_LIT>" % <EOL> self . folder_id ) <EOL> try : <EOL> result = getattr ( self , self . target_type + "<STR_LIT>" ) ( ) <EOL> except AttributeError as e : <EOL> logger . error ( e ) <EOL> raise <EOL> except Exception as e : <EOL> logger . error ( e ) <EOL> raise <EOL> return result <EOL> def get_current_user ( self ) : <EOL> pass <EOL> def raw_put ( self ) : <EOL> return self . generic_put ( False ) <EOL> def check_gas ( self ) : <EOL> with open ( self . source_file , "<STR_LIT:rb>" ) as f : <EOL> jsons = json . loads ( f . read ( ) ) <EOL> if_ok = False <EOL> if not isinstance ( jsons , dict ) : <EOL> return False <EOL> if not self . gas_new : <EOL> assert ( jsons . get ( "<STR_LIT:id>" ) ) <EOL> self . file_id = jsons . get ( "<STR_LIT:id>" ) <EOL> if ( len ( jsons [ "<STR_LIT>" ] ) > <NUM_LIT:0> ) : <EOL> for j in jsons [ "<STR_LIT>" ] : <EOL> if j [ "<STR_LIT:type>" ] and j [ "<STR_LIT:name>" ] : <EOL> if_ok = True <EOL> else : <EOL> return False <EOL> return if_ok <EOL> def gas_pack ( self ) : <EOL> map_type_ext = { "<STR_LIT>" : "<STR_LIT>" , "<STR_LIT:html>" : "<STR_LIT:html>" } <EOL> json_packed = { } <EOL> try : <EOL> with open ( self . source_file , "<STR_LIT:rb>" ) as fr1 : <EOL> jsons = json . loads ( fr1 . read ( ) ) <EOL> path = os . path . split ( self . source_file ) [ <NUM_LIT:0> ] <EOL> for j in jsons [ "<STR_LIT>" ] : <EOL> file_name = os . path . join ( path , "<STR_LIT>" % ( <EOL> j [ "<STR_LIT:name>" ] , map_type_ext [ j [ "<STR_LIT:type>" ] ] ) ) <EOL> with open ( file_name , "<STR_LIT:rb>" ) as fr2 : <EOL> file_content = fr2 . read ( ) <EOL> j [ "<STR_LIT:source>" ] = file_content <EOL> new_json = "<STR_LIT>" % self . source_file <EOL> with open ( new_json , "<STR_LIT>" ) as fw : <EOL> fw . write ( json . dumps ( jsons , indent = <NUM_LIT:4> ) ) <EOL> except : <EOL> return False <EOL> else : <EOL> return True <EOL> def gas_put ( self ) : <EOL> if not self . check_gas ( ) : <EOL> raise Exception ( <EOL> "<STR_LIT>" ) <EOL> if not self . gas_pack ( ) : <EOL> raise Exception ( "<STR_LIT>" ) <EOL> return self . generic_put ( True , file_name = "<STR_LIT>" % self . source_file ) <EOL> def check_csv ( self ) : <EOL> self . csv_delimiter = '<STR_LIT:U+002C>' <EOL> with open ( self . source_file , '<STR_LIT:rb>' ) as csv_file : <EOL> try : <EOL> dialect = csv . Sniffer ( ) . sniff ( csv_file . readline ( ) ) <EOL> if dialect . delimiter == self . csv_delimiter : <EOL> return True <EOL> except : <EOL> logger . error ( "<STR_LIT>" ) <EOL> return False <EOL> def csv_save_latlng ( self ) : <EOL> rows = [ ] <EOL> with open ( self . source_file , '<STR_LIT:rb>' ) as csv_file : <EOL> csv_reader = csv . reader ( csv_file ) <EOL> self . ft_headers = csv_reader . next ( ) <EOL> if self . ft_location_column and self . ft_latlng_column : <EOL> self . ft_headers . append ( self . ft_latlng_column ) <EOL> rows . append ( self . ft_headers ) <EOL> index_latlng = self . ft_headers . index ( self . ft_latlng_column ) <EOL> index_location = self . ft_headers . index ( self . ft_location_column ) <EOL> for row in csv_reader : <EOL> latlng = self . ft_geocoding ( row [ index_location ] ) <EOL> row . insert ( index_latlng , latlng ) <EOL> rows . append ( row ) <EOL> csv_file_dir = os . path . dirname ( self . source_file ) <EOL> csv_file_basename = os . path . basename ( self . source_file ) <EOL> csv_file_noextension = os . path . splitext ( csv_file_basename ) [ <NUM_LIT:0> ] <EOL> latlng_file = os . path . join ( <EOL> csv_file_dir , <EOL> csv_file_noextension + <EOL> self . csv_latlng_suffix ) <EOL> with open ( latlng_file , '<STR_LIT>' ) as csv_file : <EOL> csv_writer = csv . writer ( csv_file , lineterminator = '<STR_LIT:\n>' ) <EOL> csv_writer . writerows ( rows ) <EOL> return latlng_file <EOL> def ss_put ( self ) : <EOL> if not self . check_csv ( ) : <EOL> raise Exception ( <EOL> "<STR_LIT>" % <EOL> self . csv_delimiter ) <EOL> return self . generic_put ( True ) <EOL> def user_define_column ( self , cols , csv_column_define ) : <EOL> return_cols = [ ] <EOL> for ( col , col_type ) in zip ( cols , self . csv_column_define ) : <EOL> d = { "<STR_LIT:type>" : col_type , "<STR_LIT:name>" : col } <EOL> return_cols . append ( d ) <EOL> return return_cols <EOL> def create_ft ( self , target_file ) : <EOL> table = { <EOL> "<STR_LIT:name>" : self . title , <EOL> "<STR_LIT:description>" : self . description , <EOL> "<STR_LIT>" : True , <EOL> "<STR_LIT>" : [ ] <EOL> } <EOL> with open ( target_file , '<STR_LIT:rb>' ) as csv_file : <EOL> csv_reader = csv . reader ( csv_file ) <EOL> cols = csv_reader . next ( ) <EOL> self . ft_headers = cols <EOL> if self . ft_location_column and self . ft_latlng_column : <EOL> if self . ft_location_column not in cols : <EOL> raise Exception ( <EOL> "<STR_LIT>" % <EOL> self . ft_location_column ) <EOL> if self . csv_column_define is None : <EOL> for c in cols : <EOL> if c == self . ft_latlng_column : <EOL> d = { "<STR_LIT:type>" : "<STR_LIT>" } <EOL> else : <EOL> d = { "<STR_LIT:type>" : "<STR_LIT>" } <EOL> d [ "<STR_LIT:name>" ] = c <EOL> table [ "<STR_LIT>" ] . append ( d ) <EOL> else : <EOL> table [ "<STR_LIT>" ] = self . user_define_column ( <EOL> cols , self . csv_column_define ) <EOL> elif self . ft_location_column and not self . ft_latlng_column : <EOL> if self . ft_location_column not in cols : <EOL> raise Exception ( <EOL> "<STR_LIT>" % <EOL> self . ft_location_column ) <EOL> if self . csv_column_define is None : <EOL> for c in cols : <EOL> if c == self . ft_location_column : <EOL> d = { "<STR_LIT:type>" : "<STR_LIT>" } <EOL> else : <EOL> d = { "<STR_LIT:type>" : "<STR_LIT>" } <EOL> d [ "<STR_LIT:name>" ] = c <EOL> table [ "<STR_LIT>" ] . append ( d ) <EOL> else : <EOL> table [ "<STR_LIT>" ] = self . user_define_column ( <EOL> cols , self . csv_column_define ) <EOL> else : <EOL> if self . csv_column_define is None : <EOL> for c in cols : <EOL> d = { "<STR_LIT:type>" : "<STR_LIT>" , "<STR_LIT:name>" : c } <EOL> table [ "<STR_LIT>" ] . append ( d ) <EOL> else : <EOL> table [ "<STR_LIT>" ] = self . user_define_column ( <EOL> cols , self . csv_column_define ) <EOL> return table <EOL> def ft_put ( self ) : <EOL> if not self . check_csv ( ) : <EOL> raise Exception ( <EOL> "<STR_LIT>" % <EOL> self . csv_delimiter ) <EOL> if self . ft_location_column and self . ft_latlng_column : <EOL> target_file = self . csv_save_latlng ( ) <EOL> table = self . create_ft ( target_file ) <EOL> else : <EOL> table = self . create_ft ( self . source_file ) <EOL> service_response = self . ft_service . table ( ) . insert ( body = table ) . execute ( ) <EOL> table_id = service_response [ "<STR_LIT>" ] <EOL> if self . folder_id is not None : <EOL> new_parent = { '<STR_LIT:id>' : self . folder_id } <EOL> try : <EOL> self . service . parents ( ) . insert ( fileId = table_id , body = new_parent ) . execute ( ) <EOL> except apiclient . errors . HttpError as error : <EOL> raise Exception ( '<STR_LIT>' % error ) <EOL> try : <EOL> self . service . parents ( ) . delete ( fileId = table_id , parentId = self . root ) . execute ( ) <EOL> except apiclient . errors . HttpError as error : <EOL> raise Exception ( '<STR_LIT>' % error ) <EOL> if self . ft_location_column and self . ft_latlng_column : <EOL> url = self . ft_put_body ( table_id , target_file ) <EOL> else : <EOL> url = self . ft_put_body ( table_id , self . source_file ) <EOL> if self . permission is not None : <EOL> pass_action = { <EOL> "<STR_LIT:name>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : self . permission } <EOL> perm = GDPerm ( table_id , pass_action ) <EOL> result = perm . run ( ) <EOL> logger . debug ( result ) <EOL> ft_url = "<STR_LIT>" % table_id <EOL> return service_response <EOL> def ft_put_body ( self , table_id , target_file ) : <EOL> params = urllib . urlencode ( { '<STR_LIT>' : "<STR_LIT:false>" } ) <EOL> URI = "<STR_LIT>" % ( <EOL> table_id , params ) <EOL> METHOD = "<STR_LIT:POST>" <EOL> with open ( target_file ) as ft_file : <EOL> rows = ft_file . read ( ) <EOL> i_newline = rows . index ( '<STR_LIT:\n>' ) + <NUM_LIT:1> <EOL> rows = rows [ i_newline : ] <EOL> utf8_body = rows . decode ( '<STR_LIT:utf-8>' ) . encode ( '<STR_LIT:utf-8>' ) <EOL> try : <EOL> response , content = self . http . request ( <EOL> URI . encode ( '<STR_LIT:utf-8>' ) , METHOD , body = utf8_body ) <EOL> except : <EOL> raise Exception ( '<STR_LIT>' <EOL> % ( URI . encode ( '<STR_LIT:utf-8>' ) , METHOD , utf8_body ) ) <EOL> content = json . loads ( content ) <EOL> @ staticmethod <EOL> def ft_geocoding ( address ) : <EOL> GEOCODING_URL = "<STR_LIT>" <EOL> params = { '<STR_LIT:address>' : address , '<STR_LIT>' : '<STR_LIT:false>' } <EOL> response = requests . get ( GEOCODING_URL , params = params ) <EOL> response_json = ( response . json ( ) ) <EOL> lat = response_json [ "<STR_LIT>" ] [ <NUM_LIT:0> ] [ "<STR_LIT>" ] [ "<STR_LIT:location>" ] [ "<STR_LIT>" ] <EOL> lng = response_json [ "<STR_LIT>" ] [ <NUM_LIT:0> ] [ "<STR_LIT>" ] [ "<STR_LIT:location>" ] [ "<STR_LIT>" ] <EOL> latlng = str ( lat ) + "<STR_LIT:U+002C>" + str ( lng ) <EOL> return latlng <EOL> @ staticmethod <EOL> def sizeof_fmt ( num , suffix = '<STR_LIT:B>' ) : <EOL> for unit in [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:M>' , '<STR_LIT>' , '<STR_LIT:T>' , '<STR_LIT:P>' , '<STR_LIT:E>' , '<STR_LIT>' ] : <EOL> if abs ( num ) < <NUM_LIT> : <EOL> return "<STR_LIT>" % ( num , unit , suffix ) <EOL> num /= <NUM_LIT> <EOL> return "<STR_LIT>" % ( num , '<STR_LIT:Y>' , suffix ) <EOL> def generic_put ( self , if_convert , file_name = None ) : <EOL> if ( file_name ) : <EOL> self . source_file = file_name <EOL> if self . local_size < <NUM_LIT> * <NUM_LIT> : <EOL> if_resumable = False <EOL> else : <EOL> if_resumable = True <EOL> media_body = MediaFileUpload ( <EOL> self . source_file , <EOL> mimetype = self . mime_type , <EOL> resumable = if_resumable ) <EOL> if self . folder_id is None : <EOL> parents = [ ] <EOL> else : <EOL> parents = [ { <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT:id>" : self . folder_id } ] <EOL> body = { <EOL> '<STR_LIT:title>' : self . title , <EOL> '<STR_LIT:description>' : self . description , <EOL> '<STR_LIT>' : self . mime_type , <EOL> '<STR_LIT>' : parents } <EOL> if ( self . file_id ) : <EOL> if ( self . gas_new ) : <EOL> request = self . service . files ( ) . insert ( <EOL> body = body , media_body = media_body , convert = if_convert ) <EOL> else : <EOL> request = self . service . files ( ) . update ( body = body , media_body = media_body , <EOL> convert = if_convert , fileId = self . file_id ) <EOL> else : <EOL> request = self . service . files ( ) . insert ( <EOL> body = body , media_body = media_body , convert = if_convert ) <EOL> service_response = None <EOL> print "<STR_LIT>" % self . source_file <EOL> print "<STR_LIT>" % self . sizeof_fmt ( self . local_size ) <EOL> if ( if_resumable ) : <EOL> while service_response is None : <EOL> status , service_response = request . next_chunk ( num_retries = <NUM_LIT:10> ) <EOL> if status : <EOL> sys . stdout . write ( <EOL> "<STR_LIT>" % <EOL> ( status . progress ( ) * <NUM_LIT:100> ) ) <EOL> sys . stdout . flush ( ) <EOL> else : <EOL> sys . stdout . write ( "<STR_LIT>" % ( "<STR_LIT:U+0020>" * <NUM_LIT:10> ) ) <EOL> sys . stdout . flush ( ) <EOL> else : <EOL> service_response = request . execute ( ) <EOL> if self . permission is not None : <EOL> pass_action = { <EOL> "<STR_LIT:name>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : self . permission } <EOL> perm = GDPerm ( service_response [ '<STR_LIT:id>' ] , pass_action ) <EOL> result = perm . run ( ) <EOL> logger . debug ( result ) <EOL> return service_response <EOL> def pt_put ( self ) : <EOL> return self . generic_put ( True ) <EOL> def dr_put ( self ) : <EOL> return self . generic_put ( True ) <EOL> def ocr_put ( self ) : <EOL> return self . generic_put ( True ) <EOL> def doc_put ( self ) : <EOL> return self . generic_put ( True ) </s>
