<s> from oauth2client . client import OAuth2WebServerFlow , OAuth2Credentials <EOL> from oauth2client . file import Storage <EOL> from apiclient . discovery import build <EOL> from apiclient . http import MediaFileUpload <EOL> from apiclient import errors <EOL> import httplib2 <EOL> import argparse <EOL> import sys , os . path <EOL> import re <EOL> import time <EOL> import logging <EOL> from operator import itemgetter <EOL> import collections <EOL> from collections import defaultdict , OrderedDict <EOL> import webbrowser <EOL> import yaml , csv , json , pprint <EOL> import StringIO <EOL> import shelve <EOL> import pandas as pd <EOL> import datetime <EOL> LOGLEVELS = { '<STR_LIT>' : logging . DEBUG , <EOL> '<STR_LIT>' : logging . INFO , <EOL> '<STR_LIT>' : logging . WARNING , <EOL> '<STR_LIT>' : logging . ERROR , <EOL> '<STR_LIT>' : logging . CRITICAL } <EOL> logger = logging . getLogger ( __name__ ) <EOL> def default_options ( ) : <EOL> defaults = { } <EOL> defaults [ '<STR_LIT:store>' ] = os . path . expanduser ( '<STR_LIT>' ) <EOL> defaults [ '<STR_LIT>' ] = os . path . expanduser ( '<STR_LIT>' ) <EOL> defaults [ '<STR_LIT>' ] = os . path . expanduser ( '<STR_LIT>' ) <EOL> defaults [ '<STR_LIT>' ] = False <EOL> defaults [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> defaults [ '<STR_LIT>' ] = <NUM_LIT:0> <EOL> return defaults <EOL> def load_config ( opts ) : <EOL> if '<STR_LIT>' in opts : <EOL> try : <EOL> with open ( opts [ '<STR_LIT>' ] , '<STR_LIT:r>' ) as f : <EOL> config = yaml . load ( f ) <EOL> return dict ( config ) <EOL> except IOError : <EOL> logger . error ( '<STR_LIT>' , yaml_name ) <EOL> sys . exit ( ) <EOL> else : <EOL> return { } <EOL> def get_file ( title = None , fmt = '<STR_LIT>' , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> opts = default_options ( ) <EOL> opts [ '<STR_LIT:title>' ] = title <EOL> opts . update ( ( k , v ) for k , v in load_config ( opts ) . items ( ) if v is not None ) <EOL> opts . update ( ( k , v ) for k , v in kwargs . items ( ) ) <EOL> logger . info ( '<STR_LIT>' , pprint . pformat ( opts ) ) <EOL> if opts [ '<STR_LIT:title>' ] is None : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> content = get_content ( opts ) <EOL> wb = pd . ExcelFile ( StringIO . StringIO ( content ) ) <EOL> if fmt == '<STR_LIT>' : <EOL> return wb <EOL> try : <EOL> parsed_wb = OrderedDict ( [ ( sheet_name , wb . parse ( sheet_name , header = opts [ '<STR_LIT>' ] ) ) for sheet_name in wb . sheet_names ] ) <EOL> except : <EOL> print '<STR_LIT>' '<STR_LIT>' '<STR_LIT>' <EOL> logger . exception ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> raise <EOL> if fmt == '<STR_LIT:list>' : <EOL> fmt_wb = OrderedDict ( [ ( name , list ( df . itertuples ( index = False ) ) ) for name , df in parsed_wb . items ( ) ] ) <EOL> elif fmt == '<STR_LIT>' : <EOL> fmt_wb = OrderedDict ( [ ( name , [ r [ <NUM_LIT:1> ] . to_dict ( ) for r in df . iterrows ( ) ] ) for name , df in parsed_wb . items ( ) ] ) <EOL> elif fmt == '<STR_LIT>' : <EOL> fmt_wb = parsed_wb <EOL> else : <EOL> raise ValueError ( '<STR_LIT>' % fmt ) <EOL> if len ( fmt_wb ) == <NUM_LIT:1> : <EOL> return fmt_wb . values ( ) [ <NUM_LIT:0> ] <EOL> if '<STR_LIT>' in opts : <EOL> try : <EOL> return fmt_wb [ opts [ '<STR_LIT>' ] ] <EOL> except : <EOL> print '<STR_LIT>' % ( opts [ '<STR_LIT>' ] , fmt_wb . keys ( ) ) <EOL> logger . exception ( '<STR_LIT>' , opts [ '<STR_LIT>' ] , fmt_wb . keys ( ) ) <EOL> raise <EOL> else : <EOL> return fmt_wb <EOL> def write_xlsx ( data , fname , sheet_names = None ) : <EOL> logger . debug ( '<STR_LIT>' ) <EOL> if not isinstance ( data , collections . Mapping ) : <EOL> try : <EOL> logger . debug ( '<STR_LIT>' ) <EOL> dfs = { '<STR_LIT>' : pd . DataFrame ( data ) } <EOL> except : <EOL> logger . debug ( '<STR_LIT>' ) <EOL> if isinstance ( data , collections . Sequence ) : <EOL> logger . debug ( '<STR_LIT>' ) <EOL> if sheet_names is not None : <EOL> assert len ( sheet_names ) == len ( data ) , '<STR_LIT>' <EOL> else : <EOL> sheet_names = [ '<STR_LIT>' % i for i in range ( len ( data ) ) ] <EOL> dfs = dict ( zip ( sheet_names , data ) ) <EOL> else : <EOL> dfs = data <EOL> else : <EOL> dfs = data <EOL> assert isinstance ( dfs , collections . Mapping ) , '<STR_LIT>' % ( type ( dfs ) , dfs ) <EOL> writer = pd . ExcelWriter ( fname ) <EOL> for sheet_name , df in dfs . items ( ) : <EOL> try : <EOL> df = pd . DataFrame ( df ) <EOL> except : <EOL> logger . exception ( '<STR_LIT>' , df ) <EOL> df . to_excel ( writer , sheet_name , index = False ) <EOL> writer . save ( ) <EOL> def put_file ( title = None , data = None , sheet_names = None , fname = None , update = False , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> opts = default_options ( ) <EOL> opts [ '<STR_LIT:title>' ] = title <EOL> opts [ '<STR_LIT>' ] = fname <EOL> opts [ '<STR_LIT:data>' ] = data <EOL> opts [ '<STR_LIT>' ] = sheet_names <EOL> opts [ '<STR_LIT>' ] = update <EOL> opts . update ( ( k , v ) for k , v in load_config ( opts ) . items ( ) if v is not None ) <EOL> opts . update ( ( k , v ) for k , v in kwargs . items ( ) if v is not None ) <EOL> if opts [ '<STR_LIT:title>' ] is None : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> if opts [ '<STR_LIT>' ] is None : <EOL> if opts [ '<STR_LIT:data>' ] is not None : <EOL> fname = os . path . join ( os . environ [ '<STR_LIT>' ] , str ( int ( time . time ( ) ) ) ) <EOL> write_xlsx ( data , fname , sheet_names ) <EOL> else : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> mimetype = '<STR_LIT>' , <EOL> media_body = MediaFileUpload ( fname , mimetype = mimetype , resumable = True ) <EOL> body = { <EOL> '<STR_LIT:title>' : title , <EOL> '<STR_LIT:description>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : mimetype } <EOL> service = get_service ( opts ) <EOL> try : <EOL> orig_file = find_file ( service , opts ) <EOL> if orig_file is not None and opts [ '<STR_LIT>' ] : <EOL> file = service . files ( ) . update ( <EOL> fileId = orig_file [ '<STR_LIT:id>' ] , <EOL> body = body , <EOL> media_body = media_body , <EOL> newRevision = True , <EOL> convert = True ) . execute ( ) <EOL> else : <EOL> if not opts [ '<STR_LIT>' ] : <EOL> logger . warning ( '<STR_LIT>' , opts [ '<STR_LIT:title>' ] ) <EOL> file = service . files ( ) . insert ( <EOL> body = body , <EOL> media_body = media_body , <EOL> convert = True ) . execute ( ) <EOL> except errors . HttpError , error : <EOL> logger . exception ( '<STR_LIT>' , title ) <EOL> def find_file ( service , opts ) : <EOL> files = service . files ( ) <EOL> try : <EOL> res = files . list ( ) . execute ( ) <EOL> except errors . HttpError , error : <EOL> logger . error ( '<STR_LIT>' , exc_info = error ) <EOL> raise error <EOL> files = res [ '<STR_LIT>' ] <EOL> fs = [ f for f in files if f [ '<STR_LIT:title>' ] == opts [ '<STR_LIT:title>' ] ] <EOL> if not fs : <EOL> title_list = sorted ( [ f [ '<STR_LIT:title>' ] for f in files ] ) <EOL> logger . error ( '<STR_LIT>' , opts [ '<STR_LIT:title>' ] , pprint . pformat ( title_list ) ) <EOL> return None <EOL> if len ( fs ) > <NUM_LIT:1> : <EOL> dups = '<STR_LIT:\n>' . join ( [ f [ '<STR_LIT>' ] for f in fs ] ) <EOL> logger . warning ( '<STR_LIT>' , opts [ '<STR_LIT:title>' ] , dups ) <EOL> file = fs [ <NUM_LIT:0> ] <EOL> return file <EOL> def get_content ( opts ) : <EOL> cache = shelve . open ( opts [ '<STR_LIT>' ] ) <EOL> if opts [ '<STR_LIT>' ] and opts [ '<STR_LIT:title>' ] in cache : <EOL> logger . info ( '<STR_LIT>' , opts [ '<STR_LIT:title>' ] ) <EOL> content = cache [ opts [ '<STR_LIT:title>' ] ] <EOL> else : <EOL> logger . debug ( '<STR_LIT>' ) <EOL> service = get_service ( opts ) <EOL> file = find_file ( service , opts ) <EOL> if file is None : <EOL> logger . exception ( '<STR_LIT>' , opts [ '<STR_LIT:title>' ] ) <EOL> sys . exit ( ) <EOL> content = download ( service , file ) <EOL> cache [ opts [ '<STR_LIT:title>' ] ] = content <EOL> return content <EOL> def get_service ( opts ) : <EOL> flow = OAuth2WebServerFlow ( client_id = opts [ '<STR_LIT>' ] , <EOL> client_secret = opts [ '<STR_LIT>' ] , <EOL> scope = opts [ '<STR_LIT>' ] , <EOL> redirect_uri = opts [ '<STR_LIT>' ] ) <EOL> credentials = get_credentials ( flow , opts ) <EOL> http = httplib2 . Http ( ) <EOL> http = credentials . authorize ( http ) <EOL> service = build ( '<STR_LIT>' , '<STR_LIT>' , http = http ) <EOL> return service <EOL> def get_credentials ( flow , opts ) : <EOL> storage = Storage ( opts [ '<STR_LIT:store>' ] ) <EOL> credentials = storage . get ( ) <EOL> if not credentials : <EOL> auth_url = flow . step1_get_authorize_url ( ) <EOL> webbrowser . open ( auth_url ) <EOL> code = raw_input ( '<STR_LIT>' % auth_url ) <EOL> credentials = flow . step2_exchange ( code ) <EOL> storage . put ( credentials ) <EOL> if credentials . access_token_expired : <EOL> logger . info ( '<STR_LIT>' ) <EOL> refresh_http = httplib2 . Http ( ) <EOL> credentials . refresh ( refresh_http ) <EOL> return credentials <EOL> def download ( service , file ) : <EOL> logger . debug ( '<STR_LIT>' , pprint . pformat ( file . viewkeys ( ) ) ) <EOL> download_url = file . get ( '<STR_LIT>' ) [ '<STR_LIT>' ] <EOL> if download_url : <EOL> resp , content = service . _http . request ( download_url ) <EOL> if resp . status == <NUM_LIT:200> : <EOL> logger . debug ( '<STR_LIT>' , resp ) <EOL> return content <EOL> else : <EOL> logger . error ( '<STR_LIT>' % resp ) <EOL> return None <EOL> else : <EOL> logger . error ( '<STR_LIT>' ) <EOL> return None <EOL> def merge_config ( opts , yaml_name ) : <EOL> try : <EOL> with open ( yaml_name , '<STR_LIT:r>' ) as f : <EOL> config = yaml . load ( f ) <EOL> logger . debug ( '<STR_LIT>' , yaml_name ) <EOL> for k , v in config . items ( ) : <EOL> if not hasattr ( opts , k ) or getattr ( opts , k ) is None : <EOL> setattr ( opts , k , v ) <EOL> except IOError : <EOL> logger . error ( '<STR_LIT>' , yaml_name ) <EOL> sys . exit ( ) <EOL> class Join ( argparse . Action ) : <EOL> def __call__ ( self , parser , namespace , values , option_string = None ) : <EOL> if not isinstance ( values , str ) : <EOL> setattr ( namespace , self . dest , '<STR_LIT:U+0020>' . join ( values ) ) <EOL> def parse_args ( ** kwopts ) : <EOL> parser = argparse . ArgumentParser ( description = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT:title>' , <EOL> nargs = '<STR_LIT:+>' , <EOL> metavar = '<STR_LIT>' , <EOL> action = Join , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , <EOL> nargs = '<STR_LIT:+>' , <EOL> metavar = '<STR_LIT>' , <EOL> action = Join , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , <EOL> action = '<STR_LIT:store_true>' , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , <EOL> type = int , <EOL> default = <NUM_LIT:0> , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , <EOL> default = False , <EOL> action = '<STR_LIT:store_true>' , <EOL> help = '<STR_LIT>' ) <EOL> args = parser . parse_args ( ) <EOL> if args . no_header : <EOL> setattr ( args , '<STR_LIT>' , None ) <EOL> return vars ( args ) <EOL> def write_to_stdout ( content ) : <EOL> for line in content : <EOL> print '<STR_LIT:\t>' . join ( map ( str , line ) ) <EOL> def main ( ) : <EOL> """<STR_LIT>""" <EOL> logger = logging . getLogger ( __name__ ) <EOL> ch = logging . StreamHandler ( ) <EOL> formatter = logging . Formatter ( '<STR_LIT>' ) <EOL> ch . setFormatter ( formatter ) <EOL> logger . addHandler ( ch ) <EOL> logger . setLevel ( logging . DEBUG ) <EOL> content = get_file ( fmt = '<STR_LIT:list>' , ** parse_args ( ) ) <EOL> if isinstance ( content , dict ) : <EOL> content = content . values ( ) [ <NUM_LIT:0> ] <EOL> write_to_stdout ( content ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> main ( ) </s>
