<s> from googleapiclient import discovery <EOL> from googleapiclient import errors <EOL> from grow . common import oauth <EOL> from grow . common import utils <EOL> from . import base <EOL> from protorpc import messages <EOL> import bs4 <EOL> import cStringIO <EOL> import csv <EOL> import html2text <EOL> import httplib2 <EOL> import json <EOL> import logging <EOL> import os <EOL> import yaml <EOL> OAUTH_SCOPE = '<STR_LIT>' <EOL> discovery . logger . setLevel ( logging . WARNING ) <EOL> class BaseGooglePreprocessor ( base . BasePreprocessor ) : <EOL> def _create_service ( self ) : <EOL> credentials = oauth . get_credentials ( <EOL> scope = OAUTH_SCOPE , storage_key = '<STR_LIT>' ) <EOL> http = httplib2 . Http ( ca_certs = utils . get_cacerts_path ( ) ) <EOL> http = credentials . authorize ( http ) <EOL> return discovery . build ( '<STR_LIT>' , '<STR_LIT>' , http = http ) <EOL> def run ( self , build = True ) : <EOL> try : <EOL> self . download ( self . config ) <EOL> except errors . HttpError as e : <EOL> self . logger . error ( str ( e ) ) <EOL> class GoogleDocsPreprocessor ( BaseGooglePreprocessor ) : <EOL> KIND = '<STR_LIT>' <EOL> class Config ( messages . Message ) : <EOL> path = messages . StringField ( <NUM_LIT:1> ) <EOL> id = messages . StringField ( <NUM_LIT:2> ) <EOL> convert = messages . BooleanField ( <NUM_LIT:3> ) <EOL> def download ( self , config ) : <EOL> doc_id = config . id <EOL> path = config . path <EOL> ext = os . path . splitext ( config . path ) [ <NUM_LIT:1> ] <EOL> convert_to_markdown = ext == '<STR_LIT>' and config . convert is not False <EOL> service = self . _create_service ( ) <EOL> resp = service . files ( ) . get ( fileId = doc_id ) . execute ( ) <EOL> for mimetype , url in resp [ '<STR_LIT>' ] . iteritems ( ) : <EOL> if mimetype . endswith ( '<STR_LIT:html>' ) : <EOL> resp , content = service . _http . request ( url ) <EOL> if resp . status != <NUM_LIT:200> : <EOL> self . logger . error ( '<STR_LIT>' . format ( resp . status , path ) ) <EOL> break <EOL> soup = bs4 . BeautifulSoup ( content , '<STR_LIT>' ) <EOL> content = unicode ( soup . body ) <EOL> if convert_to_markdown : <EOL> h2t = html2text . HTML2Text ( ) <EOL> content = h2t . handle ( content ) <EOL> content = content . encode ( '<STR_LIT:utf-8>' ) <EOL> self . pod . write_file ( path , content ) <EOL> self . logger . info ( '<STR_LIT>' . format ( path ) ) <EOL> class GoogleSheetsPreprocessor ( BaseGooglePreprocessor ) : <EOL> KIND = '<STR_LIT>' <EOL> class Config ( messages . Message ) : <EOL> path = messages . StringField ( <NUM_LIT:1> ) <EOL> id = messages . StringField ( <NUM_LIT:2> ) <EOL> gid = messages . IntegerField ( <NUM_LIT:3> ) <EOL> output_style = messages . StringField ( <NUM_LIT:4> , default = '<STR_LIT>' ) <EOL> def download ( self , config ) : <EOL> path = config . path <EOL> sheet_id = config . id <EOL> sheet_gid = config . gid <EOL> service = self . _create_service ( ) <EOL> resp = service . files ( ) . get ( fileId = sheet_id ) . execute ( ) <EOL> ext = os . path . splitext ( self . config . path ) [ <NUM_LIT:1> ] <EOL> convert_to = None <EOL> if ext == '<STR_LIT>' : <EOL> convert_to = ext <EOL> ext = '<STR_LIT>' <EOL> elif ext in [ '<STR_LIT>' , '<STR_LIT>' ] : <EOL> convert_to = ext <EOL> ext = '<STR_LIT>' <EOL> for mimetype , url in resp [ '<STR_LIT>' ] . iteritems ( ) : <EOL> if not mimetype . endswith ( ext [ <NUM_LIT:1> : ] ) : <EOL> continue <EOL> if self . config . gid : <EOL> url += '<STR_LIT>' . format ( self . config . gid ) <EOL> resp , content = service . _http . request ( url ) <EOL> if resp . status != <NUM_LIT:200> : <EOL> self . logger . error ( '<STR_LIT>' . format ( resp . status , path ) ) <EOL> break <EOL> if convert_to in [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] : <EOL> fp = cStringIO . StringIO ( ) <EOL> fp . write ( content ) <EOL> fp . seek ( <NUM_LIT:0> ) <EOL> reader = csv . DictReader ( fp ) <EOL> kwargs = { } <EOL> if convert_to == '<STR_LIT>' : <EOL> if self . config . output_style == '<STR_LIT>' : <EOL> kwargs [ '<STR_LIT>' ] = <NUM_LIT:2> <EOL> kwargs [ '<STR_LIT>' ] = ( '<STR_LIT:U+002C>' , '<STR_LIT>' ) <EOL> kwargs [ '<STR_LIT>' ] = True <EOL> content = json . dumps ( [ row for row in reader ] , ** kwargs ) <EOL> else : <EOL> content = yaml . safe_dump ( <EOL> list ( reader ) , <EOL> default_flow_style = False ) <EOL> self . pod . write_file ( path , content ) <EOL> self . logger . info ( '<STR_LIT>' . format ( path ) ) </s>
