<s> """<STR_LIT>""" <EOL> from __future__ import with_statement <EOL> import calendar <EOL> import contextlib <EOL> import copy <EOL> import datetime <EOL> import errno <EOL> import hashlib <EOL> import logging <EOL> import mimetypes <EOL> import optparse <EOL> import os <EOL> import random <EOL> import re <EOL> import shutil <EOL> import subprocess <EOL> import sys <EOL> import tempfile <EOL> import time <EOL> import urllib <EOL> import urllib2 <EOL> import google <EOL> from oauth2client import devshell <EOL> try : <EOL> from oauth2client . contrib import gce as oauth2client_gce <EOL> except ImportError : <EOL> from oauth2client import gce as oauth2client_gce <EOL> import yaml <EOL> from google . appengine . cron import groctimespecification <EOL> from google . appengine . api import appinfo <EOL> from google . appengine . api import appinfo_includes <EOL> from google . appengine . api import backendinfo <EOL> from google . appengine . api import client_deployinfo <EOL> from google . appengine . api import croninfo <EOL> from google . appengine . api import dispatchinfo <EOL> from google . appengine . api import dosinfo <EOL> from google . appengine . api import queueinfo <EOL> from google . appengine . api import validation <EOL> from google . appengine . api import yaml_errors <EOL> from google . appengine . api import yaml_object <EOL> from google . appengine . datastore import datastore_index <EOL> from google . appengine . tools import appengine_rpc <EOL> try : <EOL> from google . appengine . tools import appengine_rpc_httplib2 <EOL> except ImportError : <EOL> appengine_rpc_httplib2 = None <EOL> if sys . version_info [ : <NUM_LIT:2> ] >= ( <NUM_LIT:2> , <NUM_LIT:7> ) : <EOL> from google . appengine . tools import appcfg_java <EOL> else : <EOL> appcfg_java = None <EOL> from google . appengine . tools import augment_mimetypes <EOL> from google . appengine . tools import bulkloader <EOL> from google . appengine . tools import sdk_update_checker <EOL> LIST_DELIMITER = '<STR_LIT:\n>' <EOL> TUPLE_DELIMITER = '<STR_LIT:|>' <EOL> BACKENDS_ACTION = '<STR_LIT>' <EOL> BACKENDS_MESSAGE = ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> _CONVERTING_URL = ( <EOL> '<STR_LIT>' ) <EOL> MAX_LOG_LEVEL = <NUM_LIT:4> <EOL> MAX_BATCH_SIZE = <NUM_LIT> <EOL> MAX_BATCH_COUNT = <NUM_LIT:100> <EOL> MAX_BATCH_FILE_SIZE = <NUM_LIT> <EOL> BATCH_OVERHEAD = <NUM_LIT> <EOL> verbosity = <NUM_LIT:1> <EOL> PREFIXED_BY_ADMIN_CONSOLE_RE = '<STR_LIT>' <EOL> SDK_PRODUCT = '<STR_LIT>' <EOL> DAY = <NUM_LIT> * <NUM_LIT> <EOL> SUNDAY = <NUM_LIT:6> <EOL> MEGA = <NUM_LIT> * <NUM_LIT> <EOL> MILLION = <NUM_LIT:1000> * <NUM_LIT:1000> <EOL> DEFAULT_RESOURCE_LIMITS = { <EOL> '<STR_LIT>' : <NUM_LIT:32> * MILLION , <EOL> '<STR_LIT>' : <NUM_LIT:32> * MILLION , <EOL> '<STR_LIT>' : <NUM_LIT:100> , <EOL> '<STR_LIT>' : <NUM_LIT> * MEGA , <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> } <EOL> APPCFG_CLIENT_ID = '<STR_LIT>' <EOL> APPCFG_CLIENT_NOTSOSECRET = '<STR_LIT>' <EOL> APPCFG_SCOPES = ( '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' ) <EOL> STATIC_FILE_PREFIX = '<STR_LIT>' <EOL> METADATA_BASE = '<STR_LIT>' <EOL> SERVICE_ACCOUNT_BASE = ( <EOL> '<STR_LIT>' ) <EOL> APP_YAML_FILENAME = '<STR_LIT>' <EOL> GO_APP_BUILDER = os . path . join ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> if sys . platform . startswith ( '<STR_LIT>' ) : <EOL> GO_APP_BUILDER += '<STR_LIT>' <EOL> GCLOUD_ONLY_RUNTIMES = set ( [ '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> augment_mimetypes . init ( ) <EOL> class Error ( Exception ) : <EOL> pass <EOL> class CannotStartServingError ( Error ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> def PrintUpdate ( msg , error_fh = sys . stderr ) : <EOL> """<STR_LIT>""" <EOL> if verbosity > <NUM_LIT:0> : <EOL> timestamp = datetime . datetime . now ( ) <EOL> print >> error_fh , '<STR_LIT>' % ( timestamp . strftime ( '<STR_LIT>' ) , msg ) <EOL> def StatusUpdate ( msg , error_fh = sys . stderr ) : <EOL> """<STR_LIT>""" <EOL> PrintUpdate ( msg , error_fh ) <EOL> def BackendsStatusUpdate ( runtime , error_fh = sys . stderr ) : <EOL> """<STR_LIT>""" <EOL> language = runtime <EOL> if language == '<STR_LIT>' : <EOL> language = '<STR_LIT>' <EOL> elif language == '<STR_LIT>' : <EOL> language = '<STR_LIT>' <EOL> if language == '<STR_LIT>' or language == '<STR_LIT>' : <EOL> StatusUpdate ( BACKENDS_MESSAGE + ( _CONVERTING_URL % language ) , error_fh ) <EOL> def ErrorUpdate ( msg , error_fh = sys . stderr ) : <EOL> """<STR_LIT>""" <EOL> PrintUpdate ( msg , error_fh ) <EOL> def _PrintErrorAndExit ( stream , msg , exit_code = <NUM_LIT:2> ) : <EOL> """<STR_LIT>""" <EOL> stream . write ( msg ) <EOL> sys . exit ( exit_code ) <EOL> def JavaSupported ( ) : <EOL> """<STR_LIT>""" <EOL> if appcfg_java : <EOL> tools_java_dir = os . path . join ( os . path . dirname ( appcfg_java . __file__ ) , '<STR_LIT>' ) <EOL> return os . path . isdir ( tools_java_dir ) <EOL> else : <EOL> return False <EOL> @ contextlib . contextmanager <EOL> def TempChangeField ( obj , field_name , new_value ) : <EOL> """<STR_LIT>""" <EOL> old_value = getattr ( obj , field_name ) <EOL> setattr ( obj , field_name , new_value ) <EOL> yield old_value <EOL> setattr ( obj , field_name , old_value ) <EOL> class FileClassification ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , config , filename , error_fh = sys . stderr ) : <EOL> """<STR_LIT>""" <EOL> self . __error_fh = error_fh <EOL> self . __static_mime_type = self . __GetMimeTypeIfStaticFile ( config , filename ) <EOL> self . __static_app_readable = self . __GetAppReadableIfStaticFile ( config , <EOL> filename ) <EOL> self . __error_mime_type , self . __error_code = self . __LookupErrorBlob ( config , <EOL> filename ) <EOL> def __GetMimeTypeIfStaticFile ( self , config , filename ) : <EOL> """<STR_LIT>""" <EOL> if self . __FileNameImpliesStaticFile ( filename ) : <EOL> return self . __MimeType ( filename ) <EOL> for handler in config . handlers : <EOL> handler_type = handler . GetHandlerType ( ) <EOL> if handler_type in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> if handler_type == '<STR_LIT>' : <EOL> regex = os . path . join ( re . escape ( handler . GetHandler ( ) ) , '<STR_LIT>' ) <EOL> else : <EOL> regex = handler . upload <EOL> if re . match ( regex , filename ) : <EOL> return handler . mime_type or self . __MimeType ( filename ) <EOL> return None <EOL> @ staticmethod <EOL> def __FileNameImpliesStaticFile ( filename ) : <EOL> """<STR_LIT>""" <EOL> static = '<STR_LIT>' + os . sep <EOL> return static in filename <EOL> @ staticmethod <EOL> def __GetAppReadableIfStaticFile ( config , filename ) : <EOL> """<STR_LIT>""" <EOL> for handler in config . handlers : <EOL> handler_type = handler . GetHandlerType ( ) <EOL> if handler_type in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> if handler_type == '<STR_LIT>' : <EOL> regex = os . path . join ( re . escape ( handler . GetHandler ( ) ) , '<STR_LIT>' ) <EOL> else : <EOL> regex = handler . upload <EOL> if re . match ( regex , filename ) : <EOL> return handler . application_readable <EOL> return False <EOL> def __LookupErrorBlob ( self , config , filename ) : <EOL> """<STR_LIT>""" <EOL> if not config . error_handlers : <EOL> return ( None , None ) <EOL> for error_handler in config . error_handlers : <EOL> if error_handler . file == filename : <EOL> error_code = error_handler . error_code <EOL> error_code = error_code or '<STR_LIT:default>' <EOL> if error_handler . mime_type : <EOL> return ( error_handler . mime_type , error_code ) <EOL> else : <EOL> return ( self . __MimeType ( filename ) , error_code ) <EOL> return ( None , None ) <EOL> def __MimeType ( self , filename , default = '<STR_LIT>' ) : <EOL> guess = mimetypes . guess_type ( filename ) [ <NUM_LIT:0> ] <EOL> if guess is None : <EOL> print >> self . __error_fh , ( '<STR_LIT>' <EOL> % ( filename , default ) ) <EOL> return default <EOL> return guess <EOL> def IsApplicationFile ( self ) : <EOL> return bool ( ( not self . IsStaticFile ( ) or self . __static_app_readable ) and <EOL> not self . IsErrorFile ( ) ) <EOL> def IsStaticFile ( self ) : <EOL> return bool ( self . __static_mime_type ) <EOL> def StaticMimeType ( self ) : <EOL> return self . __static_mime_type <EOL> def IsErrorFile ( self ) : <EOL> return bool ( self . __error_mime_type ) <EOL> def ErrorMimeType ( self ) : <EOL> return self . __error_mime_type <EOL> def ErrorCode ( self ) : <EOL> return self . __error_code <EOL> def BuildClonePostBody ( file_tuples ) : <EOL> """<STR_LIT>""" <EOL> file_list = [ ] <EOL> for tup in file_tuples : <EOL> path = tup [ <NUM_LIT:1> ] <EOL> tup = tup [ <NUM_LIT:2> : ] <EOL> file_list . append ( TUPLE_DELIMITER . join ( [ path ] + list ( tup ) ) ) <EOL> return LIST_DELIMITER . join ( file_list ) <EOL> def _GetRemoteResourceLimits ( logging_context ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> yaml_data = logging_context . Send ( '<STR_LIT>' ) <EOL> except urllib2 . HTTPError , err : <EOL> if err . code != <NUM_LIT> : <EOL> raise <EOL> return { } <EOL> return yaml . safe_load ( yaml_data ) <EOL> def GetResourceLimits ( logging_context , error_fh = sys . stderr ) : <EOL> """<STR_LIT>""" <EOL> resource_limits = DEFAULT_RESOURCE_LIMITS . copy ( ) <EOL> StatusUpdate ( '<STR_LIT>' , error_fh ) <EOL> resource_limits . update ( _GetRemoteResourceLimits ( logging_context ) ) <EOL> logging . debug ( '<STR_LIT>' , resource_limits ) <EOL> return resource_limits <EOL> def RetryWithBackoff ( callable_func , retry_notify_func , <EOL> initial_delay = <NUM_LIT:1> , backoff_factor = <NUM_LIT:2> , <EOL> max_delay = <NUM_LIT> , max_tries = <NUM_LIT:20> ) : <EOL> """<STR_LIT>""" <EOL> delay = initial_delay <EOL> num_tries = <NUM_LIT:0> <EOL> while True : <EOL> done , opaque_value = callable_func ( ) <EOL> num_tries += <NUM_LIT:1> <EOL> if done : <EOL> return True , opaque_value <EOL> if num_tries >= max_tries : <EOL> return False , opaque_value <EOL> retry_notify_func ( opaque_value , delay ) <EOL> time . sleep ( delay ) <EOL> delay = min ( delay * backoff_factor , max_delay ) <EOL> def RetryNoBackoff ( callable_func , <EOL> retry_notify_func , <EOL> delay = <NUM_LIT:5> , <EOL> max_tries = <NUM_LIT:200> ) : <EOL> """<STR_LIT>""" <EOL> return RetryWithBackoff ( callable_func , <EOL> retry_notify_func , <EOL> delay , <EOL> <NUM_LIT:1> , <EOL> delay , <EOL> max_tries ) <EOL> def MigratePython27Notice ( ) : <EOL> """<STR_LIT>""" <EOL> ErrorUpdate ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> class IndexDefinitionUpload ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , rpcserver , definitions , error_fh = sys . stderr ) : <EOL> """<STR_LIT>""" <EOL> self . rpcserver = rpcserver <EOL> self . definitions = definitions <EOL> self . error_fh = error_fh <EOL> def DoUpload ( self ) : <EOL> """<STR_LIT>""" <EOL> StatusUpdate ( '<STR_LIT>' , self . error_fh ) <EOL> with TempChangeField ( self . definitions , '<STR_LIT>' , None ) as app_id : <EOL> self . rpcserver . Send ( '<STR_LIT>' , <EOL> app_id = app_id , <EOL> payload = self . definitions . ToYAML ( ) ) <EOL> class CronEntryUpload ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , rpcserver , cron , error_fh = sys . stderr ) : <EOL> """<STR_LIT>""" <EOL> self . rpcserver = rpcserver <EOL> self . cron = cron <EOL> self . error_fh = error_fh <EOL> def DoUpload ( self ) : <EOL> """<STR_LIT>""" <EOL> StatusUpdate ( '<STR_LIT>' , self . error_fh ) <EOL> with TempChangeField ( self . cron , '<STR_LIT>' , None ) as app_id : <EOL> self . rpcserver . Send ( '<STR_LIT>' , <EOL> app_id = app_id , <EOL> payload = self . cron . ToYAML ( ) ) <EOL> class QueueEntryUpload ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , rpcserver , queue , error_fh = sys . stderr ) : <EOL> """<STR_LIT>""" <EOL> self . rpcserver = rpcserver <EOL> self . queue = queue <EOL> self . error_fh = error_fh <EOL> def DoUpload ( self ) : <EOL> """<STR_LIT>""" <EOL> StatusUpdate ( '<STR_LIT>' , self . error_fh ) <EOL> with TempChangeField ( self . queue , '<STR_LIT>' , None ) as app_id : <EOL> self . rpcserver . Send ( '<STR_LIT>' , <EOL> app_id = app_id , <EOL> payload = self . queue . ToYAML ( ) ) <EOL> class DispatchEntryUpload ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , rpcserver , dispatch , error_fh = sys . stderr ) : <EOL> """<STR_LIT>""" <EOL> self . rpcserver = rpcserver <EOL> self . dispatch = dispatch <EOL> self . error_fh = error_fh <EOL> def DoUpload ( self ) : <EOL> """<STR_LIT>""" <EOL> StatusUpdate ( '<STR_LIT>' , self . error_fh ) <EOL> self . rpcserver . Send ( '<STR_LIT>' , <EOL> app_id = self . dispatch . application , <EOL> payload = self . dispatch . ToYAML ( ) ) <EOL> class DosEntryUpload ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , rpcserver , dos , error_fh = sys . stderr ) : <EOL> """<STR_LIT>""" <EOL> self . rpcserver = rpcserver <EOL> self . dos = dos <EOL> self . error_fh = error_fh <EOL> def DoUpload ( self ) : <EOL> """<STR_LIT>""" <EOL> StatusUpdate ( '<STR_LIT>' , self . error_fh ) <EOL> with TempChangeField ( self . dos , '<STR_LIT>' , None ) as app_id : <EOL> self . rpcserver . Send ( '<STR_LIT>' , <EOL> app_id = app_id , <EOL> payload = self . dos . ToYAML ( ) ) <EOL> class DefaultVersionSet ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , rpcserver , app_id , module , version , error_fh = sys . stderr ) : <EOL> """<STR_LIT>""" <EOL> self . rpcserver = rpcserver <EOL> self . app_id = app_id <EOL> self . module = module <EOL> self . version = version <EOL> self . error_fh = error_fh <EOL> def SetVersion ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . module : <EOL> modules = self . module . split ( '<STR_LIT:U+002C>' ) <EOL> if len ( modules ) > <NUM_LIT:1> : <EOL> StatusUpdate ( '<STR_LIT>' <EOL> '<STR_LIT>' % ( '<STR_LIT:U+002CU+0020>' . join ( modules ) , <EOL> self . app_id , <EOL> self . version ) , <EOL> self . error_fh ) <EOL> params = [ ( '<STR_LIT>' , self . app_id ) , ( '<STR_LIT:version>' , self . version ) ] <EOL> params . extend ( ( '<STR_LIT>' , module ) for module in modules ) <EOL> url = '<STR_LIT>' + urllib . urlencode ( sorted ( params ) ) <EOL> self . rpcserver . Send ( url ) <EOL> return <EOL> else : <EOL> StatusUpdate ( '<STR_LIT>' <EOL> '<STR_LIT>' % ( self . module , self . app_id , self . version ) , <EOL> self . error_fh ) <EOL> else : <EOL> StatusUpdate ( '<STR_LIT>' <EOL> % ( self . app_id , self . version ) , self . error_fh ) <EOL> self . rpcserver . Send ( '<STR_LIT>' , <EOL> app_id = self . app_id , <EOL> module = self . module , <EOL> version = self . version ) <EOL> class TrafficMigrator ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , rpcserver , app_id , version , error_fh = sys . stderr ) : <EOL> """<STR_LIT>""" <EOL> self . rpcserver = rpcserver <EOL> self . app_id = app_id <EOL> self . version = version <EOL> self . error_fh = error_fh <EOL> def MigrateTraffic ( self ) : <EOL> """<STR_LIT>""" <EOL> StatusUpdate ( '<STR_LIT>' <EOL> % ( self . app_id , self . version ) , self . error_fh ) <EOL> self . rpcserver . Send ( '<STR_LIT>' , <EOL> app_id = self . app_id , <EOL> version = self . version ) <EOL> class IndexOperation ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , rpcserver , error_fh = sys . stderr ) : <EOL> """<STR_LIT>""" <EOL> self . rpcserver = rpcserver <EOL> self . error_fh = error_fh <EOL> def DoDiff ( self , definitions ) : <EOL> """<STR_LIT>""" <EOL> StatusUpdate ( '<STR_LIT>' , self . error_fh ) <EOL> with TempChangeField ( definitions , '<STR_LIT>' , None ) as app_id : <EOL> response = self . rpcserver . Send ( '<STR_LIT>' , <EOL> app_id = app_id , <EOL> payload = definitions . ToYAML ( ) ) <EOL> return datastore_index . ParseMultipleIndexDefinitions ( response ) <EOL> def DoDelete ( self , definitions , app_id ) : <EOL> """<STR_LIT>""" <EOL> StatusUpdate ( '<STR_LIT>' , self . error_fh ) <EOL> response = self . rpcserver . Send ( '<STR_LIT>' , <EOL> app_id = app_id , <EOL> payload = definitions . ToYAML ( ) ) <EOL> return datastore_index . ParseIndexDefinitions ( response ) <EOL> class VacuumIndexesOperation ( IndexOperation ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , rpcserver , force , confirmation_fn = raw_input , <EOL> error_fh = sys . stderr ) : <EOL> """<STR_LIT>""" <EOL> super ( VacuumIndexesOperation , self ) . __init__ ( rpcserver , error_fh ) <EOL> self . force = force <EOL> self . confirmation_fn = confirmation_fn <EOL> def GetConfirmation ( self , index ) : <EOL> """<STR_LIT>""" <EOL> while True : <EOL> print '<STR_LIT>' <EOL> print <EOL> print index . ToYAML ( ) <EOL> print <EOL> confirmation = self . confirmation_fn ( <EOL> '<STR_LIT>' ) <EOL> confirmation = confirmation . strip ( ) . lower ( ) <EOL> if confirmation == '<STR_LIT:y>' : <EOL> return True <EOL> elif confirmation == '<STR_LIT:n>' or not confirmation : <EOL> return False <EOL> elif confirmation == '<STR_LIT:a>' : <EOL> self . force = True <EOL> return True <EOL> else : <EOL> print '<STR_LIT>' <EOL> def DoVacuum ( self , definitions ) : <EOL> """<STR_LIT>""" <EOL> unused_new_indexes , notused_indexes = self . DoDiff ( definitions ) <EOL> deletions = datastore_index . IndexDefinitions ( indexes = [ ] ) <EOL> if notused_indexes . indexes is not None : <EOL> for index in notused_indexes . indexes : <EOL> if self . force or self . GetConfirmation ( index ) : <EOL> deletions . indexes . append ( index ) <EOL> if deletions . indexes : <EOL> not_deleted = self . DoDelete ( deletions , definitions . application ) <EOL> if not_deleted . indexes : <EOL> not_deleted_count = len ( not_deleted . indexes ) <EOL> if not_deleted_count == <NUM_LIT:1> : <EOL> warning_message = ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> else : <EOL> warning_message = ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> % not_deleted_count ) <EOL> for index in not_deleted . indexes : <EOL> warning_message += index . ToYAML ( ) <EOL> logging . warning ( warning_message ) <EOL> class LogsRequester ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , <EOL> rpcserver , <EOL> app_id , <EOL> module , <EOL> version_id , <EOL> output_file , <EOL> num_days , <EOL> append , <EOL> severity , <EOL> end , <EOL> vhost , <EOL> include_vhost , <EOL> include_all = None , <EOL> time_func = time . time , <EOL> error_fh = sys . stderr ) : <EOL> """<STR_LIT>""" <EOL> self . rpcserver = rpcserver <EOL> self . app_id = app_id <EOL> self . output_file = output_file <EOL> self . append = append <EOL> self . num_days = num_days <EOL> self . severity = severity <EOL> self . vhost = vhost <EOL> self . include_vhost = include_vhost <EOL> self . include_all = include_all <EOL> self . error_fh = error_fh <EOL> self . module = module <EOL> self . version_id = version_id <EOL> self . sentinel = None <EOL> self . write_mode = '<STR_LIT:w>' <EOL> if self . append : <EOL> self . sentinel = FindSentinel ( self . output_file ) <EOL> self . write_mode = '<STR_LIT:a>' <EOL> self . skip_until = False <EOL> now = PacificDate ( time_func ( ) ) <EOL> if end < now : <EOL> self . skip_until = end <EOL> else : <EOL> end = now <EOL> self . valid_dates = None <EOL> if self . num_days : <EOL> start = end - datetime . timedelta ( self . num_days - <NUM_LIT:1> ) <EOL> self . valid_dates = ( start , end ) <EOL> def DownloadLogs ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . module : <EOL> StatusUpdate ( '<STR_LIT>' % <EOL> ( self . app_id , self . module , self . version_id ) , self . error_fh ) <EOL> else : <EOL> StatusUpdate ( '<STR_LIT>' % <EOL> ( self . app_id , self . version_id ) , self . error_fh ) <EOL> tf = tempfile . TemporaryFile ( ) <EOL> last_offset = None <EOL> try : <EOL> while True : <EOL> try : <EOL> new_offset = self . RequestLogLines ( tf , last_offset ) <EOL> if not new_offset or new_offset == last_offset : <EOL> break <EOL> last_offset = new_offset <EOL> except KeyboardInterrupt : <EOL> StatusUpdate ( '<STR_LIT>' , <EOL> self . error_fh ) <EOL> break <EOL> StatusUpdate ( '<STR_LIT>' % self . output_file , <EOL> self . error_fh ) <EOL> if self . output_file == '<STR_LIT:->' : <EOL> of = sys . stdout <EOL> else : <EOL> try : <EOL> of = open ( self . output_file , self . write_mode ) <EOL> except IOError , err : <EOL> StatusUpdate ( '<STR_LIT>' % ( self . output_file , err ) ) <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> try : <EOL> line_count = CopyReversedLines ( tf , of ) <EOL> finally : <EOL> of . flush ( ) <EOL> if of is not sys . stdout : <EOL> of . close ( ) <EOL> finally : <EOL> tf . close ( ) <EOL> StatusUpdate ( '<STR_LIT>' % line_count , self . error_fh ) <EOL> def RequestLogLines ( self , tf , offset ) : <EOL> """<STR_LIT>""" <EOL> logging . info ( '<STR_LIT>' , offset ) <EOL> kwds = { '<STR_LIT>' : self . app_id , <EOL> '<STR_LIT:version>' : self . version_id , <EOL> '<STR_LIT>' : <NUM_LIT:1000> , <EOL> '<STR_LIT>' : <NUM_LIT:1> , <EOL> } <EOL> if self . module : <EOL> kwds [ '<STR_LIT>' ] = self . module <EOL> if offset : <EOL> kwds [ '<STR_LIT>' ] = offset <EOL> if self . severity is not None : <EOL> kwds [ '<STR_LIT>' ] = str ( self . severity ) <EOL> if self . vhost is not None : <EOL> kwds [ '<STR_LIT>' ] = str ( self . vhost ) <EOL> if self . include_vhost is not None : <EOL> kwds [ '<STR_LIT>' ] = str ( self . include_vhost ) <EOL> if self . include_all is not None : <EOL> kwds [ '<STR_LIT>' ] = str ( self . include_all ) <EOL> response = self . rpcserver . Send ( '<STR_LIT>' , payload = None , ** kwds ) <EOL> response = response . replace ( '<STR_LIT:\r>' , '<STR_LIT>' ) <EOL> lines = response . splitlines ( ) <EOL> logging . info ( '<STR_LIT>' , len ( response ) , len ( lines ) ) <EOL> offset = None <EOL> valid_dates = self . valid_dates <EOL> sentinel = self . sentinel <EOL> skip_until = self . skip_until <EOL> len_sentinel = None <EOL> if sentinel : <EOL> len_sentinel = len ( sentinel ) <EOL> for line in lines : <EOL> if line . startswith ( '<STR_LIT:#>' ) : <EOL> match = re . match ( r'<STR_LIT>' , line ) <EOL> if match and match . group ( <NUM_LIT:1> ) != '<STR_LIT:None>' : <EOL> offset = match . group ( <NUM_LIT:1> ) <EOL> continue <EOL> if ( sentinel and <EOL> line . startswith ( sentinel ) and <EOL> line [ len_sentinel : len_sentinel + <NUM_LIT:1> ] in ( '<STR_LIT>' , '<STR_LIT>' ) ) : <EOL> return None <EOL> linedate = DateOfLogLine ( line ) <EOL> if not linedate : <EOL> continue <EOL> if skip_until : <EOL> if linedate > skip_until : <EOL> continue <EOL> else : <EOL> self . skip_until = skip_until = False <EOL> if valid_dates and not valid_dates [ <NUM_LIT:0> ] <= linedate <= valid_dates [ <NUM_LIT:1> ] : <EOL> return None <EOL> tf . write ( line + '<STR_LIT:\n>' ) <EOL> if not lines : <EOL> return None <EOL> return offset <EOL> def DateOfLogLine ( line ) : <EOL> """<STR_LIT>""" <EOL> m = re . compile ( r'<STR_LIT>' ) . match ( line ) <EOL> if not m : <EOL> return None <EOL> try : <EOL> return datetime . date ( * time . strptime ( m . group ( <NUM_LIT:1> ) , '<STR_LIT>' ) [ : <NUM_LIT:3> ] ) <EOL> except ValueError : <EOL> return None <EOL> def PacificDate ( now ) : <EOL> """<STR_LIT>""" <EOL> return datetime . date ( * time . gmtime ( PacificTime ( now ) ) [ : <NUM_LIT:3> ] ) <EOL> def PacificTime ( now ) : <EOL> """<STR_LIT>""" <EOL> now -= <NUM_LIT:8> * <NUM_LIT> <EOL> if IsPacificDST ( now ) : <EOL> now += <NUM_LIT> <EOL> return now <EOL> def IsPacificDST ( now ) : <EOL> """<STR_LIT>""" <EOL> pst = time . gmtime ( now ) <EOL> year = pst [ <NUM_LIT:0> ] <EOL> assert year >= <NUM_LIT> <EOL> begin = calendar . timegm ( ( year , <NUM_LIT:3> , <NUM_LIT:8> , <NUM_LIT:2> , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> ) ) <EOL> while time . gmtime ( begin ) . tm_wday != SUNDAY : <EOL> begin += DAY <EOL> end = calendar . timegm ( ( year , <NUM_LIT:11> , <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> ) ) <EOL> while time . gmtime ( end ) . tm_wday != SUNDAY : <EOL> end += DAY <EOL> return begin <= now < end <EOL> def CopyReversedLines ( instream , outstream , blocksize = <NUM_LIT:2> ** <NUM_LIT:16> ) : <EOL> r"""<STR_LIT>""" <EOL> line_count = <NUM_LIT:0> <EOL> instream . seek ( <NUM_LIT:0> , <NUM_LIT:2> ) <EOL> last_block = instream . tell ( ) // blocksize <EOL> spillover = '<STR_LIT>' <EOL> for iblock in xrange ( last_block + <NUM_LIT:1> , - <NUM_LIT:1> , - <NUM_LIT:1> ) : <EOL> instream . seek ( iblock * blocksize ) <EOL> data = instream . read ( blocksize ) <EOL> lines = data . splitlines ( True ) <EOL> lines [ - <NUM_LIT:1> : ] = '<STR_LIT>' . join ( lines [ - <NUM_LIT:1> : ] + [ spillover ] ) . splitlines ( True ) <EOL> if lines and not lines [ - <NUM_LIT:1> ] . endswith ( '<STR_LIT:\n>' ) : <EOL> lines [ - <NUM_LIT:1> ] += '<STR_LIT:\n>' <EOL> lines . reverse ( ) <EOL> if lines and iblock > <NUM_LIT:0> : <EOL> spillover = lines . pop ( ) <EOL> if lines : <EOL> line_count += len ( lines ) <EOL> data = '<STR_LIT>' . join ( lines ) . replace ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> outstream . write ( data ) <EOL> return line_count <EOL> def FindSentinel ( filename , blocksize = <NUM_LIT:2> ** <NUM_LIT:16> , error_fh = sys . stderr ) : <EOL> """<STR_LIT>""" <EOL> if filename == '<STR_LIT:->' : <EOL> StatusUpdate ( '<STR_LIT>' , <EOL> error_fh ) <EOL> sys . exit ( <NUM_LIT:2> ) <EOL> try : <EOL> fp = open ( filename , '<STR_LIT:rb>' ) <EOL> except IOError , err : <EOL> StatusUpdate ( '<STR_LIT>' % ( filename , err ) , <EOL> error_fh ) <EOL> return None <EOL> try : <EOL> fp . seek ( <NUM_LIT:0> , <NUM_LIT:2> ) <EOL> fp . seek ( max ( <NUM_LIT:0> , fp . tell ( ) - blocksize ) ) <EOL> lines = fp . readlines ( ) <EOL> del lines [ : <NUM_LIT:1> ] <EOL> sentinel = None <EOL> for line in lines : <EOL> if not line . startswith ( '<STR_LIT:\t>' ) : <EOL> sentinel = line <EOL> if not sentinel : <EOL> StatusUpdate ( '<STR_LIT>' % <EOL> filename , error_fh ) <EOL> return None <EOL> return sentinel . rstrip ( '<STR_LIT:\n>' ) <EOL> finally : <EOL> fp . close ( ) <EOL> class UploadBatcher ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , what , logging_context ) : <EOL> """<STR_LIT>""" <EOL> assert what in ( '<STR_LIT:file>' , '<STR_LIT>' , '<STR_LIT>' ) , repr ( what ) <EOL> self . what = what <EOL> self . logging_context = logging_context <EOL> self . single_url = '<STR_LIT>' + what <EOL> self . batch_url = self . single_url + '<STR_LIT:s>' <EOL> self . batching = True <EOL> self . batch = [ ] <EOL> self . batch_size = <NUM_LIT:0> <EOL> def SendBatch ( self ) : <EOL> """<STR_LIT>""" <EOL> boundary = '<STR_LIT>' <EOL> parts = [ ] <EOL> for path , payload , mime_type in self . batch : <EOL> while boundary in payload : <EOL> boundary += '<STR_LIT>' % random . randint ( <NUM_LIT:0> , <NUM_LIT> ) <EOL> assert len ( boundary ) < <NUM_LIT> , '<STR_LIT>' <EOL> part = '<STR_LIT:\n>' . join ( [ '<STR_LIT>' , <EOL> '<STR_LIT>' % urllib . quote ( path ) , <EOL> '<STR_LIT>' % _Hash ( payload ) , <EOL> '<STR_LIT>' % mime_type , <EOL> '<STR_LIT>' % len ( payload ) , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> payload , <EOL> ] ) <EOL> parts . append ( part ) <EOL> parts . insert ( <NUM_LIT:0> , <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT:\n>' <EOL> '<STR_LIT>' % <EOL> boundary ) <EOL> parts . append ( '<STR_LIT>' ) <EOL> delimiter = '<STR_LIT>' % boundary <EOL> payload = delimiter . join ( parts ) <EOL> logging . info ( '<STR_LIT>' , <EOL> len ( self . batch ) , self . what , self . batch_url , boundary ) <EOL> self . logging_context . Send ( self . batch_url , <EOL> payload = payload , <EOL> content_type = '<STR_LIT>' ) <EOL> self . batch = [ ] <EOL> self . batch_size = <NUM_LIT:0> <EOL> def SendSingleFile ( self , path , payload , mime_type ) : <EOL> """<STR_LIT>""" <EOL> logging . info ( '<STR_LIT>' , <EOL> self . what , path , len ( payload ) , mime_type , self . single_url ) <EOL> self . logging_context . Send ( self . single_url , <EOL> payload = payload , <EOL> content_type = mime_type , <EOL> path = path ) <EOL> def Flush ( self ) : <EOL> """<STR_LIT>""" <EOL> if not self . batch : <EOL> return <EOL> try : <EOL> self . SendBatch ( ) <EOL> except urllib2 . HTTPError , err : <EOL> if err . code != <NUM_LIT> : <EOL> raise <EOL> logging . info ( '<STR_LIT>' , self . what ) <EOL> self . batching = False <EOL> for path , payload , mime_type in self . batch : <EOL> self . SendSingleFile ( path , payload , mime_type ) <EOL> self . batch = [ ] <EOL> self . batch_size = <NUM_LIT:0> <EOL> def AddToBatch ( self , path , payload , mime_type ) : <EOL> """<STR_LIT>""" <EOL> if not mime_type : <EOL> mime_type = '<STR_LIT>' <EOL> size = len ( payload ) <EOL> if size <= MAX_BATCH_FILE_SIZE : <EOL> if ( len ( self . batch ) >= MAX_BATCH_COUNT or <EOL> self . batch_size + size > MAX_BATCH_SIZE ) : <EOL> self . Flush ( ) <EOL> if self . batching : <EOL> logging . info ( '<STR_LIT>' , <EOL> self . what , path , size , mime_type ) <EOL> self . batch . append ( ( path , payload , mime_type ) ) <EOL> self . batch_size += size + BATCH_OVERHEAD <EOL> return <EOL> self . SendSingleFile ( path , payload , mime_type ) <EOL> def _FormatHash ( h ) : <EOL> """<STR_LIT>""" <EOL> return '<STR_LIT>' % ( h [ <NUM_LIT:0> : <NUM_LIT:8> ] , h [ <NUM_LIT:8> : <NUM_LIT:16> ] , h [ <NUM_LIT:16> : <NUM_LIT> ] , h [ <NUM_LIT> : <NUM_LIT:32> ] , h [ <NUM_LIT:32> : <NUM_LIT> ] ) <EOL> def _Hash ( content ) : <EOL> """<STR_LIT>""" <EOL> h = hashlib . sha1 ( content ) . hexdigest ( ) <EOL> return _FormatHash ( h ) <EOL> def _HashFromFileHandle ( file_handle ) : <EOL> """<STR_LIT>""" <EOL> pos = file_handle . tell ( ) <EOL> content_hash = _Hash ( file_handle . read ( ) ) <EOL> file_handle . seek ( pos , <NUM_LIT:0> ) <EOL> return content_hash <EOL> def EnsureDir ( path ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> os . makedirs ( path ) <EOL> except OSError , exc : <EOL> if not ( exc . errno == errno . EEXIST and os . path . isdir ( path ) ) : <EOL> raise <EOL> def DoDownloadApp ( rpcserver , out_dir , app_id , module , app_version , <EOL> error_fh = sys . stderr ) : <EOL> """<STR_LIT>""" <EOL> StatusUpdate ( '<STR_LIT>' , error_fh ) <EOL> url_args = { '<STR_LIT>' : app_id } <EOL> if module : <EOL> url_args [ '<STR_LIT>' ] = module <EOL> if app_version is not None : <EOL> url_args [ '<STR_LIT>' ] = app_version <EOL> result = rpcserver . Send ( '<STR_LIT>' , ** url_args ) <EOL> StatusUpdate ( '<STR_LIT>' , error_fh ) <EOL> lines = result . splitlines ( ) <EOL> if len ( lines ) < <NUM_LIT:1> : <EOL> logging . error ( '<STR_LIT>' ) <EOL> return <EOL> full_version = lines [ <NUM_LIT:0> ] <EOL> file_lines = lines [ <NUM_LIT:1> : ] <EOL> current_file_number = <NUM_LIT:0> <EOL> num_files = len ( file_lines ) <EOL> num_errors = <NUM_LIT:0> <EOL> for line in file_lines : <EOL> parts = line . split ( '<STR_LIT:|>' , <NUM_LIT:2> ) <EOL> if len ( parts ) != <NUM_LIT:3> : <EOL> logging . error ( '<STR_LIT>' <EOL> '<STR_LIT>' , line ) <EOL> return <EOL> current_file_number += <NUM_LIT:1> <EOL> file_id , size_str , path = parts <EOL> try : <EOL> size = int ( size_str ) <EOL> except ValueError : <EOL> logging . error ( '<STR_LIT>' <EOL> '<STR_LIT>' , size_str ) <EOL> return <EOL> StatusUpdate ( '<STR_LIT>' % ( current_file_number , num_files , path ) , <EOL> error_fh ) <EOL> def TryGet ( ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> contents = rpcserver . Send ( '<STR_LIT>' , app_id = app_id , <EOL> version = full_version , id = file_id ) <EOL> return True , contents <EOL> except urllib2 . HTTPError , exc : <EOL> if exc . code == <NUM_LIT> : <EOL> return False , exc <EOL> else : <EOL> raise <EOL> def PrintRetryMessage ( _ , delay ) : <EOL> StatusUpdate ( '<STR_LIT>' % delay , <EOL> error_fh ) <EOL> success , contents = RetryWithBackoff ( TryGet , PrintRetryMessage ) <EOL> if not success : <EOL> logging . error ( '<STR_LIT>' , path ) <EOL> num_errors += <NUM_LIT:1> <EOL> continue <EOL> if len ( contents ) != size : <EOL> logging . error ( '<STR_LIT>' <EOL> '<STR_LIT>' , path , size , len ( contents ) ) <EOL> num_errors += <NUM_LIT:1> <EOL> full_path = os . path . join ( out_dir , path ) <EOL> if os . path . exists ( full_path ) : <EOL> logging . error ( '<STR_LIT>' <EOL> '<STR_LIT>' , path ) <EOL> num_errors += <NUM_LIT:1> <EOL> continue <EOL> full_dir = os . path . dirname ( full_path ) <EOL> try : <EOL> EnsureDir ( full_dir ) <EOL> except OSError , exc : <EOL> logging . error ( '<STR_LIT>' , full_dir , exc ) <EOL> num_errors += <NUM_LIT:1> <EOL> continue <EOL> try : <EOL> out_file = open ( full_path , '<STR_LIT:wb>' ) <EOL> except IOError , exc : <EOL> logging . error ( '<STR_LIT>' , full_path , exc ) <EOL> num_errors += <NUM_LIT:1> <EOL> continue <EOL> try : <EOL> try : <EOL> out_file . write ( contents ) <EOL> except IOError , exc : <EOL> logging . error ( '<STR_LIT>' , full_path , exc ) <EOL> num_errors += <NUM_LIT:1> <EOL> continue <EOL> finally : <EOL> out_file . close ( ) <EOL> if num_errors > <NUM_LIT:0> : <EOL> logging . error ( '<STR_LIT>' , num_errors ) <EOL> class _ClientDeployLoggingContext ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , <EOL> rpcserver , <EOL> request_params , <EOL> usage_reporting , <EOL> time_func = time . time ) : <EOL> """<STR_LIT>""" <EOL> self . rpcserver = rpcserver <EOL> self . request_params = request_params <EOL> self . usage_reporting = usage_reporting <EOL> self . time_func = time_func <EOL> self . requests = [ ] <EOL> def Send ( self , url , payload = '<STR_LIT>' , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> start_time_usec = self . GetCurrentTimeUsec ( ) <EOL> request_size_bytes = len ( payload ) <EOL> try : <EOL> logging . info ( '<STR_LIT>' , url , self . request_params ) <EOL> kwargs . update ( self . request_params ) <EOL> result = self . rpcserver . Send ( url , payload = payload , ** kwargs ) <EOL> self . _RegisterReqestForLogging ( url , <NUM_LIT:200> , start_time_usec , <EOL> request_size_bytes ) <EOL> return result <EOL> except urllib2 . HTTPError , e : <EOL> self . _RegisterReqestForLogging ( url , e . code , start_time_usec , <EOL> request_size_bytes ) <EOL> raise e <EOL> def GetCurrentTimeUsec ( self ) : <EOL> """<STR_LIT>""" <EOL> return int ( round ( self . time_func ( ) * <NUM_LIT:1000> * <NUM_LIT:1000> ) ) <EOL> def GetSdkVersion ( self ) : <EOL> """<STR_LIT>""" <EOL> sdk_version = sdk_update_checker . GetVersionObject ( ) <EOL> return sdk_version . get ( '<STR_LIT>' , '<STR_LIT:?>' ) if sdk_version else '<STR_LIT:?>' <EOL> def _RegisterReqestForLogging ( self , path , response_code , start_time_usec , <EOL> request_size_bytes ) : <EOL> """<STR_LIT>""" <EOL> end_time_usec = self . GetCurrentTimeUsec ( ) <EOL> self . requests . append ( client_deployinfo . Request ( <EOL> path = path , <EOL> response_code = response_code , <EOL> start_time_usec = start_time_usec , <EOL> end_time_usec = end_time_usec , <EOL> request_size_bytes = request_size_bytes ) ) <EOL> def LogClientDeploy ( self , runtime , start_time_usec , success ) : <EOL> """<STR_LIT>""" <EOL> if not self . usage_reporting : <EOL> logging . info ( '<STR_LIT>' ) <EOL> return <EOL> end_time_usec = self . GetCurrentTimeUsec ( ) <EOL> try : <EOL> info = client_deployinfo . ClientDeployInfoExternal ( <EOL> runtime = runtime , <EOL> start_time_usec = start_time_usec , <EOL> end_time_usec = end_time_usec , <EOL> requests = self . requests , <EOL> success = success , <EOL> sdk_version = self . GetSdkVersion ( ) ) <EOL> self . Send ( '<STR_LIT>' , info . ToYAML ( ) ) <EOL> except BaseException , e : <EOL> logging . debug ( '<STR_LIT>' , e ) <EOL> class EndpointsState ( object ) : <EOL> SERVING = '<STR_LIT>' <EOL> PENDING = '<STR_LIT>' <EOL> FAILED = '<STR_LIT>' <EOL> _STATES = frozenset ( ( SERVING , PENDING , FAILED ) ) <EOL> @ classmethod <EOL> def Parse ( cls , value ) : <EOL> state = value . lower ( ) <EOL> if state not in cls . _STATES : <EOL> lst = sorted ( cls . _STATES ) <EOL> pretty_states = '<STR_LIT:U+002CU+0020>' . join ( lst [ : - <NUM_LIT:1> ] ) + '<STR_LIT>' + lst [ - <NUM_LIT:1> ] <EOL> raise ValueError ( '<STR_LIT>' % <EOL> ( value , pretty_states ) ) <EOL> return state <EOL> class AppVersionUpload ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , rpcserver , config , module_yaml_path = '<STR_LIT>' , <EOL> backend = None , <EOL> error_fh = None , <EOL> usage_reporting = False , ignore_endpoints_failures = True ) : <EOL> """<STR_LIT>""" <EOL> self . rpcserver = rpcserver <EOL> self . config = config <EOL> self . app_id = self . config . application <EOL> self . module = self . config . module <EOL> self . backend = backend <EOL> self . error_fh = error_fh or sys . stderr <EOL> self . version = self . config . version <EOL> self . params = { } <EOL> if self . app_id : <EOL> self . params [ '<STR_LIT>' ] = self . app_id <EOL> if self . module : <EOL> self . params [ '<STR_LIT>' ] = self . module <EOL> if self . backend : <EOL> self . params [ '<STR_LIT>' ] = self . backend <EOL> elif self . version : <EOL> self . params [ '<STR_LIT:version>' ] = self . version <EOL> self . files = { } <EOL> self . all_files = set ( ) <EOL> self . in_transaction = False <EOL> self . deployed = False <EOL> self . started = False <EOL> self . batching = True <EOL> self . logging_context = _ClientDeployLoggingContext ( rpcserver , <EOL> self . params , <EOL> usage_reporting ) <EOL> self . file_batcher = UploadBatcher ( '<STR_LIT:file>' , self . logging_context ) <EOL> self . blob_batcher = UploadBatcher ( '<STR_LIT>' , self . logging_context ) <EOL> self . errorblob_batcher = UploadBatcher ( '<STR_LIT>' , self . logging_context ) <EOL> if not self . config . vm_settings : <EOL> self . config . vm_settings = appinfo . VmSettings ( ) <EOL> self . config . vm_settings [ '<STR_LIT>' ] = module_yaml_path <EOL> if not self . config . auto_id_policy : <EOL> self . config . auto_id_policy = appinfo . DATASTORE_ID_POLICY_DEFAULT <EOL> self . ignore_endpoints_failures = ignore_endpoints_failures <EOL> def AddFile ( self , path , file_handle ) : <EOL> """<STR_LIT>""" <EOL> assert not self . in_transaction , '<STR_LIT>' <EOL> assert file_handle is not None <EOL> reason = appinfo . ValidFilename ( path ) <EOL> if reason : <EOL> logging . error ( reason ) <EOL> return <EOL> content_hash = _HashFromFileHandle ( file_handle ) <EOL> self . files [ path ] = content_hash <EOL> self . all_files . add ( path ) <EOL> def Describe ( self ) : <EOL> """<STR_LIT>""" <EOL> result = '<STR_LIT>' % self . app_id <EOL> if self . module is not None and self . module != appinfo . DEFAULT_MODULE : <EOL> result += '<STR_LIT>' % self . module <EOL> if self . backend : <EOL> result += '<STR_LIT>' % self . backend <EOL> elif self . version : <EOL> result += '<STR_LIT>' % self . version <EOL> return result <EOL> @ staticmethod <EOL> def _ValidateBeginYaml ( resp ) : <EOL> """<STR_LIT>""" <EOL> response_dict = yaml . safe_load ( resp ) <EOL> if not response_dict or '<STR_LIT>' not in response_dict : <EOL> return False <EOL> return response_dict <EOL> def Begin ( self ) : <EOL> """<STR_LIT>""" <EOL> assert not self . in_transaction , '<STR_LIT>' <EOL> config_copy = copy . deepcopy ( self . config ) <EOL> for url in config_copy . handlers : <EOL> handler_type = url . GetHandlerType ( ) <EOL> if url . application_readable : <EOL> if handler_type == '<STR_LIT>' : <EOL> url . static_dir = '<STR_LIT>' % ( STATIC_FILE_PREFIX , url . static_dir ) <EOL> elif handler_type == '<STR_LIT>' : <EOL> url . static_files = '<STR_LIT>' % ( STATIC_FILE_PREFIX , url . static_files ) <EOL> url . upload = '<STR_LIT>' % ( STATIC_FILE_PREFIX , url . upload ) <EOL> response = self . logging_context . Send ( <EOL> '<STR_LIT>' , <EOL> payload = config_copy . ToYAML ( ) ) <EOL> result = self . _ValidateBeginYaml ( response ) <EOL> if result : <EOL> warnings = result . get ( '<STR_LIT>' ) <EOL> for warning in warnings : <EOL> StatusUpdate ( '<STR_LIT>' % warning , self . error_fh ) <EOL> self . in_transaction = True <EOL> files_to_clone = [ ] <EOL> blobs_to_clone = [ ] <EOL> errorblobs = { } <EOL> for path , content_hash in self . files . iteritems ( ) : <EOL> file_classification = FileClassification ( <EOL> self . config , path , error_fh = self . error_fh ) <EOL> if file_classification . IsStaticFile ( ) : <EOL> upload_path = path <EOL> if file_classification . IsApplicationFile ( ) : <EOL> upload_path = '<STR_LIT>' % ( STATIC_FILE_PREFIX , path ) <EOL> blobs_to_clone . append ( ( path , upload_path , content_hash , <EOL> file_classification . StaticMimeType ( ) ) ) <EOL> if file_classification . IsErrorFile ( ) : <EOL> errorblobs [ path ] = content_hash <EOL> if file_classification . IsApplicationFile ( ) : <EOL> files_to_clone . append ( ( path , path , content_hash ) ) <EOL> files_to_upload = { } <EOL> def CloneFiles ( url , files , file_type ) : <EOL> """<STR_LIT>""" <EOL> if not files : <EOL> return <EOL> StatusUpdate ( '<STR_LIT>' % <EOL> ( len ( files ) , file_type , len ( files ) != <NUM_LIT:1> and '<STR_LIT:s>' or '<STR_LIT>' ) , <EOL> self . error_fh ) <EOL> max_files = self . resource_limits [ '<STR_LIT>' ] <EOL> for i in xrange ( <NUM_LIT:0> , len ( files ) , max_files ) : <EOL> if i > <NUM_LIT:0> and i % max_files == <NUM_LIT:0> : <EOL> StatusUpdate ( '<STR_LIT>' % i , self . error_fh ) <EOL> chunk = files [ i : min ( len ( files ) , i + max_files ) ] <EOL> result = self . logging_context . Send ( url , <EOL> payload = BuildClonePostBody ( chunk ) ) <EOL> if result : <EOL> to_upload = { } <EOL> for f in result . split ( LIST_DELIMITER ) : <EOL> for entry in files : <EOL> real_path , upload_path = entry [ : <NUM_LIT:2> ] <EOL> if f == upload_path : <EOL> to_upload [ real_path ] = self . files [ real_path ] <EOL> break <EOL> files_to_upload . update ( to_upload ) <EOL> CloneFiles ( '<STR_LIT>' , blobs_to_clone , '<STR_LIT>' ) <EOL> CloneFiles ( '<STR_LIT>' , files_to_clone , '<STR_LIT>' ) <EOL> logging . debug ( '<STR_LIT>' , files_to_upload ) <EOL> for ( path , content_hash ) in errorblobs . iteritems ( ) : <EOL> files_to_upload [ path ] = content_hash <EOL> self . files = files_to_upload <EOL> return sorted ( files_to_upload . iterkeys ( ) ) <EOL> def UploadFile ( self , path , file_handle ) : <EOL> """<STR_LIT>""" <EOL> assert self . in_transaction , '<STR_LIT>' <EOL> if path not in self . files : <EOL> raise KeyError ( '<STR_LIT>' <EOL> % path ) <EOL> del self . files [ path ] <EOL> file_classification = FileClassification ( <EOL> self . config , path , error_fh = self . error_fh ) <EOL> payload = file_handle . read ( ) <EOL> if file_classification . IsStaticFile ( ) : <EOL> upload_path = path <EOL> if file_classification . IsApplicationFile ( ) : <EOL> upload_path = '<STR_LIT>' % ( STATIC_FILE_PREFIX , path ) <EOL> self . blob_batcher . AddToBatch ( upload_path , payload , <EOL> file_classification . StaticMimeType ( ) ) <EOL> if file_classification . IsErrorFile ( ) : <EOL> self . errorblob_batcher . AddToBatch ( file_classification . ErrorCode ( ) , <EOL> payload , <EOL> file_classification . ErrorMimeType ( ) ) <EOL> if file_classification . IsApplicationFile ( ) : <EOL> self . file_batcher . AddToBatch ( path , payload , None ) <EOL> def Precompile ( self ) : <EOL> """<STR_LIT>""" <EOL> StatusUpdate ( '<STR_LIT>' , self . error_fh ) <EOL> files = [ ] <EOL> if self . config . GetEffectiveRuntime ( ) == '<STR_LIT>' : <EOL> for f in self . all_files : <EOL> if f . endswith ( '<STR_LIT>' ) and not self . config . nobuild_files . match ( f ) : <EOL> files . append ( f ) <EOL> while True : <EOL> if files : <EOL> StatusUpdate ( '<STR_LIT>' % len ( files ) , self . error_fh ) <EOL> files = self . PrecompileBatch ( files ) <EOL> if not files : <EOL> break <EOL> StatusUpdate ( '<STR_LIT>' , self . error_fh ) <EOL> def PrecompileBatch ( self , files ) : <EOL> """<STR_LIT>""" <EOL> payload = LIST_DELIMITER . join ( files ) <EOL> response = self . logging_context . Send ( '<STR_LIT>' , <EOL> payload = payload ) <EOL> if not response : <EOL> return [ ] <EOL> return response . split ( LIST_DELIMITER ) <EOL> def Commit ( self ) : <EOL> """<STR_LIT>""" <EOL> assert self . in_transaction , '<STR_LIT>' <EOL> if self . files : <EOL> raise RuntimeError ( '<STR_LIT>' ) <EOL> def PrintRetryMessage ( _ , delay ) : <EOL> StatusUpdate ( '<STR_LIT>' % delay , self . error_fh ) <EOL> app_summary = self . Deploy ( ) <EOL> success , unused_contents = RetryWithBackoff ( <EOL> lambda : ( self . IsReady ( ) , None ) , PrintRetryMessage , <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT> , <NUM_LIT:20> ) <EOL> if not success : <EOL> logging . warning ( '<STR_LIT>' ) <EOL> raise RuntimeError ( '<STR_LIT>' ) <EOL> result = self . StartServing ( ) <EOL> if not result : <EOL> self . in_transaction = False <EOL> else : <EOL> if result == '<STR_LIT:0>' : <EOL> raise CannotStartServingError ( <EOL> '<STR_LIT>' ) <EOL> success , response = RetryNoBackoff ( self . IsServing , PrintRetryMessage ) <EOL> if not success : <EOL> logging . warning ( '<STR_LIT>' ) <EOL> raise RuntimeError ( '<STR_LIT>' ) <EOL> check_config_updated = response . get ( '<STR_LIT>' ) <EOL> if check_config_updated : <EOL> unused_done , ( last_state , user_error ) = RetryWithBackoff ( <EOL> self . IsEndpointsConfigUpdated , <EOL> PrintRetryMessage , <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT> , <NUM_LIT:20> ) <EOL> if last_state != EndpointsState . SERVING : <EOL> self . HandleEndpointsError ( user_error ) <EOL> self . in_transaction = False <EOL> return app_summary <EOL> def Deploy ( self ) : <EOL> """<STR_LIT>""" <EOL> assert self . in_transaction , '<STR_LIT>' <EOL> if self . files : <EOL> raise RuntimeError ( '<STR_LIT>' ) <EOL> StatusUpdate ( '<STR_LIT>' , self . error_fh ) <EOL> result = self . logging_context . Send ( '<STR_LIT>' ) <EOL> self . deployed = True <EOL> if result : <EOL> return yaml_object . BuildSingleObject ( appinfo . AppInfoSummary , result ) <EOL> else : <EOL> return None <EOL> def IsReady ( self ) : <EOL> """<STR_LIT>""" <EOL> assert self . deployed , '<STR_LIT>' <EOL> StatusUpdate ( '<STR_LIT>' , self . error_fh ) <EOL> result = self . logging_context . Send ( '<STR_LIT>' ) <EOL> return result == '<STR_LIT:1>' <EOL> def StartServing ( self ) : <EOL> """<STR_LIT>""" <EOL> assert self . deployed , '<STR_LIT>' <EOL> StatusUpdate ( '<STR_LIT>' , self . error_fh ) <EOL> self . params [ '<STR_LIT>' ] = '<STR_LIT:1>' <EOL> result = self . logging_context . Send ( '<STR_LIT>' ) <EOL> del self . params [ '<STR_LIT>' ] <EOL> self . started = True <EOL> return result <EOL> @ staticmethod <EOL> def _ValidateIsServingYaml ( resp ) : <EOL> """<STR_LIT>""" <EOL> response_dict = yaml . safe_load ( resp ) <EOL> if '<STR_LIT>' not in response_dict : <EOL> return None <EOL> return response_dict <EOL> def IsServing ( self ) : <EOL> """<STR_LIT>""" <EOL> assert self . started , '<STR_LIT>' <EOL> StatusUpdate ( '<STR_LIT>' , self . error_fh ) <EOL> self . params [ '<STR_LIT>' ] = '<STR_LIT:1>' <EOL> result = self . logging_context . Send ( '<STR_LIT>' ) <EOL> del self . params [ '<STR_LIT>' ] <EOL> if result in [ '<STR_LIT:0>' , '<STR_LIT:1>' ] : <EOL> return result == '<STR_LIT:1>' , { } <EOL> result = AppVersionUpload . _ValidateIsServingYaml ( result ) <EOL> if not result : <EOL> raise CannotStartServingError ( <EOL> '<STR_LIT>' ) <EOL> message = result . get ( '<STR_LIT:message>' ) <EOL> fatal = result . get ( '<STR_LIT>' ) <EOL> if message : <EOL> StatusUpdate ( message , self . error_fh ) <EOL> if fatal : <EOL> raise CannotStartServingError ( message or '<STR_LIT>' ) <EOL> return result [ '<STR_LIT>' ] , result <EOL> @ staticmethod <EOL> def _ValidateIsEndpointsConfigUpdatedYaml ( resp ) : <EOL> """<STR_LIT>""" <EOL> response_dict = yaml . safe_load ( resp ) <EOL> if '<STR_LIT>' not in response_dict and '<STR_LIT>' not in response_dict : <EOL> return None <EOL> return response_dict <EOL> def GetLogUrl ( self ) : <EOL> """<STR_LIT>""" <EOL> module = '<STR_LIT>' % self . module if self . module else '<STR_LIT>' <EOL> return ( '<STR_LIT>' + <EOL> urllib . urlencode ( ( ( '<STR_LIT>' , self . app_id ) , <EOL> ( '<STR_LIT>' , module + self . version ) ) ) ) <EOL> def IsEndpointsConfigUpdated ( self ) : <EOL> """<STR_LIT>""" <EOL> assert self . started , ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> StatusUpdate ( '<STR_LIT>' , <EOL> self . error_fh ) <EOL> result = self . logging_context . Send ( '<STR_LIT>' ) <EOL> result = AppVersionUpload . _ValidateIsEndpointsConfigUpdatedYaml ( result ) <EOL> if result is None : <EOL> raise CannotStartServingError ( <EOL> '<STR_LIT>' ) <EOL> if '<STR_LIT>' in result : <EOL> updated_state = EndpointsState . Parse ( result [ '<STR_LIT>' ] ) <EOL> user_error = result . get ( '<STR_LIT>' ) <EOL> else : <EOL> updated_state = ( EndpointsState . SERVING if result [ '<STR_LIT>' ] <EOL> else EndpointsState . PENDING ) <EOL> user_error = None <EOL> return updated_state != EndpointsState . PENDING , ( updated_state , user_error ) <EOL> def HandleEndpointsError ( self , user_error ) : <EOL> """<STR_LIT>""" <EOL> detailed_error = user_error or ( <EOL> "<STR_LIT>" % self . GetLogUrl ( ) ) <EOL> error_message = ( '<STR_LIT>' % <EOL> detailed_error ) <EOL> StatusUpdate ( error_message , self . error_fh ) <EOL> doc_link = ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> StatusUpdate ( '<STR_LIT>' <EOL> '<STR_LIT>' % doc_link , self . error_fh ) <EOL> if self . ignore_endpoints_failures : <EOL> StatusUpdate ( '<STR_LIT>' , <EOL> self . error_fh ) <EOL> else : <EOL> raise RuntimeError ( error_message ) <EOL> def Rollback ( self , force_rollback = False ) : <EOL> """<STR_LIT>""" <EOL> if not self . in_transaction : <EOL> return <EOL> msg = '<STR_LIT>' <EOL> if self . config . vm and not force_rollback : <EOL> msg += ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> StatusUpdate ( msg , self . error_fh ) <EOL> self . logging_context . Send ( '<STR_LIT>' , <EOL> force_rollback = '<STR_LIT:1>' if force_rollback else '<STR_LIT:0>' ) <EOL> self . in_transaction = False <EOL> self . files = { } <EOL> def DoUpload ( self , paths , openfunc ) : <EOL> """<STR_LIT>""" <EOL> start_time_usec = self . logging_context . GetCurrentTimeUsec ( ) <EOL> logging . info ( '<STR_LIT>' ) <EOL> StatusUpdate ( '<STR_LIT>' % self . Describe ( ) , self . error_fh ) <EOL> path = '<STR_LIT>' <EOL> try : <EOL> self . resource_limits = GetResourceLimits ( self . logging_context , <EOL> self . error_fh ) <EOL> self . _AddFilesThatAreSmallEnough ( paths , openfunc ) <EOL> except KeyboardInterrupt : <EOL> logging . info ( '<STR_LIT>' ) <EOL> raise <EOL> except EnvironmentError , e : <EOL> if self . _IsExceptionClientDeployLoggable ( e ) : <EOL> self . logging_context . LogClientDeploy ( self . config . runtime , <EOL> start_time_usec , False ) <EOL> logging . error ( '<STR_LIT>' , <EOL> path , e ) <EOL> raise <EOL> try : <EOL> missing_files = self . Begin ( ) <EOL> self . _UploadMissingFiles ( missing_files , openfunc ) <EOL> if ( self . config . derived_file_type and <EOL> appinfo . PYTHON_PRECOMPILED in self . config . derived_file_type ) : <EOL> try : <EOL> self . Precompile ( ) <EOL> except urllib2 . HTTPError , e : <EOL> ErrorUpdate ( '<STR_LIT>' <EOL> '<STR_LIT>' % <EOL> ( e . code , e . read ( ) . rstrip ( '<STR_LIT:\n>' ) ) ) <EOL> if e . code == <NUM_LIT> or self . config . GetEffectiveRuntime ( ) == '<STR_LIT>' : <EOL> raise <EOL> print >> self . error_fh , ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> app_summary = self . Commit ( ) <EOL> StatusUpdate ( '<STR_LIT>' % self . Describe ( ) , self . error_fh ) <EOL> self . logging_context . LogClientDeploy ( self . config . runtime , start_time_usec , <EOL> True ) <EOL> except BaseException , e : <EOL> try : <EOL> self . _LogDoUploadException ( e ) <EOL> self . Rollback ( ) <EOL> finally : <EOL> if self . _IsExceptionClientDeployLoggable ( e ) : <EOL> self . logging_context . LogClientDeploy ( self . config . runtime , <EOL> start_time_usec , False ) <EOL> raise <EOL> logging . info ( '<STR_LIT>' ) <EOL> return app_summary <EOL> def _IsExceptionClientDeployLoggable ( self , exception ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( exception , KeyboardInterrupt ) : <EOL> return False <EOL> if ( isinstance ( exception , urllib2 . HTTPError ) <EOL> and <NUM_LIT> <= exception . code <= <NUM_LIT> ) : <EOL> return False <EOL> return True <EOL> def _AddFilesThatAreSmallEnough ( self , paths , openfunc ) : <EOL> """<STR_LIT>""" <EOL> StatusUpdate ( '<STR_LIT>' , self . error_fh ) <EOL> num_files = <NUM_LIT:0> <EOL> for path in paths : <EOL> file_handle = openfunc ( path ) <EOL> try : <EOL> file_length = GetFileLength ( file_handle ) <EOL> file_classification = FileClassification ( <EOL> self . config , path , self . error_fh ) <EOL> if file_classification . IsApplicationFile ( ) : <EOL> max_size = self . resource_limits [ '<STR_LIT>' ] <EOL> else : <EOL> max_size = self . resource_limits [ '<STR_LIT>' ] <EOL> if file_length > max_size : <EOL> extra_msg = ( '<STR_LIT>' <EOL> if JavaSupported ( ) and path . endswith ( '<STR_LIT>' ) <EOL> else '<STR_LIT>' ) <EOL> logging . error ( '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> path , max_size , file_length , extra_msg ) <EOL> else : <EOL> logging . info ( '<STR_LIT>' , path ) <EOL> self . AddFile ( path , file_handle ) <EOL> finally : <EOL> file_handle . close ( ) <EOL> num_files += <NUM_LIT:1> <EOL> if num_files % <NUM_LIT> == <NUM_LIT:0> : <EOL> StatusUpdate ( '<STR_LIT>' % num_files , self . error_fh ) <EOL> def _UploadMissingFiles ( self , missing_files , openfunc ) : <EOL> """<STR_LIT>""" <EOL> if not missing_files : <EOL> return <EOL> StatusUpdate ( '<STR_LIT>' % len ( missing_files ) , <EOL> self . error_fh ) <EOL> num_files = <NUM_LIT:0> <EOL> for missing_file in missing_files : <EOL> file_handle = openfunc ( missing_file ) <EOL> try : <EOL> self . UploadFile ( missing_file , file_handle ) <EOL> finally : <EOL> file_handle . close ( ) <EOL> num_files += <NUM_LIT:1> <EOL> if num_files % <NUM_LIT> == <NUM_LIT:0> : <EOL> StatusUpdate ( '<STR_LIT>' % <EOL> ( num_files , len ( missing_files ) ) , self . error_fh ) <EOL> self . file_batcher . Flush ( ) <EOL> self . blob_batcher . Flush ( ) <EOL> self . errorblob_batcher . Flush ( ) <EOL> StatusUpdate ( '<STR_LIT>' % num_files , self . error_fh ) <EOL> @ staticmethod <EOL> def _LogDoUploadException ( exception ) : <EOL> """<STR_LIT>""" <EOL> def InstanceOf ( tipe ) : <EOL> return isinstance ( exception , tipe ) <EOL> if InstanceOf ( KeyboardInterrupt ) : <EOL> logging . info ( '<STR_LIT>' ) <EOL> elif InstanceOf ( urllib2 . HTTPError ) : <EOL> logging . info ( '<STR_LIT>' , exception ) <EOL> elif InstanceOf ( CannotStartServingError ) : <EOL> logging . error ( exception . message ) <EOL> else : <EOL> logging . exception ( '<STR_LIT>' ) <EOL> class DoLockAction ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( <EOL> self , url , rpcserver , app_id , version , module , instance , file_handle ) : <EOL> self . url = url <EOL> self . rpcserver = rpcserver <EOL> self . app_id = app_id <EOL> self . version = version <EOL> self . module = module <EOL> self . instance = instance <EOL> self . file_handle = file_handle <EOL> def GetState ( self ) : <EOL> yaml_data = self . rpcserver . Send ( '<STR_LIT>' , <EOL> app_id = self . app_id , <EOL> version_match = self . version , <EOL> module = self . module ) <EOL> state = yaml . safe_load ( yaml_data ) <EOL> done = state [ '<STR_LIT:state>' ] != '<STR_LIT>' <EOL> if done : <EOL> print >> self . file_handle , state [ '<STR_LIT:message>' ] <EOL> return ( done , state [ '<STR_LIT:message>' ] ) <EOL> def PrintRetryMessage ( self , msg , delay ) : <EOL> StatusUpdate ( '<STR_LIT>' % ( msg , delay ) , <EOL> self . file_handle ) <EOL> def Do ( self ) : <EOL> kwargs = { '<STR_LIT>' : self . app_id , <EOL> '<STR_LIT>' : self . version , <EOL> '<STR_LIT>' : self . module } <EOL> if self . instance : <EOL> kwargs [ '<STR_LIT>' ] = self . instance <EOL> response = self . rpcserver . Send ( self . url , ** kwargs ) <EOL> print >> self . file_handle , response <EOL> RetryWithBackoff ( self . GetState , self . PrintRetryMessage , <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:5> , <NUM_LIT:20> ) <EOL> def FileIterator ( base , skip_files , runtime , separator = os . path . sep ) : <EOL> """<STR_LIT>""" <EOL> dirs = [ '<STR_LIT>' ] <EOL> while dirs : <EOL> current_dir = dirs . pop ( ) <EOL> entries = set ( os . listdir ( os . path . join ( base , current_dir ) ) ) <EOL> for entry in sorted ( entries ) : <EOL> name = os . path . join ( current_dir , entry ) <EOL> fullname = os . path . join ( base , name ) <EOL> if separator == '<STR_LIT:\\>' : <EOL> name = name . replace ( '<STR_LIT:\\>' , '<STR_LIT:/>' ) <EOL> if runtime == '<STR_LIT>' and not skip_files . match ( name ) : <EOL> root , extension = os . path . splitext ( entry ) <EOL> if extension == '<STR_LIT>' and ( root + '<STR_LIT>' ) in entries : <EOL> logging . warning ( '<STR_LIT>' <EOL> '<STR_LIT>' , name ) <EOL> continue <EOL> if os . path . isfile ( fullname ) : <EOL> if skip_files . match ( name ) : <EOL> logging . info ( '<STR_LIT>' , name ) <EOL> else : <EOL> yield name <EOL> elif os . path . isdir ( fullname ) : <EOL> if skip_files . match ( name ) : <EOL> logging . info ( <EOL> '<STR_LIT>' , <EOL> name ) <EOL> else : <EOL> dirs . append ( name ) <EOL> def GetFileLength ( fh ) : <EOL> """<STR_LIT>""" <EOL> pos = fh . tell ( ) <EOL> fh . seek ( <NUM_LIT:0> , <NUM_LIT:2> ) <EOL> length = fh . tell ( ) <EOL> fh . seek ( pos , <NUM_LIT:0> ) <EOL> return length <EOL> def GetUserAgent ( get_version = sdk_update_checker . GetVersionObject , <EOL> get_platform = appengine_rpc . GetPlatformToken , <EOL> sdk_product = SDK_PRODUCT ) : <EOL> """<STR_LIT>""" <EOL> product_tokens = [ ] <EOL> sdk_name = os . environ . get ( '<STR_LIT>' ) <EOL> if sdk_name : <EOL> product_tokens . append ( sdk_name ) <EOL> else : <EOL> version = get_version ( ) <EOL> if version is None : <EOL> release = '<STR_LIT>' <EOL> else : <EOL> release = version [ '<STR_LIT>' ] <EOL> product_tokens . append ( '<STR_LIT>' % ( sdk_product , release ) ) <EOL> product_tokens . append ( get_platform ( ) ) <EOL> python_version = '<STR_LIT:.>' . join ( str ( i ) for i in sys . version_info ) <EOL> product_tokens . append ( '<STR_LIT>' % python_version ) <EOL> return '<STR_LIT:U+0020>' . join ( product_tokens ) <EOL> def GetSourceName ( get_version = sdk_update_checker . GetVersionObject ) : <EOL> """<STR_LIT>""" <EOL> version = get_version ( ) <EOL> if version is None : <EOL> release = '<STR_LIT>' <EOL> else : <EOL> release = version [ '<STR_LIT>' ] <EOL> return '<STR_LIT>' % ( release , ) <EOL> def _ReadUrlContents ( url ) : <EOL> """<STR_LIT>""" <EOL> req = urllib2 . Request ( url , headers = { '<STR_LIT>' : '<STR_LIT>' } ) <EOL> return urllib2 . urlopen ( req ) . read ( ) <EOL> class AppCfgApp ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , argv , parser_class = optparse . OptionParser , <EOL> rpc_server_class = None , <EOL> out_fh = sys . stdout , <EOL> error_fh = sys . stderr , <EOL> update_check_class = sdk_update_checker . SDKUpdateChecker , <EOL> throttle_class = None , <EOL> opener = open , <EOL> file_iterator = FileIterator , <EOL> time_func = time . time , <EOL> wrap_server_error_message = True , <EOL> oauth_client_id = APPCFG_CLIENT_ID , <EOL> oauth_client_secret = APPCFG_CLIENT_NOTSOSECRET , <EOL> oauth_scopes = APPCFG_SCOPES ) : <EOL> """<STR_LIT>""" <EOL> self . parser_class = parser_class <EOL> self . argv = argv <EOL> self . rpc_server_class = rpc_server_class <EOL> self . out_fh = out_fh <EOL> self . error_fh = error_fh <EOL> self . update_check_class = update_check_class <EOL> self . throttle_class = throttle_class <EOL> self . time_func = time_func <EOL> self . wrap_server_error_message = wrap_server_error_message <EOL> self . oauth_client_id = oauth_client_id <EOL> self . oauth_client_secret = oauth_client_secret <EOL> self . oauth_scopes = oauth_scopes <EOL> self . read_url_contents = _ReadUrlContents <EOL> self . parser = self . _GetOptionParser ( ) <EOL> for action in self . actions . itervalues ( ) : <EOL> action . options ( self , self . parser ) <EOL> self . options , self . args = self . parser . parse_args ( argv [ <NUM_LIT:1> : ] ) <EOL> if len ( self . args ) < <NUM_LIT:1> : <EOL> self . _PrintHelpAndExit ( ) <EOL> if not self . options . allow_any_runtime : <EOL> if self . options . runtime : <EOL> if self . options . runtime not in appinfo . GetAllRuntimes ( ) : <EOL> _PrintErrorAndExit ( self . error_fh , <EOL> '<STR_LIT>' % <EOL> self . options . runtime ) <EOL> else : <EOL> appinfo . AppInfoExternal . ATTRIBUTES [ appinfo . RUNTIME ] = ( <EOL> '<STR_LIT:|>' . join ( appinfo . GetAllRuntimes ( ) ) ) <EOL> if self . options . redundant_oauth2 : <EOL> print >> sys . stderr , ( <EOL> '<STR_LIT>' ) <EOL> action = self . args . pop ( <NUM_LIT:0> ) <EOL> def RaiseParseError ( actionname , action ) : <EOL> self . parser , self . options = self . _MakeSpecificParser ( action ) <EOL> error_desc = action . error_desc <EOL> if not error_desc : <EOL> error_desc = "<STR_LIT>" % ( <EOL> actionname . split ( '<STR_LIT:U+0020>' ) [ <NUM_LIT:0> ] ) <EOL> self . parser . error ( error_desc ) <EOL> if action == BACKENDS_ACTION : <EOL> if len ( self . args ) < <NUM_LIT:1> : <EOL> RaiseParseError ( action , self . actions [ BACKENDS_ACTION ] ) <EOL> backend_action_first = BACKENDS_ACTION + '<STR_LIT:U+0020>' + self . args [ <NUM_LIT:0> ] <EOL> if backend_action_first in self . actions : <EOL> self . args . pop ( <NUM_LIT:0> ) <EOL> action = backend_action_first <EOL> elif len ( self . args ) > <NUM_LIT:1> : <EOL> backend_directory_first = BACKENDS_ACTION + '<STR_LIT:U+0020>' + self . args [ <NUM_LIT:1> ] <EOL> if backend_directory_first in self . actions : <EOL> self . args . pop ( <NUM_LIT:1> ) <EOL> action = backend_directory_first <EOL> if len ( self . args ) < <NUM_LIT:1> or action == BACKENDS_ACTION : <EOL> RaiseParseError ( action , self . actions [ action ] ) <EOL> if action not in self . actions : <EOL> self . parser . error ( "<STR_LIT>" % <EOL> ( action , self . parser . get_description ( ) ) ) <EOL> self . action = self . actions [ action ] <EOL> if not self . action . uses_basepath or self . options . help : <EOL> self . basepath = None <EOL> else : <EOL> if not self . args : <EOL> RaiseParseError ( action , self . action ) <EOL> self . basepath = self . args . pop ( <NUM_LIT:0> ) <EOL> self . parser , self . options = self . _MakeSpecificParser ( self . action ) <EOL> if self . options . help : <EOL> self . _PrintHelpAndExit ( ) <EOL> if self . options . verbose == <NUM_LIT:2> : <EOL> logging . getLogger ( ) . setLevel ( logging . INFO ) <EOL> elif self . options . verbose == <NUM_LIT:3> : <EOL> logging . getLogger ( ) . setLevel ( logging . DEBUG ) <EOL> global verbosity <EOL> verbosity = self . options . verbose <EOL> if self . options . oauth2_client_id : <EOL> self . oauth_client_id = self . options . oauth2_client_id <EOL> if self . options . oauth2_client_secret : <EOL> self . oauth_client_secret = self . options . oauth2_client_secret <EOL> self . opener = opener <EOL> self . file_iterator = file_iterator <EOL> def Run ( self ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> self . action ( self ) <EOL> except urllib2 . HTTPError , e : <EOL> body = e . read ( ) <EOL> if self . wrap_server_error_message : <EOL> error_format = ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> else : <EOL> error_format = '<STR_LIT>' <EOL> print >> self . error_fh , ( error_format % ( e . code , body . rstrip ( '<STR_LIT:\n>' ) ) ) <EOL> return <NUM_LIT:1> <EOL> except yaml_errors . EventListenerError , e : <EOL> print >> self . error_fh , ( '<STR_LIT>' % e ) <EOL> return <NUM_LIT:1> <EOL> except CannotStartServingError : <EOL> print >> self . error_fh , '<STR_LIT>' <EOL> return <NUM_LIT:1> <EOL> return <NUM_LIT:0> <EOL> def _GetActionDescriptions ( self ) : <EOL> """<STR_LIT>""" <EOL> action_names = self . actions . keys ( ) <EOL> action_names . sort ( ) <EOL> desc = '<STR_LIT>' <EOL> for action_name in action_names : <EOL> if not self . actions [ action_name ] . hidden : <EOL> desc += '<STR_LIT>' % ( action_name , <EOL> self . actions [ action_name ] . short_desc ) <EOL> return desc <EOL> def _GetOptionParser ( self ) : <EOL> """<STR_LIT>""" <EOL> def AppendSourceReference ( option , opt_str , value , parser ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> appinfo . ValidateSourceReference ( value ) <EOL> except validation . ValidationError , e : <EOL> raise optparse . OptionValueError ( '<STR_LIT>' % ( opt_str , e . message ) ) <EOL> getattr ( parser . values , option . dest ) . append ( value ) <EOL> class Formatter ( optparse . IndentedHelpFormatter ) : <EOL> """<STR_LIT>""" <EOL> def format_description ( self , description ) : <EOL> """<STR_LIT>""" <EOL> return description + '<STR_LIT:\n>' <EOL> class AppCfgOption ( optparse . Option ) : <EOL> """<STR_LIT>""" <EOL> _ACTION = '<STR_LIT>' <EOL> ACTIONS = optparse . Option . ACTIONS + ( _ACTION , ) <EOL> STORE_ACTIONS = optparse . Option . STORE_ACTIONS + ( _ACTION , ) <EOL> TYPED_ACTIONS = optparse . Option . TYPED_ACTIONS + ( _ACTION , ) <EOL> ALWAYS_TYPED_ACTIONS = optparse . Option . ALWAYS_TYPED_ACTIONS + ( _ACTION , ) <EOL> def take_action ( self , action , dest , opt , value , values , parser ) : <EOL> if action != self . _ACTION : <EOL> return optparse . Option . take_action ( <EOL> self , action , dest , opt , value , values , parser ) <EOL> try : <EOL> key , value = value . split ( '<STR_LIT::>' , <NUM_LIT:1> ) <EOL> except ValueError : <EOL> raise optparse . OptionValueError ( <EOL> '<STR_LIT>' % ( <EOL> opt , value ) ) <EOL> values . ensure_value ( dest , { } ) [ key ] = value <EOL> desc = self . _GetActionDescriptions ( ) <EOL> desc = ( '<STR_LIT>' <EOL> '<STR_LIT>' ) % desc <EOL> parser = self . parser_class ( usage = '<STR_LIT>' , <EOL> description = desc , <EOL> formatter = Formatter ( ) , <EOL> conflict_handler = '<STR_LIT>' , <EOL> option_class = AppCfgOption ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , action = '<STR_LIT:store_true>' , <EOL> dest = '<STR_LIT>' , help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , action = '<STR_LIT>' , const = <NUM_LIT:0> , <EOL> dest = '<STR_LIT>' , help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , action = '<STR_LIT>' , const = <NUM_LIT:2> , <EOL> dest = '<STR_LIT>' , default = <NUM_LIT:1> , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , action = '<STR_LIT>' , const = <NUM_LIT:3> , <EOL> dest = '<STR_LIT>' , help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , action = '<STR_LIT:store>' , dest = '<STR_LIT>' , <EOL> default = '<STR_LIT>' , <EOL> metavar = '<STR_LIT>' , help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , action = '<STR_LIT:store_true>' , dest = '<STR_LIT>' , <EOL> default = True , help = optparse . SUPPRESS_HELP ) <EOL> parser . add_option ( '<STR_LIT>' , action = '<STR_LIT:store_true>' , <EOL> dest = '<STR_LIT>' , default = False , <EOL> help = optparse . SUPPRESS_HELP ) <EOL> parser . add_option ( '<STR_LIT>' , action = '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> help = optparse . SUPPRESS_HELP ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , action = '<STR_LIT:store>' , dest = '<STR_LIT:email>' , <EOL> metavar = '<STR_LIT>' , default = None , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , action = '<STR_LIT:store>' , dest = '<STR_LIT:host>' , <EOL> metavar = '<STR_LIT>' , default = None , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , action = '<STR_LIT>' , <EOL> dest = '<STR_LIT>' , default = True , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , action = '<STR_LIT:store_true>' , <EOL> dest = '<STR_LIT>' , default = False , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , action = '<STR_LIT:store>' , dest = '<STR_LIT>' , <EOL> help = ( '<STR_LIT>' <EOL> '<STR_LIT>' ) ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , action = '<STR_LIT:store>' , dest = '<STR_LIT>' , <EOL> help = ( '<STR_LIT>' <EOL> '<STR_LIT>' ) ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , action = '<STR_LIT:store>' , dest = '<STR_LIT:version>' , <EOL> help = ( '<STR_LIT>' <EOL> '<STR_LIT>' ) ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , action = '<STR_LIT:store>' , dest = '<STR_LIT>' , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , metavar = '<STR_LIT>' , <EOL> type = '<STR_LIT:string>' , action = '<STR_LIT>' , <EOL> callback = AppendSourceReference , dest = '<STR_LIT>' , <EOL> default = [ ] , <EOL> help = optparse . SUPPRESS_HELP ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , action = '<STR_LIT>' , <EOL> dest = '<STR_LIT>' , metavar = '<STR_LIT>' , <EOL> help = ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , action = '<STR_LIT:store_true>' , <EOL> dest = '<STR_LIT>' , default = False , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , action = '<STR_LIT:store_true>' , <EOL> dest = '<STR_LIT>' , default = False , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , action = '<STR_LIT:store>' , <EOL> dest = '<STR_LIT>' , default = None , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , action = '<STR_LIT:store>' , <EOL> dest = '<STR_LIT>' , default = None , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , action = '<STR_LIT:store>' , <EOL> dest = '<STR_LIT>' , default = None , <EOL> help = optparse . SUPPRESS_HELP ) <EOL> parser . add_option ( '<STR_LIT>' , action = '<STR_LIT:store>' , <EOL> dest = '<STR_LIT>' , default = None , <EOL> help = optparse . SUPPRESS_HELP ) <EOL> parser . add_option ( '<STR_LIT>' , action = '<STR_LIT:store>' , <EOL> dest = '<STR_LIT>' , default = None , <EOL> help = optparse . SUPPRESS_HELP ) <EOL> parser . add_option ( '<STR_LIT>' , action = '<STR_LIT:store_true>' , <EOL> dest = '<STR_LIT>' , default = False , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , action = '<STR_LIT>' , <EOL> dest = '<STR_LIT>' , default = True , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , <EOL> action = '<STR_LIT:store_true>' , default = False , <EOL> help = optparse . SUPPRESS_HELP ) <EOL> parser . add_option ( '<STR_LIT>' , action = '<STR_LIT:store_true>' , <EOL> dest = '<STR_LIT>' , default = True , <EOL> help = optparse . SUPPRESS_HELP ) <EOL> parser . add_option ( '<STR_LIT>' , action = '<STR_LIT>' , <EOL> dest = '<STR_LIT>' , <EOL> help = optparse . SUPPRESS_HELP ) <EOL> return parser <EOL> def _MakeSpecificParser ( self , action ) : <EOL> """<STR_LIT>""" <EOL> parser = self . _GetOptionParser ( ) <EOL> parser . set_usage ( action . usage ) <EOL> parser . set_description ( '<STR_LIT>' % ( action . short_desc , action . long_desc ) ) <EOL> action . options ( self , parser ) <EOL> options , unused_args = parser . parse_args ( self . argv [ <NUM_LIT:1> : ] ) <EOL> return parser , options <EOL> def _PrintHelpAndExit ( self , exit_code = <NUM_LIT:2> ) : <EOL> """<STR_LIT>""" <EOL> self . parser . print_help ( ) <EOL> sys . exit ( exit_code ) <EOL> def _GetRpcServer ( self ) : <EOL> """<STR_LIT>""" <EOL> StatusUpdate ( '<STR_LIT>' % self . options . server , self . error_fh ) <EOL> source = GetSourceName ( ) <EOL> dev_appserver = self . options . host in [ '<STR_LIT:localhost>' , '<STR_LIT:127.0.0.1>' ] <EOL> if dev_appserver : <EOL> if not self . rpc_server_class : <EOL> self . rpc_server_class = appengine_rpc . HttpRpcServer <EOL> if hasattr ( self , '<STR_LIT>' ) : <EOL> self . rpc_server_class . RUNTIME = self . runtime <EOL> email = self . options . email <EOL> if email is None : <EOL> email = '<STR_LIT>' <EOL> logging . info ( '<STR_LIT>' , email ) <EOL> rpcserver = self . rpc_server_class ( <EOL> self . options . server , <EOL> lambda : ( email , '<STR_LIT:password>' ) , <EOL> GetUserAgent ( ) , <EOL> source , <EOL> host_override = self . options . host , <EOL> save_cookies = self . options . save_cookies , <EOL> secure = False ) <EOL> rpcserver . authenticated = True <EOL> return rpcserver <EOL> if not self . rpc_server_class : <EOL> self . rpc_server_class = appengine_rpc_httplib2 . HttpRpcServerOAuth2 <EOL> oauth2_parameters = self . _GetOAuth2Parameters ( ) <EOL> return self . rpc_server_class ( self . options . server , oauth2_parameters , <EOL> GetUserAgent ( ) , source , <EOL> host_override = self . options . host , <EOL> save_cookies = self . options . save_cookies , <EOL> auth_tries = <NUM_LIT:3> , <EOL> account_type = '<STR_LIT>' , <EOL> secure = self . options . secure , <EOL> ignore_certs = self . options . ignore_certs , <EOL> options = self . options ) <EOL> def _MaybeGetDevshellOAuth2AccessToken ( self ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> creds = devshell . DevshellCredentials ( ) <EOL> return creds . access_token <EOL> except devshell . NoDevshellServer : <EOL> return None <EOL> def _GetOAuth2Parameters ( self ) : <EOL> """<STR_LIT>""" <EOL> oauth2_parameters = ( <EOL> appengine_rpc_httplib2 . HttpRpcServerOAuth2 . OAuth2Parameters ( <EOL> access_token = ( self . options . oauth2_access_token or <EOL> self . _MaybeGetDevshellOAuth2AccessToken ( ) ) , <EOL> client_id = self . oauth_client_id , <EOL> client_secret = self . oauth_client_secret , <EOL> scope = self . oauth_scopes , <EOL> refresh_token = self . options . oauth2_refresh_token , <EOL> credential_file = self . options . oauth2_credential_file , <EOL> credentials = self . _GetCredentials ( ) ) ) <EOL> return oauth2_parameters <EOL> def _GetCredentials ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . options . authenticate_service_account : <EOL> url = '<STR_LIT>' % ( METADATA_BASE , SERVICE_ACCOUNT_BASE ) <EOL> try : <EOL> vm_scopes_string = self . read_url_contents ( url ) <EOL> except urllib2 . URLError , e : <EOL> raise RuntimeError ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' % ( url , e ) ) <EOL> vm_scopes = vm_scopes_string . split ( ) <EOL> missing = list ( set ( self . oauth_scopes ) . difference ( vm_scopes ) ) <EOL> if missing : <EOL> raise RuntimeError ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' % ( missing , vm_scopes ) ) <EOL> return oauth2client_gce . AppAssertionCredentials ( ) <EOL> else : <EOL> return None <EOL> def _FindYaml ( self , basepath , file_name ) : <EOL> """<STR_LIT>""" <EOL> if not os . path . isdir ( basepath ) : <EOL> self . parser . error ( '<STR_LIT>' % basepath ) <EOL> alt_basepath = os . path . join ( basepath , '<STR_LIT>' , '<STR_LIT>' ) <EOL> for yaml_basepath in ( basepath , alt_basepath ) : <EOL> for yaml_file in ( file_name + '<STR_LIT>' , file_name + '<STR_LIT>' ) : <EOL> yaml_path = os . path . join ( yaml_basepath , yaml_file ) <EOL> if os . path . isfile ( yaml_path ) : <EOL> return yaml_path <EOL> return None <EOL> def _ParseAppInfoFromYaml ( self , basepath , basename = '<STR_LIT>' ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> appyaml = self . _ParseYamlFile ( basepath , basename , appinfo_includes . Parse ) <EOL> except yaml_errors . EventListenerError , e : <EOL> self . parser . error ( '<STR_LIT>' % ( <EOL> os . path . join ( basepath , basename ) , e ) ) <EOL> if not appyaml : <EOL> if JavaSupported ( ) : <EOL> if appcfg_java . IsWarFileWithoutYaml ( basepath ) : <EOL> java_app_update = appcfg_java . JavaAppUpdate ( basepath , self . options ) <EOL> appyaml_string = java_app_update . GenerateAppYamlString ( [ ] ) <EOL> appyaml = appinfo . LoadSingleAppInfo ( appyaml_string ) <EOL> if not appyaml : <EOL> self . parser . error ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' % basename ) <EOL> else : <EOL> self . parser . error ( '<STR_LIT>' <EOL> '<STR_LIT>' % <EOL> ( os . path . abspath ( basepath ) , basename ) ) <EOL> orig_application = appyaml . application <EOL> orig_module = appyaml . module <EOL> orig_version = appyaml . version <EOL> if self . options . app_id : <EOL> appyaml . application = self . options . app_id <EOL> if self . options . module : <EOL> appyaml . module = self . options . module <EOL> if self . options . version : <EOL> appyaml . version = self . options . version <EOL> if self . options . runtime : <EOL> appyaml . SetEffectiveRuntime ( self . options . runtime ) <EOL> if self . options . env_variables : <EOL> if appyaml . env_variables is None : <EOL> appyaml . env_variables = appinfo . EnvironmentVariables ( ) <EOL> appyaml . env_variables . update ( self . options . env_variables ) <EOL> if self . options . source_ref : <EOL> try : <EOL> combined_refs = '<STR_LIT:\n>' . join ( self . options . source_ref ) <EOL> appinfo . ValidateCombinedSourceReferencesString ( combined_refs ) <EOL> if appyaml . beta_settings is None : <EOL> appyaml . beta_settings = appinfo . BetaSettings ( ) <EOL> appyaml . beta_settings [ '<STR_LIT>' ] = combined_refs <EOL> except validation . ValidationError , e : <EOL> self . parser . error ( e . message ) <EOL> if not appyaml . application : <EOL> self . parser . error ( '<STR_LIT>' <EOL> '<STR_LIT>' % basename ) <EOL> msg = '<STR_LIT>' % appyaml . application <EOL> if appyaml . application != orig_application : <EOL> msg += '<STR_LIT>' % orig_application <EOL> if self . action . function is '<STR_LIT>' : <EOL> if ( appyaml . module is not None and <EOL> appyaml . module != appinfo . DEFAULT_MODULE ) : <EOL> msg += '<STR_LIT>' % appyaml . module <EOL> if appyaml . module != orig_module : <EOL> msg += '<STR_LIT>' % orig_module <EOL> msg += '<STR_LIT>' % appyaml . version <EOL> if appyaml . version != orig_version : <EOL> msg += '<STR_LIT>' % orig_version <EOL> StatusUpdate ( msg , self . error_fh ) <EOL> return appyaml <EOL> def _ParseYamlFile ( self , basepath , basename , parser ) : <EOL> """<STR_LIT>""" <EOL> file_name = self . _FindYaml ( basepath , basename ) <EOL> if file_name is not None : <EOL> fh = self . opener ( file_name , '<STR_LIT:r>' ) <EOL> try : <EOL> defns = parser ( fh , open_fn = self . opener ) <EOL> finally : <EOL> fh . close ( ) <EOL> return defns <EOL> return None <EOL> def _ParseBackendsYaml ( self , basepath ) : <EOL> """<STR_LIT>""" <EOL> return self . _ParseYamlFile ( basepath , '<STR_LIT>' , <EOL> backendinfo . LoadBackendInfo ) <EOL> def _ParseIndexYaml ( self , basepath , appyaml = None ) : <EOL> """<STR_LIT>""" <EOL> index_yaml = self . _ParseYamlFile ( basepath , <EOL> '<STR_LIT:index>' , <EOL> datastore_index . ParseIndexDefinitions ) <EOL> if not index_yaml : <EOL> return None <EOL> self . _SetApplication ( index_yaml , '<STR_LIT:index>' , appyaml ) <EOL> return index_yaml <EOL> def _SetApplication ( self , dest_yaml , basename , appyaml = None ) : <EOL> """<STR_LIT>""" <EOL> if self . options . app_id : <EOL> dest_yaml . application = self . options . app_id <EOL> if not dest_yaml . application : <EOL> if not appyaml : <EOL> appyaml = self . _ParseYamlFile ( self . basepath , <EOL> '<STR_LIT>' , <EOL> appinfo_includes . Parse ) <EOL> if appyaml : <EOL> dest_yaml . application = appyaml . application <EOL> else : <EOL> self . parser . error ( '<STR_LIT>' <EOL> '<STR_LIT>' % basename ) <EOL> def _ParseCronYaml ( self , basepath , appyaml = None ) : <EOL> """<STR_LIT>""" <EOL> cron_yaml = self . _ParseYamlFile ( basepath , '<STR_LIT>' , croninfo . LoadSingleCron ) <EOL> if not cron_yaml : <EOL> return None <EOL> self . _SetApplication ( cron_yaml , '<STR_LIT>' , appyaml ) <EOL> return cron_yaml <EOL> def _ParseQueueYaml ( self , basepath , appyaml = None ) : <EOL> """<STR_LIT>""" <EOL> queue_yaml = self . _ParseYamlFile ( basepath , <EOL> '<STR_LIT>' , <EOL> queueinfo . LoadSingleQueue ) <EOL> if not queue_yaml : <EOL> return None <EOL> self . _SetApplication ( queue_yaml , '<STR_LIT>' , appyaml ) <EOL> return queue_yaml <EOL> def _ParseDispatchYaml ( self , basepath , appyaml = None ) : <EOL> """<STR_LIT>""" <EOL> dispatch_yaml = self . _ParseYamlFile ( basepath , <EOL> '<STR_LIT>' , <EOL> dispatchinfo . LoadSingleDispatch ) <EOL> if not dispatch_yaml : <EOL> return None <EOL> self . _SetApplication ( dispatch_yaml , '<STR_LIT>' , appyaml ) <EOL> return dispatch_yaml <EOL> def _ParseDosYaml ( self , basepath , appyaml = None ) : <EOL> """<STR_LIT>""" <EOL> dos_yaml = self . _ParseYamlFile ( basepath , '<STR_LIT>' , dosinfo . LoadSingleDos ) <EOL> if not dos_yaml : <EOL> return None <EOL> self . _SetApplication ( dos_yaml , '<STR_LIT>' , appyaml ) <EOL> return dos_yaml <EOL> def Help ( self , action = None ) : <EOL> """<STR_LIT>""" <EOL> if not action : <EOL> if len ( self . args ) > <NUM_LIT:1> : <EOL> self . args = [ '<STR_LIT:U+0020>' . join ( self . args ) ] <EOL> if len ( self . args ) != <NUM_LIT:1> or self . args [ <NUM_LIT:0> ] not in self . actions : <EOL> self . parser . error ( '<STR_LIT>' <EOL> '<STR_LIT>' + <EOL> self . _GetActionDescriptions ( ) ) <EOL> action = self . args [ <NUM_LIT:0> ] <EOL> action = self . actions [ action ] <EOL> self . parser , unused_options = self . _MakeSpecificParser ( action ) <EOL> self . _PrintHelpAndExit ( exit_code = <NUM_LIT:0> ) <EOL> def DownloadApp ( self ) : <EOL> """<STR_LIT>""" <EOL> if len ( self . args ) != <NUM_LIT:1> : <EOL> self . parser . error ( '<STR_LIT>' <EOL> '<STR_LIT>' + str ( len ( self . args ) ) + '<STR_LIT:.>' ) <EOL> out_dir = self . args [ <NUM_LIT:0> ] <EOL> app_id = self . options . app_id <EOL> if app_id is None : <EOL> self . parser . error ( '<STR_LIT>' ) <EOL> module = self . options . module <EOL> app_version = self . options . version <EOL> if os . path . exists ( out_dir ) : <EOL> if not os . path . isdir ( out_dir ) : <EOL> self . parser . error ( '<STR_LIT>' <EOL> '<STR_LIT>' % out_dir ) <EOL> elif os . listdir ( out_dir ) : <EOL> self . parser . error ( '<STR_LIT>' <EOL> '<STR_LIT>' % out_dir ) <EOL> rpcserver = self . _GetRpcServer ( ) <EOL> DoDownloadApp ( rpcserver , out_dir , app_id , module , app_version ) <EOL> def UpdateVersion ( self , rpcserver , basepath , appyaml , module_yaml_path , <EOL> backend = None ) : <EOL> """<STR_LIT>""" <EOL> runtime = appyaml . GetEffectiveRuntime ( ) <EOL> if appyaml . vm and ( self . options . called_by_gcloud or runtime != '<STR_LIT>' ) : <EOL> self . options . precompilation = False <EOL> elif runtime == '<STR_LIT>' : <EOL> self . options . precompilation = False <EOL> elif runtime == '<STR_LIT>' and not self . options . precompilation : <EOL> logging . warning ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> self . options . precompilation = True <EOL> elif ( runtime . startswith ( '<STR_LIT>' ) and <EOL> appinfo . JAVA_PRECOMPILED not in ( appyaml . derived_file_type or [ ] ) ) : <EOL> self . options . precompilation = False <EOL> if runtime in GCLOUD_ONLY_RUNTIMES : <EOL> raise RuntimeError ( '<STR_LIT>' <EOL> '<STR_LIT>' % runtime ) <EOL> if self . options . precompilation : <EOL> if not appyaml . derived_file_type : <EOL> appyaml . derived_file_type = [ ] <EOL> if appinfo . PYTHON_PRECOMPILED not in appyaml . derived_file_type : <EOL> appyaml . derived_file_type . append ( appinfo . PYTHON_PRECOMPILED ) <EOL> paths = self . file_iterator ( basepath , appyaml . skip_files , appyaml . runtime ) <EOL> openfunc = lambda path : self . opener ( os . path . join ( basepath , path ) , '<STR_LIT:rb>' ) <EOL> if appyaml . GetEffectiveRuntime ( ) == '<STR_LIT>' : <EOL> if appyaml . runtime == '<STR_LIT>' : <EOL> raise RuntimeError ( <EOL> '<STR_LIT>' ) <EOL> sdk_base = os . path . normpath ( os . path . join ( <EOL> google . appengine . __file__ , '<STR_LIT:..>' , '<STR_LIT:..>' , '<STR_LIT:..>' ) ) <EOL> gopath = os . environ . get ( '<STR_LIT>' ) <EOL> if not gopath : <EOL> gopath = os . path . join ( sdk_base , '<STR_LIT>' ) <EOL> goroot = os . path . join ( sdk_base , '<STR_LIT>' ) <EOL> if not os . path . exists ( goroot ) : <EOL> goroot = None <EOL> gab = os . path . join ( sdk_base , GO_APP_BUILDER ) <EOL> if os . path . exists ( gab ) : <EOL> app_paths = list ( paths ) <EOL> go_files = [ f for f in app_paths <EOL> if f . endswith ( '<STR_LIT>' ) and not appyaml . nobuild_files . match ( f ) ] <EOL> if not go_files : <EOL> raise RuntimeError ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> gab_argv = [ <EOL> gab , <EOL> '<STR_LIT>' , appyaml . api_version , <EOL> '<STR_LIT>' , self . basepath , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , gopath , <EOL> '<STR_LIT>' , <EOL> ] <EOL> if goroot : <EOL> gab_argv . extend ( [ '<STR_LIT>' , goroot ] ) <EOL> gab_argv . extend ( go_files ) <EOL> env = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } <EOL> logging . info ( '<STR_LIT>' , '<STR_LIT:U+0020>' . join ( gab_argv ) ) <EOL> try : <EOL> p = subprocess . Popen ( gab_argv , stdout = subprocess . PIPE , <EOL> stderr = subprocess . PIPE , env = env ) <EOL> ( stdout , stderr ) = p . communicate ( ) <EOL> except Exception , e : <EOL> raise RuntimeError ( '<STR_LIT>' , e ) <EOL> if p . returncode != <NUM_LIT:0> : <EOL> raise RuntimeError ( stderr ) <EOL> overlay = dict ( [ l . split ( '<STR_LIT:|>' ) for l in stdout . split ( '<STR_LIT:\n>' ) if l ] ) <EOL> logging . info ( '<STR_LIT>' , overlay ) <EOL> def Open ( path ) : <EOL> if path in overlay : <EOL> return self . opener ( overlay [ path ] , '<STR_LIT:rb>' ) <EOL> return self . opener ( os . path . join ( basepath , path ) , '<STR_LIT:rb>' ) <EOL> paths = app_paths + overlay . keys ( ) <EOL> openfunc = Open <EOL> appversion = AppVersionUpload ( <EOL> rpcserver , <EOL> appyaml , <EOL> module_yaml_path = module_yaml_path , <EOL> backend = backend , <EOL> error_fh = self . error_fh , <EOL> usage_reporting = self . options . usage_reporting , <EOL> ignore_endpoints_failures = self . options . ignore_endpoints_failures ) <EOL> return appversion . DoUpload ( paths , openfunc ) <EOL> def UpdateUsingSpecificFiles ( self ) : <EOL> """<STR_LIT>""" <EOL> rpcserver = self . _GetRpcServer ( ) <EOL> all_files = [ self . basepath ] + self . args <EOL> has_python25_version = False <EOL> for yaml_path in all_files : <EOL> file_name = os . path . basename ( yaml_path ) <EOL> self . basepath = os . path . dirname ( yaml_path ) <EOL> if not self . basepath : <EOL> self . basepath = '<STR_LIT:.>' <EOL> module_yaml = self . _ParseAppInfoFromYaml ( self . basepath , <EOL> os . path . splitext ( file_name ) [ <NUM_LIT:0> ] ) <EOL> if module_yaml . runtime == '<STR_LIT>' : <EOL> has_python25_version = True <EOL> if not module_yaml . module and file_name != '<STR_LIT>' : <EOL> ErrorUpdate ( "<STR_LIT>" % <EOL> yaml_path ) <EOL> continue <EOL> self . UpdateVersion ( rpcserver , self . basepath , module_yaml , file_name ) <EOL> if has_python25_version : <EOL> MigratePython27Notice ( ) <EOL> def Update ( self ) : <EOL> """<STR_LIT>""" <EOL> if not os . path . isdir ( self . basepath ) : <EOL> self . UpdateUsingSpecificFiles ( ) <EOL> return <EOL> if JavaSupported ( ) and appcfg_java . IsWarFileWithoutYaml ( self . basepath ) : <EOL> java_app_update = appcfg_java . JavaAppUpdate ( self . basepath , self . options ) <EOL> self . options . compile_jsps = not java_app_update . app_engine_web_xml . vm <EOL> sdk_root = os . path . dirname ( appcfg_java . __file__ ) <EOL> self . stage_dir = java_app_update . CreateStagingDirectory ( sdk_root ) <EOL> try : <EOL> appyaml = self . _ParseAppInfoFromYaml ( <EOL> self . stage_dir , <EOL> basename = os . path . splitext ( APP_YAML_FILENAME ) [ <NUM_LIT:0> ] ) <EOL> self . _UpdateWithParsedAppYaml ( appyaml , self . stage_dir ) <EOL> finally : <EOL> if self . options . retain_upload_dir : <EOL> StatusUpdate ( <EOL> '<STR_LIT>' % self . stage_dir , <EOL> self . error_fh ) <EOL> else : <EOL> shutil . rmtree ( self . stage_dir ) <EOL> else : <EOL> appyaml = self . _ParseAppInfoFromYaml ( <EOL> self . basepath , <EOL> basename = os . path . splitext ( APP_YAML_FILENAME ) [ <NUM_LIT:0> ] ) <EOL> self . _UpdateWithParsedAppYaml ( appyaml , self . basepath ) <EOL> def _UpdateWithParsedAppYaml ( self , appyaml , basepath ) : <EOL> """<STR_LIT>""" <EOL> self . runtime = appyaml . runtime <EOL> rpcserver = self . _GetRpcServer ( ) <EOL> if self . options . skip_sdk_update_check : <EOL> logging . info ( '<STR_LIT>' ) <EOL> else : <EOL> updatecheck = self . update_check_class ( rpcserver , appyaml ) <EOL> updatecheck . CheckForUpdates ( ) <EOL> def _AbortAppMismatch ( yaml_name ) : <EOL> StatusUpdate ( '<STR_LIT>' <EOL> '<STR_LIT>' % yaml_name , self . error_fh ) <EOL> dos_yaml = self . _ParseDosYaml ( basepath , appyaml ) <EOL> if dos_yaml and dos_yaml . application != appyaml . application : <EOL> _AbortAppMismatch ( '<STR_LIT>' ) <EOL> return <EOL> queue_yaml = self . _ParseQueueYaml ( basepath , appyaml ) <EOL> if queue_yaml and queue_yaml . application != appyaml . application : <EOL> _AbortAppMismatch ( '<STR_LIT>' ) <EOL> return <EOL> cron_yaml = self . _ParseCronYaml ( basepath , appyaml ) <EOL> if cron_yaml and cron_yaml . application != appyaml . application : <EOL> _AbortAppMismatch ( '<STR_LIT>' ) <EOL> return <EOL> index_defs = self . _ParseIndexYaml ( basepath , appyaml ) <EOL> if index_defs and index_defs . application != appyaml . application : <EOL> _AbortAppMismatch ( '<STR_LIT>' ) <EOL> return <EOL> dispatch_yaml = self . _ParseDispatchYaml ( basepath , appyaml ) <EOL> if dispatch_yaml and dispatch_yaml . application != appyaml . application : <EOL> _AbortAppMismatch ( '<STR_LIT>' ) <EOL> return <EOL> self . UpdateVersion ( rpcserver , basepath , appyaml , APP_YAML_FILENAME ) <EOL> if appyaml . runtime == '<STR_LIT>' : <EOL> MigratePython27Notice ( ) <EOL> if self . options . backends : <EOL> self . BackendsUpdate ( ) <EOL> if index_defs : <EOL> index_upload = IndexDefinitionUpload ( rpcserver , index_defs , self . error_fh ) <EOL> try : <EOL> index_upload . DoUpload ( ) <EOL> except urllib2 . HTTPError , e : <EOL> ErrorUpdate ( '<STR_LIT>' <EOL> '<STR_LIT>' % <EOL> ( e . code , e . read ( ) . rstrip ( '<STR_LIT:\n>' ) ) ) <EOL> print >> self . error_fh , ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> if cron_yaml : <EOL> cron_upload = CronEntryUpload ( rpcserver , cron_yaml , self . error_fh ) <EOL> try : <EOL> cron_upload . DoUpload ( ) <EOL> except urllib2 . HTTPError , e : <EOL> ErrorUpdate ( '<STR_LIT>' <EOL> '<STR_LIT>' % <EOL> ( e . code , e . read ( ) . rstrip ( '<STR_LIT:\n>' ) ) ) <EOL> print >> self . error_fh , ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> if queue_yaml : <EOL> queue_upload = QueueEntryUpload ( rpcserver , queue_yaml , self . error_fh ) <EOL> try : <EOL> queue_upload . DoUpload ( ) <EOL> except urllib2 . HTTPError , e : <EOL> ErrorUpdate ( '<STR_LIT>' <EOL> '<STR_LIT>' % <EOL> ( e . code , e . read ( ) . rstrip ( '<STR_LIT:\n>' ) ) ) <EOL> print >> self . error_fh , ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> if dos_yaml : <EOL> dos_upload = DosEntryUpload ( rpcserver , dos_yaml , self . error_fh ) <EOL> dos_upload . DoUpload ( ) <EOL> if dispatch_yaml : <EOL> dispatch_upload = DispatchEntryUpload ( rpcserver , <EOL> dispatch_yaml , <EOL> self . error_fh ) <EOL> dispatch_upload . DoUpload ( ) <EOL> def _UpdateOptions ( self , parser ) : <EOL> """<STR_LIT>""" <EOL> parser . add_option ( '<STR_LIT>' , action = '<STR_LIT>' , <EOL> dest = '<STR_LIT>' , default = True , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , action = '<STR_LIT:store_true>' , <EOL> dest = '<STR_LIT>' , default = False , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , action = '<STR_LIT>' , <EOL> dest = '<STR_LIT>' , default = True , <EOL> help = '<STR_LIT>' ) <EOL> if JavaSupported ( ) : <EOL> appcfg_java . AddUpdateOptions ( parser ) <EOL> def VacuumIndexes ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . args : <EOL> self . parser . error ( '<STR_LIT>' ) <EOL> index_defs = self . _ParseIndexYaml ( self . basepath ) <EOL> if index_defs is None : <EOL> index_defs = datastore_index . IndexDefinitions ( ) <EOL> rpcserver = self . _GetRpcServer ( ) <EOL> vacuum = VacuumIndexesOperation ( rpcserver , <EOL> self . options . force_delete ) <EOL> vacuum . DoVacuum ( index_defs ) <EOL> def _VacuumIndexesOptions ( self , parser ) : <EOL> """<STR_LIT>""" <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , action = '<STR_LIT:store_true>' , dest = '<STR_LIT>' , <EOL> default = False , <EOL> help = '<STR_LIT>' ) <EOL> def UpdateCron ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . args : <EOL> self . parser . error ( '<STR_LIT>' ) <EOL> rpcserver = self . _GetRpcServer ( ) <EOL> cron_yaml = self . _ParseCronYaml ( self . basepath ) <EOL> if cron_yaml : <EOL> cron_upload = CronEntryUpload ( rpcserver , cron_yaml , self . error_fh ) <EOL> cron_upload . DoUpload ( ) <EOL> else : <EOL> print >> self . error_fh , ( <EOL> '<STR_LIT>' ) <EOL> def UpdateIndexes ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . args : <EOL> self . parser . error ( '<STR_LIT>' ) <EOL> rpcserver = self . _GetRpcServer ( ) <EOL> index_defs = self . _ParseIndexYaml ( self . basepath ) <EOL> if index_defs : <EOL> index_upload = IndexDefinitionUpload ( rpcserver , index_defs , self . error_fh ) <EOL> index_upload . DoUpload ( ) <EOL> else : <EOL> print >> self . error_fh , ( <EOL> '<STR_LIT>' ) <EOL> def UpdateQueues ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . args : <EOL> self . parser . error ( '<STR_LIT>' ) <EOL> rpcserver = self . _GetRpcServer ( ) <EOL> queue_yaml = self . _ParseQueueYaml ( self . basepath ) <EOL> if queue_yaml : <EOL> queue_upload = QueueEntryUpload ( rpcserver , queue_yaml , self . error_fh ) <EOL> queue_upload . DoUpload ( ) <EOL> else : <EOL> print >> self . error_fh , ( <EOL> '<STR_LIT>' ) <EOL> def UpdateDispatch ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . args : <EOL> self . parser . error ( '<STR_LIT>' ) <EOL> rpcserver = self . _GetRpcServer ( ) <EOL> dispatch_yaml = self . _ParseDispatchYaml ( self . basepath ) <EOL> if dispatch_yaml : <EOL> dispatch_upload = DispatchEntryUpload ( rpcserver , <EOL> dispatch_yaml , <EOL> self . error_fh ) <EOL> dispatch_upload . DoUpload ( ) <EOL> else : <EOL> print >> self . error_fh , ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> def UpdateDos ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . args : <EOL> self . parser . error ( '<STR_LIT>' ) <EOL> rpcserver = self . _GetRpcServer ( ) <EOL> dos_yaml = self . _ParseDosYaml ( self . basepath ) <EOL> if dos_yaml : <EOL> dos_upload = DosEntryUpload ( rpcserver , dos_yaml , self . error_fh ) <EOL> dos_upload . DoUpload ( ) <EOL> else : <EOL> print >> self . error_fh , ( <EOL> '<STR_LIT>' ) <EOL> def BackendsAction ( self ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> def BackendsPhpCheck ( self , appyaml ) : <EOL> """<STR_LIT>""" <EOL> if appyaml . runtime . startswith ( '<STR_LIT>' ) : <EOL> _PrintErrorAndExit ( <EOL> self . error_fh , <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> def BackendsYamlCheck ( self , basepath , appyaml , backend = None ) : <EOL> """<STR_LIT>""" <EOL> if appyaml . backends : <EOL> self . parser . error ( '<STR_LIT>' ) <EOL> backends_yaml = self . _ParseBackendsYaml ( basepath ) <EOL> appyaml . backends = backends_yaml . backends <EOL> if not appyaml . backends : <EOL> self . parser . error ( '<STR_LIT>' ) <EOL> backends = [ ] <EOL> for backend_entry in appyaml . backends : <EOL> entry = backendinfo . LoadBackendEntry ( backend_entry . ToYAML ( ) ) <EOL> if entry . name in backends : <EOL> self . parser . error ( '<STR_LIT>' % entry . name ) <EOL> else : <EOL> backends . append ( entry . name ) <EOL> backends_to_update = [ ] <EOL> if backend : <EOL> if backend in backends : <EOL> backends_to_update = [ backend ] <EOL> else : <EOL> self . parser . error ( "<STR_LIT>" % <EOL> backend ) <EOL> else : <EOL> backends_to_update = backends <EOL> return backends_to_update <EOL> def BackendsUpdate ( self ) : <EOL> """<STR_LIT>""" <EOL> self . backend = None <EOL> if len ( self . args ) == <NUM_LIT:1> : <EOL> self . backend = self . args [ <NUM_LIT:0> ] <EOL> elif len ( self . args ) > <NUM_LIT:1> : <EOL> self . parser . error ( '<STR_LIT>' ) <EOL> if JavaSupported ( ) and appcfg_java . IsWarFileWithoutYaml ( self . basepath ) : <EOL> java_app_update = appcfg_java . JavaAppUpdate ( self . basepath , self . options ) <EOL> self . options . compile_jsps = True <EOL> sdk_root = os . path . dirname ( appcfg_java . __file__ ) <EOL> basepath = java_app_update . CreateStagingDirectory ( sdk_root ) <EOL> else : <EOL> basepath = self . basepath <EOL> yaml_file_basename = '<STR_LIT>' <EOL> appyaml = self . _ParseAppInfoFromYaml ( basepath , <EOL> basename = yaml_file_basename ) <EOL> BackendsStatusUpdate ( appyaml . runtime , self . error_fh ) <EOL> self . BackendsPhpCheck ( appyaml ) <EOL> rpcserver = self . _GetRpcServer ( ) <EOL> backends_to_update = self . BackendsYamlCheck ( basepath , appyaml , self . backend ) <EOL> for backend in backends_to_update : <EOL> self . UpdateVersion ( rpcserver , basepath , appyaml , yaml_file_basename , <EOL> backend = backend ) <EOL> def BackendsList ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . args : <EOL> self . parser . error ( '<STR_LIT>' ) <EOL> appyaml = self . _ParseAppInfoFromYaml ( self . basepath ) <EOL> BackendsStatusUpdate ( appyaml . runtime , self . error_fh ) <EOL> rpcserver = self . _GetRpcServer ( ) <EOL> response = rpcserver . Send ( '<STR_LIT>' , app_id = appyaml . application ) <EOL> print >> self . out_fh , response <EOL> def BackendsRollback ( self ) : <EOL> """<STR_LIT>""" <EOL> if len ( self . args ) != <NUM_LIT:1> : <EOL> self . parser . error ( '<STR_LIT>' ) <EOL> self . _Rollback ( self . args [ <NUM_LIT:0> ] ) <EOL> def BackendsStart ( self ) : <EOL> """<STR_LIT>""" <EOL> if len ( self . args ) != <NUM_LIT:1> : <EOL> self . parser . error ( '<STR_LIT>' ) <EOL> backend = self . args [ <NUM_LIT:0> ] <EOL> appyaml = self . _ParseAppInfoFromYaml ( self . basepath ) <EOL> BackendsStatusUpdate ( appyaml . runtime , self . error_fh ) <EOL> self . BackendsPhpCheck ( appyaml ) <EOL> rpcserver = self . _GetRpcServer ( ) <EOL> response = rpcserver . Send ( '<STR_LIT>' , <EOL> app_id = appyaml . application , <EOL> backend = backend ) <EOL> print >> self . out_fh , response <EOL> def BackendsStop ( self ) : <EOL> """<STR_LIT>""" <EOL> if len ( self . args ) != <NUM_LIT:1> : <EOL> self . parser . error ( '<STR_LIT>' ) <EOL> backend = self . args [ <NUM_LIT:0> ] <EOL> appyaml = self . _ParseAppInfoFromYaml ( self . basepath ) <EOL> BackendsStatusUpdate ( appyaml . runtime , self . error_fh ) <EOL> rpcserver = self . _GetRpcServer ( ) <EOL> response = rpcserver . Send ( '<STR_LIT>' , <EOL> app_id = appyaml . application , <EOL> backend = backend ) <EOL> print >> self . out_fh , response <EOL> def BackendsDelete ( self ) : <EOL> """<STR_LIT>""" <EOL> if len ( self . args ) != <NUM_LIT:1> : <EOL> self . parser . error ( '<STR_LIT>' ) <EOL> backend = self . args [ <NUM_LIT:0> ] <EOL> appyaml = self . _ParseAppInfoFromYaml ( self . basepath ) <EOL> BackendsStatusUpdate ( appyaml . runtime , self . error_fh ) <EOL> rpcserver = self . _GetRpcServer ( ) <EOL> response = rpcserver . Send ( '<STR_LIT>' , <EOL> app_id = appyaml . application , <EOL> backend = backend ) <EOL> print >> self . out_fh , response <EOL> def BackendsConfigure ( self ) : <EOL> """<STR_LIT>""" <EOL> if len ( self . args ) != <NUM_LIT:1> : <EOL> self . parser . error ( '<STR_LIT>' ) <EOL> backend = self . args [ <NUM_LIT:0> ] <EOL> appyaml = self . _ParseAppInfoFromYaml ( self . basepath ) <EOL> BackendsStatusUpdate ( appyaml . runtime , self . error_fh ) <EOL> self . BackendsPhpCheck ( appyaml ) <EOL> backends_yaml = self . _ParseBackendsYaml ( self . basepath ) <EOL> rpcserver = self . _GetRpcServer ( ) <EOL> response = rpcserver . Send ( '<STR_LIT>' , <EOL> app_id = appyaml . application , <EOL> backend = backend , <EOL> payload = backends_yaml . ToYAML ( ) ) <EOL> print >> self . out_fh , response <EOL> def ListVersions ( self ) : <EOL> """<STR_LIT>""" <EOL> if len ( self . args ) == <NUM_LIT:0> : <EOL> if not self . options . app_id : <EOL> self . parser . error ( '<STR_LIT>' ) <EOL> app_id = self . options . app_id <EOL> elif len ( self . args ) == <NUM_LIT:1> : <EOL> if self . options . app_id : <EOL> self . parser . error ( '<STR_LIT>' ) <EOL> appyaml = self . _ParseAppInfoFromYaml ( self . args [ <NUM_LIT:0> ] ) <EOL> app_id = appyaml . application <EOL> else : <EOL> self . parser . error ( '<STR_LIT>' % len ( self . args ) ) <EOL> rpcserver = self . _GetRpcServer ( ) <EOL> response = rpcserver . Send ( '<STR_LIT>' , app_id = app_id ) <EOL> parsed_response = yaml . safe_load ( response ) <EOL> if not parsed_response : <EOL> print >> self . out_fh , ( '<STR_LIT>' % app_id ) <EOL> else : <EOL> print >> self . out_fh , response <EOL> def DeleteVersion ( self ) : <EOL> """<STR_LIT>""" <EOL> if not ( self . options . app_id and self . options . version ) : <EOL> self . parser . error ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> if self . options . module : <EOL> module = self . options . module <EOL> else : <EOL> module = '<STR_LIT>' <EOL> rpcserver = self . _GetRpcServer ( ) <EOL> response = rpcserver . Send ( '<STR_LIT>' , <EOL> app_id = self . options . app_id , <EOL> version_match = self . options . version , <EOL> module = module ) <EOL> print >> self . out_fh , response <EOL> def _LockingAction ( self , url ) : <EOL> """<STR_LIT>""" <EOL> if len ( self . args ) == <NUM_LIT:1> : <EOL> appyaml = self . _ParseAppInfoFromYaml ( self . args [ <NUM_LIT:0> ] ) <EOL> app_id = appyaml . application <EOL> module = appyaml . module or '<STR_LIT>' <EOL> version = appyaml . version <EOL> elif not self . args : <EOL> if not ( self . options . app_id and self . options . version ) : <EOL> self . parser . error ( <EOL> ( '<STR_LIT>' <EOL> '<STR_LIT>' ) ) <EOL> module = '<STR_LIT>' <EOL> else : <EOL> self . _PrintHelpAndExit ( ) <EOL> if self . options . app_id : <EOL> app_id = self . options . app_id <EOL> if self . options . module : <EOL> module = self . options . module <EOL> if self . options . version : <EOL> version = self . options . version <EOL> rpcserver = self . _GetRpcServer ( ) <EOL> DoLockAction ( <EOL> url , <EOL> rpcserver , <EOL> app_id , version , module , <EOL> self . options . instance , <EOL> self . out_fh ) . Do ( ) <EOL> def DebugAction ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _LockingAction ( '<STR_LIT>' ) <EOL> def LockAction ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _LockingAction ( '<STR_LIT>' ) <EOL> def _LockActionOptions ( self , parser ) : <EOL> """<STR_LIT>""" <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , type = '<STR_LIT:string>' , dest = '<STR_LIT>' , <EOL> help = '<STR_LIT>' ) <EOL> def PrepareVmRuntimeAction ( self ) : <EOL> """<STR_LIT>""" <EOL> if not self . options . app_id : <EOL> self . parser . error ( '<STR_LIT>' ) <EOL> rpcserver = self . _GetRpcServer ( ) <EOL> response = rpcserver . Send ( '<STR_LIT>' , <EOL> app_id = self . options . app_id ) <EOL> print >> self . out_fh , response <EOL> def _ParseAndValidateModuleYamls ( self , yaml_paths ) : <EOL> """<STR_LIT>""" <EOL> results = [ ] <EOL> app_id = None <EOL> last_yaml_path = None <EOL> for yaml_path in yaml_paths : <EOL> if not os . path . isfile ( yaml_path ) : <EOL> _PrintErrorAndExit ( <EOL> self . error_fh , <EOL> ( "<STR_LIT>" <EOL> "<STR_LIT>" ) % yaml_path ) <EOL> file_name = os . path . basename ( yaml_path ) <EOL> base_path = os . path . dirname ( yaml_path ) <EOL> if not base_path : <EOL> base_path = '<STR_LIT:.>' <EOL> module_yaml = self . _ParseAppInfoFromYaml ( base_path , <EOL> os . path . splitext ( file_name ) [ <NUM_LIT:0> ] ) <EOL> if not module_yaml . module and file_name != '<STR_LIT>' : <EOL> _PrintErrorAndExit ( <EOL> self . error_fh , <EOL> "<STR_LIT>" % yaml_path ) <EOL> if app_id is not None and module_yaml . application != app_id : <EOL> _PrintErrorAndExit ( <EOL> self . error_fh , <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" % ( module_yaml . application , <EOL> yaml_path , <EOL> app_id , <EOL> last_yaml_path ) ) <EOL> app_id = module_yaml . application <EOL> last_yaml_path = yaml_path <EOL> results . append ( module_yaml ) <EOL> return results <EOL> def _ModuleAction ( self , action_path ) : <EOL> """<STR_LIT>""" <EOL> modules_to_process = [ ] <EOL> if not self . args : <EOL> if not ( self . options . app_id and <EOL> self . options . module and <EOL> self . options . version ) : <EOL> _PrintErrorAndExit ( self . error_fh , <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> else : <EOL> modules_to_process . append ( ( self . options . app_id , <EOL> self . options . module , <EOL> self . options . version ) ) <EOL> else : <EOL> if self . options . module : <EOL> _PrintErrorAndExit ( self . error_fh , <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> module_yamls = self . _ParseAndValidateModuleYamls ( self . args ) <EOL> for serv_yaml in module_yamls : <EOL> app_id = serv_yaml . application <EOL> modules_to_process . append ( ( self . options . app_id or serv_yaml . application , <EOL> serv_yaml . module or appinfo . DEFAULT_MODULE , <EOL> self . options . version or serv_yaml . version ) ) <EOL> rpcserver = self . _GetRpcServer ( ) <EOL> for app_id , module , version in modules_to_process : <EOL> response = rpcserver . Send ( action_path , <EOL> app_id = app_id , <EOL> module = module , <EOL> version = version ) <EOL> print >> self . out_fh , response <EOL> def StartModuleVersion ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _ModuleAction ( '<STR_LIT>' ) <EOL> def StopModuleVersion ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _ModuleAction ( '<STR_LIT>' ) <EOL> def Rollback ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _Rollback ( ) <EOL> def _RollbackOptions ( self , parser ) : <EOL> """<STR_LIT>""" <EOL> parser . add_option ( '<STR_LIT>' , action = '<STR_LIT:store_true>' , <EOL> dest = '<STR_LIT>' , default = False , <EOL> help = '<STR_LIT>' ) <EOL> def _Rollback ( self , backend = None ) : <EOL> """<STR_LIT>""" <EOL> if os . path . isdir ( self . basepath ) : <EOL> module_yaml = self . _ParseAppInfoFromYaml ( self . basepath ) <EOL> else : <EOL> file_name = os . path . basename ( self . basepath ) <EOL> self . basepath = os . path . dirname ( self . basepath ) <EOL> if not self . basepath : <EOL> self . basepath = '<STR_LIT:.>' <EOL> module_yaml = self . _ParseAppInfoFromYaml ( self . basepath , <EOL> os . path . splitext ( file_name ) [ <NUM_LIT:0> ] ) <EOL> appversion = AppVersionUpload ( self . _GetRpcServer ( ) , module_yaml , <EOL> module_yaml_path = '<STR_LIT>' , <EOL> backend = backend ) <EOL> appversion . in_transaction = True <EOL> force_rollback = False <EOL> if hasattr ( self . options , '<STR_LIT>' ) : <EOL> force_rollback = self . options . force_rollback <EOL> appversion . Rollback ( force_rollback ) <EOL> def SetDefaultVersion ( self ) : <EOL> """<STR_LIT>""" <EOL> module = '<STR_LIT>' <EOL> if len ( self . args ) == <NUM_LIT:1> : <EOL> stored_modules = self . options . module <EOL> self . options . module = None <EOL> try : <EOL> appyaml = self . _ParseAppInfoFromYaml ( self . args [ <NUM_LIT:0> ] ) <EOL> finally : <EOL> self . options . module = stored_modules <EOL> app_id = appyaml . application <EOL> module = appyaml . module or '<STR_LIT>' <EOL> version = appyaml . version <EOL> elif not self . args : <EOL> if not ( self . options . app_id and self . options . version ) : <EOL> self . parser . error ( <EOL> ( '<STR_LIT>' <EOL> '<STR_LIT>' ) ) <EOL> else : <EOL> self . _PrintHelpAndExit ( ) <EOL> if self . options . app_id : <EOL> app_id = self . options . app_id <EOL> if self . options . module : <EOL> module = self . options . module <EOL> if self . options . version : <EOL> version = self . options . version <EOL> version_setter = DefaultVersionSet ( self . _GetRpcServer ( ) , <EOL> app_id , <EOL> module , <EOL> version , <EOL> self . error_fh ) <EOL> version_setter . SetVersion ( ) <EOL> def MigrateTraffic ( self ) : <EOL> """<STR_LIT>""" <EOL> module = '<STR_LIT:default>' <EOL> if len ( self . args ) == <NUM_LIT:1> : <EOL> appyaml = self . _ParseAppInfoFromYaml ( self . args [ <NUM_LIT:0> ] ) <EOL> app_id = appyaml . application <EOL> module = appyaml . module or '<STR_LIT:default>' <EOL> version = appyaml . version <EOL> elif not self . args : <EOL> if not ( self . options . app_id and self . options . version ) : <EOL> self . parser . error ( <EOL> ( '<STR_LIT>' <EOL> '<STR_LIT>' ) ) <EOL> else : <EOL> self . _PrintHelpAndExit ( ) <EOL> if self . options . app_id : <EOL> app_id = self . options . app_id <EOL> if self . options . module : <EOL> module = self . options . module <EOL> if self . options . version : <EOL> version = self . options . version <EOL> if module not in [ '<STR_LIT>' , '<STR_LIT:default>' ] : <EOL> StatusUpdate ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> return <EOL> traffic_migrator = TrafficMigrator ( <EOL> self . _GetRpcServer ( ) , app_id , version , self . error_fh ) <EOL> traffic_migrator . MigrateTraffic ( ) <EOL> def RequestLogs ( self ) : <EOL> """<STR_LIT>""" <EOL> args_length = len ( self . args ) <EOL> module = '<STR_LIT>' <EOL> if args_length == <NUM_LIT:2> : <EOL> appyaml = self . _ParseAppInfoFromYaml ( self . args . pop ( <NUM_LIT:0> ) ) <EOL> app_id = appyaml . application <EOL> module = appyaml . module or '<STR_LIT>' <EOL> version = appyaml . version <EOL> elif args_length == <NUM_LIT:1> : <EOL> if not ( self . options . app_id and self . options . version ) : <EOL> self . parser . error ( <EOL> ( '<STR_LIT>' <EOL> '<STR_LIT>' ) ) <EOL> else : <EOL> self . _PrintHelpAndExit ( ) <EOL> if self . options . app_id : <EOL> app_id = self . options . app_id <EOL> if self . options . module : <EOL> module = self . options . module <EOL> if self . options . version : <EOL> version = self . options . version <EOL> if ( self . options . severity is not None and <EOL> not <NUM_LIT:0> <= self . options . severity <= MAX_LOG_LEVEL ) : <EOL> self . parser . error ( <EOL> '<STR_LIT>' % MAX_LOG_LEVEL ) <EOL> if self . options . num_days is None : <EOL> self . options . num_days = int ( not self . options . append ) <EOL> try : <EOL> end_date = self . _ParseEndDate ( self . options . end_date ) <EOL> except ( TypeError , ValueError ) : <EOL> self . parser . error ( '<STR_LIT>' ) <EOL> rpcserver = self . _GetRpcServer ( ) <EOL> logs_requester = LogsRequester ( rpcserver , <EOL> app_id , <EOL> module , <EOL> version , <EOL> self . args [ <NUM_LIT:0> ] , <EOL> self . options . num_days , <EOL> self . options . append , <EOL> self . options . severity , <EOL> end_date , <EOL> self . options . vhost , <EOL> self . options . include_vhost , <EOL> self . options . include_all , <EOL> time_func = self . time_func ) <EOL> logs_requester . DownloadLogs ( ) <EOL> @ staticmethod <EOL> def _ParseEndDate ( date , time_func = time . time ) : <EOL> """<STR_LIT>""" <EOL> if not date : <EOL> return PacificDate ( time_func ( ) ) <EOL> return datetime . date ( * [ int ( i ) for i in date . split ( '<STR_LIT:->' ) ] ) <EOL> def _RequestLogsOptions ( self , parser ) : <EOL> """<STR_LIT>""" <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , type = '<STR_LIT:int>' , dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store>' , default = None , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store_true>' , default = False , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , type = '<STR_LIT:int>' , dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store>' , default = None , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , type = '<STR_LIT:string>' , dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store>' , default = None , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store_true>' , default = False , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store_true>' , default = None , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store>' , default = '<STR_LIT>' , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> def CronInfo ( self , now = None , output = sys . stdout ) : <EOL> """<STR_LIT>""" <EOL> if self . args : <EOL> self . parser . error ( '<STR_LIT>' ) <EOL> if now is None : <EOL> now = datetime . datetime . utcnow ( ) <EOL> cron_yaml = self . _ParseCronYaml ( self . basepath ) <EOL> if cron_yaml and cron_yaml . cron : <EOL> for entry in cron_yaml . cron : <EOL> description = entry . description <EOL> if not description : <EOL> description = '<STR_LIT>' <EOL> if not entry . timezone : <EOL> entry . timezone = '<STR_LIT>' <EOL> print >> output , '<STR_LIT>' % ( description , <EOL> entry . url , <EOL> entry . schedule , <EOL> entry . timezone ) <EOL> if entry . timezone != '<STR_LIT>' : <EOL> print >> output , ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> schedule = groctimespecification . GrocTimeSpecification ( entry . schedule ) <EOL> matches = schedule . GetMatches ( now , self . options . num_runs ) <EOL> for match in matches : <EOL> print >> output , '<STR_LIT>' % ( <EOL> match . strftime ( '<STR_LIT>' ) , match - now ) <EOL> def _CronInfoOptions ( self , parser ) : <EOL> """<STR_LIT>""" <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , type = '<STR_LIT:int>' , dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store>' , default = <NUM_LIT:5> , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> def _CheckRequiredLoadOptions ( self ) : <EOL> """<STR_LIT>""" <EOL> for option in [ '<STR_LIT:filename>' ] : <EOL> if getattr ( self . options , option ) is None : <EOL> self . parser . error ( '<STR_LIT>' % option ) <EOL> if not self . options . url : <EOL> self . parser . error ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> def InferRemoteApiUrl ( self , appyaml ) : <EOL> """<STR_LIT>""" <EOL> handlers = appyaml . handlers <EOL> handler_suffixes = [ '<STR_LIT>' , <EOL> '<STR_LIT>' ] <EOL> app_id = appyaml . application <EOL> for handler in handlers : <EOL> if hasattr ( handler , '<STR_LIT>' ) and handler . script : <EOL> if any ( handler . script . endswith ( suffix ) for suffix in handler_suffixes ) : <EOL> server = self . options . server <EOL> url = handler . url <EOL> if url . endswith ( '<STR_LIT>' ) : <EOL> url = url [ : - <NUM_LIT:6> ] <EOL> if server == '<STR_LIT>' : <EOL> return '<STR_LIT>' % ( app_id , url ) <EOL> else : <EOL> match = re . match ( PREFIXED_BY_ADMIN_CONSOLE_RE , server ) <EOL> if match : <EOL> return '<STR_LIT>' % ( app_id , match . group ( <NUM_LIT:1> ) , url ) <EOL> else : <EOL> return '<STR_LIT>' % ( server , url ) <EOL> return None <EOL> def RunBulkloader ( self , arg_dict ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> import sqlite3 <EOL> except ImportError : <EOL> logging . error ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> sys . exit ( bulkloader . Run ( arg_dict , self . _GetOAuth2Parameters ( ) ) ) <EOL> def _SetupLoad ( self ) : <EOL> """<STR_LIT>""" <EOL> if len ( self . args ) != <NUM_LIT:1> and not self . options . url : <EOL> self . parser . error ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> if len ( self . args ) == <NUM_LIT:1> : <EOL> self . basepath = self . args [ <NUM_LIT:0> ] <EOL> appyaml = self . _ParseAppInfoFromYaml ( self . basepath ) <EOL> self . options . app_id = appyaml . application <EOL> if not self . options . url : <EOL> url = self . InferRemoteApiUrl ( appyaml ) <EOL> if url is not None : <EOL> self . options . url = url <EOL> self . _CheckRequiredLoadOptions ( ) <EOL> if self . options . batch_size < <NUM_LIT:1> : <EOL> self . parser . error ( '<STR_LIT>' ) <EOL> if verbosity == <NUM_LIT:1> : <EOL> logging . getLogger ( ) . setLevel ( logging . INFO ) <EOL> self . options . debug = False <EOL> else : <EOL> logging . getLogger ( ) . setLevel ( logging . DEBUG ) <EOL> self . options . debug = True <EOL> def _MakeLoaderArgs ( self ) : <EOL> """<STR_LIT>""" <EOL> args = dict ( [ ( arg_name , getattr ( self . options , arg_name , None ) ) for <EOL> arg_name in ( <EOL> '<STR_LIT:url>' , <EOL> '<STR_LIT:filename>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> ) ] ) <EOL> args [ '<STR_LIT>' ] = self . options . app_id <EOL> args [ '<STR_LIT>' ] = self . throttle_class <EOL> return args <EOL> def PerformDownload ( self , run_fn = None ) : <EOL> """<STR_LIT>""" <EOL> if run_fn is None : <EOL> run_fn = self . RunBulkloader <EOL> self . _SetupLoad ( ) <EOL> StatusUpdate ( '<STR_LIT>' , self . error_fh ) <EOL> args = self . _MakeLoaderArgs ( ) <EOL> args [ '<STR_LIT>' ] = bool ( args [ '<STR_LIT>' ] ) <EOL> args [ '<STR_LIT>' ] = False <EOL> args [ '<STR_LIT>' ] = False <EOL> args [ '<STR_LIT>' ] = not args [ '<STR_LIT>' ] <EOL> args [ '<STR_LIT>' ] = False <EOL> args [ '<STR_LIT>' ] = False <EOL> run_fn ( args ) <EOL> def PerformUpload ( self , run_fn = None ) : <EOL> """<STR_LIT>""" <EOL> if run_fn is None : <EOL> run_fn = self . RunBulkloader <EOL> self . _SetupLoad ( ) <EOL> StatusUpdate ( '<STR_LIT>' , self . error_fh ) <EOL> args = self . _MakeLoaderArgs ( ) <EOL> args [ '<STR_LIT>' ] = False <EOL> args [ '<STR_LIT>' ] = False <EOL> args [ '<STR_LIT>' ] = False <EOL> args [ '<STR_LIT>' ] = not args [ '<STR_LIT>' ] <EOL> args [ '<STR_LIT>' ] = False <EOL> run_fn ( args ) <EOL> def CreateBulkloadConfig ( self , run_fn = None ) : <EOL> """<STR_LIT>""" <EOL> if run_fn is None : <EOL> run_fn = self . RunBulkloader <EOL> self . _SetupLoad ( ) <EOL> StatusUpdate ( '<STR_LIT>' , self . error_fh ) <EOL> args = self . _MakeLoaderArgs ( ) <EOL> args [ '<STR_LIT>' ] = False <EOL> args [ '<STR_LIT>' ] = False <EOL> args [ '<STR_LIT>' ] = False <EOL> args [ '<STR_LIT>' ] = False <EOL> args [ '<STR_LIT>' ] = False <EOL> args [ '<STR_LIT>' ] = True <EOL> run_fn ( args ) <EOL> def _PerformLoadOptions ( self , parser ) : <EOL> """<STR_LIT>""" <EOL> parser . add_option ( '<STR_LIT>' , type = '<STR_LIT:string>' , dest = '<STR_LIT:url>' , <EOL> action = '<STR_LIT:store>' , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , type = '<STR_LIT:int>' , dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store>' , default = <NUM_LIT:10> , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , type = '<STR_LIT:int>' , dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store>' , default = <NUM_LIT> , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , type = '<STR_LIT:int>' , dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store>' , default = <NUM_LIT:20> , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , type = '<STR_LIT:int>' , dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store>' , default = <NUM_LIT:8> , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , type = '<STR_LIT:string>' , dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store>' , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , type = '<STR_LIT:string>' , dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store>' , default = '<STR_LIT>' , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , type = '<STR_LIT:string>' , dest = '<STR_LIT>' , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , action = '<STR_LIT:store_true>' , <EOL> dest = '<STR_LIT>' , default = False , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , type = '<STR_LIT:string>' , dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store>' , default = '<STR_LIT>' , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , type = '<STR_LIT:int>' , dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store>' , default = <NUM_LIT:10> , <EOL> help = '<STR_LIT>' ) <EOL> def _PerformUploadOptions ( self , parser ) : <EOL> """<STR_LIT>""" <EOL> self . _PerformLoadOptions ( parser ) <EOL> parser . add_option ( '<STR_LIT>' , type = '<STR_LIT:string>' , dest = '<STR_LIT:filename>' , <EOL> action = '<STR_LIT:store>' , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , type = '<STR_LIT:string>' , dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store>' , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store_true>' , default = False , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , type = '<STR_LIT:string>' , dest = '<STR_LIT>' , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , type = '<STR_LIT:string>' , dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store>' , <EOL> help = '<STR_LIT>' ) <EOL> def _PerformDownloadOptions ( self , parser ) : <EOL> """<STR_LIT>""" <EOL> self . _PerformLoadOptions ( parser ) <EOL> parser . add_option ( '<STR_LIT>' , type = '<STR_LIT:string>' , dest = '<STR_LIT:filename>' , <EOL> action = '<STR_LIT:store>' , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , type = '<STR_LIT:string>' , dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store>' , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , type = '<STR_LIT:string>' , dest = '<STR_LIT>' , <EOL> help = '<STR_LIT>' <EOL> ) <EOL> parser . add_option ( '<STR_LIT>' , type = '<STR_LIT:string>' , <EOL> dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store>' , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , type = '<STR_LIT:string>' , dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store>' , <EOL> help = '<STR_LIT>' ) <EOL> def _CreateBulkloadConfigOptions ( self , parser ) : <EOL> """<STR_LIT>""" <EOL> self . _PerformLoadOptions ( parser ) <EOL> parser . add_option ( '<STR_LIT>' , type = '<STR_LIT:string>' , dest = '<STR_LIT:filename>' , <EOL> action = '<STR_LIT:store>' , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , type = '<STR_LIT:string>' , <EOL> dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store>' , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> def ResourceLimitsInfo ( self , output = None ) : <EOL> """<STR_LIT>""" <EOL> rpcserver = self . _GetRpcServer ( ) <EOL> appyaml = self . _ParseAppInfoFromYaml ( self . basepath ) <EOL> request_params = { '<STR_LIT>' : appyaml . application , '<STR_LIT:version>' : appyaml . version } <EOL> logging_context = _ClientDeployLoggingContext ( rpcserver , request_params , <EOL> usage_reporting = False ) <EOL> resource_limits = GetResourceLimits ( logging_context , self . error_fh ) <EOL> for attr_name in sorted ( resource_limits ) : <EOL> print >> output , '<STR_LIT>' % ( attr_name , resource_limits [ attr_name ] ) <EOL> class Action ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , function , usage , short_desc , long_desc = '<STR_LIT>' , <EOL> error_desc = None , options = lambda obj , parser : None , <EOL> uses_basepath = True , hidden = False ) : <EOL> """<STR_LIT>""" <EOL> self . function = function <EOL> self . usage = usage <EOL> self . short_desc = short_desc <EOL> self . long_desc = long_desc <EOL> self . error_desc = error_desc <EOL> self . options = options <EOL> self . uses_basepath = uses_basepath <EOL> self . hidden = hidden <EOL> def __call__ ( self , appcfg ) : <EOL> """<STR_LIT>""" <EOL> method = getattr ( appcfg , self . function ) <EOL> return method ( ) <EOL> actions = { <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> short_desc = '<STR_LIT>' , <EOL> uses_basepath = False ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> options = _UpdateOptions , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" , <EOL> uses_basepath = False ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" , <EOL> error_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> options = _UpdateOptions , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> options = _VacuumIndexesOptions , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> options = _RollbackOptions , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> options = _RequestLogsOptions , <EOL> uses_basepath = False , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" , <EOL> error_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> options = _CronInfoOptions , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> uses_basepath = False , <EOL> usage = '<STR_LIT>' , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> uses_basepath = False , <EOL> usage = '<STR_LIT>' , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> options = _PerformUploadOptions , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" , <EOL> uses_basepath = False ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> options = _PerformDownloadOptions , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" , <EOL> uses_basepath = False ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> options = _CreateBulkloadConfigOptions , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" , <EOL> uses_basepath = False ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" , <EOL> uses_basepath = False ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" , <EOL> uses_basepath = False , <EOL> hidden = True ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" , <EOL> uses_basepath = False ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> uses_basepath = False , <EOL> short_desc = '<STR_LIT>' , <EOL> long_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> options = _LockActionOptions , <EOL> short_desc = '<STR_LIT>' , <EOL> hidden = True , <EOL> uses_basepath = False , <EOL> long_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> options = _LockActionOptions , <EOL> short_desc = '<STR_LIT>' , <EOL> hidden = True , <EOL> uses_basepath = False , <EOL> long_desc = """<STR_LIT>""" ) , <EOL> '<STR_LIT>' : Action ( <EOL> function = '<STR_LIT>' , <EOL> usage = '<STR_LIT>' , <EOL> short_desc = '<STR_LIT>' , <EOL> hidden = True , <EOL> uses_basepath = False , <EOL> long_desc = """<STR_LIT>""" ) , <EOL> } <EOL> def main ( argv ) : <EOL> logging . basicConfig ( format = ( '<STR_LIT>' <EOL> '<STR_LIT>' ) ) <EOL> try : <EOL> result = AppCfgApp ( argv ) . Run ( ) <EOL> if result : <EOL> sys . exit ( result ) <EOL> except KeyboardInterrupt : <EOL> StatusUpdate ( '<STR_LIT>' ) <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> main ( sys . argv ) </s>
