<s> '''<STR_LIT>''' <EOL> from contextlib import closing <EOL> import os <EOL> import sys <EOL> import urllib <EOL> import tempfile <EOL> import xmlrpclib <EOL> import shutil <EOL> import socket <EOL> from codalab . client import get_address_host <EOL> from codalab . client . bundle_client import BundleClient <EOL> from codalab . common import ( <EOL> NotFoundError , <EOL> PermissionError , <EOL> UsageError , <EOL> AuthorizationError , <EOL> ) <EOL> from codalab . lib import ( <EOL> file_util , <EOL> path_util , <EOL> zip_util , <EOL> ) <EOL> from codalab . server . rpc_file_handle import RPCFileHandle <EOL> from worker . file_util import gzip_file , tar_gzip_directory , un_tar_directory , un_gzip_stream , un_gzip_string <EOL> xmlrpclib . Marshaller . dispatch [ int ] = lambda _ , v , w : w ( "<STR_LIT>" % v ) <EOL> class AuthenticatedTransport ( xmlrpclib . SafeTransport ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , address , get_auth_token ) : <EOL> '''<STR_LIT>''' <EOL> xmlrpclib . SafeTransport . __init__ ( self , use_datetime = <NUM_LIT:0> ) <EOL> url_type , _ = urllib . splittype ( address ) <EOL> if url_type not in ( "<STR_LIT:http>" , "<STR_LIT>" ) : <EOL> raise UsageError ( "<STR_LIT>" % address ) <EOL> self . _url_type = url_type <EOL> self . _bearer_token = get_auth_token <EOL> def send_content ( self , connection , request_body ) : <EOL> '''<STR_LIT>''' <EOL> _ , command = xmlrpclib . loads ( request_body ) <EOL> token = self . _bearer_token ( command ) <EOL> if token is not None and len ( token ) > <NUM_LIT:0> : <EOL> connection . putheader ( "<STR_LIT>" , "<STR_LIT>" . format ( token ) ) <EOL> xmlrpclib . SafeTransport . send_content ( self , connection , request_body ) <EOL> def make_connection ( self , host ) : <EOL> '''<STR_LIT>''' <EOL> if self . _url_type == "<STR_LIT>" : <EOL> return xmlrpclib . SafeTransport . make_connection ( self , host ) <EOL> else : <EOL> return xmlrpclib . Transport . make_connection ( self , host ) <EOL> class RemoteBundleClient ( BundleClient ) : <EOL> CLIENT_COMMANDS = ( <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> ) <EOL> SERVER_COMMANDS = ( <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> ) <EOL> FILE_COMMANDS = ( <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> ) <EOL> COMMANDS = CLIENT_COMMANDS + SERVER_COMMANDS + FILE_COMMANDS <EOL> def __init__ ( self , address , get_auth_token , verbose ) : <EOL> self . address = address <EOL> self . verbose = verbose <EOL> host = get_address_host ( address ) <EOL> transport = AuthenticatedTransport ( host , lambda cmd : None if cmd == '<STR_LIT>' else get_auth_token ( self ) ) <EOL> self . proxy = xmlrpclib . ServerProxy ( host , transport = transport , allow_none = True ) <EOL> def do_command ( command ) : <EOL> def inner ( * args , ** kwargs ) : <EOL> import time <EOL> time_delay = <NUM_LIT:1> <EOL> if self . verbose >= <NUM_LIT:2> : <EOL> print '<STR_LIT>' % ( command , args , kwargs ) <EOL> while True : <EOL> try : <EOL> return getattr ( self . proxy , command ) ( * args , ** kwargs ) <EOL> except xmlrpclib . ProtocolError , e : <EOL> raise UsageError ( "<STR_LIT>" % ( host , e ) ) <EOL> except xmlrpclib . Fault , e : <EOL> if '<STR_LIT>' in e . faultString : <EOL> index = e . faultString . find ( '<STR_LIT::>' ) <EOL> raise UsageError ( e . faultString [ index + <NUM_LIT:1> : ] ) <EOL> if '<STR_LIT>' in e . faultString : <EOL> index = e . faultString . find ( '<STR_LIT::>' ) <EOL> raise NotFoundError ( e . faultString [ index + <NUM_LIT:1> : ] ) <EOL> elif '<STR_LIT>' in e . faultString : <EOL> index = e . faultString . find ( '<STR_LIT::>' ) <EOL> raise PermissionError ( e . faultString [ index + <NUM_LIT:1> : ] ) <EOL> elif '<STR_LIT>' in e . faultString : <EOL> index = e . faultString . find ( '<STR_LIT::>' ) <EOL> raise AuthorizationError ( e . faultString [ index + <NUM_LIT:1> : ] ) <EOL> else : <EOL> raise <EOL> except socket . error , e : <EOL> print >> sys . stderr , "<STR_LIT>" % ( host , e , time_delay ) <EOL> time . sleep ( time_delay ) <EOL> time_delay *= <NUM_LIT:2> <EOL> if time_delay > <NUM_LIT> : <EOL> raise UsageError ( '<STR_LIT>' % ( host , e ) ) <EOL> return inner <EOL> for command in self . COMMANDS : <EOL> setattr ( self , command , do_command ( command ) ) <EOL> def upload_bundle ( self , sources , follow_symlinks , exclude_patterns , git , unpack , remove_sources , info , worksheet_uuid , add_to_worksheet ) : <EOL> """<STR_LIT>""" <EOL> if all ( path_util . path_is_url ( source ) for source in sources ) : <EOL> return self . upload_bundle_url ( sources , follow_symlinks , exclude_patterns , git , unpack , remove_sources , info , worksheet_uuid , add_to_worksheet ) <EOL> remote_file_uuids = [ ] <EOL> try : <EOL> for source in sources : <EOL> if zip_util . path_is_archive ( source ) : <EOL> source_handle = open ( source ) <EOL> temp_file_name = os . path . basename ( source ) <EOL> elif os . path . isdir ( source ) : <EOL> source_handle = tar_gzip_directory ( source , follow_symlinks , exclude_patterns ) <EOL> temp_file_name = os . path . basename ( source ) + '<STR_LIT>' <EOL> unpack = True <EOL> else : <EOL> resolved_source = source <EOL> if follow_symlinks : <EOL> resolved_source = os . path . realpath ( source ) <EOL> if not os . path . exists ( resolved_source ) : <EOL> raise UsageError ( '<STR_LIT>' ) <EOL> elif os . path . islink ( source ) : <EOL> raise UsageError ( '<STR_LIT>' ) <EOL> source_handle = gzip_file ( resolved_source ) <EOL> temp_file_name = os . path . basename ( source ) + '<STR_LIT>' <EOL> unpack = True <EOL> remote_file_uuid = self . open_temp_file ( temp_file_name ) <EOL> remote_file_uuids . append ( remote_file_uuid ) <EOL> with closing ( RPCFileHandle ( remote_file_uuid , self . proxy ) ) as dest_handle : <EOL> status = '<STR_LIT>' % ( source , '<STR_LIT>' + info [ '<STR_LIT>' ] + '<STR_LIT:)>' if '<STR_LIT>' in info else '<STR_LIT>' , self . address ) <EOL> file_util . copy ( source_handle , dest_handle , autoflush = False , print_status = status ) <EOL> return self . finish_upload_bundle ( remote_file_uuids , unpack , info , worksheet_uuid , add_to_worksheet ) <EOL> except : <EOL> for remote_file_uuid in remote_file_uuids : <EOL> self . finalize_file ( remote_file_uuid ) <EOL> raise <EOL> def download_directory ( self , target , download_path ) : <EOL> """<STR_LIT>""" <EOL> remote_file_uuid = self . open_tarred_gzipped_directory ( target ) <EOL> with closing ( RPCFileHandle ( remote_file_uuid , self . proxy , finalize_on_close = True ) ) as fileobj : <EOL> un_tar_directory ( fileobj , download_path , '<STR_LIT>' ) <EOL> def download_file ( self , target , download_path ) : <EOL> """<STR_LIT>""" <EOL> self . _do_download_file ( target , out_path = download_path ) <EOL> def cat_target ( self , target , out ) : <EOL> """<STR_LIT>""" <EOL> self . _do_download_file ( target , out_fileobj = out ) <EOL> def _do_download_file ( self , target , out_path = None , out_fileobj = None ) : <EOL> remote_file_uuid = self . open_gzipped_file ( target ) <EOL> with closing ( un_gzip_stream ( RPCFileHandle ( remote_file_uuid , self . proxy , finalize_on_close = True ) ) ) as fileobj : <EOL> if out_path is not None : <EOL> with open ( out_path , '<STR_LIT:wb>' ) as out : <EOL> shutil . copyfileobj ( fileobj , out ) <EOL> elif out_fileobj is not None : <EOL> shutil . copyfileobj ( fileobj , out_fileobj ) <EOL> def read_file_section ( self , target , offset , length ) : <EOL> """<STR_LIT>""" <EOL> return un_gzip_string ( <EOL> self . read_gzipped_file_section ( target , offset , length ) . data ) </s>
