<s> import boto <EOL> import boto . s3 . connection <EOL> import cherrypy <EOL> import json <EOL> import re <EOL> import requests <EOL> import six <EOL> import uuid <EOL> from . abstract_assetstore_adapter import AbstractAssetstoreAdapter <EOL> from girder . models . model_base import ValidationException <EOL> from girder import logger , events <EOL> BUF_LEN = <NUM_LIT> <EOL> boto . config . add_section ( '<STR_LIT>' ) <EOL> boto . config . set ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:True>' ) <EOL> def authv4_determine_region_name ( self , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> result = authv4_orig_determine_region_name ( self , * args , ** kwargs ) <EOL> except UnboundLocalError : <EOL> result = '<STR_LIT>' <EOL> return result <EOL> def required_auth_capability_wrapper ( fun ) : <EOL> def wrapper ( self , * args , ** kwargs ) : <EOL> if self . anon : <EOL> return [ '<STR_LIT>' ] <EOL> else : <EOL> return fun ( self , * args , ** kwargs ) <EOL> return wrapper <EOL> authv4_orig_determine_region_name = boto . auth . S3HmacAuthV4Handler . determine_region_name <EOL> boto . auth . S3HmacAuthV4Handler . determine_region_name = authv4_determine_region_name <EOL> boto . s3 . connection . S3Connection . _required_auth_capability = required_auth_capability_wrapper ( <EOL> boto . s3 . connection . S3Connection . _required_auth_capability ) <EOL> def _generate_url_sigv4 ( self , expires_in , method , bucket = '<STR_LIT>' , key = '<STR_LIT>' , <EOL> headers = None , response_headers = None , version_id = None , <EOL> iso_date = None , params = None ) : <EOL> """<STR_LIT>""" <EOL> path = self . calling_format . build_path_base ( bucket , key ) <EOL> auth_path = self . calling_format . build_auth_path ( bucket , key ) <EOL> host = self . calling_format . build_host ( self . server_name ( ) , bucket ) <EOL> if host . endswith ( '<STR_LIT>' ) : <EOL> host = host [ : - <NUM_LIT:4> ] <EOL> if params is None : <EOL> params = { } <EOL> if version_id is not None : <EOL> params [ '<STR_LIT>' ] = version_id <EOL> http_request = self . build_base_http_request ( <EOL> method , path , auth_path , headers = headers , host = host , params = params ) <EOL> return self . _auth_handler . presign ( http_request , expires_in , <EOL> iso_date = iso_date ) <EOL> class S3AssetstoreAdapter ( AbstractAssetstoreAdapter ) : <EOL> """<STR_LIT>""" <EOL> CHUNK_LEN = <NUM_LIT> * <NUM_LIT> * <NUM_LIT:32> <EOL> HMAC_TTL = <NUM_LIT> <EOL> @ staticmethod <EOL> def validateInfo ( doc ) : <EOL> """<STR_LIT>""" <EOL> if '<STR_LIT>' not in doc : <EOL> doc [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> doc [ '<STR_LIT>' ] = doc [ '<STR_LIT>' ] . strip ( '<STR_LIT:/>' ) <EOL> if not doc . get ( '<STR_LIT>' ) : <EOL> raise ValidationException ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> if not doc . get ( '<STR_LIT>' ) : <EOL> if not doc . get ( '<STR_LIT>' ) : <EOL> raise ValidationException ( <EOL> '<STR_LIT>' , '<STR_LIT>' ) <EOL> if not doc . get ( '<STR_LIT>' ) : <EOL> raise ValidationException ( <EOL> '<STR_LIT>' , '<STR_LIT>' ) <EOL> if '<STR_LIT>' not in doc : <EOL> doc [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> if doc [ '<STR_LIT>' ] != '<STR_LIT>' : <EOL> service = re . match ( "<STR_LIT>" , <EOL> doc [ '<STR_LIT>' ] ) <EOL> if not service : <EOL> raise ValidationException ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' , '<STR_LIT>' ) <EOL> doc [ '<STR_LIT>' ] = makeBotoConnectParams ( <EOL> doc [ '<STR_LIT>' ] , doc [ '<STR_LIT>' ] , doc [ '<STR_LIT>' ] ) <EOL> conn = botoConnectS3 ( doc [ '<STR_LIT>' ] ) <EOL> if doc . get ( '<STR_LIT>' ) : <EOL> try : <EOL> conn . get_bucket ( bucket_name = doc [ '<STR_LIT>' ] , validate = True ) <EOL> except Exception : <EOL> logger . exception ( '<STR_LIT>' ) <EOL> raise ValidationException ( '<STR_LIT>' % <EOL> doc [ '<STR_LIT>' ] , '<STR_LIT>' ) <EOL> else : <EOL> try : <EOL> bucket = conn . get_bucket ( bucket_name = doc [ '<STR_LIT>' ] , <EOL> validate = True ) <EOL> testKey = boto . s3 . key . Key ( <EOL> bucket = bucket , name = '<STR_LIT:/>' . join ( <EOL> filter ( None , ( doc [ '<STR_LIT>' ] , '<STR_LIT:test>' ) ) ) ) <EOL> testKey . set_contents_from_string ( '<STR_LIT>' ) <EOL> except Exception : <EOL> logger . exception ( '<STR_LIT>' ) <EOL> raise ValidationException ( '<STR_LIT>' % <EOL> doc [ '<STR_LIT>' ] , '<STR_LIT>' ) <EOL> return doc <EOL> def __init__ ( self , assetstore ) : <EOL> """<STR_LIT>""" <EOL> super ( S3AssetstoreAdapter , self ) . __init__ ( assetstore ) <EOL> if ( '<STR_LIT>' in self . assetstore and '<STR_LIT>' in self . assetstore and <EOL> '<STR_LIT>' in self . assetstore ) : <EOL> self . assetstore [ '<STR_LIT>' ] = makeBotoConnectParams ( <EOL> self . assetstore [ '<STR_LIT>' ] , self . assetstore [ '<STR_LIT>' ] , <EOL> self . assetstore [ '<STR_LIT>' ] ) <EOL> def _getRequestHeaders ( self , upload ) : <EOL> return { <EOL> '<STR_LIT>' : '<STR_LIT>' % upload [ '<STR_LIT:name>' ] , <EOL> '<STR_LIT:Content-Type>' : upload . get ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : str ( upload [ '<STR_LIT>' ] ) , <EOL> '<STR_LIT>' : str ( cherrypy . request . remote . ip ) <EOL> } <EOL> def initUpload ( self , upload ) : <EOL> """<STR_LIT>""" <EOL> if upload [ '<STR_LIT:size>' ] <= <NUM_LIT:0> : <EOL> return upload <EOL> uid = uuid . uuid4 ( ) . hex <EOL> key = '<STR_LIT:/>' . join ( filter ( None , ( self . assetstore . get ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> uid [ <NUM_LIT:0> : <NUM_LIT:2> ] , uid [ <NUM_LIT:2> : <NUM_LIT:4> ] , uid ) ) ) <EOL> path = '<STR_LIT>' % ( self . assetstore [ '<STR_LIT>' ] , key ) <EOL> headers = self . _getRequestHeaders ( upload ) <EOL> chunked = upload [ '<STR_LIT:size>' ] > self . CHUNK_LEN <EOL> upload [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> upload [ '<STR_LIT>' ] = { <EOL> '<STR_LIT>' : chunked , <EOL> '<STR_LIT>' : self . CHUNK_LEN , <EOL> '<STR_LIT>' : path , <EOL> '<STR_LIT:key>' : key <EOL> } <EOL> if chunked : <EOL> upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] = { '<STR_LIT>' : '<STR_LIT:POST>' } <EOL> alsoSignHeaders = { } <EOL> queryParams = { '<STR_LIT>' : None } <EOL> else : <EOL> upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] = { '<STR_LIT>' : '<STR_LIT>' } <EOL> alsoSignHeaders = { <EOL> '<STR_LIT>' : upload [ '<STR_LIT:size>' ] <EOL> } <EOL> queryParams = None <EOL> url = self . _botoGenerateUrl ( <EOL> method = upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] , key = key , <EOL> headers = dict ( headers , ** alsoSignHeaders ) , queryParams = queryParams , <EOL> chunkedUpload = chunked ) <EOL> upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT:url>' ] = url <EOL> upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] = headers <EOL> return upload <EOL> def uploadChunk ( self , upload , chunk ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( chunk , six . string_types ) : <EOL> return self . _clientUploadChunk ( upload , chunk ) <EOL> else : <EOL> return self . _proxiedUploadChunk ( upload , chunk ) <EOL> def _clientUploadChunk ( self , upload , chunk ) : <EOL> """<STR_LIT>""" <EOL> info = json . loads ( chunk ) <EOL> index = int ( info [ '<STR_LIT>' ] ) - <NUM_LIT:1> <EOL> length = min ( self . CHUNK_LEN , upload [ '<STR_LIT:size>' ] - index * self . CHUNK_LEN ) <EOL> if '<STR_LIT>' in info and int ( info [ '<STR_LIT>' ] ) != length : <EOL> raise ValidationException ( '<STR_LIT>' % ( <EOL> length , info [ '<STR_LIT>' ] ) ) <EOL> if length <= <NUM_LIT:0> : <EOL> raise ValidationException ( '<STR_LIT>' % length ) <EOL> queryParams = { <EOL> '<STR_LIT>' : info [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : info [ '<STR_LIT>' ] <EOL> } <EOL> url = self . _botoGenerateUrl ( <EOL> method = '<STR_LIT>' , key = upload [ '<STR_LIT>' ] [ '<STR_LIT:key>' ] , queryParams = queryParams , <EOL> headers = { <EOL> '<STR_LIT>' : length <EOL> } ) <EOL> upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] = info [ '<STR_LIT>' ] <EOL> upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] = info [ '<STR_LIT>' ] <EOL> upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT:url>' : url <EOL> } <EOL> return upload <EOL> def _getBucket ( self , validate = True ) : <EOL> conn = botoConnectS3 ( self . assetstore [ '<STR_LIT>' ] ) <EOL> bucket = conn . lookup ( bucket_name = self . assetstore [ '<STR_LIT>' ] , <EOL> validate = validate ) <EOL> if not bucket : <EOL> raise Exception ( '<STR_LIT>' ) <EOL> return bucket <EOL> def _proxiedUploadChunk ( self , upload , chunk ) : <EOL> """<STR_LIT>""" <EOL> bucket = self . _getBucket ( ) <EOL> if upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] : <EOL> if '<STR_LIT>' in upload [ '<STR_LIT>' ] : <EOL> mp = boto . s3 . multipart . MultiPartUpload ( bucket ) <EOL> mp . id = upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> mp . key_name = upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> else : <EOL> mp = bucket . initiate_multipart_upload ( <EOL> upload [ '<STR_LIT>' ] [ '<STR_LIT:key>' ] , <EOL> headers = self . _getRequestHeaders ( upload ) ) <EOL> upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] = mp . id <EOL> upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] = mp . key_name <EOL> upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] = <NUM_LIT:0> <EOL> upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] += <NUM_LIT:1> <EOL> key = mp . upload_part_from_file ( <EOL> chunk , upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] , <EOL> headers = self . _getRequestHeaders ( upload ) ) <EOL> upload [ '<STR_LIT>' ] += key . size <EOL> else : <EOL> key = bucket . new_key ( upload [ '<STR_LIT>' ] [ '<STR_LIT:key>' ] ) <EOL> key . set_contents_from_file ( chunk , <EOL> headers = self . _getRequestHeaders ( upload ) ) <EOL> if key . size < upload [ '<STR_LIT:size>' ] : <EOL> bucket . delete_key ( key ) <EOL> raise ValidationException ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> upload [ '<STR_LIT>' ] = key . size <EOL> return upload <EOL> def requestOffset ( self , upload ) : <EOL> if upload [ '<STR_LIT>' ] > <NUM_LIT:0> : <EOL> return upload [ '<STR_LIT>' ] <EOL> if upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] : <EOL> raise ValidationException ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> headers = self . _getRequestHeaders ( upload ) <EOL> url = self . _botoGenerateUrl ( method = '<STR_LIT>' , key = upload [ '<STR_LIT>' ] [ '<STR_LIT:key>' ] , <EOL> headers = headers ) <EOL> return { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT:url>' : url , <EOL> '<STR_LIT>' : headers , <EOL> '<STR_LIT>' : <NUM_LIT:0> <EOL> } <EOL> def finalizeUpload ( self , upload , file ) : <EOL> if upload [ '<STR_LIT:size>' ] <= <NUM_LIT:0> : <EOL> return file <EOL> file [ '<STR_LIT>' ] = upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> file [ '<STR_LIT>' ] = upload [ '<STR_LIT>' ] [ '<STR_LIT:key>' ] <EOL> if upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] : <EOL> if upload [ '<STR_LIT>' ] > <NUM_LIT:0> : <EOL> bucket = self . _getBucket ( ) <EOL> mp = boto . s3 . multipart . MultiPartUpload ( bucket ) <EOL> mp . id = upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> mp . key_name = upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> mp . complete_upload ( ) <EOL> else : <EOL> queryParams = { '<STR_LIT>' : upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] } <EOL> headers = { '<STR_LIT:Content-Type>' : '<STR_LIT>' } <EOL> url = self . _botoGenerateUrl ( <EOL> method = '<STR_LIT:POST>' , key = upload [ '<STR_LIT>' ] [ '<STR_LIT:key>' ] , headers = headers , <EOL> queryParams = queryParams ) <EOL> file [ '<STR_LIT>' ] = { <EOL> '<STR_LIT>' : '<STR_LIT:POST>' , <EOL> '<STR_LIT:url>' : url , <EOL> '<STR_LIT>' : headers <EOL> } <EOL> file [ '<STR_LIT>' ] = ( '<STR_LIT>' , ) <EOL> return file <EOL> def downloadFile ( self , file , offset = <NUM_LIT:0> , headers = True , endByte = None , <EOL> contentDisposition = None , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> if self . assetstore . get ( '<STR_LIT>' , { } ) . get ( '<STR_LIT>' ) is True : <EOL> urlFn = self . _anonDownloadUrl <EOL> else : <EOL> urlFn = self . _botoGenerateUrl <EOL> if headers : <EOL> if file [ '<STR_LIT:size>' ] > <NUM_LIT:0> : <EOL> queryParams = { } <EOL> if contentDisposition == '<STR_LIT>' : <EOL> queryParams [ '<STR_LIT>' ] = '<STR_LIT>' % file [ '<STR_LIT:name>' ] <EOL> url = urlFn ( key = file [ '<STR_LIT>' ] , queryParams = queryParams ) <EOL> raise cherrypy . HTTPRedirect ( url ) <EOL> else : <EOL> self . setContentHeaders ( file , <NUM_LIT:0> , <NUM_LIT:0> ) <EOL> def stream ( ) : <EOL> yield '<STR_LIT>' <EOL> return stream <EOL> else : <EOL> def stream ( ) : <EOL> if file [ '<STR_LIT:size>' ] > <NUM_LIT:0> : <EOL> pipe = requests . get ( urlFn ( key = file [ '<STR_LIT>' ] ) , stream = True ) <EOL> for chunk in pipe . iter_content ( chunk_size = BUF_LEN ) : <EOL> if chunk : <EOL> yield chunk <EOL> else : <EOL> yield '<STR_LIT>' <EOL> return stream <EOL> def importData ( self , parent , parentType , params , progress , user , <EOL> bucket = None , ** kwargs ) : <EOL> importPath = params . get ( '<STR_LIT>' , '<STR_LIT>' ) . strip ( ) . lstrip ( '<STR_LIT:/>' ) <EOL> if importPath and not importPath . endswith ( '<STR_LIT:/>' ) : <EOL> importPath += '<STR_LIT:/>' <EOL> if bucket is None : <EOL> bucket = self . _getBucket ( ) <EOL> for obj in bucket . list ( importPath , '<STR_LIT:/>' ) : <EOL> if progress : <EOL> progress . update ( message = obj . name ) <EOL> if isinstance ( obj , boto . s3 . prefix . Prefix ) : <EOL> name = obj . name . rstrip ( '<STR_LIT:/>' ) . rsplit ( '<STR_LIT:/>' , <NUM_LIT:1> ) [ - <NUM_LIT:1> ] <EOL> folder = self . model ( '<STR_LIT>' ) . createFolder ( <EOL> parent = parent , name = name , parentType = parentType , <EOL> creator = user , reuseExisting = True ) <EOL> self . importData ( parent = folder , parentType = '<STR_LIT>' , params = { <EOL> '<STR_LIT>' : obj . name <EOL> } , progress = progress , user = user , bucket = bucket , ** kwargs ) <EOL> elif isinstance ( obj , boto . s3 . key . Key ) : <EOL> name = obj . name . rsplit ( '<STR_LIT:/>' , <NUM_LIT:1> ) [ - <NUM_LIT:1> ] <EOL> if not name : <EOL> continue <EOL> if parentType != '<STR_LIT>' : <EOL> raise ValidationException ( <EOL> '<STR_LIT>' % <EOL> parentType ) <EOL> item = self . model ( '<STR_LIT>' ) . createItem ( <EOL> name = name , creator = user , folder = parent , reuseExisting = True ) <EOL> file = self . model ( '<STR_LIT:file>' ) . createFile ( <EOL> name = name , creator = user , item = item , reuseExisting = True , <EOL> assetstore = self . assetstore , mimeType = None , size = obj . size ) <EOL> file [ '<STR_LIT>' ] = obj . name <EOL> file [ '<STR_LIT>' ] = True <EOL> self . model ( '<STR_LIT:file>' ) . save ( file ) <EOL> def deleteFile ( self , file ) : <EOL> """<STR_LIT>""" <EOL> if file [ '<STR_LIT:size>' ] > <NUM_LIT:0> and '<STR_LIT>' in file : <EOL> q = { <EOL> '<STR_LIT>' : file [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : self . assetstore [ '<STR_LIT>' ] <EOL> } <EOL> matching = self . model ( '<STR_LIT:file>' ) . find ( q , limit = <NUM_LIT:2> , fields = [ ] ) <EOL> if matching . count ( True ) == <NUM_LIT:1> : <EOL> events . daemon . trigger ( '<STR_LIT>' , { <EOL> '<STR_LIT>' : self . assetstore . get ( '<STR_LIT>' , { } ) , <EOL> '<STR_LIT>' : self . assetstore [ '<STR_LIT>' ] , <EOL> '<STR_LIT:key>' : file [ '<STR_LIT>' ] <EOL> } ) <EOL> def fileUpdated ( self , file ) : <EOL> """<STR_LIT>""" <EOL> if file . get ( '<STR_LIT>' ) : <EOL> return <EOL> bucket = self . _getBucket ( ) <EOL> key = bucket . get_key ( file [ '<STR_LIT>' ] , validate = True ) <EOL> if not key : <EOL> return <EOL> disp = '<STR_LIT>' % file [ '<STR_LIT:name>' ] <EOL> mime = file . get ( '<STR_LIT>' ) or '<STR_LIT>' <EOL> if key . content_type != mime or key . content_disposition != disp : <EOL> key . set_remote_metadata ( metadata_plus = { <EOL> '<STR_LIT:Content-Type>' : mime , <EOL> '<STR_LIT>' : disp . encode ( '<STR_LIT:utf8>' ) <EOL> } , metadata_minus = [ ] , preserve_acl = True ) <EOL> def cancelUpload ( self , upload ) : <EOL> """<STR_LIT>""" <EOL> if '<STR_LIT>' not in upload : <EOL> return <EOL> if '<STR_LIT:key>' not in upload [ '<STR_LIT>' ] : <EOL> return <EOL> bucket = self . _getBucket ( ) <EOL> if bucket : <EOL> key = bucket . get_key ( upload [ '<STR_LIT>' ] [ '<STR_LIT:key>' ] , validate = True ) <EOL> if key : <EOL> bucket . delete_key ( key ) <EOL> if ( '<STR_LIT>' in upload and '<STR_LIT>' in upload [ '<STR_LIT>' ] and <EOL> '<STR_LIT:key>' in upload [ '<STR_LIT>' ] ) : <EOL> getParams = { } <EOL> while True : <EOL> try : <EOL> multipartUploads = bucket . get_all_multipart_uploads ( <EOL> ** getParams ) <EOL> except boto . exception . S3ResponseError : <EOL> break <EOL> if not len ( multipartUploads ) : <EOL> break <EOL> for multipartUpload in multipartUploads : <EOL> if ( multipartUpload . id == upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] and <EOL> multipartUpload . key_name == <EOL> upload [ '<STR_LIT>' ] [ '<STR_LIT:key>' ] ) : <EOL> multipartUpload . cancel_upload ( ) <EOL> if not multipartUploads . is_truncated : <EOL> break <EOL> getParams [ '<STR_LIT>' ] = multipartUploads . next_key_marker <EOL> getParams [ '<STR_LIT>' ] = multipartUploads . next_upload_id_marker <EOL> def untrackedUploads ( self , knownUploads = None , delete = False ) : <EOL> """<STR_LIT>""" <EOL> if self . assetstore . get ( '<STR_LIT>' ) : <EOL> return [ ] <EOL> untrackedList = [ ] <EOL> prefix = self . assetstore . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> if prefix : <EOL> prefix += '<STR_LIT:/>' <EOL> if knownUploads is None : <EOL> knownUploads = [ ] <EOL> bucket = self . _getBucket ( ) <EOL> if not bucket : <EOL> return [ ] <EOL> getParams = { } <EOL> while True : <EOL> try : <EOL> multipartUploads = bucket . get_all_multipart_uploads ( ** getParams ) <EOL> except boto . exception . S3ResponseError : <EOL> break <EOL> if not len ( multipartUploads ) : <EOL> break <EOL> for multipartUpload in multipartUploads : <EOL> if self . _uploadIsKnown ( multipartUpload , knownUploads ) : <EOL> continue <EOL> if not multipartUpload . key_name . startswith ( prefix ) : <EOL> continue <EOL> unknown = { '<STR_LIT>' : { '<STR_LIT>' : multipartUpload . id , <EOL> '<STR_LIT:key>' : multipartUpload . key_name } } <EOL> untrackedList . append ( unknown ) <EOL> if delete : <EOL> multipartUpload . cancel_upload ( ) <EOL> if not multipartUploads . is_truncated : <EOL> break <EOL> getParams [ '<STR_LIT>' ] = multipartUploads . next_key_marker <EOL> getParams [ '<STR_LIT>' ] = multipartUploads . next_upload_id_marker <EOL> return untrackedList <EOL> def _uploadIsKnown ( self , multipartUpload , knownUploads ) : <EOL> """<STR_LIT>""" <EOL> for upload in knownUploads : <EOL> if ( '<STR_LIT>' in upload and '<STR_LIT>' in upload [ '<STR_LIT>' ] and <EOL> '<STR_LIT:key>' in upload [ '<STR_LIT>' ] ) : <EOL> if ( multipartUpload . id == upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] and <EOL> multipartUpload . key_name == upload [ '<STR_LIT>' ] [ '<STR_LIT:key>' ] ) : <EOL> return True <EOL> return False <EOL> def _botoGenerateUrl ( self , key , method = '<STR_LIT:GET>' , headers = None , <EOL> queryParams = None , chunkedUpload = False ) : <EOL> """<STR_LIT>""" <EOL> conn = botoConnectS3 ( self . assetstore . get ( '<STR_LIT>' , { } ) ) <EOL> url = _generate_url_sigv4 ( <EOL> conn , expires_in = self . HMAC_TTL , method = method , <EOL> bucket = self . assetstore [ '<STR_LIT>' ] , key = key , headers = headers , <EOL> params = queryParams ) <EOL> config = self . assetstore . get ( '<STR_LIT>' , { } ) <EOL> if ( not config . get ( '<STR_LIT>' , True ) and <EOL> config . get ( '<STR_LIT:host>' ) == '<STR_LIT:127.0.0.1>' ) : <EOL> if url . startswith ( '<STR_LIT>' ) : <EOL> url = '<STR_LIT:http>' + url [ <NUM_LIT:5> : ] <EOL> if chunkedUpload : <EOL> url = url . split ( '<STR_LIT:?>' ) [ <NUM_LIT:0> ] + '<STR_LIT>' <EOL> return url <EOL> def _anonDownloadUrl ( self , key , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> if self . assetstore [ '<STR_LIT>' ] : <EOL> return '<STR_LIT:/>' . join ( ( <EOL> self . assetstore [ '<STR_LIT>' ] , self . assetstore [ '<STR_LIT>' ] , <EOL> key . lstrip ( '<STR_LIT:/>' ) ) ) <EOL> else : <EOL> service = '<STR_LIT>' % self . assetstore [ '<STR_LIT>' ] <EOL> return '<STR_LIT:/>' . join ( ( service , key . lstrip ( '<STR_LIT:/>' ) ) ) <EOL> class BotoCallingFormat ( boto . s3 . connection . OrdinaryCallingFormat ) : <EOL> def build_auth_path ( self , bucket , key = '<STR_LIT>' ) : <EOL> path = '<STR_LIT>' <EOL> if bucket : <EOL> path = '<STR_LIT:/>' + bucket <EOL> return path + '<STR_LIT:/>' + key <EOL> def build_path_base ( self , bucket , key = '<STR_LIT>' ) : <EOL> path_base = '<STR_LIT:/>' <EOL> if bucket : <EOL> path_base += bucket + '<STR_LIT:/>' <EOL> return path_base + key <EOL> def botoConnectS3 ( connectParams ) : <EOL> """<STR_LIT>""" <EOL> if '<STR_LIT>' not in connectParams or not connectParams [ '<STR_LIT>' ] : <EOL> connectParams = connectParams . copy ( ) <EOL> connectParams [ '<STR_LIT>' ] = BotoCallingFormat ( ) <EOL> try : <EOL> return boto . connect_s3 ( ** connectParams ) <EOL> except Exception : <EOL> logger . exception ( '<STR_LIT>' ) <EOL> raise ValidationException ( '<STR_LIT>' ) <EOL> def makeBotoConnectParams ( accessKeyId , secret , service = None ) : <EOL> """<STR_LIT>""" <EOL> if accessKeyId and secret : <EOL> connect = { <EOL> '<STR_LIT>' : accessKeyId , <EOL> '<STR_LIT>' : secret , <EOL> } <EOL> else : <EOL> connect = { <EOL> '<STR_LIT>' : True <EOL> } <EOL> if service : <EOL> service = re . match ( "<STR_LIT>" , service ) <EOL> if service . groups ( ) [ <NUM_LIT:1> ] == '<STR_LIT:http>' : <EOL> connect [ '<STR_LIT>' ] = False <EOL> connect [ '<STR_LIT:host>' ] = service . groups ( ) [ <NUM_LIT:2> ] <EOL> if service . groups ( ) [ <NUM_LIT:4> ] is not None : <EOL> connect [ '<STR_LIT:port>' ] = int ( service . groups ( ) [ <NUM_LIT:4> ] ) <EOL> else : <EOL> connect [ '<STR_LIT:host>' ] = boto . config . get ( '<STR_LIT>' , '<STR_LIT:host>' , '<STR_LIT>' ) <EOL> return connect <EOL> def _deleteFileImpl ( event ) : <EOL> """<STR_LIT>""" <EOL> info = event . info <EOL> conn = botoConnectS3 ( info . get ( '<STR_LIT>' , { } ) ) <EOL> bucket = conn . lookup ( bucket_name = info [ '<STR_LIT>' ] , validate = False ) <EOL> key = bucket . get_key ( info [ '<STR_LIT:key>' ] , validate = True ) <EOL> if key : <EOL> bucket . delete_key ( key ) <EOL> events . bind ( '<STR_LIT>' , '<STR_LIT>' , <EOL> _deleteFileImpl ) </s>
