<s> import json <EOL> import os <EOL> import re <EOL> import requests <EOL> from . . import base <EOL> from . . import mongo_replicaset <EOL> from girder . utility . s3_assetstore_adapter import botoConnectS3 <EOL> Chunk1 , Chunk2 = ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> def setUpModule ( ) : <EOL> base . startServer ( mockS3 = True ) <EOL> def tearDownModule ( ) : <EOL> base . stopServer ( ) <EOL> def _send_s3_request ( request , data = None ) : <EOL> """<STR_LIT>""" <EOL> if request [ '<STR_LIT>' ] == '<STR_LIT>' : <EOL> req = requests . put ( url = request [ '<STR_LIT:url>' ] , data = data , <EOL> headers = request . get ( '<STR_LIT>' , { } ) ) <EOL> elif request [ '<STR_LIT>' ] == '<STR_LIT:POST>' : <EOL> req = requests . post ( url = request [ '<STR_LIT:url>' ] , data = data , <EOL> headers = request . get ( '<STR_LIT>' , { } ) ) <EOL> return req <EOL> class UploadTestCase ( base . TestCase ) : <EOL> def setUp ( self ) : <EOL> base . TestCase . setUp ( self ) <EOL> admin = { <EOL> '<STR_LIT:email>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT:password>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : True <EOL> } <EOL> self . admin = self . model ( '<STR_LIT:user>' ) . createUser ( ** admin ) <EOL> user = { <EOL> '<STR_LIT:email>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT:password>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : False <EOL> } <EOL> self . user = self . model ( '<STR_LIT:user>' ) . createUser ( ** user ) <EOL> folders = self . model ( '<STR_LIT>' ) . childFolders ( <EOL> parent = self . user , parentType = '<STR_LIT:user>' , user = self . user ) <EOL> for folder in folders : <EOL> if folder [ '<STR_LIT>' ] is True : <EOL> self . folder = folder <EOL> def _uploadFile ( self , name , partial = False , largeFile = False ) : <EOL> """<STR_LIT>""" <EOL> if not largeFile : <EOL> chunk1 = Chunk1 <EOL> chunk2 = Chunk2 <EOL> else : <EOL> chunk1 = '<STR_LIT:->' * ( <NUM_LIT> * <NUM_LIT> * <NUM_LIT:32> ) <EOL> chunk2 = '<STR_LIT:->' * ( <NUM_LIT> * <NUM_LIT> * <NUM_LIT:1> ) <EOL> resp = self . request ( <EOL> path = '<STR_LIT>' , method = '<STR_LIT:POST>' , user = self . user , params = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : self . folder [ '<STR_LIT>' ] , <EOL> '<STR_LIT:name>' : name , <EOL> '<STR_LIT:size>' : len ( chunk1 ) + len ( chunk2 ) , <EOL> '<STR_LIT>' : '<STR_LIT>' <EOL> } ) <EOL> self . assertStatusOk ( resp ) <EOL> upload = resp . json <EOL> if partial is not False and partial == <NUM_LIT:0> : <EOL> return upload <EOL> if '<STR_LIT>' not in upload : <EOL> fields = [ ( '<STR_LIT>' , <NUM_LIT:0> ) , ( '<STR_LIT>' , upload [ '<STR_LIT>' ] ) ] <EOL> files = [ ( '<STR_LIT>' , '<STR_LIT>' , chunk1 ) ] <EOL> resp = self . multipartRequest ( <EOL> path = '<STR_LIT>' , user = self . user , fields = fields , files = files ) <EOL> self . assertStatusOk ( resp ) <EOL> if partial is not False : <EOL> return resp . json <EOL> fields = [ ( '<STR_LIT>' , len ( chunk1 ) ) , ( '<STR_LIT>' , upload [ '<STR_LIT>' ] ) ] <EOL> files = [ ( '<STR_LIT>' , '<STR_LIT>' , chunk2 ) ] <EOL> resp = self . multipartRequest ( <EOL> path = '<STR_LIT>' , user = self . user , fields = fields , files = files ) <EOL> self . assertStatusOk ( resp ) <EOL> return upload <EOL> if not upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] : <EOL> _send_s3_request ( upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] , chunk1 + chunk2 ) <EOL> if partial is not False : <EOL> return <EOL> else : <EOL> chunk1 = chunk1 + chunk2 <EOL> s3resp = _send_s3_request ( upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] ) <EOL> matches = re . search ( '<STR_LIT>' , s3resp . text ) <EOL> s3uploadId = matches . groups ( ) [ <NUM_LIT:0> ] <EOL> offset = <NUM_LIT:0> <EOL> chunkN = <NUM_LIT:1> <EOL> while len ( chunk1 ) : <EOL> params = { '<STR_LIT>' : offset , '<STR_LIT>' : upload [ '<STR_LIT>' ] } <EOL> params [ "<STR_LIT>" ] = json . dumps ( { '<STR_LIT>' : s3uploadId , <EOL> '<STR_LIT>' : chunkN } ) <EOL> resp = self . request ( path = '<STR_LIT>' , method = '<STR_LIT:POST>' , <EOL> user = self . user , params = params ) <EOL> self . assertStatusOk ( resp ) <EOL> upload = resp . json <EOL> if len ( chunk1 ) > upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] : <EOL> chunk2 = chunk1 [ upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] : ] <EOL> chunk1 = chunk1 [ : upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] ] <EOL> else : <EOL> chunk2 = "<STR_LIT>" <EOL> _send_s3_request ( upload [ '<STR_LIT>' ] [ '<STR_LIT>' ] , chunk1 ) <EOL> chunk1 = chunk2 <EOL> if partial is not False : <EOL> partial -= <NUM_LIT:1> <EOL> chunkN += <NUM_LIT:1> <EOL> if partial is not False and not partial : <EOL> return upload <EOL> resp = self . request ( path = '<STR_LIT>' , method = '<STR_LIT:POST>' , <EOL> user = self . user , <EOL> params = { '<STR_LIT>' : upload [ '<STR_LIT>' ] } ) <EOL> self . assertStatusOk ( resp ) <EOL> if '<STR_LIT>' in resp . json : <EOL> _send_s3_request ( resp . json [ '<STR_LIT>' ] ) <EOL> return upload <EOL> def _testUpload ( self ) : <EOL> """<STR_LIT>""" <EOL> completeUpload = self . _uploadFile ( '<STR_LIT>' ) <EOL> self . _uploadFile ( '<STR_LIT>' , largeFile = True ) <EOL> partialUploads = [ ] <EOL> for largeFile in ( False , True ) : <EOL> for partial in range ( <NUM_LIT:3> ) : <EOL> partialUploads . append ( self . _uploadFile ( <EOL> '<STR_LIT>' % ( partial , str ( largeFile ) ) , <EOL> partial , largeFile ) ) <EOL> resp = self . request ( path = '<STR_LIT>' , method = '<STR_LIT:GET>' , <EOL> user = self . user ) <EOL> self . assertStatus ( resp , <NUM_LIT> ) <EOL> resp = self . request ( path = '<STR_LIT>' , method = '<STR_LIT:GET>' , <EOL> user = self . admin ) <EOL> self . assertStatusOk ( resp ) <EOL> foundUploads = resp . json <EOL> self . assertEqual ( len ( foundUploads ) , len ( partialUploads ) ) <EOL> resp = self . request ( path = '<STR_LIT>' , method = '<STR_LIT>' , <EOL> user = self . user , <EOL> params = { '<STR_LIT>' : partialUploads [ <NUM_LIT:0> ] [ '<STR_LIT>' ] } ) <EOL> self . assertStatus ( resp , <NUM_LIT> ) <EOL> resp = self . request ( path = '<STR_LIT>' , method = '<STR_LIT>' , <EOL> user = self . admin , <EOL> params = { '<STR_LIT>' : completeUpload [ '<STR_LIT>' ] } ) <EOL> self . assertStatusOk ( resp ) <EOL> self . assertEqual ( resp . json , [ ] ) <EOL> resp = self . request ( path = '<STR_LIT>' , method = '<STR_LIT:GET>' , <EOL> user = self . admin ) <EOL> self . assertEqual ( len ( resp . json ) , len ( partialUploads ) ) <EOL> resp = self . request ( path = '<STR_LIT>' , method = '<STR_LIT:GET>' , <EOL> user = self . admin , <EOL> params = { '<STR_LIT>' : partialUploads [ <NUM_LIT:0> ] [ '<STR_LIT>' ] } ) <EOL> self . assertStatusOk ( resp ) <EOL> self . assertEqual ( resp . json [ <NUM_LIT:0> ] [ '<STR_LIT>' ] , partialUploads [ <NUM_LIT:0> ] [ '<STR_LIT>' ] ) <EOL> resp = self . request ( path = '<STR_LIT>' , method = '<STR_LIT:GET>' , <EOL> user = self . admin , <EOL> params = { '<STR_LIT>' : self . assetstore [ '<STR_LIT>' ] } ) <EOL> self . assertStatusOk ( resp ) <EOL> self . assertEqual ( len ( resp . json ) , len ( partialUploads ) ) <EOL> resp = self . request ( path = '<STR_LIT>' , method = '<STR_LIT:GET>' , <EOL> user = self . admin , <EOL> params = { '<STR_LIT>' : <NUM_LIT:0> } ) <EOL> self . assertStatusOk ( resp ) <EOL> self . assertEqual ( len ( resp . json ) , len ( partialUploads ) ) <EOL> resp = self . request ( path = '<STR_LIT>' , method = '<STR_LIT>' , <EOL> user = self . admin , <EOL> params = { '<STR_LIT>' : partialUploads [ <NUM_LIT:0> ] [ '<STR_LIT>' ] } ) <EOL> self . assertStatusOk ( resp ) <EOL> self . assertEqual ( resp . json [ <NUM_LIT:0> ] [ '<STR_LIT>' ] , partialUploads [ <NUM_LIT:0> ] [ '<STR_LIT>' ] ) <EOL> resp = self . request ( path = '<STR_LIT>' , method = '<STR_LIT:GET>' , <EOL> user = self . admin ) <EOL> self . assertEqual ( len ( resp . json ) , len ( partialUploads ) - <NUM_LIT:1> ) <EOL> resp = self . request ( path = '<STR_LIT>' , method = '<STR_LIT>' , <EOL> user = self . admin , <EOL> params = { '<STR_LIT>' : <NUM_LIT:1> } ) <EOL> self . assertStatusOk ( resp ) <EOL> self . assertEqual ( resp . json , [ ] ) <EOL> resp = self . request ( path = '<STR_LIT>' , method = '<STR_LIT>' , <EOL> user = self . admin ) <EOL> self . assertStatusOk ( resp ) <EOL> resp = self . request ( path = '<STR_LIT>' , method = '<STR_LIT:GET>' , <EOL> user = self . admin ) <EOL> self . assertEqual ( resp . json , [ ] ) <EOL> def testFilesystemAssetstoreUpload ( self ) : <EOL> self . _testUpload ( ) <EOL> def testGridFSAssetstoreUpload ( self ) : <EOL> base . dropGridFSDatabase ( '<STR_LIT>' ) <EOL> self . model ( '<STR_LIT>' ) . remove ( self . model ( '<STR_LIT>' ) . getCurrent ( ) ) <EOL> assetstore = self . model ( '<STR_LIT>' ) . createGridFsAssetstore ( <EOL> name = '<STR_LIT>' , db = '<STR_LIT>' ) <EOL> self . assetstore = assetstore <EOL> self . _testUpload ( ) <EOL> def testGridFSReplicaSetAssetstoreUpload ( self ) : <EOL> verbose = <NUM_LIT:0> <EOL> if '<STR_LIT>' in os . environ . get ( '<STR_LIT>' , '<STR_LIT>' ) . split ( ) : <EOL> verbose = <NUM_LIT:2> <EOL> mongo_replicaset . startMongoReplicaSet ( verbose = verbose ) <EOL> self . model ( '<STR_LIT>' ) . remove ( self . model ( '<STR_LIT>' ) . getCurrent ( ) ) <EOL> assetstore = self . model ( '<STR_LIT>' ) . createGridFsAssetstore ( <EOL> name = '<STR_LIT>' , db = '<STR_LIT>' , <EOL> mongohost = '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> replicaset = '<STR_LIT>' ) <EOL> self . assetstore = assetstore <EOL> self . _testUpload ( ) <EOL> mongo_replicaset . stepDownMongoReplicaSet ( <NUM_LIT:0> ) <EOL> mongo_replicaset . waitForRSStatus ( <EOL> mongo_replicaset . getMongoClient ( <NUM_LIT:0> ) , status = [ <NUM_LIT:2> , ( <NUM_LIT:1> , <NUM_LIT:2> ) , ( <NUM_LIT:1> , <NUM_LIT:2> ) ] , <EOL> verbose = verbose ) <EOL> mongo_replicaset . pauseMongoReplicaSet ( [ True ] , verbose = verbose ) <EOL> self . _uploadFile ( '<STR_LIT>' ) <EOL> mongo_replicaset . pauseMongoReplicaSet ( [ False , False ] , verbose = verbose ) <EOL> self . _uploadFile ( '<STR_LIT>' ) <EOL> mongo_replicaset . stopMongoReplicaSet ( ) <EOL> def testS3AssetstoreUpload ( self ) : <EOL> self . model ( '<STR_LIT>' ) . remove ( self . assetstore ) <EOL> params = { <EOL> '<STR_LIT:name>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : base . mockS3Server . service <EOL> } <EOL> assetstore = self . model ( '<STR_LIT>' ) . createS3Assetstore ( ** params ) <EOL> self . assetstore = assetstore <EOL> self . _testUpload ( ) <EOL> conn = botoConnectS3 ( base . mockS3Server . botoConnect ) <EOL> bucket = conn . lookup ( bucket_name = '<STR_LIT>' , validate = True ) <EOL> bucket . initiate_multipart_upload ( '<STR_LIT>' ) <EOL> resp = self . request ( path = '<STR_LIT>' , method = '<STR_LIT:GET>' , <EOL> user = self . admin ) <EOL> self . assertStatusOk ( resp ) <EOL> self . assertEqual ( len ( resp . json ) , <NUM_LIT:1> ) <EOL> resp = self . request ( path = '<STR_LIT>' , method = '<STR_LIT>' , <EOL> user = self . admin ) <EOL> self . assertStatusOk ( resp ) <EOL> resp = self . request ( path = '<STR_LIT>' , method = '<STR_LIT:GET>' , <EOL> user = self . admin ) <EOL> self . assertStatusOk ( resp ) <EOL> self . assertEqual ( resp . json , [ ] ) </s>
