lib/s3/right_s3_interface.rb
# New experimental API for uploading objects using the multipart upload API.
# store_object_multipart is similar in function to the store_object method, but breaks the input into parts and transmits each
# part separately.  The multipart upload API has the benefit of being be able to retransmit a part in isolation without needing to
# restart the entire upload.  This makes it ideal for uploading large files over unreliable networks.  It also does not
# require the file size to be known before starting the upload, making it useful for stream data as it is created (say via reading a pipe or socket).
# The hash of the response headers contains useful information like the location (the URI for the newly created object), bucket, key, and etag).
#
# The optional argument of :headers allows the caller to specify arbitrary request header values.
#
# s3.store_object_multipart(:bucket => "foobucket", :key => "foo", :data => "polemonium" )
#   => {:location=>"https://s3.amazonaws.com/right_s3_awesome_test_bucket_000B1_officedrop/test%2Flarge_multipart_file",
#       :e_tag=>"\"72b81ac08aed4d4d1055c11f56c2a258-1\"",
#       :key=>"test/large_multipart_file",
#       :bucket=>"right_s3_awesome_test_bucket_000B1_officedrop"}
#
# f = File.new("some_file", "r")
# s3.store_object_multipart(:bucket => "foobucket", :key => "foo", :data => f )
#   => {:location=>"https://s3.amazonaws.com/right_s3_awesome_test_bucket_000B1_officedrop/test%2Flarge_multipart_file",
#       :e_tag=>"\"72b81ac08aed4d4d1055c11f56c2a258-1\"",
#       :key=>"test/large_multipart_file",
#       :bucket=>"right_s3_awesome_test_bucket_000B1_officedrop"}
New experimental API for uploading objects using the multipart upload API.
store_object_multipart is similar in function to the store_object method, but breaks the input into parts and transmits each
part separately.  The multipart upload API has the benefit of being be able to retransmit a part in isolation without needing to
restart the entire upload.  This makes it ideal for uploading large files over unreliable networks.  It also does not
require the file size to be known before starting the upload, making it useful for stream data as it is created (say via reading a pipe or socket).
The hash of the response headers contains useful information like the location (the URI for the newly created object), bucket, key, and etag).

The optional argument of :headers allows the caller to specify arbitrary request header values.




New experimental API for uploading objects using the multipart upload API . store_object_multipart is similar in function to the store_object method but breaks the input into parts and transmits each part separately . The multipart upload API has the benefit of being be able to retransmit a part in isolation without needing to restart the entire upload . This makes it ideal for uploading large files over unreliable networks . It also does not require the file size to be known before starting the upload making it useful for stream data as it is created ( say via reading a pipe or socket ) . The hash of the response headers contains useful information like the location ( the URI for the newly created object ) bucket key and etag ) . The optional argument of : headers allows the caller to specify arbitrary request header values .
def store_object_multipart(params)
      AwsUtils.allow_only([:bucket, :key, :data, :headers, :part_size, :retry_count], params)
      AwsUtils.mandatory_arguments([:bucket, :key, :data], params)
      params[:headers] = {} unless params[:headers]
      params[:data].binmode if(params[:data].respond_to?(:binmode)) 
      unless(params[:data].respond_to?(:read))
        params[:data] = StringIO.new(params[:data].to_s)
      end
      params[:part_size] ||= MINIMUM_PART_SIZE
      if params[:part_size] < MINIMUM_PART_SIZE
        raise AwsError.new("Part size for a multipart upload must be greater than or equal to #{5 * 1024 * 1024} bytes.  #{params[:part_size]} bytes was provided.")
      end
      params[:retry_count] ||= DEFAULT_RETRY_COUNT
      if params[:retry_count] < 0
        raise AwsError.new("Retry count must be positive.  #{params[:retry_count]} bytes was provided.")
      end
      if (params[:part_size] >= USE_100_CONTINUE_PUT_SIZE)
        params[:headers]['expect'] = '100-continue'
      end
      initiate_hash = generate_rest_request('POST', params[:headers].merge(:url=>"#{params[:bucket]}/#{CGI::escape params[:key]}?uploads"))
      initiate_resp = request_info(initiate_hash, S3MultipartUploadInitiateResponseParser.new)
      upload_id = initiate_resp[:upload_id]
      part_etags = []
      part_data = ""
      index = 1
      until params[:data].eof?
        part_data = params[:data].read(params[:part_size])
        unless part_data.size == 0
          retry_attempts = 1
          while true
            begin
              send_part_hash = generate_rest_request('PUT', params[:headers].merge({ :url=>"#{params[:bucket]}/#{CGI::escape params[:key]}?partNumber=#{index}&uploadId=#{upload_id}", :data=>part_data } ))
              send_part_resp = request_info(send_part_hash, S3HttpResponseHeadParser.new)
              part_etags << {:part_num => index, :etag => send_part_resp['etag']}
              index += 1
              break 
            rescue AwsError => e
              if retry_attempts >= params[:retry_count]
                raise e
              else
                retry_attempts += 1
              end
            end
          end
        end
      end
      complete_body = "<CompleteMultipartUpload>"
      part_etags.each do |part_hash|
        complete_body << "<Part><PartNumber>#{part_hash[:part_num]}</PartNumber><ETag>#{part_hash[:etag]}</ETag></Part>"
      end
      complete_body << "</CompleteMultipartUpload>"
      complete_req_hash = generate_rest_request('POST', {:url=>"#{params[:bucket]}/#{CGI::escape params[:key]}?uploadId=#{upload_id}", :data => complete_body})
      return request_info(complete_req_hash, S3CompleteMultipartParser.new)
    rescue
      on_exception
    end
def store_object_multipart ( params ) AwsUtils . allow_only ( [ :bucket , :key , :data , :headers , :part_size , :retry_count ] , params ) AwsUtils . mandatory_arguments ( [ :bucket , :key , :data ] , params ) params [ :headers ] = { } unless params [ :headers ] params [ :data ] . binmode if ( params [ :data ] . respond_to? ( :binmode ) ) unless ( params [ :data ] . respond_to? ( :read ) ) params [ :data ] = StringIO . new ( params [ :data ] . to_s ) end params [ :part_size ] ||= MINIMUM_PART_SIZE if params [ :part_size ] < MINIMUM_PART_SIZE raise AwsError . new ( "Part size for a multipart upload must be greater than or equal to #{5 * 1024 * 1024} bytes.  #{params[:part_size]} bytes was provided." ) end params [ :retry_count ] ||= DEFAULT_RETRY_COUNT if params [ :retry_count ] < 0 raise AwsError . new ( "Retry count must be positive.  #{params[:retry_count]} bytes was provided." ) end if ( params [ :part_size ] >= USE_100_CONTINUE_PUT_SIZE ) params [ :headers ] [ 'expect' ] = '100-continue' end initiate_hash = generate_rest_request ( 'POST' , params [ :headers ] . merge ( :url => "#{params[:bucket]}/#{CGI::escape params[:key]}?uploads" ) ) initiate_resp = request_info ( initiate_hash , S3MultipartUploadInitiateResponseParser . new ) upload_id = initiate_resp [ :upload_id ] part_etags = [ ] part_data = "" index = 1 until params [ :data ] . eof? part_data = params [ :data ] . read ( params [ :part_size ] ) unless part_data . size == 0 retry_attempts = 1 while true begin send_part_hash = generate_rest_request ( 'PUT' , params [ :headers ] . merge ( { :url => "#{params[:bucket]}/#{CGI::escape params[:key]}?partNumber=#{index}&uploadId=#{upload_id}" , :data => part_data } ) ) send_part_resp = request_info ( send_part_hash , S3HttpResponseHeadParser . new ) part_etags << { :part_num => index , :etag => send_part_resp [ 'etag' ] } index += 1 break rescue AwsError => e if retry_attempts >= params [ :retry_count ] raise e else retry_attempts += 1 end end end end end complete_body = "<CompleteMultipartUpload>" part_etags . each do | part_hash | complete_body << "<Part><PartNumber>#{part_hash[:part_num]}</PartNumber><ETag>#{part_hash[:etag]}</ETag></Part>" end complete_body << "</CompleteMultipartUpload>" complete_req_hash = generate_rest_request ( 'POST' , { :url => "#{params[:bucket]}/#{CGI::escape params[:key]}?uploadId=#{upload_id}" , :data => complete_body } ) return request_info ( complete_req_hash , S3CompleteMultipartParser . new ) rescue on_exception end
New experimental API for uploading objects using the multipart upload API.
New experimental API for uploading objects using the multipart upload API .
# On Windows, if someone opens a file in text mode, we must reset it to binary mode for streaming to work properly # detect whether we are using straight read or converting to string first # make sure part size is > 5 MB minimum # make sure retry_count is positive # Set 100-continue for large part sizes # initiate upload # split into parts and upload each one, re-trying if necessary #   upload occurs serially at this time. # successful, can move to next part #Hit an error attempting to transmit part, retry until retry_attemts have been exhausted # assemble complete upload message
params
<not_specific>