lib/logstash/outputs/s3.rb
# The upload process will use a separate uploader/threadpool with less resource allocated to it.
# but it will use an unbounded queue for the work, it may take some time before all the older files get processed.
The upload process will use a separate uploader/threadpool with less resource allocated to it.
but it will use an unbounded queue for the work, it may take some time before all the older files get processed.
The upload process will use a separate uploader / threadpool with less resource allocated to it . but it will use an unbounded queue for the work it may take some time before all the older files get processed .
def restore_from_crash
    @crash_uploader = Uploader.new(bucket_resource, @logger, CRASH_RECOVERY_THREADPOOL)
    temp_folder_path = Pathname.new(@temporary_directory)
    Dir.glob(::File.join(@temporary_directory, "**/*"))
      .select { |file| ::File.file?(file) }
      .each do |file|
      temp_file = TemporaryFile.create_from_existing_file(file, temp_folder_path)
      if temp_file.size > 0
        @logger.debug("Recovering from crash and uploading", :file => temp_file.path)
        @crash_uploader.upload_async(temp_file, :on_complete => method(:clean_temporary_file), :upload_options => upload_options)
      else
        clean_temporary_file(temp_file)
      end
    end
  end
def restore_from_crash @crash_uploader = Uploader . new ( bucket_resource , @logger , CRASH_RECOVERY_THREADPOOL ) temp_folder_path = Pathname . new ( @temporary_directory ) Dir . glob ( :: File . join ( @temporary_directory , "**/*" ) ) . select { | file | :: File . file? ( file ) } . each do | file | temp_file = TemporaryFile . create_from_existing_file ( file , temp_folder_path ) if temp_file . size > 0 @logger . debug ( "Recovering from crash and uploading" , :file => temp_file . path ) @crash_uploader . upload_async ( temp_file , :on_complete => method ( :clean_temporary_file ) , :upload_options => upload_options ) else clean_temporary_file ( temp_file ) end end end
The upload process will use a separate uploader/threadpool with less resource allocated to it.
The upload process will use a separate uploader / threadpool with less resource allocated to it .