rlib/object_put_dir.rb
# Recursive copy of the directory to the object store bucket
# Looks for .fit files, and if found, extracts fits metadata into a separate .info object.
# @param aws_fd [AWS_connection] S3 connection to the object store
# @param directory [String] Top level directory to recursively copy from
# @param strip_base_dir [String] Remove this from the directory path for processing naming
# @param misc_prefix [String] Using this as the object prefix, if we can't determine what it should really be
# @param metadata_log [String] Filename of log file we will add fits metadata to, as a log of the processing.
Recursive copy of the directory to the object store bucket
Looks for .fit files, and if found, extracts fits metadata into a separate .info object.
Recursive copy of the directory to the object store bucket Looks for . fit files and if found extracts fits metadata into a separate . info object .
def object_put_dir(aws_fd:, directory:, strip_base_dir: nil, misc_prefix: 'misc', metadata_log: nil )
  test = false
  begin
    strip_base_dir.gsub!(/\/$/, '') if strip_base_dir != nil && strip_base_dir[-1] == '/' 
    misc_prefix.gsub!(/\/$/, '') if misc_prefix != nil && misc_prefix[-1] == '/' 
    Dir.open(directory).each do |filename|
      next unless filename != '.' && filename != '..' 
      qualified_filename = "#{directory}/#{filename}"
      begin
        stat_record = File.stat(qualified_filename) 
        begin
          if stat_record.symlink? == false 
            if stat_record.directory?
              object_put_dir(aws_fd: aws_fd, directory: qualified_filename, strip_base_dir: strip_base_dir, misc_prefix: misc_prefix, metadata_log: metadata_log)
            elsif stat_record.file?
              begin
                case File.extname(filename)
                when '.fit', '.fits' 
                  fm = Fits_metadata.new(directory: directory, filename: filename)
                  log_metadata(metadata_log: metadata_log, metadata: fm.full_metadata) if metadata_log != nil
                  puts "aws_fd.object_put_file(filename: #{qualified_filename}, key: #{fm.obj_path}/#{filename})"
                  if test
                    puts "aws_fd.object_put_file(info_file, key: #{fm.obj_path(filetype: 'info')}/#{fm.info_filename})"
                  else
                    aws_fd.object_put_file(filename: qualified_filename, key: "#{fm.obj_path}/#{filename}", metadata: fm.obj_metadata, not_if_exits: true)
                    if fm.info_filename != nil
                      puts "aws_fd.object_put_file(info_file, key: #{fm.obj_path(filetype: 'info')}/#{fm.info_filename})"
                      aws_fd.object_put_mem(content: fm.raw_fits_metadata,
                                            key: "#{fm.obj_path(filetype: 'info')}/#{fm.info_filename}",
                                            metadata: { 'fits_url' => "#{fm.obj_path}/#{filename}" }, 
                                            not_if_exits: true
                                           )
                    end
                  end
                when '.info' 
                  fm = Fits_metadata.new(directory: directory, filename: filename)
                  puts "aws_fd.object_put_file(filename: #{qualified_filename}, key: #{fm.obj_path(filetype: 'info')}/#{filename})"
                  aws_fd.object_put_file(filename: qualified_filename,
                                         key: "#{fm.obj_path(filetype: 'info')}/#{filename}",
                                         metadata: { 'fits_url' => "#{fm.obj_path}/#{filename.gsub(/info$/, 'fit')}" },  
                                         not_if_exits: true
                                        )
                when '.dat' 
                  diphot = Diphot_metadata.new(directory: directory, filename: filename)
                  puts "aws_fd.object_put_file(filename: #{qualified_filename}, key: #{diphot.obj_path}/#{filename})"
                  aws_fd.object_put_file(filename: qualified_filename, key: "#{diphot.obj_path}/#{filename}", metadata: diphot.obj_metadata, not_if_exits: true)
                else
                  object_name = strip_base_dir != nil && strip_base_dir != '' ? qualified_filename.gsub(/^#{strip_base_dir}\//, '') : filename
                  object_name = misc_prefix != nil && misc_prefix != '' ? "#{misc_prefix}/#{object_name}" : object_name
                  puts "aws_fd.object_put_file(filename: #{qualified_filename}, key: #{object_name})"
                  aws_fd.object_put_file(filename: qualified_filename, key: object_name, not_if_exits: true)
                end
              rescue S3_Exists => _e 
              rescue StandardError => e
                $stderr.puts "Copy of \"#{qualified_filename}\" to object store failed: #{e}"
              end
            end
          end
        end
      rescue StandardError => e
        $stderr.puts "Stat of #{qualified_filename} failed with error: #{e}"
      end
    end
  rescue StandardError => e
    $stderr.puts "object_put_dir(#{directory}) : #{e}"
  end
  puts "Completed Dir #{directory}"
end
def object_put_dir ( aws_fd : , directory : , strip_base_dir : nil , misc_prefix : 'misc' , metadata_log : nil ) test = false begin strip_base_dir . gsub! ( / \/ $ / , '' ) if strip_base_dir != nil && strip_base_dir [ - 1 ] == '/' misc_prefix . gsub! ( / \/ $ / , '' ) if misc_prefix != nil && misc_prefix [ - 1 ] == '/' Dir . open ( directory ) . each do | filename | next unless filename != '.' && filename != '..' qualified_filename = "#{directory}/#{filename}" begin stat_record = File . stat ( qualified_filename ) begin if stat_record . symlink? == false if stat_record . directory? object_put_dir ( aws_fd : aws_fd , directory : qualified_filename , strip_base_dir : strip_base_dir , misc_prefix : misc_prefix , metadata_log : metadata_log ) elsif stat_record . file? begin case File . extname ( filename ) when '.fit' , '.fits' fm = Fits_metadata . new ( directory : directory , filename : filename ) log_metadata ( metadata_log : metadata_log , metadata : fm . full_metadata ) if metadata_log != nil puts "aws_fd.object_put_file(filename: #{qualified_filename}, key: #{fm.obj_path}/#{filename})" if test puts "aws_fd.object_put_file(info_file, key: #{fm.obj_path(filetype: 'info')}/#{fm.info_filename})" else aws_fd . object_put_file ( filename : qualified_filename , key : "#{fm.obj_path}/#{filename}" , metadata : fm . obj_metadata , not_if_exits : true ) if fm . info_filename != nil puts "aws_fd.object_put_file(info_file, key: #{fm.obj_path(filetype: 'info')}/#{fm.info_filename})" aws_fd . object_put_mem ( content : fm . raw_fits_metadata , key : "#{fm.obj_path(filetype: 'info')}/#{fm.info_filename}" , metadata : { 'fits_url' => "#{fm.obj_path}/#{filename}" } , not_if_exits : true ) end end when '.info' fm = Fits_metadata . new ( directory : directory , filename : filename ) puts "aws_fd.object_put_file(filename: #{qualified_filename}, key: #{fm.obj_path(filetype: 'info')}/#{filename})" aws_fd . object_put_file ( filename : qualified_filename , key : "#{fm.obj_path(filetype: 'info')}/#{filename}" , metadata : { 'fits_url' => "#{fm.obj_path}/#{filename.gsub(/info$/, 'fit')}" } , not_if_exits : true ) when '.dat' diphot = Diphot_metadata . new ( directory : directory , filename : filename ) puts "aws_fd.object_put_file(filename: #{qualified_filename}, key: #{diphot.obj_path}/#{filename})" aws_fd . object_put_file ( filename : qualified_filename , key : "#{diphot.obj_path}/#{filename}" , metadata : diphot . obj_metadata , not_if_exits : true ) else object_name = strip_base_dir != nil && strip_base_dir != '' ? qualified_filename . gsub ( / ^ #{ strip_base_dir } \/ / , '' ) : filename object_name = misc_prefix != nil && misc_prefix != '' ? "#{misc_prefix}/#{object_name}" : object_name puts "aws_fd.object_put_file(filename: #{qualified_filename}, key: #{object_name})" aws_fd . object_put_file ( filename : qualified_filename , key : object_name , not_if_exits : true ) end rescue S3_Exists => _e rescue StandardError => e $stderr . puts "Copy of \"#{qualified_filename}\" to object store failed: #{e}" end end end end rescue StandardError => e $stderr . puts "Stat of #{qualified_filename} failed with error: #{e}" end end rescue StandardError => e $stderr . puts "object_put_dir(#{directory}) : #{e}" end puts "Completed Dir #{directory}" end
Recursive copy of the directory to the object store bucket
Looks for .fit files, and if found, extracts fits metadata into a separate .info object.
Recursive copy of the directory to the object store bucket Looks for . fit files and if found extracts fits metadata into a separate . info object .
# Will reinsert a / later. # Will reinsert a / later. # ignore parent, and current directories. # It is possible for this to cause an exception if symlink points no where. # Otherwise ignore # recurse through sub-directories. # Process files # Astro image format. # Also create a .info object (Should get skipped if object exists in store, so raised an S3_Exists exception) # reference to the fits object as metadata # Dumps of the .fit file metadata into a separate file # reference to the fits object as metadata # Diphot difference files # Get this if the file existed in the object store with the same MD5 sum. # puts e
aws_fd
S3 connection to the object store
S3 connection to the object store
directory
Top level directory to recursively copy from
Top level directory to recursively copy from
strip_base_dir
Remove this from the directory path for processing naming
Remove this from the directory path for processing naming
misc_prefix
Using this as the object prefix, if we can't determine what it should really be
Using this as the object prefix if we can ' t determine what it should really be
metadata_log
Filename of log file we will add fits metadata to, as a log of the processing.
Filename of log file we will add fits metadata to as a log of the processing .