app/jobs/marc_profiling_job.rb
# Tally up some statistics about the MARC records in the file:
#   - how many times a field/subfield is used
#   - how many records have a field/subfield
#   - how many times a field/subfield is used in a record
# and collect a sampling of field/subfield occurences.
# rubocop:disable all (Ha!)
Tally up some statistics about the MARC records in the file:
how many times a field/subfield is used
how many records have a field/subfield
how many times a field/subfield is used in a record
and collect a sampling of field/subfield occurences.
rubocop:disable all (Ha!)
Tally up some statistics about the MARC records in the file : how many times a field / subfield is used how many records have a field / subfield how many times a field / subfield is used in a record and collect a sampling of field / subfield occurences . rubocop : disable all ( Ha! )
def perform(blob, count: nil, sample_size: 25)
    count ||= blob.metadata['count']
    return unless count
    progress.total = count
    sampled_values = Hash.new { |hash, key| hash[key] = Sample.new(sample_size, count) }
    instance_frequency = Hash.new { |hash, key| hash[key] = 0 }
    record_frequency = Hash.new { |hash, key| hash[key] = 0 }
    histogram_frequency = Hash.new { |hash, key| hash[key] = Hash.new { |h, k| h[k] = 0 } }
    marc_record_service(blob).each_slice(100) do |batch|
      progress.increment(batch.size)
      batch.each do |record|
        id = record['001']&.value
        record_stats = Hash.new { |hash, key| hash[key] = 0 }
        record_subfield_stats = Hash.new { |hash, key| hash[key] = 0 }
        record.fields.each do |field|
          record_frequency[field.tag] += 1 if record_stats[field.tag].zero?
          record_stats[field.tag] += 1
          instance_frequency[field.tag] += 1
          if field.is_a? MARC::ControlField
            sampled_values[field.tag].add([field.value, id])
          else
            sampled_values[field.tag].add([field.to_s, id])
            subfield_frequency = Hash.new { |hash, key| hash[key] = 0 }
            field.each do |subfield|
              key = "#{field.tag}$#{subfield.code}"
              sampled_values[key].add([subfield.value, id])
              record_frequency[key] += 1 if record_subfield_stats[key].zero?
              record_subfield_stats[key] += 1
              subfield_frequency[key] += 1
              instance_frequency[key] += 1
            end
            subfield_frequency.each do |key, count|
              histogram_frequency[key][count] += 1 unless count == 0
            end
          end
        end
        record_stats.each do |key, value|
          histogram_frequency[key][value] += 1
        end
      end
    end
    profile = MarcProfile.where(blob: blob).first_or_initialize do |p|
      p.upload = blob.attachments.first.record if blob.attachments.first.record.is_a? Upload
    end
    profile.update(
      field_frequency: instance_frequency,
      record_frequency: record_frequency,
      histogram_frequency: histogram_frequency,
      sampled_values: sampled_values.transform_values(&:to_a)
    )
  end
def perform ( blob , count : nil , sample_size : 25 ) count ||= blob . metadata [ 'count' ] return unless count progress . total = count sampled_values = Hash . new { | hash , key | hash [ key ] = Sample . new ( sample_size , count ) } instance_frequency = Hash . new { | hash , key | hash [ key ] = 0 } record_frequency = Hash . new { | hash , key | hash [ key ] = 0 } histogram_frequency = Hash . new { | hash , key | hash [ key ] = Hash . new { | h , k | h [ k ] = 0 } } marc_record_service ( blob ) . each_slice ( 100 ) do | batch | progress . increment ( batch . size ) batch . each do | record | id = record [ '001' ] &. value record_stats = Hash . new { | hash , key | hash [ key ] = 0 } record_subfield_stats = Hash . new { | hash , key | hash [ key ] = 0 } record . fields . each do | field | record_frequency [ field . tag ] += 1 if record_stats [ field . tag ] . zero? record_stats [ field . tag ] += 1 instance_frequency [ field . tag ] += 1 if field . is_a? MARC :: ControlField sampled_values [ field . tag ] . add ( [ field . value , id ] ) else sampled_values [ field . tag ] . add ( [ field . to_s , id ] ) subfield_frequency = Hash . new { | hash , key | hash [ key ] = 0 } field . each do | subfield | key = "#{field.tag}$#{subfield.code}" sampled_values [ key ] . add ( [ subfield . value , id ] ) record_frequency [ key ] += 1 if record_subfield_stats [ key ] . zero? record_subfield_stats [ key ] += 1 subfield_frequency [ key ] += 1 instance_frequency [ key ] += 1 end subfield_frequency . each do | key , count | histogram_frequency [ key ] [ count ] += 1 unless count == 0 end end end record_stats . each do | key , value | histogram_frequency [ key ] [ value ] += 1 end end end profile = MarcProfile . where ( blob : blob ) . first_or_initialize do | p | p . upload = blob . attachments . first . record if blob . attachments . first . record . is_a? Upload end profile . update ( field_frequency : instance_frequency , record_frequency : record_frequency , histogram_frequency : histogram_frequency , sampled_values : sampled_values . transform_values ( & :to_a ) ) end
Tally up some statistics about the MARC records in the file:
 how many times a field/subfield is used
 how many records have a field/subfield
 how many times a field/subfield is used in a record
and collect a sampling of field/subfield occurences.
Tally up some statistics about the MARC records in the file : how many times a field / subfield is used how many records have a field / subfield how many times a field / subfield is used in a record and collect a sampling of field / subfield occurences .
blob
<not_specific>
count
<not_specific>
sample_size
<not_specific>