lib/browser_web_data_entity_sumarization/entity_sumarization_statistics.rb
###
# The method process all generated result files from nif dataset (by entity class type)
# to one result knowledge base file.
#
# @param [String] type Type from http://mappings.dbpedia.org/server/ontology/classes/
# @param [TrueClass, FalseClass] identify_identical Flag for process identify and group identical properties as one item.
The method process all generated result files from nif dataset (by entity class type)
to one result knowledge base file.
The method process all generated result files from nif dataset ( by entity class type ) to one result knowledge base file .
def generate_knowledge_base_for_entity(type, identify_identical = true)
        puts "_____ #{type} _____" if @console_output
        files = Dir.glob("#{@results_dir_path}/#{type}/*.json")
        type = type.to_s.to_sym
        knowledge_data = {type => []}
        global_properties = get_global_statistic_by_type(type) || {}
        if identify_identical
          try_this_identical = {}
          files.each { |file_path|
            file_data = JSON.parse(File.read(file_path).force_encoding('utf-8'), symbolize_names: true)
            file_data[:nif_data].each { |data|
              try_this_identical.merge!(data[:properties][type]) { |_, x, y| x + y }
            }
          }
          try_this_identical.merge!(global_properties) { |_, x, y| x + y }
          if try_this_identical.size > 0
            try_this_identical = Hash[try_this_identical.sort_by { |_, v| v }.reverse]
            puts "- prepare to identify identical: total count #{try_this_identical.size}" if @console_output
            @predicates_similarity.identify_identical_predicates(try_this_identical.keys)
          end
        end
        puts "- calculate: files count #{files.size}" if @console_output
        files.each { |file_path|
          file_data = JSON.parse(File.read(file_path).force_encoding('utf-8'), symbolize_names: true)
          file_data[:nif_data].each { |found|
            properties = found[:properties][type.to_sym]
            strict_properties = (found[:strict_properties] ||{})[type] || {}
            weight = found[:weight]
            strict_properties.each { |property, count|
              property = property.to_s
              value = count.to_i * weight
              prepare_property_to_knowledge(property, knowledge_data[type]) { |from_knowledge|
                old_score = from_knowledge[:score] * from_knowledge[:counter]
                from_knowledge[:counter] += 1
                (old_score + value) / from_knowledge[:counter]
              }
            }
            properties.each { |property, count|
              property = property.to_s
              value = count.to_i * weight
              prepare_property_to_knowledge(property, knowledge_data[type]) { |from_knowledge|
                old_score = from_knowledge[:score] * from_knowledge[:counter]
                from_knowledge[:counter] += 1
                (old_score + value) / from_knowledge[:counter]
              }
            }
          }
          unless knowledge_data[type].empty?
            max_weight = knowledge_data[type].max_by { |data| data[:score] }[:score]
            knowledge_data[type] = knowledge_data[type].map { |hash|
              hash[:score] = (hash[:score] / max_weight).round(4)
              hash
            }
          end
        }
        if global_properties.size > 0
          max_count = global_properties.max_by { |_, count| count }[1].to_f
          global_properties.each { |property, count|
            value = count / max_count
            prepare_property_to_knowledge(property, knowledge_data[type]) { |from_knowledge|
              from_knowledge[:score] > 0 ? ((from_knowledge[:score] + value) / 2.0).round(4) : value.round(4)
            }
          }
        end
        knowledge_data[type].map! { |hash|
          hash.delete(:counter)
          hash
        }
        knowledge_data[type] = knowledge_data[type].keep_if { |hash|
          hash[:score] > 0
        }.sort_by { |hash|
          hash[:score]
        }.reverse
        if identify_identical
          @predicates_similarity.reduce_identical
        end
        update_knowledge_base(knowledge_data)
      end
def generate_knowledge_base_for_entity ( type , identify_identical = true ) puts "_____ #{type} _____" if @console_output files = Dir . glob ( "#{@results_dir_path}/#{type}/*.json" ) type = type . to_s . to_sym knowledge_data = { type => [ ] } global_properties = get_global_statistic_by_type ( type ) || { } if identify_identical try_this_identical = { } files . each { | file_path | file_data = JSON . parse ( File . read ( file_path ) . force_encoding ( 'utf-8' ) , symbolize_names : true ) file_data [ :nif_data ] . each { | data | try_this_identical . merge! ( data [ :properties ] [ type ] ) { | _ , x , y | x + y } } } try_this_identical . merge! ( global_properties ) { | _ , x , y | x + y } if try_this_identical . size > 0 try_this_identical = Hash [ try_this_identical . sort_by { | _ , v | v } . reverse ] puts "- prepare to identify identical: total count #{try_this_identical.size}" if @console_output @predicates_similarity . identify_identical_predicates ( try_this_identical . keys ) end end puts "- calculate: files count #{files.size}" if @console_output files . each { | file_path | file_data = JSON . parse ( File . read ( file_path ) . force_encoding ( 'utf-8' ) , symbolize_names : true ) file_data [ :nif_data ] . each { | found | properties = found [ :properties ] [ type . to_sym ] strict_properties = ( found [ :strict_properties ] || { } ) [ type ] || { } weight = found [ :weight ] strict_properties . each { | property , count | property = property . to_s value = count . to_i * weight prepare_property_to_knowledge ( property , knowledge_data [ type ] ) { | from_knowledge | old_score = from_knowledge [ :score ] * from_knowledge [ :counter ] from_knowledge [ :counter ] += 1 ( old_score + value ) / from_knowledge [ :counter ] } } properties . each { | property , count | property = property . to_s value = count . to_i * weight prepare_property_to_knowledge ( property , knowledge_data [ type ] ) { | from_knowledge | old_score = from_knowledge [ :score ] * from_knowledge [ :counter ] from_knowledge [ :counter ] += 1 ( old_score + value ) / from_knowledge [ :counter ] } } } unless knowledge_data [ type ] . empty? max_weight = knowledge_data [ type ] . max_by { | data | data [ :score ] } [ :score ] knowledge_data [ type ] = knowledge_data [ type ] . map { | hash | hash [ :score ] = ( hash [ :score ] / max_weight ) . round ( 4 ) hash } end } if global_properties . size > 0 max_count = global_properties . max_by { | _ , count | count } [ 1 ] . to_f global_properties . each { | property , count | value = count / max_count prepare_property_to_knowledge ( property , knowledge_data [ type ] ) { | from_knowledge | from_knowledge [ :score ] > 0 ? ( ( from_knowledge [ :score ] + value ) / 2.0 ) . round ( 4 ) : value . round ( 4 ) } } end knowledge_data [ type ] . map! { | hash | hash . delete ( :counter ) hash } knowledge_data [ type ] = knowledge_data [ type ] . keep_if { | hash | hash [ :score ] > 0 } . sort_by { | hash | hash [ :score ] } . reverse if identify_identical @predicates_similarity . reduce_identical end update_knowledge_base ( knowledge_data ) end
The method process all generated result files from nif dataset (by entity class type)
to one result knowledge base file.
The method process all generated result files from nif dataset ( by entity class type ) to one result knowledge base file .
type
<not_specific>
identify_identical
<not_specific>
<not_specific>
identify_identical Flag for process identify and group identical properties as one item.
identify_identical Flag for process identify and group identical properties as one item .