lib/linguist/classifier.rb
# Internal: show a table of probabilities for each <token,language> pair.
#
# The number in each table entry is the number of "points" that each
# token contributes toward the belief that the file under test is a
# particular language.  Points are additive.
#
# Points are the number of times a token appears in the file, times
# how much more likely (log of probability ratio) that token is to
# appear in one language vs. the least-likely language.  Dashes
# indicate the least-likely language (and zero points) for each token.
show a table of probabilities for each  pair.
The number in each table entry is the number of "points" that each
token contributes toward the belief that the file under test is a
particular language.  Points are additive.

Points are the number of times a token appears in the file, times
how much more likely (log of probability ratio) that token is to
appear in one language vs. the least-likely language.  Dashes
indicate the least-likely language (and zero points) for each token.
show a table of probabilities for each pair . The number in each table entry is the number of " points " that each token contributes toward the belief that the file under test is a particular language . Points are additive . Points are the number of times a token appears in the file times how much more likely ( log of probability ratio ) that token is to appear in one language vs . the least - likely language . Dashes indicate the least - likely language ( and zero points ) for each token .
def debug_dump_all_tokens(tokens, languages)
        maxlen = tokens.map { |tok| tok.size }.max
        printf "%#{maxlen}s", ""
        puts "    #" + languages.map { |lang| sprintf("%10s", lang) }.join
        token_map = Hash.new(0)
        tokens.each { |tok| token_map[tok] += 1 }
        token_map.sort.each { |tok, count|
          arr = languages.map { |lang| [lang, token_probability(tok, lang)] }
          min = arr.map { |a,b| b }.min
          minlog = Math.log(min)
          if !arr.inject(true) { |result, n| result && n[1] == arr[0][1] }
            printf "%#{maxlen}s%5d", tok, count
            puts arr.map { |ent|
              ent[1] == min ? "         -" : sprintf("%10.3f", count * (Math.log(ent[1]) - minlog))
            }.join
          end
        }
      end
def debug_dump_all_tokens ( tokens , languages ) maxlen = tokens . map { | tok | tok . size } . max printf "%#{maxlen}s" , "" puts "    #" + languages . map { | lang | sprintf ( "%10s" , lang ) } . join token_map = Hash . new ( 0 ) tokens . each { | tok | token_map [ tok ] += 1 } token_map . sort . each { | tok , count | arr = languages . map { | lang | [ lang , token_probability ( tok , lang ) ] } min = arr . map { | a , b | b } . min minlog = Math . log ( min ) if ! arr . inject ( true ) { | result , n | result && n [ 1 ] == arr [ 0 ] [ 1 ] } printf "%#{maxlen}s%5d" , tok , count puts arr . map { | ent | ent [ 1 ] == min ? "         -" : sprintf ( "%10.3f" , count * ( Math . log ( ent [ 1 ] ) - minlog ) ) } . join end } end
Internal: show a table of probabilities for each <token,language> pair.
Internal : show a table of probabilities for each <token language > pair .
tokens
<not_specific>
languages
<not_specific>