public float score(IntTaggedWord iTW, int loc, String word) {
    // both actual
    double c_TW = seenCounter.getCount(iTW);
    // double x_TW = xferCounter.getCount(iTW);

    IntTaggedWord temp = new IntTaggedWord(iTW.word, nullTag);
    // word counts
    double c_W = seenCounter.getCount(temp);
    // double x_W = xferCounter.getCount(temp);

    // totals
    double total = seenCounter.getCount(NULL_ITW);
    double totalUnseen = uwModel.unSeenCounter().getCount(NULL_ITW);

    temp = new IntTaggedWord(nullWord, iTW.tag);
    // tag counts
    double c_T = seenCounter.getCount(temp);
    double c_Tunseen = uwModel.unSeenCounter().getCount(temp);

    double pb_W_T; // always set below

    if (DEBUG_LEXICON) {
      // dump info about last word
      if (iTW.word != debugLastWord) {
        if (debugLastWord >= 0 && debugPrefix != null) {
          // the 2nd conjunct in test above handles older serialized files
          EncodingPrintWriter.err.println(debugPrefix + debugProbs + debugNoProbs, "UTF-8");
        }
      }
    }

    boolean seen = (c_W > 0.0);

    if (seen) {

      // known word model for P(T|W)
      if (DEBUG_LEXICON_SCORE) {
        System.err.println("Lexicon.score " + wordIndex.get(iTW.word) + "/" + tagIndex.get(iTW.tag) + " as known word.");
      }

      // c_TW = Math.sqrt(c_TW); [cdm: funny math scaling? dunno who played with this]
      // c_TW += 0.5;

      double p_T_U;
      if (useSignatureForKnownSmoothing) { // only works for English currently
        p_T_U = getUnknownWordModel().scoreProbTagGivenWordSignature(iTW, loc, smooth[0], word);
        if (DEBUG_LEXICON_SCORE) System.err.println("With useSignatureForKnownSmoothing, P(T|U) is " + p_T_U + " rather than " + (c_Tunseen / totalUnseen));
      } else {
        p_T_U = c_Tunseen / totalUnseen;
      }
      double pb_T_W; // always set below

      if (DEBUG_LEXICON_SCORE) {
        System.err.println("c_W is " + c_W  + " mle = " + (c_TW/c_W)+ " smoothInUnknownsThresh is " +
                smoothInUnknownsThreshold + " base p_T_U is " + c_Tunseen + "/" + totalUnseen + " = " + p_T_U);
      }
      if (c_W > smoothInUnknownsThreshold && c_TW > 0.0 && c_W > 0.0) {
        // we've seen the word enough times to have confidence in its tagging
        pb_T_W = c_TW / c_W;
      } else {
        // we haven't seen the word enough times to have confidence in its
        // tagging
        if (smartMutation) {
          int numTags = tagIndex.size();
          if (m_TT == null || numTags != m_T.length) {
            buildPT_T();
          }
          p_T_U *= 0.1;
          // System.out.println("Checking "+iTW);
          for (int t = 0; t < numTags; t++) {
            IntTaggedWord iTW2 = new IntTaggedWord(iTW.word, t);
            double p_T_W2 = seenCounter.getCount(iTW2) / c_W;
            if (p_T_W2 > 0) {
              // System.out.println(" Observation of "+tagIndex.get(t)+"
              // ("+seenCounter.getCount(iTW2)+") mutated to
              // "+tagIndex.get(iTW.tag)+" at rate
              // "+(m_TT[tag][t]/m_T[t]));
              p_T_U += p_T_W2 * m_TT[iTW.tag][t] / m_T[t] * 0.9;
            }
          }
        }
        if (DEBUG_LEXICON_SCORE) {
          System.err.println("c_TW = " + c_TW + " c_W = " + c_W +
                             " p_T_U = " + p_T_U);
        }
        // double pb_T_W = (c_TW+smooth[1]*x_TW)/(c_W+smooth[1]*x_W);
        pb_T_W = (c_TW + smooth[1] * p_T_U) / (c_W + smooth[1]);
      }
      double p_T = (c_T / total);
      double p_W = (c_W / total);
      pb_W_T = Math.log(pb_T_W * p_W / p_T);

      if (DEBUG_LEXICON) {
        if (iTW.word != debugLastWord) {
          debugLastWord = iTW.word;
          debugLoc = loc;
          debugProbs = new StringBuilder();
          debugNoProbs = new StringBuilder("impossible: ");
          debugPrefix = "Lexicon: " + wordIndex.get(debugLastWord) + " (known): ";
        }
        if (pb_W_T > Double.NEGATIVE_INFINITY) {
          NumberFormat nf = NumberFormat.getNumberInstance();
          nf.setMaximumFractionDigits(3);
          debugProbs.append(tagIndex.get(iTW.tag) + ": cTW=" + c_TW + " c_T=" + c_T
                  + " pb_T_W=" + nf.format(pb_T_W) + " log pb_W_T=" + nf.format(pb_W_T)
                  + ", ");
          // debugProbs.append("\n" + "smartMutation=" + smartMutation + "
          // smoothInUnknownsThreshold=" + smoothInUnknownsThreshold + "
          // smooth0=" + smooth[0] + "smooth1=" + smooth[1] + " p_T_U=" + p_T_U
          // + " c_W=" + c_W);
        } else {
          debugNoProbs.append(tagIndex.get(iTW.tag)).append(' ');
        }
      } // end if (DEBUG_LEXICON)

    } else { // when unseen
      if (loc >= 0) {
        pb_W_T = getUnknownWordModel().score(iTW, loc, c_T, total, smooth[0], word);
      } else {
        // For negative we now do a weighted average for the dependency grammar :-)
        double pb_W0_T = getUnknownWordModel().score(iTW, 0, c_T, total, smooth[0], word);
        double pb_W1_T = getUnknownWordModel().score(iTW, 1, c_T, total, smooth[0], word);
        pb_W_T = Math.log((Math.exp(pb_W0_T) + 2 * Math.exp(pb_W1_T))/3);
      }
    }

    // Categorical cutoff if score is too low
    if (pb_W_T > -100.0) {
      return (float) pb_W_T;
    }
    return Float.NEGATIVE_INFINITY;
  }
public float score ( IntTaggedWord iTW , int loc , String word ) { double c_TW = seenCounter . getCount ( iTW ) ; IntTaggedWord temp = new IntTaggedWord ( iTW . word , nullTag ) ; double c_W = seenCounter . getCount ( temp ) ; double total = seenCounter . getCount ( NULL_ITW ) ; double totalUnseen = uwModel . unSeenCounter ( ) . getCount ( NULL_ITW ) ; temp = new IntTaggedWord ( nullWord , iTW . tag ) ; double c_T = seenCounter . getCount ( temp ) ; double c_Tunseen = uwModel . unSeenCounter ( ) . getCount ( temp ) ; double pb_W_T ; if ( DEBUG_LEXICON ) { if ( iTW . word != debugLastWord ) { if ( debugLastWord >= 0 && debugPrefix != null ) { EncodingPrintWriter . err . println ( debugPrefix + debugProbs + debugNoProbs , "UTF-8" ) ; } } } boolean seen = ( c_W > 0.0 ) ; if ( seen ) { if ( DEBUG_LEXICON_SCORE ) { System . err . println ( "Lexicon.score " + wordIndex . get ( iTW . word ) + "/" + tagIndex . get ( iTW . tag ) + " as known word." ) ; } double p_T_U ; if ( useSignatureForKnownSmoothing ) { p_T_U = getUnknownWordModel ( ) . scoreProbTagGivenWordSignature ( iTW , loc , smooth [ 0 ] , word ) ; if ( DEBUG_LEXICON_SCORE ) System . err . println ( "With useSignatureForKnownSmoothing, P(T|U) is " + p_T_U + " rather than " + ( c_Tunseen / totalUnseen ) ) ; } else { p_T_U = c_Tunseen / totalUnseen ; } double pb_T_W ; if ( DEBUG_LEXICON_SCORE ) { System . err . println ( "c_W is " + c_W + " mle = " + ( c_TW / c_W ) + " smoothInUnknownsThresh is " + smoothInUnknownsThreshold + " base p_T_U is " + c_Tunseen + "/" + totalUnseen + " = " + p_T_U ) ; } if ( c_W > smoothInUnknownsThreshold && c_TW > 0.0 && c_W > 0.0 ) { pb_T_W = c_TW / c_W ; } else { if ( smartMutation ) { int numTags = tagIndex . size ( ) ; if ( m_TT == null || numTags != m_T . length ) { buildPT_T ( ) ; } p_T_U *= 0.1 ; for ( int t = 0 ; t < numTags ; t ++ ) { IntTaggedWord iTW2 = new IntTaggedWord ( iTW . word , t ) ; double p_T_W2 = seenCounter . getCount ( iTW2 ) / c_W ; if ( p_T_W2 > 0 ) { p_T_U += p_T_W2 * m_TT [ iTW . tag ] [ t ] / m_T [ t ] * 0.9 ; } } } if ( DEBUG_LEXICON_SCORE ) { System . err . println ( "c_TW = " + c_TW + " c_W = " + c_W + " p_T_U = " + p_T_U ) ; } pb_T_W = ( c_TW + smooth [ 1 ] * p_T_U ) / ( c_W + smooth [ 1 ] ) ; } double p_T = ( c_T / total ) ; double p_W = ( c_W / total ) ; pb_W_T = Math . log ( pb_T_W * p_W / p_T ) ; if ( DEBUG_LEXICON ) { if ( iTW . word != debugLastWord ) { debugLastWord = iTW . word ; debugLoc = loc ; debugProbs = new StringBuilder ( ) ; debugNoProbs = new StringBuilder ( "impossible: " ) ; debugPrefix = "Lexicon: " + wordIndex . get ( debugLastWord ) + " (known): " ; } if ( pb_W_T > Double . NEGATIVE_INFINITY ) { NumberFormat nf = NumberFormat . getNumberInstance ( ) ; nf . setMaximumFractionDigits ( 3 ) ; debugProbs . append ( tagIndex . get ( iTW . tag ) + ": cTW=" + c_TW + " c_T=" + c_T + " pb_T_W=" + nf . format ( pb_T_W ) + " log pb_W_T=" + nf . format ( pb_W_T ) + ", " ) ; } else { debugNoProbs . append ( tagIndex . get ( iTW . tag ) ) . append ( ' ' ) ; } } } else { if ( loc >= 0 ) { pb_W_T = getUnknownWordModel ( ) . score ( iTW , loc , c_T , total , smooth [ 0 ] , word ) ; } else { double pb_W0_T = getUnknownWordModel ( ) . score ( iTW , 0 , c_T , total , smooth [ 0 ] , word ) ; double pb_W1_T = getUnknownWordModel ( ) . score ( iTW , 1 , c_T , total , smooth [ 0 ] , word ) ; pb_W_T = Math . log ( ( Math . exp ( pb_W0_T ) + 2 * Math . exp ( pb_W1_T ) ) / 3 ) ; } } if ( pb_W_T > - 100.0 ) { return ( float ) pb_W_T ; } return Float . NEGATIVE_INFINITY ; }
Get the score of this word with this tag (as an IntTaggedWord) at this
location. (Presumably an estimate of P(word | tag).)
<p>
<i>Implementation documentation:</i>
Seen:
c_W = count(W)      c_TW = count(T,W)
c_T = count(T)      c_Tunseen = count(T) among new words in 2nd half
total = count(seen words)   totalUnseen = count("unseen" words)
p_T_U = Pmle(T|"unseen")
pb_T_W = P(T|W). If (c_W &gt; smoothInUnknownsThreshold) = c_TW/c_W
Else (if not smart mutation) pb_T_W = bayes prior smooth[1] with p_T_U
p_T= Pmle(T)          p_W = Pmle(W)
pb_W_T = log(pb_T_W * p_W / p_T) [Bayes rule]
Note that this doesn't really properly reserve mass to unknowns.

Unseen:
c_TS = count(T,Sig|Unseen)      c_S = count(Sig)   c_T = count(T|Unseen)
c_U = totalUnseen above
p_T_U = Pmle(T|Unseen)
pb_T_S = Bayes smooth of Pmle(T|S) with P(T|Unseen) [smooth[0]]
pb_W_T = log(P(W|T)) inverted

@param iTW An IntTaggedWord pairing a word and POS tag
@param loc The position in the sentence. <i>In the default implementation
this is used only for unknown words to change their probability
distribution when sentence initial</i>
@return A float score, usually, log P(word|tag)