public static void main(String[] args) {
    if (args.length < 3) {
      System.err.println("java BaseLexicon treebankPath fileRange unknownWordModel words*");
      return;
    }
    System.out.print("Training BaseLexicon from " + args[0] + ' ' + args[1] + " ... ");
    Treebank tb = new DiskTreebank();
    tb.loadPath(args[0], new NumberRangesFileFilter(args[1], true));
    // TODO: change this interface so the lexicon creates its own indices?
    Index<String> wordIndex = new HashIndex<String>();
    Index<String> tagIndex = new HashIndex<String>();
    BaseLexicon lex = new BaseLexicon(wordIndex, tagIndex);
    lex.getUnknownWordModel().setUnknownLevel(Integer.parseInt(args[2]));
    lex.initializeTraining(tb.size());
    lex.train(tb);
    lex.finishTraining();
    System.out.println("done.");
    System.out.println();
    NumberFormat nf = NumberFormat.getNumberInstance();
    nf.setMaximumFractionDigits(4);
    List<String> impos = new ArrayList<String>();
    for (int i = 3; i < args.length; i++) {
      if (lex.isKnown(args[i])) {
        System.out.println(args[i] + " is a known word.  Log probabilities [log P(w|t)] for its taggings are:");
        for (Iterator<IntTaggedWord> it = lex.ruleIteratorByWord(wordIndex.indexOf(args[i], true), i - 3, null); it.hasNext(); ) {
          IntTaggedWord iTW = it.next();
          System.out.println(StringUtils.pad(iTW, 24) + nf.format(lex.score(iTW, i - 3, wordIndex.get(iTW.word))));
        }
      } else {
        String sig = lex.getUnknownWordModel().getSignature(args[i], i-3);
        System.out.println(args[i] + " is an unknown word.  Signature with uwm " + lex.getUnknownWordModel().getUnknownLevel() + ((i == 3) ? " init": "non-init") + " is: " + sig);
        impos.clear();
        List<String> lis = new ArrayList<String>(tagIndex.objectsList());
        Collections.sort(lis);
        for (String tStr : lis) {
          IntTaggedWord iTW = new IntTaggedWord(args[i], tStr, wordIndex, tagIndex);
          double score = lex.score(iTW, 1, args[i]);
          if (score == Float.NEGATIVE_INFINITY) {
            impos.add(tStr);
          } else {
            System.out.println(StringUtils.pad(iTW, 24) + nf.format(score));
          }
        }
        if (impos.size() > 0) {
          System.out.println(args[i] + " impossible tags: " + impos);
        }
      }
      System.out.println();
    }
  }
public static void main ( String [ ] args ) { if ( args . length < 3 ) { System . err . println ( "java BaseLexicon treebankPath fileRange unknownWordModel words*" ) ; return ; } System . out . print ( "Training BaseLexicon from " + args [ 0 ] + ' ' + args [ 1 ] + " ... " ) ; Treebank tb = new DiskTreebank ( ) ; tb . loadPath ( args [ 0 ] , new NumberRangesFileFilter ( args [ 1 ] , true ) ) ; Index < String > wordIndex = new HashIndex < String > ( ) ; Index < String > tagIndex = new HashIndex < String > ( ) ; BaseLexicon lex = new BaseLexicon ( wordIndex , tagIndex ) ; lex . getUnknownWordModel ( ) . setUnknownLevel ( Integer . parseInt ( args [ 2 ] ) ) ; lex . initializeTraining ( tb . size ( ) ) ; lex . train ( tb ) ; lex . finishTraining ( ) ; System . out . println ( "done." ) ; System . out . println ( ) ; NumberFormat nf = NumberFormat . getNumberInstance ( ) ; nf . setMaximumFractionDigits ( 4 ) ; List < String > impos = new ArrayList < String > ( ) ; for ( int i = 3 ; i < args . length ; i ++ ) { if ( lex . isKnown ( args [ i ] ) ) { System . out . println ( args [ i ] + " is a known word.  Log probabilities [log P(w|t)] for its taggings are:" ) ; for ( Iterator < IntTaggedWord > it = lex . ruleIteratorByWord ( wordIndex . indexOf ( args [ i ] , true ) , i - 3 , null ) ; it . hasNext ( ) ; ) { IntTaggedWord iTW = it . next ( ) ; System . out . println ( StringUtils . pad ( iTW , 24 ) + nf . format ( lex . score ( iTW , i - 3 , wordIndex . get ( iTW . word ) ) ) ) ; } } else { String sig = lex . getUnknownWordModel ( ) . getSignature ( args [ i ] , i - 3 ) ; System . out . println ( args [ i ] + " is an unknown word.  Signature with uwm " + lex . getUnknownWordModel ( ) . getUnknownLevel ( ) + ( ( i == 3 ) ? " init" : "non-init" ) + " is: " + sig ) ; impos . clear ( ) ; List < String > lis = new ArrayList < String > ( tagIndex . objectsList ( ) ) ; Collections . sort ( lis ) ; for ( String tStr : lis ) { IntTaggedWord iTW = new IntTaggedWord ( args [ i ] , tStr , wordIndex , tagIndex ) ; double score = lex . score ( iTW , 1 , args [ i ] ) ; if ( score == Float . NEGATIVE_INFINITY ) { impos . add ( tStr ) ; } else { System . out . println ( StringUtils . pad ( iTW , 24 ) + nf . format ( score ) ) ; } } if ( impos . size ( ) > 0 ) { System . out . println ( args [ i ] + " impossible tags: " + impos ) ; } } System . out . println ( ) ; } }
Provides some testing and opportunities for exploration of the
probabilities of a BaseLexicon.  What's here currently probably
only works for the English Penn Treeebank, as it uses default
constructors.  Of the words given to test on,
the first is treated as sentence initial, and the rest as not
sentence initial.

@param args The command line arguments:
java BaseLexicon treebankPath fileRange unknownWordModel words*