public static SemanticParserLoss testSemanticParser(List<CcgExample> testExamples, CcgParser parser,
      CcgInference inferenceAlg, ExpressionSimplifier simplifier, ExpressionComparator comparator,
      List<SemanticParserExampleLoss> exampleLossAccumulator, boolean print) {

    int numCorrect = 0;
    int numCorrectLfPossible = 0;
    int numParsed = 0;

    LogFunction log = new NullLogFunction();
    for (CcgExample example : testExamples) {
      Expression2 correctLf = simplifier.apply(example.getLogicalForm());

      List<CcgParse> parses = inferenceAlg.beamSearch(parser, example.getSentence(), null, log);
      CountAccumulator<Expression2> lfProbs = estimateLfProbabilities(parses, simplifier);
      
      if (print) {
        System.out.println("====");
        System.out.println("SENT: " + example.getSentence().getWords());
      }

      if (lfProbs.keySet().size() > 0) {
        Expression2 best = lfProbs.getSortedKeys().get(0); 
        // Pick the best parse that produces this expression:
        CcgParse parse = null;
        for (CcgParse p : parses) {
          if (simplifier.apply(p.getLogicalForm()).equals(best)) {
            parse = p;
            break;
          }
        }
        Preconditions.checkState(parse != null);

        int correct = 0;
        int correctLfPossible = 0;
        Expression2 lf = lfProbs.getSortedKeys().get(0);

        correct = comparator.equals(lf, correctLf) ? 1 : 0;

        List<CcgParse> conditionalParses = CcgLoglikelihoodOracle.filterParsesByLogicalForm(
            correctLf, comparator, parses);
        if (conditionalParses.size() > 0) {
          correctLfPossible = 1;
        }

        List<DependencyStructure> deps = parse.getAllDependencies();

        if (print) {
          System.out.println("PREDICTED: " + lf);
          List<Expression2> sorted = lfProbs.getSortedKeys();
          int numToPrint = Math.min(sorted.size(), 5);
          for (int i = 0; i < numToPrint; i++) {
            Expression2 key = sorted.get(i);
            System.out.println("   " + lfProbs.getProbability(key) + " " + key);
          }

          System.out.println("TRUE:      " + correctLf);
          System.out.println("DEPS: " + deps);
          System.out.println("CORRECT: " + correct);
          System.out.println("LICENSED: " + correctLfPossible);
          System.out.println("LEX: ");

          List<LexiconEntryInfo> entries = parse.getSpannedLexiconEntries();
          for (int i = 0; i < entries.size(); i++) {
            System.out.println("   " + entries.get(i));
          }
        }

        numCorrect += correct;
        numCorrectLfPossible += correctLfPossible;
        numParsed++;

        if (exampleLossAccumulator != null) {
          exampleLossAccumulator.add(new SemanticParserExampleLoss(example, lf,
              correctLf, true, correct > 0, correctLfPossible > 0));
        }
      } else {
        if (print) {
          System.out.println("NO PARSE");
        }

        if (exampleLossAccumulator != null) {
          exampleLossAccumulator.add(new SemanticParserExampleLoss(example, null,
              correctLf, false, false, false));
        }
      }
    }

    double precision = ((double) numCorrect) / numParsed;
    double recall = ((double) numCorrect) / testExamples.size();
    double licensedRecall = ((double) numCorrectLfPossible) / testExamples.size();

    if (print) {
      System.out.println("\nPrecision: " + precision);
      System.out.println("Recall: " + recall);
      System.out.println("Licensed Recall: " + licensedRecall);
    }

    return new SemanticParserLoss(testExamples.size(), numParsed,
        numCorrect, numCorrectLfPossible);
  }
public static SemanticParserLoss testSemanticParser ( List < CcgExample > testExamples , CcgParser parser , CcgInference inferenceAlg , ExpressionSimplifier simplifier , ExpressionComparator comparator , List < SemanticParserExampleLoss > exampleLossAccumulator , boolean print ) { int numCorrect = 0 ; int numCorrectLfPossible = 0 ; int numParsed = 0 ; LogFunction log = new NullLogFunction ( ) ; for ( CcgExample example : testExamples ) { Expression2 correctLf = simplifier . apply ( example . getLogicalForm ( ) ) ; List < CcgParse > parses = inferenceAlg . beamSearch ( parser , example . getSentence ( ) , null , log ) ; CountAccumulator < Expression2 > lfProbs = estimateLfProbabilities ( parses , simplifier ) ; if ( print ) { System . out . println ( "====" ) ; System . out . println ( "SENT: " + example . getSentence ( ) . getWords ( ) ) ; } if ( lfProbs . keySet ( ) . size ( ) > 0 ) { Expression2 best = lfProbs . getSortedKeys ( ) . get ( 0 ) ; CcgParse parse = null ; for ( CcgParse p : parses ) { if ( simplifier . apply ( p . getLogicalForm ( ) ) . equals ( best ) ) { parse = p ; break ; } } Preconditions . checkState ( parse != null ) ; int correct = 0 ; int correctLfPossible = 0 ; Expression2 lf = lfProbs . getSortedKeys ( ) . get ( 0 ) ; correct = comparator . equals ( lf , correctLf ) ? 1 : 0 ; List < CcgParse > conditionalParses = CcgLoglikelihoodOracle . filterParsesByLogicalForm ( correctLf , comparator , parses ) ; if ( conditionalParses . size ( ) > 0 ) { correctLfPossible = 1 ; } List < DependencyStructure > deps = parse . getAllDependencies ( ) ; if ( print ) { System . out . println ( "PREDICTED: " + lf ) ; List < Expression2 > sorted = lfProbs . getSortedKeys ( ) ; int numToPrint = Math . min ( sorted . size ( ) , 5 ) ; for ( int i = 0 ; i < numToPrint ; i ++ ) { Expression2 key = sorted . get ( i ) ; System . out . println ( "   " + lfProbs . getProbability ( key ) + " " + key ) ; } System . out . println ( "TRUE:      " + correctLf ) ; System . out . println ( "DEPS: " + deps ) ; System . out . println ( "CORRECT: " + correct ) ; System . out . println ( "LICENSED: " + correctLfPossible ) ; System . out . println ( "LEX: " ) ; List < LexiconEntryInfo > entries = parse . getSpannedLexiconEntries ( ) ; for ( int i = 0 ; i < entries . size ( ) ; i ++ ) { System . out . println ( "   " + entries . get ( i ) ) ; } } numCorrect += correct ; numCorrectLfPossible += correctLfPossible ; numParsed ++ ; if ( exampleLossAccumulator != null ) { exampleLossAccumulator . add ( new SemanticParserExampleLoss ( example , lf , correctLf , true , correct > 0 , correctLfPossible > 0 ) ) ; } } else { if ( print ) { System . out . println ( "NO PARSE" ) ; } if ( exampleLossAccumulator != null ) { exampleLossAccumulator . add ( new SemanticParserExampleLoss ( example , null , correctLf , false , false , false ) ) ; } } } double precision = ( ( double ) numCorrect ) / numParsed ; double recall = ( ( double ) numCorrect ) / testExamples . size ( ) ; double licensedRecall = ( ( double ) numCorrectLfPossible ) / testExamples . size ( ) ; if ( print ) { System . out . println ( "\nPrecision: " + precision ) ; System . out . println ( "Recall: " + recall ) ; System . out . println ( "Licensed Recall: " + licensedRecall ) ; } return new SemanticParserLoss ( testExamples . size ( ) , numParsed , numCorrect , numCorrectLfPossible ) ; }
Evaluate {@code parser} by measuring its precision and
recall at reproducing the annotated logical forms in
{@code testExamples}.

@param testExamples
@param parser
@param inferenceAlg
@param simplifier
@param comparator
@param exampleLossAccumulator if non-null, the predictions for each example are
added to this list.
@param print if true, print out loss information.
@return