src/English/relations/en_ComboRelationFinder.cpp
/** This confidence metric was hand-crafted based on Python scripts that
 *  analyzed the effect of various features and conditions on the system's
 *  precision.  See text/Projects/SERIF/experiments/relation-confidence
 *  in svn for the code that generated it.  In the long run, it would 
 *  probably be better to do this with a trained machine learning model; 
 *  or if we had a less ad-hoc relation model, it might be possible to use
 *  a simple analysis of features to come up with a confidence score.  But
 *  this will do for now.  It's tuned for English ACE, and may not work 
 *  well in other domains.  Here are the conditions it checks, and their
 *  (positive) impact on the confidence score:
 *
 *    0.152 overlap=='e=e'
 *    0.131 alt-model(LDC2004 PHYS.Part-Whole) (w=1.75)
 *    0.130 entity_types=='PER+GPE'
 *    0.126 rhs_entity_type=='GPE'
 *    0.124 rhs_entity_type!='PER'
 *    0.112 rhs_mention_type=='name'
 *    0.110 rhs_entity_subtype!='UNDET'
 *    0.103 reversed>9999.40587
 *    0.101 rhs_entity_subtype=='Nation'
 *    0.101 entity_types=='PER+ORG'
 *    0.099 alt-model(LDC2004 EMP-ORG.Employ-Staff) (w=2.12)
 *    0.097 reversed>9999.631035
 *    0.094 alt-model-enttype(LDC2004 PHYS.Part-Whole GPE GPE) (w=1.71)
 *
 *  The following graphs give an indication of the performance of the
 *  confidence metric (on heldout data):
 *
 *  1.000| Prec vs Score (Running avg)    100.0|      Precision vs Rank    :   
 *  0.857|                                 85.7|                           :   
 *  0.714|                      .:         71.4|                        .  :   
 *  0.571|                  .:  :: ...     57.1|                 .  .:. :..:   
 *  0.429|          ::... ::::.:::::::     42.9|           : :.  ::::::.::::   
 *  0.286|      : . ::::: ::::::::::::     28.6|       . .::::: .:::::::::::   
 *  0.143|   :: :.::::::::::::::::::::     14.3|.....:.:::::::::::::::::::::   
 *  0.000|::::::::::::::::::::::::::::      0.0|::::::::::::::::::::::::::::   
 *      0.00        0.44            1.0       0.00        0.44            1.0  
 * 
 *  1.000|:          Density              1.000|        Score vs Rank    .::   
 *  0.857|:                               0.857|                        .:::   
 *  0.714|:                               0.714|                   ...::::::   
 *  0.571|:               . .             0.571|             ..:::::::::::::   
 *  0.429|:              :: :        .    0.429|          .:::::::::::::::::   
 *  0.286|:  : ..  .  :..:: : :      :    0.286|       ..:::::::::::::::::::   
 *  0.143|: :: ::.::. ::::::: ::  :  :    0.143|    ..::::::::::::::::::::::   
 *  0.000|:_::::::::::::::::::::::::_:    0.000|____::::::::::::::::::::::::   
 *      0.00        0.44            1.0       0.00        0.44            1.0  
 * 
 *      Prec(top 10%): 56%       Gradient: 0.538
 *      Prec(top 25%): 54%      R-Squared: 0.118
 *      Prec(top 50%): 45%  Rank Gradient: 0.561
 *      Prec(bot 50%): 20%    x-intercept:-0.148
 *      Prec(bot 25%):  9%    y-intercept: 0.080
 */
This confidence metric was hand-crafted based on Python scripts that
analyzed the effect of various features and conditions on the system's
precision.  See text/Projects/SERIF/experiments/relation-confidence
in svn for the code that generated it.  In the long run, it would
probably be better to do this with a trained machine learning model;
or if we had a less ad-hoc relation model, it might be possible to use
a simple analysis of features to come up with a confidence score.  But
this will do for now.  It's tuned for English ACE, and may not work
well in other domains.  Here are the conditions it checks, and their
(positive) impact on the confidence score.



The following graphs give an indication of the performance of the
confidence metric (on heldout data).






This confidence metric was hand - crafted based on Python scripts that analyzed the effect of various features and conditions on the system ' s precision . See text / Projects / SERIF / experiments / relation - confidence in svn for the code that generated it . In the long run it would probably be better to do this with a trained machine learning model ; or if we had a less ad - hoc relation model it might be possible to use a simple analysis of features to come up with a confidence score . But this will do for now . It ' s tuned for English ACE and may not work well in other domains . Here are the conditions it checks and their ( positive ) impact on the confidence score . The following graphs give an indication of the performance of the confidence metric ( on heldout data ) .
double EnglishComboRelationFinder::calculateConfidenceScore(const Mention* mention1, const Mention* mention2, 
													 float reversed_score, Symbol answer, RelationObservation *obs) {
	double score = 0;
	if (DEBUG) 
		_debugStream << "Confidence Metric:\n";
	if (mention1->getNode()->getEndToken() == mention2->getNode()->getEndToken()) {
		score += 0.152;
		if (DEBUG) _debugStream << "  0.152 overlap=='e=e'\n";
	}
	for (int i = 0; i < obs->getNAltDecoderPredictions(); i++) {
		if ((obs->getNthAltDecoderName(i) == Symbol(L"LDC2004")) &&
			(obs->getNthAltDecoderPrediction(i) == Symbol(L"PHYS.Part-Whole"))) {
			score += 0.131;
			if (DEBUG) _debugStream << "  0.131 alt-model(LDC2004 PHYS.Part-Whole)\n";
		}
	}
	if ((mention1->getEntityType().matchesPER()) && 
		(mention2->getEntityType().matchesGPE())) {
		score += 0.130;
		if (DEBUG) _debugStream << "  0.130 entity_types=='PER+GPE'\n";
	}
	if (mention2->getEntityType().matchesGPE()) {
		score += 0.126;
		if (DEBUG) _debugStream << "  0.126 rhs_entity_type=='GPE'\n";
	}
	if (!mention2->getEntityType().matchesPER()) {
		score += 0.124;
		if (DEBUG) _debugStream << "  0.124 rhs_entity_type!='PER'\n";
	}
	if (mention2->getMentionType() == Mention::NAME) {
		score += 0.112;
		if (DEBUG) _debugStream << "  0.112 rhs_mention_type=='name'\n";
	}
	if (mention2->getEntitySubtype().isDetermined()) {
		score += 0.110;
		if (DEBUG) _debugStream << "  0.110 rhs_entity_subtype!='UNDET'\n";
	}
	if (reversed_score > 9999.40587) {
		score += 0.103;
		if (DEBUG) _debugStream << "  0.103 reversed>9999.40587\n";
	}
	if (mention2->getEntitySubtype().getName() == Symbol(L"Nation")) {
		score += 0.101;
		if (DEBUG) _debugStream << "  0.101 rhs_entity_subtype=='Nation'\n";
	}
	if ((mention1->getEntityType().matchesPER()) && 
		(mention2->getEntityType().matchesORG())) {
		score += 0.101;
		if (DEBUG) _debugStream << "  0.101 entity_types=='PER+ORG'\n";
	}
	for (int i = 0; i < obs->getNAltDecoderPredictions(); i++) {
		if ((obs->getNthAltDecoderName(i) == Symbol(L"LDC2004")) &&
			(obs->getNthAltDecoderPrediction(i) == Symbol(L"EMP-ORG.Employ-Staff"))) {
			score += 0.099;
			if (DEBUG) _debugStream << "  0.099 alt-model(LDC2004 EMP-ORG.Employ-Staff)\n";
		}
	}
	if (reversed_score > 9999.631035) {
		score += 0.097;
		if (DEBUG) _debugStream << "  0.097 reversed>9999.631035\n";
	}
	for (int i = 0; i < obs->getNAltDecoderPredictions(); i++) {
		if ((obs->getNthAltDecoderName(i) == Symbol(L"LDC2004")) &&
			(obs->getNthAltDecoderPrediction(i) == Symbol(L"PHYS.Part-Whole")) &&
			(mention1->getEntityType().matchesGPE()) && 
			(mention2->getEntityType().matchesGPE())) {
			score += 0.094;
			if (DEBUG) _debugStream << "  0.094 alt-model-enttype(LDC2004 PHYS.Part-Whole GPE GPE)\n";
		}
	}
	return std::min(score, 1.0);
}
double EnglishComboRelationFinder :: calculateConfidenceScore ( const Mention * mention1 , const Mention * mention2 , float reversed_score , Symbol answer , RelationObservation * obs ) { double score = 0 ; if ( DEBUG ) _debugStream << " \n " ; if ( mention1 -> getNode ( ) -> getEndToken ( ) == mention2 -> getNode ( ) -> getEndToken ( ) ) { score += 0.152 ; if ( DEBUG ) _debugStream << " \n " ; } for ( int i = 0 ; i < obs -> getNAltDecoderPredictions ( ) ; i ++ ) { if ( ( obs -> getNthAltDecoderName ( i ) == Symbol ( L" " ) ) && ( obs -> getNthAltDecoderPrediction ( i ) == Symbol ( L" " ) ) ) { score += 0.131 ; if ( DEBUG ) _debugStream << " \n " ; } } if ( ( mention1 -> getEntityType ( ) . matchesPER ( ) ) && ( mention2 -> getEntityType ( ) . matchesGPE ( ) ) ) { score += 0.130 ; if ( DEBUG ) _debugStream << " \n " ; } if ( mention2 -> getEntityType ( ) . matchesGPE ( ) ) { score += 0.126 ; if ( DEBUG ) _debugStream << " \n " ; } if ( ! mention2 -> getEntityType ( ) . matchesPER ( ) ) { score += 0.124 ; if ( DEBUG ) _debugStream << " \n " ; } if ( mention2 -> getMentionType ( ) == Mention :: NAME ) { score += 0.112 ; if ( DEBUG ) _debugStream << " \n " ; } if ( mention2 -> getEntitySubtype ( ) . isDetermined ( ) ) { score += 0.110 ; if ( DEBUG ) _debugStream << " \n " ; } if ( reversed_score > 9999.40587 ) { score += 0.103 ; if ( DEBUG ) _debugStream << " \n " ; } if ( mention2 -> getEntitySubtype ( ) . getName ( ) == Symbol ( L" " ) ) { score += 0.101 ; if ( DEBUG ) _debugStream << " \n " ; } if ( ( mention1 -> getEntityType ( ) . matchesPER ( ) ) && ( mention2 -> getEntityType ( ) . matchesORG ( ) ) ) { score += 0.101 ; if ( DEBUG ) _debugStream << " \n " ; } for ( int i = 0 ; i < obs -> getNAltDecoderPredictions ( ) ; i ++ ) { if ( ( obs -> getNthAltDecoderName ( i ) == Symbol ( L" " ) ) && ( obs -> getNthAltDecoderPrediction ( i ) == Symbol ( L" " ) ) ) { score += 0.099 ; if ( DEBUG ) _debugStream << " \n " ; } } if ( reversed_score > 9999.631035 ) { score += 0.097 ; if ( DEBUG ) _debugStream << " \n " ; } for ( int i = 0 ; i < obs -> getNAltDecoderPredictions ( ) ; i ++ ) { if ( ( obs -> getNthAltDecoderName ( i ) == Symbol ( L" " ) ) && ( obs -> getNthAltDecoderPrediction ( i ) == Symbol ( L" " ) ) && ( mention1 -> getEntityType ( ) . matchesGPE ( ) ) && ( mention2 -> getEntityType ( ) . matchesGPE ( ) ) ) { score += 0.094 ; if ( DEBUG ) _debugStream << " \n " ; } } return std :: min ( score , 1.0 ) ; }
This confidence metric was hand-crafted based on Python scripts that
  analyzed the effect of various features and conditions on the system's
  precision.
This confidence metric was hand - crafted based on Python scripts that analyzed the effect of various features and conditions on the system ' s precision .