src/Generic/names/IdFModel.cpp
/**
 * Given dataTable (a set of events), fill in probability tables (future + history, 
 * # times occured), history tables (history, # times occured), and unique tables 
 * (history, # unique futures given this history). 
 *
 * Calculate the un-backed-off probabilities in the prob tables, then call 
 * deriveWordLambdas() to fill in the lambda tables, using the counts from 
 * the history and unique tables.
 */
Given dataTable (a set of events), fill in probability tables (future + history,
# times occured), history tables (history, # times occured), and unique tables
(history, # unique futures given this history).

Calculate the un-backed-off probabilities in the prob tables, then call
deriveWordLambdas() to fill in the lambda tables, using the counts from
the history and unique tables.
Given dataTable ( a set of events ) fill in probability tables ( future + history # times occured ) history tables ( history # times occured ) and unique tables ( history # unique futures given this history ) . Calculate the un - backed - off probabilities in the prob tables then call deriveWordLambdas () to fill in the lambda tables using the counts from the history and unique tables .
void IdFModel::deriveWordTables(NgramScoreTable* dataTable) {
	if (DECODE) {
		SessionLogger::info("SERIF") << "can't derive word tables during decode\n";
		return;
	}
	NgramScoreTable::Table::iterator iter;
	for (iter = dataTable->get_start(); iter != dataTable->get_end(); ++iter) {
		Symbol ngram[3];
		ngram[0] = (*iter).first[0]; 
		ngram[1] = (*iter).first[1]; 
		_wordTagProbabilities->add(ngram, (*iter).second);
		_wordTagHistories->add(ngram + 1, (*iter).second);
		ngram[1] = (*iter).first[2]; 
		_wordReducedTagProbabilities->add(ngram, (*iter).second);
		_wordReducedTagHistories->add(ngram + 1, (*iter).second);
		if (_nameClassTags->isContinue((*iter).first[3])) {
			ngram[0] = (*iter).first[0]; 
			ngram[1] = (*iter).first[1]; 
			ngram[2] = (*iter).first[5]; 
			_wordContinueProbabilities->add(ngram, (*iter).second);
			_wordContinueHistories->add(ngram + 1, (*iter).second);
		}
		else {		
			ngram[0] = (*iter).first[0]; 
			ngram[1] = (*iter).first[1]; 
			ngram[2] = (*iter).first[4]; 
			_wordStartProbabilities->add(ngram, (*iter).second);
			_wordStartHistories->add(ngram + 1, (*iter).second);
		}
	}
	for (iter = _wordStartProbabilities->get_start(); iter != _wordStartProbabilities->get_end(); ++iter) {
		_startUniqueCounts->add((*iter).first + 1, 1);
		float transitions = (*iter).second;
		float histories = _wordStartHistories->lookup((*iter).first + 1);
		(*iter).second = transitions / histories;
	}
	for (iter = _wordContinueProbabilities->get_start(); iter != _wordContinueProbabilities->get_end(); ++iter) {
		_continueUniqueCounts->add((*iter).first + 1, 1);
		float transitions = (*iter).second;
		float histories = _wordContinueHistories->lookup((*iter).first + 1);
		(*iter).second = transitions / histories;
	}
	for (iter = _wordTagProbabilities->get_start(); iter != _wordTagProbabilities->get_end(); ++iter) {
		_wordTagUniqueCounts->add((*iter).first + 1, 1);
		float transitions = (*iter).second;
		float histories = _wordTagHistories->lookup((*iter).first + 1);
		(*iter).second = transitions / histories;
	}
	for (iter = _wordReducedTagProbabilities->get_start(); iter != _wordReducedTagProbabilities->get_end(); ++iter) {
		_wordReducedTagUniqueCounts->add((*iter).first + 1, 1);
		float transitions = (*iter).second;
		float histories = _wordReducedTagHistories->lookup((*iter).first + 1);
		(*iter).second = transitions / histories;
	}
	deriveWordLambdas();
}
void IdFModel :: deriveWordTables ( NgramScoreTable * dataTable ) { if ( DECODE ) { SessionLogger :: info ( " " ) << " \n " ; return ; } NgramScoreTable :: Table :: iterator iter ; for ( iter = dataTable -> get_start ( ) ; iter != dataTable -> get_end ( ) ; ++ iter ) { Symbol ngram [ 3 ] ; ngram [ 0 ] = ( * iter ) . first [ 0 ] ; ngram [ 1 ] = ( * iter ) . first [ 1 ] ; _wordTagProbabilities -> add ( ngram , ( * iter ) . second ) ; _wordTagHistories -> add ( ngram + 1 , ( * iter ) . second ) ; ngram [ 1 ] = ( * iter ) . first [ 2 ] ; _wordReducedTagProbabilities -> add ( ngram , ( * iter ) . second ) ; _wordReducedTagHistories -> add ( ngram + 1 , ( * iter ) . second ) ; if ( _nameClassTags -> isContinue ( ( * iter ) . first [ 3 ] ) ) { ngram [ 0 ] = ( * iter ) . first [ 0 ] ; ngram [ 1 ] = ( * iter ) . first [ 1 ] ; ngram [ 2 ] = ( * iter ) . first [ 5 ] ; _wordContinueProbabilities -> add ( ngram , ( * iter ) . second ) ; _wordContinueHistories -> add ( ngram + 1 , ( * iter ) . second ) ; } else { ngram [ 0 ] = ( * iter ) . first [ 0 ] ; ngram [ 1 ] = ( * iter ) . first [ 1 ] ; ngram [ 2 ] = ( * iter ) . first [ 4 ] ; _wordStartProbabilities -> add ( ngram , ( * iter ) . second ) ; _wordStartHistories -> add ( ngram + 1 , ( * iter ) . second ) ; } } for ( iter = _wordStartProbabilities -> get_start ( ) ; iter != _wordStartProbabilities -> get_end ( ) ; ++ iter ) { _startUniqueCounts -> add ( ( * iter ) . first + 1 , 1 ) ; float transitions = ( * iter ) . second ; float histories = _wordStartHistories -> lookup ( ( * iter ) . first + 1 ) ; ( * iter ) . second = transitions / histories ; } for ( iter = _wordContinueProbabilities -> get_start ( ) ; iter != _wordContinueProbabilities -> get_end ( ) ; ++ iter ) { _continueUniqueCounts -> add ( ( * iter ) . first + 1 , 1 ) ; float transitions = ( * iter ) . second ; float histories = _wordContinueHistories -> lookup ( ( * iter ) . first + 1 ) ; ( * iter ) . second = transitions / histories ; } for ( iter = _wordTagProbabilities -> get_start ( ) ; iter != _wordTagProbabilities -> get_end ( ) ; ++ iter ) { _wordTagUniqueCounts -> add ( ( * iter ) . first + 1 , 1 ) ; float transitions = ( * iter ) . second ; float histories = _wordTagHistories -> lookup ( ( * iter ) . first + 1 ) ; ( * iter ) . second = transitions / histories ; } for ( iter = _wordReducedTagProbabilities -> get_start ( ) ; iter != _wordReducedTagProbabilities -> get_end ( ) ; ++ iter ) { _wordReducedTagUniqueCounts -> add ( ( * iter ) . first + 1 , 1 ) ; float transitions = ( * iter ) . second ; float histories = _wordReducedTagHistories -> lookup ( ( * iter ) . first + 1 ) ; ( * iter ) . second = transitions / histories ; } deriveWordLambdas ( ) ; }
Given dataTable (a set of events), fill in probability tables (future + history,
 # times occured), history tables (history, # times occured), and unique tables
 (history, # unique futures given this history).
Given dataTable ( a set of events ) fill in probability tables ( future + history # times occured ) history tables ( history # times occured ) and unique tables ( history # unique futures given this history ) .
// word
 // full tag
 // reduced tag
 // word
 // tag
 // word - 1
 // mid-start tags, not typically classified as "start") are treated as
 // start for probability model purposes
 // word
 // tag
 // tag - 1
