src/Generic/names/IdFTrainer.cpp
/**
 * transfers events from _wordEventCounts and _tagEventCounts into 
 * _wordEventCountsPlusUnknown and _tagEventCountsPlusUnknown
 *
 * Every event is added to the relevant table once as is. Then, any words 
 * qualifying as unknown are replaced by their feature-symbol, and
 * the events that contain them are added again, in their modified form, 
 * to the relevant table.
 */


Every event is added to the relevant table once as is. Then, any words
qualifying as unknown are replaced by their feature-symbol, and
the events that contain them are added again, in their modified form,
to the relevant table.
Every event is added to the relevant table once as is . Then any words qualifying as unknown are replaced by their feature - symbol and the events that contain them are added again in their modified form to the relevant table .
void IdFTrainer::addUnknownWords() {
	float threshold = 5;
	NgramScoreTable::Table::iterator iter;
	for (iter = _tagEventCounts->get_start(); iter != _tagEventCounts->get_end(); ++iter) {
		Symbol ngram[4];
		ngram[0] = (*iter).first[0];
		ngram[1] = (*iter).first[1];
		ngram[2] = (*iter).first[2];
		ngram[3] = (*iter).first[3];
		_tagEventCountsPlusUnknown->add(ngram, (*iter).second);
		if (_temporaryVocabTable->lookup(&ngram[2]) == 0) {
			ngram[2] = _model->getWordFeatures(ngram[2], 
				ngram[3] == FIRSTWORD, 
				_model->normalizedWordIsInVocab(ngram[2]));
			_tagEventCountsPlusUnknown->add(ngram, (*iter).second);
		}
	}
	for (iter = _wordEventCounts->get_start(); iter != _wordEventCounts->get_end(); ++iter) {
		Symbol ngram[6];
		ngram[0] = (*iter).first[0];
		ngram[1] = (*iter).first[2];
		ngram[2] = (*iter).first[3];
		ngram[3] = (*iter).first[4];
		ngram[4] = (*iter).first[5];
		ngram[5] = (*iter).first[6];
		_wordEventCountsPlusUnknown->add(ngram, (*iter).second);
		bool word_not_found = false;
		if (_temporaryVocabTable->lookup(&ngram[0]) == 0) {
			ngram[0] = _model->getWordFeatures(ngram[0],
				(*iter).first[1] == FIRSTWORD, 
				_model->normalizedWordIsInVocab(ngram[0]));
			word_not_found = true;
		}
		if (_temporaryVocabTable->lookup(&ngram[5]) == 0) {
			ngram[5] = _model->getWordFeatures(ngram[5],
				(*iter).first[7] == FIRSTWORD, 				
				_model->normalizedWordIsInVocab(ngram[5]));
			word_not_found = true;
		}
		if (word_not_found)
			_wordEventCountsPlusUnknown->add(ngram, (*iter).second);
	}	
}
void IdFTrainer :: addUnknownWords ( ) { float threshold = 5 ; NgramScoreTable :: Table :: iterator iter ; for ( iter = _tagEventCounts -> get_start ( ) ; iter != _tagEventCounts -> get_end ( ) ; ++ iter ) { Symbol ngram [ 4 ] ; ngram [ 0 ] = ( * iter ) . first [ 0 ] ; ngram [ 1 ] = ( * iter ) . first [ 1 ] ; ngram [ 2 ] = ( * iter ) . first [ 2 ] ; ngram [ 3 ] = ( * iter ) . first [ 3 ] ; _tagEventCountsPlusUnknown -> add ( ngram , ( * iter ) . second ) ; if ( _temporaryVocabTable -> lookup ( & ngram [ 2 ] ) == 0 ) { ngram [ 2 ] = _model -> getWordFeatures ( ngram [ 2 ] , ngram [ 3 ] == FIRSTWORD , _model -> normalizedWordIsInVocab ( ngram [ 2 ] ) ) ; _tagEventCountsPlusUnknown -> add ( ngram , ( * iter ) . second ) ; } } for ( iter = _wordEventCounts -> get_start ( ) ; iter != _wordEventCounts -> get_end ( ) ; ++ iter ) { Symbol ngram [ 6 ] ; ngram [ 0 ] = ( * iter ) . first [ 0 ] ; ngram [ 1 ] = ( * iter ) . first [ 2 ] ; ngram [ 2 ] = ( * iter ) . first [ 3 ] ; ngram [ 3 ] = ( * iter ) . first [ 4 ] ; ngram [ 4 ] = ( * iter ) . first [ 5 ] ; ngram [ 5 ] = ( * iter ) . first [ 6 ] ; _wordEventCountsPlusUnknown -> add ( ngram , ( * iter ) . second ) ; bool word_not_found = false ; if ( _temporaryVocabTable -> lookup ( & ngram [ 0 ] ) == 0 ) { ngram [ 0 ] = _model -> getWordFeatures ( ngram [ 0 ] , ( * iter ) . first [ 1 ] == FIRSTWORD , _model -> normalizedWordIsInVocab ( ngram [ 0 ] ) ) ; word_not_found = true ; } if ( _temporaryVocabTable -> lookup ( & ngram [ 5 ] ) == 0 ) { ngram [ 5 ] = _model -> getWordFeatures ( ngram [ 5 ] , ( * iter ) . first [ 7 ] == FIRSTWORD , _model -> normalizedWordIsInVocab ( ngram [ 5 ] ) ) ; word_not_found = true ; } if ( word_not_found ) _wordEventCountsPlusUnknown -> add ( ngram , ( * iter ) . second ) ; } }
transfers events from _wordEventCounts and _tagEventCounts into
 _wordEventCountsPlusUnknown and _tagEventCountsPlusUnknown
transfers events from _wordEventCounts and _tagEventCounts into _wordEventCountsPlusUnknown and _tagEventCountsPlusUnknown