src/lmtable.cpp
//Add method inserts n-grams in the table structure. It is ONLY used during
//loading of LMs in text format. It searches for the prefix, then it adds the
//suffix to the last level and updates the start-end positions.
Add method inserts n-grams in the table structure. It is ONLY used during
loading of LMs in text format. It searches for the prefix, then it adds the
suffix to the last level and updates the start-end positions.
Add method inserts n - grams in the table structure . It is ONLY used during loading of LMs in text format . It searches for the prefix then it adds the suffix to the last level and updates the start - end positions .
int lmtable::addwithoffset(ngram& ng, float iprob, float ibow)
	{
		char *found;
		LMT_TYPE ndt=tbltype[1]; 
		int ndsz=nodesize(ndt); 
		static int no_more_msg = 0;
		if (ng.size>1) {
			table_entry_pos_t start=0;
			table_entry_pos_t end=cursize[1];		
			table_entry_pos_t position;
			for (int l=1; l<ng.size; l++) {
				ndt=tbltype[l];
				ndsz=nodesize(ndt);
				if (search(l,start,(end-start),ndsz, ng.wordp(ng.size-l+1),LMT_FIND, &found)) {
					if (l < (ng.size-1)) {
						if (found==table[l]){
							start=0; 
						}
						else {
							position=(table_entry_pos_t) (((table_pos_t) (found)-(table_pos_t) table[l])/ndsz);
							start=startpos[l][position];
						}
						end=boundwithoffset(found,ndt,l);
					}
				} else {
					if (!no_more_msg)
					{
						VERBOSE(2, "warning: missing back-off (at level " << l << ") for ngram " << ng << " (and possibly for others)" << std::endl);
					}
					no_more_msg++;
					if (!(no_more_msg % 5000000))
					{
						VERBOSE(2, "!" << std::endl);
					}
					return 0;
				}
			}
			position=(table_entry_pos_t) (((table_pos_t) found-(table_pos_t) table[ng.size-1])/ndsz);
			if (startpos[ng.size-1][position]==BOUND_EMPTY1)
				startpos[ng.size-1][position]=cursize[ng.size];
			boundwithoffset(found,ndt,cursize[ng.size]+1,ng.size-1);
		}
		MY_ASSERT(cursize[ng.size]< maxsize[ng.size]); 
		ndt=tbltype[ng.size];
		ndsz=nodesize(ndt);
		found=table[ng.size] + ((table_pos_t) cursize[ng.size] * ndsz);
		word(found,*ng.wordp(1));
		prob(found,ndt,iprob);
		if (ng.size<maxlev) {
			table_entry_pos_t newend;
			if (found==table[ng.size])			newend=0; 
			else 			newend=boundwithoffset(found - ndsz,ndt,ng.size);
			bow(found,ndt,ibow);
			boundwithoffset(found,ndt,newend,ng.size);
		}
		cursize[ng.size]++;
		if (!(cursize[ng.size]%5000000))
		{			
			VERBOSE(1, "." << std::endl);
		}
		return 1;
	}
int lmtable :: addwithoffset ( ngram & ng , float iprob , float ibow ) { char * found ; LMT_TYPE ndt = tbltype [ 1 ] ; int ndsz = nodesize ( ndt ) ; static int no_more_msg = 0 ; if ( ng . size > 1 ) { table_entry_pos_t start = 0 ; table_entry_pos_t end = cursize [ 1 ] ; table_entry_pos_t position ; for ( int l = 1 ; l < ng . size ; l ++ ) { ndt = tbltype [ l ] ; ndsz = nodesize ( ndt ) ; if ( search ( l , start , ( end - start ) , ndsz , ng . wordp ( ng . size - l + 1 ) , LMT_FIND , & found ) ) { if ( l < ( ng . size - 1 ) ) { if ( found == table [ l ] ) { start = 0 ; } else { position = ( table_entry_pos_t ) ( ( ( table_pos_t ) ( found ) - ( table_pos_t ) table [ l ] ) / ndsz ) ; start = startpos [ l ] [ position ] ; } end = boundwithoffset ( found , ndt , l ) ; } } else { if ( ! no_more_msg ) { VERBOSE ( 2 , " " << l << " " << ng << " " << std :: endl ) ; } no_more_msg ++ ; if ( ! ( no_more_msg % 5000000 ) ) { VERBOSE ( 2 , " " << std :: endl ) ; } return 0 ; } } position = ( table_entry_pos_t ) ( ( ( table_pos_t ) found - ( table_pos_t ) table [ ng . size - 1 ] ) / ndsz ) ; if ( startpos [ ng . size - 1 ] [ position ] == BOUND_EMPTY1 ) startpos [ ng . size - 1 ] [ position ] = cursize [ ng . size ] ; boundwithoffset ( found , ndt , cursize [ ng . size ] + 1 , ng . size - 1 ) ; } MY_ASSERT ( cursize [ ng . size ] < maxsize [ ng . size ] ) ; ndt = tbltype [ ng . size ] ; ndsz = nodesize ( ndt ) ; found = table [ ng . size ] + ( ( table_pos_t ) cursize [ ng . size ] * ndsz ) ; word ( found , * ng . wordp ( 1 ) ) ; prob ( found , ndt , iprob ) ; if ( ng . size < maxlev ) { table_entry_pos_t newend ; if ( found == table [ ng . size ] ) newend = 0 ; else newend = boundwithoffset ( found - ndsz , ndt , ng . size ) ; bow ( found , ndt , ibow ) ; boundwithoffset ( found , ndt , newend , ng . size ) ; } cursize [ ng . size ] ++ ; if ( ! ( cursize [ ng . size ] % 5000000 ) ) { VERBOSE ( 1 , " " << std :: endl ) ; } return 1 ; }
Add method inserts n-grams in the table structure.
Add method inserts n - grams in the table structure .
//default initialization //default initialization // find the prefix starting from the first level //update start and end positions for next step //set start position //first pos in table // update book keeping information about level ng-size -1. // if this is the first successor update start position in the previous level //always update ending position // just add at the end of table[ng.size] // is there enough space? //find the bound of the previous entry //first pos in table