src/Generic/patterns/MatchStorage.cpp
// Take the next combination of MatchData, and create a sentence out of it.
// Store a sentence offset to Token map

Take the next combination of MatchData, and create a sentence out of it.
Store a sentence offset to Token map
Take the next combination of MatchData and create a sentence out of it . Store a sentence offset to Token map
bool MatchStorage::getNextSentence(TokenSequence *ts, int search_start_token, int search_end_token, 
								   std::wstring &result, MatchData **currentMatchData, 
								   int* offsetToStartTokenMap, int* offsetToEndTokenMap)
{
	if (_finished) return false;
	while (!matchesAreInOrder()) {
		advance();
		if (_finished) return false;
	}
	int token_number = 0;
	int offset_pos = 0;
	result = L"";
	appendAndRecord(result, L" ", offset_pos, 0, 0, offsetToStartTokenMap, offsetToEndTokenMap);
	for (int i = 0; i <= _highest; i++) {
		currentMatchData[i] = _currentMatches[i];
		if (_patternsArray[i] == 0) continue; 
		MatchData *md = _currentMatches[i];
		while (token_number < md->start_token) {
			if (token_number >= search_start_token && token_number <= search_end_token) {
				const wchar_t* token_str = ts->getToken(token_number)->getSymbol().to_string();
				appendAndRecord(result, token_str, offset_pos, token_number, token_number, offsetToStartTokenMap, offsetToEndTokenMap);
				appendAndRecord(result, L" ", offset_pos, token_number + 1, token_number, offsetToStartTokenMap, offsetToEndTokenMap);
			}
			token_number++;
		}
		if (token_number >= search_start_token && token_number <= search_end_token) {
			appendAndRecord(result, L"!@RP_MATCH@! ", offset_pos, token_number, md->end_token, offsetToStartTokenMap, offsetToEndTokenMap);
		}
		token_number = md->end_token + 1;
	}
	while (token_number < ts->getNTokens()) {
		if (token_number >= search_start_token && token_number <= search_end_token) {
			const wchar_t* token_str = ts->getToken(token_number)->getSymbol().to_string();
			appendAndRecord(result, token_str, offset_pos, token_number, token_number, offsetToStartTokenMap, offsetToEndTokenMap);
			int space_start_token = token_number + 1;
			if (space_start_token >= ts->getNTokens())
				space_start_token = ts->getNTokens() - 1;
			appendAndRecord(result, L" ", offset_pos, space_start_token, token_number, offsetToStartTokenMap, offsetToEndTokenMap);
		}
		token_number++;
	}
	if (offset_pos == 1) {
		return false;
	}	
	if (offset_pos >= 2) {
		offsetToStartTokenMap[0] = offsetToStartTokenMap[1];
		offsetToEndTokenMap[0] = offsetToEndTokenMap[1];
	}
	advance();
	return true;
}
bool MatchStorage :: getNextSentence ( TokenSequence * ts , int search_start_token , int search_end_token , std :: wstring & result , MatchData * * currentMatchData , int * offsetToStartTokenMap , int * offsetToEndTokenMap ) { if ( _finished ) return false ; while ( ! matchesAreInOrder ( ) ) { advance ( ) ; if ( _finished ) return false ; } int token_number = 0 ; int offset_pos = 0 ; result = L" " ; appendAndRecord ( result , L" " , offset_pos , 0 , 0 , offsetToStartTokenMap , offsetToEndTokenMap ) ; for ( int i = 0 ; i <= _highest ; i ++ ) { currentMatchData [ i ] = _currentMatches [ i ] ; if ( _patternsArray [ i ] == 0 ) continue ; MatchData * md = _currentMatches [ i ] ; while ( token_number < md -> start_token ) { if ( token_number >= search_start_token && token_number <= search_end_token ) { const wchar_t * token_str = ts -> getToken ( token_number ) -> getSymbol ( ) . to_string ( ) ; appendAndRecord ( result , token_str , offset_pos , token_number , token_number , offsetToStartTokenMap , offsetToEndTokenMap ) ; appendAndRecord ( result , L" " , offset_pos , token_number + 1 , token_number , offsetToStartTokenMap , offsetToEndTokenMap ) ; } token_number ++ ; } if ( token_number >= search_start_token && token_number <= search_end_token ) { appendAndRecord ( result , L" " , offset_pos , token_number , md -> end_token , offsetToStartTokenMap , offsetToEndTokenMap ) ; } token_number = md -> end_token + 1 ; } while ( token_number < ts -> getNTokens ( ) ) { if ( token_number >= search_start_token && token_number <= search_end_token ) { const wchar_t * token_str = ts -> getToken ( token_number ) -> getSymbol ( ) . to_string ( ) ; appendAndRecord ( result , token_str , offset_pos , token_number , token_number , offsetToStartTokenMap , offsetToEndTokenMap ) ; int space_start_token = token_number + 1 ; if ( space_start_token >= ts -> getNTokens ( ) ) space_start_token = ts -> getNTokens ( ) - 1 ; appendAndRecord ( result , L" " , offset_pos , space_start_token , token_number , offsetToStartTokenMap , offsetToEndTokenMap ) ; } token_number ++ ; } if ( offset_pos == 1 ) { return false ; } if ( offset_pos >= 2 ) { offsetToStartTokenMap [ 0 ] = offsetToStartTokenMap [ 1 ] ; offsetToEndTokenMap [ 0 ] = offsetToEndTokenMap [ 1 ] ; } advance ( ) ; return true ; }
Take the next combination of MatchData, and create a sentence out of it.
Take the next combination of MatchData and create a sentence out of it .
// check to make sure current matches pointed to are in right order and don't overlap
 // current matches do not overlap, create sentence
 // We put a space at the beginning of a regex pattern, so we don't start matching in the middle of a word.
 // This requires us to put a space in here as well. 
 // text pattern
 // Nothing was added to string. This could happen if there are no subpatterns in the RegexPattern.
 // make the space at the beginning that we put in earlier match the earliest token
