src/English/tokens/en_Tokenizer.cpp
//Replace any letter repeated more than 2 times in a row with just 2 of that letter
//This is helpful with badly formed internet text, where people put excessive letters for emphasis

Replace any letter repeated more than 2 times in a row with just 2 of that letter
This is helpful with badly formed internet text, where people put excessive letters for emphasis
Replace any letter repeated more than 2 times in a row with just 2 of that letter This is helpful with badly formed internet text where people put excessive letters for emphasis
void EnglishTokenizer::replaceExcessiveCharacters(LocatedString *input) {
	int start = 0;
	int end;
	for(int start = 0; start < input->length(); start++) {
		if(isalpha(input->charAt(start))) {
			wchar_t let = input->charAt(start);
			for(end = start + 1; end < input->length() && input->charAt(end) == let; end++) {}
			int letStrLen = end-start;
			if(letStrLen > 2) {
				input->replace(start, letStrLen-1, wstring(1, let).c_str());
			}
		}
	}
}
void EnglishTokenizer :: replaceExcessiveCharacters ( LocatedString * input ) { int start = 0 ; int end ; for ( int start = 0 ; start < input -> length ( ) ; start ++ ) { if ( isalpha ( input -> charAt ( start ) ) ) { wchar_t let = input -> charAt ( start ) ; for ( end = start + 1 ; end < input -> length ( ) && input -> charAt ( end ) == let ; end ++ ) { } int letStrLen = end - start ; if ( letStrLen > 2 ) { input -> replace ( start , letStrLen - 1 , wstring ( 1 , let ) . c_str ( ) ) ; } } } }
Replace any letter repeated more than 2 times in a row with just 2 of that letter
This is helpful with badly formed internet text, where people put excessive letters for emphasis
Replace any letter repeated more than 2 times in a row with just 2 of that letter This is helpful with badly formed internet text where people put excessive letters for emphasis
//end is currently 1 past the end of the punctuation string
 //replace replaces from pos to pos+len inclusive, so we need to subtract 1 from the length
