src/Generic/common/LocatedString.cpp
/* Changes located string to make nonstandard whitespace into more standard chars  */
Changes located string to make nonstandard whitespace into more standard chars
Changes located string to make nonstandard whitespace into more standard chars
void LocatedString::replaceNonstandardUnicodeWhitespace(){
	int len = static_cast<int>(_text.length());
	char msg[200];
	for (int index = 0; index < len; index++) {
		wchar_t wch = charAt(index);
		msg[0] = 0;
		if (
			(wch == 0x00a0) ||  
			((wch >= 0x2000) && (wch <= 0x200f)) || 
		    ((wch >= 0x202a) && (wch <= 0x202e)) || 
			(wch == 0x202f) ||  
			((wch >= 0x205f) && (wch <= 0x2063))){ 
				sprintf_s(msg, "U+%x with blank at index %d\n",
				  wch, index);
				replace(index,1,L" ");
		}else if ((wch == 0x0085) ||  
				  (wch == 0x2028))  { 
				sprintf_s(msg, "U+%x with line feed at index %d\n",
						wch, index);
				replace(index,1,L"\x0a");
		}else if (wch == 0x2029){ 
				sprintf_s(msg, "U+%x with double line feed at index %d\n",
						wch, index);
				replace(index,1,L"\x0a\x0a"); 
				len++;
		}
		if (msg[0] != 0){
			SessionLogger::dbg("unicode") <<"LocatedString::replaceNonstandardUnicodeWhitespace replacing char " <<msg;
		}
	}
}
void LocatedString :: replaceNonstandardUnicodeWhitespace ( ) { int len = static_cast < int > ( _text . length ( ) ) ; char msg [ 200 ] ; for ( int index = 0 ; index < len ; index ++ ) { wchar_t wch = charAt ( index ) ; msg [ 0 ] = 0 ; if ( ( wch == 0x00a0 ) || ( ( wch >= 0x2000 ) && ( wch <= 0x200f ) ) || ( ( wch >= 0x202a ) && ( wch <= 0x202e ) ) || ( wch == 0x202f ) || ( ( wch >= 0x205f ) && ( wch <= 0x2063 ) ) ) { sprintf_s ( msg , " \n " , wch , index ) ; replace ( index , 1 , L" " ) ; } else if ( ( wch == 0x0085 ) || ( wch == 0x2028 ) ) { sprintf_s ( msg , " \n " , wch , index ) ; replace ( index , 1 , L" \x0a " ) ; } else if ( wch == 0x2029 ) { sprintf_s ( msg , " \n " , wch , index ) ; replace ( index , 1 , L" \x0a \x0a " ) ; len ++ ; } if ( msg [ 0 ] != 0 ) { SessionLogger :: dbg ( " " ) << " " << msg ; } } }
Changes located string to make nonstandard whitespace into more standard chars
Changes located string to make nonstandard whitespace into more standard chars
// this code is a portion of Tokenizer::replaceNonstandardUnicodes
 // it belongs in a utility directory but is being tested locally where first used
 // "horizontal tab"   -- keep this one unchanged for segmenting -- 
 // (wch == 0x0009) ||  
 // "no-break space"
 // 2000-200b are fancy white spaces
 // 0x200b is "no break space" used for some languages
 // 200c-200f are typography (include ltr and rtl)
 // embedding direction to print
 // these typography codes fall between tokens so we replace with whitespace
 // "narrow no-break space"
 // 0x205f is mathematical space
 // 0x2060 is "word joiner" 0-width no-break space
 // 0x2061-0x2063 math separators
 // replace with vanilla space
 // new Unicode "next line"
 // "line separator"	
 // replace with old-fashioned line feed (Unix new line)
 // "paragraph separator"
 // two new lines
 //std::cerr<<msg;
 //std::cerr<<"debug tokenizer: replaceNonstandardUnicodeWhitespace replacing char " <<msg;
