src/Urdu/tokens/ur_Tokenizer.cpp
/**
* Get rid of characters that encode left-to-right directionality
*/
Get rid of characters that encode left-to-right directionality
Get rid of characters that encode left - to - right directionality
void UrduTokenizer::replaceDirectionalChar(LocatedString *string) {
	for (int i=0; i<string->length(); i++) {
		wchar_t wch = string->charAt(i);
		if (wcschr(DIRECTIONAL_SYMBOLS, wch) != NULL) {
			string->replace(i,1,L" ");
			SessionLogger::info("tokenizer") << "used replaceDirectionalChar";
		}
	}
}
void UrduTokenizer :: replaceDirectionalChar ( LocatedString * string ) { for ( int i = 0 ; i < string -> length ( ) ; i ++ ) { wchar_t wch = string -> charAt ( i ) ; if ( wcschr ( DIRECTIONAL_SYMBOLS , wch ) != NULL ) { string -> replace ( i , 1 , L" " ) ; SessionLogger :: info ( " " ) << " " ; } } }
Get rid of characters that encode left-to-right directionality
Get rid of characters that encode left - to - right directionality