src/Urdu/tokens/ur_Tokenizer.cpp
/**
* Place a space before and after any instance of the given characters,
* if the optional additional constraints regarding urls and digits are met
*/
Place a space before and after any instance of the given characters,
if the optional additional constraints regarding urls and digits are met
Place a space before and after any instance of the given characters if the optional additional constraints regarding urls and digits are met
void UrduTokenizer::splitPunctuationWithConstraints(LocatedString *string, const wchar_t punc[], bool checkURL, bool checkDigits) {
	wchar_t wch;
	bool replaceable;
	for (int i=0; i < string->length(); i++) {
		wch = string->charAt(i);
		replaceable = true;
		if (isRelevantPunctuation(wch, punc)) {
			if (checkURL && (matchURL(string, i) || matchEmail(string, i))) {
				replaceable = false;
			}
			if (checkDigits) {
				if (!isCommaBreakable(string, i)) {
					replaceable = false; 
				}
			}
			if (replaceable) {
				if (i < string->length()-1) {
					string->insert(L" ",i+1);
					string->insert(L" ",i);
					i = i+2;
				}
				else {
					string->insert(L" ",i);
					i++;
				}
			}
		}
	}
}
void UrduTokenizer :: splitPunctuationWithConstraints ( LocatedString * string , const wchar_t punc [ ] , bool checkURL , bool checkDigits ) { wchar_t wch ; bool replaceable ; for ( int i = 0 ; i < string -> length ( ) ; i ++ ) { wch = string -> charAt ( i ) ; replaceable = true ; if ( isRelevantPunctuation ( wch , punc ) ) { if ( checkURL && ( matchURL ( string , i ) || matchEmail ( string , i ) ) ) { replaceable = false ; } if ( checkDigits ) { if ( ! isCommaBreakable ( string , i ) ) { replaceable = false ; } } if ( replaceable ) { if ( i < string -> length ( ) - 1 ) { string -> insert ( L" " , i + 1 ) ; string -> insert ( L" " , i ) ; i = i + 2 ; } else { string -> insert ( L" " , i ) ; i ++ ; } } } } }
Place a space before and after any instance of the given characters,
if the optional additional constraints regarding urls and digits are met
Place a space before and after any instance of the given characters if the optional additional constraints regarding urls and digits are met
//if (isRelevantPunctuation(wch, punc)) {