src/Arabic/tokens/ar_Tokenizer.cpp
//TODO:  don't split sentences on number separators

don't split sentences on number separators
don ' t split sentences on number separators
bool ArabicTokenizer::matchNumberSeparator(const LocatedString *string, int index) const {
	return (index > 0) &&
		   ((index + 1) < string->length()) &&
		   ((string->charAt(index) == L',')||(string->charAt(index) ==L'.')||
		   (string->charAt(index)==0x066B)||(string->charAt(index)==0x066C))
		   &&((iswdigit(string->charAt(index - 1))
			||((string->charAt(index - 1)>=0x0660) && (string->charAt(index - 1)<=0x0669))))
		   && ((iswdigit(string->charAt(index + 1))
			||((string->charAt(index + 1)>=0x0660) && (string->charAt(index + 1)<=0x0669))));
}
bool ArabicTokenizer :: matchNumberSeparator ( const LocatedString * string , int index ) const { return ( index > 0 ) && ( ( index + 1 ) < string -> length ( ) ) && ( ( string -> charAt ( index ) == L' ' ) || ( string -> charAt ( index ) == L' ' ) || ( string -> charAt ( index ) == 0x066B ) || ( string -> charAt ( index ) == 0x066C ) ) && ( ( iswdigit ( string -> charAt ( index - 1 ) ) || ( ( string -> charAt ( index - 1 ) >= 0x0660 ) && ( string -> charAt ( index - 1 ) <= 0x0669 ) ) ) ) && ( ( iswdigit ( string -> charAt ( index + 1 ) ) || ( ( string -> charAt ( index + 1 ) >= 0x0660 ) && ( string -> charAt ( index + 1 ) <= 0x0669 ) ) ) ) ; }
TODO:  don't split sentences on number separators
TODO : don ' t split sentences on number separators
//66B is Arabic Thousand Separator, 66C is Arabic decimal point-
 //660-669 are Arabic digits
 //TODO check that Arabic comma is not used as a number separator
