src/Generic/sentences/DefaultSentenceBreaker.cpp
// Returns a count of the number of whitespace/punctuation spans
// between index start and index end in string; multiple 
// contiguous chars count as one span. Estimate of number of tokens.

Returns a count of the number of whitespace/punctuation spans
between index start and index end in string; multiple
contiguous chars count as one span. Estimate of number of tokens.
Returns a count of the number of whitespace / punctuation spans between index start and index end in string ; multiple contiguous chars count as one span . Estimate of number of tokens .
int DefaultSentenceBreaker::countPossibleTokenBreaks(const LocatedString* string, int start, int end) {
	int tb = 0;
	for (int i = start; i < end && i < string->length(); i++) {
		if (iswspace(string->charAt(i)) || iswpunct(string->charAt(i))) {
			tb++;
			while (i < string->length()-1 && (iswspace(string->charAt(i+1)) || iswpunct(string->charAt(i+1))))
				i++;
		}
	}
	return tb;
}
int DefaultSentenceBreaker :: countPossibleTokenBreaks ( const LocatedString * string , int start , int end ) { int tb = 0 ; for ( int i = start ; i < end && i < string -> length ( ) ; i ++ ) { if ( iswspace ( string -> charAt ( i ) ) || iswpunct ( string -> charAt ( i ) ) ) { tb ++ ; while ( i < string -> length ( ) - 1 && ( iswspace ( string -> charAt ( i + 1 ) ) || iswpunct ( string -> charAt ( i + 1 ) ) ) ) i ++ ; } } return tb ; }
Returns a count of the number of whitespace/punctuation spans
between index start and index end in string; multiple
contiguous chars count as one span.
Returns a count of the number of whitespace / punctuation spans between index start and index end in string ; multiple contiguous chars count as one span .