src/Urdu/tokens/ur_Tokenizer.cpp
/**
 * @param string the located string source of the sentence.
 * @param i the position at which to check for metadata.
 * @return <code>true</code> if the document metadata enforces a token break
 *         immediately before the given position; <code>false</code> if not.
 */
@param string the located string source of the sentence.
@param i the position at which to check for metadata.
@return true if the document metadata enforces a token break
immediately before the given position; false if not.
@param string the located string source of the sentence . @param i the position at which to check for metadata . @return true if the document metadata enforces a token break immediately before the given position ; false if not .
bool UrduTokenizer::enforceTokenBreak(const LocatedString *string, int i) const {
	Metadata *metadata = _document->getMetadata();
	Span *span;
	span = metadata->getEndingSpan(string->end<EDTOffset>(i - 1));
	if (span != NULL &&
		string->end<EDTOffset>(i - 1) != string->end<EDTOffset>(i) &&
		span->enforceTokenBreak())
	{
		return true;
	}
	span = metadata->getStartingSpan(string->start<EDTOffset>(i));
	if (span != NULL &&
		(i == 0 || string->end<EDTOffset>(i - 1) != string->end<EDTOffset>(i)) &&
		span->enforceTokenBreak()) {
		return true;
	}
	return false;
}
bool UrduTokenizer :: enforceTokenBreak ( const LocatedString * string , int i ) const { Metadata * metadata = _document -> getMetadata ( ) ; Span * span ; span = metadata -> getEndingSpan ( string -> end < EDTOffset > ( i - 1 ) ) ; if ( span != NULL && string -> end < EDTOffset > ( i - 1 ) != string -> end < EDTOffset > ( i ) && span -> enforceTokenBreak ( ) ) { return true ; } span = metadata -> getStartingSpan ( string -> start < EDTOffset > ( i ) ) ; if ( span != NULL && ( i == 0 || string -> end < EDTOffset > ( i - 1 ) != string -> end < EDTOffset > ( i ) ) && span -> enforceTokenBreak ( ) ) { return true ; } return false ; }
@param string the located string source of the sentence.
@param string the located string source of the sentence .
// Is there a span ending just before this position? // don't break in middle of replaced token // (replaced tokens causes several characters in the string to have the same offset) // Is there a span starting right at this position? // don't break in middle of replaced token // (replaced tokens causes several characters in the string to have the same offset)