src/Chinese/tokens/ch_Tokenizer.cpp
// Note: this only looks at the smallest span starting/ending at
// a given offset. To do this correctly, need to look at every span.

this only looks at the smallest span starting/ending at
a given offset. To do this correctly, need to look at every span.
this only looks at the smallest span starting / ending at a given offset . To do this correctly need to look at every span .
bool ChineseTokenizer::constraintViolation(EDTOffset begin, EDTOffset end) {
	Metadata *metadata = _document->getMetadata();
	Span *span;
	if (metadata->get_span_count() == 0)
		return false;
	for (EDTOffset i = begin; i < end; ++i) {	 
		span = metadata->getEndingSpan(i);
		if ((span != NULL) && span->enforceTokenBreak()) {
			return true;
		}
	}
	for (EDTOffset j = EDTOffset(begin.value() + 1); j <= end; ++j) { 
		span = metadata->getStartingSpan(j);
		if ((span != NULL) && span->enforceTokenBreak()) {
			return true;
		}
	}
	return false;
}
bool ChineseTokenizer :: constraintViolation ( EDTOffset begin , EDTOffset end ) { Metadata * metadata = _document -> getMetadata ( ) ; Span * span ; if ( metadata -> get_span_count ( ) == 0 ) return false ; for ( EDTOffset i = begin ; i < end ; ++ i ) { span = metadata -> getEndingSpan ( i ) ; if ( ( span != NULL ) && span -> enforceTokenBreak ( ) ) { return true ; } } for ( EDTOffset j = EDTOffset ( begin . value ( ) + 1 ) ; j <= end ; ++ j ) { span = metadata -> getStartingSpan ( j ) ; if ( ( span != NULL ) && span -> enforceTokenBreak ( ) ) { return true ; } } return false ; }
Note: this only looks at the smallest span starting/ending at
a given offset.
Note : this only looks at the smallest span starting / ending at a given offset .
// No spans exist, so skip check
 // Is there a span ending in middle of token? 
 // Is there a span starting in middle of token?
