src/Generic/sentences/DefaultSentenceBreaker.cpp
/*
	Break up a sentence using punctuation if it exists:
		period+space
		period+quotation
		european-style quotation <<
	Else use white space to break it up

	If the resulting sentence is very long, break it up again

	In Arabic or other right-to-left languages,
	the << and ellipses will appear in the wrong place in the XML,
	but careful inspection of the unicode shows that the right thing
	is happening; everything between << xxx >> inclusive is in the sentence,
	the '...' comes at the end of the sentence
*/
Break up a sentence using punctuation if it exists:
period+space
period+quotation
european-style quotation <<
Else use white space to break it up

If the resulting sentence is very long, break it up again

In Arabic or other right-to-left languages,
the << and ellipses will appear in the wrong place in the XML,
but careful inspection of the unicode shows that the right thing
is happening; everything between << xxx >> inclusive is in the sentence,
the '...' comes at the end of the sentence
Break up a sentence using punctuation if it exists : period + space period + quotation european - style quotation << Else use white space to break it up If the resulting sentence is very long break it up again In Arabic or other right - to - left languages the << and ellipses will appear in the wrong place in the XML but careful inspection of the unicode shows that the right thing is happening ; everything between << xxx >> inclusive is in the sentence the ' ... ' comes at the end of the sentence
DefaultSentenceBreaker::DefaultSentenceBreaker()
{
	std::string non_final_abbrevs_file = ParamReader::getParam("tokenizer_non_final_abbrevs"); 
	if (non_final_abbrevs_file != "") {
		boost::scoped_ptr<UTF8InputStream> stream_scoped_ptr(UTF8InputStream::build(non_final_abbrevs_file));
		UTF8InputStream& stream(*stream_scoped_ptr);
		if (!stream.fail()) {
			std::wstring line;
			while (!stream.eof()) {
				stream.getLine(line);
				boost::algorithm::trim(line);
				if (line.size() == 0 || line.at(0) == L'#')
					continue;
				_nonFinalAbbrevs.insert(line);
				std::transform(line.begin(), line.end(), line.begin(), towlower);
				_nonFinalAbbrevs.insert(line);
			}
		}
		stream.close();
	}
	_max_token_breaks = 100; 
	_skipTableSentences = ParamReader::isParamTrue("skip_table_sentences");
	_breakTableSentences = ParamReader::isParamTrue("break_table_sentences") || _skipTableSentences;
	_minTableRows = (size_t)ParamReader::getOptionalIntParamWithDefaultValue("sentence_breaker_min_table_rows", 20);
}
DefaultSentenceBreaker :: DefaultSentenceBreaker ( ) { std :: string non_final_abbrevs_file = ParamReader :: getParam ( " " ) ; if ( non_final_abbrevs_file != " " ) { boost :: scoped_ptr < UTF8InputStream > stream_scoped_ptr ( UTF8InputStream :: build ( non_final_abbrevs_file ) ) ; UTF8InputStream & stream ( * stream_scoped_ptr ) ; if ( ! stream . fail ( ) ) { std :: wstring line ; while ( ! stream . eof ( ) ) { stream . getLine ( line ) ; boost :: algorithm :: trim ( line ) ; if ( line . size ( ) == 0 || line . at ( 0 ) == L' ' ) continue ; _nonFinalAbbrevs . insert ( line ) ; std :: transform ( line . begin ( ) , line . end ( ) , line . begin ( ) , towlower ) ; _nonFinalAbbrevs . insert ( line ) ; } } stream . close ( ) ; } _max_token_breaks = 100 ; _skipTableSentences = ParamReader :: isParamTrue ( " " ) ; _breakTableSentences = ParamReader :: isParamTrue ( " " ) || _skipTableSentences ; _minTableRows = ( size_t ) ParamReader :: getOptionalIntParamWithDefaultValue ( " " , 20 ) ; }
Break up a sentence using punctuation if it exists:
period+space
period+quotation
european-style quotation <<
Else use white space to break it up
Break up a sentence using punctuation if it exists : period + space period + quotation european - style quotation << Else use white space to break it up
// parameter used in both tokenization and sentence-breaking
 // includes both the word as-is and also a lowercase version
 // Overridden by language
