src/Generic/sentences/DefaultSentenceBreaker.cpp
// Attempts to break sentence into ~n_split_sents smaller sentences

Attempts to break sentence into ~n_split_sents smaller sentences
Attempts to break sentence into ~n_split_sents smaller sentences
std::vector<Sentence*> DefaultSentenceBreaker::attemptLongSentenceBreak(const Sentence *sentence, int n_split_sents) {
	std::vector<Sentence*> results;
	if (n_split_sents == 0 ||  n_split_sents == 1) 
		return results;
	int orig_sent_len = sentence->getString()->length();
	int avg_num_chars = orig_sent_len/n_split_sents;
	SessionLogger::dbg("brk_sent_0") << "DefaultSentenceBreaker::attemptLongSentenceBreak(),"
							 << " Attempting to break sentence with " << orig_sent_len
							 << " characters into smaller sentences with " << avg_num_chars
							 << " characters on average.\n";
	int sent_start = 0;
	int curr_char = avg_num_chars;
	while (curr_char < orig_sent_len) { 
		int window_size, left_idx, right_idx;
		bool found_breakpoint = false;
		int breakpoint = -1;
		std::string justification = "";
		for (window_size = 0; window_size < 100 && !found_breakpoint; window_size++) {
			left_idx = curr_char - window_size;
			right_idx = curr_char + window_size;
			if ((left_idx <= sent_start) && ((right_idx >= orig_sent_len) || (right_idx - sent_start + 1 > getMaxSentenceChars())))
				break;
			if ((left_idx > sent_start) &&
				isSecondaryEOSChar(sentence->getString()->charAt(left_idx)) && 
				!inRestrictedBreakSpan(_curDoc->getMetadata(), sentence->getString(), left_idx + 1)) 
			{
				breakpoint = left_idx;
				justification = "punctuation";
				found_breakpoint = true;
			}
			else if ((right_idx < orig_sent_len) && (right_idx - sent_start + 1 <= getMaxSentenceChars()) &&
					isSecondaryEOSChar(sentence->getString()->charAt(right_idx)) &&
				    !inRestrictedBreakSpan(_curDoc->getMetadata(), sentence->getString(), right_idx + 1))
			{
				breakpoint = right_idx;
				justification = "punctuation";
				found_breakpoint = true;
			}
		}
		for (window_size = 0; !found_breakpoint; window_size++) {
			left_idx = curr_char - window_size;
			right_idx = curr_char + window_size;
			if ((left_idx <= sent_start) && ((right_idx >= orig_sent_len) || (right_idx - sent_start + 1 > getMaxSentenceChars())))
				break;
			if ((left_idx > sent_start) &&
				iswspace(sentence->getString()->charAt(left_idx)) &&
				!inRestrictedBreakSpan(_curDoc->getMetadata(), sentence->getString(), left_idx + 1)) 
			{
				breakpoint = left_idx;
				justification = "whitespace";
				found_breakpoint = true;
			}
			else if ((right_idx < orig_sent_len) && (right_idx - sent_start + 1 <= getMaxSentenceChars()) &&
					iswspace(sentence->getString()->charAt(right_idx)) &&
					!inRestrictedBreakSpan(_curDoc->getMetadata(), sentence->getString(), right_idx + 1)) 
			{
				breakpoint = right_idx;
				justification = "whitespace";
				found_breakpoint = true;
			}
		}
		if (!found_breakpoint) {
			SessionLogger::warn("bad_sentence_break") << " Could not find good break point, breaking at random.";
		}
		for (window_size = 0; !found_breakpoint; window_size++) {
			left_idx = curr_char - window_size;
			right_idx = curr_char + window_size;
			if ((left_idx <= sent_start) && ((right_idx >= orig_sent_len) || (right_idx - sent_start + 1 > getMaxSentenceChars())))
				break;
			if ((left_idx > sent_start) &&
				!inRestrictedBreakSpan(_curDoc->getMetadata(), sentence->getString(), left_idx + 1)) 
			{
				breakpoint = left_idx;
				justification = "random";
				found_breakpoint = true;
			}
			else if ((right_idx < orig_sent_len) && (right_idx - sent_start + 1 <= getMaxSentenceChars()) &&
					!inRestrictedBreakSpan(_curDoc->getMetadata(), sentence->getString(), right_idx + 1)) 
			{
				breakpoint = right_idx;
				justification = "random";
				found_breakpoint = true;
			}
		}
		if (!found_breakpoint) {
			throw UnexpectedInputException("DefaultSentenceBreaker::attemptLongSentenceBreak()",
										   "Could not find unrestricted sentence break point within max_sentence_chars range.");
		}
		int next_sent_no = sentence->getSentNumber() + static_cast<int>(results.size());
		results.push_back(createSubSentence(sentence, next_sent_no, sent_start, breakpoint, justification));
		sent_start = breakpoint + 1;
		curr_char = sent_start + avg_num_chars;
		if ((sent_start < orig_sent_len) && isAcceptableSentenceString(sentence->getString(), sent_start, orig_sent_len)) {
			next_sent_no = sentence->getSentNumber() + static_cast<int>(results.size());
			results.push_back(createFinalSubSentence(sentence, next_sent_no, sent_start));
			sent_start = orig_sent_len + 1;
			curr_char = orig_sent_len + 1;
		}
	}
	return results;
}
std :: vector < Sentence * > DefaultSentenceBreaker :: attemptLongSentenceBreak ( const Sentence * sentence , int n_split_sents ) { std :: vector < Sentence * > results ; if ( n_split_sents == 0 || n_split_sents == 1 ) return results ; int orig_sent_len = sentence -> getString ( ) -> length ( ) ; int avg_num_chars = orig_sent_len / n_split_sents ; SessionLogger :: dbg ( " " ) << " " << " " << orig_sent_len << " " << avg_num_chars << " \n " ; int sent_start = 0 ; int curr_char = avg_num_chars ; while ( curr_char < orig_sent_len ) { int window_size , left_idx , right_idx ; bool found_breakpoint = false ; int breakpoint = -1 ; std :: string justification = " " ; for ( window_size = 0 ; window_size < 100 && ! found_breakpoint ; window_size ++ ) { left_idx = curr_char - window_size ; right_idx = curr_char + window_size ; if ( ( left_idx <= sent_start ) && ( ( right_idx >= orig_sent_len ) || ( right_idx - sent_start + 1 > getMaxSentenceChars ( ) ) ) ) break ; if ( ( left_idx > sent_start ) && isSecondaryEOSChar ( sentence -> getString ( ) -> charAt ( left_idx ) ) && ! inRestrictedBreakSpan ( _curDoc -> getMetadata ( ) , sentence -> getString ( ) , left_idx + 1 ) ) { breakpoint = left_idx ; justification = " " ; found_breakpoint = true ; } else if ( ( right_idx < orig_sent_len ) && ( right_idx - sent_start + 1 <= getMaxSentenceChars ( ) ) && isSecondaryEOSChar ( sentence -> getString ( ) -> charAt ( right_idx ) ) && ! inRestrictedBreakSpan ( _curDoc -> getMetadata ( ) , sentence -> getString ( ) , right_idx + 1 ) ) { breakpoint = right_idx ; justification = " " ; found_breakpoint = true ; } } for ( window_size = 0 ; ! found_breakpoint ; window_size ++ ) { left_idx = curr_char - window_size ; right_idx = curr_char + window_size ; if ( ( left_idx <= sent_start ) && ( ( right_idx >= orig_sent_len ) || ( right_idx - sent_start + 1 > getMaxSentenceChars ( ) ) ) ) break ; if ( ( left_idx > sent_start ) && iswspace ( sentence -> getString ( ) -> charAt ( left_idx ) ) && ! inRestrictedBreakSpan ( _curDoc -> getMetadata ( ) , sentence -> getString ( ) , left_idx + 1 ) ) { breakpoint = left_idx ; justification = " " ; found_breakpoint = true ; } else if ( ( right_idx < orig_sent_len ) && ( right_idx - sent_start + 1 <= getMaxSentenceChars ( ) ) && iswspace ( sentence -> getString ( ) -> charAt ( right_idx ) ) && ! inRestrictedBreakSpan ( _curDoc -> getMetadata ( ) , sentence -> getString ( ) , right_idx + 1 ) ) { breakpoint = right_idx ; justification = " " ; found_breakpoint = true ; } } if ( ! found_breakpoint ) { SessionLogger :: warn ( " " ) << " " ; } for ( window_size = 0 ; ! found_breakpoint ; window_size ++ ) { left_idx = curr_char - window_size ; right_idx = curr_char + window_size ; if ( ( left_idx <= sent_start ) && ( ( right_idx >= orig_sent_len ) || ( right_idx - sent_start + 1 > getMaxSentenceChars ( ) ) ) ) break ; if ( ( left_idx > sent_start ) && ! inRestrictedBreakSpan ( _curDoc -> getMetadata ( ) , sentence -> getString ( ) , left_idx + 1 ) ) { breakpoint = left_idx ; justification = " " ; found_breakpoint = true ; } else if ( ( right_idx < orig_sent_len ) && ( right_idx - sent_start + 1 <= getMaxSentenceChars ( ) ) && ! inRestrictedBreakSpan ( _curDoc -> getMetadata ( ) , sentence -> getString ( ) , right_idx + 1 ) ) { breakpoint = right_idx ; justification = " " ; found_breakpoint = true ; } } if ( ! found_breakpoint ) { throw UnexpectedInputException ( " " , " " ) ; } int next_sent_no = sentence -> getSentNumber ( ) + static_cast < int > ( results . size ( ) ) ; results . push_back ( createSubSentence ( sentence , next_sent_no , sent_start , breakpoint , justification ) ) ; sent_start = breakpoint + 1 ; curr_char = sent_start + avg_num_chars ; if ( ( sent_start < orig_sent_len ) && isAcceptableSentenceString ( sentence -> getString ( ) , sent_start , orig_sent_len ) ) { next_sent_no = sentence -> getSentNumber ( ) + static_cast < int > ( results . size ( ) ) ; results . push_back ( createFinalSubSentence ( sentence , next_sent_no , sent_start ) ) ; sent_start = orig_sent_len + 1 ; curr_char = orig_sent_len + 1 ; } } return results ; }
Attempts to break sentence into ~n_split_sents smaller sentences
Attempts to break sentence into ~n_split_sents smaller sentences
/* First, lets look to see if there's a comma nearby */ // stop searching if both left and right index are out of bounds
 /* No comma, so just break at a whitespace point */ // stop searching when both left and right index are out of bounds
 /* Can't find a good break, so find the first non-restricted breakpoint near curr_char */ // stop searching when both left and right index are out of bounds
 // if the remaining string is an acceptable size, finish out the rest of sentence
