src/Generic/reader/ProseDocumentZoner.cpp
// ProcessRegion will delete r, and replace it with a new Region in newRegions

ProcessRegion will delete r, and replace it with a new Region in newRegions
ProcessRegion will delete r and replace it with a new Region in newRegions
void ProseDocumentZoner::processRegion(Document *document, const Region *r, std::vector<Region *> &newRegions) {
	const LocatedString* contents = r->getString();
	std::vector<Region*> regions;
	int word_start = -1;
	int i = 0;
	bool in_word = false;
	std::vector<ClassifiedWord> words;
	while (true) {
		if (i >= contents->length()) {
			if (word_start > -1) {
				Classification classification = classifyWord(contents, word_start, i);
				words.push_back(ClassifiedWord(word_start, i, classification));
			}
			break;
		}
		wchar_t c = contents->charAt(i);
		if (!in_word) {
			if (iswspace(c)) {
				;
			} else {
				in_word = true;
				word_start = i;
			}
		}
		if (in_word) {
			if (iswspace(c)) {
				Classification classification = classifyWord(contents, word_start, i);
				words.push_back(ClassifiedWord(word_start, i, classification));
				in_word = false;
				word_start = -1;
			} else {
				;
			}
		}
		i++;
		continue;
	}
	size_t current_word = 0;
	std::vector<OffsetPair> nonProseOffsets;
	while (true) {
		if (current_word >= words.size())
			break;
		int last_non_prose_index = -1;
		if (words[current_word].classification == NON_PROSE) {
			last_non_prose_index = getIndexOfLastNonProse(words, current_word);
		}
		if (last_non_prose_index == -1) { 
			current_word++;
			continue;
		}
		nonProseOffsets.push_back(
			OffsetPair(words[current_word].start_offset, words[last_non_prose_index].end_offset));
		current_word = last_non_prose_index + 1;
	}
	int start_of_region = 0;
	BOOST_FOREACH (OffsetPair offsetPair, nonProseOffsets) {
		if (offsetPair.first > start_of_region) {
			LocatedString *newRegionString = contents->substring(start_of_region, offsetPair.first);
			Region *proseRegion = new Region(document, PROSE_TAG, static_cast<int>(newRegions.size()), newRegionString);
			delete newRegionString;
			newRegions.push_back(proseRegion);
		}
		LocatedString *newRegionString = contents->substring(offsetPair.first, offsetPair.second);
		Region *nonProseRegion = new Region(document, NON_PROSE_TAG, static_cast<int>(newRegions.size()), newRegionString);
		delete newRegionString;
		newRegions.push_back(nonProseRegion);
		start_of_region = offsetPair.second;
	}
	if (start_of_region < contents->length()) {
		LocatedString *newRegionString = contents->substring(start_of_region);
		Region *proseRegion = new Region(document, PROSE_TAG, static_cast<int>(newRegions.size()), newRegionString);
		delete newRegionString;
		std::wstring regionString = proseRegion->getString()->toString();
		boost::algorithm::trim(regionString);
		if (regionString.length() > 0) {
			newRegions.push_back(proseRegion);
		} else {
			delete proseRegion;
		}		
	}
	delete r;
}
void ProseDocumentZoner :: processRegion ( Document * document , const Region * r , std :: vector < Region * > & newRegions ) { const LocatedString * contents = r -> getString ( ) ; std :: vector < Region * > regions ; int word_start = -1 ; int i = 0 ; bool in_word = false ; std :: vector < ClassifiedWord > words ; while ( true ) { if ( i >= contents -> length ( ) ) { if ( word_start > -1 ) { Classification classification = classifyWord ( contents , word_start , i ) ; words . push_back ( ClassifiedWord ( word_start , i , classification ) ) ; } break ; } wchar_t c = contents -> charAt ( i ) ; if ( ! in_word ) { if ( iswspace ( c ) ) { ; } else { in_word = true ; word_start = i ; } } if ( in_word ) { if ( iswspace ( c ) ) { Classification classification = classifyWord ( contents , word_start , i ) ; words . push_back ( ClassifiedWord ( word_start , i , classification ) ) ; in_word = false ; word_start = -1 ; } else { ; } } i ++ ; continue ; } size_t current_word = 0 ; std :: vector < OffsetPair > nonProseOffsets ; while ( true ) { if ( current_word >= words . size ( ) ) break ; int last_non_prose_index = -1 ; if ( words [ current_word ] . classification == NON_PROSE ) { last_non_prose_index = getIndexOfLastNonProse ( words , current_word ) ; } if ( last_non_prose_index == -1 ) { current_word ++ ; continue ; } nonProseOffsets . push_back ( OffsetPair ( words [ current_word ] . start_offset , words [ last_non_prose_index ] . end_offset ) ) ; current_word = last_non_prose_index + 1 ; } int start_of_region = 0 ; BOOST_FOREACH ( OffsetPair offsetPair , nonProseOffsets ) { if ( offsetPair . first > start_of_region ) { LocatedString * newRegionString = contents -> substring ( start_of_region , offsetPair . first ) ; Region * proseRegion = new Region ( document , PROSE_TAG , static_cast < int > ( newRegions . size ( ) ) , newRegionString ) ; delete newRegionString ; newRegions . push_back ( proseRegion ) ; } LocatedString * newRegionString = contents -> substring ( offsetPair . first , offsetPair . second ) ; Region * nonProseRegion = new Region ( document , NON_PROSE_TAG , static_cast < int > ( newRegions . size ( ) ) , newRegionString ) ; delete newRegionString ; newRegions . push_back ( nonProseRegion ) ; start_of_region = offsetPair . second ; } if ( start_of_region < contents -> length ( ) ) { LocatedString * newRegionString = contents -> substring ( start_of_region ) ; Region * proseRegion = new Region ( document , PROSE_TAG , static_cast < int > ( newRegions . size ( ) ) , newRegionString ) ; delete newRegionString ; std :: wstring regionString = proseRegion -> getString ( ) -> toString ( ) ; boost :: algorithm :: trim ( regionString ) ; if ( regionString . length ( ) > 0 ) { newRegions . push_back ( proseRegion ) ; } else { delete proseRegion ; } } delete r ; }
ProcessRegion will delete r, and replace it with a new Region in newRegions
ProcessRegion will delete r and replace it with a new Region in newRegions
// Cycle over contents one at a time finding words based on whitespace
 // and classifying them when found
 // words vector is filled with all words in the document and they have all been classified
 // Find non-prose sections
 //std::cout << "Looking at: " << UnicodeUtil::toUTF8StdString(contents->substring(words[current_word].start_offset, words[current_word].end_offset)->toString()) << "\n";
 //std::cout << "Last non prose index : " << last_non_prose_index << "\n";
 // Didn't find non-prose section
 // Found non-prose section
 // create PROSE region
 //std::cout << "\n---------------------\nPROSE: \n";
 //std::cout << UnicodeUtil::toUTF8StdString(proseRegion->getString()->toString());
 // create NON_PROSE region
 //std::cout << "\n---------------------\nNON PROSE: \n";
 //std::cout << UnicodeUtil::toUTF8StdString(nonProseRegion->getString()->toString());
 // final PROSE region
 //std::cout << "\n---------------------\nPROSE: \n";
 //std::cout << UnicodeUtil::toUTF8StdString(proseRegion->getString()->toString());
