src/Arabic/parse/ar_ChartDecoder.cpp
/* decode a sentence using bw derived tokenization */
decode a sentence using bw derived tokenization
decode a sentence using bw derived tokenization
ParseNode* ArabicChartDecoder::decode(SplitTokenSequence* bw_tokens, bool collapseNPlabels)
{
	WordSegment seg;
	int num_segments = 0;
	for (int i = 0; i < bw_tokens->getNTokens(); i++)
		num_segments += bw_tokens->getWordSegment(i)->maxLength();
	if (num_segments >= MAX_SENTENCE_LENGTH) {
		SessionLogger::logger->beginWarning();
		*SessionLogger::logger << "Using a flat parse because there are"
								  "too many possible segments.\n";
		throw UnexpectedInputException(
			"ArabicChartDecoder::Decode()","SplitTokenSequence has too many segments");
	}
	if(bw_tokens ==NULL){
		std::cerr<<"bw_tokens is NULL"<<std::endl;
	}
	if(bw_tokens->getWordSegment(bw_tokens->getNTokens()-1) ==NULL){
		std::cerr<<"bw_tokens last word is NULL"<<
			bw_tokens->getNTokens()<<" tokens "<<
			std::endl;
	}
	if(bw_tokens->getWordSegment(bw_tokens->getNTokens()-1)->getPossibility(0).segments ==NULL){
		std::cerr<<"bw_tokens last word first segment segments is NULL"
			<<bw_tokens->getNTokens()<< " tokens "<<std::endl;
	}
	bool lastIsPunct = LanguageSpecificFunctions::isSentenceEndingPunctuation(bw_tokens->getWordSegment(
		bw_tokens->getNTokens()-1)->getPossibility(0).segments[0]->getText());
	Symbol split_sentence[MAX_SENTENCE_LENGTH];
	int chart_size = initChart(bw_tokens, split_sentence);
	initPunctuationUpperBound(split_sentence, chart_size);
	for (int span = 2; span <= chart_size; span++) {
		for (int start = 0; start <= (chart_size - span); start++) {
			int end = start + span;
			numTheories = 0;
			if(DEBUG1){
				_debugStream <<"---------start: "<<start<<" end: "<<end<<"\n";
				_debugStream.flush();
			}
			for (int mid = (start + 1); mid < end; mid++) {
				if ((end == chart_size) && lastIsPunct &&
					!((mid == (chart_size - 1)) && (start == 0))) {
					continue;
					}
				if (punctuationCrossing(start, end, chart_size)) {
					continue;
				}
				leftClosable = !crossingConstraintViolation(start, mid);
				rightClosable = !crossingConstraintViolation(mid, end);
				for (ChartEntry** leftEntry = chart[start][mid - 1];
				*leftEntry; ++leftEntry)
				{
					for (ChartEntry** rightEntry = chart[mid][end - 1];
					*rightEntry; ++rightEntry)
					{
						addKernelTheories(*leftEntry, *rightEntry);
						addExtensionTheories(*leftEntry, *rightEntry);
					}
				}
			}
			transferTheoriesToChart(start, end);
		}
	}
	float finalScore;
	postProcessChart();
	ParseNode* returnValue = getBestParse(finalScore, 0, chart_size, false);
	ParseNode* iterReturnValue = returnValue;
	while (iterReturnValue != 0) {
		int replacementPosition = 0;
		postprocessParse(iterReturnValue, collapseNPlabels);
		addNameToParse(iterReturnValue, replacementPosition);
		iterReturnValue = iterReturnValue->next;
	}
	cleanupChart(chart_size);
	std::wstring resultNoSpace = returnValue->readLeaves();
	std::wstring sentNoSpace = L"";
	for(int i =0; i< bw_tokens->getNTokens(); i++){
		sentNoSpace += bw_tokens->getWordSegment(i)->getOriginalWord().to_string();
	}
	if(wcscmp(resultNoSpace.c_str(),sentNoSpace.c_str())!=0){
		std::cerr<<"\nToken Sequence After Parsing is wrong Because of Marjorie\n";
		Symbol sns = Symbol(sentNoSpace.c_str());
		Symbol rns = Symbol(resultNoSpace.c_str());
		std::cerr<<"Initial: "<<sns.to_debug_string()<<"\n";
		std::cerr<<"After:   "<<rns.to_debug_string()<<"\n";
	}
	return returnValue;
}
ParseNode * ArabicChartDecoder :: decode ( SplitTokenSequence * bw_tokens , bool collapseNPlabels ) { WordSegment seg ; int num_segments = 0 ; for ( int i = 0 ; i < bw_tokens -> getNTokens ( ) ; i ++ ) num_segments += bw_tokens -> getWordSegment ( i ) -> maxLength ( ) ; if ( num_segments >= MAX_SENTENCE_LENGTH ) { SessionLogger :: logger -> beginWarning ( ) ; * SessionLogger :: logger << " " " \n " ; throw UnexpectedInputException ( " " , " " ) ; } if ( bw_tokens == NULL ) { std :: cerr << " " << std :: endl ; } if ( bw_tokens -> getWordSegment ( bw_tokens -> getNTokens ( ) - 1 ) == NULL ) { std :: cerr << " " << bw_tokens -> getNTokens ( ) << " " << std :: endl ; } if ( bw_tokens -> getWordSegment ( bw_tokens -> getNTokens ( ) - 1 ) -> getPossibility ( 0 ) . segments == NULL ) { std :: cerr << " " << bw_tokens -> getNTokens ( ) << " " << std :: endl ; } bool lastIsPunct = LanguageSpecificFunctions :: isSentenceEndingPunctuation ( bw_tokens -> getWordSegment ( bw_tokens -> getNTokens ( ) - 1 ) -> getPossibility ( 0 ) . segments [ 0 ] -> getText ( ) ) ; Symbol split_sentence [ MAX_SENTENCE_LENGTH ] ; int chart_size = initChart ( bw_tokens , split_sentence ) ; initPunctuationUpperBound ( split_sentence , chart_size ) ; for ( int span = 2 ; span <= chart_size ; span ++ ) { for ( int start = 0 ; start <= ( chart_size - span ) ; start ++ ) { int end = start + span ; numTheories = 0 ; if ( DEBUG1 ) { _debugStream << " " << start << " " << end << " \n " ; _debugStream . flush ( ) ; } for ( int mid = ( start + 1 ) ; mid < end ; mid ++ ) { if ( ( end == chart_size ) && lastIsPunct && ! ( ( mid == ( chart_size - 1 ) ) && ( start == 0 ) ) ) { continue ; } if ( punctuationCrossing ( start , end , chart_size ) ) { continue ; } leftClosable = ! crossingConstraintViolation ( start , mid ) ; rightClosable = ! crossingConstraintViolation ( mid , end ) ; for ( ChartEntry * * leftEntry = chart [ start ] [ mid - 1 ] ; * leftEntry ; ++ leftEntry ) { for ( ChartEntry * * rightEntry = chart [ mid ] [ end - 1 ] ; * rightEntry ; ++ rightEntry ) { addKernelTheories ( * leftEntry , * rightEntry ) ; addExtensionTheories ( * leftEntry , * rightEntry ) ; } } } transferTheoriesToChart ( start , end ) ; } } float finalScore ; postProcessChart ( ) ; ParseNode * returnValue = getBestParse ( finalScore , 0 , chart_size , false ) ; ParseNode * iterReturnValue = returnValue ; while ( iterReturnValue != 0 ) { int replacementPosition = 0 ; postprocessParse ( iterReturnValue , collapseNPlabels ) ; addNameToParse ( iterReturnValue , replacementPosition ) ; iterReturnValue = iterReturnValue -> next ; } cleanupChart ( chart_size ) ; std :: wstring resultNoSpace = returnValue -> readLeaves ( ) ; std :: wstring sentNoSpace = L" " ; for ( int i = 0 ; i < bw_tokens -> getNTokens ( ) ; i ++ ) { sentNoSpace += bw_tokens -> getWordSegment ( i ) -> getOriginalWord ( ) . to_string ( ) ; } if ( wcscmp ( resultNoSpace . c_str ( ) , sentNoSpace . c_str ( ) ) != 0 ) { std :: cerr << " \n \n " ; Symbol sns = Symbol ( sentNoSpace . c_str ( ) ) ; Symbol rns = Symbol ( resultNoSpace . c_str ( ) ) ; std :: cerr << " " << sns . to_debug_string ( ) << " \n " ; std :: cerr << " " << rns . to_debug_string ( ) << " \n " ; } return returnValue ; }
decode a sentence using bw derived tokenization
decode a sentence using bw derived tokenization
// make sure that the maximally segmented version still fits in the chart
 // -- SRS
 //	std::cout<<"D 1"<<std::endl;
 //	std::cout<<"D 2"<<std::endl;
 //Returning an empty parse here leaves us without a name theory
 //instead build a flat "FRAGMENT" parse with NPP for names
 //	std::cout<<"D 3"<<std::endl;
 //	std::cout<<"D 4"<<std::endl;
 //	std::cout<<"D 5"<<std::endl;
 //	std::cout<<"chart_size: "<<chart_size<<" \n";
 //	std::cout<<"D 6"<<std::endl;
 // diversity parsing iteration
 //	std::cout<<"D 7"<<std::endl;
 //	std::cout<<"D 8"<<std::endl;
 /*	if(returnValue == NULL){
		std::cout<<"EMPTY RETURN!!!\n";
	}
	else{
		std::cout<<"RETURN VALUE EXISTS !! "<<finalScore<<"\n";
	}
*/ //if(DEBUG2){
 //	_parseStream << returnValue->toWString();
 //	_parseStream <<"\n\n";
 //}
 //	std::cout<<"D 9"<<std::endl;
 //	std::cout<<"D 10"<<std::endl;
