src/PredFinder/PredFinder.cpp
/**
 * Validates LearnIt and manual patterns against reading macro sets to make sure that each relation
 * produced by a pattern will be consumed by a macro set, and that each macro set consumes at least
 * one relation.
 * @param force_verbose Derived from the parameter "verbose_pattern_verification". If true, output
 * is highly verbose. To gain finer control of verbosity, set "verbose_pattern_verification" to false
 * and set the individual parameters (e.g., "rel_union") mentioned in the body of this method to true
 * as desired.
 *
 * @author afrankel@bbn.com
 **/
Validates LearnIt and manual patterns against reading macro sets to make sure that each relation
produced by a pattern will be consumed by a macro set, and that each macro set consumes at least
one relation.
@param force_verbose Derived from the parameter "verbose_pattern_verification". If true, output
is highly verbose. To gain finer control of verbosity, set "verbose_pattern_verification" to false
and set the individual parameters  mentioned in the body of this method to true
as desired.


Validates LearnIt and manual patterns against reading macro sets to make sure that each relation produced by a pattern will be consumed by a macro set and that each macro set consumes at least one relation . @param force_verbose Derived from the parameter " verbose_pattern_verification " . If true output is highly verbose . To gain finer control of verbosity set " verbose_pattern_verification " to false and set the individual parameters mentioned in the body of this method to true as desired .
bool PredFinder::validatePatterns() {
	bool validate_manual_patterns = ParamReader::getOptionalTrueFalseParamWithDefaultVal(
		"validate_manual_patterns",  false);
	bool verbose_pattern_verification = ParamReader::getOptionalTrueFalseParamWithDefaultVal(
		"verbose_pattern_verification",  false);
	if (!validate_manual_patterns) {
		return false;
	}
	bool bAllRelationsUsed(true);
	bool bAllMacrosUsed(true);
	ElfRelationMultisetSortedByName rel_from_li_patt_set;
	ElfRelationMultisetSortedByName rel_from_man_patt_set;
	std::set<std::wstring> tbd_pattern_set_names;
	std::set<std::wstring> tbd_predicates;
	ElfRelationFactory::fake_relations_from_li_pattern_set(_learnit_patterns, rel_from_li_patt_set);
	ElfRelationFactory::fake_relations_from_man_pattern_set(_manual_patterns, rel_from_man_patt_set, tbd_pattern_set_names,
		tbd_predicates, verbose_pattern_verification);
	ElfRelationMultisetSortedByName filtered_rel_from_man_patt_set;
	BOOST_FOREACH(ElfRelation_ptr rel, rel_from_man_patt_set) {
		std::wstring rel_name = rel->get_name();
		std::set<std::wstring>::const_iterator iter = tbd_pattern_set_names.find(rel_name);
		if (iter == tbd_pattern_set_names.end()) {
			filtered_rel_from_man_patt_set.insert(rel);
		}
	}
	size_t rel_from_man_patt_set_size(filtered_rel_from_man_patt_set.size());
	std::vector<ElfRelation_ptr> rel_from_man_patt_vec(filtered_rel_from_man_patt_set.begin(), filtered_rel_from_man_patt_set.end());
	if (verbose_pattern_verification || SessionLogger::dbg_or_msg_enabled("verbose_pattern_verification")) {
	   printInfoAboutPatterns(rel_from_li_patt_set, filtered_rel_from_man_patt_set);
    }
	std::vector<ElfRelation_ptr> rel_union;
	set_union(rel_from_li_patt_set.begin(), rel_from_li_patt_set.end(),
		filtered_rel_from_man_patt_set.begin(), filtered_rel_from_man_patt_set.end(), std::inserter(rel_union, rel_union.end()), 
		ElfRelation_less_than());
    if (verbose_pattern_verification || SessionLogger::dbg_or_msg_enabled("rel_union")) {
		SessionLogger::dbg("rel_union") << "UNION OF RELATIONS FROM LEARNIT AND MANUAL PATTERNS:";
    	BOOST_FOREACH(ElfRelation_ptr rel, rel_union) {
    		std::ostringstream ostr;
    		rel->dump(ostr);
    		SessionLogger::dbg("rel_union") << ostr.str();
    	}
    }
	std::vector<std::wstring> unconsuming_macro_sets;
	std::vector<std::wstring> unconsumed_relations;
	size_t rel_union_size(rel_union.size());
	std::vector<bool> relation_used(rel_union_size, false);
	std::set<std::wstring> preds_generated_in_bbn_stage;
	BOOST_FOREACH(ReadingMacroSet_ptr reading_macro_set, _reading_macros) {
		bool bUsed(false);
		reading_macro_set->retrieve_predicates_generated_in_stage(ReadingMacro::STAGE_BBN_SYM, preds_generated_in_bbn_stage);
		std::wstring macro_set_name = reading_macro_set->get_macro_set_name().to_string();
		std::set<std::wstring>::const_iterator iter = tbd_predicates.find(macro_set_name);
		if (iter != tbd_predicates.end()) {
			bUsed = true; 
		} else {
			for (size_t i = 0; i < rel_union_size; ++i) {
				if (reading_macro_set->matches(L"bbn", rel_union[i])) {
					relation_used[i] = true;
					bUsed = true;
				} else if (reading_macro_set->matches(L"erudite", rel_union[i])) {
					relation_used[i] = true;
					bUsed = true;
				}
			}
		}
		if (!bUsed && !reading_macro_set->has_operator(ReadingMacroOperator::CATCH_SYM)) {
			std::wstring name(reading_macro_set->get_macro_set_name().to_string());
			unconsuming_macro_sets.push_back(name);
		}
	}
	std::set<std::wstring> filtered_unconsuming_macro_sets;
	BOOST_FOREACH(std::wstring unconsuming_macro_set, unconsuming_macro_sets) {
		if (preds_generated_in_bbn_stage.find(unconsuming_macro_set) == preds_generated_in_bbn_stage.end()) {
			filtered_unconsuming_macro_sets.insert(unconsuming_macro_set);
		}
	}
	ostringstream unconsumed_rel_info;
	for (size_t i = 0; i < rel_union_size; ++i) {
		if (!relation_used[i]) {
			wstringstream wstr;
			wstr << rel_union[i]->get_name() << " (" << rel_union[i]->get_source() << ")";
			unconsumed_relations.push_back(wstr.str());
			if (unconsumed_relations.size() == 1) { 
				unconsumed_rel_info << "Information about relations not consumed by macro sets:\n";
			}
			rel_union[i]->dump(unconsumed_rel_info,  0);
			bAllRelationsUsed = false;
		}
	}
	wstringstream unconsuming_macro_info;
	if (!filtered_unconsuming_macro_sets.empty()) {
		bAllMacrosUsed = false;
		unconsuming_macro_info << L"\nMacro sets that consumed no relations:\n";
		BOOST_FOREACH(std::wstring wstr, filtered_unconsuming_macro_sets) {
			unconsuming_macro_info << wstr << endl;
		}
	}
	if (!unconsuming_macro_info.str().empty()) {
		SessionLogger::warn("unconsuming_macro_0") << unconsuming_macro_info.str();
	}
	wstringstream unconsumed_rel_list;
	if (!unconsumed_relations.empty()) {
		unconsumed_rel_list << "\nRelations not consumed by macro sets:\n";
		BOOST_FOREACH(std::wstring wstr, unconsumed_relations) {
			unconsumed_rel_list << wstr << endl;
		}
	}
	if (!unconsumed_rel_info.str().empty()) {
		SessionLogger::warn("unconsumed_rel_0") << unconsumed_rel_info.str();
	}
	if (!unconsumed_rel_list.str().empty()) {
		SessionLogger::warn("unconsumed_rel_list_0") << unconsumed_rel_list.str();
	}
	return(bAllRelationsUsed && bAllMacrosUsed); 
}
bool PredFinder :: validatePatterns ( ) { bool validate_manual_patterns = ParamReader :: getOptionalTrueFalseParamWithDefaultVal ( " " , false ) ; bool verbose_pattern_verification = ParamReader :: getOptionalTrueFalseParamWithDefaultVal ( " " , false ) ; if ( ! validate_manual_patterns ) { return false ; } bool bAllRelationsUsed ( true ) ; bool bAllMacrosUsed ( true ) ; ElfRelationMultisetSortedByName rel_from_li_patt_set ; ElfRelationMultisetSortedByName rel_from_man_patt_set ; std :: set < std :: wstring > tbd_pattern_set_names ; std :: set < std :: wstring > tbd_predicates ; ElfRelationFactory :: fake_relations_from_li_pattern_set ( _learnit_patterns , rel_from_li_patt_set ) ; ElfRelationFactory :: fake_relations_from_man_pattern_set ( _manual_patterns , rel_from_man_patt_set , tbd_pattern_set_names , tbd_predicates , verbose_pattern_verification ) ; ElfRelationMultisetSortedByName filtered_rel_from_man_patt_set ; BOOST_FOREACH ( ElfRelation_ptr rel , rel_from_man_patt_set ) { std :: wstring rel_name = rel -> get_name ( ) ; std :: set < std :: wstring > :: const_iterator iter = tbd_pattern_set_names . find ( rel_name ) ; if ( iter == tbd_pattern_set_names . end ( ) ) { filtered_rel_from_man_patt_set . insert ( rel ) ; } } size_t rel_from_man_patt_set_size ( filtered_rel_from_man_patt_set . size ( ) ) ; std :: vector < ElfRelation_ptr > rel_from_man_patt_vec ( filtered_rel_from_man_patt_set . begin ( ) , filtered_rel_from_man_patt_set . end ( ) ) ; if ( verbose_pattern_verification || SessionLogger :: dbg_or_msg_enabled ( " " ) ) { printInfoAboutPatterns ( rel_from_li_patt_set , filtered_rel_from_man_patt_set ) ; } std :: vector < ElfRelation_ptr > rel_union ; set_union ( rel_from_li_patt_set . begin ( ) , rel_from_li_patt_set . end ( ) , filtered_rel_from_man_patt_set . begin ( ) , filtered_rel_from_man_patt_set . end ( ) , std :: inserter ( rel_union , rel_union . end ( ) ) , ElfRelation_less_than ( ) ) ; if ( verbose_pattern_verification || SessionLogger :: dbg_or_msg_enabled ( " " ) ) { SessionLogger :: dbg ( " " ) << " " ; BOOST_FOREACH ( ElfRelation_ptr rel , rel_union ) { std :: ostringstream ostr ; rel -> dump ( ostr ) ; SessionLogger :: dbg ( " " ) << ostr . str ( ) ; } } std :: vector < std :: wstring > unconsuming_macro_sets ; std :: vector < std :: wstring > unconsumed_relations ; size_t rel_union_size ( rel_union . size ( ) ) ; std :: vector < bool > relation_used ( rel_union_size , false ) ; std :: set < std :: wstring > preds_generated_in_bbn_stage ; BOOST_FOREACH ( ReadingMacroSet_ptr reading_macro_set , _reading_macros ) { bool bUsed ( false ) ; reading_macro_set -> retrieve_predicates_generated_in_stage ( ReadingMacro :: STAGE_BBN_SYM , preds_generated_in_bbn_stage ) ; std :: wstring macro_set_name = reading_macro_set -> get_macro_set_name ( ) . to_string ( ) ; std :: set < std :: wstring > :: const_iterator iter = tbd_predicates . find ( macro_set_name ) ; if ( iter != tbd_predicates . end ( ) ) { bUsed = true ; } else { for ( size_t i = 0 ; i < rel_union_size ; ++ i ) { if ( reading_macro_set -> matches ( L" " , rel_union [ i ] ) ) { relation_used [ i ] = true ; bUsed = true ; } else if ( reading_macro_set -> matches ( L" " , rel_union [ i ] ) ) { relation_used [ i ] = true ; bUsed = true ; } } } if ( ! bUsed && ! reading_macro_set -> has_operator ( ReadingMacroOperator :: CATCH_SYM ) ) { std :: wstring name ( reading_macro_set -> get_macro_set_name ( ) . to_string ( ) ) ; unconsuming_macro_sets . push_back ( name ) ; } } std :: set < std :: wstring > filtered_unconsuming_macro_sets ; BOOST_FOREACH ( std :: wstring unconsuming_macro_set , unconsuming_macro_sets ) { if ( preds_generated_in_bbn_stage . find ( unconsuming_macro_set ) == preds_generated_in_bbn_stage . end ( ) ) { filtered_unconsuming_macro_sets . insert ( unconsuming_macro_set ) ; } } ostringstream unconsumed_rel_info ; for ( size_t i = 0 ; i < rel_union_size ; ++ i ) { if ( ! relation_used [ i ] ) { wstringstream wstr ; wstr << rel_union [ i ] -> get_name ( ) << " " << rel_union [ i ] -> get_source ( ) << " " ; unconsumed_relations . push_back ( wstr . str ( ) ) ; if ( unconsumed_relations . size ( ) == 1 ) { unconsumed_rel_info << " \n " ; } rel_union [ i ] -> dump ( unconsumed_rel_info , 0 ) ; bAllRelationsUsed = false ; } } wstringstream unconsuming_macro_info ; if ( ! filtered_unconsuming_macro_sets . empty ( ) ) { bAllMacrosUsed = false ; unconsuming_macro_info << L" \n \n " ; BOOST_FOREACH ( std :: wstring wstr , filtered_unconsuming_macro_sets ) { unconsuming_macro_info << wstr << endl ; } } if ( ! unconsuming_macro_info . str ( ) . empty ( ) ) { SessionLogger :: warn ( " " ) << unconsuming_macro_info . str ( ) ; } wstringstream unconsumed_rel_list ; if ( ! unconsumed_relations . empty ( ) ) { unconsumed_rel_list << " \n \n " ; BOOST_FOREACH ( std :: wstring wstr , unconsumed_relations ) { unconsumed_rel_list << wstr << endl ; } } if ( ! unconsumed_rel_info . str ( ) . empty ( ) ) { SessionLogger :: warn ( " " ) << unconsumed_rel_info . str ( ) ; } if ( ! unconsumed_rel_list . str ( ) . empty ( ) ) { SessionLogger :: warn ( " " ) << unconsumed_rel_list . str ( ) ; } return ( bAllRelationsUsed && bAllMacrosUsed ) ; }
Validates LearnIt and manual patterns against reading macro sets to make sure that each relation
 produced by a pattern will be consumed by a macro set, and that each macro set consumes at least
 one relation.
Validates LearnIt and manual patterns against reading macro sets to make sure that each relation produced by a pattern will be consumed by a macro set and that each macro set consumes at least one relation .
/*defaultVal=*/ /* defaultVal=*/ // Fill rel_union with the union of relations from LearnIt and manual patterns.
 // Determine whether each reading macro consumes at least one relation.
 // e.g., we encountered a pattern that returns "tbd bombing", which translates (via EITbdAdapter::getEventName())
 // into "eru:bombingAttackEvent", which matches the macro set name in the macro set file eru_bombingAttackEvent.sexp.
 // Macros with a "catch" operator are only for catching relations that unexpectedly 
 // slip through. We don't need to validate those macros.
 // Make another pass through unconsuming_macro_sets to weed out the ones that consume relations
 // produced during the earlier ("bbn") stage of macro application.
 // Determine whether each relation has been used.
 // first item
 /*indent=*/ // Unconsumed relations will cause the function to return false, but 
 // unconsuming macro sets just generate warnings.
