src/LearnIt2Trainer/LearnIt2Trainer.cpp
// this creates a simple sample problem to calibrate the weights of the
// different loss functions.
// It has one instance i which is the source of all information. For sentence features,
// it has the sentence bias feature and one feature g annotated at 85% expectation.
// For slot features, it has the prior feature and a single feature h.
// There are ten more instances which all have the same slot features as i.
// However, they all have (in addition to the sentence bias feature) a distinct
// sentence feature apiece. There is a further layer of 10 instances, one for 
// each in the middle layer, which have once apiece the sentence features along
// with distinct slot features. We expect the slot predictions on this to be around 0.4.
// Finally, to set the prior, we have 90 instances which
// each have the two priors and one distinct slot and sentence feature apiece.
// The goal is to have p_sent(i) > 80% and p_sent(x) > 40% for all instances x
// with slot feature g, while maintaining p_sent(y) < 12% for all other instance
// y.

this creates a simple sample problem to calibrate the weights of the
different loss functions.
It has one instance i which is the source of all information. For sentence features,
it has the sentence bias feature and one feature g annotated at 85% expectation.
For slot features, it has the prior feature and a single feature h.
There are ten more instances which all have the same slot features as i.
However, they all have (in addition to the sentence bias feature) a distinct
sentence feature apiece. There is a further layer of 10 instances, one for
each in the middle layer, which have once apiece the sentence features along
with distinct slot features. We expect the slot predictions on this to be around 0.4.
Finally, to set the prior, we have 90 instances which
each have the two priors and one distinct slot and sentence feature apiece.
The goal is to have p_sent(i) > 80% and p_sent(x) > 40% for all instances x
with slot feature g, while maintaining p_sent(y) < 12% for all other instance
y.
this creates a simple sample problem to calibrate the weights of the different loss functions . It has one instance i which is the source of all information . For sentence features it has the sentence bias feature and one feature g annotated at 85% expectation . For slot features it has the prior feature and a single feature h . There are ten more instances which all have the same slot features as i . However they all have ( in addition to the sentence bias feature ) a distinct sentence feature apiece . There is a further layer of 10 instances one for each in the middle layer which have once apiece the sentence features along with distinct slot features . We expect the slot predictions on this to be around 0 . 4 . Finally to set the prior we have 90 instances which each have the two priors and one distinct slot and sentence feature apiece . The goal is to have p_sent ( i ) > 80% and p_sent ( x ) > 40% for all instances x with slot feature g while maintaining p_sent ( y ) < 12% for all other instance y .
void calibrateWeights(const std::string& dummyDB, const std::string& dummyStringsFile) {
	throw UnrecoverableException("LearnIt2Trainer.cpp:calibrateWeights",
		"Need to update for alternating optimization.");
}
void calibrateWeights ( const std :: string & dummyDB , const std :: string & dummyStringsFile ) { throw UnrecoverableException ( " " , " " ) ; }
this creates a simple sample problem to calibrate the weights of the
different loss functions.
this creates a simple sample problem to calibrate the weights of the different loss functions .
//LearnIt2DB_ptr db = make_shared<LearnIt2DB>(dummyDB, false);
 //FromDBFeatureAlphabet_ptr alphabet = db->getAlphabet(0);
 //InstanceHashes_ptr instanceHashes = make_shared<InstanceHashes>();
 //int nInstances = 111;
 //int nFeatures = alphabet->size();
 //InferenceDataView_ptr slotView =
 //	make_shared<InferenceDataView>(nInstances, nFeatures);
 //InferenceDataView_ptr sentenceView = 
 //	make_shared<InferenceDataView>(nInstances, nFeatures);
 //InferenceDataView_ptr combinedView =
 //	make_shared<InferenceDataView>(nInstances, nFeatures);
 //int slotBias = 0;
 //int sentBias = 1;
 //int labeledSentFeature = 2;
 //int transmittingSlotFeature = 3;
 //std::set<unsigned int> featuresInInstance;
 //for (int i=0; i<nInstances; ++i) {
 //	instanceHashes->registerInstance(i, lexical_cast<string>(i));
 //}
 //// create our only (indirectly) labeled instance
 //featuresInInstance.insert(0);
 //featuresInInstance.insert(3);
 //slotView->observeFeaturesForInstance(0, featuresInInstance);
 //combinedView->observeFeaturesForInstance(0, featuresInInstance);
 //featuresInInstance.clear();
 //featuresInInstance.insert(1);
 //featuresInInstance.insert(2);
 //sentenceView->observeFeaturesForInstance(0, featuresInInstance);
 //combinedView->observeFeaturesForInstance(0, featuresInInstance);
 //// make the ten instances we want to propogate information to from the
 //// labeled instance
 //int nextFeature = 4;
 //int nextInstance = 1;
 //std::vector<int> middle_layer_sentence_features;
 //std::vector<int> middle_layer_instances;
 //for (int i=0; i<10; ++i) {
 //	int inst = nextInstance++;
 //	featuresInInstance.clear();
 //	featuresInInstance.insert(slotBias);
 //	featuresInInstance.insert(transmittingSlotFeature);
 //	slotView->observeFeaturesForInstance(inst, featuresInInstance);
 //	combinedView->observeFeaturesForInstance(inst, featuresInInstance);
 //	featuresInInstance.clear();
 //	featuresInInstance.insert(sentBias);
 //	int feat = nextFeature++;
 //	featuresInInstance.insert(feat);
 //	middle_layer_sentence_features.push_back(feat);
 //	sentenceView->observeFeaturesForInstance(inst, featuresInInstance);
 //	combinedView->observeFeaturesForInstance(inst, featuresInInstance);
 //	middle_layer_instances.push_back(inst);
 //}
 //std::vector<int> bottom_layer_instances;
 //// the second layer of propagated information
 //for (int i=0; i<10; ++i) {
 //	int inst = nextInstance++;
 //	featuresInInstance.clear();
 //	featuresInInstance.insert(sentBias);
 //	featuresInInstance.insert(middle_layer_sentence_features[i]);
 //	sentenceView->observeFeaturesForInstance(inst, featuresInInstance);
 //	combinedView->observeFeaturesForInstance(inst, featuresInInstance);
 //	featuresInInstance.clear();
 //	featuresInInstance.insert(slotBias);
 //	featuresInInstance.insert(nextFeature++);
 //	slotView->observeFeaturesForInstance(inst, featuresInInstance);
 //	combinedView->observeFeaturesForInstance(inst, featuresInInstance);
 //	bottom_layer_instances.push_back(inst);
 //}
 //// these are a bunch of dummies just to make the whole distribution
 //// approximately match the prior
 //std::vector<int> prior_tuning_instances;
 //for (int i=0; i<90; ++i) {
 //	int inst = nextInstance++;
 //	featuresInInstance.clear();
 //	featuresInInstance.insert(slotBias);
 //	featuresInInstance.insert(nextFeature++);
 //	slotView->observeFeaturesForInstance(inst, featuresInInstance);
 //	combinedView->observeFeaturesForInstance(inst, featuresInInstance);
 //	featuresInInstance.clear();
 //	featuresInInstance.insert(sentBias);
 //	featuresInInstance.insert(nextFeature++);
 //	sentenceView->observeFeaturesForInstance(inst, featuresInInstance);
 //	combinedView->observeFeaturesForInstance(inst, featuresInInstance);
 //	prior_tuning_instances.push_back(inst);
 //}
 //ActiveLearningData_ptr alData = make_shared<ActiveLearningData>(
 //	db->instanceAnnotationView(), 
 //	dynamic_pointer_cast<FeatureAlphabet>(alphabet),
 //	instanceHashes, dynamic_pointer_cast<DataView>(combinedView));
 //alData->_test_clear_annotations();
 //alData->registerAnnotation(2, "positive");
 //alData->updateBetweenIterations();
 //Eigen::VectorXd regWeights(nFeatures);
 //for (int i=0; i<nextFeature; ++i) {
 //	regWeights.coeffRef(i) = 1.0;
 //}
 //for (int i=nextFeature; i<nFeatures; ++i) {
 //	regWeights.coeffRef(i) = 1.0;
 //}
 //if (ParamReader::isParamTrue("learnit2_calibration_exponential_ballpark_search")) {
 //	typedef std::pair<double, std::string> Result;
 //	vector<Result> results;
 //	
 //	LearnIt2Trainer_ptr trainer = make_shared<LearnIt2Trainer>(db, 
 //		dynamic_pointer_cast<FeatureAlphabet>(alphabet),
 //		slotView, sentenceView, slotView->nInstances(), false, false);
 //	bool l1 = false;
 //	double inst_weight = 1.0;
 //	double prior_weight = 1.0;
 //	//double reg_weight = 1.0;
 //	for (double reg_weight = 0.01; reg_weight < 1.1; reg_weight *= 10.0) {
 //		for (double diss_ann_weight = 50.0; diss_ann_weight < 200; diss_ann_weight += 50.0) {
 //			for (double fan_weight = 400.0; fan_weight < 1001.0; fan_weight +=100.0) {
 //				for (double diss_not_weight = 1.0; diss_not_weight < 50.0; diss_not_weight += 10.0) {
 //					/*reg_weight = 1.0;
 //					diss_ann_weight = 100.0;
 //					fan_weight = 1000.0;
 //					diss_not_weight = 10.0;*/
 //					/*fan_weight = 1000.0;
 //					diss_ann_weight = 200.0;
 //					diss_not_weight = 50.0;*/
 //					trainer->_test_clear_params();
 //					trainer->_test_clear_objective_components();
 //					trainer->addObjectiveComponent(make_shared<RegularizationLoss>(
 //						trainer->parameters(), regWeights, 
 //						reg_weight));
 //					trainer->addObjectiveComponent(make_shared<InstanceLoss>(sentenceView,
 //						alData, inst_weight));
 //					trainer->addObjectiveComponent(make_shared<InstanceLoss>(slotView, 
 //						alData, inst_weight));
 //					if (l1) {
 //						trainer->addObjectiveComponent(make_shared<FeatureAnnotationLoss>(
 //							slotView, alData, fan_weight, FeatureAnnotationLoss::L1));
 //						trainer->addObjectiveComponent(make_shared<FeatureAnnotationLoss>(
 //							sentenceView, alData, fan_weight, FeatureAnnotationLoss::L1));
 //						trainer->addObjectiveComponent(make_shared<UnlabeledDisagreementLoss>(
 //							slotView, sentenceView, alData, diss_ann_weight, diss_not_weight,
 //							UnlabeledDisagreementLoss::L1));
 //						//if (ParamReader::isParamTrue("learnit2_prior_in_objective") {
 //						trainer->addObjectiveComponent(make_shared<PriorLoss>(slotView, 
 //							ParamReader::getRequiredFloatParam("relation_prior"),
 //							prior_weight, PriorLoss::L1));
 //						trainer->addObjectiveComponent(make_shared<PriorLoss>(sentenceView, 
 //							ParamReader::getRequiredFloatParam("relation_prior"),
 //							prior_weight, PriorLoss::L1));
 //					} else {
 //						trainer->addObjectiveComponent(make_shared<FeatureAnnotationLoss>(
 //							slotView, alData, fan_weight, FeatureAnnotationLoss::KL));
 //						trainer->addObjectiveComponent(make_shared<FeatureAnnotationLoss>(
 //							sentenceView, alData, fan_weight, FeatureAnnotationLoss::KL));
 //						trainer->addObjectiveComponent(make_shared<UnlabeledDisagreementLoss>(
 //							slotView, sentenceView, alData, diss_ann_weight, diss_not_weight,
 //							UnlabeledDisagreementLoss::KL));
 //						//if (ParamReader::isParamTrue("learnit2_prior_in_objective") {
 //						trainer->addObjectiveComponent(make_shared<PriorLoss>(slotView, 
 //							ParamReader::getRequiredFloatParam("relation_prior"),
 //							prior_weight, PriorLoss::KL));
 //						trainer->addObjectiveComponent(make_shared<PriorLoss>(sentenceView, 
 //							ParamReader::getRequiredFloatParam("relation_prior"),
 //							prior_weight, PriorLoss::KL));
 //					}
 //					trainer->optimize(50);
 //					// we score a set of parameters by the the squared differences
 //					// between the predictions of the following probabilities
 //					// and what we want them to be
 //					// feature-labeled instance, sentence prob: >= 80
 //					// feature-labeled instance, slot prob: >= 75
 //					// 'transmitted' instances, sent_prob: >= 40
 //					// prior instance, slot_prob: <= 12
 //					// prior instance, sent_prob: <= 12
 //					// feature labeled instance is zero
 //					double loss = 0.0;
 //					// feature-labeled instance, sentence prob: >= 80
 //					double pred = sentenceView->prediction(0);
 //					double fl_sent_pred = pred;
 //					if (pred < 0.8) {
 //						loss += abs((0.8 - pred)) /** (0.8 - pred)*/;
 //					}
 //					// feature-labeled instance, slot prob: should be around .70
 //					// we multiply by 11 because this is also the slot prob for 
 //					// all the transmitted instances
 //					pred = slotView->prediction(0);
 //					double fl_slot_pred = pred;
 //					loss += abs(11.0*(pred - .70)) /** (pred-.70)*/;
 //					// 'transmitted' instances, sent prob, around .55
 //					double trans_sent_pred = 0.0;
 //					BOOST_FOREACH(int inst, middle_layer_instances) {
 //						pred = sentenceView->prediction(inst);
 //						trans_sent_pred += pred/20.0;
 //						loss += abs((pred - 0.55)) /** (pred - 0.55)*/;
 //					}
 //					double trans_slot_pred = 0.0;
 //					BOOST_FOREACH(int inst, bottom_layer_instances) {
 //						pred = sentenceView->prediction(inst);
 //						trans_sent_pred += pred/20.0;
 //						loss += (pred - 0.55) * (pred - 0.55);
 //						pred = slotView->prediction(inst);
 //						trans_slot_pred= slotView->prediction(inst);
 //						loss += abs((pred - 0.3)) /** (pred - 0.3)*/;
 //					}
 //					// prior instances; slot and sent probs <= 12, >= .08
 //					double prior_sent_pred = 0.0;
 //					double prior_slot_pred = 0.0;
 //					BOOST_FOREACH(int inst, prior_tuning_instances) {
 //						pred = sentenceView->prediction(inst);
 //						prior_sent_pred += pred/90.0;
 //						loss += (pred-.1)*(pred-.1);
 //						pred = slotView->prediction(inst);
 //						prior_slot_pred += pred/90.0;
 //						loss += abs((pred - .1)) /** (pred - .1)*/ ;
 //					}
 //					stringstream str;
 //					str << fixed;
 //					str.precision(2);
 //					str << "loss=" << loss << "; reg=" << reg_weight << "; fan="
 //						<< fan_weight << "; disa=" << diss_ann_weight << "; disn=" 
 //						<< diss_not_weight << endl << "\tp_st(fl): "
 //						<< fl_sent_pred << "; p_sl(fl): " << fl_slot_pred <<
 //						"; trans t: " << trans_sent_pred << "; trans l: "
 //						<< trans_slot_pred << "; pr: "
 //						<< prior_sent_pred << "; pr_sl: " << prior_slot_pred;
 //					results.push_back(make_pair(loss, str.str()));
 //				}
 //			}
 //		}
 //	}
 //	sort(results.begin(), results.end());
 //	BOOST_FOREACH(const Result& result, results) {
 //		cout << result.second << endl;
 //	}
 //}
