src/LearnIt/lbfgs/lbfgs.cpp
// do a line search on function func in the direction of the direction vector
// on termination, will return the size of the step taken in that direction,
// and the parameters (but not gradient!) will be updated for the new position 
// (that is, for their values *after* moving the selected distance along
// the direction vector)
// see Numerical Recipes in C section 9.7

do a line search on function func in the direction of the direction vector
on termination, will return the size of the step taken in that direction,
and the parameters (but not gradient!) will be updated for the new position
(that is, for their values *after* moving the selected distance along
the direction vector)
see Numerical Recipes in C section 9.7
do a line search on function func in the direction of the direction vector on termination will return the size of the step taken in that direction and the parameters ( but not gradient! ) will be updated for the new position ( that is for their values * after * moving the selected distance along the direction vector ) see Numerical Recipes in C section 9 . 7
double LBFGS::lineSearch(FunctionWithGradient& func, VectorXd& direction) 
{
	static const double MAX_STEP_LENGTH = 100.0;
	static const double ALPHA = 1e-4;
	size_t n_params = func.parameters().size();
	VectorXd original_parameters = func.parameters();
	VectorXd original_gradient = func.gradient();
	double step_length = direction.norm();
	if (step_length > MAX_STEP_LENGTH) {
		original_gradient *= MAX_STEP_LENGTH / step_length;
	}
	double slope = original_gradient.dot(direction);
	if (slope <= 0.0) {
		throw LineSearchSlopeNegative();
	}
	double lambda_min_denom = 0.0;
	for (size_t i=0; i<n_params; ++i) {
		double temp = std::fabs(func.gradient()(i))/(std::max)(func.parameters()(i), 1.0);
		if (temp > lambda_min_denom) {
			lambda_min_denom = temp;
		}
	}
	double lambda_min = 1e-7 / lambda_min_denom;
	double lambda = 1.0;
	double previous_lambda = 0.0; 
	double previous_f = 0.0;
	double new_lambda = 1.0; 
	double f_old = func.value();
	while (true) {
		func.setParams(func.parameters() + (lambda - previous_lambda) * direction);
		if (lambda < lambda_min) {
			func.setParams(original_parameters);
			return 0.0;
		}
		double f = func.value();
		if (f >= f_old + lambda * slope * ALPHA) {
			return lambda;
		} else {
			if (lambda == 1.0) {
				new_lambda = -slope / (2.0 * (f - f_old - slope));
			} else {
				double rhs1 = f - f_old - lambda * slope;
				double rhs2 = previous_f - f_old - previous_lambda * slope;
				double a = (rhs1/(lambda*lambda) - rhs2/(previous_lambda*previous_lambda))
								/(lambda-previous_lambda);
				double b = (-previous_lambda*rhs1/(lambda*lambda)
							+lambda*rhs2/(previous_lambda*previous_lambda))
							/(lambda-previous_lambda);
				if (a == 0.0) {
					new_lambda = -slope / (2.0 * b);
				} else {
					double discriminant = b*b - 3.0*a*slope;
					if (discriminant < 0) {
						new_lambda = 0.5 * lambda;
					} else if (b <= 0.0) {
						new_lambda = (-b + sqrt(discriminant))/(3.0*a);
					} else {
						new_lambda = -slope / (b + sqrt(discriminant));
					}
					new_lambda = (std::min)(new_lambda, 0.5 * lambda);
				}
			}
			previous_lambda = lambda;
			previous_f = f;
			lambda = (std::max)(new_lambda, 0.1*lambda);
		}
	}
}
double LBFGS :: lineSearch ( FunctionWithGradient & func , VectorXd & direction ) { static const double MAX_STEP_LENGTH = 100.0 ; static const double ALPHA = 1e-4 ; size_t n_params = func . parameters ( ) . size ( ) ; VectorXd original_parameters = func . parameters ( ) ; VectorXd original_gradient = func . gradient ( ) ; double step_length = direction . norm ( ) ; if ( step_length > MAX_STEP_LENGTH ) { original_gradient *= MAX_STEP_LENGTH / step_length ; } double slope = original_gradient . dot ( direction ) ; if ( slope <= 0.0 ) { throw LineSearchSlopeNegative ( ) ; } double lambda_min_denom = 0.0 ; for ( size_t i = 0 ; i < n_params ; ++ i ) { double temp = std :: fabs ( func . gradient ( ) ( i ) ) / ( std :: max ) ( func . parameters ( ) ( i ) , 1.0 ) ; if ( temp > lambda_min_denom ) { lambda_min_denom = temp ; } } double lambda_min = 1e-7 / lambda_min_denom ; double lambda = 1.0 ; double previous_lambda = 0.0 ; double previous_f = 0.0 ; double new_lambda = 1.0 ; double f_old = func . value ( ) ; while ( true ) { func . setParams ( func . parameters ( ) + ( lambda - previous_lambda ) * direction ) ; if ( lambda < lambda_min ) { func . setParams ( original_parameters ) ; return 0.0 ; } double f = func . value ( ) ; if ( f >= f_old + lambda * slope * ALPHA ) { return lambda ; } else { if ( lambda == 1.0 ) { new_lambda = - slope / ( 2.0 * ( f - f_old - slope ) ) ; } else { double rhs1 = f - f_old - lambda * slope ; double rhs2 = previous_f - f_old - previous_lambda * slope ; double a = ( rhs1 / ( lambda * lambda ) - rhs2 / ( previous_lambda * previous_lambda ) ) / ( lambda - previous_lambda ) ; double b = ( - previous_lambda * rhs1 / ( lambda * lambda ) + lambda * rhs2 / ( previous_lambda * previous_lambda ) ) / ( lambda - previous_lambda ) ; if ( a == 0.0 ) { new_lambda = - slope / ( 2.0 * b ) ; } else { double discriminant = b * b - 3.0 * a * slope ; if ( discriminant < 0 ) { new_lambda = 0.5 * lambda ; } else if ( b <= 0.0 ) { new_lambda = ( - b + sqrt ( discriminant ) ) / ( 3.0 * a ) ; } else { new_lambda = - slope / ( b + sqrt ( discriminant ) ) ; } new_lambda = ( std :: min ) ( new_lambda , 0.5 * lambda ) ; } } previous_lambda = lambda ; previous_f = f ; lambda = ( std :: max ) ( new_lambda , 0.1 * lambda ) ; } } }
do a line search on function func in the direction of the direction vector
on termination, will return the size of the step taken in that direction,
and the parameters (but not gradient!)
do a line search on function func in the direction of the direction vector on termination will return the size of the step taken in that direction and the parameters ( but not gradient! )
// ensure we don't step too far
 // This is actually unnecessary when used with L-BFGS because we always pass
 // a direction vector of unit length, but I put it in in case someone uses
 // this function for something else
 //	double step_length = norm_2(direction);
 //	double slope = inner_prod(original_gradient, direction);
 // check we're going up like we ought to be
 // calculate minimum step length
 // lambda is our fraction of the full Newton step
 // our previous estimate of lambda
 // our new estimate of lambda
 // for the first iteration, old_lambda is zero, so we just update
 // our parameters by the full step size.  After than lamba will be 
 // shrinking, so lambda - old_lambda will be negative, so we will
 // be walking along the direction vector back towards our 
 // original parameters
 // Yay, we increased the function enough.
 // 'enough' means the average slope (f-f_old)/lambda over
 // our step was at least ALPHA times the slope 
 // at our initial point
 // for our first backtrack, we use a quadratic approximation
 // (from Numerical Recipes 9.7.11)
 // on the following iterations we use a cubic approximation
 // (Numericla Recipes 9.7.14)
 // decrease by at least half each iteration
 // don't backtrack more than 10x in one iteration
