public MinimizationOutput FindMinimum(IObjectiveFunction objective, Vector <double> initial_guess)
{
if (!objective.GradientSupported)
{
throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for Newton minimization.");
}
if (!objective.HessianSupported)
{
throw new IncompatibleObjectiveException("Hessian not supported in objective function, but required for Newton minimization.");
}
if (!(objective is ObjectiveChecker))
{
objective = new ObjectiveChecker(objective, this.ValidateObjective, this.ValidateGradient, this.ValidateHessian);
}
IEvaluation initial_eval = objective.Evaluate(initial_guess);
// Check that we're not already done
if (this.ExitCriteriaSatisfied(initial_guess, initial_eval.Gradient))
{
return(new MinimizationOutput(initial_eval, 0, ExitCondition.AbsoluteGradient));
}
// Set up line search algorithm
var line_searcher = new WeakWolfeLineSearch(1e-4, 0.9, 1e-4, max_iterations: 1000);
// Declare state variables
IEvaluation candidate_point = initial_eval;
Vector <double> search_direction;
LineSearchOutput result;
// Subsequent steps
int iterations = 0;
int total_line_search_steps = 0;
int iterations_with_nontrivial_line_search = 0;
int steepest_descent_resets = 0;
bool tmp_line_search = false;
while (!this.ExitCriteriaSatisfied(candidate_point.Point, candidate_point.Gradient) && iterations < this.MaximumIterations)
{
search_direction = candidate_point.Hessian.LU().Solve(-candidate_point.Gradient);
if (search_direction * candidate_point.Gradient >= 0)
{
search_direction = -candidate_point.Gradient;
steepest_descent_resets += 1;
tmp_line_search = true;
}
if (this.UseLineSearch || tmp_line_search)
{
try
{
result = line_searcher.FindConformingStep(objective, candidate_point, search_direction, 1.0);
}
catch (Exception e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
iterations_with_nontrivial_line_search += result.Iterations > 0 ? 1 : 0;
total_line_search_steps += result.Iterations;
candidate_point = result.FunctionInfoAtMinimum;
}
else
{
candidate_point = objective.Evaluate(candidate_point.Point + search_direction);
}
tmp_line_search = false;
iterations += 1;
}
if (iterations == this.MaximumIterations)
{
throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", this.MaximumIterations));
}
return(new MinimizationWithLineSearchOutput(candidate_point, iterations, ExitCondition.AbsoluteGradient, total_line_search_steps, iterations_with_nontrivial_line_search));
}
// Implemented following http://www.math.washington.edu/~burke/crs/408/lectures/L9-weak-Wolfe.pdf
public LineSearchOutput FindConformingStep(IObjectiveFunction objective, IEvaluation starting_point, Vector <double> search_direction, double initial_step, double upper_bound = Double.PositiveInfinity)
{
if (!(objective is ObjectiveChecker))
{
objective = new ObjectiveChecker(objective, this.ValidateValue, this.ValidateGradient, null);
}
double lower_bound = 0.0;
double step = initial_step;
double initial_value = starting_point.Value;
Vector <double> initial_gradient = starting_point.Gradient;
double initial_dd = search_direction * initial_gradient;
int ii;
IEvaluation candidate_eval = null;
ExitCondition reason_for_exit = ExitCondition.None;
for (ii = 0; ii < this.MaximumIterations; ++ii)
{
candidate_eval = objective.Evaluate(starting_point.Point + search_direction * step);
double step_dd = search_direction * candidate_eval.Gradient;
if (candidate_eval.Value > initial_value + this.C1 * step * initial_dd)
{
upper_bound = step;
step = 0.5 * (lower_bound + upper_bound);
}
else if (Math.Abs(step_dd) > this.C2 * Math.Abs(initial_dd))
{
lower_bound = step;
step = Double.IsPositiveInfinity(upper_bound) ? 2 * lower_bound : 0.5 * (lower_bound + upper_bound);
}
else
{
reason_for_exit = ExitCondition.StrongWolfeCriteria;
break;
}
if (!Double.IsInfinity(upper_bound))
{
double max_rel_change = 0.0;
for (int jj = 0; jj < candidate_eval.Point.Count; ++jj)
{
double tmp = Math.Abs(search_direction[jj] * (upper_bound - lower_bound)) / Math.Max(Math.Abs(candidate_eval.Point[jj]), 1.0);
max_rel_change = Math.Max(max_rel_change, tmp);
}
if (max_rel_change < this.ParameterTolerance)
{
reason_for_exit = ExitCondition.LackOfProgress;
break;
}
}
}
if (ii == this.MaximumIterations && Double.IsPositiveInfinity(upper_bound))
{
throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached. Function appears to be unbounded in search direction.", this.MaximumIterations));
}
else if (ii == this.MaximumIterations)
{
throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", this.MaximumIterations));
}
else
{
return(new LineSearchOutput(candidate_eval, ii, step, reason_for_exit));
}
}
public MinimizationOutput FindMinimum(IObjectiveFunction objective, Vector <double> initial_guess)
{
if (!objective.GradientSupported)
{
throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for ConjugateGradient minimization.");
}
if (!(objective is ObjectiveChecker))
{
objective = new ObjectiveChecker(objective, this.ValidateObjective, this.ValidateGradient, null);
}
IEvaluation initial_eval = objective.Evaluate(initial_guess);
var gradient = initial_eval.Gradient;
// Check that we're not already done
if (this.ExitCriteriaSatisfied(initial_guess, gradient))
{
return(new MinimizationOutput(initial_eval, 0, ExitCondition.AbsoluteGradient));
}
// Set up line search algorithm
var line_searcher = new WeakWolfeLineSearch(1e-4, 0.1, 1e-4, max_iterations: 1000);
// Declare state variables
IEvaluation candidate_point;
Vector <double> steepest_direction, previous_steepest_direction, search_direction;
// First step
steepest_direction = -gradient;
search_direction = steepest_direction;
double initial_step_size = 100 * this.GradientTolerance / (gradient * gradient);
LineSearchOutput result;
try
{
result = line_searcher.FindConformingStep(objective, initial_eval, search_direction, initial_step_size);
}
catch (Exception e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
candidate_point = result.FunctionInfoAtMinimum;
double step_size = result.FinalStep;
// Subsequent steps
int iterations = 1;
int total_line_search_steps = result.Iterations;
int iterations_with_nontrivial_line_search = result.Iterations > 0 ? 0 : 1;
int steepest_descent_resets = 0;
while (!this.ExitCriteriaSatisfied(candidate_point.Point, candidate_point.Gradient) && iterations < this.MaximumIterations)
{
previous_steepest_direction = steepest_direction;
steepest_direction = -candidate_point.Gradient;
var search_direction_adjuster = Math.Max(0, steepest_direction * (steepest_direction - previous_steepest_direction) / (previous_steepest_direction * previous_steepest_direction));
//double prev_grad_mag = previous_steepest_direction*previous_steepest_direction;
//double grad_overlap = steepest_direction*previous_steepest_direction;
//double search_grad_overlap = candidate_point.Gradient*search_direction;
//if (iterations % initial_guess.Count == 0 || (Math.Abs(grad_overlap) >= 0.2 * prev_grad_mag) || (-2 * prev_grad_mag >= search_grad_overlap) || (search_grad_overlap >= -0.2 * prev_grad_mag))
// search_direction = steepest_direction;
//else
// search_direction = steepest_direction + search_direction_adjuster * search_direction;
search_direction = steepest_direction + search_direction_adjuster * search_direction;
if (search_direction * candidate_point.Gradient >= 0)
{
search_direction = steepest_direction;
steepest_descent_resets += 1;
}
try
{
result = line_searcher.FindConformingStep(objective, candidate_point, search_direction, step_size);
}
catch (Exception e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
iterations_with_nontrivial_line_search += result.Iterations > 0 ? 1 : 0;
total_line_search_steps += result.Iterations;
step_size = result.FinalStep;
candidate_point = result.FunctionInfoAtMinimum;
iterations += 1;
}
if (iterations == this.MaximumIterations)
{
throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", this.MaximumIterations));
}
return(new MinimizationWithLineSearchOutput(candidate_point, iterations, ExitCondition.AbsoluteGradient, total_line_search_steps, iterations_with_nontrivial_line_search));
}
public CheckedEvaluation(ObjectiveChecker checker, IEvaluation evaluation)
{
this.Checker = checker;
this.InnerEvaluation = evaluation;
}
public MinimizationOutput FindMinimum(IObjectiveFunction objective, Vector <double> lower_bound, Vector <double> upper_bound, Vector <double> initial_guess)
{
if (!objective.GradientSupported)
{
throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for BFGS minimization.");
}
if (!(objective is ObjectiveChecker))
{
objective = new ObjectiveChecker(objective, this.ValidateObjective, this.ValidateGradient, null);
}
// Check that dimensions match
if (lower_bound.Count != upper_bound.Count || lower_bound.Count != initial_guess.Count)
{
throw new ArgumentException("Dimensions of bounds and/or initial guess do not match.");
}
// Check that initial guess is feasible
for (int ii = 0; ii < initial_guess.Count; ++ii)
{
if (initial_guess[ii] < lower_bound[ii] || initial_guess[ii] > upper_bound[ii])
{
throw new ArgumentException("Initial guess is not in the feasible region");
}
}
IEvaluation initial_eval = objective.Evaluate(initial_guess);
// Check that we're not already done
ExitCondition current_exit_condition = this.ExitCriteriaSatisfied(initial_eval, null, lower_bound, upper_bound, 0);
if (current_exit_condition != ExitCondition.None)
{
return(new MinimizationOutput(initial_eval, 0, current_exit_condition));
}
// Set up line search algorithm
var line_searcher = new StrongWolfeLineSearch(1e-4, 0.9, Math.Max(this.ParameterTolerance, 1e-5), max_iterations: 1000);
// Declare state variables
IEvaluation candidate_point, previous_point;
double step_size;
Vector <double> gradient, step, line_search_direction, reduced_solution1, reduced_gradient, reduced_initial_point, reduced_cauchy_point, solution1;
Matrix <double> pseudo_hessian, reduced_hessian;
List <int> reduced_map;
// First step
pseudo_hessian = DiagonalMatrix.CreateIdentity(initial_guess.Count);
// Determine active set
var gradient_projection_result = QuadraticGradientProjectionSearch.search(initial_eval.Point, initial_eval.Gradient, pseudo_hessian, lower_bound, upper_bound);
var cauchy_point = gradient_projection_result.Item1;
var fixed_count = gradient_projection_result.Item2;
var is_fixed = gradient_projection_result.Item3;
var free_count = lower_bound.Count - fixed_count;
if (free_count > 0)
{
reduced_gradient = new DenseVector(free_count);
reduced_hessian = new DenseMatrix(free_count, free_count);
reduced_map = new List <int>(free_count);
reduced_initial_point = new DenseVector(free_count);
reduced_cauchy_point = new DenseVector(free_count);
CreateReducedData(initial_eval.Point, cauchy_point, is_fixed, lower_bound, upper_bound, initial_eval.Gradient, pseudo_hessian, reduced_initial_point, reduced_cauchy_point, reduced_gradient, reduced_hessian, reduced_map);
// Determine search direction and maximum step size
reduced_solution1 = reduced_initial_point + reduced_hessian.Cholesky().Solve(-reduced_gradient);
solution1 = reduced_to_full(reduced_map, reduced_solution1, cauchy_point);
}
else
{
solution1 = cauchy_point;
}
var direction_from_cauchy = solution1 - cauchy_point;
var max_step_from_cauchy_point = FindMaxStep(cauchy_point, direction_from_cauchy, lower_bound, upper_bound);
var solution2 = cauchy_point + Math.Min(max_step_from_cauchy_point, 1.0) * direction_from_cauchy;
line_search_direction = solution2 - initial_eval.Point;
var max_line_search_step = FindMaxStep(initial_eval.Point, line_search_direction, lower_bound, upper_bound);
var est_step_size = -initial_eval.Gradient * line_search_direction / (line_search_direction * pseudo_hessian * line_search_direction);
var starting_step_size = Math.Min(Math.Max(est_step_size, 1.0), max_line_search_step);
// Line search
LineSearchOutput result;
try
{
result = line_searcher.FindConformingStep(objective, initial_eval, line_search_direction, starting_step_size, upper_bound: max_line_search_step);
}
catch (Exception e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
previous_point = initial_eval;
candidate_point = result.FunctionInfoAtMinimum;
gradient = candidate_point.Gradient;
step = candidate_point.Point - initial_guess;
step_size = result.FinalStep;
// Subsequent steps
int iterations;
int total_line_search_steps = result.Iterations;
int iterations_with_nontrivial_line_search = result.Iterations > 0 ? 0 : 1;
int steepest_descent_resets = 0;
for (iterations = 1; iterations < this.MaximumIterations; ++iterations)
{
// Do BFGS update
var y = candidate_point.Gradient - previous_point.Gradient;
double sy = step * y;
if (sy > 0.0) // only do update if it will create a positive definite matrix
{
double sts = step * step;
//inverse_pseudo_hessian = inverse_pseudo_hessian + ((sy + y * inverse_pseudo_hessian * y) / Math.Pow(sy, 2.0)) * step.OuterProduct(step) - ((inverse_pseudo_hessian * y.ToColumnMatrix()) * step.ToRowMatrix() + step.ToColumnMatrix() * (y.ToRowMatrix() * inverse_pseudo_hessian)) * (1.0 / sy);
var Hs = pseudo_hessian * step;
var sHs = step * pseudo_hessian * step;
pseudo_hessian = pseudo_hessian + y.OuterProduct(y) * (1.0 / sy) - Hs.OuterProduct(Hs) * (1.0 / sHs);
}
else
{
steepest_descent_resets += 1;
//pseudo_hessian = LinearAlgebra.Double.DiagonalMatrix.Identity(initial_guess.Count);
}
// Determine active set
gradient_projection_result = QuadraticGradientProjectionSearch.search(candidate_point.Point, candidate_point.Gradient, pseudo_hessian, lower_bound, upper_bound);
cauchy_point = gradient_projection_result.Item1;
fixed_count = gradient_projection_result.Item2;
is_fixed = gradient_projection_result.Item3;
free_count = lower_bound.Count - fixed_count;
if (free_count > 0)
{
reduced_gradient = new DenseVector(free_count);
reduced_hessian = new DenseMatrix(free_count, free_count);
reduced_map = new List <int>(free_count);
reduced_initial_point = new DenseVector(free_count);
reduced_cauchy_point = new DenseVector(free_count);
CreateReducedData(candidate_point.Point, cauchy_point, is_fixed, lower_bound, upper_bound, candidate_point.Gradient, pseudo_hessian, reduced_initial_point, reduced_cauchy_point, reduced_gradient, reduced_hessian, reduced_map);
// Determine search direction and maximum step size
reduced_solution1 = reduced_initial_point + reduced_hessian.Cholesky().Solve(-reduced_gradient);
solution1 = reduced_to_full(reduced_map, reduced_solution1, cauchy_point);
}
else
{
solution1 = cauchy_point;
}
direction_from_cauchy = solution1 - cauchy_point;
max_step_from_cauchy_point = FindMaxStep(cauchy_point, direction_from_cauchy, lower_bound, upper_bound);
//var cauchy_eval = objective.Evaluate(cauchy_point);
solution2 = cauchy_point + Math.Min(max_step_from_cauchy_point, 1.0) * direction_from_cauchy;
line_search_direction = solution2 - candidate_point.Point;
max_line_search_step = FindMaxStep(candidate_point.Point, line_search_direction, lower_bound, upper_bound);
//line_search_direction = solution1 - candidate_point.Point;
//max_line_search_step = FindMaxStep(candidate_point.Point, line_search_direction, lower_bound, upper_bound);
if (max_line_search_step == 0.0)
{
line_search_direction = cauchy_point - candidate_point.Point;
max_line_search_step = FindMaxStep(candidate_point.Point, line_search_direction, lower_bound, upper_bound);
}
est_step_size = -candidate_point.Gradient * line_search_direction / (line_search_direction * pseudo_hessian * line_search_direction);
starting_step_size = Math.Min(Math.Max(est_step_size, 1.0), max_line_search_step);
// Line search
try
{
result = line_searcher.FindConformingStep(objective, candidate_point, line_search_direction, starting_step_size, upper_bound: max_line_search_step);
//result = line_searcher.FindConformingStep(objective, cauchy_eval, direction_from_cauchy, Math.Min(1.0, max_step_from_cauchy_point), upper_bound: max_step_from_cauchy_point);
}
catch (Exception e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
iterations_with_nontrivial_line_search += result.Iterations > 0 ? 1 : 0;
total_line_search_steps += result.Iterations;
step_size = result.FinalStep;
step = result.FunctionInfoAtMinimum.Point - candidate_point.Point;
previous_point = candidate_point;
candidate_point = result.FunctionInfoAtMinimum;
current_exit_condition = this.ExitCriteriaSatisfied(candidate_point, previous_point, lower_bound, upper_bound, iterations);
if (current_exit_condition != ExitCondition.None)
{
break;
}
}
if (iterations == this.MaximumIterations && current_exit_condition == ExitCondition.None)
{
throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", this.MaximumIterations));
}
return(new MinimizationWithLineSearchOutput(candidate_point, iterations, current_exit_condition, total_line_search_steps, iterations_with_nontrivial_line_search));
}
public MinimizationOutput FindMinimum(IObjectiveFunction objective, Vector <double> initial_guess)
{
if (!objective.GradientSupported)
{
throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for BFGS minimization.");
}
if (!(objective is ObjectiveChecker))
{
objective = new ObjectiveChecker(objective, this.ValidateObjective, this.ValidateGradient, null);
}
IEvaluation initial_eval = objective.Evaluate(initial_guess);
// Check that we're not already done
ExitCondition current_exit_condition = this.ExitCriteriaSatisfied(initial_eval, null);
if (current_exit_condition != ExitCondition.None)
{
return(new MinimizationOutput(initial_eval, 0, current_exit_condition));
}
// Set up line search algorithm
var line_searcher = new WeakWolfeLineSearch(1e-4, 0.9, Math.Max(this.ParameterTolerance, 1e-10), max_iterations: 1000);
// Declare state variables
IEvaluation candidate_point, previous_point;
double step_size;
Vector <double> gradient, step, search_direction;
Matrix <double> inverse_pseudo_hessian;
// First step
inverse_pseudo_hessian = LinearAlgebra.Double.DiagonalMatrix.CreateIdentity(initial_guess.Count);
search_direction = -initial_eval.Gradient;
step_size = 100 * this.GradientTolerance / (search_direction * search_direction);
LineSearchOutput result;
try
{
result = line_searcher.FindConformingStep(objective, initial_eval, search_direction, step_size);
}
catch (Exception e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
previous_point = initial_eval;
candidate_point = result.FunctionInfoAtMinimum;
gradient = candidate_point.Gradient;
step = candidate_point.Point - initial_guess;
step_size = result.FinalStep;
// Subsequent steps
Matrix <double> I = LinearAlgebra.Double.DiagonalMatrix.CreateIdentity(initial_guess.Count);
int iterations;
int total_line_search_steps = result.Iterations;
int iterations_with_nontrivial_line_search = result.Iterations > 0 ? 0 : 1;
int steepest_descent_resets = 0;
for (iterations = 1; iterations < this.MaximumIterations; ++iterations)
{
var y = candidate_point.Gradient - previous_point.Gradient;
double sy = step * y;
inverse_pseudo_hessian = inverse_pseudo_hessian + ((sy + y * inverse_pseudo_hessian * y) / Math.Pow(sy, 2.0)) * step.OuterProduct(step) - ((inverse_pseudo_hessian * y.ToColumnMatrix()) * step.ToRowMatrix() + step.ToColumnMatrix() * (y.ToRowMatrix() * inverse_pseudo_hessian)) * (1.0 / sy);
search_direction = -inverse_pseudo_hessian * candidate_point.Gradient;
if (search_direction * candidate_point.Gradient >= 0.0)
{
search_direction = -candidate_point.Gradient;
inverse_pseudo_hessian = LinearAlgebra.Double.DiagonalMatrix.CreateIdentity(initial_guess.Count);
steepest_descent_resets += 1;
}
//else if (search_direction * candidate_point.Gradient >= -this.GradientTolerance*this.GradientTolerance)
//{
// search_direction = -candidate_point.Gradient;
// inverse_pseudo_hessian = LinearAlgebra.Double.DiagonalMatrix.Identity(initial_guess.Count);
// steepest_descent_resets += 1;
//}
try
{
result = line_searcher.FindConformingStep(objective, candidate_point, search_direction, 1.0);
}
catch (Exception e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
iterations_with_nontrivial_line_search += result.Iterations > 0 ? 1 : 0;
total_line_search_steps += result.Iterations;
step_size = result.FinalStep;
step = result.FunctionInfoAtMinimum.Point - candidate_point.Point;
previous_point = candidate_point;
candidate_point = result.FunctionInfoAtMinimum;
current_exit_condition = this.ExitCriteriaSatisfied(candidate_point, previous_point);
if (current_exit_condition != ExitCondition.None)
{
break;
}
}
if (iterations == this.MaximumIterations && current_exit_condition == ExitCondition.None)
{
throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", this.MaximumIterations));
}
return(new MinimizationWithLineSearchOutput(candidate_point, iterations, current_exit_condition, total_line_search_steps, iterations_with_nontrivial_line_search));
}
