public MinimizationOutput FindMinimum(IObjectiveFunction objective, Vector <double> initial_guess) { if (!objective.GradientSupported) { throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for Newton minimization."); } if (!objective.HessianSupported) { throw new IncompatibleObjectiveException("Hessian not supported in objective function, but required for Newton minimization."); } if (!(objective is ObjectiveChecker)) { objective = new ObjectiveChecker(objective, this.ValidateObjective, this.ValidateGradient, this.ValidateHessian); } IEvaluation initial_eval = objective.Evaluate(initial_guess); // Check that we're not already done if (this.ExitCriteriaSatisfied(initial_guess, initial_eval.Gradient)) { return(new MinimizationOutput(initial_eval, 0, ExitCondition.AbsoluteGradient)); } // Set up line search algorithm var line_searcher = new WeakWolfeLineSearch(1e-4, 0.9, 1e-4, max_iterations: 1000); // Declare state variables IEvaluation candidate_point = initial_eval; Vector <double> search_direction; LineSearchOutput result; // Subsequent steps int iterations = 0; int total_line_search_steps = 0; int iterations_with_nontrivial_line_search = 0; int steepest_descent_resets = 0; bool tmp_line_search = false; while (!this.ExitCriteriaSatisfied(candidate_point.Point, candidate_point.Gradient) && iterations < this.MaximumIterations) { search_direction = candidate_point.Hessian.LU().Solve(-candidate_point.Gradient); if (search_direction * candidate_point.Gradient >= 0) { search_direction = -candidate_point.Gradient; steepest_descent_resets += 1; tmp_line_search = true; } if (this.UseLineSearch || tmp_line_search) { try { result = line_searcher.FindConformingStep(objective, candidate_point, search_direction, 1.0); } catch (Exception e) { throw new InnerOptimizationException("Line search failed.", e); } iterations_with_nontrivial_line_search += result.Iterations > 0 ? 1 : 0; total_line_search_steps += result.Iterations; candidate_point = result.FunctionInfoAtMinimum; } else { candidate_point = objective.Evaluate(candidate_point.Point + search_direction); } tmp_line_search = false; iterations += 1; } if (iterations == this.MaximumIterations) { throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", this.MaximumIterations)); } return(new MinimizationWithLineSearchOutput(candidate_point, iterations, ExitCondition.AbsoluteGradient, total_line_search_steps, iterations_with_nontrivial_line_search)); }
// Implemented following http://www.math.washington.edu/~burke/crs/408/lectures/L9-weak-Wolfe.pdf public LineSearchOutput FindConformingStep(IObjectiveFunction objective, IEvaluation starting_point, Vector <double> search_direction, double initial_step, double upper_bound = Double.PositiveInfinity) { if (!(objective is ObjectiveChecker)) { objective = new ObjectiveChecker(objective, this.ValidateValue, this.ValidateGradient, null); } double lower_bound = 0.0; double step = initial_step; double initial_value = starting_point.Value; Vector <double> initial_gradient = starting_point.Gradient; double initial_dd = search_direction * initial_gradient; int ii; IEvaluation candidate_eval = null; ExitCondition reason_for_exit = ExitCondition.None; for (ii = 0; ii < this.MaximumIterations; ++ii) { candidate_eval = objective.Evaluate(starting_point.Point + search_direction * step); double step_dd = search_direction * candidate_eval.Gradient; if (candidate_eval.Value > initial_value + this.C1 * step * initial_dd) { upper_bound = step; step = 0.5 * (lower_bound + upper_bound); } else if (Math.Abs(step_dd) > this.C2 * Math.Abs(initial_dd)) { lower_bound = step; step = Double.IsPositiveInfinity(upper_bound) ? 2 * lower_bound : 0.5 * (lower_bound + upper_bound); } else { reason_for_exit = ExitCondition.StrongWolfeCriteria; break; } if (!Double.IsInfinity(upper_bound)) { double max_rel_change = 0.0; for (int jj = 0; jj < candidate_eval.Point.Count; ++jj) { double tmp = Math.Abs(search_direction[jj] * (upper_bound - lower_bound)) / Math.Max(Math.Abs(candidate_eval.Point[jj]), 1.0); max_rel_change = Math.Max(max_rel_change, tmp); } if (max_rel_change < this.ParameterTolerance) { reason_for_exit = ExitCondition.LackOfProgress; break; } } } if (ii == this.MaximumIterations && Double.IsPositiveInfinity(upper_bound)) { throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached. Function appears to be unbounded in search direction.", this.MaximumIterations)); } else if (ii == this.MaximumIterations) { throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", this.MaximumIterations)); } else { return(new LineSearchOutput(candidate_eval, ii, step, reason_for_exit)); } }
public MinimizationOutput FindMinimum(IObjectiveFunction objective, Vector <double> initial_guess) { if (!objective.GradientSupported) { throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for ConjugateGradient minimization."); } if (!(objective is ObjectiveChecker)) { objective = new ObjectiveChecker(objective, this.ValidateObjective, this.ValidateGradient, null); } IEvaluation initial_eval = objective.Evaluate(initial_guess); var gradient = initial_eval.Gradient; // Check that we're not already done if (this.ExitCriteriaSatisfied(initial_guess, gradient)) { return(new MinimizationOutput(initial_eval, 0, ExitCondition.AbsoluteGradient)); } // Set up line search algorithm var line_searcher = new WeakWolfeLineSearch(1e-4, 0.1, 1e-4, max_iterations: 1000); // Declare state variables IEvaluation candidate_point; Vector <double> steepest_direction, previous_steepest_direction, search_direction; // First step steepest_direction = -gradient; search_direction = steepest_direction; double initial_step_size = 100 * this.GradientTolerance / (gradient * gradient); LineSearchOutput result; try { result = line_searcher.FindConformingStep(objective, initial_eval, search_direction, initial_step_size); } catch (Exception e) { throw new InnerOptimizationException("Line search failed.", e); } candidate_point = result.FunctionInfoAtMinimum; double step_size = result.FinalStep; // Subsequent steps int iterations = 1; int total_line_search_steps = result.Iterations; int iterations_with_nontrivial_line_search = result.Iterations > 0 ? 0 : 1; int steepest_descent_resets = 0; while (!this.ExitCriteriaSatisfied(candidate_point.Point, candidate_point.Gradient) && iterations < this.MaximumIterations) { previous_steepest_direction = steepest_direction; steepest_direction = -candidate_point.Gradient; var search_direction_adjuster = Math.Max(0, steepest_direction * (steepest_direction - previous_steepest_direction) / (previous_steepest_direction * previous_steepest_direction)); //double prev_grad_mag = previous_steepest_direction*previous_steepest_direction; //double grad_overlap = steepest_direction*previous_steepest_direction; //double search_grad_overlap = candidate_point.Gradient*search_direction; //if (iterations % initial_guess.Count == 0 || (Math.Abs(grad_overlap) >= 0.2 * prev_grad_mag) || (-2 * prev_grad_mag >= search_grad_overlap) || (search_grad_overlap >= -0.2 * prev_grad_mag)) // search_direction = steepest_direction; //else // search_direction = steepest_direction + search_direction_adjuster * search_direction; search_direction = steepest_direction + search_direction_adjuster * search_direction; if (search_direction * candidate_point.Gradient >= 0) { search_direction = steepest_direction; steepest_descent_resets += 1; } try { result = line_searcher.FindConformingStep(objective, candidate_point, search_direction, step_size); } catch (Exception e) { throw new InnerOptimizationException("Line search failed.", e); } iterations_with_nontrivial_line_search += result.Iterations > 0 ? 1 : 0; total_line_search_steps += result.Iterations; step_size = result.FinalStep; candidate_point = result.FunctionInfoAtMinimum; iterations += 1; } if (iterations == this.MaximumIterations) { throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", this.MaximumIterations)); } return(new MinimizationWithLineSearchOutput(candidate_point, iterations, ExitCondition.AbsoluteGradient, total_line_search_steps, iterations_with_nontrivial_line_search)); }
public CheckedEvaluation(ObjectiveChecker checker, IEvaluation evaluation) { this.Checker = checker; this.InnerEvaluation = evaluation; }
public MinimizationOutput FindMinimum(IObjectiveFunction objective, Vector <double> lower_bound, Vector <double> upper_bound, Vector <double> initial_guess) { if (!objective.GradientSupported) { throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for BFGS minimization."); } if (!(objective is ObjectiveChecker)) { objective = new ObjectiveChecker(objective, this.ValidateObjective, this.ValidateGradient, null); } // Check that dimensions match if (lower_bound.Count != upper_bound.Count || lower_bound.Count != initial_guess.Count) { throw new ArgumentException("Dimensions of bounds and/or initial guess do not match."); } // Check that initial guess is feasible for (int ii = 0; ii < initial_guess.Count; ++ii) { if (initial_guess[ii] < lower_bound[ii] || initial_guess[ii] > upper_bound[ii]) { throw new ArgumentException("Initial guess is not in the feasible region"); } } IEvaluation initial_eval = objective.Evaluate(initial_guess); // Check that we're not already done ExitCondition current_exit_condition = this.ExitCriteriaSatisfied(initial_eval, null, lower_bound, upper_bound, 0); if (current_exit_condition != ExitCondition.None) { return(new MinimizationOutput(initial_eval, 0, current_exit_condition)); } // Set up line search algorithm var line_searcher = new StrongWolfeLineSearch(1e-4, 0.9, Math.Max(this.ParameterTolerance, 1e-5), max_iterations: 1000); // Declare state variables IEvaluation candidate_point, previous_point; double step_size; Vector <double> gradient, step, line_search_direction, reduced_solution1, reduced_gradient, reduced_initial_point, reduced_cauchy_point, solution1; Matrix <double> pseudo_hessian, reduced_hessian; List <int> reduced_map; // First step pseudo_hessian = DiagonalMatrix.CreateIdentity(initial_guess.Count); // Determine active set var gradient_projection_result = QuadraticGradientProjectionSearch.search(initial_eval.Point, initial_eval.Gradient, pseudo_hessian, lower_bound, upper_bound); var cauchy_point = gradient_projection_result.Item1; var fixed_count = gradient_projection_result.Item2; var is_fixed = gradient_projection_result.Item3; var free_count = lower_bound.Count - fixed_count; if (free_count > 0) { reduced_gradient = new DenseVector(free_count); reduced_hessian = new DenseMatrix(free_count, free_count); reduced_map = new List <int>(free_count); reduced_initial_point = new DenseVector(free_count); reduced_cauchy_point = new DenseVector(free_count); CreateReducedData(initial_eval.Point, cauchy_point, is_fixed, lower_bound, upper_bound, initial_eval.Gradient, pseudo_hessian, reduced_initial_point, reduced_cauchy_point, reduced_gradient, reduced_hessian, reduced_map); // Determine search direction and maximum step size reduced_solution1 = reduced_initial_point + reduced_hessian.Cholesky().Solve(-reduced_gradient); solution1 = reduced_to_full(reduced_map, reduced_solution1, cauchy_point); } else { solution1 = cauchy_point; } var direction_from_cauchy = solution1 - cauchy_point; var max_step_from_cauchy_point = FindMaxStep(cauchy_point, direction_from_cauchy, lower_bound, upper_bound); var solution2 = cauchy_point + Math.Min(max_step_from_cauchy_point, 1.0) * direction_from_cauchy; line_search_direction = solution2 - initial_eval.Point; var max_line_search_step = FindMaxStep(initial_eval.Point, line_search_direction, lower_bound, upper_bound); var est_step_size = -initial_eval.Gradient * line_search_direction / (line_search_direction * pseudo_hessian * line_search_direction); var starting_step_size = Math.Min(Math.Max(est_step_size, 1.0), max_line_search_step); // Line search LineSearchOutput result; try { result = line_searcher.FindConformingStep(objective, initial_eval, line_search_direction, starting_step_size, upper_bound: max_line_search_step); } catch (Exception e) { throw new InnerOptimizationException("Line search failed.", e); } previous_point = initial_eval; candidate_point = result.FunctionInfoAtMinimum; gradient = candidate_point.Gradient; step = candidate_point.Point - initial_guess; step_size = result.FinalStep; // Subsequent steps int iterations; int total_line_search_steps = result.Iterations; int iterations_with_nontrivial_line_search = result.Iterations > 0 ? 0 : 1; int steepest_descent_resets = 0; for (iterations = 1; iterations < this.MaximumIterations; ++iterations) { // Do BFGS update var y = candidate_point.Gradient - previous_point.Gradient; double sy = step * y; if (sy > 0.0) // only do update if it will create a positive definite matrix { double sts = step * step; //inverse_pseudo_hessian = inverse_pseudo_hessian + ((sy + y * inverse_pseudo_hessian * y) / Math.Pow(sy, 2.0)) * step.OuterProduct(step) - ((inverse_pseudo_hessian * y.ToColumnMatrix()) * step.ToRowMatrix() + step.ToColumnMatrix() * (y.ToRowMatrix() * inverse_pseudo_hessian)) * (1.0 / sy); var Hs = pseudo_hessian * step; var sHs = step * pseudo_hessian * step; pseudo_hessian = pseudo_hessian + y.OuterProduct(y) * (1.0 / sy) - Hs.OuterProduct(Hs) * (1.0 / sHs); } else { steepest_descent_resets += 1; //pseudo_hessian = LinearAlgebra.Double.DiagonalMatrix.Identity(initial_guess.Count); } // Determine active set gradient_projection_result = QuadraticGradientProjectionSearch.search(candidate_point.Point, candidate_point.Gradient, pseudo_hessian, lower_bound, upper_bound); cauchy_point = gradient_projection_result.Item1; fixed_count = gradient_projection_result.Item2; is_fixed = gradient_projection_result.Item3; free_count = lower_bound.Count - fixed_count; if (free_count > 0) { reduced_gradient = new DenseVector(free_count); reduced_hessian = new DenseMatrix(free_count, free_count); reduced_map = new List <int>(free_count); reduced_initial_point = new DenseVector(free_count); reduced_cauchy_point = new DenseVector(free_count); CreateReducedData(candidate_point.Point, cauchy_point, is_fixed, lower_bound, upper_bound, candidate_point.Gradient, pseudo_hessian, reduced_initial_point, reduced_cauchy_point, reduced_gradient, reduced_hessian, reduced_map); // Determine search direction and maximum step size reduced_solution1 = reduced_initial_point + reduced_hessian.Cholesky().Solve(-reduced_gradient); solution1 = reduced_to_full(reduced_map, reduced_solution1, cauchy_point); } else { solution1 = cauchy_point; } direction_from_cauchy = solution1 - cauchy_point; max_step_from_cauchy_point = FindMaxStep(cauchy_point, direction_from_cauchy, lower_bound, upper_bound); //var cauchy_eval = objective.Evaluate(cauchy_point); solution2 = cauchy_point + Math.Min(max_step_from_cauchy_point, 1.0) * direction_from_cauchy; line_search_direction = solution2 - candidate_point.Point; max_line_search_step = FindMaxStep(candidate_point.Point, line_search_direction, lower_bound, upper_bound); //line_search_direction = solution1 - candidate_point.Point; //max_line_search_step = FindMaxStep(candidate_point.Point, line_search_direction, lower_bound, upper_bound); if (max_line_search_step == 0.0) { line_search_direction = cauchy_point - candidate_point.Point; max_line_search_step = FindMaxStep(candidate_point.Point, line_search_direction, lower_bound, upper_bound); } est_step_size = -candidate_point.Gradient * line_search_direction / (line_search_direction * pseudo_hessian * line_search_direction); starting_step_size = Math.Min(Math.Max(est_step_size, 1.0), max_line_search_step); // Line search try { result = line_searcher.FindConformingStep(objective, candidate_point, line_search_direction, starting_step_size, upper_bound: max_line_search_step); //result = line_searcher.FindConformingStep(objective, cauchy_eval, direction_from_cauchy, Math.Min(1.0, max_step_from_cauchy_point), upper_bound: max_step_from_cauchy_point); } catch (Exception e) { throw new InnerOptimizationException("Line search failed.", e); } iterations_with_nontrivial_line_search += result.Iterations > 0 ? 1 : 0; total_line_search_steps += result.Iterations; step_size = result.FinalStep; step = result.FunctionInfoAtMinimum.Point - candidate_point.Point; previous_point = candidate_point; candidate_point = result.FunctionInfoAtMinimum; current_exit_condition = this.ExitCriteriaSatisfied(candidate_point, previous_point, lower_bound, upper_bound, iterations); if (current_exit_condition != ExitCondition.None) { break; } } if (iterations == this.MaximumIterations && current_exit_condition == ExitCondition.None) { throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", this.MaximumIterations)); } return(new MinimizationWithLineSearchOutput(candidate_point, iterations, current_exit_condition, total_line_search_steps, iterations_with_nontrivial_line_search)); }
public MinimizationOutput FindMinimum(IObjectiveFunction objective, Vector <double> initial_guess) { if (!objective.GradientSupported) { throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for BFGS minimization."); } if (!(objective is ObjectiveChecker)) { objective = new ObjectiveChecker(objective, this.ValidateObjective, this.ValidateGradient, null); } IEvaluation initial_eval = objective.Evaluate(initial_guess); // Check that we're not already done ExitCondition current_exit_condition = this.ExitCriteriaSatisfied(initial_eval, null); if (current_exit_condition != ExitCondition.None) { return(new MinimizationOutput(initial_eval, 0, current_exit_condition)); } // Set up line search algorithm var line_searcher = new WeakWolfeLineSearch(1e-4, 0.9, Math.Max(this.ParameterTolerance, 1e-10), max_iterations: 1000); // Declare state variables IEvaluation candidate_point, previous_point; double step_size; Vector <double> gradient, step, search_direction; Matrix <double> inverse_pseudo_hessian; // First step inverse_pseudo_hessian = LinearAlgebra.Double.DiagonalMatrix.CreateIdentity(initial_guess.Count); search_direction = -initial_eval.Gradient; step_size = 100 * this.GradientTolerance / (search_direction * search_direction); LineSearchOutput result; try { result = line_searcher.FindConformingStep(objective, initial_eval, search_direction, step_size); } catch (Exception e) { throw new InnerOptimizationException("Line search failed.", e); } previous_point = initial_eval; candidate_point = result.FunctionInfoAtMinimum; gradient = candidate_point.Gradient; step = candidate_point.Point - initial_guess; step_size = result.FinalStep; // Subsequent steps Matrix <double> I = LinearAlgebra.Double.DiagonalMatrix.CreateIdentity(initial_guess.Count); int iterations; int total_line_search_steps = result.Iterations; int iterations_with_nontrivial_line_search = result.Iterations > 0 ? 0 : 1; int steepest_descent_resets = 0; for (iterations = 1; iterations < this.MaximumIterations; ++iterations) { var y = candidate_point.Gradient - previous_point.Gradient; double sy = step * y; inverse_pseudo_hessian = inverse_pseudo_hessian + ((sy + y * inverse_pseudo_hessian * y) / Math.Pow(sy, 2.0)) * step.OuterProduct(step) - ((inverse_pseudo_hessian * y.ToColumnMatrix()) * step.ToRowMatrix() + step.ToColumnMatrix() * (y.ToRowMatrix() * inverse_pseudo_hessian)) * (1.0 / sy); search_direction = -inverse_pseudo_hessian * candidate_point.Gradient; if (search_direction * candidate_point.Gradient >= 0.0) { search_direction = -candidate_point.Gradient; inverse_pseudo_hessian = LinearAlgebra.Double.DiagonalMatrix.CreateIdentity(initial_guess.Count); steepest_descent_resets += 1; } //else if (search_direction * candidate_point.Gradient >= -this.GradientTolerance*this.GradientTolerance) //{ // search_direction = -candidate_point.Gradient; // inverse_pseudo_hessian = LinearAlgebra.Double.DiagonalMatrix.Identity(initial_guess.Count); // steepest_descent_resets += 1; //} try { result = line_searcher.FindConformingStep(objective, candidate_point, search_direction, 1.0); } catch (Exception e) { throw new InnerOptimizationException("Line search failed.", e); } iterations_with_nontrivial_line_search += result.Iterations > 0 ? 1 : 0; total_line_search_steps += result.Iterations; step_size = result.FinalStep; step = result.FunctionInfoAtMinimum.Point - candidate_point.Point; previous_point = candidate_point; candidate_point = result.FunctionInfoAtMinimum; current_exit_condition = this.ExitCriteriaSatisfied(candidate_point, previous_point); if (current_exit_condition != ExitCondition.None) { break; } } if (iterations == this.MaximumIterations && current_exit_condition == ExitCondition.None) { throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", this.MaximumIterations)); } return(new MinimizationWithLineSearchOutput(candidate_point, iterations, current_exit_condition, total_line_search_steps, iterations_with_nontrivial_line_search)); }