protected int DoBfgsUpdate(ref ExitCondition currentExitCondition, WolfeLineSearch lineSearcher, ref Matrix <double> inversePseudoHessian, ref Vector <double> lineSearchDirection, ref IObjectiveFunction previousPoint, ref LineSearchResult lineSearchResult, ref IObjectiveFunction candidate, ref Vector <double> step, ref int totalLineSearchSteps, ref int iterationsWithNontrivialLineSearch) { int iterations; for (iterations = 1; iterations < MaximumIterations; ++iterations) { double startingStepSize; double maxLineSearchStep; lineSearchDirection = CalculateSearchDirection(ref inversePseudoHessian, out maxLineSearchStep, out startingStepSize, previousPoint, candidate, step); try { lineSearchResult = lineSearcher.FindConformingStep(candidate, lineSearchDirection, startingStepSize, maxLineSearchStep); } catch (Exception e) { throw new InnerOptimizationException("Line search failed.", e); } iterationsWithNontrivialLineSearch += lineSearchResult.Iterations > 0 ? 1 : 0; totalLineSearchSteps += lineSearchResult.Iterations; step = lineSearchResult.FunctionInfoAtMinimum.Point - candidate.Point; previousPoint = candidate; candidate = lineSearchResult.FunctionInfoAtMinimum; currentExitCondition = ExitCriteriaSatisfied(candidate, previousPoint, iterations); if (currentExitCondition != ExitCondition.None) { break; } } return(iterations); }
public NonlinearMinimizationResult(IObjectiveModel modelInfo, int iterations, ExitCondition reasonForExit) { ModelInfoAtMinimum = modelInfo; Iterations = iterations; ReasonForExit = reasonForExit; EvaluateCovariance(modelInfo); }
private void InterpreterOnExecutionComplete(object sender, ExitCondition exitCondition) { void MethodInvokerDelegate() { statusRunTime.Text = string.Format(Resources.LogExitedAfter, _stopwatch.Elapsed); tsbHalt.Enabled = false; } Invoke((MethodInvoker)MethodInvokerDelegate); _stopwatch.Stop(); }
public void RunSteps() { st.Start(); while (exitCondition == ExitCondition.Running) { Actions.Clear(); DoorActions.Clear(); SaveState(); actualCost = CostCalculationService.CalculateCostNew(Model); actualCost.Index = CurrentIndex; CalculateCostsForState(); bool m = Model.IsInInvalidState; MakeAStepByTheCalculatedCosts(); //CalculateDoorCosts(); //MakeStepByDoorChanges(); HandleModelChangeUpdate(); if (CurrentIndex >= MaxIndex) { exitCondition = ExitCondition.isFinished; } if (st.ElapsedMilliseconds > maxSeconds * 1000) { exitCondition = ExitCondition.isTimeout; } if (ActualTreshold >= MaxTreshold) { exitCondition = ExitCondition.isTreshold; } CurrentIndex++; Thread.Sleep(5); } Logger.WriteLog($"Run Ended.\nExitCondition : {exitCondition}"); exitCondition = ExitCondition.Running; ActualTreshold = 0; MaxIndex += MaxIndex; st.Reset(); }
public LineSearchResult(IObjectiveFunction functionInfo, int iterations, double finalStep, ExitCondition reasonForExit) : base(functionInfo, iterations, reasonForExit) { FinalStep = finalStep; }
/// <summary> /// Non-linear least square fitting by the Levenberg-Marduardt algorithm. /// </summary> /// <param name="objective">The objective function, including model, observations, and parameter bounds.</param> /// <param name="initialGuess">The initial guess values.</param> /// <param name="initialMu">The initial damping parameter of mu.</param> /// <param name="gradientTolerance">The stopping threshold for infinity norm of the gradient vector.</param> /// <param name="stepTolerance">The stopping threshold for L2 norm of the change of parameters.</param> /// <param name="functionTolerance">The stopping threshold for L2 norm of the residuals.</param> /// <param name="maximumIterations">The max iterations.</param> /// <returns>The result of the Levenberg-Marquardt minimization</returns> public static NonlinearMinimizationResult Minimum(IObjectiveModel objective, Vector <double> initialGuess, Vector <double> lowerBound = null, Vector <double> upperBound = null, Vector <double> scales = null, List <bool> isFixed = null, double initialMu = 1E-3, double gradientTolerance = 1E-15, double stepTolerance = 1E-15, double functionTolerance = 1E-15, int maximumIterations = -1) { // Non-linear least square fitting by the Levenberg-Marduardt algorithm. // // Levenberg-Marquardt is finding the minimum of a function F(p) that is a sum of squares of nonlinear functions. // // For given datum pair (x, y), uncertainties σ (or weighting W = 1 / σ^2) and model function f = f(x; p), // let's find the parameters of the model so that the sum of the quares of the deviations is minimized. // // F(p) = 1/2 * ∑{ Wi * (yi - f(xi; p))^2 } // pbest = argmin F(p) // // We will use the following terms: // Weighting W is the diagonal matrix and can be decomposed as LL', so L = 1/σ // Residuals, R = L(y - f(x; p)) // Residual sum of squares, RSS = ||R||^2 = R.DotProduct(R) // Jacobian J = df(x; p)/dp // Gradient g = -J'W(y − f(x; p)) = -J'LR // Approximated Hessian H = J'WJ // // The Levenberg-Marquardt algorithm is summarized as follows: // initially let μ = τ * max(diag(H)). // repeat // solve linear equations: (H + μI)ΔP = -g // let ρ = (||R||^2 - ||Rnew||^2) / (Δp'(μΔp - g)). // if ρ > ε, P = P + ΔP; μ = μ * max(1/3, 1 - (2ρ - 1)^3); ν = 2; // otherwise μ = μ*ν; ν = 2*ν; // // References: // [1]. Madsen, K., H. B. Nielsen, and O. Tingleff. // "Methods for Non-Linear Least Squares Problems. Technical University of Denmark, 2004. Lecture notes." (2004). // Available Online from: http://orbit.dtu.dk/files/2721358/imm3215.pdf // [2]. Gavin, Henri. // "The Levenberg-Marquardt method for nonlinear least squares curve-fitting problems." // Department of Civil and Environmental Engineering, Duke University (2017): 1-19. // Availble Online from: http://people.duke.edu/~hpgavin/ce281/lm.pdf if (objective == null) { throw new ArgumentNullException("objective"); } ValidateBounds(initialGuess, lowerBound, upperBound, scales); objective.SetParameters(initialGuess, isFixed); ExitCondition exitCondition = ExitCondition.None; // First, calculate function values and setup variables var P = ProjectToInternalParameters(initialGuess); // current internal parameters var Pstep = Vector <double> .Build.Dense(P.Count); // the change of parameters var RSS = EvaluateFunction(objective, P); // Residual Sum of Squares = R'R if (maximumIterations < 0) { maximumIterations = 200 * (initialGuess.Count + 1); } // if RSS == NaN, stop if (double.IsNaN(RSS)) { exitCondition = ExitCondition.InvalidValues; return(new NonlinearMinimizationResult(objective, -1, exitCondition)); } // When only function evaluation is needed, set maximumIterations to zero, if (maximumIterations == 0) { exitCondition = ExitCondition.ManuallyStopped; } // if RSS <= fTol, stop if (RSS <= functionTolerance) { exitCondition = ExitCondition.Converged; // SmallRSS } // Evaluate gradient and Hessian var jac = EvaluateJacobian(objective, P); var Gradient = jac.Item1; // objective.Gradient; var Hessian = jac.Item2; // objective.Hessian; var diagonalOfHessian = Hessian.Diagonal(); // diag(H) // if ||g||oo <= gtol, found and stop if (Gradient.InfinityNorm() <= gradientTolerance) { exitCondition = ExitCondition.RelativeGradient; } if (exitCondition != ExitCondition.None) { return(new NonlinearMinimizationResult(objective, -1, exitCondition)); } double mu = initialMu * diagonalOfHessian.Max(); // μ double nu = 2; // ν int iterations = 0; while (iterations < maximumIterations && exitCondition == ExitCondition.None) { iterations++; while (true) { Hessian.SetDiagonal(Hessian.Diagonal() + mu); // hessian[i, i] = hessian[i, i] + mu; // solve normal equations Pstep = Hessian.Solve(-Gradient); // if ||ΔP|| <= xTol * (||P|| + xTol), found and stop if (Pstep.L2Norm() <= stepTolerance * (stepTolerance + P.DotProduct(P))) { exitCondition = ExitCondition.RelativePoints; break; } var Pnew = P + Pstep; // new parameters to test // evaluate function at Pnew var RSSnew = EvaluateFunction(objective, Pnew); if (double.IsNaN(RSSnew)) { exitCondition = ExitCondition.InvalidValues; break; } // calculate the ratio of the actual to the predicted reduction. // ρ = (RSS - RSSnew) / (Δp'(μΔp - g)) var predictedReduction = Pstep.DotProduct(mu * Pstep - Gradient); var rho = (predictedReduction != 0) ? (RSS - RSSnew) / predictedReduction : 0; if (rho > 0.0) { // accepted Pnew.CopyTo(P); RSS = RSSnew; // update gradient and Hessian jac = EvaluateJacobian(objective, P); Gradient = jac.Item1; // objective.Gradient; Hessian = jac.Item2; // objective.Hessian; diagonalOfHessian = Hessian.Diagonal(); // if ||g||_oo <= gtol, found and stop if (Gradient.InfinityNorm() <= gradientTolerance) { exitCondition = ExitCondition.RelativeGradient; } // if ||R||^2 < fTol, found and stop if (RSS <= functionTolerance) { exitCondition = ExitCondition.Converged; // SmallRSS } mu = mu * Math.Max(1.0 / 3.0, 1.0 - Math.Pow(2.0 * rho - 1.0, 3)); nu = 2; break; } else { // rejected, increased μ mu = mu * nu; nu = 2 * nu; Hessian.SetDiagonal(diagonalOfHessian); } } } if (iterations >= maximumIterations) { exitCondition = ExitCondition.ExceedIterations; } return(new NonlinearMinimizationResult(objective, iterations, exitCondition)); }
public MinimizationWithLineSearchResult(IObjectiveFunction functionInfo, int iterations, ExitCondition reasonForExit, int totalLineSearchIterations, int iterationsWithNonTrivialLineSearch) : base(functionInfo, iterations, reasonForExit) { TotalLineSearchIterations = totalLineSearchIterations; IterationsWithNonTrivialLineSearch = iterationsWithNonTrivialLineSearch; }
/// <summary> /// Non-linear least square fitting by the trust-region algorithm. /// </summary> /// <param name="objective">The objective model, including function, jacobian, observations, and parameter bounds.</param> /// <param name="subproblem">The subproblem</param> /// <param name="initialGuess">The initial guess values.</param> /// <param name="functionTolerance">The stopping threshold for L2 norm of the residuals.</param> /// <param name="gradientTolerance">The stopping threshold for infinity norm of the gradient vector.</param> /// <param name="stepTolerance">The stopping threshold for L2 norm of the change of parameters.</param> /// <param name="radiusTolerance">The stopping threshold for trust region radius</param> /// <param name="maximumIterations">The max iterations.</param> /// <returns></returns> public static NonlinearMinimizationResult Minimum(ITrustRegionSubproblem subproblem, IObjectiveModel objective, Vector <double> initialGuess, Vector <double> lowerBound = null, Vector <double> upperBound = null, Vector <double> scales = null, List <bool> isFixed = null, double gradientTolerance = 1E-8, double stepTolerance = 1E-8, double functionTolerance = 1E-8, double radiusTolerance = 1E-18, int maximumIterations = -1) { // Non-linear least square fitting by the trust-region algorithm. // // For given datum pair (x, y), uncertainties σ (or weighting W = 1 / σ^2) and model function f = f(x; p), // let's find the parameters of the model so that the sum of the quares of the deviations is minimized. // // F(p) = 1/2 * ∑{ Wi * (yi - f(xi; p))^2 } // pbest = argmin F(p) // // Here, we will use the following terms: // Weighting W is the diagonal matrix and can be decomposed as LL', so L = 1/σ // Residuals, R = L(y - f(x; p)) // Residual sum of squares, RSS = ||R||^2 = R.DotProduct(R) // Jacobian J = df(x; p)/dp // Gradient g = -J'W(y − f(x; p)) = -J'LR // Approximated Hessian H = J'WJ // // The trust region algorithm is summarized as follows: // initially set trust-region radius, Δ // repeat // solve subproblem // update Δ: // let ρ = (RSS - RSSnew) / predRed // if ρ > 0.75, Δ = 2Δ // if ρ < 0.25, Δ = Δ/4 // if ρ > eta, P = P + ΔP // // References: // [1]. Madsen, K., H. B. Nielsen, and O. Tingleff. // "Methods for Non-Linear Least Squares Problems. Technical University of Denmark, 2004. Lecture notes." (2004). // Available Online from: http://orbit.dtu.dk/files/2721358/imm3215.pdf // [2]. Nocedal, Jorge, and Stephen J. Wright. // Numerical optimization (2006): 101-134. // [3]. SciPy // Available Online from: https://github.com/scipy/scipy/blob/master/scipy/optimize/_trustregion.py double maxDelta = 1000; double eta = 0; if (objective == null) { throw new ArgumentNullException("objective"); } ValidateBounds(initialGuess, lowerBound, upperBound, scales); objective.SetParameters(initialGuess, isFixed); ExitCondition exitCondition = ExitCondition.None; // First, calculate function values and setup variables var P = ProjectToInternalParameters(initialGuess); // current internal parameters var Pstep = Vector <double> .Build.Dense(P.Count); // the change of parameters var RSS = EvaluateFunction(objective, initialGuess); // Residual Sum of Squares if (maximumIterations < 0) { maximumIterations = 200 * (initialGuess.Count + 1); } // if RSS == NaN, stop if (double.IsNaN(RSS)) { exitCondition = ExitCondition.InvalidValues; return(new NonlinearMinimizationResult(objective, -1, exitCondition)); } // When only function evaluation is needed, set maximumIterations to zero, if (maximumIterations == 0) { exitCondition = ExitCondition.ManuallyStopped; } // if ||R||^2 <= fTol, stop if (RSS <= functionTolerance) { exitCondition = ExitCondition.Converged; // SmallRSS } // evaluate projected gradient and Hessian var jac = EvaluateJacobian(objective, P); var Gradient = jac.Item1; // objective.Gradient; var Hessian = jac.Item2; // objective.Hessian; // if ||g||_oo <= gtol, found and stop if (Gradient.InfinityNorm() <= gradientTolerance) { exitCondition = ExitCondition.RelativeGradient; // SmallGradient } if (exitCondition != ExitCondition.None) { return(new NonlinearMinimizationResult(objective, -1, exitCondition)); } // initialize trust-region radius, Δ double delta = Gradient.DotProduct(Gradient) / (Hessian * Gradient).DotProduct(Gradient); delta = Math.Max(1.0, Math.Min(delta, maxDelta)); int iterations = 0; bool hitBoundary = false; while (iterations < maximumIterations && exitCondition == ExitCondition.None) { iterations++; // solve the subproblem subproblem.Solve(objective, delta); Pstep = subproblem.Pstep; hitBoundary = subproblem.HitBoundary; // predicted reduction = L(0) - L(Δp) = -Δp'g - 1/2 * Δp'HΔp var predictedReduction = -Gradient.DotProduct(Pstep) - 0.5 * Pstep.DotProduct(Hessian * Pstep); if (Pstep.L2Norm() <= stepTolerance * (stepTolerance + P.L2Norm())) { exitCondition = ExitCondition.RelativePoints; // SmallRelativeParameters break; } var Pnew = P + Pstep; // parameters to test // evaluate function at Pnew var RSSnew = EvaluateFunction(objective, Pnew); // if RSS == NaN, stop if (double.IsNaN(RSSnew)) { exitCondition = ExitCondition.InvalidValues; break; } // calculate the ratio of the actual to the predicted reduction. double rho = (predictedReduction != 0) ? (RSS - RSSnew) / predictedReduction : 0.0; if (rho > 0.75 && hitBoundary) { delta = Math.Min(2.0 * delta, maxDelta); } else if (rho < 0.25) { delta = delta * 0.25; if (delta <= radiusTolerance * (radiusTolerance + P.DotProduct(P))) { exitCondition = ExitCondition.LackOfProgress; break; } } if (rho > eta) { // accepted Pnew.CopyTo(P); RSS = RSSnew; // evaluate projected gradient and Hessian jac = EvaluateJacobian(objective, P); Gradient = jac.Item1; // objective.Gradient; Hessian = jac.Item2; // objective.Hessian; // if ||g||_oo <= gtol, found and stop if (Gradient.InfinityNorm() <= gradientTolerance) { exitCondition = ExitCondition.RelativeGradient; } // if ||R||^2 < fTol, found and stop if (RSS <= functionTolerance) { exitCondition = ExitCondition.Converged; // SmallRSS } } } if (iterations >= maximumIterations) { exitCondition = ExitCondition.ExceedIterations; } return(new NonlinearMinimizationResult(objective, iterations, exitCondition)); }
// Implemented following http://www.math.washington.edu/~burke/crs/408/lectures/L9-weak-Wolfe.pdf public LineSearchOutput FindConformingStep(IObjectiveFunction objective, IEvaluation starting_point, Vector <double> search_direction, double initial_step, double upper_bound = Double.PositiveInfinity) { if (!(objective is ObjectiveChecker)) { objective = new ObjectiveChecker(objective, this.ValidateValue, this.ValidateGradient, null); } double lower_bound = 0.0; double step = initial_step; double initial_value = starting_point.Value; Vector <double> initial_gradient = starting_point.Gradient; double initial_dd = search_direction * initial_gradient; int ii; IEvaluation candidate_eval = null; ExitCondition reason_for_exit = ExitCondition.None; for (ii = 0; ii < this.MaximumIterations; ++ii) { candidate_eval = objective.Evaluate(starting_point.Point + search_direction * step); double step_dd = search_direction * candidate_eval.Gradient; if (candidate_eval.Value > initial_value + this.C1 * step * initial_dd) { upper_bound = step; step = 0.5 * (lower_bound + upper_bound); } else if (Math.Abs(step_dd) > this.C2 * Math.Abs(initial_dd)) { lower_bound = step; step = Double.IsPositiveInfinity(upper_bound) ? 2 * lower_bound : 0.5 * (lower_bound + upper_bound); } else { reason_for_exit = ExitCondition.StrongWolfeCriteria; break; } if (!Double.IsInfinity(upper_bound)) { double max_rel_change = 0.0; for (int jj = 0; jj < candidate_eval.Point.Count; ++jj) { double tmp = Math.Abs(search_direction[jj] * (upper_bound - lower_bound)) / Math.Max(Math.Abs(candidate_eval.Point[jj]), 1.0); max_rel_change = Math.Max(max_rel_change, tmp); } if (max_rel_change < this.ParameterTolerance) { reason_for_exit = ExitCondition.LackOfProgress; break; } } } if (ii == this.MaximumIterations && Double.IsPositiveInfinity(upper_bound)) { throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached. Function appears to be unbounded in search direction.", this.MaximumIterations)); } else if (ii == this.MaximumIterations) { throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", this.MaximumIterations)); } else { return(new LineSearchOutput(candidate_eval, ii, step, reason_for_exit)); } }
/// <summary></summary> /// <param name="startingPoint">The objective function being optimized, evaluated at the starting point of the search</param> /// <param name="searchDirection">Search direction</param> /// <param name="initialStep">Initial size of the step in the search direction</param> /// <param name="upperBound">The upper bound</param> public LineSearchResult FindConformingStep(IObjectiveFunctionEvaluation startingPoint, Vector <double> searchDirection, double initialStep, double upperBound) { ValidateInputArguments(startingPoint, searchDirection, initialStep, upperBound); double lowerBound = 0.0; double step = initialStep; double initialValue = startingPoint.Value; Vector <double> initialGradient = startingPoint.Gradient; double initialDd = searchDirection * initialGradient; IObjectiveFunction objective = startingPoint.CreateNew(); int ii; ExitCondition reasonForExit = ExitCondition.None; for (ii = 0; ii < MaximumIterations; ++ii) { objective.EvaluateAt(startingPoint.Point + searchDirection * step); ValidateGradient(objective); ValidateValue(objective); double stepDd = searchDirection * objective.Gradient; if (objective.Value > initialValue + C1 * step * initialDd) { upperBound = step; step = 0.5 * (lowerBound + upperBound); } else if (WolfeCondition(stepDd, initialDd)) { lowerBound = step; step = double.IsPositiveInfinity(upperBound) ? 2 * lowerBound : 0.5 * (lowerBound + upperBound); } else { reasonForExit = WolfeExitCondition; break; } if (!double.IsInfinity(upperBound)) { double maxRelChange = 0.0; for (int jj = 0; jj < objective.Point.Count; ++jj) { double tmp = Math.Abs(searchDirection[jj] * (upperBound - lowerBound)) / Math.Max(Math.Abs(objective.Point[jj]), 1.0); maxRelChange = Math.Max(maxRelChange, tmp); } if (maxRelChange < ParameterTolerance) { reasonForExit = ExitCondition.LackOfProgress; break; } } } if (ii == MaximumIterations && Double.IsPositiveInfinity(upperBound)) { throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached. Function appears to be unbounded in search direction.", MaximumIterations)); } if (ii == MaximumIterations) { throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", MaximumIterations)); } return(new LineSearchResult(objective, ii, step, reasonForExit)); }
public MinimizationWithLineSearchOutput(IEvaluation function_info, int iterations, ExitCondition reason_for_exit, int total_line_search_iterations, int iterations_with_non_trivial_line_search) : base(function_info, iterations, reason_for_exit) { this.TotalLineSearchIterations = total_line_search_iterations; this.IterationsWithNonTrivialLineSearch = iterations_with_non_trivial_line_search; }
public MinimizationOutput FindMinimum(IObjectiveFunction objective, Vector <double> lower_bound, Vector <double> upper_bound, Vector <double> initial_guess) { if (!objective.GradientSupported) { throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for BFGS minimization."); } if (!(objective is ObjectiveChecker)) { objective = new ObjectiveChecker(objective, this.ValidateObjective, this.ValidateGradient, null); } // Check that dimensions match if (lower_bound.Count != upper_bound.Count || lower_bound.Count != initial_guess.Count) { throw new ArgumentException("Dimensions of bounds and/or initial guess do not match."); } // Check that initial guess is feasible for (int ii = 0; ii < initial_guess.Count; ++ii) { if (initial_guess[ii] < lower_bound[ii] || initial_guess[ii] > upper_bound[ii]) { throw new ArgumentException("Initial guess is not in the feasible region"); } } IEvaluation initial_eval = objective.Evaluate(initial_guess); // Check that we're not already done ExitCondition current_exit_condition = this.ExitCriteriaSatisfied(initial_eval, null, lower_bound, upper_bound, 0); if (current_exit_condition != ExitCondition.None) { return(new MinimizationOutput(initial_eval, 0, current_exit_condition)); } // Set up line search algorithm var line_searcher = new StrongWolfeLineSearch(1e-4, 0.9, Math.Max(this.ParameterTolerance, 1e-5), max_iterations: 1000); // Declare state variables IEvaluation candidate_point, previous_point; double step_size; Vector <double> gradient, step, line_search_direction, reduced_solution1, reduced_gradient, reduced_initial_point, reduced_cauchy_point, solution1; Matrix <double> pseudo_hessian, reduced_hessian; List <int> reduced_map; // First step pseudo_hessian = DiagonalMatrix.CreateIdentity(initial_guess.Count); // Determine active set var gradient_projection_result = QuadraticGradientProjectionSearch.search(initial_eval.Point, initial_eval.Gradient, pseudo_hessian, lower_bound, upper_bound); var cauchy_point = gradient_projection_result.Item1; var fixed_count = gradient_projection_result.Item2; var is_fixed = gradient_projection_result.Item3; var free_count = lower_bound.Count - fixed_count; if (free_count > 0) { reduced_gradient = new DenseVector(free_count); reduced_hessian = new DenseMatrix(free_count, free_count); reduced_map = new List <int>(free_count); reduced_initial_point = new DenseVector(free_count); reduced_cauchy_point = new DenseVector(free_count); CreateReducedData(initial_eval.Point, cauchy_point, is_fixed, lower_bound, upper_bound, initial_eval.Gradient, pseudo_hessian, reduced_initial_point, reduced_cauchy_point, reduced_gradient, reduced_hessian, reduced_map); // Determine search direction and maximum step size reduced_solution1 = reduced_initial_point + reduced_hessian.Cholesky().Solve(-reduced_gradient); solution1 = reduced_to_full(reduced_map, reduced_solution1, cauchy_point); } else { solution1 = cauchy_point; } var direction_from_cauchy = solution1 - cauchy_point; var max_step_from_cauchy_point = FindMaxStep(cauchy_point, direction_from_cauchy, lower_bound, upper_bound); var solution2 = cauchy_point + Math.Min(max_step_from_cauchy_point, 1.0) * direction_from_cauchy; line_search_direction = solution2 - initial_eval.Point; var max_line_search_step = FindMaxStep(initial_eval.Point, line_search_direction, lower_bound, upper_bound); var est_step_size = -initial_eval.Gradient * line_search_direction / (line_search_direction * pseudo_hessian * line_search_direction); var starting_step_size = Math.Min(Math.Max(est_step_size, 1.0), max_line_search_step); // Line search LineSearchOutput result; try { result = line_searcher.FindConformingStep(objective, initial_eval, line_search_direction, starting_step_size, upper_bound: max_line_search_step); } catch (Exception e) { throw new InnerOptimizationException("Line search failed.", e); } previous_point = initial_eval; candidate_point = result.FunctionInfoAtMinimum; gradient = candidate_point.Gradient; step = candidate_point.Point - initial_guess; step_size = result.FinalStep; // Subsequent steps int iterations; int total_line_search_steps = result.Iterations; int iterations_with_nontrivial_line_search = result.Iterations > 0 ? 0 : 1; int steepest_descent_resets = 0; for (iterations = 1; iterations < this.MaximumIterations; ++iterations) { // Do BFGS update var y = candidate_point.Gradient - previous_point.Gradient; double sy = step * y; if (sy > 0.0) // only do update if it will create a positive definite matrix { double sts = step * step; //inverse_pseudo_hessian = inverse_pseudo_hessian + ((sy + y * inverse_pseudo_hessian * y) / Math.Pow(sy, 2.0)) * step.OuterProduct(step) - ((inverse_pseudo_hessian * y.ToColumnMatrix()) * step.ToRowMatrix() + step.ToColumnMatrix() * (y.ToRowMatrix() * inverse_pseudo_hessian)) * (1.0 / sy); var Hs = pseudo_hessian * step; var sHs = step * pseudo_hessian * step; pseudo_hessian = pseudo_hessian + y.OuterProduct(y) * (1.0 / sy) - Hs.OuterProduct(Hs) * (1.0 / sHs); } else { steepest_descent_resets += 1; //pseudo_hessian = LinearAlgebra.Double.DiagonalMatrix.Identity(initial_guess.Count); } // Determine active set gradient_projection_result = QuadraticGradientProjectionSearch.search(candidate_point.Point, candidate_point.Gradient, pseudo_hessian, lower_bound, upper_bound); cauchy_point = gradient_projection_result.Item1; fixed_count = gradient_projection_result.Item2; is_fixed = gradient_projection_result.Item3; free_count = lower_bound.Count - fixed_count; if (free_count > 0) { reduced_gradient = new DenseVector(free_count); reduced_hessian = new DenseMatrix(free_count, free_count); reduced_map = new List <int>(free_count); reduced_initial_point = new DenseVector(free_count); reduced_cauchy_point = new DenseVector(free_count); CreateReducedData(candidate_point.Point, cauchy_point, is_fixed, lower_bound, upper_bound, candidate_point.Gradient, pseudo_hessian, reduced_initial_point, reduced_cauchy_point, reduced_gradient, reduced_hessian, reduced_map); // Determine search direction and maximum step size reduced_solution1 = reduced_initial_point + reduced_hessian.Cholesky().Solve(-reduced_gradient); solution1 = reduced_to_full(reduced_map, reduced_solution1, cauchy_point); } else { solution1 = cauchy_point; } direction_from_cauchy = solution1 - cauchy_point; max_step_from_cauchy_point = FindMaxStep(cauchy_point, direction_from_cauchy, lower_bound, upper_bound); //var cauchy_eval = objective.Evaluate(cauchy_point); solution2 = cauchy_point + Math.Min(max_step_from_cauchy_point, 1.0) * direction_from_cauchy; line_search_direction = solution2 - candidate_point.Point; max_line_search_step = FindMaxStep(candidate_point.Point, line_search_direction, lower_bound, upper_bound); //line_search_direction = solution1 - candidate_point.Point; //max_line_search_step = FindMaxStep(candidate_point.Point, line_search_direction, lower_bound, upper_bound); if (max_line_search_step == 0.0) { line_search_direction = cauchy_point - candidate_point.Point; max_line_search_step = FindMaxStep(candidate_point.Point, line_search_direction, lower_bound, upper_bound); } est_step_size = -candidate_point.Gradient * line_search_direction / (line_search_direction * pseudo_hessian * line_search_direction); starting_step_size = Math.Min(Math.Max(est_step_size, 1.0), max_line_search_step); // Line search try { result = line_searcher.FindConformingStep(objective, candidate_point, line_search_direction, starting_step_size, upper_bound: max_line_search_step); //result = line_searcher.FindConformingStep(objective, cauchy_eval, direction_from_cauchy, Math.Min(1.0, max_step_from_cauchy_point), upper_bound: max_step_from_cauchy_point); } catch (Exception e) { throw new InnerOptimizationException("Line search failed.", e); } iterations_with_nontrivial_line_search += result.Iterations > 0 ? 1 : 0; total_line_search_steps += result.Iterations; step_size = result.FinalStep; step = result.FunctionInfoAtMinimum.Point - candidate_point.Point; previous_point = candidate_point; candidate_point = result.FunctionInfoAtMinimum; current_exit_condition = this.ExitCriteriaSatisfied(candidate_point, previous_point, lower_bound, upper_bound, iterations); if (current_exit_condition != ExitCondition.None) { break; } } if (iterations == this.MaximumIterations && current_exit_condition == ExitCondition.None) { throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", this.MaximumIterations)); } return(new MinimizationWithLineSearchOutput(candidate_point, iterations, current_exit_condition, total_line_search_steps, iterations_with_nontrivial_line_search)); }
public MinimizationOutput(IEvaluation function_info, int iterations, ExitCondition reason_for_exit) { this.FunctionInfoAtMinimum = function_info; this.Iterations = iterations; this.ReasonForExit = reason_for_exit; }
/// <summary> /// Find the minimum of the objective function given lower and upper bounds /// </summary> /// <param name="objective">The objective function, must support a gradient</param> /// <param name="lowerBound">The lower bound</param> /// <param name="upperBound">The upper bound</param> /// <param name="initialGuess">The initial guess</param> /// <returns>The MinimizationResult which contains the minimum and the ExitCondition</returns> public MinimizationResult FindMinimum(IObjectiveFunction objective, Vector <double> lowerBound, Vector <double> upperBound, Vector <double> initialGuess) { _lowerBound = lowerBound; _upperBound = upperBound; if (!objective.IsGradientSupported) { throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for BFGS minimization."); } // Check that dimensions match if (lowerBound.Count != upperBound.Count || lowerBound.Count != initialGuess.Count) { throw new ArgumentException("Dimensions of bounds and/or initial guess do not match."); } // Check that initial guess is feasible for (int ii = 0; ii < initialGuess.Count; ++ii) { if (initialGuess[ii] < lowerBound[ii] || initialGuess[ii] > upperBound[ii]) { throw new ArgumentException("Initial guess is not in the feasible region"); } } objective.EvaluateAt(initialGuess); ValidateGradientAndObjective(objective); // Check that we're not already done ExitCondition currentExitCondition = ExitCriteriaSatisfied(objective, null, 0); if (currentExitCondition != ExitCondition.None) { return(new MinimizationResult(objective, 0, currentExitCondition)); } // Set up line search algorithm var lineSearcher = new StrongWolfeLineSearch(1e-4, 0.9, Math.Max(ParameterTolerance, 1e-5), maxIterations: 1000); // Declare state variables Vector <double> reducedSolution1, reducedGradient, reducedInitialPoint, reducedCauchyPoint, solution1; Matrix <double> reducedHessian; List <int> reducedMap; // First step var pseudoHessian = CreateMatrix.DiagonalIdentity <double>(initialGuess.Count); // Determine active set var gradientProjectionResult = QuadraticGradientProjectionSearch.Search(objective.Point, objective.Gradient, pseudoHessian, lowerBound, upperBound); var cauchyPoint = gradientProjectionResult.CauchyPoint; var fixedCount = gradientProjectionResult.FixedCount; var isFixed = gradientProjectionResult.IsFixed; var freeCount = lowerBound.Count - fixedCount; if (freeCount > 0) { reducedGradient = new DenseVector(freeCount); reducedHessian = new DenseMatrix(freeCount, freeCount); reducedMap = new List <int>(freeCount); reducedInitialPoint = new DenseVector(freeCount); reducedCauchyPoint = new DenseVector(freeCount); CreateReducedData(objective.Point, cauchyPoint, isFixed, lowerBound, upperBound, objective.Gradient, pseudoHessian, reducedInitialPoint, reducedCauchyPoint, reducedGradient, reducedHessian, reducedMap); // Determine search direction and maximum step size reducedSolution1 = reducedInitialPoint + reducedHessian.Cholesky().Solve(-reducedGradient); solution1 = ReducedToFull(reducedMap, reducedSolution1, cauchyPoint); } else { solution1 = cauchyPoint; } var directionFromCauchy = solution1 - cauchyPoint; var maxStepFromCauchyPoint = FindMaxStep(cauchyPoint, directionFromCauchy, lowerBound, upperBound); var solution2 = cauchyPoint + Math.Min(maxStepFromCauchyPoint, 1.0) * directionFromCauchy; var lineSearchDirection = solution2 - objective.Point; var maxLineSearchStep = FindMaxStep(objective.Point, lineSearchDirection, lowerBound, upperBound); var estStepSize = -objective.Gradient * lineSearchDirection / (lineSearchDirection * pseudoHessian * lineSearchDirection); var startingStepSize = Math.Min(Math.Max(estStepSize, 1.0), maxLineSearchStep); // Line search LineSearchResult lineSearchResult; try { lineSearchResult = lineSearcher.FindConformingStep(objective, lineSearchDirection, startingStepSize, upperBound: maxLineSearchStep); } catch (Exception e) { throw new InnerOptimizationException("Line search failed.", e); } var previousPoint = objective.Fork(); var candidatePoint = lineSearchResult.FunctionInfoAtMinimum; var gradient = candidatePoint.Gradient; var step = candidatePoint.Point - initialGuess; // Subsequent steps int totalLineSearchSteps = lineSearchResult.Iterations; int iterationsWithNontrivialLineSearch = lineSearchResult.Iterations > 0 ? 0 : 1; int iterations = DoBfgsUpdate(ref currentExitCondition, lineSearcher, ref pseudoHessian, ref lineSearchDirection, ref previousPoint, ref lineSearchResult, ref candidatePoint, ref step, ref totalLineSearchSteps, ref iterationsWithNontrivialLineSearch); if (iterations == MaximumIterations && currentExitCondition == ExitCondition.None) { throw new MaximumIterationsException(string.Format("Maximum iterations ({0}) reached.", MaximumIterations)); } return(new MinimizationWithLineSearchResult(candidatePoint, iterations, currentExitCondition, totalLineSearchSteps, iterationsWithNontrivialLineSearch)); }
public MinimizationResult1D(IEvaluation1D functionInfo, int iterations, ExitCondition reasonForExit) { FunctionInfoAtMinimum = functionInfo; Iterations = iterations; ReasonForExit = reasonForExit; }
public LineSearchOutput(IEvaluation function_info, int iterations, double final_step, ExitCondition reason_for_exit) : base(function_info, iterations, reason_for_exit) { this.FinalStep = final_step; }
public MinimizationResult(IObjectiveFunction functionInfo, int iterations, ExitCondition reasonForExit) { FunctionInfoAtMinimum = functionInfo; Iterations = iterations; ReasonForExit = reasonForExit; }
public ExitStateCondition(ExitCondition test, State state) { ExitTest = test; ToState = state; }
/// <summary> /// Find the minimum of the objective function given lower and upper bounds /// </summary> /// <param name="objective">The objective function, must support a gradient</param> /// <param name="initialGuess">The initial guess</param> /// <returns>The MinimizationResult which contains the minimum and the ExitCondition</returns> public MinimizationResult FindMinimum(IObjectiveFunction objective, Vector <double> initialGuess) { if (!objective.IsGradientSupported) { throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for L-BFGS minimization."); } objective.EvaluateAt(initialGuess); ValidateGradientAndObjective(objective); // Check that we're not already done ExitCondition currentExitCondition = ExitCriteriaSatisfied(objective, null, 0); if (currentExitCondition != ExitCondition.None) { return(new MinimizationResult(objective, 0, currentExitCondition)); } // Set up line search algorithm var lineSearcher = new WeakWolfeLineSearch(1e-4, 0.9, Math.Max(ParameterTolerance, 1e-10), 1000); // First step var lineSearchDirection = -objective.Gradient; var stepSize = (100 * GradientTolerance) / (lineSearchDirection * lineSearchDirection); var previousPoint = objective; LineSearchResult lineSearchResult; try { lineSearchResult = lineSearcher.FindConformingStep(objective, lineSearchDirection, stepSize); } catch (OptimizationException e) { throw new InnerOptimizationException("Line search failed.", e); } catch (ArgumentException e) { throw new InnerOptimizationException("Line search failed.", e); } var candidate = lineSearchResult.FunctionInfoAtMinimum; ValidateGradientAndObjective(candidate); var gradient = candidate.Gradient; var step = candidate.Point - initialGuess; var yk = candidate.Gradient - previousPoint.Gradient; var ykhistory = new List <Vector <double> >() { yk }; var skhistory = new List <Vector <double> >() { step }; var rhokhistory = new List <double>() { 1.0 / yk.DotProduct(step) }; // Subsequent steps int iterations = 1; int totalLineSearchSteps = lineSearchResult.Iterations; int iterationsWithNontrivialLineSearch = lineSearchResult.Iterations > 0 ? 0 : 1; previousPoint = candidate; while ((iterations++ < MaximumIterations) && (previousPoint.Gradient.Norm(2) >= GradientTolerance)) { lineSearchDirection = -ApplyLbfgsUpdate(previousPoint, ykhistory, skhistory, rhokhistory); var directionalDerivative = previousPoint.Gradient.DotProduct(lineSearchDirection); if (directionalDerivative > 0) { throw new InnerOptimizationException("Direction is not a descent direction."); } try { lineSearchResult = lineSearcher.FindConformingStep(previousPoint, lineSearchDirection, 1.0); } catch (OptimizationException e) { throw new InnerOptimizationException("Line search failed.", e); } catch (ArgumentException e) { throw new InnerOptimizationException("Line search failed.", e); } iterationsWithNontrivialLineSearch += lineSearchResult.Iterations > 0 ? 1 : 0; totalLineSearchSteps += lineSearchResult.Iterations; candidate = lineSearchResult.FunctionInfoAtMinimum; currentExitCondition = ExitCriteriaSatisfied(candidate, previousPoint, iterations); if (currentExitCondition != ExitCondition.None) { break; } step = candidate.Point - previousPoint.Point; yk = candidate.Gradient - previousPoint.Gradient; ykhistory.Add(yk); skhistory.Add(step); rhokhistory.Add(1.0 / yk.DotProduct(step)); previousPoint = candidate; if (ykhistory.Count > Memory) { ykhistory.RemoveAt(0); skhistory.RemoveAt(0); rhokhistory.RemoveAt(0); } } if ((iterations == MaximumIterations) && (currentExitCondition == ExitCondition.None)) { throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", MaximumIterations)); } return(new MinimizationWithLineSearchResult(candidate, iterations, ExitCondition.AbsoluteGradient, totalLineSearchSteps, iterationsWithNontrivialLineSearch)); }
/// <summary> /// Finds the minimum of the objective function with an intial pertubation /// </summary> /// <param name="objectiveFunction">The objective function, no gradient or hessian needed</param> /// <param name="initialGuess">The intial guess</param> /// <param name="initalPertubation">The inital pertubation</param> /// <returns>The minimum point</returns> public MinimizationResult FindMinimum(IObjectiveFunction objectiveFunction, Vector <double> initialGuess, Vector <double> initalPertubation) { // confirm that we are in a position to commence if (objectiveFunction == null) { throw new ArgumentNullException("objectiveFunction", "ObjectiveFunction must be set to a valid ObjectiveFunctionDelegate"); } if (initialGuess == null) { throw new ArgumentNullException("initialGuess", "initialGuess must be initialized"); } if (initialGuess == null) { throw new ArgumentNullException("initalPertubation", "initalPertubation must be initialized, if unknown use overloaded version of FindMinimum()"); } SimplexConstant[] simplexConstants = SimplexConstant.CreateSimplexConstantsFromVectors(initialGuess, initalPertubation); // create the initial simplex int numDimensions = simplexConstants.Length; int numVertices = numDimensions + 1; Vector <double>[] vertices = InitializeVertices(simplexConstants); double[] errorValues = new double[numVertices]; int evaluationCount = 0; ExitCondition exitCondition = ExitCondition.None; ErrorProfile errorProfile; errorValues = InitializeErrorValues(vertices, objectiveFunction); // iterate until we converge, or complete our permitted number of iterations while (true) { errorProfile = EvaluateSimplex(errorValues); // see if the range in point heights is small enough to exit if (HasConverged(ConvergenceTolerance, errorProfile, errorValues)) { exitCondition = ExitCondition.Converged; break; } // attempt a reflection of the simplex double reflectionPointValue = TryToScaleSimplex(-1.0, ref errorProfile, vertices, errorValues, objectiveFunction); ++evaluationCount; if (reflectionPointValue <= errorValues[errorProfile.LowestIndex]) { // it's better than the best point, so attempt an expansion of the simplex double expansionPointValue = TryToScaleSimplex(2.0, ref errorProfile, vertices, errorValues, objectiveFunction); ++evaluationCount; } else if (reflectionPointValue >= errorValues[errorProfile.NextHighestIndex]) { // it would be worse than the second best point, so attempt a contraction to look // for an intermediate point double currentWorst = errorValues[errorProfile.HighestIndex]; double contractionPointValue = TryToScaleSimplex(0.5, ref errorProfile, vertices, errorValues, objectiveFunction); ++evaluationCount; if (contractionPointValue >= currentWorst) { // that would be even worse, so let's try to contract uniformly towards the low point; // don't bother to update the error profile, we'll do it at the start of the // next iteration ShrinkSimplex(errorProfile, vertices, errorValues, objectiveFunction); evaluationCount += numVertices; // that required one function evaluation for each vertex; keep track } } // check to see if we have exceeded our alloted number of evaluations if (evaluationCount >= MaximumIterations) { throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", MaximumIterations)); } } var regressionResult = new MinimizationResult(objectiveFunction, evaluationCount, exitCondition); return(regressionResult); }
/// <summary>Creates a new <see cref="ClosedNewtonCotesFormula"/> object. /// </summary> /// <param name="rule">The Newton-Cotes rule, i.e. the degree of the Newton-Cotes method.</param> /// <param name="exitCondition">The exit condition.</param> /// <returns>A new <see cref="ClosedNewtonCotesFormula"/> object.</returns> public static ClosedNewtonCotesFormula Create(ClosedNewtonCotesFormula.Rule rule, ExitCondition exitCondition) { return(new ClosedNewtonCotesFormula(rule, exitCondition)); }
/// <summary> /// Find the minimum of the objective function given lower and upper bounds /// </summary> /// <param name="objective">The objective function, must support a gradient</param> /// <param name="initialGuess">The initial guess</param> /// <returns>The MinimizationResult which contains the minimum and the ExitCondition</returns> public MinimizationResult FindMinimum(IObjectiveFunction objective, Vector <double> initialGuess) { if (!objective.IsGradientSupported) { throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for BFGS minimization."); } objective.EvaluateAt(initialGuess); ValidateGradientAndObjective(objective); // Check that we're not already done ExitCondition currentExitCondition = ExitCriteriaSatisfied(objective, null, 0); if (currentExitCondition != ExitCondition.None) { return(new MinimizationResult(objective, 0, currentExitCondition)); } // Set up line search algorithm var lineSearcher = new WeakWolfeLineSearch(1e-4, 0.9, Math.Max(ParameterTolerance, 1e-10), 1000); // First step var inversePseudoHessian = CreateMatrix.DenseIdentity <double>(initialGuess.Count); var lineSearchDirection = -objective.Gradient; var stepSize = 100 * GradientTolerance / (lineSearchDirection * lineSearchDirection); var previousPoint = objective; LineSearchResult lineSearchResult; try { lineSearchResult = lineSearcher.FindConformingStep(objective, lineSearchDirection, stepSize); } catch (OptimizationException e) { throw new InnerOptimizationException("Line search failed.", e); } catch (ArgumentException e) { throw new InnerOptimizationException("Line search failed.", e); } var candidate = lineSearchResult.FunctionInfoAtMinimum; ValidateGradientAndObjective(candidate); var gradient = candidate.Gradient; var step = candidate.Point - initialGuess; // Subsequent steps Matrix <double> I = CreateMatrix.DiagonalIdentity <double>(initialGuess.Count); int iterations; int totalLineSearchSteps = lineSearchResult.Iterations; int iterationsWithNontrivialLineSearch = lineSearchResult.Iterations > 0 ? 0 : 1; iterations = DoBfgsUpdate(ref currentExitCondition, lineSearcher, ref inversePseudoHessian, ref lineSearchDirection, ref previousPoint, ref lineSearchResult, ref candidate, ref step, ref totalLineSearchSteps, ref iterationsWithNontrivialLineSearch); if (iterations == MaximumIterations && currentExitCondition == ExitCondition.None) { throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", MaximumIterations)); } return(new MinimizationWithLineSearchResult(candidate, iterations, ExitCondition.AbsoluteGradient, totalLineSearchSteps, iterationsWithNontrivialLineSearch)); }
public MinimizationOutput FindMinimum(IObjectiveFunction objective, Vector <double> initial_guess) { if (!objective.GradientSupported) { throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for BFGS minimization."); } if (!(objective is ObjectiveChecker)) { objective = new ObjectiveChecker(objective, this.ValidateObjective, this.ValidateGradient, null); } IEvaluation initial_eval = objective.Evaluate(initial_guess); // Check that we're not already done ExitCondition current_exit_condition = this.ExitCriteriaSatisfied(initial_eval, null); if (current_exit_condition != ExitCondition.None) { return(new MinimizationOutput(initial_eval, 0, current_exit_condition)); } // Set up line search algorithm var line_searcher = new WeakWolfeLineSearch(1e-4, 0.9, Math.Max(this.ParameterTolerance, 1e-10), max_iterations: 1000); // Declare state variables IEvaluation candidate_point, previous_point; double step_size; Vector <double> gradient, step, search_direction; Matrix <double> inverse_pseudo_hessian; // First step inverse_pseudo_hessian = LinearAlgebra.Double.DiagonalMatrix.CreateIdentity(initial_guess.Count); search_direction = -initial_eval.Gradient; step_size = 100 * this.GradientTolerance / (search_direction * search_direction); LineSearchOutput result; try { result = line_searcher.FindConformingStep(objective, initial_eval, search_direction, step_size); } catch (Exception e) { throw new InnerOptimizationException("Line search failed.", e); } previous_point = initial_eval; candidate_point = result.FunctionInfoAtMinimum; gradient = candidate_point.Gradient; step = candidate_point.Point - initial_guess; step_size = result.FinalStep; // Subsequent steps Matrix <double> I = LinearAlgebra.Double.DiagonalMatrix.CreateIdentity(initial_guess.Count); int iterations; int total_line_search_steps = result.Iterations; int iterations_with_nontrivial_line_search = result.Iterations > 0 ? 0 : 1; int steepest_descent_resets = 0; for (iterations = 1; iterations < this.MaximumIterations; ++iterations) { var y = candidate_point.Gradient - previous_point.Gradient; double sy = step * y; inverse_pseudo_hessian = inverse_pseudo_hessian + ((sy + y * inverse_pseudo_hessian * y) / Math.Pow(sy, 2.0)) * step.OuterProduct(step) - ((inverse_pseudo_hessian * y.ToColumnMatrix()) * step.ToRowMatrix() + step.ToColumnMatrix() * (y.ToRowMatrix() * inverse_pseudo_hessian)) * (1.0 / sy); search_direction = -inverse_pseudo_hessian * candidate_point.Gradient; if (search_direction * candidate_point.Gradient >= 0.0) { search_direction = -candidate_point.Gradient; inverse_pseudo_hessian = LinearAlgebra.Double.DiagonalMatrix.CreateIdentity(initial_guess.Count); steepest_descent_resets += 1; } //else if (search_direction * candidate_point.Gradient >= -this.GradientTolerance*this.GradientTolerance) //{ // search_direction = -candidate_point.Gradient; // inverse_pseudo_hessian = LinearAlgebra.Double.DiagonalMatrix.Identity(initial_guess.Count); // steepest_descent_resets += 1; //} try { result = line_searcher.FindConformingStep(objective, candidate_point, search_direction, 1.0); } catch (Exception e) { throw new InnerOptimizationException("Line search failed.", e); } iterations_with_nontrivial_line_search += result.Iterations > 0 ? 1 : 0; total_line_search_steps += result.Iterations; step_size = result.FinalStep; step = result.FunctionInfoAtMinimum.Point - candidate_point.Point; previous_point = candidate_point; candidate_point = result.FunctionInfoAtMinimum; current_exit_condition = this.ExitCriteriaSatisfied(candidate_point, previous_point); if (current_exit_condition != ExitCondition.None) { break; } } if (iterations == this.MaximumIterations && current_exit_condition == ExitCondition.None) { throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", this.MaximumIterations)); } return(new MinimizationWithLineSearchOutput(candidate_point, iterations, current_exit_condition, total_line_search_steps, iterations_with_nontrivial_line_search)); }