/// <summary> /// Finds a minimum of a function by the BFGS quasi-Newton method /// This uses the function and it's gradient (partial derivatives in each direction) and approximates the Hessian /// </summary> /// <param name="initialGuess">An initial guess</param> /// <param name="functionValue">Evaluates the function at a point</param> /// <param name="functionGradient">Evaluates the gradient of the function at a point</param> /// <returns>The minimum found</returns> public static Vector <double> Solve(Vector initialGuess, Func <Vector <double>, double> functionValue, Func <Vector <double>, Vector <double> > functionGradient) { var objectiveFunction = ObjectiveFunction.Gradient(functionValue, functionGradient); objectiveFunction.EvaluateAt(initialGuess); int dim = initialGuess.Count; int iter = 0; // H represents the approximation of the inverse hessian matrix // it is updated via the Sherman–Morrison formula (http://en.wikipedia.org/wiki/Sherman%E2%80%93Morrison_formula) Matrix <double> H = DenseMatrix.CreateIdentity(dim); Vector <double> x = initialGuess; Vector <double> x_old = x; Vector <double> grad; WolfeLineSearch wolfeLineSearch = new WeakWolfeLineSearch(1e-4, 0.9, 1e-5, 200); do { // search along the direction of the gradient grad = objectiveFunction.Gradient; Vector <double> p = -1 * H * grad; var lineSearchResult = wolfeLineSearch.FindConformingStep(objectiveFunction, p, 1.0); double rate = lineSearchResult.FinalStep; x = x + rate * p; Vector <double> grad_old = grad; // update the gradient objectiveFunction.EvaluateAt(x); grad = objectiveFunction.Gradient;// functionGradient(x); Vector <double> s = x - x_old; Vector <double> y = grad - grad_old; double rho = 1.0 / (y * s); if (iter == 0) { // set up an initial hessian H = (y * s) / (y * y) * DenseMatrix.CreateIdentity(dim); } var sM = s.ToColumnMatrix(); var yM = y.ToColumnMatrix(); // Update the estimate of the hessian H = H - rho * (sM * (yM.TransposeThisAndMultiply(H)) + (H * yM).TransposeAndMultiply(sM)) + rho * rho * (y.DotProduct(H * y) + 1.0 / rho) * (sM.TransposeAndMultiply(sM)); x_old = x; iter++; }while ((grad.InfinityNorm() > GradientTolerance) && (iter < MaxIterations)); return(x); }
public static MinimizationResult Minimum(IObjectiveFunction objective, Vector <double> initialGuess, double gradientTolerance = 1e-8, int maxIterations = 1000) { if (!objective.IsGradientSupported) { throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for ConjugateGradient minimization."); } objective.EvaluateAt(initialGuess); var gradient = objective.Gradient; ValidateGradient(objective); // Check that we're not already done if (gradient.Norm(2.0) < gradientTolerance) { return(new MinimizationResult(objective, 0, ExitCondition.AbsoluteGradient)); } // Set up line search algorithm var lineSearcher = new WeakWolfeLineSearch(1e-4, 0.1, 1e-4, 1000); // First step var steepestDirection = -gradient; var searchDirection = steepestDirection; double initialStepSize = 100 * gradientTolerance / (gradient * gradient); LineSearchResult result; try { result = lineSearcher.FindConformingStep(objective, searchDirection, initialStepSize); } catch (Exception e) { throw new InnerOptimizationException("Line search failed.", e); } objective = result.FunctionInfoAtMinimum; ValidateGradient(objective); double stepSize = result.FinalStep; // Subsequent steps int iterations = 1; int totalLineSearchSteps = result.Iterations; int iterationsWithNontrivialLineSearch = result.Iterations > 0 ? 0 : 1; int steepestDescentResets = 0; while (objective.Gradient.Norm(2.0) >= gradientTolerance && iterations < maxIterations) { var previousSteepestDirection = steepestDirection; steepestDirection = -objective.Gradient; var searchDirectionAdjuster = Math.Max(0, steepestDirection * (steepestDirection - previousSteepestDirection) / (previousSteepestDirection * previousSteepestDirection)); searchDirection = steepestDirection + searchDirectionAdjuster * searchDirection; if (searchDirection * objective.Gradient >= 0) { searchDirection = steepestDirection; steepestDescentResets += 1; } try { result = lineSearcher.FindConformingStep(objective, searchDirection, stepSize); } catch (Exception e) { throw new InnerOptimizationException("Line search failed.", e); } iterationsWithNontrivialLineSearch += result.Iterations == 0 ? 1 : 0; totalLineSearchSteps += result.Iterations; stepSize = result.FinalStep; objective = result.FunctionInfoAtMinimum; iterations += 1; } //if (iterations == maxIterations) //{ // throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", maxIterations)); //} return(new MinimizationWithLineSearchResult(objective, iterations, ExitCondition.AbsoluteGradient, totalLineSearchSteps, iterationsWithNontrivialLineSearch)); }
/// <summary> /// Find the minimum of the objective function given lower and upper bounds /// </summary> /// <param name="objective">The objective function, must support a gradient</param> /// <param name="initialGuess">The initial guess</param> /// <returns>The MinimizationResult which contains the minimum and the ExitCondition</returns> public MinimizationResult FindMinimum(IObjectiveFunction objective, Vector <double> initialGuess) { if (!objective.IsGradientSupported) { throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for L-BFGS minimization."); } objective.EvaluateAt(initialGuess); ValidateGradientAndObjective(objective); // Check that we're not already done ExitCondition currentExitCondition = ExitCriteriaSatisfied(objective, null, 0); if (currentExitCondition != ExitCondition.None) { return(new MinimizationResult(objective, 0, currentExitCondition)); } // Set up line search algorithm var lineSearcher = new WeakWolfeLineSearch(1e-4, 0.9, Math.Max(ParameterTolerance, 1e-10), 1000); // First step var lineSearchDirection = -objective.Gradient; var stepSize = 100 * GradientTolerance / (lineSearchDirection * lineSearchDirection); var previousPoint = objective; LineSearchResult lineSearchResult; try { lineSearchResult = lineSearcher.FindConformingStep(objective, lineSearchDirection, stepSize); } catch (OptimizationException e) { throw new InnerOptimizationException("Line search failed.", e); } catch (ArgumentException e) { throw new InnerOptimizationException("Line search failed.", e); } var candidate = lineSearchResult.FunctionInfoAtMinimum; ValidateGradientAndObjective(candidate); var gradient = candidate.Gradient; var step = candidate.Point - initialGuess; var yk = candidate.Gradient - previousPoint.Gradient; var ykhistory = new List <Vector <double> >() { yk }; var skhistory = new List <Vector <double> >() { step }; var rhokhistory = new List <double>() { 1.0 / yk.DotProduct(step) }; // Subsequent steps int iterations = 1; int totalLineSearchSteps = lineSearchResult.Iterations; int iterationsWithNontrivialLineSearch = lineSearchResult.Iterations > 0 ? 0 : 1; previousPoint = candidate; while (iterations++ < MaximumIterations && previousPoint.Gradient.Norm(2) >= GradientTolerance) { lineSearchDirection = -ApplyLbfgsUpdate(previousPoint, ykhistory, skhistory, rhokhistory); var directionalDerivative = previousPoint.Gradient.DotProduct(lineSearchDirection); if (directionalDerivative > 0) { throw new InnerOptimizationException("Direction is not a descent direction."); } try { lineSearchResult = lineSearcher.FindConformingStep(previousPoint, lineSearchDirection, 1.0); } catch (OptimizationException e) { throw new InnerOptimizationException("Line search failed.", e); } catch (ArgumentException e) { throw new InnerOptimizationException("Line search failed.", e); } iterationsWithNontrivialLineSearch += lineSearchResult.Iterations > 0 ? 1 : 0; totalLineSearchSteps += lineSearchResult.Iterations; candidate = lineSearchResult.FunctionInfoAtMinimum; currentExitCondition = ExitCriteriaSatisfied(candidate, previousPoint, iterations); if (currentExitCondition != ExitCondition.None) { break; } step = candidate.Point - previousPoint.Point; yk = candidate.Gradient - previousPoint.Gradient; ykhistory.Add(yk); skhistory.Add(step); rhokhistory.Add(1.0 / yk.DotProduct(step)); previousPoint = candidate; if (ykhistory.Count > Memory) { ykhistory.RemoveAt(0); skhistory.RemoveAt(0); rhokhistory.RemoveAt(0); } } if (iterations == MaximumIterations && currentExitCondition == ExitCondition.None) { throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", MaximumIterations)); } return(new MinimizationWithLineSearchResult(candidate, iterations, ExitCondition.AbsoluteGradient, totalLineSearchSteps, iterationsWithNontrivialLineSearch)); }
/// <summary> /// Find the minimum of the objective function given lower and upper bounds /// </summary> /// <param name="objective">The objective function, must support a gradient</param> /// <param name="initialGuess">The initial guess</param> /// <returns>The MinimizationResult which contains the minimum and the ExitCondition</returns> public MinimizationResult FindMinimum(IObjectiveFunction objective, Vector <double> initialGuess) { if (!objective.IsGradientSupported) { throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for BFGS minimization."); } objective.EvaluateAt(initialGuess); ValidateGradientAndObjective(objective); // Check that we're not already done ExitCondition currentExitCondition = ExitCriteriaSatisfied(objective, null, 0); if (currentExitCondition != ExitCondition.None) { return(new MinimizationResult(objective, 0, currentExitCondition)); } // Set up line search algorithm var lineSearcher = new WeakWolfeLineSearch(1e-4, 0.9, Math.Max(ParameterTolerance, 1e-10), 1000); // First step var inversePseudoHessian = CreateMatrix.DenseIdentity <double>(initialGuess.Count); var lineSearchDirection = -objective.Gradient; var stepSize = 100 * GradientTolerance / (lineSearchDirection * lineSearchDirection); var previousPoint = objective; LineSearchResult lineSearchResult; try { lineSearchResult = lineSearcher.FindConformingStep(objective, lineSearchDirection, stepSize); } catch (OptimizationException e) { throw new InnerOptimizationException("Line search failed.", e); } catch (ArgumentException e) { throw new InnerOptimizationException("Line search failed.", e); } var candidate = lineSearchResult.FunctionInfoAtMinimum; ValidateGradientAndObjective(candidate); var gradient = candidate.Gradient; var step = candidate.Point - initialGuess; // Subsequent steps Matrix <double> I = CreateMatrix.DiagonalIdentity <double>(initialGuess.Count); int iterations; int totalLineSearchSteps = lineSearchResult.Iterations; int iterationsWithNontrivialLineSearch = lineSearchResult.Iterations > 0 ? 0 : 1; iterations = DoBfgsUpdate(ref currentExitCondition, lineSearcher, ref inversePseudoHessian, ref lineSearchDirection, ref previousPoint, ref lineSearchResult, ref candidate, ref step, ref totalLineSearchSteps, ref iterationsWithNontrivialLineSearch); if (iterations == MaximumIterations && currentExitCondition == ExitCondition.None) { throw new MaximumIterationsException(FormattableString.Invariant($"Maximum iterations ({MaximumIterations}) reached.")); } return(new MinimizationWithLineSearchResult(candidate, iterations, ExitCondition.AbsoluteGradient, totalLineSearchSteps, iterationsWithNontrivialLineSearch)); }
public static MinimizationResult Minimum(IObjectiveFunction objective, Vector <double> initialGuess, double gradientTolerance = 1e-8, int maxIterations = 1000, bool useLineSearch = false) { if (!objective.IsGradientSupported) { throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for Newton minimization."); } if (!objective.IsHessianSupported) { throw new IncompatibleObjectiveException("Hessian not supported in objective function, but required for Newton minimization."); } // Check that we're not already done objective.EvaluateAt(initialGuess); ValidateGradient(objective); if (objective.Gradient.Norm(2.0) < gradientTolerance) { return(new MinimizationResult(objective, 0, ExitCondition.AbsoluteGradient)); } // Set up line search algorithm var lineSearcher = new WeakWolfeLineSearch(1e-4, 0.9, 1e-4, maxIterations: 1000); // Subsequent steps int iterations = 0; int totalLineSearchSteps = 0; int iterationsWithNontrivialLineSearch = 0; bool tmpLineSearch = false; while (objective.Gradient.Norm(2.0) >= gradientTolerance && iterations < maxIterations) { ValidateHessian(objective); var searchDirection = objective.Hessian.LU().Solve(-objective.Gradient); if (searchDirection * objective.Gradient >= 0) { searchDirection = -objective.Gradient; tmpLineSearch = true; } if (useLineSearch || tmpLineSearch) { LineSearchResult result; try { result = lineSearcher.FindConformingStep(objective, searchDirection, 1.0); } catch (Exception e) { throw new InnerOptimizationException("Line search failed.", e); } iterationsWithNontrivialLineSearch += result.Iterations > 0 ? 1 : 0; totalLineSearchSteps += result.Iterations; objective = result.FunctionInfoAtMinimum; } else { objective.EvaluateAt(objective.Point + searchDirection); } ValidateGradient(objective); tmpLineSearch = false; iterations += 1; } if (iterations == maxIterations) { throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", maxIterations)); } return(new MinimizationWithLineSearchResult(objective, iterations, ExitCondition.AbsoluteGradient, totalLineSearchSteps, iterationsWithNontrivialLineSearch)); }