/// <summary>
/// Finds a minimum of a function by the BFGS quasi-Newton method
/// This uses the function and it's gradient (partial derivatives in each direction) and approximates the Hessian
/// </summary>
/// <param name="initialGuess">An initial guess</param>
/// <param name="functionValue">Evaluates the function at a point</param>
/// <param name="functionGradient">Evaluates the gradient of the function at a point</param>
/// <returns>The minimum found</returns>
public static Vector <double> Solve(Vector initialGuess, Func <Vector <double>, double> functionValue, Func <Vector <double>, Vector <double> > functionGradient)
{
var objectiveFunction = ObjectiveFunction.Gradient(functionValue, functionGradient);
objectiveFunction.EvaluateAt(initialGuess);
int dim = initialGuess.Count;
int iter = 0;
// H represents the approximation of the inverse hessian matrix
// it is updated via the Sherman–Morrison formula (http://en.wikipedia.org/wiki/Sherman%E2%80%93Morrison_formula)
Matrix <double> H = DenseMatrix.CreateIdentity(dim);
Vector <double> x = initialGuess;
Vector <double> x_old = x;
Vector <double> grad;
WolfeLineSearch wolfeLineSearch = new WeakWolfeLineSearch(1e-4, 0.9, 1e-5, 200);
do
{
// search along the direction of the gradient
grad = objectiveFunction.Gradient;
Vector <double> p = -1 * H * grad;
var lineSearchResult = wolfeLineSearch.FindConformingStep(objectiveFunction, p, 1.0);
double rate = lineSearchResult.FinalStep;
x = x + rate * p;
Vector <double> grad_old = grad;
// update the gradient
objectiveFunction.EvaluateAt(x);
grad = objectiveFunction.Gradient;// functionGradient(x);
Vector <double> s = x - x_old;
Vector <double> y = grad - grad_old;
double rho = 1.0 / (y * s);
if (iter == 0)
{
// set up an initial hessian
H = (y * s) / (y * y) * DenseMatrix.CreateIdentity(dim);
}
var sM = s.ToColumnMatrix();
var yM = y.ToColumnMatrix();
// Update the estimate of the hessian
H = H
- rho * (sM * (yM.TransposeThisAndMultiply(H)) + (H * yM).TransposeAndMultiply(sM))
+ rho * rho * (y.DotProduct(H * y) + 1.0 / rho) * (sM.TransposeAndMultiply(sM));
x_old = x;
iter++;
}while ((grad.InfinityNorm() > GradientTolerance) && (iter < MaxIterations));
return(x);
}
public static MinimizationResult Minimum(IObjectiveFunction objective, Vector <double> initialGuess, double gradientTolerance = 1e-8, int maxIterations = 1000)
{
if (!objective.IsGradientSupported)
{
throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for ConjugateGradient minimization.");
}
objective.EvaluateAt(initialGuess);
var gradient = objective.Gradient;
ValidateGradient(objective);
// Check that we're not already done
if (gradient.Norm(2.0) < gradientTolerance)
{
return(new MinimizationResult(objective, 0, ExitCondition.AbsoluteGradient));
}
// Set up line search algorithm
var lineSearcher = new WeakWolfeLineSearch(1e-4, 0.1, 1e-4, 1000);
// First step
var steepestDirection = -gradient;
var searchDirection = steepestDirection;
double initialStepSize = 100 * gradientTolerance / (gradient * gradient);
LineSearchResult result;
try
{
result = lineSearcher.FindConformingStep(objective, searchDirection, initialStepSize);
}
catch (Exception e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
objective = result.FunctionInfoAtMinimum;
ValidateGradient(objective);
double stepSize = result.FinalStep;
// Subsequent steps
int iterations = 1;
int totalLineSearchSteps = result.Iterations;
int iterationsWithNontrivialLineSearch = result.Iterations > 0 ? 0 : 1;
int steepestDescentResets = 0;
while (objective.Gradient.Norm(2.0) >= gradientTolerance && iterations < maxIterations)
{
var previousSteepestDirection = steepestDirection;
steepestDirection = -objective.Gradient;
var searchDirectionAdjuster = Math.Max(0, steepestDirection * (steepestDirection - previousSteepestDirection) / (previousSteepestDirection * previousSteepestDirection));
searchDirection = steepestDirection + searchDirectionAdjuster * searchDirection;
if (searchDirection * objective.Gradient >= 0)
{
searchDirection = steepestDirection;
steepestDescentResets += 1;
}
try
{
result = lineSearcher.FindConformingStep(objective, searchDirection, stepSize);
}
catch (Exception e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
iterationsWithNontrivialLineSearch += result.Iterations == 0 ? 1 : 0;
totalLineSearchSteps += result.Iterations;
stepSize = result.FinalStep;
objective = result.FunctionInfoAtMinimum;
iterations += 1;
}
//if (iterations == maxIterations)
//{
// throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", maxIterations));
//}
return(new MinimizationWithLineSearchResult(objective, iterations, ExitCondition.AbsoluteGradient, totalLineSearchSteps, iterationsWithNontrivialLineSearch));
}
/// <summary>
/// Find the minimum of the objective function given lower and upper bounds
/// </summary>
/// <param name="objective">The objective function, must support a gradient</param>
/// <param name="initialGuess">The initial guess</param>
/// <returns>The MinimizationResult which contains the minimum and the ExitCondition</returns>
public MinimizationResult FindMinimum(IObjectiveFunction objective, Vector <double> initialGuess)
{
if (!objective.IsGradientSupported)
{
throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for L-BFGS minimization.");
}
objective.EvaluateAt(initialGuess);
ValidateGradientAndObjective(objective);
// Check that we're not already done
ExitCondition currentExitCondition = ExitCriteriaSatisfied(objective, null, 0);
if (currentExitCondition != ExitCondition.None)
{
return(new MinimizationResult(objective, 0, currentExitCondition));
}
// Set up line search algorithm
var lineSearcher = new WeakWolfeLineSearch(1e-4, 0.9, Math.Max(ParameterTolerance, 1e-10), 1000);
// First step
var lineSearchDirection = -objective.Gradient;
var stepSize = 100 * GradientTolerance / (lineSearchDirection * lineSearchDirection);
var previousPoint = objective;
LineSearchResult lineSearchResult;
try
{
lineSearchResult = lineSearcher.FindConformingStep(objective, lineSearchDirection, stepSize);
}
catch (OptimizationException e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
catch (ArgumentException e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
var candidate = lineSearchResult.FunctionInfoAtMinimum;
ValidateGradientAndObjective(candidate);
var gradient = candidate.Gradient;
var step = candidate.Point - initialGuess;
var yk = candidate.Gradient - previousPoint.Gradient;
var ykhistory = new List <Vector <double> >()
{
yk
};
var skhistory = new List <Vector <double> >()
{
step
};
var rhokhistory = new List <double>()
{
1.0 / yk.DotProduct(step)
};
// Subsequent steps
int iterations = 1;
int totalLineSearchSteps = lineSearchResult.Iterations;
int iterationsWithNontrivialLineSearch = lineSearchResult.Iterations > 0 ? 0 : 1;
previousPoint = candidate;
while (iterations++ < MaximumIterations && previousPoint.Gradient.Norm(2) >= GradientTolerance)
{
lineSearchDirection = -ApplyLbfgsUpdate(previousPoint, ykhistory, skhistory, rhokhistory);
var directionalDerivative = previousPoint.Gradient.DotProduct(lineSearchDirection);
if (directionalDerivative > 0)
{
throw new InnerOptimizationException("Direction is not a descent direction.");
}
try
{
lineSearchResult = lineSearcher.FindConformingStep(previousPoint, lineSearchDirection, 1.0);
}
catch (OptimizationException e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
catch (ArgumentException e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
iterationsWithNontrivialLineSearch += lineSearchResult.Iterations > 0 ? 1 : 0;
totalLineSearchSteps += lineSearchResult.Iterations;
candidate = lineSearchResult.FunctionInfoAtMinimum;
currentExitCondition = ExitCriteriaSatisfied(candidate, previousPoint, iterations);
if (currentExitCondition != ExitCondition.None)
{
break;
}
step = candidate.Point - previousPoint.Point;
yk = candidate.Gradient - previousPoint.Gradient;
ykhistory.Add(yk);
skhistory.Add(step);
rhokhistory.Add(1.0 / yk.DotProduct(step));
previousPoint = candidate;
if (ykhistory.Count > Memory)
{
ykhistory.RemoveAt(0);
skhistory.RemoveAt(0);
rhokhistory.RemoveAt(0);
}
}
if (iterations == MaximumIterations && currentExitCondition == ExitCondition.None)
{
throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", MaximumIterations));
}
return(new MinimizationWithLineSearchResult(candidate, iterations, ExitCondition.AbsoluteGradient, totalLineSearchSteps, iterationsWithNontrivialLineSearch));
}
/// <summary>
/// Find the minimum of the objective function given lower and upper bounds
/// </summary>
/// <param name="objective">The objective function, must support a gradient</param>
/// <param name="initialGuess">The initial guess</param>
/// <returns>The MinimizationResult which contains the minimum and the ExitCondition</returns>
public MinimizationResult FindMinimum(IObjectiveFunction objective, Vector <double> initialGuess)
{
if (!objective.IsGradientSupported)
{
throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for BFGS minimization.");
}
objective.EvaluateAt(initialGuess);
ValidateGradientAndObjective(objective);
// Check that we're not already done
ExitCondition currentExitCondition = ExitCriteriaSatisfied(objective, null, 0);
if (currentExitCondition != ExitCondition.None)
{
return(new MinimizationResult(objective, 0, currentExitCondition));
}
// Set up line search algorithm
var lineSearcher = new WeakWolfeLineSearch(1e-4, 0.9, Math.Max(ParameterTolerance, 1e-10), 1000);
// First step
var inversePseudoHessian = CreateMatrix.DenseIdentity <double>(initialGuess.Count);
var lineSearchDirection = -objective.Gradient;
var stepSize = 100 * GradientTolerance / (lineSearchDirection * lineSearchDirection);
var previousPoint = objective;
LineSearchResult lineSearchResult;
try
{
lineSearchResult = lineSearcher.FindConformingStep(objective, lineSearchDirection, stepSize);
}
catch (OptimizationException e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
catch (ArgumentException e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
var candidate = lineSearchResult.FunctionInfoAtMinimum;
ValidateGradientAndObjective(candidate);
var gradient = candidate.Gradient;
var step = candidate.Point - initialGuess;
// Subsequent steps
Matrix <double> I = CreateMatrix.DiagonalIdentity <double>(initialGuess.Count);
int iterations;
int totalLineSearchSteps = lineSearchResult.Iterations;
int iterationsWithNontrivialLineSearch = lineSearchResult.Iterations > 0 ? 0 : 1;
iterations = DoBfgsUpdate(ref currentExitCondition, lineSearcher, ref inversePseudoHessian, ref lineSearchDirection, ref previousPoint, ref lineSearchResult, ref candidate, ref step, ref totalLineSearchSteps, ref iterationsWithNontrivialLineSearch);
if (iterations == MaximumIterations && currentExitCondition == ExitCondition.None)
{
throw new MaximumIterationsException(FormattableString.Invariant($"Maximum iterations ({MaximumIterations}) reached."));
}
return(new MinimizationWithLineSearchResult(candidate, iterations, ExitCondition.AbsoluteGradient, totalLineSearchSteps, iterationsWithNontrivialLineSearch));
}
public static MinimizationResult Minimum(IObjectiveFunction objective, Vector <double> initialGuess, double gradientTolerance = 1e-8, int maxIterations = 1000, bool useLineSearch = false)
{
if (!objective.IsGradientSupported)
{
throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for Newton minimization.");
}
if (!objective.IsHessianSupported)
{
throw new IncompatibleObjectiveException("Hessian not supported in objective function, but required for Newton minimization.");
}
// Check that we're not already done
objective.EvaluateAt(initialGuess);
ValidateGradient(objective);
if (objective.Gradient.Norm(2.0) < gradientTolerance)
{
return(new MinimizationResult(objective, 0, ExitCondition.AbsoluteGradient));
}
// Set up line search algorithm
var lineSearcher = new WeakWolfeLineSearch(1e-4, 0.9, 1e-4, maxIterations: 1000);
// Subsequent steps
int iterations = 0;
int totalLineSearchSteps = 0;
int iterationsWithNontrivialLineSearch = 0;
bool tmpLineSearch = false;
while (objective.Gradient.Norm(2.0) >= gradientTolerance && iterations < maxIterations)
{
ValidateHessian(objective);
var searchDirection = objective.Hessian.LU().Solve(-objective.Gradient);
if (searchDirection * objective.Gradient >= 0)
{
searchDirection = -objective.Gradient;
tmpLineSearch = true;
}
if (useLineSearch || tmpLineSearch)
{
LineSearchResult result;
try
{
result = lineSearcher.FindConformingStep(objective, searchDirection, 1.0);
}
catch (Exception e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
iterationsWithNontrivialLineSearch += result.Iterations > 0 ? 1 : 0;
totalLineSearchSteps += result.Iterations;
objective = result.FunctionInfoAtMinimum;
}
else
{
objective.EvaluateAt(objective.Point + searchDirection);
}
ValidateGradient(objective);
tmpLineSearch = false;
iterations += 1;
}
if (iterations == maxIterations)
{
throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", maxIterations));
}
return(new MinimizationWithLineSearchResult(objective, iterations, ExitCondition.AbsoluteGradient, totalLineSearchSteps, iterationsWithNontrivialLineSearch));
}
