/// <summary>
/// Performs a line search by using the secant method.
/// </summary>
/// <param name="costFunction">The cost function.</param>
/// <param name="theta">The starting point.</param>
/// <param name="direction">The search direction.</param>
/// <param name="previousStepWidth"></param>
/// <returns>The best found minimum point along the <paramref name="direction"/>.</returns>
public override double Minimize(IDifferentiableCostFunction <double> costFunction, Vector <double> theta, Vector <double> direction, double previousStepWidth)
{
var maxLineSearchIteration = MaxLineSearchIterations;
var initialStepSize = LineSearchStepSize;
var epsilonSquare = ErrorToleranceSquared;
// the eta value determines how much the gradient at the current step
// point differs from the gradient that is orthogonal to the search direction.
var gradient = costFunction.Jacobian(theta + initialStepSize * direction);
var eta = gradient * direction;
var previousEta = eta;
// if our step size is small enough to drop this error term under the
// threshold, we terminate the search.
var deltaD = direction * direction;
// the step size used during line search.
// this parameter will be adapted during search according to the change in gradient.
var alpha = -initialStepSize;
// welp
var cumulativeAlpha = 0.0D;
// iteratively find the minimum along the search direction
for (var j = 0; j < maxLineSearchIteration; ++j)
{
// terminate line search if alpha is close enough to zero
var alphaSquare = alpha * alpha;
Debug.Assert(!double.IsNaN(alphaSquare) && !double.IsInfinity(alphaSquare), string.Format("Numerical instability in alpha² with alpha={0}", alpha));
if (alphaSquare * deltaD <= epsilonSquare)
{
break;
}
// by multiplying the current gradient with the search direction,
// we'll end up with a (close to ) zero term if both directions are orthogonal.
// At this point, alpha will be zero (or close to it), which terminates our search.
gradient = costFunction.Jacobian(theta);
eta = gradient * direction;
Debug.Assert(!double.IsInfinity(eta), "!double.IsInfinity(eta)");
// determine the change in orthogonality error
var etaChange = previousEta - eta;
previousEta = eta;
if (etaChange == 0.0D)
{
break;
}
// adjust the step size
alpha = alpha * eta / etaChange;
Debug.Assert(alpha.IsFinite(), "alpha.IsFinite()");
// step to the new location along the line
theta += alpha * direction;
cumulativeAlpha += alpha;
}
return(cumulativeAlpha);
}
/// <summary>
/// Convenience function to obtain often-used function values.
/// </summary>
/// <param name="function">The function.</param>
/// <param name="location">The location.</param>
/// <param name="direction">The direction.</param>
/// <returns>FunctionValues.</returns>
private static FunctionValues GetFunctionValues(IDifferentiableCostFunction <double> function, Vector <double> location, Vector <double> direction)
{
// define the functions
var φ = Getφ(function, location, direction);
var dφ = GetDφ(function, location, direction);
// determine the starting values
var φ0 = φ(0);
var dφ0 = dφ(0);
var Δf0 = function.Jacobian(location);
// bundle the helper
var values = new FunctionValues(
φ: φ, dφ: dφ,
φ0: φ0, dφ0: dφ0,
Δf0: Δf0);
return(values);
}
/// <summary>
/// Minimizes the <paramref name="function" /> by performing a line search along the <paramref name="direction" />, starting from the given <paramref name="location" />.
/// </summary>
/// <param name="function">The cost function.</param>
/// <param name="location">The starting point.</param>
/// <param name="direction">The search direction.</param>
/// <param name="previousStepWidth">The previous step width α. In the initial iteration, this value should be <c>0.0D</c>.</param>
/// <returns>The best found minimum point along the <paramref name="direction" />.</returns>
/// <exception cref="System.NotImplementedException">aww yeah</exception>
public double Minimize(IDifferentiableCostFunction <double> function, Vector <double> location, Vector <double> direction, double previousStepWidth)
{
// prefetch
var γ = _γ;
// convenience function for the evaluation
var values = GetFunctionValues(function, location, direction);
// find a starting point and check if that solution is already good enough
var c = DetermineInitialSearchPoint(previousStepWidth, location, ref values);
if (ShouldTerminate(c, ref values))
{
return(c);
}
// bracket the initial starting point
var bracket = BracketStartingPoint(c, values);
if (ShouldTerminate(bracket.Start, ref values))
{
return(bracket.Start);
}
if (ShouldTerminate(bracket.End, ref values))
{
return(bracket.End);
}
// iterate along the line
var maxIterations = _maxIterations;
for (var i = 0; i < maxIterations; ++i)
{
// L1: perform a double secant step to optimize the search interval
var candidate = DoubleSecant(bracket, ref values);
if (ShouldTerminate(candidate.Start, ref values))
{
return(candidate.Start);
}
if (ShouldTerminate(candidate.End, ref values))
{
return(candidate.End);
}
// L2
if ((candidate.End - candidate.Start) > γ * (bracket.End - bracket.Start))
{
// find a new midpoint
c = (candidate.Start + candidate.End) / 2;
if (ShouldTerminate(c, ref values))
{
return(c);
}
// update the bracketing interval
candidate = UpdateBracketing(candidate, c, ref values);
}
// L3: wrap around
bracket = candidate;
}
// welp
return(c);
}
/// <summary>
/// Gets the directional derivative φ'(α).
/// </summary>
/// <param name="function">The function.</param>
/// <param name="location">The location.</param>
/// <param name="direction">The direction.</param>
/// <returns>Func<System.Double, System.Double>.</returns>
private static Func <double, double> GetDφ([NotNull] IDifferentiableCostFunction <double> function, [NotNull] Vector <double> location, [NotNull] Vector <double> direction)
{
return(alpha => function.Jacobian(location + alpha * direction) * direction);
}