/// <summary>
/// Performs a line search by using the secant method.
/// </summary>
/// <param name="costFunction">The cost function.</param>
/// <param name="theta">The starting point.</param>
/// <param name="direction">The search direction.</param>
/// <param name="previousStepWidth"></param>
/// <returns>The best found minimum point along the <paramref name="direction"/>.</returns>
public override double Minimize(IDifferentiableCostFunction <double> costFunction, Vector <double> theta, Vector <double> direction, double previousStepWidth)
{
var maxLineSearchIteration = MaxLineSearchIterations;
var initialStepSize = LineSearchStepSize;
var epsilonSquare = ErrorToleranceSquared;
// the eta value determines how much the gradient at the current step
// point differs from the gradient that is orthogonal to the search direction.
var gradient = costFunction.Jacobian(theta + initialStepSize * direction);
var eta = gradient * direction;
var previousEta = eta;
// if our step size is small enough to drop this error term under the
// threshold, we terminate the search.
var deltaD = direction * direction;
// the step size used during line search.
// this parameter will be adapted during search according to the change in gradient.
var alpha = -initialStepSize;
// welp
var cumulativeAlpha = 0.0D;
// iteratively find the minimum along the search direction
for (var j = 0; j < maxLineSearchIteration; ++j)
{
// terminate line search if alpha is close enough to zero
var alphaSquare = alpha * alpha;
Debug.Assert(!double.IsNaN(alphaSquare) && !double.IsInfinity(alphaSquare), string.Format("Numerical instability in alpha² with alpha={0}", alpha));
if (alphaSquare * deltaD <= epsilonSquare)
{
break;
}
// by multiplying the current gradient with the search direction,
// we'll end up with a (close to ) zero term if both directions are orthogonal.
// At this point, alpha will be zero (or close to it), which terminates our search.
gradient = costFunction.Jacobian(theta);
eta = gradient * direction;
Debug.Assert(!double.IsInfinity(eta), "!double.IsInfinity(eta)");
// determine the change in orthogonality error
var etaChange = previousEta - eta;
previousEta = eta;
if (etaChange == 0.0D)
{
break;
}
// adjust the step size
alpha = alpha * eta / etaChange;
Debug.Assert(alpha.IsFinite(), "alpha.IsFinite()");
// step to the new location along the line
theta += alpha * direction;
cumulativeAlpha += alpha;
}
return(cumulativeAlpha);
}
/// <summary>
/// Convenience function to obtain often-used function values.
/// </summary>
/// <param name="function">The function.</param>
/// <param name="location">The location.</param>
/// <param name="direction">The direction.</param>
/// <returns>FunctionValues.</returns>
private static FunctionValues GetFunctionValues(IDifferentiableCostFunction <double> function, Vector <double> location, Vector <double> direction)
{
// define the functions
var φ = Getφ(function, location, direction);
var dφ = GetDφ(function, location, direction);
// determine the starting values
var φ0 = φ(0);
var dφ0 = dφ(0);
var Δf0 = function.Jacobian(location);
// bundle the helper
var values = new FunctionValues(
φ: φ, dφ: dφ,
φ0: φ0, dφ0: dφ0,
Δf0: Δf0);
return(values);
}
/// <summary>
/// Gets the directional derivative φ'(α).
/// </summary>
/// <param name="function">The function.</param>
/// <param name="location">The location.</param>
/// <param name="direction">The direction.</param>
/// <returns>Func<System.Double, System.Double>.</returns>
private static Func <double, double> GetDφ([NotNull] IDifferentiableCostFunction <double> function, [NotNull] Vector <double> location, [NotNull] Vector <double> direction)
{
return(alpha => function.Jacobian(location + alpha * direction) * direction);
}
