/// <summary> /// Performs a line search by using the secant method. /// </summary> /// <param name="costFunction">The cost function.</param> /// <param name="theta">The starting point.</param> /// <param name="direction">The search direction.</param> /// <param name="previousStepWidth"></param> /// <returns>The best found minimum point along the <paramref name="direction"/>.</returns> public override double Minimize(IDifferentiableCostFunction <double> costFunction, Vector <double> theta, Vector <double> direction, double previousStepWidth) { var maxLineSearchIteration = MaxLineSearchIterations; var initialStepSize = LineSearchStepSize; var epsilonSquare = ErrorToleranceSquared; // the eta value determines how much the gradient at the current step // point differs from the gradient that is orthogonal to the search direction. var gradient = costFunction.Jacobian(theta + initialStepSize * direction); var eta = gradient * direction; var previousEta = eta; // if our step size is small enough to drop this error term under the // threshold, we terminate the search. var deltaD = direction * direction; // the step size used during line search. // this parameter will be adapted during search according to the change in gradient. var alpha = -initialStepSize; // welp var cumulativeAlpha = 0.0D; // iteratively find the minimum along the search direction for (var j = 0; j < maxLineSearchIteration; ++j) { // terminate line search if alpha is close enough to zero var alphaSquare = alpha * alpha; Debug.Assert(!double.IsNaN(alphaSquare) && !double.IsInfinity(alphaSquare), string.Format("Numerical instability in alpha² with alpha={0}", alpha)); if (alphaSquare * deltaD <= epsilonSquare) { break; } // by multiplying the current gradient with the search direction, // we'll end up with a (close to ) zero term if both directions are orthogonal. // At this point, alpha will be zero (or close to it), which terminates our search. gradient = costFunction.Jacobian(theta); eta = gradient * direction; Debug.Assert(!double.IsInfinity(eta), "!double.IsInfinity(eta)"); // determine the change in orthogonality error var etaChange = previousEta - eta; previousEta = eta; if (etaChange == 0.0D) { break; } // adjust the step size alpha = alpha * eta / etaChange; Debug.Assert(alpha.IsFinite(), "alpha.IsFinite()"); // step to the new location along the line theta += alpha * direction; cumulativeAlpha += alpha; } return(cumulativeAlpha); }
/// <summary> /// Convenience function to obtain often-used function values. /// </summary> /// <param name="function">The function.</param> /// <param name="location">The location.</param> /// <param name="direction">The direction.</param> /// <returns>FunctionValues.</returns> private static FunctionValues GetFunctionValues(IDifferentiableCostFunction <double> function, Vector <double> location, Vector <double> direction) { // define the functions var φ = Getφ(function, location, direction); var dφ = GetDφ(function, location, direction); // determine the starting values var φ0 = φ(0); var dφ0 = dφ(0); var Δf0 = function.Jacobian(location); // bundle the helper var values = new FunctionValues( φ: φ, dφ: dφ, φ0: φ0, dφ0: dφ0, Δf0: Δf0); return(values); }
/// <summary> /// Minimizes the <paramref name="function" /> by performing a line search along the <paramref name="direction" />, starting from the given <paramref name="location" />. /// </summary> /// <param name="function">The cost function.</param> /// <param name="location">The starting point.</param> /// <param name="direction">The search direction.</param> /// <param name="previousStepWidth">The previous step width α. In the initial iteration, this value should be <c>0.0D</c>.</param> /// <returns>The best found minimum point along the <paramref name="direction" />.</returns> /// <exception cref="System.NotImplementedException">aww yeah</exception> public double Minimize(IDifferentiableCostFunction <double> function, Vector <double> location, Vector <double> direction, double previousStepWidth) { // prefetch var γ = _γ; // convenience function for the evaluation var values = GetFunctionValues(function, location, direction); // find a starting point and check if that solution is already good enough var c = DetermineInitialSearchPoint(previousStepWidth, location, ref values); if (ShouldTerminate(c, ref values)) { return(c); } // bracket the initial starting point var bracket = BracketStartingPoint(c, values); if (ShouldTerminate(bracket.Start, ref values)) { return(bracket.Start); } if (ShouldTerminate(bracket.End, ref values)) { return(bracket.End); } // iterate along the line var maxIterations = _maxIterations; for (var i = 0; i < maxIterations; ++i) { // L1: perform a double secant step to optimize the search interval var candidate = DoubleSecant(bracket, ref values); if (ShouldTerminate(candidate.Start, ref values)) { return(candidate.Start); } if (ShouldTerminate(candidate.End, ref values)) { return(candidate.End); } // L2 if ((candidate.End - candidate.Start) > γ * (bracket.End - bracket.Start)) { // find a new midpoint c = (candidate.Start + candidate.End) / 2; if (ShouldTerminate(c, ref values)) { return(c); } // update the bracketing interval candidate = UpdateBracketing(candidate, c, ref values); } // L3: wrap around bracket = candidate; } // welp return(c); }
/// <summary> /// Gets the directional derivative φ'(α). /// </summary> /// <param name="function">The function.</param> /// <param name="location">The location.</param> /// <param name="direction">The direction.</param> /// <returns>Func<System.Double, System.Double>.</returns> private static Func <double, double> GetDφ([NotNull] IDifferentiableCostFunction <double> function, [NotNull] Vector <double> location, [NotNull] Vector <double> direction) { return(alpha => function.Jacobian(location + alpha * direction) * direction); }