/// <summary> /// L-BFGS two-loop recursion (see Nocedal & Wright 2006, Numerical Optimization, p. 178) /// </summary> /// <param name="direction">The direction.</param> private void ComputeDirection(double[] direction) { // Implemented two-loop Hessian update method. var k = updateInfo.kCounter; var rho = updateInfo.rho; var alpha = updateInfo.alpha; // just to avoid recreating alpha var S = updateInfo.S; var Y = updateInfo.Y; // First loop for (var i = k - 1; i >= 0; i--) { alpha[i] = rho[i] * ArrayMath.InnerProduct(S[i], direction); for (var j = 0; j < dimension; j++) { direction[j] = direction[j] - alpha[i] * Y[i][j]; } } // Second loop for (var i = 0; i < k; i++) { var beta = rho[i] * ArrayMath.InnerProduct(Y[i], direction); for (var j = 0; j < dimension; j++) { direction[j] = direction[j] + S[i][j] * (alpha[i] - beta); } } for (var i = 0; i < dimension; i++) { direction[i] = -direction[i]; } }
/// <summary> /// Gets the function value at the given input vector. /// </summary> /// <param name="x">The input vector.</param> /// <returns>The function value.</returns> public double ValueAt(double[] x) { CheckDimension(x); var value = func.ValueAt(x); if (l2Cost > 0) { value += l2Cost * ArrayMath.InnerProduct(x, x); } return(value); }
private static readonly double RHO = 0.5; // decrease of step size (must be from 0 to 1) /// <summary> /// Backtracking line search. (see Nocedal & Wright 2006, Numerical Optimization, p. 37) /// </summary> /// <param name="function">The function.</param> /// <param name="direction">The direction.</param> /// <param name="lsr">The result.</param> /// <param name="initialStepSize">Initial step size.</param> public static void DoLineSearch( IFunction function, double[] direction, LineSearchResult lsr, double initialStepSize) { var stepSize = initialStepSize; var currFctEvalCount = lsr.FctEvalCount; var x = lsr.NextPoint; var gradAtX = lsr.GradAtNext; var valueAtX = lsr.ValueAtNext; var dimension = x.Length; // Retrieve current points and gradient for array reuse purpose var nextPoint = lsr.CurrPoint; var gradAtNextPoint = lsr.GradAtCurr; double valueAtNextPoint; var dirGradientAtX = ArrayMath.InnerProduct(direction, gradAtX); // To avoid recomputing in the loop var cachedProd = C * dirGradientAtX; while (true) { // Get next point for (var i = 0; i < dimension; i++) { nextPoint[i] = x[i] + direction[i] * stepSize; } // New value valueAtNextPoint = function.ValueAt(nextPoint); currFctEvalCount++; // Check Armijo condition if (valueAtNextPoint <= valueAtX + cachedProd * stepSize) { break; } // Shrink step size stepSize *= RHO; } // Compute and save gradient at the new point Array.Copy(function.GradientAt(nextPoint), 0, gradAtNextPoint, 0, gradAtNextPoint.Length); // Update line search result lsr.SetAll(stepSize, valueAtX, valueAtNextPoint, gradAtX, gradAtNextPoint, x, nextPoint, currFctEvalCount); }