/// <summary>
/// Finds a minimum of a function by the BFGS quasi-Newton method
/// This uses the function and it's gradient (partial derivatives in each direction) and approximates the Hessian
/// </summary>
/// <param name="initialGuess">An initial guess</param>
/// <param name="functionValue">Evaluates the function at a point</param>
/// <param name="functionGradient">Evaluates the gradient of the function at a point</param>
/// <returns>The minimum found</returns>
public static Vector <double> Solve(Vector initialGuess, Func <Vector <double>, double> functionValue, Func <Vector <double>, Vector <double> > functionGradient)
{
var objectiveFunction = ObjectiveFunction.Gradient(functionValue, functionGradient);
objectiveFunction.EvaluateAt(initialGuess);
int dim = initialGuess.Count;
int iter = 0;
// H represents the approximation of the inverse hessian matrix
// it is updated via the Sherman–Morrison formula (http://en.wikipedia.org/wiki/Sherman%E2%80%93Morrison_formula)
Matrix <double> H = DenseMatrix.CreateIdentity(dim);
Vector <double> x = initialGuess;
Vector <double> x_old = x;
Vector <double> grad;
WolfeLineSearch wolfeLineSearch = new WeakWolfeLineSearch(1e-4, 0.9, 1e-5, 200);
do
{
// search along the direction of the gradient
grad = objectiveFunction.Gradient;
Vector <double> p = -1 * H * grad;
var lineSearchResult = wolfeLineSearch.FindConformingStep(objectiveFunction, p, 1.0);
double rate = lineSearchResult.FinalStep;
x = x + (rate * p);
Vector <double> grad_old = grad;
// update the gradient
objectiveFunction.EvaluateAt(x);
grad = objectiveFunction.Gradient;// functionGradient(x);
Vector <double> s = x - x_old;
Vector <double> y = grad - grad_old;
double rho = 1.0 / (y * s);
if (iter == 0)
{
// set up an initial hessian
H = ((y * s) / (y * y)) * DenseMatrix.CreateIdentity(dim);
}
var sM = s.ToColumnMatrix();
var yM = y.ToColumnMatrix();
// Update the estimate of the hessian
H = (H
- (rho * ((sM * (yM.TransposeThisAndMultiply(H))) + (H * yM).TransposeAndMultiply(sM))))
+ (rho * rho * (y.DotProduct(H * y) + (1.0 / rho)) * (sM.TransposeAndMultiply(sM)));
x_old = x;
iter++;
}while ((grad.InfinityNorm() > GradientTolerance) && (iter < MaxIterations));
return(x);
}
/// <summary>
/// Find the minimum of the objective function given lower and upper bounds
/// </summary>
/// <param name="objective">The objective function, must support a gradient</param>
/// <param name="initialGuess">The initial guess</param>
/// <returns>The MinimizationResult which contains the minimum and the ExitCondition</returns>
public MinimizationResult FindMinimum(IObjectiveFunction objective, Vector <double> initialGuess)
{
if (!objective.IsGradientSupported)
{
throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for BFGS minimization.");
}
objective.EvaluateAt(initialGuess);
ValidateGradientAndObjective(objective);
// Check that we're not already done
ExitCondition currentExitCondition = ExitCriteriaSatisfied(objective, null, 0);
if (currentExitCondition != ExitCondition.None)
{
return(new MinimizationResult(objective, 0, currentExitCondition));
}
// Set up line search algorithm
var lineSearcher = new WeakWolfeLineSearch(1e-4, 0.9, Math.Max(ParameterTolerance, 1e-10), 1000);
// First step
var inversePseudoHessian = CreateMatrix.DenseIdentity <double>(initialGuess.Count);
var lineSearchDirection = -objective.Gradient;
var stepSize = 100 * GradientTolerance / (lineSearchDirection * lineSearchDirection);
var previousPoint = objective;
LineSearchResult lineSearchResult;
try
{
lineSearchResult = lineSearcher.FindConformingStep(objective, lineSearchDirection, stepSize);
}
catch (OptimizationException e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
catch (ArgumentException e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
var candidate = lineSearchResult.FunctionInfoAtMinimum;
ValidateGradientAndObjective(candidate);
var gradient = candidate.Gradient;
var step = candidate.Point - initialGuess;
// Subsequent steps
Matrix <double> I = CreateMatrix.DiagonalIdentity <double>(initialGuess.Count);
int iterations;
int totalLineSearchSteps = lineSearchResult.Iterations;
int iterationsWithNontrivialLineSearch = lineSearchResult.Iterations > 0 ? 0 : 1;
iterations = DoBfgsUpdate(ref currentExitCondition, lineSearcher, ref inversePseudoHessian, ref lineSearchDirection, ref previousPoint, ref lineSearchResult, ref candidate, ref step, ref totalLineSearchSteps, ref iterationsWithNontrivialLineSearch);
if (iterations == MaximumIterations && currentExitCondition == ExitCondition.None)
{
throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", MaximumIterations));
}
return(new MinimizationWithLineSearchResult(candidate, iterations, ExitCondition.AbsoluteGradient, totalLineSearchSteps, iterationsWithNontrivialLineSearch));
}
public MinimizationResult FindMinimum(IObjectiveFunction objective, Vector <double> initialGuess)
{
if (!objective.IsGradientSupported)
{
throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for Newton minimization.");
}
if (!objective.IsHessianSupported)
{
throw new IncompatibleObjectiveException("Hessian not supported in objective function, but required for Newton minimization.");
}
// Check that we're not already done
objective.EvaluateAt(initialGuess);
ValidateGradient(objective);
if (ExitCriteriaSatisfied(objective.Gradient))
{
return(new MinimizationResult(objective, 0, ExitCondition.AbsoluteGradient));
}
// Set up line search algorithm
var lineSearcher = new WeakWolfeLineSearch(1e-4, 0.9, 1e-4, maxIterations: 1000);
// Subsequent steps
int iterations = 0;
int totalLineSearchSteps = 0;
int iterationsWithNontrivialLineSearch = 0;
bool tmpLineSearch = false;
while (!ExitCriteriaSatisfied(objective.Gradient) && iterations < MaximumIterations)
{
ValidateHessian(objective);
var searchDirection = objective.Hessian.LU().Solve(-objective.Gradient);
if (searchDirection * objective.Gradient >= 0)
{
searchDirection = -objective.Gradient;
tmpLineSearch = true;
}
if (UseLineSearch || tmpLineSearch)
{
LineSearchResult result;
try
{
result = lineSearcher.FindConformingStep(objective, searchDirection, 1.0);
}
catch (Exception e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
iterationsWithNontrivialLineSearch += result.Iterations > 0 ? 1 : 0;
totalLineSearchSteps += result.Iterations;
objective = result.FunctionInfoAtMinimum;
}
else
{
objective.EvaluateAt(objective.Point + searchDirection);
}
ValidateGradient(objective);
tmpLineSearch = false;
iterations += 1;
}
if (iterations == MaximumIterations)
{
throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", MaximumIterations));
}
return(new MinimizationWithLineSearchResult(objective, iterations, ExitCondition.AbsoluteGradient, totalLineSearchSteps, iterationsWithNontrivialLineSearch));
}
public MinimizationOutput FindMinimum(IObjectiveFunction objective, Vector <double> initial_guess)
{
if (!objective.GradientSupported)
{
throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for Newton minimization.");
}
if (!objective.HessianSupported)
{
throw new IncompatibleObjectiveException("Hessian not supported in objective function, but required for Newton minimization.");
}
if (!(objective is ObjectiveChecker))
{
objective = new ObjectiveChecker(objective, this.ValidateObjective, this.ValidateGradient, this.ValidateHessian);
}
IEvaluation initial_eval = objective.Evaluate(initial_guess);
// Check that we're not already done
if (this.ExitCriteriaSatisfied(initial_guess, initial_eval.Gradient))
{
return(new MinimizationOutput(initial_eval, 0, ExitCondition.AbsoluteGradient));
}
// Set up line search algorithm
var line_searcher = new WeakWolfeLineSearch(1e-4, 0.9, 1e-4, max_iterations: 1000);
// Declare state variables
IEvaluation candidate_point = initial_eval;
Vector <double> search_direction;
LineSearchOutput result;
// Subsequent steps
int iterations = 0;
int total_line_search_steps = 0;
int iterations_with_nontrivial_line_search = 0;
int steepest_descent_resets = 0;
bool tmp_line_search = false;
while (!this.ExitCriteriaSatisfied(candidate_point.Point, candidate_point.Gradient) && iterations < this.MaximumIterations)
{
search_direction = candidate_point.Hessian.LU().Solve(-candidate_point.Gradient);
if (search_direction * candidate_point.Gradient >= 0)
{
search_direction = -candidate_point.Gradient;
steepest_descent_resets += 1;
tmp_line_search = true;
}
if (this.UseLineSearch || tmp_line_search)
{
try
{
result = line_searcher.FindConformingStep(objective, candidate_point, search_direction, 1.0);
}
catch (Exception e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
iterations_with_nontrivial_line_search += result.Iterations > 0 ? 1 : 0;
total_line_search_steps += result.Iterations;
candidate_point = result.FunctionInfoAtMinimum;
}
else
{
candidate_point = objective.Evaluate(candidate_point.Point + search_direction);
}
tmp_line_search = false;
iterations += 1;
}
if (iterations == this.MaximumIterations)
{
throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", this.MaximumIterations));
}
return(new MinimizationWithLineSearchOutput(candidate_point, iterations, ExitCondition.AbsoluteGradient, total_line_search_steps, iterations_with_nontrivial_line_search));
}
public MinimizationResult FindMinimum(IObjectiveFunction objective, Vector <double> initialGuess)
{
if (!objective.IsGradientSupported)
{
throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for ConjugateGradient minimization.");
}
objective.EvaluateAt(initialGuess);
var gradient = objective.Gradient;
ValidateGradient(objective);
// Check that we're not already done
if (ExitCriteriaSatisfied(initialGuess, gradient))
{
return(new MinimizationResult(objective, 0, ExitCondition.AbsoluteGradient));
}
// Set up line search algorithm
var lineSearcher = new WeakWolfeLineSearch(1e-4, 0.1, 1e-4, 1000);
// First step
var steepestDirection = -gradient;
var searchDirection = steepestDirection;
double initialStepSize = 100 * GradientTolerance / (gradient * gradient);
LineSearchResult result;
try
{
result = lineSearcher.FindConformingStep(objective, searchDirection, initialStepSize);
}
catch (Exception e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
objective = result.FunctionInfoAtMinimum;
ValidateGradient(objective);
double stepSize = result.FinalStep;
// Subsequent steps
int iterations = 1;
int totalLineSearchSteps = result.Iterations;
int iterationsWithNontrivialLineSearch = result.Iterations > 0 ? 0 : 1;
int steepestDescentResets = 0;
while (!ExitCriteriaSatisfied(objective.Point, objective.Gradient) && iterations < MaximumIterations)
{
var previousSteepestDirection = steepestDirection;
steepestDirection = -objective.Gradient;
var searchDirectionAdjuster = Math.Max(0, steepestDirection * (steepestDirection - previousSteepestDirection) / (previousSteepestDirection * previousSteepestDirection));
searchDirection = steepestDirection + searchDirectionAdjuster * searchDirection;
if (searchDirection * objective.Gradient >= 0)
{
searchDirection = steepestDirection;
steepestDescentResets += 1;
}
try
{
result = lineSearcher.FindConformingStep(objective, searchDirection, stepSize);
}
catch (Exception e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
iterationsWithNontrivialLineSearch += result.Iterations == 0 ? 1 : 0;
totalLineSearchSteps += result.Iterations;
stepSize = result.FinalStep;
objective = result.FunctionInfoAtMinimum;
iterations += 1;
}
if (iterations == MaximumIterations)
{
throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", MaximumIterations));
}
return(new MinimizationWithLineSearchResult(objective, iterations, ExitCondition.AbsoluteGradient, totalLineSearchSteps, iterationsWithNontrivialLineSearch));
}
public MinimizationOutput FindMinimum(IObjectiveFunction objective, Vector <double> initial_guess)
{
if (!objective.GradientSupported)
{
throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for ConjugateGradient minimization.");
}
if (!(objective is ObjectiveChecker))
{
objective = new ObjectiveChecker(objective, this.ValidateObjective, this.ValidateGradient, null);
}
IEvaluation initial_eval = objective.Evaluate(initial_guess);
var gradient = initial_eval.Gradient;
// Check that we're not already done
if (this.ExitCriteriaSatisfied(initial_guess, gradient))
{
return(new MinimizationOutput(initial_eval, 0, ExitCondition.AbsoluteGradient));
}
// Set up line search algorithm
var line_searcher = new WeakWolfeLineSearch(1e-4, 0.1, 1e-4, max_iterations: 1000);
// Declare state variables
IEvaluation candidate_point;
Vector <double> steepest_direction, previous_steepest_direction, search_direction;
// First step
steepest_direction = -gradient;
search_direction = steepest_direction;
double initial_step_size = 100 * this.GradientTolerance / (gradient * gradient);
LineSearchOutput result;
try
{
result = line_searcher.FindConformingStep(objective, initial_eval, search_direction, initial_step_size);
}
catch (Exception e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
candidate_point = result.FunctionInfoAtMinimum;
double step_size = result.FinalStep;
// Subsequent steps
int iterations = 1;
int total_line_search_steps = result.Iterations;
int iterations_with_nontrivial_line_search = result.Iterations > 0 ? 0 : 1;
int steepest_descent_resets = 0;
while (!this.ExitCriteriaSatisfied(candidate_point.Point, candidate_point.Gradient) && iterations < this.MaximumIterations)
{
previous_steepest_direction = steepest_direction;
steepest_direction = -candidate_point.Gradient;
var search_direction_adjuster = Math.Max(0, steepest_direction * (steepest_direction - previous_steepest_direction) / (previous_steepest_direction * previous_steepest_direction));
//double prev_grad_mag = previous_steepest_direction*previous_steepest_direction;
//double grad_overlap = steepest_direction*previous_steepest_direction;
//double search_grad_overlap = candidate_point.Gradient*search_direction;
//if (iterations % initial_guess.Count == 0 || (Math.Abs(grad_overlap) >= 0.2 * prev_grad_mag) || (-2 * prev_grad_mag >= search_grad_overlap) || (search_grad_overlap >= -0.2 * prev_grad_mag))
// search_direction = steepest_direction;
//else
// search_direction = steepest_direction + search_direction_adjuster * search_direction;
search_direction = steepest_direction + search_direction_adjuster * search_direction;
if (search_direction * candidate_point.Gradient >= 0)
{
search_direction = steepest_direction;
steepest_descent_resets += 1;
}
try
{
result = line_searcher.FindConformingStep(objective, candidate_point, search_direction, step_size);
}
catch (Exception e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
iterations_with_nontrivial_line_search += result.Iterations > 0 ? 1 : 0;
total_line_search_steps += result.Iterations;
step_size = result.FinalStep;
candidate_point = result.FunctionInfoAtMinimum;
iterations += 1;
}
if (iterations == this.MaximumIterations)
{
throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", this.MaximumIterations));
}
return(new MinimizationWithLineSearchOutput(candidate_point, iterations, ExitCondition.AbsoluteGradient, total_line_search_steps, iterations_with_nontrivial_line_search));
}
/// <summary>
/// Find the minimum of the objective function given lower and upper bounds
/// </summary>
/// <param name="objective">The objective function, must support a gradient</param>
/// <param name="initialGuess">The initial guess</param>
/// <returns>The MinimizationResult which contains the minimum and the ExitCondition</returns>
public MinimizationResult FindMinimum(IObjectiveFunction objective, Vector <double> initialGuess)
{
if (!objective.IsGradientSupported)
{
throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for L-BFGS minimization.");
}
objective.EvaluateAt(initialGuess);
ValidateGradientAndObjective(objective);
// Check that we're not already done
ExitCondition currentExitCondition = ExitCriteriaSatisfied(objective, null, 0);
if (currentExitCondition != ExitCondition.None)
{
return(new MinimizationResult(objective, 0, currentExitCondition));
}
// Set up line search algorithm
var lineSearcher = new WeakWolfeLineSearch(1e-4, 0.9, Math.Max(ParameterTolerance, 1e-10), 1000);
// First step
var lineSearchDirection = -objective.Gradient;
var stepSize = (100 * GradientTolerance) / (lineSearchDirection * lineSearchDirection);
var previousPoint = objective;
LineSearchResult lineSearchResult;
try
{
lineSearchResult = lineSearcher.FindConformingStep(objective, lineSearchDirection, stepSize);
}
catch (OptimizationException e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
catch (ArgumentException e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
var candidate = lineSearchResult.FunctionInfoAtMinimum;
ValidateGradientAndObjective(candidate);
var gradient = candidate.Gradient;
var step = candidate.Point - initialGuess;
var yk = candidate.Gradient - previousPoint.Gradient;
var ykhistory = new List <Vector <double> >()
{
yk
};
var skhistory = new List <Vector <double> >()
{
step
};
var rhokhistory = new List <double>()
{
1.0 / yk.DotProduct(step)
};
// Subsequent steps
int iterations = 1;
int totalLineSearchSteps = lineSearchResult.Iterations;
int iterationsWithNontrivialLineSearch = lineSearchResult.Iterations > 0 ? 0 : 1;
previousPoint = candidate;
while ((iterations++ < MaximumIterations) && (previousPoint.Gradient.Norm(2) >= GradientTolerance))
{
lineSearchDirection = -ApplyLbfgsUpdate(previousPoint, ykhistory, skhistory, rhokhistory);
var directionalDerivative = previousPoint.Gradient.DotProduct(lineSearchDirection);
if (directionalDerivative > 0)
{
throw new InnerOptimizationException("Direction is not a descent direction.");
}
try
{
lineSearchResult = lineSearcher.FindConformingStep(previousPoint, lineSearchDirection, 1.0);
}
catch (OptimizationException e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
catch (ArgumentException e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
iterationsWithNontrivialLineSearch += lineSearchResult.Iterations > 0 ? 1 : 0;
totalLineSearchSteps += lineSearchResult.Iterations;
candidate = lineSearchResult.FunctionInfoAtMinimum;
currentExitCondition = ExitCriteriaSatisfied(candidate, previousPoint, iterations);
if (currentExitCondition != ExitCondition.None)
{
break;
}
step = candidate.Point - previousPoint.Point;
yk = candidate.Gradient - previousPoint.Gradient;
ykhistory.Add(yk);
skhistory.Add(step);
rhokhistory.Add(1.0 / yk.DotProduct(step));
previousPoint = candidate;
if (ykhistory.Count > Memory)
{
ykhistory.RemoveAt(0);
skhistory.RemoveAt(0);
rhokhistory.RemoveAt(0);
}
}
if ((iterations == MaximumIterations) && (currentExitCondition == ExitCondition.None))
{
throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", MaximumIterations));
}
return(new MinimizationWithLineSearchResult(candidate, iterations, ExitCondition.AbsoluteGradient, totalLineSearchSteps, iterationsWithNontrivialLineSearch));
}
public MinimizationOutput FindMinimum(IObjectiveFunction objective, Vector <double> initial_guess)
{
if (!objective.GradientSupported)
{
throw new IncompatibleObjectiveException("Gradient not supported in objective function, but required for BFGS minimization.");
}
if (!(objective is ObjectiveChecker))
{
objective = new ObjectiveChecker(objective, this.ValidateObjective, this.ValidateGradient, null);
}
IEvaluation initial_eval = objective.Evaluate(initial_guess);
// Check that we're not already done
ExitCondition current_exit_condition = this.ExitCriteriaSatisfied(initial_eval, null);
if (current_exit_condition != ExitCondition.None)
{
return(new MinimizationOutput(initial_eval, 0, current_exit_condition));
}
// Set up line search algorithm
var line_searcher = new WeakWolfeLineSearch(1e-4, 0.9, Math.Max(this.ParameterTolerance, 1e-10), max_iterations: 1000);
// Declare state variables
IEvaluation candidate_point, previous_point;
double step_size;
Vector <double> gradient, step, search_direction;
Matrix <double> inverse_pseudo_hessian;
// First step
inverse_pseudo_hessian = LinearAlgebra.Double.DiagonalMatrix.CreateIdentity(initial_guess.Count);
search_direction = -initial_eval.Gradient;
step_size = 100 * this.GradientTolerance / (search_direction * search_direction);
LineSearchOutput result;
try
{
result = line_searcher.FindConformingStep(objective, initial_eval, search_direction, step_size);
}
catch (Exception e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
previous_point = initial_eval;
candidate_point = result.FunctionInfoAtMinimum;
gradient = candidate_point.Gradient;
step = candidate_point.Point - initial_guess;
step_size = result.FinalStep;
// Subsequent steps
Matrix <double> I = LinearAlgebra.Double.DiagonalMatrix.CreateIdentity(initial_guess.Count);
int iterations;
int total_line_search_steps = result.Iterations;
int iterations_with_nontrivial_line_search = result.Iterations > 0 ? 0 : 1;
int steepest_descent_resets = 0;
for (iterations = 1; iterations < this.MaximumIterations; ++iterations)
{
var y = candidate_point.Gradient - previous_point.Gradient;
double sy = step * y;
inverse_pseudo_hessian = inverse_pseudo_hessian + ((sy + y * inverse_pseudo_hessian * y) / Math.Pow(sy, 2.0)) * step.OuterProduct(step) - ((inverse_pseudo_hessian * y.ToColumnMatrix()) * step.ToRowMatrix() + step.ToColumnMatrix() * (y.ToRowMatrix() * inverse_pseudo_hessian)) * (1.0 / sy);
search_direction = -inverse_pseudo_hessian * candidate_point.Gradient;
if (search_direction * candidate_point.Gradient >= 0.0)
{
search_direction = -candidate_point.Gradient;
inverse_pseudo_hessian = LinearAlgebra.Double.DiagonalMatrix.CreateIdentity(initial_guess.Count);
steepest_descent_resets += 1;
}
//else if (search_direction * candidate_point.Gradient >= -this.GradientTolerance*this.GradientTolerance)
//{
// search_direction = -candidate_point.Gradient;
// inverse_pseudo_hessian = LinearAlgebra.Double.DiagonalMatrix.Identity(initial_guess.Count);
// steepest_descent_resets += 1;
//}
try
{
result = line_searcher.FindConformingStep(objective, candidate_point, search_direction, 1.0);
}
catch (Exception e)
{
throw new InnerOptimizationException("Line search failed.", e);
}
iterations_with_nontrivial_line_search += result.Iterations > 0 ? 1 : 0;
total_line_search_steps += result.Iterations;
step_size = result.FinalStep;
step = result.FunctionInfoAtMinimum.Point - candidate_point.Point;
previous_point = candidate_point;
candidate_point = result.FunctionInfoAtMinimum;
current_exit_condition = this.ExitCriteriaSatisfied(candidate_point, previous_point);
if (current_exit_condition != ExitCondition.None)
{
break;
}
}
if (iterations == this.MaximumIterations && current_exit_condition == ExitCondition.None)
{
throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", this.MaximumIterations));
}
return(new MinimizationWithLineSearchOutput(candidate_point, iterations, current_exit_condition, total_line_search_steps, iterations_with_nontrivial_line_search));
}