/// <summary>
/// Creates a new instance of the Augmented Lagrangian algorithm.
/// </summary>
///
/// <param name="innerSolver">The <see cref="IGradientOptimizationMethod">unconstrained optimization
/// method</see> used internally to solve the dual of this optimization problem.</param>
/// <param name="constraints">The <see cref="NonlinearConstraint"/>s to which the solution must be subjected.</param>
///
public AugmentedLagrangianSolver(IGradientOptimizationMethod innerSolver,
IEnumerable <NonlinearConstraint> constraints)
{
int numberOfVariables = innerSolver.Parameters;
init(numberOfVariables, constraints, innerSolver);
}
private void init(NonlinearObjectiveFunction function,
IEnumerable <NonlinearConstraint> constraints, IGradientOptimizationMethod innerSolver)
{
if (function != null)
{
if (function.NumberOfVariables != NumberOfVariables)
{
throw new ArgumentOutOfRangeException("function",
"Incorrect number of variables in the objective function. " +
"The number of variables must match the number of variables set in the solver.");
}
this.Function = function.Function;
this.Gradient = function.Gradient;
}
if (innerSolver == null)
{
innerSolver = new BroydenFletcherGoldfarbShanno(NumberOfVariables)
{
LineSearch = Optimization.LineSearch.BacktrackingArmijo,
Corrections = 10
};
}
List <NonlinearConstraint> equality = new List <NonlinearConstraint>();
List <NonlinearConstraint> lesserThan = new List <NonlinearConstraint>();
List <NonlinearConstraint> greaterThan = new List <NonlinearConstraint>();
foreach (var c in constraints)
{
switch (c.ShouldBe)
{
case ConstraintType.EqualTo:
equality.Add(c); break;
case ConstraintType.GreaterThanOrEqualTo:
greaterThan.Add(c); break;
case ConstraintType.LesserThanOrEqualTo:
lesserThan.Add(c); break;
default:
throw new ArgumentException("Unknown constraint type.", "constraints");
}
}
this.lesserThanConstraints = lesserThan.ToArray();
this.greaterThanConstraints = greaterThan.ToArray();
this.equalityConstraints = equality.ToArray();
this.lambda = new double[equalityConstraints.Length];
this.mu = new double[lesserThanConstraints.Length];
this.nu = new double[greaterThanConstraints.Length];
this.dualSolver = innerSolver;
dualSolver.Function = objectiveFunction;
dualSolver.Gradient = objectiveGradient;
}
/// <summary>
/// Creates a new instance of the Augmented Lagrangian algorithm.
/// </summary>
///
/// <param name="innerSolver">The <see cref="IGradientOptimizationMethod">unconstrained
/// optimization method</see> used internally to solve the dual of this optimization
/// problem.</param>
/// <param name="function">The objective function to be optimized.</param>
/// <param name="constraints">
/// The <see cref="NonlinearConstraint"/>s to which the solution must be subjected.</param>
///
public AugmentedLagrangian(IGradientOptimizationMethod innerSolver,
NonlinearObjectiveFunction function, IEnumerable <NonlinearConstraint> constraints)
: base(innerSolver.NumberOfVariables)
{
if (innerSolver.NumberOfVariables != function.NumberOfVariables)
{
throw new ArgumentException("The inner unconstrained optimization algorithm and the "
+ "objective function should have the same number of variables.", "function");
}
init(function, constraints, innerSolver);
}
private void init(int numberOfVariables, IEnumerable <NonlinearConstraint> constraints, IGradientOptimizationMethod innerSolver)
{
this.numberOfVariables = numberOfVariables;
List <NonlinearConstraint> equality = new List <NonlinearConstraint>();
List <NonlinearConstraint> lesserThan = new List <NonlinearConstraint>();
List <NonlinearConstraint> greaterThan = new List <NonlinearConstraint>();
foreach (var c in constraints)
{
switch (c.ShouldBe)
{
case ConstraintType.EqualTo:
equality.Add(c); break;
case ConstraintType.GreaterThanOrEqualTo:
greaterThan.Add(c); break;
case ConstraintType.LesserThanOrEqualTo:
lesserThan.Add(c); break;
default:
throw new ArgumentException("Unknown constraint type.", "constraints");
}
}
this.lesserThanConstraints = lesserThan.ToArray();
this.greaterThanConstraints = greaterThan.ToArray();
this.equalityConstraints = equality.ToArray();
this.Solution = new double[numberOfVariables];
this.dualSolver = innerSolver;
dualSolver.Function = objectiveFunction;
dualSolver.Gradient = objectiveGradient;
for (int i = 0; i < Solution.Length; i++)
{
Solution[i] = Accord.Math.Tools.Random.NextDouble() * 2 - 1;
}
}
private static void test2(IGradientOptimizationMethod inner)
{
// maximize 2x + 3y, s.t. 2x² + 2y² <= 50
//
// http://www.wolframalpha.com/input/?i=max+2x+%2B+3y%2C+s.t.+2x%C2%B2+%2B+2y%C2%B2+%3C%3D+50
// Max x' * c
// x
// s.t. x' * A * x <= k
// x' * i = 1
// lower_bound < x < upper_bound
double[] c = { 2, 3 };
double[,] A = { { 2, 0 }, { 0, 2 } };
double k = 50;
// Create the objective function
var objective = new NonlinearObjectiveFunction(2,
function: (x) => x.InnerProduct(c),
gradient: (x) => c
);
// Test objective
for (int i = 0; i < 10; i++)
{
for (int j = 0; j < 10; j++)
{
double expected = i * 2 + j * 3;
double actual = objective.Function(new double[] { i, j });
Assert.AreEqual(expected, actual);
}
}
// Create the optimization constraints
var constraints = new List <NonlinearConstraint>();
constraints.Add(new QuadraticConstraint(objective,
quadraticTerms: A,
shouldBe: ConstraintType.LesserThanOrEqualTo, value: k
));
// Test first constraint
for (int i = 0; i < 10; i++)
{
for (int j = 0; j < 10; j++)
{
var input = new double[] { i, j };
double expected = i * (2 * i + 0 * j) + j * (0 * i + 2 * j);
double actual = constraints[0].Function(input);
Assert.AreEqual(expected, actual);
}
}
// Create the solver algorithm
AugmentedLagrangian solver =
new AugmentedLagrangian(inner, objective, constraints);
Assert.AreEqual(inner, solver.Optimizer);
Assert.IsTrue(solver.Maximize());
double maxValue = solver.Value;
Assert.AreEqual(18.02, maxValue, 1e-2);
Assert.AreEqual(2.77, solver.Solution[0], 1e-2);
Assert.AreEqual(4.16, solver.Solution[1], 1e-2);
}
private static void test1(IGradientOptimizationMethod inner, double tol)
{
// maximize 2x + 3y, s.t. 2x² + 2y² <= 50 and x+y = 1
// Max x' * c
// x
// s.t. x' * A * x <= k
// x' * i = 1
// lower_bound < x < upper_bound
double[] c = { 2, 3 };
double[,] A = { { 2, 0 }, { 0, 2 } };
double k = 50;
// Create the objective function
var objective = new NonlinearObjectiveFunction(2,
function: (x) => x.InnerProduct(c),
gradient: (x) => c
);
// Test objective
for (int i = 0; i < 10; i++)
{
for (int j = 0; j < 10; j++)
{
double expected = i * 2 + j * 3;
double actual = objective.Function(new double[] { i, j });
Assert.AreEqual(expected, actual);
}
}
// Create the optimization constraints
var constraints = new List <NonlinearConstraint>();
constraints.Add(new QuadraticConstraint(objective,
quadraticTerms: A,
shouldBe: ConstraintType.LesserThanOrEqualTo, value: k
));
constraints.Add(new NonlinearConstraint(objective,
function: (x) => x.Sum(),
gradient: (x) => new[] { 1.0, 1.0 },
shouldBe: ConstraintType.EqualTo, value: 1,
withinTolerance: 1e-10
));
// Test first constraint
for (int i = 0; i < 10; i++)
{
for (int j = 0; j < 10; j++)
{
double expected = i * (2 * i + 0 * j) + j * (0 * i + 2 * j);
double actual = constraints[0].Function(new double[] { i, j });
Assert.AreEqual(expected, actual);
}
}
// Test second constraint
for (int i = 0; i < 10; i++)
{
for (int j = 0; j < 10; j++)
{
double expected = i + j;
double actual = constraints[1].Function(new double[] { i, j });
Assert.AreEqual(expected, actual);
}
}
AugmentedLagrangian solver =
new AugmentedLagrangian(inner, objective, constraints);
Assert.AreEqual(inner, solver.Optimizer);
Assert.IsTrue(solver.Maximize());
double maxValue = solver.Value;
Assert.AreEqual(6, maxValue, tol);
Assert.AreEqual(-3, solver.Solution[0], tol);
Assert.AreEqual(4, solver.Solution[1], tol);
}
private void init(NonlinearObjectiveFunction function,
IEnumerable<NonlinearConstraint> constraints, IGradientOptimizationMethod innerSolver)
{
if (function != null)
{
if (function.NumberOfVariables != NumberOfVariables)
{
throw new ArgumentOutOfRangeException("function",
"Incorrect number of variables in the objective function. " +
"The number of variables must match the number of variables set in the solver.");
}
this.Function = function.Function;
this.Gradient = function.Gradient;
}
if (innerSolver == null)
{
innerSolver = new BroydenFletcherGoldfarbShanno(NumberOfVariables)
{
LineSearch = Optimization.LineSearch.BacktrackingArmijo,
Corrections = 10
};
}
List<NonlinearConstraint> equality = new List<NonlinearConstraint>();
List<NonlinearConstraint> lesserThan = new List<NonlinearConstraint>();
List<NonlinearConstraint> greaterThan = new List<NonlinearConstraint>();
foreach (var c in constraints)
{
switch (c.ShouldBe)
{
case ConstraintType.EqualTo:
equality.Add(c); break;
case ConstraintType.GreaterThanOrEqualTo:
greaterThan.Add(c); break;
case ConstraintType.LesserThanOrEqualTo:
lesserThan.Add(c); break;
default:
throw new ArgumentException("Unknown constraint type.", "constraints");
}
}
this.lesserThanConstraints = lesserThan.ToArray();
this.greaterThanConstraints = greaterThan.ToArray();
this.equalityConstraints = equality.ToArray();
this.lambda = new double[equalityConstraints.Length];
this.mu = new double[lesserThanConstraints.Length];
this.nu = new double[greaterThanConstraints.Length];
this.dualSolver = innerSolver;
dualSolver.Function = objectiveFunction;
dualSolver.Gradient = objectiveGradient;
}
/// <summary>
/// Creates a new instance of the Augmented Lagrangian algorithm.
/// </summary>
///
/// <param name="innerSolver">The <see cref="IGradientOptimizationMethod">unconstrained
/// optimization method</see> used internally to solve the dual of this optimization
/// problem.</param>
/// <param name="constraints">
/// The <see cref="NonlinearConstraint"/>s to which the solution must be subjected.</param>
///
public AugmentedLagrangian(IGradientOptimizationMethod innerSolver, IEnumerable<NonlinearConstraint> constraints)
: base(innerSolver.NumberOfVariables)
{
init(null, constraints, innerSolver);
}
/// <summary>
/// Creates a new instance of the Augmented Lagrangian algorithm.
/// </summary>
///
/// <param name="innerSolver">The <see cref="IGradientOptimizationMethod">unconstrained
/// optimization method</see> used internally to solve the dual of this optimization
/// problem.</param>
/// <param name="function">The objective function to be optimized.</param>
/// <param name="constraints">
/// The <see cref="NonlinearConstraint"/>s to which the solution must be subjected.</param>
///
public AugmentedLagrangian(IGradientOptimizationMethod innerSolver,
NonlinearObjectiveFunction function, IEnumerable<NonlinearConstraint> constraints)
: base(innerSolver.NumberOfVariables)
{
if (innerSolver.NumberOfVariables != function.NumberOfVariables)
throw new ArgumentException("The inner unconstrained optimization algorithm and the "
+ "objective function should have the same number of variables.", "function");
init(function, constraints, innerSolver);
}
/// <summary>
/// Creates a new instance of the Augmented Lagrangian algorithm.
/// </summary>
///
/// <param name="innerSolver">The <see cref="IGradientOptimizationMethod">unconstrained
/// optimization method</see> used internally to solve the dual of this optimization
/// problem.</param>
/// <param name="constraints">
/// The <see cref="NonlinearConstraint"/>s to which the solution must be subjected.</param>
///
public AugmentedLagrangian(IGradientOptimizationMethod innerSolver, IEnumerable <NonlinearConstraint> constraints)
: base(innerSolver.NumberOfVariables)
{
init(null, constraints, innerSolver);
}
private void init(int numberOfVariables, IEnumerable<NonlinearConstraint> constraints, IGradientOptimizationMethod innerSolver)
{
this.numberOfVariables = numberOfVariables;
List<NonlinearConstraint> equality = new List<NonlinearConstraint>();
List<NonlinearConstraint> lesserThan = new List<NonlinearConstraint>();
List<NonlinearConstraint> greaterThan = new List<NonlinearConstraint>();
foreach (var c in constraints)
{
switch (c.ShouldBe)
{
case ConstraintType.EqualTo:
equality.Add(c); break;
case ConstraintType.GreaterThanOrEqualTo:
greaterThan.Add(c); break;
case ConstraintType.LesserThanOrEqualTo:
lesserThan.Add(c); break;
default:
throw new ArgumentException("Unknown constraint type.", "constraints");
}
}
this.lesserThanConstraints = lesserThan.ToArray();
this.greaterThanConstraints = greaterThan.ToArray();
this.equalityConstraints = equality.ToArray();
this.Solution = new double[numberOfVariables];
this.dualSolver = innerSolver;
dualSolver.Function = objectiveFunction;
dualSolver.Gradient = objectiveGradient;
for (int i = 0; i < Solution.Length; i++)
Solution[i] = Accord.Math.Tools.Random.NextDouble() * 2 - 1;
}
/// <summary>
/// Creates a new instance of the Augmented Lagrangian algorithm.
/// </summary>
///
/// <param name="innerSolver">The <see cref="IGradientOptimizationMethod">unconstrained optimization
/// method</see> used internally to solve the dual of this optimization problem.</param>
/// <param name="constraints">The <see cref="NonlinearConstraint"/>s to which the solution must be subjected.</param>
///
public AugmentedLagrangianSolver(IGradientOptimizationMethod innerSolver,
IEnumerable<NonlinearConstraint> constraints)
{
int numberOfVariables = innerSolver.Parameters;
init(numberOfVariables, constraints, innerSolver);
}
private static void test2(IGradientOptimizationMethod inner)
{
// maximize 2x + 3y, s.t. 2x² + 2y² <= 50
//
// http://www.wolframalpha.com/input/?i=max+2x+%2B+3y%2C+s.t.+2x%C2%B2+%2B+2y%C2%B2+%3C%3D+50
// Max x' * c
// x
// s.t. x' * A * x <= k
// x' * i = 1
// lower_bound < x < upper_bound
double[] c = { 2, 3 };
double[,] A = { { 2, 0 }, { 0, 2 } };
double k = 50;
// Create the objective function
var objective = new NonlinearObjectiveFunction(2,
function: (x) => x.InnerProduct(c),
gradient: (x) => c
);
// Test objective
for (int i = 0; i < 10; i++)
{
for (int j = 0; j < 10; j++)
{
double expected = i * 2 + j * 3;
double actual = objective.Function(new double[] { i, j });
Assert.AreEqual(expected, actual);
}
}
// Create the optimization constraints
var constraints = new List<NonlinearConstraint>();
constraints.Add(new QuadraticConstraint(objective,
quadraticTerms: A,
shouldBe: ConstraintType.LesserThanOrEqualTo, value: k
));
// Test first constraint
for (int i = 0; i < 10; i++)
{
for (int j = 0; j < 10; j++)
{
var input = new double[] { i, j };
double expected = i * (2 * i + 0 * j) + j * (0 * i + 2 * j);
double actual = constraints[0].Function(input);
Assert.AreEqual(expected, actual);
}
}
// Create the solver algorithm
AugmentedLagrangian solver =
new AugmentedLagrangian(inner, objective, constraints);
Assert.AreEqual(inner, solver.Optimizer);
Assert.IsTrue(solver.Maximize());
double maxValue = solver.Value;
Assert.AreEqual(18.02, maxValue, 1e-2);
Assert.AreEqual(2.77, solver.Solution[0], 1e-2);
Assert.AreEqual(4.16, solver.Solution[1], 1e-2);
}
private static void test1(IGradientOptimizationMethod inner, double tol)
{
// maximize 2x + 3y, s.t. 2x² + 2y² <= 50 and x+y = 1
// Max x' * c
// x
// s.t. x' * A * x <= k
// x' * i = 1
// lower_bound < x < upper_bound
double[] c = { 2, 3 };
double[,] A = { { 2, 0 }, { 0, 2 } };
double k = 50;
// Create the objective function
var objective = new NonlinearObjectiveFunction(2,
function: (x) => x.InnerProduct(c),
gradient: (x) => c
);
// Test objective
for (int i = 0; i < 10; i++)
{
for (int j = 0; j < 10; j++)
{
double expected = i * 2 + j * 3;
double actual = objective.Function(new double[] { i, j });
Assert.AreEqual(expected, actual);
}
}
// Create the optimization constraints
var constraints = new List<NonlinearConstraint>();
constraints.Add(new QuadraticConstraint(objective,
quadraticTerms: A,
shouldBe: ConstraintType.LesserThanOrEqualTo, value: k
));
constraints.Add(new NonlinearConstraint(objective,
function: (x) => x.Sum(),
gradient: (x) => new[] { 1.0, 1.0 },
shouldBe: ConstraintType.EqualTo, value: 1,
withinTolerance: 1e-10
));
// Test first constraint
for (int i = 0; i < 10; i++)
{
for (int j = 0; j < 10; j++)
{
double expected = i * (2 * i + 0 * j) + j * (0 * i + 2 * j);
double actual = constraints[0].Function(new double[] { i, j });
Assert.AreEqual(expected, actual);
}
}
// Test second constraint
for (int i = 0; i < 10; i++)
{
for (int j = 0; j < 10; j++)
{
double expected = i + j;
double actual = constraints[1].Function(new double[] { i, j });
Assert.AreEqual(expected, actual);
}
}
AugmentedLagrangian solver =
new AugmentedLagrangian(inner, objective, constraints);
Assert.AreEqual(inner, solver.Optimizer);
Assert.IsTrue(solver.Maximize());
double maxValue = solver.Value;
Assert.AreEqual(6, maxValue, tol);
Assert.AreEqual(-3, solver.Solution[0], tol);
Assert.AreEqual(4, solver.Solution[1], tol);
}