public void ConstructorTest1()
{
Func <double[], double> function = // min f(x) = 10 * (x+1)^2 + y^2
x => 10.0 * Math.Pow(x[0] + 1.0, 2.0) + Math.Pow(x[1], 2.0);
Func <double[], double[]> gradient = x => new[] { 20 * (x[0] + 1), 2 * x[1] };
var target = new BoundedBroydenFletcherGoldfarbShanno(2)
{
Function = function,
Gradient = gradient
};
bool success = target.Minimize();
double minimum = target.Value;
double[] solution = target.Solution;
Assert.IsTrue(success);
Assert.AreEqual(0, minimum, 1e-10);
Assert.AreEqual(-1, solution[0], 1e-5);
Assert.AreEqual(0, solution[1], 1e-5);
double expectedMinimum = function(target.Solution);
Assert.AreEqual(expectedMinimum, minimum);
}
public void lbfgsTest3()
{
Accord.Math.Tools.SetupGenerator(0);
Func <double[], double> f;
Func <double[], double[]> g;
createExpDiff(out f, out g);
int errors = 0;
for (int i = 0; i < 10000; i++)
{
double[] start = Accord.Math.Matrix.Random(2, -1.0, 1.0);
var lbfgs = new BoundedBroydenFletcherGoldfarbShanno(numberOfVariables: 2,
function: f, gradient: g);
lbfgs.FunctionTolerance = 1e3;
Assert.IsTrue(lbfgs.Minimize(start));
double minValue = lbfgs.Value;
double[] solution = lbfgs.Solution;
double expected = -2;
if (Math.Abs(expected - minValue) > 1e-2)
{
errors++;
}
}
Assert.IsTrue(errors < 1000);
}
private void fitMLE(double sum1, double sum2, double n)
{
double[] gradient = new double[2];
var bfgs = new BoundedBroydenFletcherGoldfarbShanno(numberOfVariables: 2);
bfgs.LowerBounds[0] = 1e-100;
bfgs.LowerBounds[1] = 1e-100;
bfgs.Solution[0] = this.alpha;
bfgs.Solution[1] = this.beta;
bfgs.Function = (double[] parameters) =>
LogLikelihood(sum1, sum2, n, parameters[0], parameters[1]);
bfgs.Gradient = (double[] parameters) =>
Gradient(sum1, sum2, n, parameters[0], parameters[1], gradient);
if (!bfgs.Minimize())
{
throw new ConvergenceException();
}
this.alpha = bfgs.Solution[0];
this.beta = bfgs.Solution[1];
}
public void ConstructorTest2()
{
Function function = // min f(x) = 10 * (x+1)^2 + y^2
x => 10.0 * Math.Pow(x[0] + 1.0, 2.0) + Math.Pow(x[1], 2.0);
Gradient gradient = x => new[] { 20 * (x[0] + 1), 2 * x[1] };
double[] start = new double[2];
var target = new BoundedBroydenFletcherGoldfarbShanno(2,
function.Invoke, gradient.Invoke);
Assert.IsTrue(target.Minimize());
double minimum = target.Value;
double[] solution = target.Solution;
Assert.AreEqual(0, minimum, 1e-10);
Assert.AreEqual(-1, solution[0], 1e-5);
Assert.AreEqual(0, solution[1], 1e-5);
double expectedMinimum = function(target.Solution);
Assert.AreEqual(expectedMinimum, minimum);
}
public void NoGradientTest()
{
var target = new BoundedBroydenFletcherGoldfarbShanno(2)
{
Function = (x) => 0.0
};
target.Minimize();
}
public void MutableGradientSizeTest()
{
var target = new BoundedBroydenFletcherGoldfarbShanno(2)
{
Function = (x) => 0.0,
Gradient = (x) => x
};
Assert.Throws <InvalidOperationException>(() => target.Minimize(), "");
}
public void WrongGradientSizeTest()
{
var target = new BoundedBroydenFletcherGoldfarbShanno(2)
{
Function = (x) => 0.0,
Gradient = (x) => new double[1]
};
target.Minimize();
}
public void MutableGradientSizeTest()
{
var target = new BoundedBroydenFletcherGoldfarbShanno(2)
{
Function = (x) => 0.0,
Gradient = (x) => x
};
target.Minimize();
}
public void NoGradientTest()
{
var target = new BoundedBroydenFletcherGoldfarbShanno(2)
{
Function = (x) => 0.0
};
Assert.IsTrue(target.Minimize());
// The optimizer should use finite differences as the gradient
}
/// <summary>
/// Constructs a new L-BFGS learning algorithm.
/// </summary>
///
public QuasiNewtonLearning(ConditionalRandomField <T> model)
{
this.model = model;
this.lbfgs = new BoundedBroydenFletcherGoldfarbShanno(model.Function.Weights.Length);
this.lbfgs.FunctionTolerance = 1e-3;
for (int i = 0; i < lbfgs.UpperBounds.Length; i++)
{
lbfgs.UpperBounds[i] = 1e10;
lbfgs.LowerBounds[i] = -1e100;
}
}
private void init()
{
this.lbfgs = new BoundedBroydenFletcherGoldfarbShanno(Model.Function.Weights.Length);
this.lbfgs.FunctionTolerance = Tolerance;
this.lbfgs.MaxIterations = MaxIterations;
for (int i = 0; i < lbfgs.UpperBounds.Length; i++)
{
lbfgs.UpperBounds[i] = 1e10;
lbfgs.LowerBounds[i] = -1e100;
}
}
public void lbfgsTest2()
{
Accord.Math.Tools.SetupGenerator(0);
// Suppose we would like to find the minimum of the function
//
// f(x,y) = -exp{-(x-1)²} - exp{-(y-2)²/2}
//
// First we need write down the function either as a named
// method, an anonymous method or as a lambda function:
Func <double[], double> f = (x) =>
- Math.Exp(-Math.Pow(x[0] - 1, 2)) - Math.Exp(-0.5 * Math.Pow(x[1] - 2, 2));
// Now, we need to write its gradient, which is just the
// vector of first partial derivatives del_f / del_x, as:
//
// g(x,y) = { del f / del x, del f / del y }
//
Func <double[], double[]> g = (x) => new double[]
{
// df/dx = {-2 e^(- (x-1)^2) (x-1)}
2 * Math.Exp(-Math.Pow(x[0] - 1, 2)) * (x[0] - 1),
// df/dy = {- e^(-1/2 (y-2)^2) (y-2)}
Math.Exp(-0.5 * Math.Pow(x[1] - 2, 2)) * (x[1] - 2)
};
// Finally, we can create the L-BFGS solver, passing the functions as arguments
var lbfgs = new BoundedBroydenFletcherGoldfarbShanno(numberOfVariables: 2, function: f, gradient: g);
// And then minimize the function:
Assert.IsTrue(lbfgs.Minimize());
double minValue = lbfgs.Value;
double[] solution = lbfgs.Solution;
// The resultant minimum value should be -2, and the solution
// vector should be { 1.0, 2.0 }. The answer can be checked on
// Wolfram Alpha by clicking the following the link:
// http://www.wolframalpha.com/input/?i=maximize+%28exp%28-%28x-1%29%C2%B2%29+%2B+exp%28-%28y-2%29%C2%B2%2F2%29%29
double expected = -2;
Assert.AreEqual(expected, minValue, 1e-10);
Assert.AreEqual(1, solution[0], 1e-3);
Assert.AreEqual(2, solution[1], 1e-3);
}
public OptimizationProgressEventArgs[] Actual(Specification problem)
{
ActualMessage = String.Empty;
BoundedBroydenFletcherGoldfarbShanno target =
new BoundedBroydenFletcherGoldfarbShanno(problem.Variables)
{
FunctionTolerance = factr,
GradientTolerance = pgtol,
Corrections = m,
MaxIterations = max_iterations
};
for (int i = 0; i < target.LowerBounds.Length; i++)
{
if (l != null)
{
target.LowerBounds[i] = l[i];
}
if (u != null)
{
target.UpperBounds[i] = u[i];
}
}
target.Function = problem.Function;
target.Gradient = problem.Gradient;
actual.Clear();
target.Progress += new EventHandler <OptimizationProgressEventArgs>(target_Progress);
target.Minimize((double[])problem.Start.Clone());
if (target.Status == BoundedBroydenFletcherGoldfarbShannoStatus.GradientConvergence)
{
ActualMessage = "CONVERGENCE: NORM_OF_PROJECTED_GRADIENT_<=_PGTOL";
}
else if (target.Status == BoundedBroydenFletcherGoldfarbShannoStatus.FunctionConvergence)
{
ActualMessage = "CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH";
}
else if (target.Status == BoundedBroydenFletcherGoldfarbShannoStatus.LineSearchFailed)
{
ActualMessage = "ABNORMAL_TERMINATION_IN_LNSRCH";
}
return(actual.ToArray());
}
/// <summary>
/// Inheritors of this class should create the optimization algorithm in this
/// method, using the current <see cref="P:Accord.Statistics.Models.Fields.Learning.BaseHiddenGradientOptimizationLearning`2.MaxIterations" /> and <see cref="P:Accord.Statistics.Models.Fields.Learning.BaseHiddenGradientOptimizationLearning`2.Tolerance" />
/// settings.
/// </summary>
/// <returns>BoundedBroydenFletcherGoldfarbShanno.</returns>
protected override BoundedBroydenFletcherGoldfarbShanno CreateOptimizer()
{
var lbfgs = new BoundedBroydenFletcherGoldfarbShanno(Model.Function.Weights.Length)
{
FunctionTolerance = Tolerance,
MaxIterations = MaxIterations,
};
for (int i = 0; i < lbfgs.UpperBounds.Length; i++)
{
lbfgs.UpperBounds[i] = 1e10;
lbfgs.LowerBounds[i] = -1e100;
}
return(lbfgs);
}
/// <summary>
/// Constructs a new L-BFGS learning algorithm.
/// </summary>
///
public HiddenQuasiNewtonLearning(HiddenConditionalRandomField <T> model)
{
Model = model;
calculator = new ForwardBackwardGradient <T>(model);
lbfgs = new BoundedBroydenFletcherGoldfarbShanno(model.Function.Weights.Length);
lbfgs.FunctionTolerance = 1e-3;
lbfgs.Function = calculator.Objective;
lbfgs.Gradient = calculator.Gradient;
for (int i = 0; i < lbfgs.UpperBounds.Length; i++)
{
lbfgs.UpperBounds[i] = 1e10;
lbfgs.LowerBounds[i] = -1e100;
}
}
public void lbfgsTest()
{
Func <double[], double> f = rosenbrockFunction;
Func <double[], double[]> g = rosenbrockGradient;
Assert.AreEqual(104, f(new[] { -1.0, 2.0 }));
int n = 2; // number of variables
double[] initial = { -1.2, 1 };
var lbfgs = new BoundedBroydenFletcherGoldfarbShanno(n, f, g);
lbfgs.GradientTolerance = 1e-10;
lbfgs.FunctionTolerance = 1e-10;
double expected = 0;
Assert.IsTrue(lbfgs.Minimize(initial));
double actual = lbfgs.Value;
Assert.AreEqual(expected, actual, 1e-10);
double[] result = lbfgs.Solution;
//Assert.AreEqual(49, lbfgs.Evaluations);
//Assert.AreEqual(40, lbfgs.Iterations);
Assert.AreEqual(1.0, result[0], 1e-6);
Assert.AreEqual(1.0, result[1], 1e-6);
double y = f(result);
double[] d = g(result);
Assert.AreEqual(0, y, 1e-10);
Assert.AreEqual(0, d[0], 1e-6);
Assert.AreEqual(0, d[1], 1e-6);
}
public void InvalidLineSearchTest2()
{
int n = 10;
Func <double[], double> function = (parameters) =>
{
return(-(n * Math.Log(0) - n * Math.Log(Math.PI)));
};
Func <double[], double[]> gradient = (parameters) =>
{
return(new[] { 2.0, Double.NegativeInfinity });
};
double[] start = { 0, 0 };
var lbfgs = new BoundedBroydenFletcherGoldfarbShanno(2, function, gradient);
lbfgs.Minimize(start);
Assert.AreEqual(BoundedBroydenFletcherGoldfarbShannoStatus.LineSearchFailed, lbfgs.Status);
}
public void NoFunctionTest()
{
var target = new BoundedBroydenFletcherGoldfarbShanno(2);
Assert.Throws <InvalidOperationException>(() => target.Minimize(), "");
}
public void NoFunctionTest()
{
var target = new BoundedBroydenFletcherGoldfarbShanno(2);
target.Minimize();
}
/// <summary>
/// Fits the underlying distribution to a given set of observations.
/// </summary>
///
/// <param name="observations">The array of observations to fit the model against. The array
/// elements can be either of type double (for univariate data) or
/// type double[] (for multivariate data).</param>
/// <param name="weights">The weight vector containing the weight for each of the samples.</param>
/// <param name="options">Optional arguments which may be used during fitting, such
/// as regularization constants and additional parameters.</param>
///
/// <remarks>
/// Although both double[] and double[][] arrays are supported,
/// providing a double[] for a multivariate distribution or a
/// double[][] for a univariate distribution may have a negative
/// impact in performance.
/// </remarks>
///
/// <example>
/// See <see cref="CauchyDistribution"/>.
/// </example>
///
public void Fit(double[] observations, double[] weights, CauchyOptions options)
{
if (immutable)
{
throw new InvalidOperationException("This object can not be modified.");
}
if (weights != null)
{
throw new ArgumentException("This distribution does not support weighted samples.");
}
bool useMLE = true;
bool estimateT = true;
bool estimateS = true;
if (options != null)
{
useMLE = options.MaximumLikelihood;
estimateT = options.EstimateLocation;
estimateS = options.EstimateScale;
}
double t0 = location;
double s0 = scale;
int n = observations.Length;
DoubleRange range;
double median = Measures.Quartiles(observations, out range, alreadySorted: false);
if (estimateT)
{
t0 = median;
}
if (estimateS)
{
s0 = range.Length;
}
if (useMLE)
{
// Minimize the log-likelihood through numerical optimization
var lbfgs = new BoundedBroydenFletcherGoldfarbShanno(2);
lbfgs.LowerBounds[1] = 0; // scale must be positive
// Define the negative log-likelihood function,
// which is the objective we want to minimize:
lbfgs.Function = (parameters) =>
{
// Assume location is the first
// parameter, shape is the second
double t = (estimateT) ? parameters[0] : t0;
double s = (estimateS) ? parameters[1] : s0;
if (s < 0)
{
s = -s;
}
double sum = 0;
for (int i = 0; i < observations.Length; i++)
{
double y = (observations[i] - t);
sum += Math.Log(s * s + y * y);
}
return(-(n * Math.Log(s) - sum - n * Math.Log(Math.PI)));
};
lbfgs.Gradient = (parameters) =>
{
// Assume location is the first
// parameter, shape is the second
double t = (estimateT) ? parameters[0] : t0;
double s = (estimateS) ? parameters[1] : s0;
double sum1 = 0, sum2 = 0;
for (int i = 0; i < observations.Length; i++)
{
double y = (observations[i] - t);
sum1 += y / (s * s + y * y);
sum2 += s / (s * s + y * y);
}
double dt = -2.0 * sum1;
double ds = +2.0 * sum2 - n / s;
double[] g = new double[2];
g[0] = estimateT ? dt : 0;
g[1] = estimateS ? ds : 0;
return(g);
};
// Initialize using the sample median as starting
// value for location, and half interquartile range
// for shape.
double[] values = { t0, s0 };
// Minimize
lbfgs.Minimize(values);
// Check solution
t0 = lbfgs.Solution[0];
s0 = lbfgs.Solution[1];
}
init(t0, s0); // Become the new distribution
}
private ConjugateGradientCode cvsmod(ref double f, double[] s, ref double stp, ref int info,
ref int nfev, double[] wa, ref double dginit, ref double dgout)
{
var n = NumberOfVariables;
var x = Solution;
if (info == 1)
{
goto L321;
}
infoc = 1;
if (stp <= 0) // Check the input parameters for errors
{
return(ConjugateGradientCode.StepSize);
}
// Compute the initial gradient in the search direction
// and check that S is a descent direction.
if (dginit >= 0)
{
throw new LineSearchFailedException(0, "The search direction is not a descent direction.");
}
// Initialize local variables
brackt = false;
stage1 = true;
nfev = 0;
finit = f;
dgtest = ftol * dginit;
width = stpmax - stpmin;
width1 = width / 0.5;
for (var j = 0; j < x.Length; ++j)
{
wa[j] = x[j];
}
// The variables STX, FX, DGX contain the values of the step,
// function, and directional derivative at the best step.
// The variables STY, FY, DGY contain the value of the step,
// function, and derivative at the other endpoint of the interval
// of uncertainty.
// The variables STP, F, DG contain the values of the step,
// function, and derivative at the current step.
stx = 0;
fx = finit;
dgx = dginit;
sty = 0;
fy = finit;
dgy = dginit;
L30: // Start of iteration.
// Set the minimum and maximum steps to correspond
// to the present interval of uncertainty.
if (brackt)
{
stmin = Math.Min(stx, sty);
stmax = Math.Max(stx, sty);
}
else
{
stmin = stx;
stmax = stp + 4 * (stp - stx);
}
// Force the step to be within
// the bounds STPMAX and STPMIN.
stp = Math.Max(stp, stpmin);
stp = Math.Min(stp, stpmax);
// If an unusual termination is to occur then
// let STP be the lowest point obtained so far.
if (brackt && (stp <= stmin || stp >= stmax) || nfev >= maxfev - 1 ||
infoc == 0 || brackt && stmax - stmin <= xtol * stmax)
{
stp = stx;
}
// Evaluate the function and gradient at STP
// and compute the directional derivative.
for (var j = 0; j < s.Length; ++j)
{
x[j] = wa[j] + stp * s[j];
}
// Fetch function and gradient
f = Function(x);
g = Gradient(x);
info = 0;
nfev++;
dg2 = 0;
for (var j = 0; j < g.Length; ++j)
{
dg2 += g[j] * s[j];
}
ftest1 = finit + stp * dgtest;
if (brackt && (stp <= stmin || stp >= stmax) || infoc == 0)
{
return(ConjugateGradientCode.RoundingErrors);
}
if (stp == stpmax && f <= ftest1 && dg2 <= dgtest)
{
return(ConjugateGradientCode.StepHigh);
}
if (stp == stpmin && (f > ftest1 || dg2 >= dgtest))
{
return(ConjugateGradientCode.StepLow);
}
if (nfev >= maxfev)
{
return(ConjugateGradientCode.MaximumEvaluations);
}
if (brackt && stmax - stmin <= xtol * stmax)
{
return(ConjugateGradientCode.Precision);
}
// More's code has been modified so that at least one new
// function value is computed during the line search (enforcing
// at least one interpolation is not easy, since the code may
// override an interpolation)
if (f <= ftest1 && Math.Abs(dg2) <= gtol * -dginit && nfev > 1)
{
info = 1;
dgout = dg2;
return(ConjugateGradientCode.Success);
}
L321:
// In the first stage we seek a step for which the modified
// function has a nonpositive value and nonnegative derivative.
if (stage1 && f <= ftest1 && dg2 >= Math.Min(ftol, gtol) * dginit)
{
stage1 = false;
}
// A modified function is used to predict the step only if
// we have not obtained a step for which the modified function
// has a nonpositive function value and nonnegative derivative,
// and if a lower function value has been obtained but the
// decrease is not sufficient.
if (stage1 && f <= fx && f > ftest1)
{
// Define the modified function and derivative values
var fm = f - stp * dgtest;
var fxm = fx - stx * dgtest;
var fym = fy - sty * dgtest;
var dgm = dg2 - dgtest;
var dgxm = dgx - dgtest;
var dgym = dgy - dgtest;
// Call CSTEPM to update the interval of
// uncertainty and to compute the new step.
BoundedBroydenFletcherGoldfarbShanno.dcstep(ref stx, ref fxm, ref dgxm,
ref sty, ref fym, ref dgym, ref stp, fm, dgm, ref brackt, stpmin, stpmax);
// Reset the function and gradient values for f.
fx = fxm + stx * dgtest;
fy = fym + sty * dgtest;
dgx = dgxm + dgtest;
dgy = dgym + dgtest;
}
else
{
// Call CSTEPM to update the interval of
// uncertainty and to compute the new step.
BoundedBroydenFletcherGoldfarbShanno.dcstep(ref stx, ref fx, ref dgx,
ref sty, ref fy, ref dgy, ref stp, f, dg2, ref brackt, stpmin, stpmax);
}
// Force a sufficient decrease in the
// size of the interval of uncertainty.
if (brackt)
{
if (Math.Abs(sty - stx) >= 0.66 * width1)
{
stp = stx + 0.5 * (sty - stx);
}
width1 = width;
width = Math.Abs(sty - stx);
}
goto L30;
}
