public void ConstructorTest1() { Func <double[], double> function = // min f(x) = 10 * (x+1)^2 + y^2 x => 10.0 * Math.Pow(x[0] + 1.0, 2.0) + Math.Pow(x[1], 2.0); Func <double[], double[]> gradient = x => new[] { 20 * (x[0] + 1), 2 * x[1] }; var target = new BoundedBroydenFletcherGoldfarbShanno(2) { Function = function, Gradient = gradient }; bool success = target.Minimize(); double minimum = target.Value; double[] solution = target.Solution; Assert.IsTrue(success); Assert.AreEqual(0, minimum, 1e-10); Assert.AreEqual(-1, solution[0], 1e-5); Assert.AreEqual(0, solution[1], 1e-5); double expectedMinimum = function(target.Solution); Assert.AreEqual(expectedMinimum, minimum); }
public void lbfgsTest3() { Accord.Math.Tools.SetupGenerator(0); Func <double[], double> f; Func <double[], double[]> g; createExpDiff(out f, out g); int errors = 0; for (int i = 0; i < 10000; i++) { double[] start = Accord.Math.Matrix.Random(2, -1.0, 1.0); var lbfgs = new BoundedBroydenFletcherGoldfarbShanno(numberOfVariables: 2, function: f, gradient: g); lbfgs.FunctionTolerance = 1e3; Assert.IsTrue(lbfgs.Minimize(start)); double minValue = lbfgs.Value; double[] solution = lbfgs.Solution; double expected = -2; if (Math.Abs(expected - minValue) > 1e-2) { errors++; } } Assert.IsTrue(errors < 1000); }
private void fitMLE(double sum1, double sum2, double n) { double[] gradient = new double[2]; var bfgs = new BoundedBroydenFletcherGoldfarbShanno(numberOfVariables: 2); bfgs.LowerBounds[0] = 1e-100; bfgs.LowerBounds[1] = 1e-100; bfgs.Solution[0] = this.alpha; bfgs.Solution[1] = this.beta; bfgs.Function = (double[] parameters) => LogLikelihood(sum1, sum2, n, parameters[0], parameters[1]); bfgs.Gradient = (double[] parameters) => Gradient(sum1, sum2, n, parameters[0], parameters[1], gradient); if (!bfgs.Minimize()) { throw new ConvergenceException(); } this.alpha = bfgs.Solution[0]; this.beta = bfgs.Solution[1]; }
public void ConstructorTest2() { Function function = // min f(x) = 10 * (x+1)^2 + y^2 x => 10.0 * Math.Pow(x[0] + 1.0, 2.0) + Math.Pow(x[1], 2.0); Gradient gradient = x => new[] { 20 * (x[0] + 1), 2 * x[1] }; double[] start = new double[2]; var target = new BoundedBroydenFletcherGoldfarbShanno(2, function.Invoke, gradient.Invoke); Assert.IsTrue(target.Minimize()); double minimum = target.Value; double[] solution = target.Solution; Assert.AreEqual(0, minimum, 1e-10); Assert.AreEqual(-1, solution[0], 1e-5); Assert.AreEqual(0, solution[1], 1e-5); double expectedMinimum = function(target.Solution); Assert.AreEqual(expectedMinimum, minimum); }
public void NoGradientTest() { var target = new BoundedBroydenFletcherGoldfarbShanno(2) { Function = (x) => 0.0 }; target.Minimize(); }
public void MutableGradientSizeTest() { var target = new BoundedBroydenFletcherGoldfarbShanno(2) { Function = (x) => 0.0, Gradient = (x) => x }; Assert.Throws <InvalidOperationException>(() => target.Minimize(), ""); }
public void WrongGradientSizeTest() { var target = new BoundedBroydenFletcherGoldfarbShanno(2) { Function = (x) => 0.0, Gradient = (x) => new double[1] }; target.Minimize(); }
public void MutableGradientSizeTest() { var target = new BoundedBroydenFletcherGoldfarbShanno(2) { Function = (x) => 0.0, Gradient = (x) => x }; target.Minimize(); }
public void NoGradientTest() { var target = new BoundedBroydenFletcherGoldfarbShanno(2) { Function = (x) => 0.0 }; Assert.IsTrue(target.Minimize()); // The optimizer should use finite differences as the gradient }
/// <summary> /// Constructs a new L-BFGS learning algorithm. /// </summary> /// public QuasiNewtonLearning(ConditionalRandomField <T> model) { this.model = model; this.lbfgs = new BoundedBroydenFletcherGoldfarbShanno(model.Function.Weights.Length); this.lbfgs.FunctionTolerance = 1e-3; for (int i = 0; i < lbfgs.UpperBounds.Length; i++) { lbfgs.UpperBounds[i] = 1e10; lbfgs.LowerBounds[i] = -1e100; } }
private void init() { this.lbfgs = new BoundedBroydenFletcherGoldfarbShanno(Model.Function.Weights.Length); this.lbfgs.FunctionTolerance = Tolerance; this.lbfgs.MaxIterations = MaxIterations; for (int i = 0; i < lbfgs.UpperBounds.Length; i++) { lbfgs.UpperBounds[i] = 1e10; lbfgs.LowerBounds[i] = -1e100; } }
public void lbfgsTest2() { Accord.Math.Tools.SetupGenerator(0); // Suppose we would like to find the minimum of the function // // f(x,y) = -exp{-(x-1)²} - exp{-(y-2)²/2} // // First we need write down the function either as a named // method, an anonymous method or as a lambda function: Func <double[], double> f = (x) => - Math.Exp(-Math.Pow(x[0] - 1, 2)) - Math.Exp(-0.5 * Math.Pow(x[1] - 2, 2)); // Now, we need to write its gradient, which is just the // vector of first partial derivatives del_f / del_x, as: // // g(x,y) = { del f / del x, del f / del y } // Func <double[], double[]> g = (x) => new double[] { // df/dx = {-2 e^(- (x-1)^2) (x-1)} 2 * Math.Exp(-Math.Pow(x[0] - 1, 2)) * (x[0] - 1), // df/dy = {- e^(-1/2 (y-2)^2) (y-2)} Math.Exp(-0.5 * Math.Pow(x[1] - 2, 2)) * (x[1] - 2) }; // Finally, we can create the L-BFGS solver, passing the functions as arguments var lbfgs = new BoundedBroydenFletcherGoldfarbShanno(numberOfVariables: 2, function: f, gradient: g); // And then minimize the function: Assert.IsTrue(lbfgs.Minimize()); double minValue = lbfgs.Value; double[] solution = lbfgs.Solution; // The resultant minimum value should be -2, and the solution // vector should be { 1.0, 2.0 }. The answer can be checked on // Wolfram Alpha by clicking the following the link: // http://www.wolframalpha.com/input/?i=maximize+%28exp%28-%28x-1%29%C2%B2%29+%2B+exp%28-%28y-2%29%C2%B2%2F2%29%29 double expected = -2; Assert.AreEqual(expected, minValue, 1e-10); Assert.AreEqual(1, solution[0], 1e-3); Assert.AreEqual(2, solution[1], 1e-3); }
public OptimizationProgressEventArgs[] Actual(Specification problem) { ActualMessage = String.Empty; BoundedBroydenFletcherGoldfarbShanno target = new BoundedBroydenFletcherGoldfarbShanno(problem.Variables) { FunctionTolerance = factr, GradientTolerance = pgtol, Corrections = m, MaxIterations = max_iterations }; for (int i = 0; i < target.LowerBounds.Length; i++) { if (l != null) { target.LowerBounds[i] = l[i]; } if (u != null) { target.UpperBounds[i] = u[i]; } } target.Function = problem.Function; target.Gradient = problem.Gradient; actual.Clear(); target.Progress += new EventHandler <OptimizationProgressEventArgs>(target_Progress); target.Minimize((double[])problem.Start.Clone()); if (target.Status == BoundedBroydenFletcherGoldfarbShannoStatus.GradientConvergence) { ActualMessage = "CONVERGENCE: NORM_OF_PROJECTED_GRADIENT_<=_PGTOL"; } else if (target.Status == BoundedBroydenFletcherGoldfarbShannoStatus.FunctionConvergence) { ActualMessage = "CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH"; } else if (target.Status == BoundedBroydenFletcherGoldfarbShannoStatus.LineSearchFailed) { ActualMessage = "ABNORMAL_TERMINATION_IN_LNSRCH"; } return(actual.ToArray()); }
/// <summary> /// Inheritors of this class should create the optimization algorithm in this /// method, using the current <see cref="P:Accord.Statistics.Models.Fields.Learning.BaseHiddenGradientOptimizationLearning`2.MaxIterations" /> and <see cref="P:Accord.Statistics.Models.Fields.Learning.BaseHiddenGradientOptimizationLearning`2.Tolerance" /> /// settings. /// </summary> /// <returns>BoundedBroydenFletcherGoldfarbShanno.</returns> protected override BoundedBroydenFletcherGoldfarbShanno CreateOptimizer() { var lbfgs = new BoundedBroydenFletcherGoldfarbShanno(Model.Function.Weights.Length) { FunctionTolerance = Tolerance, MaxIterations = MaxIterations, }; for (int i = 0; i < lbfgs.UpperBounds.Length; i++) { lbfgs.UpperBounds[i] = 1e10; lbfgs.LowerBounds[i] = -1e100; } return(lbfgs); }
/// <summary> /// Constructs a new L-BFGS learning algorithm. /// </summary> /// public HiddenQuasiNewtonLearning(HiddenConditionalRandomField <T> model) { Model = model; calculator = new ForwardBackwardGradient <T>(model); lbfgs = new BoundedBroydenFletcherGoldfarbShanno(model.Function.Weights.Length); lbfgs.FunctionTolerance = 1e-3; lbfgs.Function = calculator.Objective; lbfgs.Gradient = calculator.Gradient; for (int i = 0; i < lbfgs.UpperBounds.Length; i++) { lbfgs.UpperBounds[i] = 1e10; lbfgs.LowerBounds[i] = -1e100; } }
public void lbfgsTest() { Func <double[], double> f = rosenbrockFunction; Func <double[], double[]> g = rosenbrockGradient; Assert.AreEqual(104, f(new[] { -1.0, 2.0 })); int n = 2; // number of variables double[] initial = { -1.2, 1 }; var lbfgs = new BoundedBroydenFletcherGoldfarbShanno(n, f, g); lbfgs.GradientTolerance = 1e-10; lbfgs.FunctionTolerance = 1e-10; double expected = 0; Assert.IsTrue(lbfgs.Minimize(initial)); double actual = lbfgs.Value; Assert.AreEqual(expected, actual, 1e-10); double[] result = lbfgs.Solution; //Assert.AreEqual(49, lbfgs.Evaluations); //Assert.AreEqual(40, lbfgs.Iterations); Assert.AreEqual(1.0, result[0], 1e-6); Assert.AreEqual(1.0, result[1], 1e-6); double y = f(result); double[] d = g(result); Assert.AreEqual(0, y, 1e-10); Assert.AreEqual(0, d[0], 1e-6); Assert.AreEqual(0, d[1], 1e-6); }
public void InvalidLineSearchTest2() { int n = 10; Func <double[], double> function = (parameters) => { return(-(n * Math.Log(0) - n * Math.Log(Math.PI))); }; Func <double[], double[]> gradient = (parameters) => { return(new[] { 2.0, Double.NegativeInfinity }); }; double[] start = { 0, 0 }; var lbfgs = new BoundedBroydenFletcherGoldfarbShanno(2, function, gradient); lbfgs.Minimize(start); Assert.AreEqual(BoundedBroydenFletcherGoldfarbShannoStatus.LineSearchFailed, lbfgs.Status); }
public void NoFunctionTest() { var target = new BoundedBroydenFletcherGoldfarbShanno(2); Assert.Throws <InvalidOperationException>(() => target.Minimize(), ""); }
public void NoFunctionTest() { var target = new BoundedBroydenFletcherGoldfarbShanno(2); target.Minimize(); }
/// <summary> /// Fits the underlying distribution to a given set of observations. /// </summary> /// /// <param name="observations">The array of observations to fit the model against. The array /// elements can be either of type double (for univariate data) or /// type double[] (for multivariate data).</param> /// <param name="weights">The weight vector containing the weight for each of the samples.</param> /// <param name="options">Optional arguments which may be used during fitting, such /// as regularization constants and additional parameters.</param> /// /// <remarks> /// Although both double[] and double[][] arrays are supported, /// providing a double[] for a multivariate distribution or a /// double[][] for a univariate distribution may have a negative /// impact in performance. /// </remarks> /// /// <example> /// See <see cref="CauchyDistribution"/>. /// </example> /// public void Fit(double[] observations, double[] weights, CauchyOptions options) { if (immutable) { throw new InvalidOperationException("This object can not be modified."); } if (weights != null) { throw new ArgumentException("This distribution does not support weighted samples."); } bool useMLE = true; bool estimateT = true; bool estimateS = true; if (options != null) { useMLE = options.MaximumLikelihood; estimateT = options.EstimateLocation; estimateS = options.EstimateScale; } double t0 = location; double s0 = scale; int n = observations.Length; DoubleRange range; double median = Measures.Quartiles(observations, out range, alreadySorted: false); if (estimateT) { t0 = median; } if (estimateS) { s0 = range.Length; } if (useMLE) { // Minimize the log-likelihood through numerical optimization var lbfgs = new BoundedBroydenFletcherGoldfarbShanno(2); lbfgs.LowerBounds[1] = 0; // scale must be positive // Define the negative log-likelihood function, // which is the objective we want to minimize: lbfgs.Function = (parameters) => { // Assume location is the first // parameter, shape is the second double t = (estimateT) ? parameters[0] : t0; double s = (estimateS) ? parameters[1] : s0; if (s < 0) { s = -s; } double sum = 0; for (int i = 0; i < observations.Length; i++) { double y = (observations[i] - t); sum += Math.Log(s * s + y * y); } return(-(n * Math.Log(s) - sum - n * Math.Log(Math.PI))); }; lbfgs.Gradient = (parameters) => { // Assume location is the first // parameter, shape is the second double t = (estimateT) ? parameters[0] : t0; double s = (estimateS) ? parameters[1] : s0; double sum1 = 0, sum2 = 0; for (int i = 0; i < observations.Length; i++) { double y = (observations[i] - t); sum1 += y / (s * s + y * y); sum2 += s / (s * s + y * y); } double dt = -2.0 * sum1; double ds = +2.0 * sum2 - n / s; double[] g = new double[2]; g[0] = estimateT ? dt : 0; g[1] = estimateS ? ds : 0; return(g); }; // Initialize using the sample median as starting // value for location, and half interquartile range // for shape. double[] values = { t0, s0 }; // Minimize lbfgs.Minimize(values); // Check solution t0 = lbfgs.Solution[0]; s0 = lbfgs.Solution[1]; } init(t0, s0); // Become the new distribution }
private ConjugateGradientCode cvsmod(ref double f, double[] s, ref double stp, ref int info, ref int nfev, double[] wa, ref double dginit, ref double dgout) { var n = NumberOfVariables; var x = Solution; if (info == 1) { goto L321; } infoc = 1; if (stp <= 0) // Check the input parameters for errors { return(ConjugateGradientCode.StepSize); } // Compute the initial gradient in the search direction // and check that S is a descent direction. if (dginit >= 0) { throw new LineSearchFailedException(0, "The search direction is not a descent direction."); } // Initialize local variables brackt = false; stage1 = true; nfev = 0; finit = f; dgtest = ftol * dginit; width = stpmax - stpmin; width1 = width / 0.5; for (var j = 0; j < x.Length; ++j) { wa[j] = x[j]; } // The variables STX, FX, DGX contain the values of the step, // function, and directional derivative at the best step. // The variables STY, FY, DGY contain the value of the step, // function, and derivative at the other endpoint of the interval // of uncertainty. // The variables STP, F, DG contain the values of the step, // function, and derivative at the current step. stx = 0; fx = finit; dgx = dginit; sty = 0; fy = finit; dgy = dginit; L30: // Start of iteration. // Set the minimum and maximum steps to correspond // to the present interval of uncertainty. if (brackt) { stmin = Math.Min(stx, sty); stmax = Math.Max(stx, sty); } else { stmin = stx; stmax = stp + 4 * (stp - stx); } // Force the step to be within // the bounds STPMAX and STPMIN. stp = Math.Max(stp, stpmin); stp = Math.Min(stp, stpmax); // If an unusual termination is to occur then // let STP be the lowest point obtained so far. if (brackt && (stp <= stmin || stp >= stmax) || nfev >= maxfev - 1 || infoc == 0 || brackt && stmax - stmin <= xtol * stmax) { stp = stx; } // Evaluate the function and gradient at STP // and compute the directional derivative. for (var j = 0; j < s.Length; ++j) { x[j] = wa[j] + stp * s[j]; } // Fetch function and gradient f = Function(x); g = Gradient(x); info = 0; nfev++; dg2 = 0; for (var j = 0; j < g.Length; ++j) { dg2 += g[j] * s[j]; } ftest1 = finit + stp * dgtest; if (brackt && (stp <= stmin || stp >= stmax) || infoc == 0) { return(ConjugateGradientCode.RoundingErrors); } if (stp == stpmax && f <= ftest1 && dg2 <= dgtest) { return(ConjugateGradientCode.StepHigh); } if (stp == stpmin && (f > ftest1 || dg2 >= dgtest)) { return(ConjugateGradientCode.StepLow); } if (nfev >= maxfev) { return(ConjugateGradientCode.MaximumEvaluations); } if (brackt && stmax - stmin <= xtol * stmax) { return(ConjugateGradientCode.Precision); } // More's code has been modified so that at least one new // function value is computed during the line search (enforcing // at least one interpolation is not easy, since the code may // override an interpolation) if (f <= ftest1 && Math.Abs(dg2) <= gtol * -dginit && nfev > 1) { info = 1; dgout = dg2; return(ConjugateGradientCode.Success); } L321: // In the first stage we seek a step for which the modified // function has a nonpositive value and nonnegative derivative. if (stage1 && f <= ftest1 && dg2 >= Math.Min(ftol, gtol) * dginit) { stage1 = false; } // A modified function is used to predict the step only if // we have not obtained a step for which the modified function // has a nonpositive function value and nonnegative derivative, // and if a lower function value has been obtained but the // decrease is not sufficient. if (stage1 && f <= fx && f > ftest1) { // Define the modified function and derivative values var fm = f - stp * dgtest; var fxm = fx - stx * dgtest; var fym = fy - sty * dgtest; var dgm = dg2 - dgtest; var dgxm = dgx - dgtest; var dgym = dgy - dgtest; // Call CSTEPM to update the interval of // uncertainty and to compute the new step. BoundedBroydenFletcherGoldfarbShanno.dcstep(ref stx, ref fxm, ref dgxm, ref sty, ref fym, ref dgym, ref stp, fm, dgm, ref brackt, stpmin, stpmax); // Reset the function and gradient values for f. fx = fxm + stx * dgtest; fy = fym + sty * dgtest; dgx = dgxm + dgtest; dgy = dgym + dgtest; } else { // Call CSTEPM to update the interval of // uncertainty and to compute the new step. BoundedBroydenFletcherGoldfarbShanno.dcstep(ref stx, ref fx, ref dgx, ref sty, ref fy, ref dgy, ref stp, f, dg2, ref brackt, stpmin, stpmax); } // Force a sufficient decrease in the // size of the interval of uncertainty. if (brackt) { if (Math.Abs(sty - stx) >= 0.66 * width1) { stp = stx + 0.5 * (sty - stx); } width1 = width; width = Math.Abs(sty - stx); } goto L30; }