public void GradientTest3() { var hmm = MultivariateNormalHiddenMarkovClassifierPotentialFunctionTest.CreateModel1(); var function = new MultivariateNormalMarkovClassifierFunction(hmm); var model = new HiddenConditionalRandomField <double[]>(function); var target = new QuasiNewtonHiddenLearning <double[]>(model); target.Regularization = 2; var inputs = inputs1; var outputs = outputs1; FiniteDifferences diff = new FiniteDifferences(function.Weights.Length); diff.Function = parameters => func(model, parameters, inputs, outputs, target.Regularization); double[] expected = diff.Compute(function.Weights); double[] actual = target.Gradient(function.Weights, inputs, outputs); for (int i = 0; i < actual.Length; i++) { Assert.AreEqual(expected[i], actual[i], 1e-3); Assert.IsFalse(double.IsNaN(actual[i])); Assert.IsFalse(double.IsNaN(expected[i])); } }
public void QuadraticConstraintConstructorTest() { IObjectiveFunction objective = null; double[,] quadraticTerms = { { 1, 2, 3 }, { 4, 5, 6 }, { 7, 8, 9 }, }; double[] linearTerms = { 1, 2, 3 }; objective = new NonlinearObjectiveFunction(3, f => f[0] + f[1] + f[2]); QuadraticConstraint target = new QuadraticConstraint(objective, quadraticTerms, linearTerms, ConstraintType.LesserThanOrEqualTo, 0); var function = target.Function; var gradient = target.Gradient; FiniteDifferences fd = new FiniteDifferences(3, function); double[][] x = { new double[] { 1, 2, 3 }, new double[] { 3, 1, 4 }, new double[] { -6, 5, 9 }, new double[] { 31, 25, 246 }, new double[] { -0.102, 0, 10 }, }; { // Function test for (int i = 0; i < x.Length; i++) { double expected = (x[i].Multiply(quadraticTerms)).InnerProduct(x[i]) + linearTerms.InnerProduct(x[i]); double actual = function(x[i]); Assert.AreEqual(expected, actual, 1e-8); } } { // Gradient test for (int i = 0; i < x.Length; i++) { double[] expected = fd.Compute(x[i]); double[] actual = gradient(x[i]); for (int j = 0; j < actual.Length; j++) { Assert.AreEqual(expected[j], actual[j], 1e-8); } } } }
private static double[] finiteDifferences(double[][] input, double[] output, bool stochastic) { LogisticRegression regression; LogisticGradientDescent teacher; regression = new LogisticRegression(inputs: 2); teacher = new LogisticGradientDescent(regression) { Stochastic = stochastic, LearningRate = 1e-4, }; FiniteDifferences diff = new FiniteDifferences(3); diff.Function = (x) => { for (int i = 0; i < x.Length; i++) { regression.Coefficients[i] = x[i]; } return(regression.GetLogLikelihood(input, output)); }; return(diff.Compute(regression.Coefficients)); }
public void GradientTest4() { var hmm = IndependentMarkovClassifierPotentialFunctionTest.CreateModel2(); var function = new MarkovMultivariateFunction(hmm); var model = new HiddenConditionalRandomField <double[]>(function); var target = new ForwardBackwardGradient <double[]>(model); target.Regularization = 0; FiniteDifferences diff = new FiniteDifferences(function.Weights.Length); diff.Function = parameters => func(model, parameters, IndependentMarkovClassifierPotentialFunctionTest.sequences, IndependentMarkovClassifierPotentialFunctionTest.labels); double[] expected = diff.Compute(function.Weights); double[] actual = target.Gradient(function.Weights, IndependentMarkovClassifierPotentialFunctionTest.sequences, IndependentMarkovClassifierPotentialFunctionTest.labels); for (int i = 0; i < actual.Length; i++) { if (double.IsNaN(expected[i])) { continue; } Assert.AreEqual(expected[i], actual[i], 1e-5); Assert.IsFalse(double.IsNaN(actual[i])); } }
public void GradientTest3() { double[][][] sequences2; int[] labels2; var hmm = CreateModel3(out sequences2, out labels2); var function = new MarkovMultivariateFunction(hmm); var model = new HiddenConditionalRandomField <double[]>(function); var target = new ForwardBackwardGradient <double[]>(model); target.Regularization = 2; var inputs = sequences2; var outputs = labels2; FiniteDifferences diff = new FiniteDifferences(function.Weights.Length); diff.Function = parameters => func(model, parameters, inputs, outputs, target.Regularization); double[] expected = diff.Compute(function.Weights); double[] actual = target.Gradient(function.Weights, inputs, outputs); for (int i = 0; i < actual.Length; i++) { double e = expected[i]; double a = actual[i]; Assert.AreEqual(e, a, 1e-3); Assert.IsFalse(double.IsNaN(actual[i])); Assert.IsFalse(double.IsNaN(expected[i])); } }
public void GradientTest_MarkovIndependentNormal_NoPriors() { double[][][] observations; int[] labels; HiddenMarkovClassifier <Independent <NormalDistribution> > hmm = IndependentMarkovFunctionTest.CreateModel4(out observations, out labels, usePriors: false); var function = new MarkovMultivariateFunction(hmm); var model = new HiddenConditionalRandomField <double[]>(function); var target = new ForwardBackwardGradient <double[]>(model); target.Regularization = 0; FiniteDifferences diff = new FiniteDifferences(function.Weights.Length); diff.Function = parameters => func(model, parameters, observations, labels); double[] expected = diff.Compute(function.Weights); double[] actual = target.Gradient(function.Weights, observations, labels); for (int i = 0; i < actual.Length; i++) { if (double.IsNaN(expected[i])) { continue; } Assert.AreEqual(expected[i], actual[i], 1e-5); Assert.IsFalse(double.IsNaN(actual[i])); } }
public void GradientDeoptimizeTest2() { double[][][] sequences2; int[] labels2; var hmm = CreateModel3(out sequences2, out labels2); var function = new MarkovMultivariateFunction(hmm); #pragma warning disable 0618 function.Deoptimize(); #pragma warning restore 0618 var model = new HiddenConditionalRandomField <double[]>(function); var target = new ForwardBackwardGradient <double[]>(model); var inputs = sequences2; var outputs = labels2; double[] actual = target.Gradient(function.Weights, inputs, outputs); FiniteDifferences diff = new FiniteDifferences(function.Weights.Length); diff.Function = parameters => func(model, parameters, inputs, outputs); double[] expected = diff.Compute(function.Weights); for (int i = 0; i < actual.Length; i++) { Assert.AreEqual(expected[i], actual[i], 1e-3); Assert.IsFalse(double.IsNaN(actual[i])); Assert.IsFalse(double.IsNaN(expected[i])); } }
public void GradientTest_DiscreteMarkov() { var function = new MarkovDiscreteFunction(2, 2, 2); var model = new HiddenConditionalRandomField <int>(function); var target = new ForwardBackwardGradient <int>(model); FiniteDifferences diff = new FiniteDifferences(function.Weights.Length) { StepSize = 1e-5 }; var inputs = QuasiNewtonHiddenLearningTest.inputs; var outputs = QuasiNewtonHiddenLearningTest.outputs; diff.Function = parameters => func(model, parameters, inputs, outputs); double[] expected = diff.Compute(function.Weights); double[] actual = target.Gradient(function.Weights, inputs, outputs); for (int i = 0; i < actual.Length; i++) { Assert.AreEqual(expected[i], actual[i], 1e-4); } }
public void GradientTest3() { HiddenMarkovClassifier hmm = HiddenMarkovClassifierPotentialFunctionTest.CreateModel1(); var function = new MarkovDiscreteFunction(hmm); var model = new HiddenConditionalRandomField <int>(function); var target = new ForwardBackwardGradient <int>(model); target.Regularization = 2; FiniteDifferences diff = new FiniteDifferences(function.Weights.Length); diff.Function = parameters => func(model, parameters, target.Regularization); double[] expected = diff.Compute(function.Weights); double[] actual = target.Gradient(function.Weights, inputs, outputs); for (int i = 0; i < actual.Length; i++) { Assert.AreEqual(expected[i], actual[i], 1e-5); Assert.IsFalse(double.IsNaN(actual[i])); Assert.IsFalse(double.IsNaN(expected[i])); } }
public void GradientTest_MarkovNormal_Regularization() { var hmm = MarkovContinuousFunctionTest.CreateModel1(); var function = new MarkovContinuousFunction(hmm); var model = new HiddenConditionalRandomField <double>(function); var target = new ForwardBackwardGradient <double>(model); target.Regularization = 2; var inputs = NormalQuasiNewtonHiddenLearningTest.inputs; var outputs = NormalQuasiNewtonHiddenLearningTest.outputs; FiniteDifferences diff = new FiniteDifferences(function.Weights.Length); diff.Function = parameters => func(model, parameters, inputs, outputs, target.Regularization); double[] expected = diff.Compute(function.Weights); double[] actual = target.Gradient(function.Weights, inputs, outputs); for (int i = 0; i < actual.Length; i++) { Assert.AreEqual(expected[i], actual[i], 1e-2); Assert.IsFalse(double.IsNaN(actual[i])); Assert.IsFalse(double.IsNaN(expected[i])); } }
public void Hessian_test_3() { // x² + log(y) + xy + exp(x+y) + 47 Func <double[], double> function = x => Math.Pow(x[0], 2) + Math.Log(x[1]) + x[0] * x[1] + Math.Exp(x[0] + x[1]) + 47; var calculator = new FiniteDifferences(variables: 2) { Function = function, NumberOfPoints = 7 }; Func <double[], double[][]> expectedFormula = (double[] x) => new double[][] { new double[] { Math.Exp(x[0] + x[1]) + 2, Math.Exp(x[0] + x[1]) + 1 }, new double[] { Math.Exp(x[0] + x[1]) + 1, Math.Exp(x[0] + x[1]) - 1.0 / Math.Pow(x[1], 2) }, }; for (double i = 1; i < 10; i++) { for (double j = 1; j < 10; j++) { double[] value = new double[] { i, j }; double[][] actual = calculator.Hessian(value); double[][] expected = expectedFormula(value); Assert.IsTrue(actual.IsEqual(expected, rtol: 1e-5)); } } }
double[] diffedRange(double z, double[] a) { Func <double[], double> fbeta3 = x => fD(z, a); var fDiffs = new FiniteDifferences(N, fbeta3); double[] result = fDiffs.Gradient(a); return(result); }
/// <summary> /// Gets the probability density function (pdf) for /// this distribution evaluated at point <c>x</c>. /// </summary> /// /// <param name="x">A single point in the distribution range.</param> /// /// <returns> /// The probability of <c>x</c> occurring /// in the current distribution. /// </returns> /// protected internal override double InnerProbabilityDensityFunction(double x) { if (pdf != null) { return(pdf(x)); } return(FiniteDifferences.Derivative(cdf, x, 1, 1e-6)); }
/// <summary> /// Gets the probability density function (pdf) for /// this distribution evaluated at point <c>x</c>. /// </summary> /// /// <param name="x">A single point in the distribution range.</param> /// /// <returns> /// The probability of <c>x</c> occurring /// in the current distribution. /// </returns> /// public override double ProbabilityDensityFunction(double x) { if (pdf != null) { return(pdf(x)); } return(FiniteDifferences.Derivative(cdf, x, 1, 1e-6)); }
public void LinearTest() { double[,] quadraticTerms = { { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, }; double[] linearTerms = { 1, 2, 3 }; var target = new QuadraticObjectiveFunction(quadraticTerms, linearTerms); var function = target.Function; var gradient = target.Gradient; FiniteDifferences fd = new FiniteDifferences(3, function); double[][] x = { new double[] { 1, 2, 3 }, new double[] { 3, 1, 4 }, new double[] { -6, 5, 9 }, new double[] { 31, 25, 246 }, new double[] { -0.102, 0, 10 }, }; { // Function test for (int i = 0; i < x.Length; i++) { double expected = 0.5 * (x[i] .Dot(quadraticTerms)) .Dot(x[i]) + linearTerms .Dot(x[i]); double actual = function(x[i]); Assert.AreEqual(expected, actual, 1e-8); } } { // Gradient test for (int i = 0; i < x.Length; i++) { double[] expected = fd.Gradient(x[i]); double[] actual = gradient(x[i]); for (int j = 0; j < actual.Length; j++) { Assert.AreEqual(expected[j], actual[j], 1e-8); } } } }
/// <summary> /// Finds the minimum value of a function. The solution vector /// will be made available at the <see cref="IOptimizationMethod{TInput, TOutput}.Solution"/> property. /// </summary> /// /// <returns>Returns <c>true</c> if the method converged to a <see cref="IOptimizationMethod{TInput, TOutput}.Solution"/>. /// In this case, the found value will also be available at the <see cref="IOptimizationMethod{TInput, TOutput}.Value"/> /// property.</returns> /// public override bool Minimize() { if (Gradient == null) { this.Gradient = FiniteDifferences.Gradient(Function, NumberOfVariables); } NonlinearObjectiveFunction.CheckGradient(Gradient, Solution); return(base.Minimize()); }
public static double[] InverseKinematics(List <MDHParameters> dht, double[] target, ref bool success) { Func <double[], double> f = x => Distance(dht, target, x); var calculator = new FiniteDifferences(dht.Count, f); Func <double[], double[]> g = calculator.Gradient; var optimizer = new BroydenFletcherGoldfarbShanno(numberOfVariables: dht.Count, function: f, gradient: g); optimizer.Minimize(); success = !(optimizer.Value >= 0.01); return(optimizer.Solution); }
public void ComputeTest() { int numberOfParameters = 2; FiniteDifferences target = new FiniteDifferences(numberOfParameters); double[] inputs = { -1, 0.4 }; target.Function = BroydenFletcherGoldfarbShannoTest.rosenbrockFunction; double[] expected = BroydenFletcherGoldfarbShannoTest.rosenbrockGradient(inputs); double[] actual = target.Compute(inputs); Assert.IsTrue(expected.IsEqual(actual, 0.05)); }
public void Hessian_test_2() { Func <double[], double> function = x => Math.Pow(x[0], 2) + x[1] + x[0] * x[1] + 47; var calculator = new FiniteDifferences(2, function); double[][] result = calculator.Hessian(new[] { 2.0, -1.0 }); double[][] expected = { new double[] { 2, 1 }, new double[] { 1, 0 }, }; Assert.IsTrue(result.IsEqual(expected, 1e-8)); }
private static double[] finiteDifferences(double[][] input, double[] output, bool stochastic) { var regression = new LogisticRegression(inputs: 2); var diff = new FiniteDifferences(3); diff.Function = (x) => { for (int i = 0; i < x.Length; i++) { regression.Coefficients[i] = x[i]; } return(regression.GetLogLikelihood(input, output)); }; return(diff.Compute(regression.Coefficients)); }
/// <summary> /// Finds the maximum value of a function. The solution vector /// will be made available at the <see cref="IOptimizationMethod{TInput, TOutput}.Solution"/> property. /// </summary> /// /// <returns>Returns <c>true</c> if the method converged to a <see cref="IOptimizationMethod{TInput, TOutput}.Solution"/>. /// In this case, the found value will also be available at the <see cref="IOptimizationMethod{TInput, TOutput}.Value"/> /// property.</returns> /// public override bool Maximize() { if (Gradient == null) { this.Gradient = FiniteDifferences.Gradient(Function, NumberOfVariables); } NonlinearObjectiveFunction.CheckGradient(Gradient, Solution); var g = Gradient; Gradient = (x) => g(x).Multiply(-1); bool success = base.Maximize(); Gradient = g; return(success); }
public void GradientTest_MarkovMultivariate() { // Creates a sequence classifier containing 2 hidden Markov Models // with 2 states and an underlying Normal distribution as density. MultivariateNormalDistribution density = new MultivariateNormalDistribution(3); var hmm = new HiddenMarkovClassifier <MultivariateNormalDistribution>(2, new Ergodic(2), density); double[][][] inputs = { new [] { new double[] { 0, 1, 0 }, new double[] { 0, 1, 0 }, new double[] { 0, 1, 0 } }, new [] { new double[] { 1, 6, 2 }, new double[] { 2, 1, 6 }, new double[] { 1, 1, 0 } }, new [] { new double[] { 9, 1, 0 }, new double[] { 0, 1, 5 }, new double[] { 0, 0, 0 } }, }; int[] outputs = { 0, 0, 1 }; var function = new MarkovMultivariateFunction(hmm); var model = new HiddenConditionalRandomField <double[]>(function); var target = new ForwardBackwardGradient <double[]>(model); FiniteDifferences diff = new FiniteDifferences(function.Weights.Length) { StepSize = 1e-5 }; diff.Function = parameters => func(model, parameters, inputs, outputs); double[] expected = diff.Compute(function.Weights); double[] actual = target.Gradient(function.Weights, inputs, outputs); for (int i = 0; i < actual.Length; i++) { Assert.AreEqual(expected[i], actual[i], 0.05); Assert.IsFalse(double.IsNaN(actual[i])); Assert.IsFalse(double.IsNaN(expected[i])); } }
public void Hessian_test_4() { const int Size = 30; const double Tolerance = 1e-8; for (int i = 0; i < 10; i++) { double[,] mat = Matrix.Random(Size); double[,] Q = mat.DotWithTransposed(mat); double[] d = Vector.Random(Size); var qof = new QuadraticObjectiveFunction(Q, d); var calculator = new FiniteDifferences(Size, qof.Function); double[][] result = calculator.Hessian(Vector.Random(Size)); Assert.IsTrue(result.IsEqual(Q, Tolerance)); } }
public void ComputeTest2() { // Create a simple function with two parameters: f(x,y) = x² + y Func <double[], double> function = x => Math.Pow(x[0], 2) + x[1]; // The gradient w.r.t to x should be 2x, // the gradient w.r.t to y should be 1 // Create a new finite differences calculator var calculator = new FiniteDifferences(2, function); // Evaluate the gradient function at the point (2, -1) double[] result = calculator.Compute(2, -1); // answer is (4, 1) Assert.AreEqual(4, result[0], 1e-10); Assert.AreEqual(1, result[1], 1e-10); Assert.IsFalse(Double.IsNaN(result[0])); Assert.IsFalse(Double.IsNaN(result[1])); }
public void test_order() { // https://www.wolframalpha.com/input/?i=third+derivative+of+(1+-+x)%5E2+%2B+100(y+-+x%5E2)%5E2+at+(-1,0.4) int numberOfParameters = 2; FiniteDifferences target = new FiniteDifferences(numberOfParameters) { NumberOfPoints = 7, Order = 3, }; double[] inputs = { -1, 0.4 }; target.Function = BroydenFletcherGoldfarbShannoTest.rosenbrockFunction; double[] expected = { -2400, 0 }; double[] actual = target.Compute(inputs); Assert.IsTrue(expected.IsEqual(actual, 1e-5)); }
public void GradientTest() { var function = new MarkovDiscreteFunction(2, 2, 2); var model = new HiddenConditionalRandomField <int>(function); var target = new ForwardBackwardGradient <int>(model); FiniteDifferences diff = new FiniteDifferences(function.Weights.Length); diff.Function = parameters => func(model, parameters); double[] expected = diff.Compute(function.Weights); double[] actual = target.Gradient(function.Weights, inputs, outputs); for (int i = 0; i < actual.Length; i++) { Assert.AreEqual(expected[i], actual[i], 1e-4); Assert.IsFalse(double.IsNaN(actual[i])); Assert.IsFalse(double.IsNaN(expected[i])); } }
public void HomogeneousTest() { double[,] quadraticTerms = { { 8, 3, 1 }, { 3, 4, 2 }, { 1, 2, 6 }, }; double[] linearTerms = { 0, 0, 0 }; var target = new QuadraticObjectiveFunction(quadraticTerms, linearTerms); var function = target.Function; var gradient = target.Gradient; FiniteDifferences fd = new FiniteDifferences(3, function); double[][] x = { new double[] { 1, 2, 3 }, new double[] { 3, 1, 4 }, new double[] { -6, 5, 9 }, new double[] { 31, 25, 246 }, new double[] { -0.102, 0, 10 }, }; { // Gradient test for (int i = 0; i < x.Length; i++) { double[] expected = fd.Gradient(x[i]); double[] actual = gradient(x[i]); for (int j = 0; j < actual.Length; j++) { Assert.AreEqual(expected[j], actual[j], 1e-6); } } } }
public void GradientTest() { for (double a = 0.1; a < 3; a += 0.1) { for (double b = 0.1; b < 3; b += 0.1) { var target = new BetaDistribution(a, b); Assert.AreEqual(a, target.Alpha); Assert.AreEqual(b, target.Beta); FiniteDifferences fd = new FiniteDifferences(2); fd.Function = (double[] parameters) => BetaDistribution.LogLikelihood(samples, parameters[0], parameters[1]); double[] expected = fd.Compute(a, b); double[] actual = BetaDistribution.Gradient(samples, a, b); Assert.IsTrue(expected[0].IsRelativelyEqual(actual[0], 0.05)); Assert.IsTrue(expected[1].IsRelativelyEqual(actual[1], 0.05)); } } }
public void GradientTest() { double[][] inputs; int[] outputs; MultinomialLogisticRegressionTest.CreateInputOutputsExample1(out inputs, out outputs); // Create an algorithm to estimate the regression var msgd = new MultinomialLogisticLearning <ConjugateGradient>(); msgd.Method.MaxIterations = 1; msgd.Learn(inputs, outputs); int variables = inputs.Columns() * outputs.DistinctCount(); var fd = new FiniteDifferences(variables, msgd.crossEntropy); double[] probe = { 0.1, 0.2, 0.5, 0.6, 0.2, 0.1 }; double[] expected = fd.Compute(probe); double[] actual = msgd.crossEntropyGradient(probe); Assert.IsTrue(expected.IsEqual(actual, 1e-5)); }
public void Hessian_test() { #region doc_hessian // Create a simple function with two parameters: f(x,y) = x² + y Func <double[], double> function = x => Math.Pow(x[0], 2) + x[1]; // The gradient w.r.t to x should be 2x, // the gradient w.r.t to y should be 1 // Create a new finite differences calculator var calculator = new FiniteDifferences(2, function); // Evaluate the gradient function at the point (2, -1) double[][] result = calculator.Hessian(new[] { 2.0, -1.0 }); // answer is [(2, 0), (0, 0)] #endregion double[][] expected = { new double[] { 2, 0 }, new double[] { 0, 0 }, }; Assert.IsTrue(result.IsEqual(expected, 1e-8)); }