/// <summary> /// Tests the gradient using finite differences on each axis in the list /// </summary> /// <param name="f">Function to test</param> /// <param name="x">Point at which to test</param> /// <param name="coords">List of coordinates to test</param> public static void TestCoords(DifferentiableFunction f, ref VBuffer <Float> x, IList <int> coords) { // REVIEW: Delete this method? VBuffer <Float> grad = default(VBuffer <Float>); VBuffer <Float> newGrad = default(VBuffer <Float>); VBuffer <Float> newX = default(VBuffer <Float>); Float val = f(ref x, ref grad, null); Float normX = VectorUtils.Norm(x); Console.WriteLine(Header); Random r = new Random(5); VBuffer <Float> dir = new VBuffer <Float>(x.Length, 1, new Float[] { 1 }, new int[] { 0 }); foreach (int n in coords) { dir.Values[0] = n; VectorUtils.AddMultInto(ref x, Eps, ref dir, ref newX); Float rVal = f(ref newX, ref newGrad, null); VectorUtils.AddMultInto(ref x, -Eps, ref dir, ref newX); Float lVal = f(ref newX, ref newGrad, null); Float dirDeriv = VectorUtils.DotProduct(ref grad, ref dir); Float numDeriv = (rVal - lVal) / (2 * Eps); Float normDiff = Math.Abs(1 - numDeriv / dirDeriv); Float diff = numDeriv - dirDeriv; Console.WriteLine("{0,-9}{1,-18:0.0000e0}{2,-18:0.0000e0}{3,-15:0.0000e0}{4,0:0.0000e0}", n, numDeriv, dirDeriv, diff, normDiff); } }
private double[] computeDirection(DifferentiableFunction monitor, LineSearchResult lsr) { // implemented two-loop hessian update method. double[] direction = lsr.GradAtNext.Clone() as double[]; double[] @as = new double[m]; // first loop for (int i = updateInfo.kCounter - 1; i >= 0; i--) { @as[i] = updateInfo.getRho(i) * ArrayMath.innerProduct(updateInfo.getS(i), direction); for (int ii = 0; ii < dimension; ii++) { direction[ii] = direction[ii] - @as[i] * updateInfo.getY(i)[ii]; } } // second loop for (int i = 0; i < updateInfo.kCounter; i++) { double b = updateInfo.getRho(i) * ArrayMath.innerProduct(updateInfo.getY(i), direction); for (int ii = 0; ii < dimension; ii++) { direction[ii] = direction[ii] + (@as[i] - b) * updateInfo.getS(i)[ii]; } } for (int i = 0; i < dimension; i++) { direction[i] *= -1.0; } return(direction); }
/// <summary> /// Tests the gradient reported by <paramref name="f"/>. /// </summary> /// <param name="f">Function to test</param> /// <param name="x">Point at which to test</param> /// <param name="dir">Direction to test derivative</param> /// <param name="quiet">Whether to disable output</param> /// <param name="newGrad">This is a reusable working buffer for intermediate calculations</param> /// <param name="newX">This is a reusable working buffer for intermediate calculations</param> /// <returns>Normalized difference between analytic and numeric directional derivative</returns> public static Float Test(DifferentiableFunction f, ref VBuffer <Float> x, ref VBuffer <Float> dir, bool quiet, ref VBuffer <Float> newGrad, ref VBuffer <Float> newX) { Float normDir = VectorUtils.Norm(dir); Float val = f(ref x, ref newGrad, null); Float dirDeriv = VectorUtils.DotProduct(ref newGrad, ref dir); Float scaledEps = Eps / normDir; VectorUtils.AddMultInto(ref x, scaledEps, ref dir, ref newX); Float rVal = f(ref newX, ref newGrad, null); VectorUtils.AddMultInto(ref x, -scaledEps, ref dir, ref newX); Float lVal = f(ref newX, ref newGrad, null); Float numDeriv = (rVal - lVal) / (2 * scaledEps); Float normDiff = Math.Abs(1 - numDeriv / dirDeriv); Float diff = numDeriv - dirDeriv; if (!quiet) { Console.WriteLine("{0,-18:0.0000e0}{1,-18:0.0000e0}{2,-15:0.0000e0}{3,0:0.0000e0}", numDeriv, dirDeriv, diff, normDiff); } return(normDiff); }
internal FunctionOptimizerState(IChannel ch, IProgressChannelProvider progress, DifferentiableFunction function, ref VBuffer <Float> initial, int m, long totalMemLimit, bool keepDense, bool enforceNonNegativity) : base(ch, progress, ref initial, m, totalMemLimit, keepDense, enforceNonNegativity) { Function = function; Init(); }
public void Evaluate(DifferentiableFunction function, Jet[] parameters, double[] values, double[][] jacobian) { if (this.index < this.domain) { throw new Exception("Insufficient parameters added"); } if (this.domain != parameters.Length) { throw new ArgumentException("Insufficient parameters provided"); } var f = new Jet[this.range]; function(f, parameters); for (int i = 0; i < this.range; ++i) { values[i] = f[i].Real; for (int j = 0; j < this.domain; ++j) { jacobian[i][j] = f[i].Infinitesimals[j]; } } }
public LineFunc(DifferentiableFunction function, ref VBuffer <Float> initial, bool useCG = false) { int dim = initial.Length; initial.CopyTo(ref _point); _func = function; // REVIEW: plumb the IProgressChannelProvider through. _value = _func(ref _point, ref _grad, null); VectorUtils.ScaleInto(ref _grad, -1, ref _dir); _useCG = useCG; }
internal L1OptimizerState(IChannel ch, IProgressChannelProvider progress, DifferentiableFunction function, ref VBuffer <Float> initial, int m, long totalMemLimit, int biasCount, Float l1Weight, bool keepDense, bool enforceNonNegativity) : base(ch, progress, ref initial, m, totalMemLimit, keepDense, enforceNonNegativity) { Contracts.AssertValue(ch); ch.Assert(0 <= biasCount && biasCount < initial.Length); ch.Assert(l1Weight > 0); _biasCount = biasCount; _l1weight = l1Weight; _function = function; Init(); }
public void T01_OneDimensional() { const double Accuracy = 1.5e-7; // sin(x+1) + x/2 has local minima at -2/3PI-1, 4/3PI-1, 10/3PI-1, etc. DifferentiableFunction function = new DifferentiableFunction(x => Math.Sin(x + 1) + x / 2, x => Math.Cos(x + 1) + 0.5); Func <double, double> ndFunction = function.Evaluate; // create a version without derivative information // the three minima above are located between -5 and 13, so we should be able to find them with BracketInward() List <MinimumBracket> brackets = Minimize.BracketInward(function, -5, 13, 3).ToList(); Assert.AreEqual(3, brackets.Count); // ensure we found them all List <double> minima = new List <double>(); // for each bracket, try to find it using all available methods foreach (MinimumBracket bracket in brackets) { double x = Minimize.GoldenSection(function, bracket); // first use golden section search, which is the most reliable Assert.AreEqual(x, Minimize.Brent(function, bracket), Accuracy); // then make sure Brent's method gives a similar answer, both with Assert.AreEqual(x, Minimize.Brent(ndFunction, bracket), Accuracy); // and without the derivative minima.Add(x); } minima.Sort(); // then sort the results to put them in a known order and make sure they're equal to the expected values Assert.AreEqual(3, minima.Count); Assert.AreEqual(Math.PI * -2 / 3 - 1, minima[0], Accuracy); Assert.AreEqual(Math.PI * 4 / 3 - 1, minima[1], Accuracy); Assert.AreEqual(Math.PI * 10 / 3 - 1, minima[2], Accuracy); // now test BracketOutward MinimumBracket b; Assert.IsFalse(Minimize.BracketOutward(x => x, 0, 1, out b)); // make sure it fails with functions that have no minimum Assert.IsTrue(Minimize.BracketOutward(x => 5, 0, 1, out b)); // but succeeds with constant functions Assert.IsTrue(Minimize.BracketOutward(function, 0, 1, out b)); // and with our sample function // make sure it searches in a downhill direction, as designed Assert.AreEqual(Math.PI * -2 / 3 - 1, Minimize.GoldenSection(function, b), Accuracy); Assert.IsTrue(Minimize.BracketOutward(function, 1, 2, out b)); Assert.AreEqual(Math.PI * 4 / 3 - 1, Minimize.GoldenSection(function, b), Accuracy); // try a function with a singularity, for kicks ndFunction = x => Math.Cos(x) / (x - 1); Assert.AreEqual(1, Minimize.GoldenSection(ndFunction, new MinimumBracket(-1, -0.1, 1)), Accuracy); Assert.AreEqual(1, Minimize.Brent(ndFunction, new MinimumBracket(-1, -0.1, 1)), Accuracy); }
/// <summary> /// Finds approximate minimum of the function /// </summary> /// <param name="function">Function to minimize</param> /// <param name="initial">Initial point</param> /// <param name="result">Approximate minimum</param> public void Minimize(DifferentiableFunction function, ref VBuffer <Float> initial, ref VBuffer <Float> result) { Contracts.Check(FloatUtils.IsFinite(initial.Values, initial.Count), "The initial vector contains NaNs or infinite values."); LineFunc lineFunc = new LineFunc(function, ref initial, UseCG); VBuffer <Float> prev = default(VBuffer <Float>); initial.CopyTo(ref prev); for (int n = 0; _maxSteps == 0 || n < _maxSteps; ++n) { Float step = LineSearch.Minimize(lineFunc.Eval, lineFunc.Value, lineFunc.Deriv); var newPoint = lineFunc.NewPoint; bool terminateNow = n > 0 && TerminateTester.ShouldTerminate(ref newPoint, ref prev); if (terminateNow || Terminate(ref newPoint)) { break; } newPoint.CopyTo(ref prev); lineFunc.ChangeDir(); } lineFunc.NewPoint.CopyTo(ref result); }
/// <summary> /// Minimize a function using the supplied termination criterion /// </summary> /// <param name="function">The function to minimize</param> /// <param name="initial">The initial point</param> /// <param name="term">termination criterion to use</param> /// <param name="result">The point at the optimum</param> /// <param name="optimum">The optimum function value</param> /// <exception cref="PrematureConvergenceException">Thrown if successive points are within numeric precision of each other, but termination condition is still unsatisfied.</exception> public void Minimize(DifferentiableFunction function, ref VBuffer <Float> initial, ITerminationCriterion term, ref VBuffer <Float> result, out Float optimum) { const string computationName = "LBFGS Optimizer"; using (var pch = Env.StartProgressChannel(computationName)) using (var ch = Env.Start(computationName)) { ch.Info("Beginning optimization"); ch.Info("num vars: {0}", initial.Length); ch.Info("improvement criterion: {0}", term.FriendlyName); OptimizerState state = MakeState(ch, pch, function, ref initial); term.Reset(); var header = new ProgressHeader(new[] { "Loss", "Improvement" }, new[] { "iterations", "gradients" }); pch.SetHeader(header, (Action <IProgressEntry>)(e => { e.SetProgress(0, (double)(state.Iter - 1)); e.SetProgress(1, state.GradientCalculations); })); bool finished = false; pch.Checkpoint(state.Value, null, 0); state.UpdateDir(); while (!finished) { bool success = state.LineSearch(ch, false); if (!success) { // problem may be numerical errors in previous gradients // try to save state of optimization by discarding them // and starting over with gradient descent. state.DiscardOldVectors(); state.UpdateDir(); state.LineSearch(ch, true); } string message; finished = term.Terminate(state, out message); double?improvement = null; double x; int end; if (message != null && DoubleParser.TryParse(out x, message, 0, message.Length, out end)) { improvement = x; } pch.Checkpoint(state.Value, improvement, state.Iter); if (!finished) { state.Shift(); state.UpdateDir(); } } state.X.CopyTo(ref result); optimum = state.Value; ch.Done(); } }
/// <summary> /// Minimize a function. /// </summary> /// <param name="function">The function to minimize</param> /// <param name="initial">The initial point</param> /// <param name="result">The point at the optimum</param> /// <param name="optimum">The optimum function value</param> /// <exception cref="PrematureConvergenceException">Thrown if successive points are within numeric precision of each other, but termination condition is still unsatisfied.</exception> public void Minimize(DifferentiableFunction function, ref VBuffer <Float> initial, ref VBuffer <Float> result, out Float optimum) { Minimize(function, ref initial, _staticTerm, ref result, out optimum); }
/// <summary> /// Minimize a function using the MeanRelativeImprovement termination criterion with the supplied tolerance level /// </summary> /// <param name="function">The function to minimize</param> /// <param name="initial">The initial point</param> /// <param name="tolerance">Convergence tolerance (smaller means more iterations, closer to exact optimum)</param> /// <param name="result">The point at the optimum</param> /// <param name="optimum">The optimum function value</param> /// <exception cref="PrematureConvergenceException">Thrown if successive points are within numeric precision of each other, but termination condition is still unsatisfied.</exception> public void Minimize(DifferentiableFunction function, ref VBuffer <Float> initial, Float tolerance, ref VBuffer <Float> result, out Float optimum) { ITerminationCriterion term = new MeanRelativeImprovementCriterion(tolerance); Minimize(function, ref initial, term, ref result, out optimum); }
public static LineSearchResult doLineSearch(DifferentiableFunction function, double[] direction, LineSearchResult lsr) { return(doLineSearch(function, direction, lsr, false)); }
public static LineSearchResult doLineSearch(DifferentiableFunction function, double[] direction, LineSearchResult lsr, bool verbose) { int currFctEvalCount = lsr.FctEvalCount; double stepSize = INITIAL_STEP_SIZE; double[] x = lsr.NextPoint; double valueAtX = lsr.ValueAtNext; double[] gradAtX = lsr.GradAtNext; double[] nextPoint = null; double[] gradAtNextPoint = null; double valueAtNextPoint = 0.0; double mu = 0; double upsilon = double.PositiveInfinity; long startTime = DateTimeHelperClass.CurrentUnixTimeMillis(); while (true) { nextPoint = ArrayMath.updatePoint(x, direction, stepSize); valueAtNextPoint = function.valueAt(nextPoint); currFctEvalCount++; gradAtNextPoint = function.gradientAt(nextPoint); if (!checkArmijoCond(valueAtX, valueAtNextPoint, gradAtX, direction, stepSize, true)) { upsilon = stepSize; } else if (!checkCurvature(gradAtNextPoint, gradAtX, direction, x.Length, true)) { mu = stepSize; } else { break; } if (upsilon < double.PositiveInfinity) { stepSize = (mu + upsilon) / TT; } else { stepSize *= TT; } if (stepSize < MIN_STEP_SIZE + mu) { stepSize = 0.0; break; } } long endTime = DateTimeHelperClass.CurrentUnixTimeMillis(); long duration = endTime - startTime; if (verbose) { Console.Write("\t" + valueAtX); Console.Write("\t" + (valueAtNextPoint - valueAtX)); Console.Write("\t" + (duration / 1000.0) + "\n"); } LineSearchResult result = new LineSearchResult(stepSize, valueAtX, valueAtNextPoint, gradAtX, gradAtNextPoint, x, nextPoint, currFctEvalCount); return(result); }
public void T01_OneDimensional() { // this is pretty close to the minimum that i can make it while still passing, given the original implementation. if an // implementation degrades substantially (with these functions, anyway), this should catch it const double Accuracy = 1.77636e-15; // this is a simple parabola with a double root at x=1. because of the double root, which means the function never crosses zero, only // touches it, many methods have more trouble with it. in particular, only unbounded newton raphson is able to find it without having // the root at one of the interval boundaries DifferentiableFunction function = new DifferentiableFunction(x => (x - 1) * (x - 1), x => 2 * x - 2); // f(x) = (x-1)^2 // test unbounded newton raphson with a wide interval Assert.AreEqual(1, FindRoot.UnboundedNewtonRaphson(function, new RootBracket(-10, 10)), Accuracy); // the others need the root to be at one of the boundaries, although this is a trivial case for any method. make sure it works from // both edges for all methods Assert.AreEqual(1, FindRoot.BoundedNewtonRaphson(function, new RootBracket(1, 10)), Accuracy); Assert.AreEqual(1, FindRoot.BoundedNewtonRaphson(function, new RootBracket(-10, 1)), Accuracy); Assert.AreEqual(1, FindRoot.Brent(function, new RootBracket(1, 10)), Accuracy); Assert.AreEqual(1, FindRoot.Brent(function, new RootBracket(-10, 1)), Accuracy); Assert.AreEqual(1, FindRoot.Subdivide(function, new RootBracket(1, 10)), Accuracy); Assert.AreEqual(1, FindRoot.Subdivide(function, new RootBracket(-10, 1)), Accuracy); // this is a parabola with roots at x=0 and x=2. since it crosses zero, it should be amenable to many different methods function = new DifferentiableFunction(x => (x - 1) * (x - 1) - 1, x => 2 * x - 2); // f(x) = (x-1)^2 - 1 // first, let's try some root bracketing RootBracket interval = new RootBracket(0.5, 1.5); // bracket outwards Assert.IsTrue(FindRoot.BracketOutward(function, ref interval)); Assert.IsTrue(interval.Min <= 0 && interval.Max >= 0 || interval.Min <= 2 && interval.Max >= 2); // make sure it brackets a root // bracket inwards. since interval, when divided into 20 pieces, will have the roots exactly on the boundaries, the sub intervals // should also (although that's not something we need to verify) interval = new RootBracket(-10, 10); bool foundZero = false, foundTwo = false; foreach (RootBracket sub in FindRoot.BracketInward(function, interval, 20)) { if (sub.Min <= 0 && sub.Max >= 0) { foundZero = true; } if (sub.Min <= 2 && sub.Max >= 2) { foundTwo = true; } Assert.IsTrue(sub.Min <= 0 && sub.Max >= 0 || sub.Min <= 2 && sub.Max >= 2); } Assert.IsTrue(foundZero && foundTwo); // try again, using an interval that doesn't divide evenly (and therefore won't provide cases that are trivial to solve) interval = new RootBracket(-8, 9); foundZero = foundTwo = false; foreach (RootBracket sub in FindRoot.BracketInward(function, interval, 20)) { double root = -1; if (sub.Min <= 0 && sub.Max >= 0) { foundZero = true; root = 0; } else if (sub.Min <= 2 && sub.Max >= 2) { foundTwo = true; root = 2; } else { Assert.Fail(); } // ensure that all methods find the root Assert.AreEqual(root, FindRoot.BoundedNewtonRaphson(function, sub), Accuracy); Assert.AreEqual(root, FindRoot.Brent(function, sub), Accuracy); Assert.AreEqual(root, FindRoot.Subdivide(function, sub), Accuracy); Assert.AreEqual(root, FindRoot.UnboundedNewtonRaphson(function, sub), Accuracy); } Assert.IsTrue(foundZero && foundTwo); // ensure that unbounded newton-raphson fails properly when there's no root function = new DifferentiableFunction(x => x * x + 1, x => 2 * x); // f(x) = x^2+1, a parabola with no root interval = new RootBracket(-1, 1); TestHelpers.TestException <RootNotFoundException>(delegate { FindRoot.UnboundedNewtonRaphson(function, interval); }); // ensure that the others complain about the root not being bracketed TestHelpers.TestException <ArgumentException>(delegate { FindRoot.BoundedNewtonRaphson(function, interval); }); TestHelpers.TestException <ArgumentException>(delegate { FindRoot.Brent(function, interval); }); TestHelpers.TestException <ArgumentException>(delegate { FindRoot.Subdivide(function, interval); }); // ensure that bracketing fails as it should Assert.IsFalse(FindRoot.BracketOutward(function, ref interval)); Assert.AreEqual(0, FindRoot.BracketInward(function, new RootBracket(-10, 10), 20).Count()); }
internal override OptimizerState MakeState(IChannel ch, IProgressChannelProvider progress, DifferentiableFunction function, ref VBuffer <Float> initial) { Contracts.AssertValue(ch); ch.AssertValue(progress); if (EnforceNonNegativity) { VBufferUtils.Apply(ref initial, delegate(int ind, ref Float initialVal) { if (initialVal < 0.0 && ind >= _biasCount) { initialVal = 0; } }); } if (_l1weight > 0 && _biasCount < initial.Length) { return(new L1OptimizerState(ch, progress, function, in initial, M, TotalMemoryLimit, _biasCount, _l1weight, KeepDense, EnforceNonNegativity)); } return(new FunctionOptimizerState(ch, progress, function, in initial, M, TotalMemoryLimit, KeepDense, EnforceNonNegativity)); }
internal FunctionOptimizerState(IChannel ch, IProgressChannelProvider progress, DifferentiableFunction function, in VBuffer <float> initial, int m,
public LineFunc(DifferentiableFunction function, in VBuffer <Float> initial, bool useCG = false)
internal virtual OptimizerState MakeState(IChannel ch, IProgressChannelProvider progress, DifferentiableFunction function, ref VBuffer <float> initial) { return(new FunctionOptimizerState(ch, progress, function, in initial, M, TotalMemoryLimit, KeepDense, EnforceNonNegativity)); }
/// <summary> /// Tests the gradient reported by f. /// </summary> /// <param name="f">function to test</param> /// <param name="x">point at which to test</param> /// <returns>maximum normalized difference between analytic and numeric directional derivative over multiple tests</returns> public static Float Test(DifferentiableFunction f, ref VBuffer <Float> x) { // REVIEW: Delete this method? return(Test(f, ref x, false)); }
internal L1OptimizerState(IChannel ch, IProgressChannelProvider progress, DifferentiableFunction function, in VBuffer <Float> initial, int m, long totalMemLimit,
/// <summary> /// Tests the gradient reported by f. /// </summary> /// <param name="f">function to test</param> /// <param name="x">point at which to test</param> /// <param name="quiet">If false, outputs detailed info.</param> /// <returns>maximum normalized difference between analytic and numeric directional derivative over multiple tests</returns> public static Float Test(DifferentiableFunction f, ref VBuffer <Float> x, bool quiet) { // REVIEW: Delete this method? VBuffer <Float> grad = default(VBuffer <Float>); VBuffer <Float> newGrad = default(VBuffer <Float>); VBuffer <Float> newX = default(VBuffer <Float>); Float normX = VectorUtils.Norm(x); f(ref x, ref grad, null); if (!quiet) { Console.WriteLine(Header); } Float maxNormDiff = Float.NegativeInfinity; int numIters = Math.Min((int)x.Length, 10); int maxDirCount = Math.Min((int)x.Length / 2, 100); for (int n = 1; n <= numIters; n++) { int dirCount = Math.Min(n * 10, maxDirCount); List <int> indices = new List <int>(dirCount); List <Float> values = new List <Float>(dirCount); for (int i = 0; i < dirCount; i++) { int index = _r.Next((int)x.Length); while (indices.IndexOf(index) >= 0) { index = _r.Next((int)x.Length); } indices.Add(index); values.Add(SampleFromGaussian(_r)); } VBuffer <Float> dir = new VBuffer <Float>(x.Length, values.Count, values.ToArray(), indices.ToArray()); Float norm = VectorUtils.Norm(dir); VectorUtils.ScaleBy(ref dir, 1 / norm); VectorUtils.AddMultInto(ref x, Eps, ref dir, ref newX); Float rVal = f(ref newX, ref newGrad, null); VectorUtils.AddMultInto(ref x, -Eps, ref dir, ref newX); Float lVal = f(ref newX, ref newGrad, null); Float dirDeriv = VectorUtils.DotProduct(ref grad, ref dir); Float numDeriv = (rVal - lVal) / (2 * Eps); Float normDiff = Math.Abs(1 - numDeriv / dirDeriv); Float diff = numDeriv - dirDeriv; if (!quiet) { Console.WriteLine("{0,-9}{1,-18:0.0000e0}{2,-18:0.0000e0}{3,-15:0.0000e0}{4,0:0.0000e0}", n, numDeriv, dirDeriv, diff, normDiff); } maxNormDiff = Math.Max(maxNormDiff, normDiff); } return(maxNormDiff); }