/// <summary>
/// Tests the gradient using finite differences on each axis in the list
/// </summary>
/// <param name="f">Function to test</param>
/// <param name="x">Point at which to test</param>
/// <param name="coords">List of coordinates to test</param>
public static void TestCoords(DifferentiableFunction f, ref VBuffer <Float> x, IList <int> coords)
{
// REVIEW: Delete this method?
VBuffer <Float> grad = default(VBuffer <Float>);
VBuffer <Float> newGrad = default(VBuffer <Float>);
VBuffer <Float> newX = default(VBuffer <Float>);
Float val = f(ref x, ref grad, null);
Float normX = VectorUtils.Norm(x);
Console.WriteLine(Header);
Random r = new Random(5);
VBuffer <Float> dir = new VBuffer <Float>(x.Length, 1, new Float[] { 1 }, new int[] { 0 });
foreach (int n in coords)
{
dir.Values[0] = n;
VectorUtils.AddMultInto(ref x, Eps, ref dir, ref newX);
Float rVal = f(ref newX, ref newGrad, null);
VectorUtils.AddMultInto(ref x, -Eps, ref dir, ref newX);
Float lVal = f(ref newX, ref newGrad, null);
Float dirDeriv = VectorUtils.DotProduct(ref grad, ref dir);
Float numDeriv = (rVal - lVal) / (2 * Eps);
Float normDiff = Math.Abs(1 - numDeriv / dirDeriv);
Float diff = numDeriv - dirDeriv;
Console.WriteLine("{0,-9}{1,-18:0.0000e0}{2,-18:0.0000e0}{3,-15:0.0000e0}{4,0:0.0000e0}", n, numDeriv, dirDeriv, diff, normDiff);
}
}
private double[] computeDirection(DifferentiableFunction monitor, LineSearchResult lsr)
{
// implemented two-loop hessian update method.
double[] direction = lsr.GradAtNext.Clone() as double[];
double[] @as = new double[m];
// first loop
for (int i = updateInfo.kCounter - 1; i >= 0; i--)
{
@as[i] = updateInfo.getRho(i) * ArrayMath.innerProduct(updateInfo.getS(i), direction);
for (int ii = 0; ii < dimension; ii++)
{
direction[ii] = direction[ii] - @as[i] * updateInfo.getY(i)[ii];
}
}
// second loop
for (int i = 0; i < updateInfo.kCounter; i++)
{
double b = updateInfo.getRho(i) * ArrayMath.innerProduct(updateInfo.getY(i), direction);
for (int ii = 0; ii < dimension; ii++)
{
direction[ii] = direction[ii] + (@as[i] - b) * updateInfo.getS(i)[ii];
}
}
for (int i = 0; i < dimension; i++)
{
direction[i] *= -1.0;
}
return(direction);
}
/// <summary>
/// Tests the gradient reported by <paramref name="f"/>.
/// </summary>
/// <param name="f">Function to test</param>
/// <param name="x">Point at which to test</param>
/// <param name="dir">Direction to test derivative</param>
/// <param name="quiet">Whether to disable output</param>
/// <param name="newGrad">This is a reusable working buffer for intermediate calculations</param>
/// <param name="newX">This is a reusable working buffer for intermediate calculations</param>
/// <returns>Normalized difference between analytic and numeric directional derivative</returns>
public static Float Test(DifferentiableFunction f, ref VBuffer <Float> x, ref VBuffer <Float> dir, bool quiet,
ref VBuffer <Float> newGrad, ref VBuffer <Float> newX)
{
Float normDir = VectorUtils.Norm(dir);
Float val = f(ref x, ref newGrad, null);
Float dirDeriv = VectorUtils.DotProduct(ref newGrad, ref dir);
Float scaledEps = Eps / normDir;
VectorUtils.AddMultInto(ref x, scaledEps, ref dir, ref newX);
Float rVal = f(ref newX, ref newGrad, null);
VectorUtils.AddMultInto(ref x, -scaledEps, ref dir, ref newX);
Float lVal = f(ref newX, ref newGrad, null);
Float numDeriv = (rVal - lVal) / (2 * scaledEps);
Float normDiff = Math.Abs(1 - numDeriv / dirDeriv);
Float diff = numDeriv - dirDeriv;
if (!quiet)
{
Console.WriteLine("{0,-18:0.0000e0}{1,-18:0.0000e0}{2,-15:0.0000e0}{3,0:0.0000e0}", numDeriv, dirDeriv, diff, normDiff);
}
return(normDiff);
}
internal FunctionOptimizerState(IChannel ch, IProgressChannelProvider progress, DifferentiableFunction function, ref VBuffer <Float> initial, int m,
long totalMemLimit, bool keepDense, bool enforceNonNegativity)
: base(ch, progress, ref initial, m, totalMemLimit, keepDense, enforceNonNegativity)
{
Function = function;
Init();
}
public void Evaluate(DifferentiableFunction function, Jet[] parameters, double[] values, double[][] jacobian)
{
if (this.index < this.domain)
{
throw new Exception("Insufficient parameters added");
}
if (this.domain != parameters.Length)
{
throw new ArgumentException("Insufficient parameters provided");
}
var f = new Jet[this.range];
function(f, parameters);
for (int i = 0; i < this.range; ++i)
{
values[i] = f[i].Real;
for (int j = 0; j < this.domain; ++j)
{
jacobian[i][j] = f[i].Infinitesimals[j];
}
}
}
public LineFunc(DifferentiableFunction function, ref VBuffer <Float> initial, bool useCG = false)
{
int dim = initial.Length;
initial.CopyTo(ref _point);
_func = function;
// REVIEW: plumb the IProgressChannelProvider through.
_value = _func(ref _point, ref _grad, null);
VectorUtils.ScaleInto(ref _grad, -1, ref _dir);
_useCG = useCG;
}
internal L1OptimizerState(IChannel ch, IProgressChannelProvider progress, DifferentiableFunction function, ref VBuffer <Float> initial, int m, long totalMemLimit,
int biasCount, Float l1Weight, bool keepDense, bool enforceNonNegativity)
: base(ch, progress, ref initial, m, totalMemLimit, keepDense, enforceNonNegativity)
{
Contracts.AssertValue(ch);
ch.Assert(0 <= biasCount && biasCount < initial.Length);
ch.Assert(l1Weight > 0);
_biasCount = biasCount;
_l1weight = l1Weight;
_function = function;
Init();
}
public void T01_OneDimensional()
{
const double Accuracy = 1.5e-7;
// sin(x+1) + x/2 has local minima at -2/3PI-1, 4/3PI-1, 10/3PI-1, etc.
DifferentiableFunction function = new DifferentiableFunction(x => Math.Sin(x + 1) + x / 2, x => Math.Cos(x + 1) + 0.5);
Func <double, double> ndFunction = function.Evaluate; // create a version without derivative information
// the three minima above are located between -5 and 13, so we should be able to find them with BracketInward()
List <MinimumBracket> brackets = Minimize.BracketInward(function, -5, 13, 3).ToList();
Assert.AreEqual(3, brackets.Count); // ensure we found them all
List <double> minima = new List <double>();
// for each bracket, try to find it using all available methods
foreach (MinimumBracket bracket in brackets)
{
double x = Minimize.GoldenSection(function, bracket); // first use golden section search, which is the most reliable
Assert.AreEqual(x, Minimize.Brent(function, bracket), Accuracy); // then make sure Brent's method gives a similar answer, both with
Assert.AreEqual(x, Minimize.Brent(ndFunction, bracket), Accuracy); // and without the derivative
minima.Add(x);
}
minima.Sort(); // then sort the results to put them in a known order and make sure they're equal to the expected values
Assert.AreEqual(3, minima.Count);
Assert.AreEqual(Math.PI * -2 / 3 - 1, minima[0], Accuracy);
Assert.AreEqual(Math.PI * 4 / 3 - 1, minima[1], Accuracy);
Assert.AreEqual(Math.PI * 10 / 3 - 1, minima[2], Accuracy);
// now test BracketOutward
MinimumBracket b;
Assert.IsFalse(Minimize.BracketOutward(x => x, 0, 1, out b)); // make sure it fails with functions that have no minimum
Assert.IsTrue(Minimize.BracketOutward(x => 5, 0, 1, out b)); // but succeeds with constant functions
Assert.IsTrue(Minimize.BracketOutward(function, 0, 1, out b)); // and with our sample function
// make sure it searches in a downhill direction, as designed
Assert.AreEqual(Math.PI * -2 / 3 - 1, Minimize.GoldenSection(function, b), Accuracy);
Assert.IsTrue(Minimize.BracketOutward(function, 1, 2, out b));
Assert.AreEqual(Math.PI * 4 / 3 - 1, Minimize.GoldenSection(function, b), Accuracy);
// try a function with a singularity, for kicks
ndFunction = x => Math.Cos(x) / (x - 1);
Assert.AreEqual(1, Minimize.GoldenSection(ndFunction, new MinimumBracket(-1, -0.1, 1)), Accuracy);
Assert.AreEqual(1, Minimize.Brent(ndFunction, new MinimumBracket(-1, -0.1, 1)), Accuracy);
}
/// <summary>
/// Finds approximate minimum of the function
/// </summary>
/// <param name="function">Function to minimize</param>
/// <param name="initial">Initial point</param>
/// <param name="result">Approximate minimum</param>
public void Minimize(DifferentiableFunction function, ref VBuffer <Float> initial, ref VBuffer <Float> result)
{
Contracts.Check(FloatUtils.IsFinite(initial.Values, initial.Count), "The initial vector contains NaNs or infinite values.");
LineFunc lineFunc = new LineFunc(function, ref initial, UseCG);
VBuffer <Float> prev = default(VBuffer <Float>);
initial.CopyTo(ref prev);
for (int n = 0; _maxSteps == 0 || n < _maxSteps; ++n)
{
Float step = LineSearch.Minimize(lineFunc.Eval, lineFunc.Value, lineFunc.Deriv);
var newPoint = lineFunc.NewPoint;
bool terminateNow = n > 0 && TerminateTester.ShouldTerminate(ref newPoint, ref prev);
if (terminateNow || Terminate(ref newPoint))
{
break;
}
newPoint.CopyTo(ref prev);
lineFunc.ChangeDir();
}
lineFunc.NewPoint.CopyTo(ref result);
}
/// <summary>
/// Minimize a function using the supplied termination criterion
/// </summary>
/// <param name="function">The function to minimize</param>
/// <param name="initial">The initial point</param>
/// <param name="term">termination criterion to use</param>
/// <param name="result">The point at the optimum</param>
/// <param name="optimum">The optimum function value</param>
/// <exception cref="PrematureConvergenceException">Thrown if successive points are within numeric precision of each other, but termination condition is still unsatisfied.</exception>
public void Minimize(DifferentiableFunction function, ref VBuffer <Float> initial, ITerminationCriterion term, ref VBuffer <Float> result, out Float optimum)
{
const string computationName = "LBFGS Optimizer";
using (var pch = Env.StartProgressChannel(computationName))
using (var ch = Env.Start(computationName))
{
ch.Info("Beginning optimization");
ch.Info("num vars: {0}", initial.Length);
ch.Info("improvement criterion: {0}", term.FriendlyName);
OptimizerState state = MakeState(ch, pch, function, ref initial);
term.Reset();
var header = new ProgressHeader(new[] { "Loss", "Improvement" }, new[] { "iterations", "gradients" });
pch.SetHeader(header,
(Action <IProgressEntry>)(e =>
{
e.SetProgress(0, (double)(state.Iter - 1));
e.SetProgress(1, state.GradientCalculations);
}));
bool finished = false;
pch.Checkpoint(state.Value, null, 0);
state.UpdateDir();
while (!finished)
{
bool success = state.LineSearch(ch, false);
if (!success)
{
// problem may be numerical errors in previous gradients
// try to save state of optimization by discarding them
// and starting over with gradient descent.
state.DiscardOldVectors();
state.UpdateDir();
state.LineSearch(ch, true);
}
string message;
finished = term.Terminate(state, out message);
double?improvement = null;
double x;
int end;
if (message != null && DoubleParser.TryParse(out x, message, 0, message.Length, out end))
{
improvement = x;
}
pch.Checkpoint(state.Value, improvement, state.Iter);
if (!finished)
{
state.Shift();
state.UpdateDir();
}
}
state.X.CopyTo(ref result);
optimum = state.Value;
ch.Done();
}
}
/// <summary>
/// Minimize a function.
/// </summary>
/// <param name="function">The function to minimize</param>
/// <param name="initial">The initial point</param>
/// <param name="result">The point at the optimum</param>
/// <param name="optimum">The optimum function value</param>
/// <exception cref="PrematureConvergenceException">Thrown if successive points are within numeric precision of each other, but termination condition is still unsatisfied.</exception>
public void Minimize(DifferentiableFunction function, ref VBuffer <Float> initial, ref VBuffer <Float> result, out Float optimum)
{
Minimize(function, ref initial, _staticTerm, ref result, out optimum);
}
/// <summary>
/// Minimize a function using the MeanRelativeImprovement termination criterion with the supplied tolerance level
/// </summary>
/// <param name="function">The function to minimize</param>
/// <param name="initial">The initial point</param>
/// <param name="tolerance">Convergence tolerance (smaller means more iterations, closer to exact optimum)</param>
/// <param name="result">The point at the optimum</param>
/// <param name="optimum">The optimum function value</param>
/// <exception cref="PrematureConvergenceException">Thrown if successive points are within numeric precision of each other, but termination condition is still unsatisfied.</exception>
public void Minimize(DifferentiableFunction function, ref VBuffer <Float> initial, Float tolerance, ref VBuffer <Float> result, out Float optimum)
{
ITerminationCriterion term = new MeanRelativeImprovementCriterion(tolerance);
Minimize(function, ref initial, term, ref result, out optimum);
}
public static LineSearchResult doLineSearch(DifferentiableFunction function, double[] direction,
LineSearchResult lsr)
{
return(doLineSearch(function, direction, lsr, false));
}
public static LineSearchResult doLineSearch(DifferentiableFunction function, double[] direction,
LineSearchResult lsr, bool verbose)
{
int currFctEvalCount = lsr.FctEvalCount;
double stepSize = INITIAL_STEP_SIZE;
double[] x = lsr.NextPoint;
double valueAtX = lsr.ValueAtNext;
double[] gradAtX = lsr.GradAtNext;
double[] nextPoint = null;
double[] gradAtNextPoint = null;
double valueAtNextPoint = 0.0;
double mu = 0;
double upsilon = double.PositiveInfinity;
long startTime = DateTimeHelperClass.CurrentUnixTimeMillis();
while (true)
{
nextPoint = ArrayMath.updatePoint(x, direction, stepSize);
valueAtNextPoint = function.valueAt(nextPoint);
currFctEvalCount++;
gradAtNextPoint = function.gradientAt(nextPoint);
if (!checkArmijoCond(valueAtX, valueAtNextPoint, gradAtX, direction, stepSize, true))
{
upsilon = stepSize;
}
else if (!checkCurvature(gradAtNextPoint, gradAtX, direction, x.Length, true))
{
mu = stepSize;
}
else
{
break;
}
if (upsilon < double.PositiveInfinity)
{
stepSize = (mu + upsilon) / TT;
}
else
{
stepSize *= TT;
}
if (stepSize < MIN_STEP_SIZE + mu)
{
stepSize = 0.0;
break;
}
}
long endTime = DateTimeHelperClass.CurrentUnixTimeMillis();
long duration = endTime - startTime;
if (verbose)
{
Console.Write("\t" + valueAtX);
Console.Write("\t" + (valueAtNextPoint - valueAtX));
Console.Write("\t" + (duration / 1000.0) + "\n");
}
LineSearchResult result = new LineSearchResult(stepSize, valueAtX, valueAtNextPoint, gradAtX,
gradAtNextPoint, x, nextPoint, currFctEvalCount);
return(result);
}
public void T01_OneDimensional()
{
// this is pretty close to the minimum that i can make it while still passing, given the original implementation. if an
// implementation degrades substantially (with these functions, anyway), this should catch it
const double Accuracy = 1.77636e-15;
// this is a simple parabola with a double root at x=1. because of the double root, which means the function never crosses zero, only
// touches it, many methods have more trouble with it. in particular, only unbounded newton raphson is able to find it without having
// the root at one of the interval boundaries
DifferentiableFunction function = new DifferentiableFunction(x => (x - 1) * (x - 1), x => 2 * x - 2); // f(x) = (x-1)^2
// test unbounded newton raphson with a wide interval
Assert.AreEqual(1, FindRoot.UnboundedNewtonRaphson(function, new RootBracket(-10, 10)), Accuracy);
// the others need the root to be at one of the boundaries, although this is a trivial case for any method. make sure it works from
// both edges for all methods
Assert.AreEqual(1, FindRoot.BoundedNewtonRaphson(function, new RootBracket(1, 10)), Accuracy);
Assert.AreEqual(1, FindRoot.BoundedNewtonRaphson(function, new RootBracket(-10, 1)), Accuracy);
Assert.AreEqual(1, FindRoot.Brent(function, new RootBracket(1, 10)), Accuracy);
Assert.AreEqual(1, FindRoot.Brent(function, new RootBracket(-10, 1)), Accuracy);
Assert.AreEqual(1, FindRoot.Subdivide(function, new RootBracket(1, 10)), Accuracy);
Assert.AreEqual(1, FindRoot.Subdivide(function, new RootBracket(-10, 1)), Accuracy);
// this is a parabola with roots at x=0 and x=2. since it crosses zero, it should be amenable to many different methods
function = new DifferentiableFunction(x => (x - 1) * (x - 1) - 1, x => 2 * x - 2); // f(x) = (x-1)^2 - 1
// first, let's try some root bracketing
RootBracket interval = new RootBracket(0.5, 1.5);
// bracket outwards
Assert.IsTrue(FindRoot.BracketOutward(function, ref interval));
Assert.IsTrue(interval.Min <= 0 && interval.Max >= 0 || interval.Min <= 2 && interval.Max >= 2); // make sure it brackets a root
// bracket inwards. since interval, when divided into 20 pieces, will have the roots exactly on the boundaries, the sub intervals
// should also (although that's not something we need to verify)
interval = new RootBracket(-10, 10);
bool foundZero = false, foundTwo = false;
foreach (RootBracket sub in FindRoot.BracketInward(function, interval, 20))
{
if (sub.Min <= 0 && sub.Max >= 0)
{
foundZero = true;
}
if (sub.Min <= 2 && sub.Max >= 2)
{
foundTwo = true;
}
Assert.IsTrue(sub.Min <= 0 && sub.Max >= 0 || sub.Min <= 2 && sub.Max >= 2);
}
Assert.IsTrue(foundZero && foundTwo);
// try again, using an interval that doesn't divide evenly (and therefore won't provide cases that are trivial to solve)
interval = new RootBracket(-8, 9);
foundZero = foundTwo = false;
foreach (RootBracket sub in FindRoot.BracketInward(function, interval, 20))
{
double root = -1;
if (sub.Min <= 0 && sub.Max >= 0)
{
foundZero = true;
root = 0;
}
else if (sub.Min <= 2 && sub.Max >= 2)
{
foundTwo = true;
root = 2;
}
else
{
Assert.Fail();
}
// ensure that all methods find the root
Assert.AreEqual(root, FindRoot.BoundedNewtonRaphson(function, sub), Accuracy);
Assert.AreEqual(root, FindRoot.Brent(function, sub), Accuracy);
Assert.AreEqual(root, FindRoot.Subdivide(function, sub), Accuracy);
Assert.AreEqual(root, FindRoot.UnboundedNewtonRaphson(function, sub), Accuracy);
}
Assert.IsTrue(foundZero && foundTwo);
// ensure that unbounded newton-raphson fails properly when there's no root
function = new DifferentiableFunction(x => x * x + 1, x => 2 * x); // f(x) = x^2+1, a parabola with no root
interval = new RootBracket(-1, 1);
TestHelpers.TestException <RootNotFoundException>(delegate { FindRoot.UnboundedNewtonRaphson(function, interval); });
// ensure that the others complain about the root not being bracketed
TestHelpers.TestException <ArgumentException>(delegate { FindRoot.BoundedNewtonRaphson(function, interval); });
TestHelpers.TestException <ArgumentException>(delegate { FindRoot.Brent(function, interval); });
TestHelpers.TestException <ArgumentException>(delegate { FindRoot.Subdivide(function, interval); });
// ensure that bracketing fails as it should
Assert.IsFalse(FindRoot.BracketOutward(function, ref interval));
Assert.AreEqual(0, FindRoot.BracketInward(function, new RootBracket(-10, 10), 20).Count());
}
internal override OptimizerState MakeState(IChannel ch, IProgressChannelProvider progress, DifferentiableFunction function, ref VBuffer <Float> initial)
{
Contracts.AssertValue(ch);
ch.AssertValue(progress);
if (EnforceNonNegativity)
{
VBufferUtils.Apply(ref initial, delegate(int ind, ref Float initialVal)
{
if (initialVal < 0.0 && ind >= _biasCount)
{
initialVal = 0;
}
});
}
if (_l1weight > 0 && _biasCount < initial.Length)
{
return(new L1OptimizerState(ch, progress, function, in initial, M, TotalMemoryLimit, _biasCount, _l1weight, KeepDense, EnforceNonNegativity));
}
return(new FunctionOptimizerState(ch, progress, function, in initial, M, TotalMemoryLimit, KeepDense, EnforceNonNegativity));
}
internal FunctionOptimizerState(IChannel ch, IProgressChannelProvider progress, DifferentiableFunction function, in VBuffer <float> initial, int m,
public LineFunc(DifferentiableFunction function, in VBuffer <Float> initial, bool useCG = false)
internal virtual OptimizerState MakeState(IChannel ch, IProgressChannelProvider progress, DifferentiableFunction function, ref VBuffer <float> initial)
{
return(new FunctionOptimizerState(ch, progress, function, in initial, M, TotalMemoryLimit, KeepDense, EnforceNonNegativity));
}
/// <summary>
/// Tests the gradient reported by f.
/// </summary>
/// <param name="f">function to test</param>
/// <param name="x">point at which to test</param>
/// <returns>maximum normalized difference between analytic and numeric directional derivative over multiple tests</returns>
public static Float Test(DifferentiableFunction f, ref VBuffer <Float> x)
{
// REVIEW: Delete this method?
return(Test(f, ref x, false));
}
internal L1OptimizerState(IChannel ch, IProgressChannelProvider progress, DifferentiableFunction function, in VBuffer <Float> initial, int m, long totalMemLimit,
/// <summary>
/// Tests the gradient reported by f.
/// </summary>
/// <param name="f">function to test</param>
/// <param name="x">point at which to test</param>
/// <param name="quiet">If false, outputs detailed info.</param>
/// <returns>maximum normalized difference between analytic and numeric directional derivative over multiple tests</returns>
public static Float Test(DifferentiableFunction f, ref VBuffer <Float> x, bool quiet)
{
// REVIEW: Delete this method?
VBuffer <Float> grad = default(VBuffer <Float>);
VBuffer <Float> newGrad = default(VBuffer <Float>);
VBuffer <Float> newX = default(VBuffer <Float>);
Float normX = VectorUtils.Norm(x);
f(ref x, ref grad, null);
if (!quiet)
{
Console.WriteLine(Header);
}
Float maxNormDiff = Float.NegativeInfinity;
int numIters = Math.Min((int)x.Length, 10);
int maxDirCount = Math.Min((int)x.Length / 2, 100);
for (int n = 1; n <= numIters; n++)
{
int dirCount = Math.Min(n * 10, maxDirCount);
List <int> indices = new List <int>(dirCount);
List <Float> values = new List <Float>(dirCount);
for (int i = 0; i < dirCount; i++)
{
int index = _r.Next((int)x.Length);
while (indices.IndexOf(index) >= 0)
{
index = _r.Next((int)x.Length);
}
indices.Add(index);
values.Add(SampleFromGaussian(_r));
}
VBuffer <Float> dir = new VBuffer <Float>(x.Length, values.Count, values.ToArray(), indices.ToArray());
Float norm = VectorUtils.Norm(dir);
VectorUtils.ScaleBy(ref dir, 1 / norm);
VectorUtils.AddMultInto(ref x, Eps, ref dir, ref newX);
Float rVal = f(ref newX, ref newGrad, null);
VectorUtils.AddMultInto(ref x, -Eps, ref dir, ref newX);
Float lVal = f(ref newX, ref newGrad, null);
Float dirDeriv = VectorUtils.DotProduct(ref grad, ref dir);
Float numDeriv = (rVal - lVal) / (2 * Eps);
Float normDiff = Math.Abs(1 - numDeriv / dirDeriv);
Float diff = numDeriv - dirDeriv;
if (!quiet)
{
Console.WriteLine("{0,-9}{1,-18:0.0000e0}{2,-18:0.0000e0}{3,-15:0.0000e0}{4,0:0.0000e0}", n, numDeriv, dirDeriv, diff, normDiff);
}
maxNormDiff = Math.Max(maxNormDiff, normDiff);
}
return(maxNormDiff);
}
