/// <summary>
/// Tests the gradient reported by <paramref name="f"/>.
/// </summary>
/// <param name="f">Function to test</param>
/// <param name="x">Point at which to test</param>
/// <param name="dir">Direction to test derivative</param>
/// <param name="quiet">Whether to disable output</param>
/// <param name="newGrad">This is a reusable working buffer for intermediate calculations</param>
/// <param name="newX">This is a reusable working buffer for intermediate calculations</param>
/// <returns>Normalized difference between analytic and numeric directional derivative</returns>
public static Float Test(DifferentiableFunction f, ref VBuffer <Float> x, ref VBuffer <Float> dir, bool quiet,
ref VBuffer <Float> newGrad, ref VBuffer <Float> newX)
{
Float normDir = VectorUtils.Norm(dir);
Float val = f(ref x, ref newGrad, null);
Float dirDeriv = VectorUtils.DotProduct(ref newGrad, ref dir);
Float scaledEps = Eps / normDir;
VectorUtils.AddMultInto(ref x, scaledEps, ref dir, ref newX);
Float rVal = f(ref newX, ref newGrad, null);
VectorUtils.AddMultInto(ref x, -scaledEps, ref dir, ref newX);
Float lVal = f(ref newX, ref newGrad, null);
Float numDeriv = (rVal - lVal) / (2 * scaledEps);
Float normDiff = Math.Abs(1 - numDeriv / dirDeriv);
Float diff = numDeriv - dirDeriv;
if (!quiet)
{
Console.WriteLine("{0,-18:0.0000e0}{1,-18:0.0000e0}{2,-15:0.0000e0}{3,0:0.0000e0}", numDeriv, dirDeriv, diff, normDiff);
}
return(normDiff);
}
/// <summary>
/// Tests the gradient using finite differences on each axis in the list
/// </summary>
/// <param name="f">Function to test</param>
/// <param name="x">Point at which to test</param>
/// <param name="coords">List of coordinates to test</param>
public static void TestCoords(DifferentiableFunction f, ref VBuffer <Float> x, IList <int> coords)
{
// REVIEW: Delete this method?
VBuffer <Float> grad = default(VBuffer <Float>);
VBuffer <Float> newGrad = default(VBuffer <Float>);
VBuffer <Float> newX = default(VBuffer <Float>);
Float val = f(ref x, ref grad, null);
Float normX = VectorUtils.Norm(x);
Console.WriteLine(Header);
Random r = new Random(5);
VBuffer <Float> dir = new VBuffer <Float>(x.Length, 1, new Float[] { 1 }, new int[] { 0 });
foreach (int n in coords)
{
dir.Values[0] = n;
VectorUtils.AddMultInto(ref x, Eps, ref dir, ref newX);
Float rVal = f(ref newX, ref newGrad, null);
VectorUtils.AddMultInto(ref x, -Eps, ref dir, ref newX);
Float lVal = f(ref newX, ref newGrad, null);
Float dirDeriv = VectorUtils.DotProduct(ref grad, ref dir);
Float numDeriv = (rVal - lVal) / (2 * Eps);
Float normDiff = Math.Abs(1 - numDeriv / dirDeriv);
Float diff = numDeriv - dirDeriv;
Console.WriteLine("{0,-9}{1,-18:0.0000e0}{2,-18:0.0000e0}{3,-15:0.0000e0}{4,0:0.0000e0}", n, numDeriv, dirDeriv, diff, normDiff);
}
}
public Float Eval(Float step, out Float deriv)
{
VectorUtils.AddMultInto(ref _point, step, ref _dir, ref _newPoint);
_newValue = _func(ref _newPoint, ref _newGrad, null);
deriv = VectorUtils.DotProduct(ref _dir, ref _newGrad);
return(_newValue);
}
/// <summary>
/// Backtracking line search with Armijo-like condition, from Andrew & Gao
/// </summary>
internal override bool LineSearch(IChannel ch, bool force)
{
Float dirDeriv = -VectorUtils.DotProduct(ref _dir, ref _steepestDescDir);
if (dirDeriv == 0)
{
throw ch.Process(new PrematureConvergenceException(this, "Directional derivative is zero. You may be sitting on the optimum."));
}
// if a non-descent direction is chosen, the line search will break anyway, so throw here
// The most likely reason for this is a bug in your function's gradient computation
// It may also indicate that your function is not convex.
ch.Check(dirDeriv < 0, "L-BFGS chose a non-descent direction.");
Float alpha = (Iter == 1 ? (1 / VectorUtils.Norm(_dir)) : 1);
GetNextPoint(alpha);
Float unnormCos = VectorUtils.DotProduct(ref _steepestDescDir, ref _newX) - VectorUtils.DotProduct(ref _steepestDescDir, ref _x);
if (unnormCos < 0)
{
VBufferUtils.ApplyWith(ref _steepestDescDir, ref _dir,
(int ind, Float sdVal, ref Float dirVal) =>
{
if (sdVal * dirVal < 0 && ind >= _biasCount)
{
dirVal = 0;
}
});
GetNextPoint(alpha);
unnormCos = VectorUtils.DotProduct(ref _steepestDescDir, ref _newX) - VectorUtils.DotProduct(ref _steepestDescDir, ref _x);
}
int i = 0;
while (true)
{
Value = Eval(ref _newX, ref _newGrad);
GradientCalculations++;
if (Value <= LastValue - Gamma * unnormCos)
{
return(true);
}
++i;
if (!force && i == MaxLineSearch)
{
return(false);
}
alpha *= (Float)0.25;
GetNextPoint(alpha);
unnormCos = VectorUtils.DotProduct(ref _steepestDescDir, ref _newX) - VectorUtils.DotProduct(ref _steepestDescDir, ref _x);
}
}
internal void Shift()
{
if (_roList.Count < _m)
{
if (_totalMemLimit > 0)
{
long totalMem = GC.GetTotalMemory(true);
if (totalMem > _totalMemLimit)
{
_m = _roList.Count;
}
}
}
VBuffer <Float> nextS;
VBuffer <Float> nextY;
if (_roList.Count == _m)
{
// REVIEW: Goofy. Instead somehow consider the array
// "circular" in some sense.
nextS = _sList[0];
Array.Copy(_sList, 1, _sList, 0, _m - 1);
nextY = _yList[0];
Array.Copy(_yList, 1, _yList, 0, _m - 1);
_roList.RemoveAt(0);
}
else
{
nextS = CreateWorkingVector();
nextY = CreateWorkingVector();
}
VectorUtils.AddMultInto(ref _newX, -1, ref _x, ref nextS);
VectorUtils.AddMultInto(ref _newGrad, -1, ref _grad, ref nextY);
Float ro = VectorUtils.DotProduct(ref nextS, ref nextY);
if (ro == 0)
{
throw Ch.Process(new PrematureConvergenceException(this, "ro equals zero. Is your function linear?"));
}
_sList[_roList.Count] = nextS;
_yList[_roList.Count] = nextY;
_roList.Add(ro);
var temp = LastValue;
LastValue = Value;
Value = temp;
Utils.Swap(ref _x, ref _newX);
Utils.Swap(ref _grad, ref _newGrad);
Iter++;
GradientCalculations = 0;
}
private static Float QuadTest2D(ref VBuffer <Float> x, ref VBuffer <Float> grad, IProgressChannelProvider progress = null)
{
Float d1 = VectorUtils.DotProduct(ref x, ref _c1);
Float d2 = VectorUtils.DotProduct(ref x, ref _c2);
Float d3 = VectorUtils.DotProduct(ref x, ref _c3);
_c3.CopyTo(ref grad);
VectorUtils.AddMult(ref _c1, d1, ref grad);
VectorUtils.AddMult(ref _c2, d2, ref grad);
return((Float)0.5 * (d1 * d1 + d2 * d2) + d3 + 55);
}
internal void MapDirByInverseHessian()
{
int count = _roList.Count;
if (count != 0)
{
Float[] alphas = new Float[count];
int lastGoodRo = -1;
for (int i = count - 1; i >= 0; i--)
{
if (_roList[i] > 0)
{
alphas[i] = -VectorUtils.DotProduct(ref _sList[i], ref _dir) / _roList[i];
VectorUtils.AddMult(ref _yList[i], alphas[i], ref _dir);
if (lastGoodRo == -1)
{
lastGoodRo = i;
}
}
}
// if we have no positive ros, dir doesn't change
if (lastGoodRo == -1)
{
return;
}
Float yDotY = VectorUtils.DotProduct(ref _yList[lastGoodRo], ref _yList[lastGoodRo]);
VectorUtils.ScaleBy(ref _dir, _roList[lastGoodRo] / yDotY);
for (int i = 0; i <= lastGoodRo; i++)
{
if (_roList[i] > 0)
{
Float beta = VectorUtils.DotProduct(ref _yList[i], ref _dir) / _roList[i];
VectorUtils.AddMult(ref _sList[i], -alphas[i] - beta, ref _dir);
}
}
}
}
public void ChangeDir()
{
if (_useCG)
{
Float newByNew = VectorUtils.NormSquared(_newGrad);
Float newByOld = VectorUtils.DotProduct(ref _newGrad, ref _grad);
Float oldByOld = VectorUtils.NormSquared(_grad);
Float betaPR = (newByNew - newByOld) / oldByOld;
Float beta = Math.Max(0, betaPR);
VectorUtils.ScaleBy(ref _dir, beta);
VectorUtils.AddMult(ref _newGrad, -1, ref _dir);
}
else
{
VectorUtils.ScaleInto(ref _newGrad, -1, ref _dir);
}
_newPoint.CopyTo(ref _point);
_newGrad.CopyTo(ref _grad);
_value = _newValue;
}
/// <summary>
/// An implementation of the line search for the Wolfe conditions, from Nocedal & Wright
/// </summary>
internal virtual bool LineSearch(IChannel ch, bool force)
{
Contracts.AssertValue(ch);
Float dirDeriv = VectorUtils.DotProduct(ref _dir, ref _grad);
if (dirDeriv == 0)
{
throw ch.Process(new PrematureConvergenceException(this, "Directional derivative is zero. You may be sitting on the optimum."));
}
// if a non-descent direction is chosen, the line search will break anyway, so throw here
// The most likely reasons for this is a bug in your function's gradient computation,
ch.Check(dirDeriv < 0, "L-BFGS chose a non-descent direction.");
Float c1 = (Float)1e-4 * dirDeriv;
Float c2 = (Float)0.9 * dirDeriv;
Float alpha = (Iter == 1 ? (1 / VectorUtils.Norm(_dir)) : 1);
PointValueDeriv last = new PointValueDeriv(0, LastValue, dirDeriv);
PointValueDeriv aLo = new PointValueDeriv();
PointValueDeriv aHi = new PointValueDeriv();
// initial bracketing phase
while (true)
{
VectorUtils.AddMultInto(ref _x, alpha, ref _dir, ref _newX);
if (EnforceNonNegativity)
{
VBufferUtils.Apply(ref _newX, delegate(int ind, ref Float newXval)
{
if (newXval < 0.0)
{
newXval = 0;
}
});
}
Value = Eval(ref _newX, ref _newGrad);
GradientCalculations++;
if (Float.IsPositiveInfinity(Value))
{
alpha /= 2;
continue;
}
if (!FloatUtils.IsFinite(Value))
{
throw ch.Except("Optimizer unable to proceed with loss function yielding {0}", Value);
}
dirDeriv = VectorUtils.DotProduct(ref _dir, ref _newGrad);
PointValueDeriv curr = new PointValueDeriv(alpha, Value, dirDeriv);
if ((curr.V > LastValue + c1 * alpha) || (last.A > 0 && curr.V >= last.V))
{
aLo = last;
aHi = curr;
break;
}
else if (Math.Abs(curr.D) <= -c2)
{
return(true);
}
else if (curr.D >= 0)
{
aLo = curr;
aHi = last;
break;
}
last = curr;
if (alpha == 0)
{
alpha = Float.Epsilon; // Robust to divisional underflow.
}
else
{
alpha *= 2;
}
}
Float minChange = (Float)0.01;
int maxSteps = 10;
// this loop is the "zoom" procedure described in Nocedal & Wright
for (int step = 0; ; ++step)
{
if (step == maxSteps && !force)
{
return(false);
}
PointValueDeriv left = aLo.A < aHi.A ? aLo : aHi;
PointValueDeriv right = aLo.A < aHi.A ? aHi : aLo;
if (left.D > 0 && right.D < 0)
{
// interpolating cubic would have max in range, not min (can this happen?)
// set a to the one with smaller value
alpha = aLo.V < aHi.V ? aLo.A : aHi.A;
}
else
{
alpha = CubicInterp(aLo, aHi);
if (Float.IsNaN(alpha) || Float.IsInfinity(alpha))
{
alpha = (aLo.A + aHi.A) / 2;
}
}
// this is to ensure that the new point is within bounds
// and that the change is reasonably sized
Float ub = (minChange * left.A + (1 - minChange) * right.A);
if (alpha > ub)
{
alpha = ub;
}
Float lb = (minChange * right.A + (1 - minChange) * left.A);
if (alpha < lb)
{
alpha = lb;
}
VectorUtils.AddMultInto(ref _x, alpha, ref _dir, ref _newX);
if (EnforceNonNegativity)
{
VBufferUtils.Apply(ref _newX, delegate(int ind, ref Float newXval)
{
if (newXval < 0.0)
{
newXval = 0;
}
});
}
Value = Eval(ref _newX, ref _newGrad);
GradientCalculations++;
if (!FloatUtils.IsFinite(Value))
{
throw ch.Except("Optimizer unable to proceed with loss function yielding {0}", Value);
}
dirDeriv = VectorUtils.DotProduct(ref _dir, ref _newGrad);
PointValueDeriv curr = new PointValueDeriv(alpha, Value, dirDeriv);
if ((curr.V > LastValue + c1 * alpha) || (curr.V >= aLo.V))
{
if (aHi.A == curr.A)
{
if (force)
{
throw ch.Process(new PrematureConvergenceException(this, "Step size interval numerically zero."));
}
else
{
return(false);
}
}
aHi = curr;
}
else if (Math.Abs(curr.D) <= -c2)
{
return(true);
}
else
{
if (curr.D * (aHi.A - aLo.A) >= 0)
{
aHi = aLo;
}
if (aLo.A == curr.A)
{
if (force)
{
throw ch.Process(new PrematureConvergenceException(this, "Step size interval numerically zero."));
}
else
{
return(false);
}
}
aLo = curr;
}
}
}
/// <summary>
/// Tests the gradient reported by f.
/// </summary>
/// <param name="f">function to test</param>
/// <param name="x">point at which to test</param>
/// <param name="quiet">If false, outputs detailed info.</param>
/// <returns>maximum normalized difference between analytic and numeric directional derivative over multiple tests</returns>
public static Float Test(DifferentiableFunction f, ref VBuffer <Float> x, bool quiet)
{
// REVIEW: Delete this method?
VBuffer <Float> grad = default(VBuffer <Float>);
VBuffer <Float> newGrad = default(VBuffer <Float>);
VBuffer <Float> newX = default(VBuffer <Float>);
Float normX = VectorUtils.Norm(x);
f(ref x, ref grad, null);
if (!quiet)
{
Console.WriteLine(Header);
}
Float maxNormDiff = Float.NegativeInfinity;
int numIters = Math.Min((int)x.Length, 10);
int maxDirCount = Math.Min((int)x.Length / 2, 100);
for (int n = 1; n <= numIters; n++)
{
int dirCount = Math.Min(n * 10, maxDirCount);
List <int> indices = new List <int>(dirCount);
List <Float> values = new List <Float>(dirCount);
for (int i = 0; i < dirCount; i++)
{
int index = _r.Next((int)x.Length);
while (indices.IndexOf(index) >= 0)
{
index = _r.Next((int)x.Length);
}
indices.Add(index);
values.Add(SampleFromGaussian(_r));
}
VBuffer <Float> dir = new VBuffer <Float>(x.Length, values.Count, values.ToArray(), indices.ToArray());
Float norm = VectorUtils.Norm(dir);
VectorUtils.ScaleBy(ref dir, 1 / norm);
VectorUtils.AddMultInto(ref x, Eps, ref dir, ref newX);
Float rVal = f(ref newX, ref newGrad, null);
VectorUtils.AddMultInto(ref x, -Eps, ref dir, ref newX);
Float lVal = f(ref newX, ref newGrad, null);
Float dirDeriv = VectorUtils.DotProduct(ref grad, ref dir);
Float numDeriv = (rVal - lVal) / (2 * Eps);
Float normDiff = Math.Abs(1 - numDeriv / dirDeriv);
Float diff = numDeriv - dirDeriv;
if (!quiet)
{
Console.WriteLine("{0,-9}{1,-18:0.0000e0}{2,-18:0.0000e0}{3,-15:0.0000e0}{4,0:0.0000e0}", n, numDeriv, dirDeriv, diff, normDiff);
}
maxNormDiff = Math.Max(maxNormDiff, normDiff);
}
return(maxNormDiff);
}