private static void FillValues(Float input, ref VBuffer <Float> result)
{
var values = result.Values;
var indices = result.Indices;
if (input == 0)
{
result = new VBuffer <Float>(2, 0, values, indices);
return;
}
if (Utils.Size(values) < 1)
{
values = new Float[1];
}
if (Utils.Size(indices) < 1)
{
indices = new int[1];
}
if (Float.IsNaN(input))
{
values[0] = 1;
indices[0] = 1;
}
else
{
values[0] = input;
indices[0] = 0;
}
result = new VBuffer <Float>(2, 1, values, indices);
}
/// <summary>
/// The hyperbolic tangent function.
/// </summary>
public static Float TanhFast(Float x)
{
if (Float.IsNaN(x))
{
return(x);
}
bool neg = false;
if (x < 0)
{
x = -x;
neg = true;
}
// Multiply by 2/ln(2).
x *= 2 * RecipLn2;
if (x >= ExpInf)
{
return(neg ? (Float)(-1) : (Float)1);
}
// Get the floor and fractional part.
int n = (int)x;
Contracts.Assert(0 <= n && n < ExpInf);
Float f = x - n;
Contracts.Assert(0 <= f && f < 1);
// Get the integer power of two part.
Float r = PowerOfTwo(n);
Contracts.Assert(1 <= r && r < Float.PositiveInfinity);
// This approximates 2^f - 1 for 0 <= f <= 1. Note that it is exact at the endpoints.
Float res = f + (f - 1) * f * ((Coef1 * f + Coef2) * f + Coef3);
res *= r;
res = (res + (r - 1)) / (res + (r + 1));
if (neg)
{
res = -res;
}
return(res);
}
/// <summary>
/// The logistic sigmoid function: 1 / (1 + e^(-x)).
/// </summary>
public static Float SigmoidFast(Float x)
{
// This is a loose translation from SSE code
if (Float.IsNaN(x))
{
return(x);
}
bool neg = false;
if (x < 0)
{
x = -x;
neg = true;
}
// Multiply by 1/ln(2).
x *= RecipLn2;
if (x >= ExpInf)
{
return(neg ? (Float)0 : (Float)1);
}
// Get the floor and fractional part.
int n = (int)x;
Contracts.Assert(0 <= n && n < ExpInf);
Float f = x - n;
Contracts.Assert(0 <= f && f < 1);
// Get the integer power of two part.
Float r = PowerOfTwo(n);
Contracts.Assert(1 <= r && r < Float.PositiveInfinity);
// This approximates 2^f for 0 <= f <= 1. Note that it is exact at the endpoints.
Float res = 1 + f + (f - 1) * f * ((Coef1 * f + Coef2) * f + Coef3);
res = 1 / (1 + r * res);
if (!neg)
{
res = 1 - res;
}
return(res);
}
public static Float GetMedianInPlace(Float[] src, int count)
{
Contracts.Assert(count >= 0);
Contracts.Assert(Utils.Size(src) >= count);
if (count == 0)
{
return(Float.NaN);
}
Array.Sort(src, 0, count);
// Skip any NaNs. They sort to the low end.
int ivMin = 0;
int ivLim = count;
while (ivMin < ivLim && Float.IsNaN(src[ivMin]))
{
ivMin++;
}
Contracts.Assert(ivMin <= ivLim);
if (ivMin >= ivLim)
{
return(Float.NaN);
}
// This assert will fire if Array.Sort changes to put NaNs at the high end.
Contracts.Assert(!Float.IsNaN(src[ivLim - 1]));
// If we're dealing with an odd number of things, just grab the middel item; otherwise,
// average the two middle items.
uint cv = (uint)ivMin + (uint)ivLim;
int iv = (int)(cv / 2);
if ((cv & 1) != 0)
{
return(src[iv]);
}
return((src[iv - 1] + src[iv]) / 2);
}
private static void FillValues(Float input, ref VBuffer <Float> result)
{
if (input == 0)
{
VBufferUtils.Resize(ref result, 2, 0);
return;
}
var editor = VBufferEditor.Create(ref result, 2, 1);
if (Float.IsNaN(input))
{
editor.Values[0] = 1;
editor.Indices[0] = 1;
}
else
{
editor.Values[0] = input;
editor.Indices[0] = 0;
}
result = editor.Commit();
}
// This converts in place.
private static void FillValues(IExceptionContext ectx, ref VBuffer <Float> buffer)
{
int size = buffer.Length;
ectx.Check(0 <= size & size < int.MaxValue / 2);
int count = buffer.Count;
var values = buffer.Values;
var indices = buffer.Indices;
int iivDst = 0;
if (count >= size)
{
// Currently, it's dense. We always produce sparse.
ectx.Assert(Utils.Size(values) >= size);
if (Utils.Size(indices) < size)
{
indices = new int[size];
}
for (int ivSrc = 0; ivSrc < count; ivSrc++)
{
ectx.Assert(iivDst <= ivSrc);
var val = values[ivSrc];
if (val == 0)
{
continue;
}
if (Float.IsNaN(val))
{
values[iivDst] = 1;
indices[iivDst] = 2 * ivSrc + 1;
}
else
{
values[iivDst] = val;
indices[iivDst] = 2 * ivSrc;
}
iivDst++;
}
}
else
{
// Currently, it's sparse.
ectx.Assert(Utils.Size(values) >= count);
ectx.Assert(Utils.Size(indices) >= count);
int ivPrev = -1;
for (int iivSrc = 0; iivSrc < count; iivSrc++)
{
ectx.Assert(iivDst <= iivSrc);
var val = values[iivSrc];
if (val == 0)
{
continue;
}
int iv = indices[iivSrc];
ectx.Assert(ivPrev < iv & iv < size);
ivPrev = iv;
if (Float.IsNaN(val))
{
values[iivDst] = 1;
indices[iivDst] = 2 * iv + 1;
}
else
{
values[iivDst] = val;
indices[iivDst] = 2 * iv;
}
iivDst++;
}
}
ectx.Assert(0 <= iivDst & iivDst <= count);
buffer = new VBuffer <Float>(size * 2, iivDst, values, indices);
}
// This converts in place.
private static void FillValues(IExceptionContext ectx, ref VBuffer <Float> buffer)
{
int size = buffer.Length;
ectx.Check(0 <= size & size < int.MaxValue / 2);
var values = buffer.GetValues();
var editor = VBufferEditor.Create(ref buffer, size * 2, values.Length);
int iivDst = 0;
if (buffer.IsDense)
{
// Currently, it's dense. We always produce sparse.
for (int ivSrc = 0; ivSrc < values.Length; ivSrc++)
{
ectx.Assert(iivDst <= ivSrc);
var val = values[ivSrc];
if (val == 0)
{
continue;
}
if (Float.IsNaN(val))
{
editor.Values[iivDst] = 1;
editor.Indices[iivDst] = 2 * ivSrc + 1;
}
else
{
editor.Values[iivDst] = val;
editor.Indices[iivDst] = 2 * ivSrc;
}
iivDst++;
}
}
else
{
// Currently, it's sparse.
var indices = buffer.GetIndices();
int ivPrev = -1;
for (int iivSrc = 0; iivSrc < values.Length; iivSrc++)
{
ectx.Assert(iivDst <= iivSrc);
var val = values[iivSrc];
if (val == 0)
{
continue;
}
int iv = indices[iivSrc];
ectx.Assert(ivPrev < iv & iv < size);
ivPrev = iv;
if (Float.IsNaN(val))
{
editor.Values[iivDst] = 1;
editor.Indices[iivDst] = 2 * iv + 1;
}
else
{
editor.Values[iivDst] = val;
editor.Indices[iivDst] = 2 * iv;
}
iivDst++;
}
}
ectx.Assert(0 <= iivDst & iivDst <= values.Length);
buffer = editor.CommitTruncated(iivDst);
}
protected override bool IsNaN(ref Float score)
{
return(Float.IsNaN(score));
}
/// <summary>
/// Returns a value indicating whether the specified number evaluates to not a number
/// </summary>
/// <param name="number">a floating point number</param>
/// <returns>a boolean</returns>
public static bool IsNaN(Real number)
{
return(Numeric.IsNaN((Numeric)number));
}
/// <summary>
/// An implementation of the line search for the Wolfe conditions, from Nocedal & Wright
/// </summary>
internal virtual bool LineSearch(IChannel ch, bool force)
{
Contracts.AssertValue(ch);
Float dirDeriv = VectorUtils.DotProduct(ref _dir, ref _grad);
if (dirDeriv == 0)
{
throw ch.Process(new PrematureConvergenceException(this, "Directional derivative is zero. You may be sitting on the optimum."));
}
// if a non-descent direction is chosen, the line search will break anyway, so throw here
// The most likely reasons for this is a bug in your function's gradient computation,
ch.Check(dirDeriv < 0, "L-BFGS chose a non-descent direction.");
Float c1 = (Float)1e-4 * dirDeriv;
Float c2 = (Float)0.9 * dirDeriv;
Float alpha = (Iter == 1 ? (1 / VectorUtils.Norm(_dir)) : 1);
PointValueDeriv last = new PointValueDeriv(0, LastValue, dirDeriv);
PointValueDeriv aLo = new PointValueDeriv();
PointValueDeriv aHi = new PointValueDeriv();
// initial bracketing phase
while (true)
{
VectorUtils.AddMultInto(ref _x, alpha, ref _dir, ref _newX);
if (EnforceNonNegativity)
{
VBufferUtils.Apply(ref _newX, delegate(int ind, ref Float newXval)
{
if (newXval < 0.0)
{
newXval = 0;
}
});
}
Value = Eval(ref _newX, ref _newGrad);
GradientCalculations++;
if (Float.IsPositiveInfinity(Value))
{
alpha /= 2;
continue;
}
if (!FloatUtils.IsFinite(Value))
{
throw ch.Except("Optimizer unable to proceed with loss function yielding {0}", Value);
}
dirDeriv = VectorUtils.DotProduct(ref _dir, ref _newGrad);
PointValueDeriv curr = new PointValueDeriv(alpha, Value, dirDeriv);
if ((curr.V > LastValue + c1 * alpha) || (last.A > 0 && curr.V >= last.V))
{
aLo = last;
aHi = curr;
break;
}
else if (Math.Abs(curr.D) <= -c2)
{
return(true);
}
else if (curr.D >= 0)
{
aLo = curr;
aHi = last;
break;
}
last = curr;
if (alpha == 0)
{
alpha = Float.Epsilon; // Robust to divisional underflow.
}
else
{
alpha *= 2;
}
}
Float minChange = (Float)0.01;
int maxSteps = 10;
// this loop is the "zoom" procedure described in Nocedal & Wright
for (int step = 0; ; ++step)
{
if (step == maxSteps && !force)
{
return(false);
}
PointValueDeriv left = aLo.A < aHi.A ? aLo : aHi;
PointValueDeriv right = aLo.A < aHi.A ? aHi : aLo;
if (left.D > 0 && right.D < 0)
{
// interpolating cubic would have max in range, not min (can this happen?)
// set a to the one with smaller value
alpha = aLo.V < aHi.V ? aLo.A : aHi.A;
}
else
{
alpha = CubicInterp(aLo, aHi);
if (Float.IsNaN(alpha) || Float.IsInfinity(alpha))
{
alpha = (aLo.A + aHi.A) / 2;
}
}
// this is to ensure that the new point is within bounds
// and that the change is reasonably sized
Float ub = (minChange * left.A + (1 - minChange) * right.A);
if (alpha > ub)
{
alpha = ub;
}
Float lb = (minChange * right.A + (1 - minChange) * left.A);
if (alpha < lb)
{
alpha = lb;
}
VectorUtils.AddMultInto(ref _x, alpha, ref _dir, ref _newX);
if (EnforceNonNegativity)
{
VBufferUtils.Apply(ref _newX, delegate(int ind, ref Float newXval)
{
if (newXval < 0.0)
{
newXval = 0;
}
});
}
Value = Eval(ref _newX, ref _newGrad);
GradientCalculations++;
if (!FloatUtils.IsFinite(Value))
{
throw ch.Except("Optimizer unable to proceed with loss function yielding {0}", Value);
}
dirDeriv = VectorUtils.DotProduct(ref _dir, ref _newGrad);
PointValueDeriv curr = new PointValueDeriv(alpha, Value, dirDeriv);
if ((curr.V > LastValue + c1 * alpha) || (curr.V >= aLo.V))
{
if (aHi.A == curr.A)
{
if (force)
{
throw ch.Process(new PrematureConvergenceException(this, "Step size interval numerically zero."));
}
else
{
return(false);
}
}
aHi = curr;
}
else if (Math.Abs(curr.D) <= -c2)
{
return(true);
}
else
{
if (curr.D * (aHi.A - aLo.A) >= 0)
{
aHi = aLo;
}
if (aLo.A == curr.A)
{
if (force)
{
throw ch.Process(new PrematureConvergenceException(this, "Step size interval numerically zero."));
}
else
{
return(false);
}
}
aLo = curr;
}
}
}
