private void GetNextPoint(Float alpha)
{
VectorUtils.AddMultInto(ref _x, alpha, ref _dir, ref _newX);
if (!EnforceNonNegativity)
{
VBufferUtils.ApplyWith(ref _x, ref _newX,
delegate(int ind, Float xVal, ref Float newXval)
{
if (xVal * newXval < 0.0 && ind >= _biasCount)
{
newXval = 0;
}
});
}
else
{
VBufferUtils.Apply(ref _newX, delegate(int ind, ref Float newXval)
{
if (newXval < 0.0 && ind >= _biasCount)
{
newXval = 0;
}
});
}
}
internal override OptimizerState MakeState(IChannel ch, IProgressChannelProvider progress, DifferentiableFunction function, ref VBuffer <Float> initial)
{
Contracts.AssertValue(ch);
ch.AssertValue(progress);
if (EnforceNonNegativity)
{
VBufferUtils.Apply(ref initial, delegate(int ind, ref Float initialVal)
{
if (initialVal < 0.0 && ind >= _biasCount)
{
initialVal = 0;
}
});
}
if (_l1weight > 0 && _biasCount < initial.Length)
{
return(new L1OptimizerState(ch, progress, function, in initial, M, TotalMemoryLimit, _biasCount, _l1weight, KeepDense, EnforceNonNegativity));
}
return(new FunctionOptimizerState(ch, progress, function, in initial, M, TotalMemoryLimit, KeepDense, EnforceNonNegativity));
}
protected override Delegate GetGetterCore(IChannel ch, IRow input, int iinfo, out Action disposer)
{
Host.AssertValueOrNull(ch);
Host.AssertValue(input);
Host.Assert(0 <= iinfo && iinfo < Infos.Length);
Host.Assert(Infos[iinfo].TypeSrc.IsVector);
Host.Assert(Infos[iinfo].TypeSrc.ItemType.IsKey);
disposer = null;
var getSrc = RowCursorUtils.GetVecGetterAs <uint>(NumberType.U4, input, Infos[iinfo].Source);
var src = default(VBuffer <uint>);
var bldr = new NgramBufferBuilder(_exes[iinfo].NgramLength, _exes[iinfo].SkipLength,
_ngramMaps[iinfo].Count, GetNgramIdFinder(iinfo));
var keyCount = (uint)Infos[iinfo].TypeSrc.ItemType.KeyCount;
if (keyCount == 0)
{
keyCount = uint.MaxValue;
}
ValueGetter <VBuffer <Float> > del;
switch (_exes[iinfo].Weighting)
{
case WeightingCriteria.TfIdf:
Host.AssertValue(_invDocFreqs[iinfo]);
del =
(ref VBuffer <Float> dst) =>
{
getSrc(ref src);
if (!bldr.IsEmpty)
{
bldr.Reset();
bldr.AddNgrams(in src, 0, keyCount);
bldr.GetResult(ref dst);
VBufferUtils.Apply(ref dst, (int i, ref Float v) => v = (Float)(v * _invDocFreqs[iinfo][i]));
}
else
{
dst = new VBuffer <Float>(0, dst.Values, dst.Indices);
}
};
break;
case WeightingCriteria.Idf:
Host.AssertValue(_invDocFreqs[iinfo]);
del =
(ref VBuffer <Float> dst) =>
{
getSrc(ref src);
if (!bldr.IsEmpty)
{
bldr.Reset();
bldr.AddNgrams(in src, 0, keyCount);
bldr.GetResult(ref dst);
VBufferUtils.Apply(ref dst, (int i, ref Float v) => v = v >= 1 ? (Float)_invDocFreqs[iinfo][i] : 0);
}
else
{
dst = new VBuffer <Float>(0, dst.Values, dst.Indices);
}
};
break;
case WeightingCriteria.Tf:
del =
(ref VBuffer <Float> dst) =>
{
getSrc(ref src);
if (!bldr.IsEmpty)
{
bldr.Reset();
bldr.AddNgrams(in src, 0, keyCount);
bldr.GetResult(ref dst);
}
else
{
dst = new VBuffer <Float>(0, dst.Values, dst.Indices);
}
};
break;
default:
throw Host.Except("Unsupported weighting criteria");
}
return(del);
}
/// <summary>
/// An implementation of the line search for the Wolfe conditions, from Nocedal & Wright
/// </summary>
internal virtual bool LineSearch(IChannel ch, bool force)
{
Contracts.AssertValue(ch);
Float dirDeriv = VectorUtils.DotProduct(ref _dir, ref _grad);
if (dirDeriv == 0)
{
throw ch.Process(new PrematureConvergenceException(this, "Directional derivative is zero. You may be sitting on the optimum."));
}
// if a non-descent direction is chosen, the line search will break anyway, so throw here
// The most likely reasons for this is a bug in your function's gradient computation,
ch.Check(dirDeriv < 0, "L-BFGS chose a non-descent direction.");
Float c1 = (Float)1e-4 * dirDeriv;
Float c2 = (Float)0.9 * dirDeriv;
Float alpha = (Iter == 1 ? (1 / VectorUtils.Norm(_dir)) : 1);
PointValueDeriv last = new PointValueDeriv(0, LastValue, dirDeriv);
PointValueDeriv aLo = new PointValueDeriv();
PointValueDeriv aHi = new PointValueDeriv();
// initial bracketing phase
while (true)
{
VectorUtils.AddMultInto(ref _x, alpha, ref _dir, ref _newX);
if (EnforceNonNegativity)
{
VBufferUtils.Apply(ref _newX, delegate(int ind, ref Float newXval)
{
if (newXval < 0.0)
{
newXval = 0;
}
});
}
Value = Eval(ref _newX, ref _newGrad);
GradientCalculations++;
if (Float.IsPositiveInfinity(Value))
{
alpha /= 2;
continue;
}
if (!FloatUtils.IsFinite(Value))
{
throw ch.Except("Optimizer unable to proceed with loss function yielding {0}", Value);
}
dirDeriv = VectorUtils.DotProduct(ref _dir, ref _newGrad);
PointValueDeriv curr = new PointValueDeriv(alpha, Value, dirDeriv);
if ((curr.V > LastValue + c1 * alpha) || (last.A > 0 && curr.V >= last.V))
{
aLo = last;
aHi = curr;
break;
}
else if (Math.Abs(curr.D) <= -c2)
{
return(true);
}
else if (curr.D >= 0)
{
aLo = curr;
aHi = last;
break;
}
last = curr;
if (alpha == 0)
{
alpha = Float.Epsilon; // Robust to divisional underflow.
}
else
{
alpha *= 2;
}
}
Float minChange = (Float)0.01;
int maxSteps = 10;
// this loop is the "zoom" procedure described in Nocedal & Wright
for (int step = 0; ; ++step)
{
if (step == maxSteps && !force)
{
return(false);
}
PointValueDeriv left = aLo.A < aHi.A ? aLo : aHi;
PointValueDeriv right = aLo.A < aHi.A ? aHi : aLo;
if (left.D > 0 && right.D < 0)
{
// interpolating cubic would have max in range, not min (can this happen?)
// set a to the one with smaller value
alpha = aLo.V < aHi.V ? aLo.A : aHi.A;
}
else
{
alpha = CubicInterp(aLo, aHi);
if (Float.IsNaN(alpha) || Float.IsInfinity(alpha))
{
alpha = (aLo.A + aHi.A) / 2;
}
}
// this is to ensure that the new point is within bounds
// and that the change is reasonably sized
Float ub = (minChange * left.A + (1 - minChange) * right.A);
if (alpha > ub)
{
alpha = ub;
}
Float lb = (minChange * right.A + (1 - minChange) * left.A);
if (alpha < lb)
{
alpha = lb;
}
VectorUtils.AddMultInto(ref _x, alpha, ref _dir, ref _newX);
if (EnforceNonNegativity)
{
VBufferUtils.Apply(ref _newX, delegate(int ind, ref Float newXval)
{
if (newXval < 0.0)
{
newXval = 0;
}
});
}
Value = Eval(ref _newX, ref _newGrad);
GradientCalculations++;
if (!FloatUtils.IsFinite(Value))
{
throw ch.Except("Optimizer unable to proceed with loss function yielding {0}", Value);
}
dirDeriv = VectorUtils.DotProduct(ref _dir, ref _newGrad);
PointValueDeriv curr = new PointValueDeriv(alpha, Value, dirDeriv);
if ((curr.V > LastValue + c1 * alpha) || (curr.V >= aLo.V))
{
if (aHi.A == curr.A)
{
if (force)
{
throw ch.Process(new PrematureConvergenceException(this, "Step size interval numerically zero."));
}
else
{
return(false);
}
}
aHi = curr;
}
else if (Math.Abs(curr.D) <= -c2)
{
return(true);
}
else
{
if (curr.D * (aHi.A - aLo.A) >= 0)
{
aHi = aLo;
}
if (aLo.A == curr.A)
{
if (force)
{
throw ch.Process(new PrematureConvergenceException(this, "Step size interval numerically zero."));
}
else
{
return(false);
}
}
aLo = curr;
}
}
}
