private void GetNextPoint(Float alpha) { VectorUtils.AddMultInto(ref _x, alpha, ref _dir, ref _newX); if (!EnforceNonNegativity) { VBufferUtils.ApplyWith(ref _x, ref _newX, delegate(int ind, Float xVal, ref Float newXval) { if (xVal * newXval < 0.0 && ind >= _biasCount) { newXval = 0; } }); } else { VBufferUtils.Apply(ref _newX, delegate(int ind, ref Float newXval) { if (newXval < 0.0 && ind >= _biasCount) { newXval = 0; } }); } }
/// <summary> /// Backtracking line search with Armijo-like condition, from Andrew & Gao /// </summary> internal override bool LineSearch(IChannel ch, bool force) { Float dirDeriv = -VectorUtils.DotProduct(ref _dir, ref _steepestDescDir); if (dirDeriv == 0) { throw ch.Process(new PrematureConvergenceException(this, "Directional derivative is zero. You may be sitting on the optimum.")); } // if a non-descent direction is chosen, the line search will break anyway, so throw here // The most likely reason for this is a bug in your function's gradient computation // It may also indicate that your function is not convex. ch.Check(dirDeriv < 0, "L-BFGS chose a non-descent direction."); Float alpha = (Iter == 1 ? (1 / VectorUtils.Norm(_dir)) : 1); GetNextPoint(alpha); Float unnormCos = VectorUtils.DotProduct(ref _steepestDescDir, ref _newX) - VectorUtils.DotProduct(ref _steepestDescDir, ref _x); if (unnormCos < 0) { VBufferUtils.ApplyWith(ref _steepestDescDir, ref _dir, (int ind, Float sdVal, ref Float dirVal) => { if (sdVal * dirVal < 0 && ind >= _biasCount) { dirVal = 0; } }); GetNextPoint(alpha); unnormCos = VectorUtils.DotProduct(ref _steepestDescDir, ref _newX) - VectorUtils.DotProduct(ref _steepestDescDir, ref _x); } int i = 0; while (true) { Value = Eval(ref _newX, ref _newGrad); GradientCalculations++; if (Value <= LastValue - Gamma * unnormCos) { return(true); } ++i; if (!force && i == MaxLineSearch) { return(false); } alpha *= (Float)0.25; GetNextPoint(alpha); unnormCos = VectorUtils.DotProduct(ref _steepestDescDir, ref _newX) - VectorUtils.DotProduct(ref _steepestDescDir, ref _x); } }
public override Delegate[] CreateGetters(IRow input, Func <int, bool> activeCols, out Action disposer) { Host.Assert(LabelIndex >= 0); Host.Assert(ScoreIndex >= 0); disposer = null; long cachedPosition = -1; Float label = 0; var score = default(VBuffer <Float>); var l1 = VBufferUtils.CreateDense <Double>(_scoreSize); ValueGetter <Float> nanGetter = (ref Float value) => value = Single.NaN; var labelGetter = activeCols(L1Col) || activeCols(L2Col) ? RowCursorUtils.GetLabelGetter(input, LabelIndex) : nanGetter; ValueGetter <VBuffer <Float> > scoreGetter; if (activeCols(L1Col) || activeCols(L2Col)) { scoreGetter = input.GetGetter <VBuffer <Float> >(ScoreIndex); } else { scoreGetter = (ref VBuffer <Float> dst) => dst = default(VBuffer <Float>); } Action updateCacheIfNeeded = () => { if (cachedPosition != input.Position) { labelGetter(ref label); scoreGetter(ref score); var lab = (Double)label; foreach (var s in score.Items(all: true)) { l1.Values[s.Key] = Math.Abs(lab - s.Value); } cachedPosition = input.Position; } }; var getters = new Delegate[2]; if (activeCols(L1Col)) { ValueGetter <VBuffer <Double> > l1Fn = (ref VBuffer <Double> dst) => { updateCacheIfNeeded(); l1.CopyTo(ref dst); }; getters[L1Col] = l1Fn; } if (activeCols(L2Col)) { VBufferUtils.PairManipulator <Double, Double> sqr = (int slot, Double x, ref Double y) => y = x * x; ValueGetter <VBuffer <Double> > l2Fn = (ref VBuffer <Double> dst) => { updateCacheIfNeeded(); dst = new VBuffer <Double>(_scoreSize, 0, dst.Values, dst.Indices); VBufferUtils.ApplyWith(ref l1, ref dst, sqr); }; getters[L2Col] = l2Fn; } return(getters); }
protected override void ApplyLossFunction(ref VBuffer <float> score, float label, ref VBuffer <Double> loss) { VBufferUtils.PairManipulator <Float, Double> lossFn = (int slot, Float src, ref Double dst) => dst = LossFunction.Loss(src, label); VBufferUtils.ApplyWith(ref score, ref loss, lossFn); }