public virtual void Update(IMatrix source, IMatrix delta, ILearningContext context) { using var deltaSquared = delta.PointwiseMultiply(delta); _cache.AddInPlace(deltaSquared); using var cachedSqrt = _cache.Sqrt(); using var delta2 = delta.PointwiseDivide(cachedSqrt); _updater.Update(source, delta2, context); }
public void Update(IMatrix biasDelta, IMatrix weightDelta, float weightCoefficient, float learningRate) { if (biasDelta != null) { using (var columnSums = biasDelta.ColumnSums()) _bias.AddInPlace(columnSums, 1f / columnSums.Count, learningRate); } _weight.AddInPlace(weightDelta, weightCoefficient, learningRate); }
public void Update(IMatrix biasDelta, IMatrix weightDelta, float weightCoefficient, float learningRate) { if (biasDelta != null) { using (var columnSums = biasDelta.ColumnSums()) _bias.AddInPlace(columnSums, 1f / columnSums.Count, learningRate); } using (var transpose = weightDelta.Transpose()) _weight.AddInPlace(transpose, weightCoefficient, learningRate); _weightTranspose.Dispose(); _weightTranspose = _weight.Transpose(); }
public override void Update(IMatrix source, IMatrix delta, ILearningContext context) { var t = context.CurrentEpoch; using var deltaSquared = delta.PointwiseMultiply(delta); _cache.AddInPlace(delta, _decayRate, 1 - _decayRate); _cache2.AddInPlace(deltaSquared, _decayRate2, 1 - _decayRate2); using var mb = _cache.Clone(); using var vb = _cache2.Clone(); mb.Multiply(1f / (1f - Convert.ToSingle(Math.Pow(_decayRate, t)))); vb.Multiply(1f / (1f - Convert.ToSingle(Math.Pow(_decayRate2, t)))); using var vbSqrt = vb.Sqrt(); using var delta2 = mb.PointwiseDivide(vbSqrt); _updater.Update(source, delta2, context); }
public override void Update(IMatrix biasDelta, IMatrix weightDelta, ITrainingContext context) { var t = context.CurrentEpoch; using (var deltaSquared = weightDelta.PointwiseMultiply(weightDelta)) { _cache.AddInPlace(weightDelta, _decay, 1 - _decay); _cache2.AddInPlace(deltaSquared, _decay2, 1 - _decay2); using (var mb = _cache.Clone()) using (var vb = _cache2.Clone()) { mb.Multiply(1f / (1f - Convert.ToSingle(Math.Pow(_decay, t)))); vb.Multiply(1f / (1f - Convert.ToSingle(Math.Pow(_decay2, t)))); using (var vbSqrt = vb.Sqrt(1e-8f)) using (var delta = mb.PointwiseDivide(vbSqrt)) { _layerUpdater.Update(biasDelta, delta, context); } } } }
protected void _Update(IMatrix source, IMatrix delta, ILearningContext context, float coefficient1, float coefficient2) { source.AddInPlace(delta, coefficient1, coefficient2); }