public virtual void Update(IMatrix source, IMatrix delta, ILearningContext context)
{
using var deltaSquared = delta.PointwiseMultiply(delta);
_cache.AddInPlace(deltaSquared);
using var cachedSqrt = _cache.Sqrt();
using var delta2 = delta.PointwiseDivide(cachedSqrt);
_updater.Update(source, delta2, context);
}
public void Update(IMatrix biasDelta, IMatrix weightDelta, float weightCoefficient, float learningRate)
{
if (biasDelta != null)
{
using (var columnSums = biasDelta.ColumnSums())
_bias.AddInPlace(columnSums, 1f / columnSums.Count, learningRate);
}
_weight.AddInPlace(weightDelta, weightCoefficient, learningRate);
}
public void Update(IMatrix biasDelta, IMatrix weightDelta, float weightCoefficient, float learningRate)
{
if (biasDelta != null)
{
using (var columnSums = biasDelta.ColumnSums())
_bias.AddInPlace(columnSums, 1f / columnSums.Count, learningRate);
}
using (var transpose = weightDelta.Transpose())
_weight.AddInPlace(transpose, weightCoefficient, learningRate);
_weightTranspose.Dispose();
_weightTranspose = _weight.Transpose();
}
public override void Update(IMatrix source, IMatrix delta, ILearningContext context)
{
var t = context.CurrentEpoch;
using var deltaSquared = delta.PointwiseMultiply(delta);
_cache.AddInPlace(delta, _decayRate, 1 - _decayRate);
_cache2.AddInPlace(deltaSquared, _decayRate2, 1 - _decayRate2);
using var mb = _cache.Clone();
using var vb = _cache2.Clone();
mb.Multiply(1f / (1f - Convert.ToSingle(Math.Pow(_decayRate, t))));
vb.Multiply(1f / (1f - Convert.ToSingle(Math.Pow(_decayRate2, t))));
using var vbSqrt = vb.Sqrt();
using var delta2 = mb.PointwiseDivide(vbSqrt);
_updater.Update(source, delta2, context);
}
public override void Update(IMatrix biasDelta, IMatrix weightDelta, ITrainingContext context)
{
var t = context.CurrentEpoch;
using (var deltaSquared = weightDelta.PointwiseMultiply(weightDelta)) {
_cache.AddInPlace(weightDelta, _decay, 1 - _decay);
_cache2.AddInPlace(deltaSquared, _decay2, 1 - _decay2);
using (var mb = _cache.Clone())
using (var vb = _cache2.Clone()) {
mb.Multiply(1f / (1f - Convert.ToSingle(Math.Pow(_decay, t))));
vb.Multiply(1f / (1f - Convert.ToSingle(Math.Pow(_decay2, t))));
using (var vbSqrt = vb.Sqrt(1e-8f))
using (var delta = mb.PointwiseDivide(vbSqrt)) {
_layerUpdater.Update(biasDelta, delta, context);
}
}
}
}
protected void _Update(IMatrix source, IMatrix delta, ILearningContext context, float coefficient1, float coefficient2)
{
source.AddInPlace(delta, coefficient1, coefficient2);
}
