internal override void Update(float[] weight, float[] delta)
{
var velocity = _velocity ?? new float[delta.Length];
Blas1.scal(velocity.Length, MomentumValue, velocity, 1);
Blas1.axpy(velocity.Length, -LearningRate, delta, 1, velocity, 1);
_velocity = velocity;
Blas1.axpy(weight.Length, 1.0f, velocity, 1, weight, 1);
}
internal override void Update(float[] weight, float[] delta)
{
var v = _v ?? new float[delta.Length];
var sq = new float[delta.Length];
Parallel.For(0, sq.Length, i => { sq[i] = delta[i] * delta[i]; });
Blas1.scal(v.Length, RememberRate, v, 1);
Blas1.axpy(sq.Length, 1.0f - RememberRate, sq, 1, v, 1);
Parallel.For(0, weight.Length, i => { weight[i] -= LearningRate / (MathF.Sqrt(v[i]) + 1e-8f) * delta[i]; });
_v = v;
}
internal override void Update(float[] weight, float[] delta)
{
if (_weightMemory == null)
{
Blas1.copy(weight.Length, weight, 1, out _weightMemory, 1);
}
var velocity = _velocity ?? new float[weight.Length];
Blas1.scal(velocity.Length, MomentumValue, velocity, 1);
Blas1.axpy(velocity.Length, -LearningRate, delta, 1, velocity, 1);
_velocity = velocity;
Blas1.axpy(_weightMemory.Length, 1.0f, velocity, 1, _weightMemory, 1);
Blas1.copy(_velocity.Length, _velocity, 1, out var ahead, 1);
Blas1.scal(ahead.Length, MomentumValue, ahead, 1);
Blas1.copy(_weightMemory.Length, _weightMemory, 1, weight, 1);
Blas1.axpy(weight.Length, 1.0f, ahead, 1, weight, 1);
}
public void ScalTest()
{
const float af = 2.0f;
const double ad = -1.0;
var xf = new[] { 1.0f, 1.0f, 1.0f };
Blas1.copy(xf.Length, xf, 1, out var mxf, 1);
var xd = new[] { 1.0, 1.0, 1.0 };
Blas1.copy(xd.Length, xd, 1, out var mxd, 1);
Blas1.scal(mxf.Length, af, mxf, 1);
Blas1.scal(mxd.Length, ad, mxd, 1);
for (var i = 0; i < mxf.Length; i++)
{
Assert.AreEqual(af * xf[i], mxf[i]);
}
for (var i = 0; i < mxd.Length; i++)
{
Assert.AreEqual(ad * xd[i], mxd[i]);
}
}
internal override void Update(float[] weight, float[] delta)
{
var m = _m ?? new float[delta.Length];
var v = _v ?? new float[delta.Length];
var sq = new float[delta.Length];
Parallel.For(0, sq.Length, i => { sq[i] = delta[i] * delta[i]; });
_beta1 *= Beta1;
_beta2 *= Beta2;
Blas1.scal(m.Length, Beta1, m, 1);
Blas1.scal(v.Length, Beta2, v, 1);
Blas1.axpy(m.Length, 1.0f - Beta1, delta, 1, m, 1);
Blas1.axpy(v.Length, 1.0f - Beta2, sq, 1, v, 1);
Parallel.For(0, weight.Length, i => {
var mHat = m[i] / (1.0f - _beta1);
var vHat = v[i] / (1.0f - _beta2);
weight[i] -= LearningRate / (MathF.Sqrt(vHat) + Eps) * mHat;
});
_m = m;
_v = v;
}
