internal override void Update(float[] weight, float[] delta)
{
var velocity = _velocity ?? new float[delta.Length];
Blas1.scal(velocity.Length, MomentumValue, velocity, 1);
Blas1.axpy(velocity.Length, -LearningRate, delta, 1, velocity, 1);
_velocity = velocity;
Blas1.axpy(weight.Length, 1.0f, velocity, 1, weight, 1);
}
internal override void Update(float[] weight, float[] delta)
{
var v = _v ?? new float[delta.Length];
var sq = new float[delta.Length];
Parallel.For(0, sq.Length, i => { sq[i] = delta[i] * delta[i]; });
Blas1.scal(v.Length, RememberRate, v, 1);
Blas1.axpy(sq.Length, 1.0f - RememberRate, sq, 1, v, 1);
Parallel.For(0, weight.Length, i => { weight[i] -= LearningRate / (MathF.Sqrt(v[i]) + 1e-8f) * delta[i]; });
_v = v;
}
internal override void Update(float[] weight, float[] delta)
{
if (_weightMemory == null)
{
Blas1.copy(weight.Length, weight, 1, out _weightMemory, 1);
}
var velocity = _velocity ?? new float[weight.Length];
Blas1.scal(velocity.Length, MomentumValue, velocity, 1);
Blas1.axpy(velocity.Length, -LearningRate, delta, 1, velocity, 1);
_velocity = velocity;
Blas1.axpy(_weightMemory.Length, 1.0f, velocity, 1, _weightMemory, 1);
Blas1.copy(_velocity.Length, _velocity, 1, out var ahead, 1);
Blas1.scal(ahead.Length, MomentumValue, ahead, 1);
Blas1.copy(_weightMemory.Length, _weightMemory, 1, weight, 1);
Blas1.axpy(weight.Length, 1.0f, ahead, 1, weight, 1);
}
public void AxpyTest()
{
var xf = new[] { 1.0f, 1.0f, 1.0f };
var yf = new[] { 1.0f, 1.0f, 1.0f };
var xd = new[] { 1.0, 1.0, 1.0 };
var yd = new[] { 1.0, 1.0, 1.0 };
Blas1.axpy(3, 2.0f, xf, 1, yf, 1);
foreach (var t in yf)
{
Assert.AreEqual(3.0f, t);
}
Blas1.axpy(3, 2.0, xd, 1, yd, 1);
foreach (var t in yd)
{
Assert.AreEqual(3.0, t);
}
}
internal override void Update(float[] weight, float[] delta)
{
var m = _m ?? new float[delta.Length];
var v = _v ?? new float[delta.Length];
var sq = new float[delta.Length];
Parallel.For(0, sq.Length, i => { sq[i] = delta[i] * delta[i]; });
_beta1 *= Beta1;
_beta2 *= Beta2;
Blas1.scal(m.Length, Beta1, m, 1);
Blas1.scal(v.Length, Beta2, v, 1);
Blas1.axpy(m.Length, 1.0f - Beta1, delta, 1, m, 1);
Blas1.axpy(v.Length, 1.0f - Beta2, sq, 1, v, 1);
Parallel.For(0, weight.Length, i => {
var mHat = m[i] / (1.0f - _beta1);
var vHat = v[i] / (1.0f - _beta2);
weight[i] -= LearningRate / (MathF.Sqrt(vHat) + Eps) * mHat;
});
_m = m;
_v = v;
}