internal override void Update(float[] weight, float[] delta) { var velocity = _velocity ?? new float[delta.Length]; Blas1.scal(velocity.Length, MomentumValue, velocity, 1); Blas1.axpy(velocity.Length, -LearningRate, delta, 1, velocity, 1); _velocity = velocity; Blas1.axpy(weight.Length, 1.0f, velocity, 1, weight, 1); }
internal override void Update(float[] weight, float[] delta) { var v = _v ?? new float[delta.Length]; var sq = new float[delta.Length]; Parallel.For(0, sq.Length, i => { sq[i] = delta[i] * delta[i]; }); Blas1.scal(v.Length, RememberRate, v, 1); Blas1.axpy(sq.Length, 1.0f - RememberRate, sq, 1, v, 1); Parallel.For(0, weight.Length, i => { weight[i] -= LearningRate / (MathF.Sqrt(v[i]) + 1e-8f) * delta[i]; }); _v = v; }
internal override void Update(float[] weight, float[] delta) { if (_weightMemory == null) { Blas1.copy(weight.Length, weight, 1, out _weightMemory, 1); } var velocity = _velocity ?? new float[weight.Length]; Blas1.scal(velocity.Length, MomentumValue, velocity, 1); Blas1.axpy(velocity.Length, -LearningRate, delta, 1, velocity, 1); _velocity = velocity; Blas1.axpy(_weightMemory.Length, 1.0f, velocity, 1, _weightMemory, 1); Blas1.copy(_velocity.Length, _velocity, 1, out var ahead, 1); Blas1.scal(ahead.Length, MomentumValue, ahead, 1); Blas1.copy(_weightMemory.Length, _weightMemory, 1, weight, 1); Blas1.axpy(weight.Length, 1.0f, ahead, 1, weight, 1); }
public void ScalTest() { const float af = 2.0f; const double ad = -1.0; var xf = new[] { 1.0f, 1.0f, 1.0f }; Blas1.copy(xf.Length, xf, 1, out var mxf, 1); var xd = new[] { 1.0, 1.0, 1.0 }; Blas1.copy(xd.Length, xd, 1, out var mxd, 1); Blas1.scal(mxf.Length, af, mxf, 1); Blas1.scal(mxd.Length, ad, mxd, 1); for (var i = 0; i < mxf.Length; i++) { Assert.AreEqual(af * xf[i], mxf[i]); } for (var i = 0; i < mxd.Length; i++) { Assert.AreEqual(ad * xd[i], mxd[i]); } }
internal override void Update(float[] weight, float[] delta) { var m = _m ?? new float[delta.Length]; var v = _v ?? new float[delta.Length]; var sq = new float[delta.Length]; Parallel.For(0, sq.Length, i => { sq[i] = delta[i] * delta[i]; }); _beta1 *= Beta1; _beta2 *= Beta2; Blas1.scal(m.Length, Beta1, m, 1); Blas1.scal(v.Length, Beta2, v, 1); Blas1.axpy(m.Length, 1.0f - Beta1, delta, 1, m, 1); Blas1.axpy(v.Length, 1.0f - Beta2, sq, 1, v, 1); Parallel.For(0, weight.Length, i => { var mHat = m[i] / (1.0f - _beta1); var vHat = v[i] / (1.0f - _beta2); weight[i] -= LearningRate / (MathF.Sqrt(vHat) + Eps) * mHat; }); _m = m; _v = v; }