public override void UpdateFunctionParameters() { Real alphaT = this._optimizer.AlphaT; _optimizer.UpdateBound(); Parallel.For(0, FunctionParameter.Data.Length, i => { Real grad = this.FunctionParameter.Grad[i]; this.m[i] += (1 - this._optimizer.Beta1) * (grad - this.m[i]); this.v[i] += (1 - this._optimizer.Beta2) * (grad * grad - this.v[i]); Real step = _optimizer.Clip(alphaT / (Math.Sqrt(this.v[i]) + this._optimizer.Epsilon)); this.FunctionParameter.Data[i] -= this._optimizer.Eta * step * this.m[i]; }); }
public static void UpdateFunctionParameters(Real alpha, Real initialAlpha, Real gamma, Real beta1, Real beta2, Real epsilon, Real eta, long updateCount, NdArray <Real> functionParameter, Real[] m, Real[] v, ref Real finalLr, out Real lower, out Real upper, Func <Real, Real> clip) { Real alphaT = AdamParameter.GetAlphaT(alpha, beta1, beta2, updateCount); AdaBound.UpdateBound(alpha, initialAlpha, gamma, updateCount, ref finalLr, out lower, out upper); for (int i = 0; i < functionParameter.Data.Length; i++) { Real grad = functionParameter.Grad[i]; m[i] += (1 - beta1) * (grad - m[i]); v[i] += (1 - beta2) * (grad * grad - v[i]); Real step = clip(alphaT / (Math.Sqrt(v[i]) + epsilon)); functionParameter.Data[i] -= eta * step * m[i]; } }