public double Optimize(TrainableObj O) { double oldDelta = O.Cache1; O.Cache1 = -epochLr * O.Gradient + Momentum * O.Cache1; return(O.Cache1 + Momentum * (O.Cache1 - oldDelta)); }
public double Optimize(TrainableObj O) { O.Cache1 = Rho * O.Cache1 + (1.0 - Rho) * O.Gradient * O.Gradient; double delta = Sqrt(O.Cache2 + Epsilon) * O.Gradient / Sqrt(O.Cache1 + Epsilon); O.Cache2 = Rho * O.Cache2 + (1.0 - Rho) * delta * delta; return(-epochLr * delta); }
public double Optimize(TrainableObj O) { O.Cache1 = Beta1 * O.Cache1 + (1.0 - Beta1) * O.Gradient; O.Cache2 = Beta2 * O.Cache2 + (1.0 - Beta2) * O.Gradient * O.Gradient; double biasCorr1 = O.Cache1 / corrDiv1; double biasCorr2 = O.Cache2 / corrDiv2; return(-epochLr * biasCorr1 / Sqrt(biasCorr2 + Epsilon)); }
public double Optimize(TrainableObj O) { O.Cache1 = Rho * O.Cache1 + (1.0 - Rho) * O.Gradient * O.Gradient; return(-epochLr * O.Gradient / Sqrt(O.Cache1 + Epsilon)); }
public double Optimize(TrainableObj O) { O.Cache1 = -epochLr * O.Gradient + MomentumValue * O.Cache1; return(O.Cache1); }
public double Optimize(TrainableObj O) { return(-epochLr * O.Gradient); }