public double Optimize(TrainableObj O)
{
double oldDelta = O.Cache1;
O.Cache1 = -epochLr * O.Gradient + Momentum * O.Cache1;
return(O.Cache1 + Momentum * (O.Cache1 - oldDelta));
}
public double Optimize(TrainableObj O)
{
O.Cache1 = Rho * O.Cache1 + (1.0 - Rho) * O.Gradient * O.Gradient;
double delta = Sqrt(O.Cache2 + Epsilon) * O.Gradient / Sqrt(O.Cache1 + Epsilon);
O.Cache2 = Rho * O.Cache2 + (1.0 - Rho) * delta * delta;
return(-epochLr * delta);
}
public double Optimize(TrainableObj O)
{
O.Cache1 = Beta1 * O.Cache1 + (1.0 - Beta1) * O.Gradient;
O.Cache2 = Beta2 * O.Cache2 + (1.0 - Beta2) * O.Gradient * O.Gradient;
double biasCorr1 = O.Cache1 / corrDiv1;
double biasCorr2 = O.Cache2 / corrDiv2;
return(-epochLr * biasCorr1 / Sqrt(biasCorr2 + Epsilon));
}
public double Optimize(TrainableObj O)
{
O.Cache1 = Rho * O.Cache1 + (1.0 - Rho) * O.Gradient * O.Gradient;
return(-epochLr * O.Gradient / Sqrt(O.Cache1 + Epsilon));
}
public double Optimize(TrainableObj O)
{
O.Cache1 = -epochLr * O.Gradient + MomentumValue * O.Cache1;
return(O.Cache1);
}
public double Optimize(TrainableObj O)
{
return(-epochLr * O.Gradient);
}
