/// <summary>
/// Updates the specified iteration.
/// </summary>
/// <param name="iteration">The iteration.</param>
/// <param name="layer">The layer.</param>
internal override void Update(int iteration, BaseLayer layer)
{
if (DecayRate > 0)
{
LearningRate = LearningRate * (1 / (1 + DecayRate * iteration));
}
foreach (var p in layer.Params)
{
Parameter param = p.Value;
if (!moments.ContainsKey(param.Name))
{
moments[param.Name] = K.Constant(0, param.Data.Shape);
}
moments[param.Name] = (Momentum * moments[param.Name]) - (LearningRate * param.Grad);
if (Nesterov)
{
param.Data = param.Data + (Momentum * moments[param.Name]) - (LearningRate * param.Grad);
}
else
{
param.Data = param.Data + moments[param.Name];
}
param.ApplyConstraint();
}
}
