/// <summary>
/// Add a param group to the Optimizer s param_groups.
/// </summary>
/// <param name="param_group"></param>
/// <remarks>This can be useful when fine tuning a pre-trained network as frozen layers can be made trainable and added to the Optimizer as training progresses.</remarks>
public override void add_param_group(Modules.ParamGroup param_group)
{
var def = _defaults as Options;
if (param_group.Options is null)
{
param_group.Options = new Options();
}
var opt = param_group.Options as Options;
// Make sure all the options are set.
if (!opt.LearningRate.HasValue)
{
opt.LearningRate = def.LearningRate;
}
if (!opt.beta1.HasValue)
{
opt.beta1 = def.beta1;
}
if (!opt.beta2.HasValue)
{
opt.beta2 = def.beta2;
}
if (!opt.eps.HasValue)
{
opt.eps = def.eps;
}
if (!opt.weight_decay.HasValue)
{
opt.weight_decay = def.weight_decay;
}
if (!opt.amsgrad.HasValue)
{
opt.amsgrad = def.amsgrad;
}
if (!opt.maximize.HasValue)
{
opt.maximize = def.maximize;
}
opt.InitialLearningRate = opt.LearningRate.Value;
_parameter_groups.Add(param_group);
foreach (var p in param_group.Parameters)
{
var state = new State();
_state[p.Handle] = state;
state.step = 0;
state.exp_avg = torch.zeros_like(p);
state.exp_avg_sq = torch.zeros_like(p);
if (opt.amsgrad.Value)
{
state.max_exp_avg_sq = torch.zeros_like(p);
}
}
}
/// <summary>
/// Add a param group to the Optimizer s param_groups.
/// </summary>
/// <param name="param_group"></param>
/// <remarks>This can be useful when fine tuning a pre-trained network as frozen layers can be made trainable and added to the Optimizer as training progresses.</remarks>
public override void add_param_group(Modules.ParamGroup param_group)
{
var def = _defaults as Options;
if (param_group.Options is null)
{
param_group.Options = new Options();
}
var opt = param_group.Options as Options;
// Make sure all the options are set.
if (!opt.LearningRate.HasValue)
{
opt.LearningRate = def.LearningRate;
}
if (!opt.momentum.HasValue)
{
opt.momentum = def.momentum;
}
if (!opt.eps.HasValue)
{
opt.eps = def.eps;
}
if (!opt.alpha.HasValue)
{
opt.alpha = def.alpha;
}
if (!opt.weight_decay.HasValue)
{
opt.weight_decay = def.weight_decay;
}
if (!opt.centered.HasValue)
{
opt.centered = def.centered;
}
opt.InitialLearningRate = opt.LearningRate.Value;
_parameter_groups.Add(param_group);
foreach (var p in param_group.Parameters)
{
var state = new State();
_state[p.Handle] = state;
state.square_avg = torch.zeros_like(p);
state.grad_avg = torch.zeros_like(p);
state.momentum_buffer = torch.zeros_like(p);
}
}
/// <summary>
/// Add a param group to the Optimizer s param_groups.
/// </summary>
/// <param name="param_group"></param>
/// <remarks>This can be useful when fine tuning a pre-trained network as frozen layers can be made trainable and added to the Optimizer as training progresses.</remarks>
public override void add_param_group(Modules.ParamGroup param_group)
{
var def = _defaults as Options;
if (param_group.Options is null)
{
param_group.Options = new Options();
}
var opt = param_group.Options as Options;
// Make sure all the options are set.
if (!opt.LearningRate.HasValue)
{
opt.LearningRate = def.LearningRate;
}
if (!opt.momentum.HasValue)
{
opt.momentum = def.momentum;
}
if (!opt.dampening.HasValue)
{
opt.dampening = def.dampening;
}
if (!opt.weight_decay.HasValue)
{
opt.weight_decay = def.weight_decay;
}
if (!opt.nesterov.HasValue)
{
opt.nesterov = def.nesterov;
}
if (!opt.maximize.HasValue)
{
opt.maximize = def.maximize;
}
opt.InitialLearningRate = opt.LearningRate.Value;
_parameter_groups.Add(param_group);
foreach (var p in param_group.Parameters)
{
var state = new State();
_state[p.Handle] = state;
state.momentum_buffer = null;
}
}
/// <summary>
/// Add a param group to the Optimizer s param_groups.
/// </summary>
/// <param name="param_group"></param>
/// <remarks>This can be useful when fine tuning a pre-trained network as frozen layers can be made trainable and added to the Optimizer as training progresses.</remarks>
public override void add_param_group(Modules.ParamGroup param_group)
{
var def = _defaults as Options;
if (param_group.Options is null)
{
param_group.Options = new Options();
}
var opt = param_group.Options as Options;
// Make sure all the options are set.
if (!opt.LearningRate.HasValue)
{
opt.LearningRate = def.LearningRate;
}
if (!opt.rho.HasValue)
{
opt.rho = def.rho;
}
if (!opt.eps.HasValue)
{
opt.eps = def.eps;
}
if (!opt.weight_decay.HasValue)
{
opt.weight_decay = def.weight_decay;
}
opt.InitialLearningRate = opt.LearningRate.Value;
_parameter_groups.Add(param_group);
foreach (var p in param_group.Parameters)
{
var state = new State();
_state[p.Handle] = state;
state.step = 0;
state.square_avg = torch.zeros_like(p);
state.acc_delta = torch.zeros_like(p);
}
}
