/// <inheritdoc />
internal override Learner ToLearner(Function modelOutput)
{
var learningRatePerSample = new TrainingParameterScheduleDouble(LearningRate, 1);
return(CNTKLib.AdaDeltaLearner(new ParameterVector(((CNTK.Function)modelOutput).Parameters().ToArray()),
learningRatePerSample, Rho, Epsilon, GetAdditionalLearningOptions()));
}
internal static Learner GetInitializer(IList <Parameter> parameters, NeuralNetworkSettingsEntity s)
{
var vector = new ParameterVector((ICollection)parameters);
switch (s.Learner)
{
case NeuralNetworkLearner.Adam: return(CNTKLib.AdamLearner(vector,
s.LearningRate.ToTrainParam(),
s.LearningMomentum?.ToTrainParam(),
s.LearningUnitGain ?? false,
s.LearningVarianceMomentum?.ToTrainParam()));
case NeuralNetworkLearner.AdaDelta:
return(CNTKLib.AdaDeltaLearner(vector,
s.LearningRate.ToTrainParam()));
case NeuralNetworkLearner.AdaGrad:
return(CNTKLib.AdaGradLearner(vector,
s.LearningRate.ToTrainParam()));
case NeuralNetworkLearner.FSAdaGrad:
return(CNTKLib.FSAdaGradLearner(vector,
s.LearningRate.ToTrainParam(),
s.LearningMomentum?.ToTrainParam(),
s.LearningUnitGain ?? false,
s.LearningVarianceMomentum?.ToTrainParam()));
case NeuralNetworkLearner.RMSProp:
return(CNTKLib.FSAdaGradLearner(vector,
s.LearningRate.ToTrainParam(),
s.LearningMomentum?.ToTrainParam(),
s.LearningUnitGain ?? false,
s.LearningVarianceMomentum?.ToTrainParam()));
case NeuralNetworkLearner.MomentumSGD:
return(CNTKLib.MomentumSGDLearner(vector,
s.LearningRate.ToTrainParam(),
s.LearningMomentum?.ToTrainParam(),
s.LearningUnitGain ?? false));
case NeuralNetworkLearner.SGD:
return(CNTKLib.SGDLearner(vector,
s.LearningRate.ToTrainParam()));
default:
throw new InvalidOperationException("Unexpected Learner");
}
}
public override Learner GetOptimizer(IList <Parameter> learningParameters)
{
var learningOptions = new AdditionalLearningOptions()
{
l1RegularizationWeight = _l1RegularizationWeight,
l2RegularizationWeight = _l2RegularizationWeight,
gradientClippingWithTruncation = _gradientClippingThresholdPerSample != double.PositiveInfinity,
gradientClippingThresholdPerSample = _gradientClippingThresholdPerSample
};
return(CNTKLib.AdaDeltaLearner(new ParameterVector((ICollection)learningParameters),
new TrainingParameterScheduleDouble(LearningRate, (uint)MinibatchSize),
_rho,
_epsilon,
learningOptions));
}
/// <summary>
/// Adadelta is an extension of Adagrad that seeks to reduce its aggressive, monotonically decreasing learning rate. Instead of accumulating all past squared gradients, Adadelta restricts the window of accumulated past gradients to some fixed size w.
/// </summary>
/// <param name="modelOutput">The model output.</param>
/// <param name="learningRate">The learning rate.</param>
/// <param name="rho">The rho.</param>
/// <param name="epsilon">The epsilon.</param>
/// <param name="regulizer">The regulizer.</param>
/// <returns>Learner.</returns>
private Learner AdaDelta(Function modelOutput, double learningRate = 1.0, double rho = 0.95, double epsilon = 1e-08, Regulizers regulizer = null)
{
CNTK.TrainingParameterScheduleDouble learningRatePerSample = new CNTK.TrainingParameterScheduleDouble(learningRate, 1);
return(CNTKLib.AdaDeltaLearner(new ParameterVector(modelOutput.Parameters().ToList()), learningRatePerSample, rho, epsilon, GetAdditionalLearningOptions()));
}
protected override Learner GenerateLearner(IList <Parameter> parameters)
{
var lr = new TrainingParameterScheduleDouble(LearningRate);
return(CNTKLib.AdaDeltaLearner(new ParameterVector(parameters.ToArray()), lr, Rho, Epsilon, Options));
}
