public void Training(string filePath)
{
ImageLoader(filePath);
var imageInput = CNTKLib.InputVariable(ImageDim, imageStreamInfo.m_elementType, "features");
var labelsVar = CNTKLib.InputVariable(new int[] { NumClasses }, labelStreamInfo.m_elementType, "labels");
var trainingLoss = CNTKLib.CrossEntropyWithSoftmax(new Variable(Classifier), labelsVar, "lossFunction");
var prediction = CNTKLib.ClassificationError(new Variable(Classifier), labelsVar, 5, "predictionError");
var learningRatePerSample = new TrainingParameterScheduleDouble(0.0078125, 1);
var trainer = Trainer.CreateTrainer(Classifier, trainingLoss, prediction,
new List <Learner> {
Learner.SGDLearner(Classifier.Parameters(), learningRatePerSample)
});
uint minibatchSize = 5;
// int outputFrequencyInMinibatches = 20, miniBatchCount = 0;
while (true)
{
var minibatchData = minibatchSource.GetNextMinibatch(minibatchSize, device);
if (minibatchData.empty())
{
break;
}
var arguments = new Dictionary <Variable, MinibatchData>()
{
{ input, minibatchData[imageStreamInfo] },
{ labelsVar, minibatchData[labelStreamInfo] }
};
// Stop training once max epochs is reached.
trainer.TrainMinibatch(arguments, device);
// TestHelper.PrintTrainingProgress(trainer, miniBatchCount++, outputFrequencyInMinibatches);
}
}
private Trainer MakeTrainer(Variable expectedOutput, Variable output, Function model, uint minibatchSize)
{
double learningRate = 0.02;
TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(learningRate);
TrainingParameterScheduleDouble momentumSchedule = new TrainingParameterScheduleDouble(0.9, minibatchSize);
Function lossFunction = CNTKLib.SquaredError(expectedOutput, output);
Function evalErrorFunction = CNTKLib.SquaredError(expectedOutput, output);
var parameters = new ParameterVector();
foreach (var p in model.Parameters())
{
parameters.Add(p);
}
List <Learner> parameterLearners = new List <Learner>()
{
CNTKLib.FSAdaGradLearner(parameters, learningRatePerSample, momentumSchedule, true)
};
Trainer trainer = Trainer.CreateTrainer(model, lossFunction, evalErrorFunction, parameterLearners);
return(trainer);
}
/// <inheritdoc />
internal override Learner ToLearner(Function model)
{
var learningRatePerSample = new TrainingParameterScheduleDouble(LearningRate, 1);
return(CNTKLib.AdaGradLearner(new ParameterVector(((CNTK.Function)model).Parameters().ToArray()),
learningRatePerSample, false, GetAdditionalLearningOptions()));
}
protected override Learner GenerateLearner(IList <Parameter> parameters, TrainingParameterScheduleDouble learningRateSchedule)
{
var m = new TrainingParameterScheduleDouble(Momentum);
var vm = new TrainingParameterScheduleDouble(VarianceMomentum);
return(CNTKLib.AdamLearner(new ParameterVector(parameters.ToArray()), learningRateSchedule, m, !NoUnitGrain, vm, Epsilon, Adamax, Options));
}
/// <inheritdoc />
internal override Learner ToLearner(Function modelOutput)
{
var learningRatePerSample = new TrainingParameterScheduleDouble(LearningRate, 1);
return(CNTKLib.AdaDeltaLearner(new ParameterVector(((CNTK.Function)modelOutput).Parameters().ToArray()),
learningRatePerSample, Rho, Epsilon, GetAdditionalLearningOptions()));
}
public void Learning(string traningFile)
{
mDigits = DigitImageCSVReader.Read(traningFile);
ResetTest();
TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(0.02, 1);
var parameterLearners =
new List <Learner>()
{
Learner.SGDLearner(mFunction.Parameters(), learningRatePerSample)
};
var trainer = Trainer.CreateTrainer(mFunction, loss, evalError, parameterLearners);
int minibatchSize = 64;
int numMinibatchesToTrain = 2000;
int updatePerMinibatches = 50;
for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++)
{
Value ft, lb;
GenerateValueData(minibatchSize, INPUT_DIMENSION, OUTPUT_DIMENSION, out ft, out lb, device);
trainer.TrainMinibatch(
new Dictionary <Variable, Value>()
{
{ mFeatures, ft }, { mLabels, lb }
}, device);
//TestHelper.PrintTrainingProgress(trainer, minibatchCount, updatePerMinibatches);
}
//int count = 0;
//for(int i = 0; i < 10000; i++)
// count += (GetValue(mDigits[i]) == mDigits[i].Label) ? 1 : 0;
}
public Trainer createTrainer(Function network, Variable target)
{
//learning rate
var lrate = 0.082;
var lr = new TrainingParameterScheduleDouble(lrate);
//network parameters
var zParams = new ParameterVector(network.Parameters().ToList());
//create loss and eval
Function loss = CNTKLib.SquaredError(network, target);
Function eval = CNTKLib.SquaredError(network, target);
//learners
//
var llr = new List <Learner>();
var msgd = Learner.SGDLearner(network.Parameters(), lr);
llr.Add(msgd);
//trainer
var trainer = Trainer.CreateTrainer(network, loss, eval, llr);
//
return(trainer);
}
public BitThoughtNeuralNet(int inDim, int outDim, int LSTMDim, int cellDim, string featuresName, string labelsName)
{
// build the model
var feature = Variable.InputVariable(new int[] { inDim }, DataType.Float, featuresName, null, false);
var label = Variable.InputVariable(new int[] { outDim }, DataType.Float, labelsName, new List <CNTK.Axis>()
{
CNTK.Axis.DefaultBatchAxis()
}, false);
var device = DeviceDescriptor.CPUDevice;
var lstmModel = BitThoughtFactory.CreateModel(feature, outDim, LSTMDim, cellDim, device, "timeSeriesOutput");
Function trainingLoss = CNTKLib.SquaredError(lstmModel, label, "squarederrorLoss");
Function prediction = CNTKLib.SquaredError(lstmModel, label, "squarederrorEval");
// prepare for training
TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(0.0005, 1);
TrainingParameterScheduleDouble momentumTimeConstant = CNTKLib.MomentumAsTimeConstantSchedule(256);
IList <Learner> parameterLearners = new List <Learner>()
{
Learner.MomentumSGDLearner(lstmModel.Parameters(), learningRatePerSample, momentumTimeConstant, /*unitGainMomentum = */ true)
};
//create trainer
Trainer = Trainer.CreateTrainer(lstmModel, trainingLoss, prediction, parameterLearners);
}
/// <summary>
/// Adam (Adaptive Moment Estimation) is another method that computes adaptive learning rates for each parameter.
/// In addition to storing an exponentially decaying average of past squared gradients vtvt like Adadelta,
/// Adam also keeps an exponentially decaying average of past gradients, similar to momentum.
/// Essentially Adam is RMSProp with momentum.
/// https://arxiv.org/pdf/1412.6980.pdf.
/// </summary>
/// <param name="parameters">Learnable parameters of the model</param>
/// <param name="learningRate">Learning rate. (Default is 0.001)</param>
/// <param name="momentum">Momentum (default is 0.9).
/// Note that this is the beta1 parameter in the Adam paper.</param>
/// <param name="varianceMomentum">variance momentum schedule. (Default is 0.999).
/// Note that this is the beta2 parameter in the Adam paper.</param>
/// <param name="l1Regularization">L1 regularization term. (Default is 0, so no regularization)</param>
/// <param name="l2Regularization">L2 regularization term. (Default is 0, so no regularization)</param>
/// <param name="gradientClippingThresholdPerSample">clipping threshold per sample, defaults to infinity. (Default is infinity, so no clipping)</param>
/// <param name="gradientClippingWithTruncation">clipping threshold per sample, defaults to infinity. (Default is true)</param>
/// <param name="unitGain"></param>
/// <param name="epsilon"></param>
/// <returns></returns>
public static Learner Adam(IList <Parameter> parameters, double learningRate = 0.001,
double momentum = 0.9, double varianceMomentum = 0.999,
double l1Regularization = 0.0,
double l2Regularization = 0.0,
double gradientClippingThresholdPerSample = double.PositiveInfinity,
bool gradientClippingWithTruncation = true,
bool unitGain = true,
double epsilon = 1e-08f)
{
var learningRatePerSample = new TrainingParameterScheduleDouble(learningRate, 1);
var momentumRate = new TrainingParameterScheduleDouble(momentum, 1);
var varianceMomentumRate = new TrainingParameterScheduleDouble(varianceMomentum, 1);
var options = SetAdditionalOptions(l1Regularization,
l2Regularization,
gradientClippingThresholdPerSample,
gradientClippingWithTruncation);
return(CNTKLib.AdamLearner(CntkUtilities.CreateParameterVector(parameters),
learningRatePerSample,
momentumRate,
unitGain,
varianceMomentumRate,
epsilon,
false,
options));
}
private Trainer MakeTrainer(Variable expectedOutput, Variable output, Function model, uint minibatchSize)
{
double learningRate = 0.01;
TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(learningRate);
TrainingParameterScheduleDouble momentumSchedule = new TrainingParameterScheduleDouble(0.9983550962823424, minibatchSize);
//Function lossFunction = CrossEntropyWithSoftmax(output, expectedOutput);
Function lossFunction = CNTKLib.CrossEntropyWithSoftmax(output, expectedOutput);
Function evalErrorFunction = CNTKLib.ClassificationError(output, expectedOutput);
var parameters = new ParameterVector();
foreach (var p in model.Parameters())
{
parameters.Add(p);
}
List <Learner> parameterLearners = new List <Learner>()
{
CNTKLib.MomentumSGDLearner(parameters, learningRatePerSample, momentumSchedule, true, new AdditionalLearningOptions()
{
l2RegularizationWeight = 0.001
})
};
Trainer trainer = Trainer.CreateTrainer(model, lossFunction, evalErrorFunction, parameterLearners);
return(trainer);
}
/// <summary>
/// create model by w,h,c,outputClassNum
/// </summary>
/// <param name="w"></param>
/// <param name="h"></param>
/// <param name="c"></param>
/// <param name="outputClassNum"></param>
/// <param name="deviceName"></param>
public FCN7(int w, int h, int c, int outputClassNum, string deviceName)
{
device = NP.CNTKHelper.GetDeviceByName(deviceName);
//input output variable
int[] inputDim = new int[] { w, h, c };
int[] outputDim = new int[] { outputClassNum };
inputVariable = Variable.InputVariable(NDShape.CreateNDShape(inputDim), DataType.Float, "inputVariable");
outputVariable = Variable.InputVariable(NDShape.CreateNDShape(outputDim), DataType.Float, "labelVariable");
//build model
classifierOutput = CreateFullyChannelNetwork(inputVariable, c, outputClassNum);
Function loss = CNTKLib.SquaredError(classifierOutput, outputVariable);
Function pred = CNTKLib.ClassificationError(classifierOutput, outputVariable);
//adam leaner
ParameterVector parameterVector = new ParameterVector(classifierOutput.Parameters().ToList());
TrainingParameterScheduleDouble learningRateSchedule = new TrainingParameterScheduleDouble(0.00178125, BatchSize);
TrainingParameterScheduleDouble momentumRateSchedule = new TrainingParameterScheduleDouble(0.9, BatchSize);
Learner leaner = CNTKLib.AdamLearner(parameterVector, learningRateSchedule, momentumRateSchedule, true);
//构造leaner方法
trainer = Trainer.CreateTrainer(classifierOutput, loss, pred, new List <Learner>()
{
leaner
});
//TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(0.00178125, BatchSize); //0.00178125
//TrainingParameterScheduleDouble momentumTimeConstant = CNTKLib.MomentumAsTimeConstantSchedule(256);
//IList<Learner> parameterLearners = new List<Learner>() { Learner.MomentumSGDLearner(classifierOutput.Parameters(), learningRatePerSample, momentumTimeConstant, true) };
//trainer = Trainer.CreateTrainer(classifierOutput, loss, pred, parameterLearners);
}
public Trainer createTrainer(Function network, Variable target, double lrvalue)
{
//learning rate
// var lrate = 0.05;
var momentum = 0.9;
var lr = new TrainingParameterScheduleDouble(lrvalue);
var mm = CNTKLib.MomentumAsTimeConstantSchedule(momentum);
var l = new AdditionalLearningOptions()
{
l1RegularizationWeight = 0.001, l2RegularizationWeight = 0.1
};
//network parameters
var zParams = new ParameterVector(network.Parameters().ToList());
//create loss and eval
Function loss = CNTKLib.SquaredError(network, target);
Function eval = StatMetrics.RMSError(network, target);
//learners
//
var llr = new List <Learner>();
var msgd = Learner.SGDLearner(network.Parameters(), lr, l);
llr.Add(msgd);
//trainer
var trainer = Trainer.CreateTrainer(network, loss, eval, llr);
//
return(trainer);
}
/// <inheritdoc />
internal override Learner ToLearner(Function model)
{
var learningRatePerSample = new TrainingParameterScheduleDouble(LearningRate, 1);
return(CNTKLib.RMSPropLearner(new ParameterVector(((CNTK.Function)model).Parameters().ToArray()),
learningRatePerSample, Gamma, Increment, Decrement, Max, Min, false, GetAdditionalLearningOptions()));
}
public TrainingParameterScheduleDouble(TrainingParameterScheduleDouble arg0) : this(CNTKLibPINVOKE.new_TrainingParameterScheduleDouble__SWIG_5(TrainingParameterScheduleDouble.getCPtr(arg0)), true)
{
if (CNTKLibPINVOKE.SWIGPendingException.Pending)
{
throw CNTKLibPINVOKE.SWIGPendingException.Retrieve();
}
}
private Function PerformTraining(TrainingConfig config)
{
// Setup the criteria (loss and metric)
var crossEntropy = CNTKLib.CrossEntropyWithSoftmax(model, inputModel.LabelSequence);
var errors = CNTKLib.ClassificationError(model, inputModel.LabelSequence);
// Instantiate the trainer object to drive the model training
var learningRatePerSample = new TrainingParameterScheduleDouble(config.LearningRate, 1);
var momentumTimeConstant = CNTKLib.MomentumAsTimeConstantSchedule(config.MomentumTimeConstant);
var additionalParameters = new AdditionalLearningOptions
{
gradientClippingThresholdPerSample = 5.0,
gradientClippingWithTruncation = true
};
var learner = Learner.MomentumSGDLearner(model.Parameters(), learningRatePerSample, momentumTimeConstant, true, additionalParameters);
trainer = Trainer.CreateTrainer(model, crossEntropy, errors, new List <Learner>()
{
learner
});
for (int i = 0; i < config.Epochs; i++)
{
TrainMinibatch(config);
}
return(model);
}
public void Train()
{
sw.Start();
//Instantiate the input and the label variables
input = Variable.InputVariable(new int[] { inputDim }, DataType.Float);
label = Variable.InputVariable(new int[] { numOutputClasses }, DataType.Float);
var featureStreamInfo = trainingData.StreamInfo("features");
var labelStreamInfo = trainingData.StreamInfo("labels");
//Create the model function
model = CreateModel(input);
var loss = CNTKLib.BinaryCrossEntropy(model, label);
var evalError = CNTKLib.ClassificationError(model, label);
//Training config
uint epoch_size = 0;
uint minibatch_size = 64;
int num_sweeps_to_train_with = fullSweeps;
int num_samples_per_sweep = 950;
int num_minibatches_to_train = (num_samples_per_sweep * num_sweeps_to_train_with);
int updatePerMinibatches = 950;
//Instantiate the learner object to drive the model training
TrainingParameterScheduleDouble lr_per_sample = new TrainingParameterScheduleDouble(0.00003f, epoch_size);
TrainingParameterScheduleDouble momentum_as_time_constant = new TrainingParameterScheduleDouble(1000f, minibatch_size);
// IList<Learner> learner = new List<Learner>() { CNTKLib.AdamLearner(Helper.AsParameterVector(model.Parameters()),lr_per_sample, momentum_as_time_constant)}; //Bug? Model outputs only NaNs
IList <Learner> learner = new List <Learner>()
{
Learner.SGDLearner(model.Parameters(), lr_per_sample)
};
//Instantiate the trainer
trainer = Trainer.CreateTrainer(model, loss, evalError, learner);
//Start training
double aggregate_metric = 0;
for (int minibatchCount = 0; minibatchCount < num_minibatches_to_train; minibatchCount++)
{
var minibatchData = trainingData.GetNextMinibatch(minibatch_size, device);
var arguments = new Dictionary <Variable, MinibatchData>
{
{ input, minibatchData[featureStreamInfo] },
{ label, minibatchData[labelStreamInfo] }
};
trainer.TrainMinibatch(arguments, device);
TestHelper.PrintTrainingProgress(trainer, minibatchCount, updatePerMinibatches);
var samples = trainer.PreviousMinibatchSampleCount();
aggregate_metric += trainer.PreviousMinibatchEvaluationAverage() * samples;
}
sw.Stop();
var train_error = aggregate_metric / (trainer.TotalNumberOfSamplesSeen());
UnityEngine.Debug.Log(String.Format("Average training error: {0:P2} , in {1} hour {2} minutes and {3} seconds using {4}", train_error, sw.Elapsed.Hours, sw.Elapsed.Minutes, sw.Elapsed.Seconds, device.Type.ToString()));
}
/// <summary>
/// Stochastic Gradient Descent.
/// </summary>
/// <param name="parameters">Learnable parameters of the model</param>
/// <param name="learningRate">Learning rate. (Default is 0.001)</param>
/// <param name="l1Regularization">L1 regularization term. (Default is 0, so no regularization)</param>
/// <param name="l2Regularization">L2 regularization term. (Default is 0, so no regularization)</param>
/// <param name="gradientClippingThresholdPerSample">clipping threshold per sample, defaults to infinity. (Default is infinity, so no clipping)</param>
/// <param name="gradientClippingWithTruncation">clipping threshold per sample, defaults to infinity. (Default is true)</param>
/// <returns></returns>
public static Learner SGD(IList <Parameter> parameters,
double learningRate = 0.01,
double l1Regularization = 0.0,
double l2Regularization = 0.0,
double gradientClippingThresholdPerSample = double.PositiveInfinity,
bool gradientClippingWithTruncation = true)
{
if (parameters == null)
{
throw new ArgumentNullException("parameters");
}
if (learningRate < 0.0)
{
throw new ArgumentException(nameof(learningRate) + " Has to be larger or equal to 0");
}
if (l1Regularization < 0.0)
{
throw new ArgumentException(nameof(l1Regularization) + " Has to be larger or equal to 0");
}
if (l2Regularization < 0.0)
{
throw new ArgumentException(nameof(l2Regularization) + " Has to be larger or equal to 0");
}
var learningRatePerSample = new TrainingParameterScheduleDouble(learningRate, 1);
var options = SetAdditionalOptions(l1Regularization,
l2Regularization,
gradientClippingThresholdPerSample,
gradientClippingWithTruncation);
return(CNTKLib.SGDLearner(CntkUtilities.CreateParameterVector(parameters),
learningRatePerSample, options));
}
/// <summary>
///
/// </summary>
/// <param name="parameters"></param>
/// <param name="learningRate">A learning rate in float, or a learning rate schedule</param>
/// <param name="gamma">Trade-off factor for current and previous gradients. Common value is 0.95. Should be in range (0.0, 1.0)</param>
/// <param name="inc">Increasing factor when trying to adjust current learning_rate. Should be greater than 1</param>
/// <param name="dec">Decreasing factor when trying to adjust current learning_rate. Should be in range (0.0, 1.0)</param>
/// <param name="max">Maximum scale allowed for the initial learning_rate. Should be greater than zero and min</param>
/// <param name="min">Minimum scale allowed for the initial learning_rate. Should be greater than zero</param>
/// <param name="l1Regularization">L1 regularization term. (Default is 0, so no regularization)</param>
/// <param name="l2Regularization">L2 regularization term. (Default is 0, so no regularization)</param>
/// <param name="gradientClippingThresholdPerSample">clipping threshold per sample, defaults to infinity. (Default is infinity, so no clipping)</param>
/// <param name="gradientClippingWithTruncation">clipping threshold per sample, defaults to infinity. (Default is true)</param>
/// <param name="needAveMultiplier"></param>
/// <returns></returns>
public static Learner RMSProp(IList <Parameter> parameters,
double learningRate = 0.001,
double gamma = 0.9,
double inc = 2.0,
double dec = 0.5,
double max = 2.0,
double min = 0.5,
double l1Regularization = 0.0,
double l2Regularization = 0.0,
double gradientClippingThresholdPerSample = double.PositiveInfinity,
bool gradientClippingWithTruncation = true,
bool needAveMultiplier = true)
{
var learningRatePerSample = new TrainingParameterScheduleDouble(learningRate, 1);
var options = SetAdditionalOptions(l1Regularization,
l2Regularization,
gradientClippingThresholdPerSample,
gradientClippingWithTruncation);
return(CNTKLib.RMSPropLearner(CntkUtilities.CreateParameterVector(parameters),
learningRatePerSample,
gamma, inc, dec, max, min,
needAveMultiplier, options));
}
/// <inheritdoc />
internal override Learner ToLearner(Function model)
{
var learningRatePerSample = new TrainingParameterScheduleDouble(LearningRate, 1);
var momentumPerSample = new TrainingParameterScheduleDouble(Momentum, 1);
return(CNTKLib.MomentumSGDLearner(new ParameterVector(((CNTK.Function)model).Parameters().ToArray()),
learningRatePerSample, momentumPerSample, UnitGain, GetAdditionalLearningOptions()));
}
public void SetLearningRateSchedule(TrainingParameterScheduleDouble learningRateSchedule)
{
CNTKLibPINVOKE.Learner_SetLearningRateSchedule(swigCPtr, TrainingParameterScheduleDouble.getCPtr(learningRateSchedule));
if (CNTKLibPINVOKE.SWIGPendingException.Pending)
{
throw CNTKLibPINVOKE.SWIGPendingException.Retrieve();
}
}
private static List <Learner> CreateLerners(IList <Parameter> parameters)
{
var learningRatePerSample = new TrainingParameterScheduleDouble(0.003125, 1);
var learner = Learner.SGDLearner(parameters, learningRatePerSample);
return(new List <Learner> {
learner
});
}
/// <summary>
/// Train and evaluate an image classifier with CIFAR-10 data.
/// The classification model is saved after training.
/// For repeated runs, the caller may choose whether to retrain a model or
/// just validate an existing one.
/// </summary>
/// <param name="device">CPU or GPU device to run</param>
/// <param name="forceRetrain">whether to override an existing model.
/// if true, any existing model will be overridden and the new one evaluated.
/// if false and there is an existing model, the existing model is evaluated.</param>
public static void TrainAndEvaluate(DeviceDescriptor device, bool forceRetrain)
{
string modelFile = "../../../../Data/CIFAR-10/Cifar10Rest.model";
// If a model already exists and not set to force retrain, validate the model and return.
if (File.Exists(modelFile) && !forceRetrain)
{
ValidateModel(device, modelFile);
return;
}
// prepare training data
var minibatchSource = CreateMinibatchSource(Path.Combine("D:/Libraries/cntk-release/Examples/Image/DataSets/CIFAR-10", "train_map.txt"),
Path.Combine(CifarDataFolder, "CIFAR-10_mean.xml"), imageDim, numClasses, MaxEpochs);
var imageStreamInfo = minibatchSource.StreamInfo("features");
var labelStreamInfo = minibatchSource.StreamInfo("labels");
// build a model
var imageInput = CNTKLib.InputVariable(imageDim, imageStreamInfo.m_elementType, "Images");
var labelsVar = CNTKLib.InputVariable(new int[] { numClasses }, labelStreamInfo.m_elementType, "Labels");
var classifierOutput = ResNetClassifier(imageInput, numClasses, device, "classifierOutput");
// prepare for training
var trainingLoss = CNTKLib.CrossEntropyWithSoftmax(classifierOutput, labelsVar, "lossFunction");
var prediction = CNTKLib.ClassificationError(classifierOutput, labelsVar, 5, "predictionError");
var learningRatePerSample = new TrainingParameterScheduleDouble(0.0078125, 1);
var trainer = Trainer.CreateTrainer(classifierOutput, trainingLoss, prediction,
new List<Learner> { Learner.SGDLearner(classifierOutput.Parameters(), learningRatePerSample) });
uint minibatchSize = 64;
int outputFrequencyInMinibatches = 20, miniBatchCount = 0;
// Feed data to the trainer for number of epochs.
while (true)
{
var minibatchData = minibatchSource.GetNextMinibatch(minibatchSize, device);
// Stop training once max epochs is reached.
if (minibatchData.empty())
{
break;
}
trainer.TrainMinibatch(new Dictionary<Variable, MinibatchData>()
{ { imageInput, minibatchData[imageStreamInfo] }, { labelsVar, minibatchData[labelStreamInfo] } }, device);
TestHelper.PrintTrainingProgress(trainer, miniBatchCount++, outputFrequencyInMinibatches);
}
// save the model
var imageClassifier = Function.Combine(new List<Variable>() { trainingLoss, prediction, classifierOutput }, "ImageClassifier");
imageClassifier.Save(modelFile);
// validate the model
ValidateModel(device, modelFile);
}
public bool AreEqual(TrainingParameterScheduleDouble right)
{
bool ret = CNTKLibPINVOKE.TrainingParameterScheduleDouble_AreEqual(swigCPtr, TrainingParameterScheduleDouble.getCPtr(right));
if (CNTKLibPINVOKE.SWIGPendingException.Pending)
{
throw CNTKLibPINVOKE.SWIGPendingException.Retrieve();
}
return(ret);
}
public static TrainingParameterScheduleDouble Deserialize(CNTKDictionary dictionary)
{
TrainingParameterScheduleDouble ret = new TrainingParameterScheduleDouble(CNTKLibPINVOKE.TrainingParameterScheduleDouble_Deserialize(CNTKDictionary.getCPtr(dictionary)), true);
if (CNTKLibPINVOKE.SWIGPendingException.Pending)
{
throw CNTKLibPINVOKE.SWIGPendingException.Retrieve();
}
return(ret);
}
public TrainingParameterScheduleDouble GetLearningRateSchedule()
{
TrainingParameterScheduleDouble ret = new TrainingParameterScheduleDouble(CNTKLibPINVOKE.Learner_GetLearningRateSchedule(swigCPtr), false);
if (CNTKLibPINVOKE.SWIGPendingException.Pending)
{
throw CNTKLibPINVOKE.SWIGPendingException.Retrieve();
}
return(ret);
}
static void TestApp()
{
var sampler = new ParallelSampler(10000, 1000, true, 1000);
var runners = new BackgroundScriptRunner[2];
try
{
for (var i = 0; i < runners.Length; ++i)
{
var runner = new BackgroundScriptRunner();
var script = ScriptBlock.Create(backgroundScript);
runner.Start(script, new object[] { sampler, MINIBATCH_SIZE });
runners[i] = runner;
}
var model = GetModel();
var output = model.Item1;
var label = model.Item2;
var loss = CNTKLib.BinaryCrossEntropy(output, label);
var metric = CNTKLib.ClassificationError(output, label);
var lr = new TrainingParameterScheduleDouble(.05);
var learner = CNTKLib.SGDLearner(new ParameterVector(output.Parameters().ToArray()), lr);
var session = new TrainingSession(output, loss, metric, learner, null, sampler, null);
var progress = session.GetIterator().GetEnumerator();
for (var i = 0; i < 10000; ++i)
{
progress.MoveNext();
var p = progress.Current;
if (p.Iteration == 1 || p.Iteration % 100 == 0)
{
Console.WriteLine(string.Format("Iteration: {0} Loss: {1} Metric: {2} Validation: {3} Elapsed: {4} CountInQueue: {5}",
p.Iteration, p.Loss, p.Metric, p.GetValidationMetric(), p.Elapsed, sampler.CountInQueue));
}
}
}
finally
{
sampler.CancelAdding();
foreach (var runner in runners)
{
runner.Finish();
runner.Dispose();
}
}
}
public void TestTrainingSession2()
{
// Data
var features = DataSourceFactory.Create(new float[] { 0, 0, 0, 1, 1, 0, 1, 1, 3, 4, 3, 5, 4, 4, 4, 5 }, new int[] { 2, 1, -1 });
var labels = DataSourceFactory.Create(new float[] { 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0 }, new int[] { 2, 1, -1 });
var sampler = new DataSourceSampler(new Dictionary <string, IDataSource <float> >()
{
{ "input", features },
{ "label", labels }
}, 2);
// Model
var input = CNTKLib.InputVariable(new int[] { 2 }, false, DataType.Float, "input");
var h = Composite.Dense(input, new int[] { 100 }, CNTKLib.HeNormalInitializer(), true, null, false, 4, "relu", DeviceDescriptor.UseDefaultDevice(), "");
h = Composite.Dense(h, new int[] { 2 }, CNTKLib.GlorotNormalInitializer(), true, null, false, 4, "sigmoid", DeviceDescriptor.UseDefaultDevice(), "");
var output = h;
var label = CNTKLib.InputVariable(new int[] { 2 }, DataType.Float, "label");
// Loss and metric functions
var loss = CNTKLib.BinaryCrossEntropy(output, label);
var error = CNTKLib.ClassificationError(output, label);
// Train
var lr = new TrainingParameterScheduleDouble(.01);
var m = new TrainingParameterScheduleDouble(.9);
var learner = Learner.MomentumSGDLearner(output.Parameters(), lr, m, true);
var session = new TrainingSession(output, loss, error, learner, null, sampler, null);
var iteration = session.GetEnumerator();
for (var i = 0; i < 1000; ++i)
{
iteration.MoveNext();
var dummy = iteration.Current;
var valid = session.GetValidationMetric();
}
Assert.IsTrue(session.Metric < 0.1);
}
public VPGActor(int stateSize, int actionSize, int layerSize)
{
for (int i = 0; i < actionSize; ++i)
{
m_normDist.Add(new EDA.Gaussian());
}
//network
CreateNetwork(stateSize, actionSize, layerSize);
var learningRate = new TrainingParameterScheduleDouble(0.01);
var learner = new List <Learner>()
{
Learner.SGDLearner(m_model.Parameters(), learningRate)
};
m_trainer = Trainer.CreateTrainer(m_model, m_loss, null, learner);
}
public VPGCritic(int stateSize, int actionSize, int layerSize)
{
//network
CreateNetwork(stateSize, actionSize, layerSize);
m_targetVariable = CNTKLib.InputVariable(new int[] { actionSize }, DataType.Float);
//loss
m_loss = CNTKLib.Square(CNTKLib.Minus(m_model.Output, m_targetVariable));
var learningRate = new TrainingParameterScheduleDouble(0.01);
var learner = new List <Learner>()
{
Learner.SGDLearner(m_model.Parameters(), learningRate)
};
m_trainer = Trainer.CreateTrainer(m_model, m_loss, null, learner);
}
public void Accept(ISupportNet sourceNet)
{
//convert to bytes
byte[] bytes = sourceNet.PersistenceMemory();
//read model and set parameters
classifierOutput = Function.Load(bytes, device);
inputVariable = classifierOutput.Inputs.First(v => v.Name == "inputVariable");
outputVariable = Variable.InputVariable(classifierOutput.Output.Shape, DataType.Float, "labelVariable");
var trainingLoss = CNTKLib.SquaredError(classifierOutput, outputVariable);
var prediction = CNTKLib.SquaredError(classifierOutput, outputVariable);
TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(0.00178125, 1); //0.00178125
TrainingParameterScheduleDouble momentumTimeConstant = CNTKLib.MomentumAsTimeConstantSchedule(256);
IList <Learner> parameterLearners = new List <Learner>()
{
Learner.MomentumSGDLearner(classifierOutput.Parameters(), learningRatePerSample, momentumTimeConstant, true)
};
trainer = Trainer.CreateTrainer(classifierOutput, trainingLoss, prediction, parameterLearners);
}
