/// <summary>
/// The Vt factor in the Adam update rule scales the gradient inversely proportionally to the ℓ2 norm of the past gradients (via the vt−1 term) and current gradient.
/// </summary>
/// <param name="modelOutput">The model output.</param>
/// <param name="learningRate">The learning rate.</param>
/// <param name="momentum">The momentum.</param>
/// <param name="varianceMomentum">The variance momentum.</param>
/// <param name="unitGain">if set to <c>true</c> [unit gain].</param>
/// <param name="epsilon">The epsilon.</param>
/// <param name="regulizer">The regulizer.</param>
/// <returns>Learner.</returns>
private Learner Adamax(Function modelOutput, double learningRate = 0.002, double momentum = 0.9, double varianceMomentum = 0.999, bool unitGain = true, double epsilon = 1e-08, Regulizers regulizer = null)
{
CNTK.TrainingParameterScheduleDouble learningRatePerSample = new CNTK.TrainingParameterScheduleDouble(learningRate, 1);
CNTK.TrainingParameterScheduleDouble momentumRate = new CNTK.TrainingParameterScheduleDouble(momentum, 1);
CNTK.TrainingParameterScheduleDouble varianceMomentumRate = new CNTK.TrainingParameterScheduleDouble(varianceMomentum, 1);
return(CNTKLib.AdamLearner(new ParameterVector(modelOutput.Parameters().ToList()), learningRatePerSample, momentumRate, unitGain, varianceMomentumRate, epsilon, true, GetAdditionalLearningOptions()));
}
/// <summary>
/// Momentum of Stochastic gradient descent optimizer.
/// </summary>
/// <param name="modelOutput">The model output.</param>
/// <param name="learningRate">The learning rate.</param>
/// <param name="momentum">The momentum.</param>
/// <param name="unitGain">if set to <c>true</c> [unit gain].</param>
/// <param name="regulizer">The regulizer.</param>
/// <returns>Learner.</returns>
private Learner MomentumSGD(Function modelOutput, double learningRate = 0.01, double momentum = 0.1, bool unitGain = true, Regulizers regulizer = null)
{
CNTK.TrainingParameterScheduleDouble learningRatePerSample = new CNTK.TrainingParameterScheduleDouble(learningRate, 1);
CNTK.TrainingParameterScheduleDouble momentumPerSample = new CNTK.TrainingParameterScheduleDouble(momentum, 1);
return(CNTKLib.MomentumSGDLearner(new ParameterVector(modelOutput.Parameters().ToList()), learningRatePerSample, momentumPerSample, unitGain, GetAdditionalLearningOptions()));
}
/// <summary>
/// create a momentum SGD learner
/// </summary>
/// <param name="parameters">parameters to learn</param>
/// <param name="learningRateSchedule">learning rate schedule</param>
/// <param name="momentumSchedule">momentum schedule</param>
/// <param name="unitGain">unit gain</param>
/// <param name="additionalOptions">other additional options</param>
/// <returns></returns>
public static Learner MomentumSGDLearner(IList <Parameter> parameters, TrainingParameterScheduleDouble learningRateSchedule,
TrainingParameterScheduleDouble momentumSchedule, bool unitGain, AdditionalLearningOptions additionalOptions = null)
{
if (additionalOptions == null)
{
additionalOptions = new AdditionalLearningOptions();
}
ParameterVector parameterVector = Helper.AsParameterVector(parameters);
return(CNTKLib.MomentumSGDLearner(parameterVector, learningRateSchedule, momentumSchedule, unitGain, additionalOptions));
}
private static Trainer PrepareTraining(Function input, Variable labelData, Function loss, double learningRate, uint minibatchSize)
{
var evalError = CNTKLib.ClassificationError(input, labelData); // エラー関数（分類精度？）
CNTK.TrainingParameterScheduleDouble learningRatePerSample = new CNTK.TrainingParameterScheduleDouble(learningRate, minibatchSize);
IList <Learner> parameterLearners =
new List <Learner>()
{
Learner.SGDLearner(input.Parameters(), learningRatePerSample)
};
return(Trainer.CreateTrainer(input, loss, evalError, parameterLearners));
}
/// <summary>
/// Create an instance of the CNTK built-in SGD learner.
/// </summary>
/// <param name="parameters">Parameters of the learner.</param>
/// <param name="learningRateSchedule">Learning rate schedule.</param>
/// <param name="additionalOptions">Additional options.</param>
/// <returns></returns>
public static Learner SGDLearner(IList <Parameter> parameters, TrainingParameterScheduleDouble learningRateSchedule, AdditionalLearningOptions additionalOptions = null)
{
ParameterVector parameterVector = Helper.AsParameterVector(parameters);
if (additionalOptions == null)
{
return(CNTKLib.SGDLearner(parameterVector, learningRateSchedule));
}
else
{
return(CNTKLib.SGDLearner(parameterVector, learningRateSchedule, additionalOptions));
}
}
static Trainer CreateModelTrainer(Function model, Variable inputs, Variable labels)
{
var trainingLoss = CNTKLib.BinaryCrossEntropy(new Variable(model), labels, "lossFunction");
var prediction = CNTKLib.ReduceMean(CNTKLib.Equal(labels, CNTKLib.Round(new Variable(model))), Axis.AllAxes()); // Keras accuracy metric
// set per sample learning rate
CNTK.TrainingParameterScheduleDouble learningRatePerSample = new CNTK.TrainingParameterScheduleDouble(0.1, 1);
IList <Learner> parameterLearners = new List <Learner>()
{
Learner.SGDLearner(model.Parameters(), learningRatePerSample)
};
var trainer = Trainer.CreateTrainer(model, trainingLoss, prediction, parameterLearners);
return(trainer);
}
public CNNAgent()
{
int inputLayer = 3; // 1 layer for WALLs - Coins - Player
int inputSize = World.SIZE;
int outputSize = Enum.GetValues(typeof(PlayAction)).Length;
model = CNTKHelper.CNTKHelper.CreateCNNModel(device, inputSize, inputLayer, outputSize);
inputVariable = model.Arguments.First(a => a.IsInput);
inputDataMap = new Dictionary <Variable, Value>()
{
{ inputVariable, null }
};
outputDataMap = new Dictionary <Variable, Value>()
{
{ model.Output, null }
};
inputTrainBatch = new Dictionary <Variable, Value>()
{
{ inputVariable, null }
};
outputTrainBatch = new Dictionary <Variable, Value>()
{
{ model.Output, null }
};
// set per sample learning rate
CNTK.TrainingParameterScheduleDouble learningRatePerSample = new CNTK.TrainingParameterScheduleDouble(0.003125, 1);
actionVariable = CNTKLib.InputVariable(new int[] { World.PLAY_ACTION_COUNT }, DataType.Float, "Actions");
var trainingLoss = CNTKLib.CrossEntropyWithSoftmax(new Variable(model), actionVariable, "lossFunction");
var prediction = CNTKLib.ClassificationError(new Variable(model), actionVariable, "classificationError");
IList <Learner> parameterLearners = new List <Learner>()
{
Learner.SGDLearner(model.Parameters(), learningRatePerSample)
};
trainer = Trainer.CreateTrainer(model, trainingLoss, prediction, parameterLearners);
}
const int SIZE_IN_NB_LAYER = 2; // 1 layer for Coins - Player
public Agent2D()
{
int inputSize = SIZE_IN;
int outputSize = SIZE_OUT;
model = CNTKHelper.CNTKHelper.CreateMLPModel1D(device, inputSize, SIZE_IN_NB_LAYER, outputSize);
inputVariable = model.Arguments.First(a => a.IsInput);
inputDataMap = new Dictionary <Variable, Value>()
{
{ inputVariable, null }
};
outputDataMap = new Dictionary <Variable, Value>()
{
{ model.Output, null }
};
inputTrainBatch = new Dictionary <Variable, Value>()
{
{ inputVariable, null }
};
outputTrainBatch = new Dictionary <Variable, Value>()
{
{ model.Output, null }
};
// Set per sample learning rate
CNTK.TrainingParameterScheduleDouble learningRatePerSample = new CNTK.TrainingParameterScheduleDouble(0.001, 1);
actionVariable = CNTKLib.InputVariable(new int[] { SIZE_OUT }, DataType.Float, "Actions");
var trainingLoss = CNTKLib.CrossEntropyWithSoftmax(new Variable(model), actionVariable, "lossFunction");
var prediction = CNTKLib.ClassificationError(new Variable(model), actionVariable, "classificationError");
IList <Learner> parameterLearners = new List <Learner>()
{
Learner.SGDLearner(model.Parameters(), learningRatePerSample)
};
trainer = Trainer.CreateTrainer(model, trainingLoss, prediction, parameterLearners);
}
/// <summary>
/// Adagrad is an algorithm for gradient-based optimization that does just this: It adapts the learning rate to the parameters, performing larger updates for infrequent and smaller updates for frequent parameters
/// </summary>
/// <param name="modelOutput">The model output.</param>
/// <param name="learningRate">The learning rate.</param>
/// <param name="regulizer">The regulizer.</param>
/// <returns>Learner.</returns>
private Learner AdaGrad(Function modelOutput, double learningRate = 0.01, Regulizers regulizer = null)
{
CNTK.TrainingParameterScheduleDouble learningRatePerSample = new CNTK.TrainingParameterScheduleDouble(learningRate, 1);
return(CNTKLib.AdaGradLearner(new ParameterVector(modelOutput.Parameters().ToList()), learningRatePerSample, false, GetAdditionalLearningOptions()));
}
/// <summary>
/// Adadelta is an extension of Adagrad that seeks to reduce its aggressive, monotonically decreasing learning rate. Instead of accumulating all past squared gradients, Adadelta restricts the window of accumulated past gradients to some fixed size w.
/// </summary>
/// <param name="modelOutput">The model output.</param>
/// <param name="learningRate">The learning rate.</param>
/// <param name="rho">The rho.</param>
/// <param name="epsilon">The epsilon.</param>
/// <param name="regulizer">The regulizer.</param>
/// <returns>Learner.</returns>
private Learner AdaDelta(Function modelOutput, double learningRate = 1.0, double rho = 0.95, double epsilon = 1e-08, Regulizers regulizer = null)
{
CNTK.TrainingParameterScheduleDouble learningRatePerSample = new CNTK.TrainingParameterScheduleDouble(learningRate, 1);
return(CNTKLib.AdaDeltaLearner(new ParameterVector(modelOutput.Parameters().ToList()), learningRatePerSample, rho, epsilon, GetAdditionalLearningOptions()));
}
static public void TrainAndEvaluate(DeviceDescriptor device)
{
// build a logistic regression model
Variable featureVariable = Variable.InputVariable(new int[] { inputDim }, DataType.Float);
Variable labelVariable = Variable.InputVariable(new int[] { numOutputClasses }, DataType.Float);
var classifierOutput = CreateLinearModel(featureVariable, numOutputClasses, device);
var loss = CNTKLib.CrossEntropyWithSoftmax(classifierOutput, labelVariable);
var evalError = CNTKLib.ClassificationError(classifierOutput, labelVariable);
// prepare for training
CNTK.TrainingParameterScheduleDouble learningRatePerSample = new CNTK.TrainingParameterScheduleDouble(0.02, 1);
IList <Learner> parameterLearners =
new List <Learner>()
{
Learner.SGDLearner(classifierOutput.Parameters(), learningRatePerSample)
};
var trainer = Trainer.CreateTrainer(classifierOutput, loss, evalError, parameterLearners);
int minibatchSize = 64;
int numMinibatchesToTrain = 1000;
int updatePerMinibatches = 50;
// train the model
for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++)
{
Value features, labels;
GenerateValueData(minibatchSize, inputDim, numOutputClasses, out features, out labels, device);
//TODO: sweepEnd should be set properly instead of false.
#pragma warning disable 618
trainer.TrainMinibatch(
new Dictionary <Variable, Value>()
{
{ featureVariable, features }, { labelVariable, labels }
}, device);
#pragma warning restore 618
TestHelper.PrintTrainingProgress(trainer, minibatchCount, updatePerMinibatches);
}
// test and validate the model
int testSize = 100;
Value testFeatureValue, expectedLabelValue;
GenerateValueData(testSize, inputDim, numOutputClasses, out testFeatureValue, out expectedLabelValue, device);
// GetDenseData just needs the variable's shape
IList <IList <float> > expectedOneHot = expectedLabelValue.GetDenseData <float>(labelVariable);
IList <int> expectedLabels = expectedOneHot.Select(l => l.IndexOf(1.0F)).ToList();
var inputDataMap = new Dictionary <Variable, Value>()
{
{ featureVariable, testFeatureValue }
};
var outputDataMap = new Dictionary <Variable, Value>()
{
{ classifierOutput.Output, null }
};
classifierOutput.Evaluate(inputDataMap, outputDataMap, device);
var outputValue = outputDataMap[classifierOutput.Output];
IList <IList <float> > actualLabelSoftMax = outputValue.GetDenseData <float>(classifierOutput.Output);
var actualLabels = actualLabelSoftMax.Select((IList <float> l) => l.IndexOf(l.Max())).ToList();
int misMatches = actualLabels.Zip(expectedLabels, (a, b) => a.Equals(b) ? 0 : 1).Sum();
Console.WriteLine($"Validating Model: Total Samples = {testSize}, Misclassify Count = {misMatches}");
}
internal static void TrainSimpleFeedForwardClassifier(DeviceDescriptor device)
{
int inputDim = 2;
int numOutputClasses = 2;
int hiddenLayerDim = 50;
int numHiddenLayers = 2;
int minibatchSize = 50;
int numSamplesPerSweep = 10000;
int numSweepsToTrainWith = 2;
int numMinibatchesToTrain = (numSamplesPerSweep * numSweepsToTrainWith) / minibatchSize;
var featureStreamName = "features";
var labelsStreamName = "labels";
var input = Variable.InputVariable(new int[] { inputDim }, DataType.Float, "features");
var labels = Variable.InputVariable(new int[] { numOutputClasses }, DataType.Float, "labels");
Function classifierOutput;
Function trainingLoss;
Function prediction;
IList <StreamConfiguration> streamConfigurations = new StreamConfiguration[]
{ new StreamConfiguration(featureStreamName, inputDim), new StreamConfiguration(labelsStreamName, numOutputClasses) };
using (var minibatchSource = MinibatchSource.TextFormatMinibatchSource(
Path.Combine(DataFolder, "SimpleDataTrain_cntk_text.txt"),
streamConfigurations, MinibatchSource.FullDataSweep, true, MinibatchSource.DefaultRandomizationWindowInChunks))
{
var featureStreamInfo = minibatchSource.StreamInfo(featureStreamName);
var labelStreamInfo = minibatchSource.StreamInfo(labelsStreamName);
IDictionary <StreamInformation, Tuple <NDArrayView, NDArrayView> > inputMeansAndInvStdDevs =
new Dictionary <StreamInformation, Tuple <NDArrayView, NDArrayView> >
{
{ featureStreamInfo, new Tuple <NDArrayView, NDArrayView>(null, null) }
};
MinibatchSource.ComputeInputPerDimMeansAndInvStdDevs(minibatchSource, inputMeansAndInvStdDevs, device);
var normalizedinput = CNTKLib.PerDimMeanVarianceNormalize(input,
inputMeansAndInvStdDevs[featureStreamInfo].Item1, inputMeansAndInvStdDevs[featureStreamInfo].Item2);
Function fullyConnected = TestHelper.FullyConnectedLinearLayer(normalizedinput, hiddenLayerDim, device, "");
classifierOutput = CNTKLib.Sigmoid(fullyConnected, "");
for (int i = 1; i < numHiddenLayers; ++i)
{
fullyConnected = TestHelper.FullyConnectedLinearLayer(classifierOutput, hiddenLayerDim, device, "");
classifierOutput = CNTKLib.Sigmoid(fullyConnected, "");
}
var outputTimesParam = new Parameter(NDArrayView.RandomUniform <float>(
new int[] { numOutputClasses, hiddenLayerDim }, -0.05, 0.05, 1, device));
var outputBiasParam = new Parameter(NDArrayView.RandomUniform <float>(
new int[] { numOutputClasses }, -0.05, 0.05, 1, device));
classifierOutput = CNTKLib.Plus(outputBiasParam, outputTimesParam * classifierOutput, "classifierOutput");
trainingLoss = CNTKLib.CrossEntropyWithSoftmax(classifierOutput, labels, "lossFunction");;
prediction = CNTKLib.ClassificationError(classifierOutput, labels, "classificationError");
// Test save and reload of model
{
Variable classifierOutputVar = classifierOutput;
Variable trainingLossVar = trainingLoss;
Variable predictionVar = prediction;
var combinedNet = Function.Combine(new List <Variable>()
{
trainingLoss, prediction, classifierOutput
},
"feedForwardClassifier");
TestHelper.SaveAndReloadModel(ref combinedNet,
new List <Variable>()
{
input, labels, trainingLossVar, predictionVar, classifierOutputVar
}, device);
classifierOutput = classifierOutputVar;
trainingLoss = trainingLossVar;
prediction = predictionVar;
}
}
CNTK.TrainingParameterScheduleDouble learningRatePerSample = new CNTK.TrainingParameterScheduleDouble(
0.02, TrainingParameterScheduleDouble.UnitType.Sample);
using (var minibatchSource = MinibatchSource.TextFormatMinibatchSource(
Path.Combine(DataFolder, "SimpleDataTrain_cntk_text.txt"), streamConfigurations))
{
var featureStreamInfo = minibatchSource.StreamInfo(featureStreamName);
var labelStreamInfo = minibatchSource.StreamInfo(labelsStreamName);
streamConfigurations = new StreamConfiguration[]
{ new StreamConfiguration("features", inputDim), new StreamConfiguration("labels", numOutputClasses) };
IList <Learner> parameterLearners =
new List <Learner>()
{
Learner.SGDLearner(classifierOutput.Parameters(), learningRatePerSample)
};
var trainer = Trainer.CreateTrainer(classifierOutput, trainingLoss, prediction, parameterLearners);
int outputFrequencyInMinibatches = 20;
int trainingCheckpointFrequency = 100;
for (int i = 0; i < numMinibatchesToTrain; ++i)
{
var minibatchData = minibatchSource.GetNextMinibatch((uint)minibatchSize, device);
var arguments = new Dictionary <Variable, MinibatchData>
{
{ input, minibatchData[featureStreamInfo] },
{ labels, minibatchData[labelStreamInfo] }
};
trainer.TrainMinibatch(arguments, device);
TestHelper.PrintTrainingProgress(trainer, i, outputFrequencyInMinibatches);
if ((i % trainingCheckpointFrequency) == (trainingCheckpointFrequency - 1))
{
string ckpName = "feedForward.net";
trainer.SaveCheckpoint(ckpName);
trainer.RestoreFromCheckpoint(ckpName);
}
}
double trainLossValue = trainer.PreviousMinibatchLossAverage();
double evaluationValue = trainer.PreviousMinibatchEvaluationAverage();
if (trainLossValue > 0.3 || evaluationValue > 0.2)
{
throw new Exception($"TrainSimpleFeedForwardClassifier resulted in unusual high training loss (= {trainLossValue}) or error rate (= {evaluationValue})");
}
}
}
public static void Run_MNIST_Test()
{
//
var device = DeviceDescriptor.UseDefaultDevice();
//dims
var inDim = 784;
var outDim = 10;
// MNIST images are 28x28=784 pixels
var input = CNTKLib.InputVariable(new NDShape(1, inDim), DataType.Float, "features");
var labels = CNTKLib.InputVariable(new NDShape(1, outDim), DataType.Float, "labels");
//create network
var nnModel = createModel(input, outDim, 1, device);
//Loss and Eval functions
var trainingLoss = CNTKLib.CrossEntropyWithSoftmax(nnModel, labels, "lossFunction");
var prediction = CNTKLib.ClassificationError(nnModel, labels, "classificationError");
//create learners and trainer
// set per sample learning rate and momentum
var learningRatePerSample = new CNTK.TrainingParameterScheduleDouble(0.001, 1);
var momentumPerSample = new CNTK.TrainingParameterScheduleDouble(0.9, 1);
var nnParams = nnModel.Parameters();
var parameterLearners = new List <Learner>()
{
CNTKLib.AdamLearner(new ParameterVector(nnModel.Parameters().ToList()), learningRatePerSample, momentumPerSample)
};
var trainer = Trainer.CreateTrainer(nnModel, trainingLoss, prediction, parameterLearners);
//create minibatch source
var sConfigs = new StreamConfiguration[]
{ new StreamConfiguration("features", inDim), new StreamConfiguration("labels", outDim) };
//this file is huge and cannot be uploaded on GitHUb.
//it can be downloaded from: https://github.com/Microsoft/CNTK/tree/987b22a8350211cb4c44278951857af1289c3666/Examples/Image/DataSets/MNIST
var minibatchSource = MinibatchSource.TextFormatMinibatchSource("..\\..\\..\\Data\\MNIST-TrainData.txt", sConfigs, MinibatchSource.InfinitelyRepeat);
var minibatchSize = (uint)754;
var featureStreamInfo = minibatchSource.StreamInfo("features");
var labelStreamInfo = minibatchSource.StreamInfo("labels");
var maxIt = 250;
var curIt = 1;
while (true)
{
var minibatchData = minibatchSource.GetNextMinibatch(minibatchSize, device);
var arguments = new Dictionary <Variable, MinibatchData>
{
{ input, minibatchData[featureStreamInfo] },
{ labels, minibatchData[labelStreamInfo] }
};
trainer.TrainMinibatch(arguments, device);
//
if (minibatchData[featureStreamInfo].sweepEnd)
{
if (curIt % 50 == 0 || curIt == 1)
{
printProgress(trainer, curIt);
}
curIt++;
}
if (maxIt <= curIt)
{
break;
}
}
// save the trained model
nnModel.Save("mnist_classifier");
// validate the model
var minibatchSourceNewModel = MinibatchSource.TextFormatMinibatchSource("../../../data/MNIST-TestData.txt", sConfigs, MinibatchSource.InfinitelyRepeat);
//prepare vars to accept results
List <List <float> > X = new List <List <float> >();
List <float> Y = new List <float>();
//Model validation
ValidateModel("mnist_classifier", minibatchSourceNewModel, new int[] { 28, 28 }, 10, "features", "labels", device, 1000, X, Y, false);
//show image classification result
showResult(X, Y);
}
public void TrainAndEvaluate(DeviceDescriptor device, bool forceRetrain)
{
var featureStreamName = "features";
var labelsStreamName = "labels";
var classifierName = "classifierOutput";
Function classifierOutput;
int[] inputSize = new int[] { Layer[0] };
int imageSize = Layer[0];
int numClasses = Layer[Layer.Length - 1];
IList <StreamConfiguration> streamConfigurations = new StreamConfiguration[]
{ new StreamConfiguration(featureStreamName, imageSize), new StreamConfiguration(labelsStreamName, numClasses) };
// If a model already exists and not set to force retrain, validate the model and return.
if (File.Exists(modelFile) && !forceRetrain)
{
var minibatchSourceExistModel = MinibatchSource.TextFormatMinibatchSource(
Path.Combine(DataPath, "Test_cntk_text.txt"), streamConfigurations);
TestHelper.ValidateModelWithMinibatchSource(modelFile, minibatchSourceExistModel,
inputSize, numClasses, featureStreamName, labelsStreamName, classifierName, device);
return;
}
// build the network
var input = CNTKLib.InputVariable(inputSize, DataType.Float, featureStreamName);
// For MLP, we like to have the middle layer to have certain amount of states.
int hiddenLayerDim = 200;
var scaledInput = CNTKLib.ElementTimes(Constant.Scalar <float>(0.00390625f, device), input);
classifierOutput = CreateMLPClassifier(device, numClasses, hiddenLayerDim, scaledInput, classifierName);
var labels = CNTKLib.InputVariable(new int[] { numClasses }, DataType.Float, labelsStreamName);
var trainingLoss = CNTKLib.CrossEntropyWithSoftmax(new Variable(classifierOutput), labels, "lossFunction");
var prediction = CNTKLib.ClassificationError(new Variable(classifierOutput), labels, "classificationError");
// prepare training data
var minibatchSource = MinibatchSource.TextFormatMinibatchSource(
Path.Combine(DataPath, "Train_cntk_text.txt"), streamConfigurations, MinibatchSource.InfinitelyRepeat);
var featureStreamInfo = minibatchSource.StreamInfo(featureStreamName);
var labelStreamInfo = minibatchSource.StreamInfo(labelsStreamName);
// set per sample learning rate
CNTK.TrainingParameterScheduleDouble learningRatePerSample = new CNTK.TrainingParameterScheduleDouble(
0.003125, 1);
IList <Learner> parameterLearners = new List <Learner>()
{
Learner.SGDLearner(classifierOutput.Parameters(), learningRatePerSample)
};
var trainer = Trainer.CreateTrainer(classifierOutput, trainingLoss, prediction, parameterLearners);
const uint minibatchSize = 64;
int outputFrequencyInMinibatches = 20, i = 0;
int epochs = 5;
while (epochs > 0)
{
var minibatchData = minibatchSource.GetNextMinibatch(minibatchSize, device);
var arguments = new Dictionary <Variable, MinibatchData>
{
{ input, minibatchData[featureStreamInfo] },
{ labels, minibatchData[labelStreamInfo] }
};
trainer.TrainMinibatch(arguments, device);
TestHelper.PrintTrainingProgress(trainer, i++, outputFrequencyInMinibatches);
/*
* MinibatchSource is created with MinibatchSource.InfinitelyRepeat.
* Batching will not end. Each time minibatchSource completes an sweep (epoch),
* the last minibatch data will be marked as end of a sweep. We use this flag
* to count number of epochs.
*/
if (TestHelper.MiniBatchDataIsSweepEnd(minibatchData.Values))
{
epochs--;
}
}
// Save the trained model
classifierOutput.Save(modelFile);
// Validate the model
var minibatchSourceNewModel = MinibatchSource.TextFormatMinibatchSource(
Path.Combine(DataPath, "Test_cntk_text.txt"), streamConfigurations, MinibatchSource.FullDataSweep);
TestHelper.ValidateModelWithMinibatchSource(modelFile, minibatchSourceNewModel,
inputSize, numClasses, featureStreamName, labelsStreamName, classifierName, device);
}
public void Main()
{
// Create minibatch source
var transforms = new List <CNTKDictionary> {
CNTKLib.ReaderCrop("RandomSide",
new Tuple <int, int>(0, 0),
new Tuple <float, float>(0.8f, 1.0f),
new Tuple <float, float>(0.0f, 0.0f),
new Tuple <float, float>(1.0f, 1.0f),
"uniRatio"),
CNTKLib.ReaderScale(imageDim[0], imageDim[1], imageDim[2])
};
var conf = CNTKLib.ImageDeserializer(@"D:\Microsoft\Presentations\FY18 AI\DotNext17\Data\Train.txt",
"labels", (uint)numClasses,
"features", transforms);
MinibatchSourceConfig config = new MinibatchSourceConfig(new List <CNTKDictionary> {
conf
});
var minibatchSource = CNTKLib.CreateCompositeMinibatchSource(config);
var imageStreamInfo = minibatchSource.StreamInfo("features");
var labelStreamInfo = minibatchSource.StreamInfo("labels");
// build a model
var imageInput = CNTKLib.InputVariable(imageDim, imageStreamInfo.m_elementType, "Images");
var labelsVar = CNTKLib.InputVariable(new int[] { numClasses }, labelStreamInfo.m_elementType, "Labels");
var C1 = ConvolutionLayer(imageInput, 5, 5, 3, 32);
var C2 = ConvolutionLayer(C1, 5, 5, 32, 1);
var C3 = ConvolutionLayer(C2, 5, 5, 64, 2);
var C4 = DenseLayer(C3, 64, true);
var C5 = DenseLayer(C4, 64, true);
var z = DenseLayer(C5, numClasses, true);
var loss = CNTKLib.CrossEntropyWithSoftmax(z, labelsVar);
var evalError = CNTKLib.ClassificationError(z, labelsVar);
// prepare for training
CNTK.TrainingParameterScheduleDouble learningRatePerSample = new CNTK.TrainingParameterScheduleDouble(0.02, 1);
IList <Learner> parameterLearners =
new List <Learner>()
{
Learner.SGDLearner(z.Parameters(), learningRatePerSample)
};
var trainer = Trainer.CreateTrainer(z, loss, evalError, parameterLearners);
uint minibatchSize = 64;
// train the model
for (int ep = 0; ep < 1000; ep++)
{
var data = minibatchSource.GetNextMinibatch(minibatchSize);
trainer.TrainMinibatch(new Dictionary <Variable, MinibatchData>()
{
{ imageInput, data[imageStreamInfo] },
{ labelsVar, data[labelStreamInfo] }
}, device);
if (ep % 50 == 0)
{
var _loss = trainer.PreviousMinibatchLossAverage();
var _eval = trainer.PreviousMinibatchEvaluationAverage();
WriteLine($"Epoch={ep}, loss={_loss}, eval={_eval}");
}
}
}
static void Main(string[] args)
{
WriteLine("Execution begins");
var fn = @"c:\DEMO\Data\train.csv";
var f = File.ReadLines(fn);
var data = from t in f.Skip(1)
let zz = t.Split(',').Select(float.Parse)
select new Digit
{
Label = (int)zz.First(),
Image = zz.Skip(1).Select(x => x / 256f).ToArray()
};
var train = data.Take(40000).ToArray();
var test = data.Skip(40000).Take(1000).ToArray();
WriteLine("Creating network");
DeviceDescriptor device = DeviceDescriptor.CPUDevice;
int inputDim = 784;
int outputDim = 10;
var inputShape = new NDShape(1, inputDim);
var outputShape = new NDShape(1, outputDim);
Variable features = Variable.InputVariable(inputShape, DataType.Float);
Variable label = Variable.InputVariable(outputShape, DataType.Float);
var W = new Parameter(new [] { outputDim, inputDim }, DataType.Float, CNTKLib.GlorotUniformInitializer(), device, "w");
var b = new Parameter(new [] { outputDim }, DataType.Float, 0, device, "b");
var z = CNTKLib.Times(W, features) + b;
var loss = CNTKLib.CrossEntropyWithSoftmax(z, label);
var evalError = CNTKLib.ClassificationError(z, label);
CNTK.TrainingParameterScheduleDouble learningRatePerSample = new CNTK.TrainingParameterScheduleDouble(0.02, 1);
IList <Learner> parameterLearners =
new List <Learner>()
{
Learner.SGDLearner(z.Parameters(), learningRatePerSample)
};
var trainer = Trainer.CreateTrainer(z, loss, evalError, parameterLearners);
int minibatchSize = 64;
int numMinibatchesToTrain = 500;
var feat = new BatchSource <float[]>((from x in train select x.Image).ToArray(), minibatchSize);
var labl = new BatchSource <float[]>((from x in train select x.Label.ToOneHot10(10).ToFloatArray()).ToArray(), minibatchSize);
var gr = new List <float>();
// train the model
for (int ep = 0; ep < numMinibatchesToTrain; ep++)
{
Value ft, lb;
feat.MoveNext(); labl.MoveNext();
ft = Value.CreateBatchOfSequences <float>(inputShape, feat.Current, device);
lb = Value.CreateBatchOfSequences <float>(outputShape, labl.Current, device);
trainer.TrainMinibatch(
new Dictionary <Variable, Value>()
{
{ features, ft }, { label, lb }
}, device);
if (ep % 50 == 0)
{
var _loss = trainer.PreviousMinibatchLossAverage();
var _eval = trainer.PreviousMinibatchEvaluationAverage();
WriteLine($"Epoch={ep}, loss={_loss}, eval={_eval}");
gr.Add((float)_loss);
}
}
var G = new GraphLib();
G.Plot(gr, System.Windows.Forms.DataVisualization.Charting.SeriesChartType.Line);
int count = 0, correct = 0;
// Test the model
foreach (var x in test)
{
var imap = new Dictionary <Variable, Value> {
{ features, Value.CreateBatch(inputShape, x.Image, device) }
};
var omap = new Dictionary <Variable, Value> {
{ z, null }
};
z.Evaluate(imap, omap, device);
var o = omap[z].GetDenseData <float>(z).First();
var res = o.MaxIndex();
WriteLine("{0} => {1}", x.Label, res);
if (x.Label == res)
{
correct++;
}
count++;
}
WriteLine("Done, {0} of {1} correct ({2}%)", correct, count, (double)correct / (double)count * 100);
Console.ReadKey();
}
/// <summary>
/// Test a simple model which takes a one hot encoded digit as an input and returns the same as an output
/// </summary>
private void TrainAndEvaluateTest(Function model, Value inputValue)
{
#region Evaluate model before training
var inputDataMap = new Dictionary <Variable, Value>()
{
{ model.Arguments[0], inputValue }
};
var outputDataMap = new Dictionary <Variable, Value>()
{
{ model.Output, null }
};
model.Evaluate(inputDataMap, outputDataMap, DeviceDescriptor.CPUDevice);
IList <IList <float> > preTrainingOutput = outputDataMap[model.Output].GetDenseData <float>(model.Output);
for (int i = 0; i < TEST1_SIZE; i++)
{
Trace.WriteLine($"Argmax({i}): {CNTKHelper.ArgMax(preTrainingOutput[i].ToArray())}");
}
#endregion
#region Train Model
var labels = CNTKLib.InputVariable(new int[] { TEST1_SIZE }, DataType.Float, "Error Input");
var trainingLoss = CNTKLib.CrossEntropyWithSoftmax(new Variable(model), labels, "lossFunction");
var prediction = CNTKLib.ClassificationError(new Variable(model), labels, "classificationError");
// Set per sample learning rate
CNTK.TrainingParameterScheduleDouble learningRatePerSample = new CNTK.TrainingParameterScheduleDouble(0.003125, 1);
IList <Learner> parameterLearners = new List <Learner>()
{
Learner.SGDLearner(model.Parameters(), learningRatePerSample)
};
var trainer = Trainer.CreateTrainer(model, trainingLoss, prediction, parameterLearners);
// Create expected output
var expectedOutputValue = Value.CreateBatch <float>(new int[] { TEST1_SIZE }, ExpectedOutput(TEST1_SIZE), DeviceDescriptor.CPUDevice);
var inputMiniBatch = new MinibatchData(inputValue, TEST1_SIZE);
var outputMiniBatch = new MinibatchData(expectedOutputValue, TEST1_SIZE);
var arguments = new Dictionary <Variable, MinibatchData>
{
{ model.Arguments[0], inputMiniBatch },
{ labels, outputMiniBatch }
};
int epochs = 5;
while (epochs > 0)
{
trainer.TrainMinibatch(arguments, device);
epochs--;
}
#endregion
#region Evaluate Model after training
outputDataMap = new Dictionary <Variable, Value>()
{
{ model.Output, null }
};
model.Evaluate(inputDataMap, outputDataMap, DeviceDescriptor.CPUDevice);
IList <IList <float> > postTrainingOutput = outputDataMap[model.Output].GetDenseData <float>(model.Output);
int nbFail = 0;
for (int i = 0; i < TEST1_SIZE; i++)
{
int prepTrainValue = CNTKHelper.ArgMax(preTrainingOutput[i].ToArray());
int postTrainValue = CNTKHelper.ArgMax(postTrainingOutput[i].ToArray());
if (i != postTrainValue)
{
nbFail++;
}
Trace.WriteLine($"Argmax({i}): {prepTrainValue} ==> {postTrainValue}");
}
Trace.WriteLine($"Failure rate = ({nbFail}/{TEST1_SIZE})");
#endregion
}
private void btnLogisticRegression_Click(object sender, EventArgs e)
{
// Chart領域の初期化
Utilities.InitChart(chtTraining);
/////////////////////////////////////////////////////////////////////////////
// 学習用画像
int dstWidth = 28;
int dstHeight = 28;
int srcChannel;
// 学習用データの取得
int numOfCategories;
string[] categories;
// 学習用データセット List<Tuple<string, int, float[]>> <ファイル名、カテゴリIndex 0, 1, 2...、画像データ配列 CHW順の一次元>
var trainingDataMap = Data.PrepareTrainingDataFromSubfolders(
@"D:\NeuralNetworkConsole\neural_network_console_140\samples\sample_dataset\MNIST\test", // 画像データが格納されたフォルダ
dstWidth, // リサイズ後の幅
dstHeight, // リサイズ後の高さ
out srcChannel, // 画像のチャンネル数
out categories // 指定フォルダ内にカテゴリ別に分けられたフォルダ名の配列
);
// クラス数
numOfCategories = categories.Length;
/////////////////////////////////////////////////////////////////////////////
// ニューラルネットワークモデルの構築
// 入力サイズ28x28
var inputData = Layers.Input(dstWidth, dstHeight, srcChannel, DataType.Float);
// ネットワークの構築
var model =
inputData
//.Scale(0.003125f) // 1/256
//.BatchNormalization(0.5f, 0f) // ±２σが±１になるぐらいを想定
.Convolution(
3, 3, // kernelSize 幅、高さ
4, // kernel数
1, 1 // stride 横、縦
)
.ReLU()
.MaxPooling(
3, 3, // Poolingのサイズ 幅、高さ
2, 2, // stride 横、縦
true // paddingするか？
)
.Convolution(
3, 3, // kernelSize 幅、高さ
8, // kernel数
1, 1 // stride 横、縦
)
.ReLU()
.MaxPooling(
3, 3, // Poolingのサイズ 幅、高さ
2, 2, // stride 横、縦
true // paddingするか？
)
.Dense(numOfCategories)
;
/////////////////////////////////////////////////////////////////////////////
// ラベルデータの確保(出力と同じサイズ)
var labelData = model.LabelData();
// 損失関数
Function trainingLoss = CNTKLib.CrossEntropyWithSoftmax(model, labelData); // Softmax → CrossEntropy
//Function trainingLoss = CNTKLib.BinaryCrossEntropy(model, labelData); // 出力が１つの場合
// 分類誤差
Function predictionError = CNTKLib.ClassificationError(model, labelData);
/////////////////////////////////////////////////////////////////////////////
// 学習
int numMinibatches = 10;
// 学習率
CNTK.TrainingParameterScheduleDouble learningRatePerSample = new CNTK.TrainingParameterScheduleDouble(0.0002, 1); //(0.003125, 1);//(0.02, 1);
// 確率的勾配降下法（Stochastic Gradient Descent）の適応
var list = model.Parameters();
IList <Learner> parameterLearners =
new List <Learner>()
{
Learner.SGDLearner(model.Parameters(), learningRatePerSample)
};
// 学習の構築
var trainer = Trainer.CreateTrainer(model, trainingLoss, predictionError, parameterLearners);
int updatePerMinibatches = numMinibatches / 10;
if (updatePerMinibatches == 0)
{
updatePerMinibatches = 1;
}
// Chart領域の初期化
Utilities.InitChart(chtTraining);
// 学習
for (int minibatchCount = 0; minibatchCount < numMinibatches; ++minibatchCount)
{
Value imageBatch, labelBatch;
int batchCount = 0, batchSize = 10;// 15;
while (Data.GetImageAndLabelMinibatch(trainingDataMap, batchSize, batchCount++,
dstWidth, dstHeight, srcChannel, numOfCategories, out imageBatch, out labelBatch))
{
//TODO: sweepEnd should be set properly.
#pragma warning disable 618
trainer.TrainMinibatch(new Dictionary <Variable, Value>()
{
{ (Variable)inputData, imageBatch },
{ (Variable)labelData, labelBatch }
}, Layers.GetDevice());
#pragma warning restore 618
//PrintTrainingProgress(trainer, minibatchCount, 1);
}
// 学習過程の表示
Utilities.DrawTrainingProgress(trainer, minibatchCount, updatePerMinibatches, chtTraining);
}
////////////////////////////////////////////////////////
// 学習結果の保存、読み出し
// 本プログラム的には意味はないが、学習と推論を切り分けるためのサンプル
var modelFilename = "cntk_cnn.model";
// モデルの保存
model.Save(modelFilename);
// モデルの読み出し
// Softmax()は学習時のモデルには含まれていないため追加（CrossEntropyに含まれているため）
Function loadModel = Layers.LoadModel(modelFilename).Softmax();
////////////////////////////////////////////////////////
// 画像データの評価(推論)
var val = loadModel.EvaluateImage(@"D:\NeuralNetworkConsole\neural_network_console_140\samples\sample_dataset\MNIST\validation\2\186.png");
Console.WriteLine($"{categories[0]}:{val[0]}, {categories[1]}:{val[1]}, {categories[2]}:{val[2]}, {categories[3]}:{val[3]}");
val = loadModel.EvaluateImage(@"D:\NeuralNetworkConsole\neural_network_console_140\samples\sample_dataset\MNIST\validation\4\109.png");
Console.WriteLine($"{categories[0]}:{val[0]}, {categories[1]}:{val[1]}, {categories[2]}:{val[2]}, {categories[3]}:{val[3]}");
val = loadModel.EvaluateImage(@"D:\NeuralNetworkConsole\neural_network_console_140\samples\sample_dataset\MNIST\validation\8\299.png");
Console.WriteLine($"{categories[0]}:{val[0]}, {categories[1]}:{val[1]}, {categories[2]}:{val[2]}, {categories[3]}:{val[3]}");
val = loadModel.EvaluateImage(@"D:\NeuralNetworkConsole\neural_network_console_140\samples\sample_dataset\MNIST\validation\9\118.png");
Console.WriteLine($"{categories[0]}:{val[0]}, {categories[1]}:{val[1]}, {categories[2]}:{val[2]}, {categories[3]}:{val[3]}");
}
/// <summary>
/// Create an instance of the CNTK built-in SGD learner.
/// </summary>
/// <param name="parameters">Parameters of the learner.</param>
/// <param name="learningRateSchedule">Learning rate schedule.</param>
/// <param name="additionalOptions">Additional options.</param>
/// <returns></returns>
public static Learner SGDLearner(IList <Parameter> parameters, TrainingParameterScheduleDouble learningRateSchedule, AdditionalLearningOptions additionalOptions)
{
ParameterVector parameterVector = Helper.AsParameterVector(parameters);
return(SGDLearner(parameterVector, learningRateSchedule, additionalOptions));
}
public void Main()
{
WriteLine("Reading data");
var fn = @"c:\DEMO\Data\train.csv";
var f = File.ReadLines(fn);
var data = from t in f.Skip(1)
let zz = t.Split(',').Select(float.Parse)
select new Digit
{
Label = (int)zz.First(),
Image = zz.Skip(1).Select(x => x / 256f).ToArray()
};
var train = data.Take(40000).ToArray();
var test = data.Skip(40000).Take(1000).ToArray();
WriteLine("Creating network");
DeviceDescriptor device = DeviceDescriptor.CPUDevice;
// Create network model
int inputDim = 784;
int outputDim = 10;
NDShape inputShape = new int[] { 28, 28, 1 };
var outputShape = new NDShape(1, outputDim);
Variable features = Variable.InputVariable(inputShape, DataType.Float);
Variable label = Variable.InputVariable(outputShape, DataType.Float);
// 28x28x1 -> 14x14x4, window 3x3
var ConvParam1 = new Parameter(new int[] { 3, 3, 1, 4 }, DataType.Float, CNTKLib.GlorotUniformInitializer(0.26, -1, 2), device);
var Layer1 = CNTKLib.ReLU(CNTKLib.Convolution(ConvParam1, features, new int[] { 1, 1, 1 }));
var Layer1Pool = CNTKLib.Pooling(Layer1, PoolingType.Max, new int[] { 3, 3 }, new int[] { 2, 2 }, true.AsArray());
// 14x14x4 -> 7x7x8, window 3
var ConvParam2 = new Parameter(new int[] { 3, 3, 4, 8 }, DataType.Float, CNTKLib.GlorotUniformInitializer(0.26, -1, 2), device);
var Layer2 = CNTKLib.ReLU(CNTKLib.Convolution(ConvParam2, Layer1Pool, new int[] { 1, 1, 4 }));
var Layer2Pool = CNTKLib.Pooling(Layer2, PoolingType.Max, new int[] { 3, 3 }, new int[] { 2, 2 }, new bool[] { true });
var ConvOut = CNTKLib.Reshape(Layer2Pool, (7 * 7 * 8).AsArray());
var W = new Parameter(new int[] { outputDim, 7 * 7 * 8 }, DataType.Float, CNTKLib.GlorotUniformInitializer(), device, "w2");
var b = new Parameter(new int[] { outputDim }, DataType.Float, 0, device, "b2");
var z = CNTKLib.Times(W, ConvOut) + b;
var loss = CNTKLib.CrossEntropyWithSoftmax(z, label);
var evalError = CNTKLib.ClassificationError(z, label);
// prepare for training
CNTK.TrainingParameterScheduleDouble learningRatePerSample = new CNTK.TrainingParameterScheduleDouble(0.02, 1);
IList <Learner> parameterLearners =
new List <Learner>()
{
Learner.SGDLearner(z.Parameters(), learningRatePerSample)
};
var trainer = Trainer.CreateTrainer(z, loss, evalError, parameterLearners);
int minibatchSize = 64;
int numMinibatchesToTrain = 10000;
var feat = new BatchSource <float[]>((from x in train select x.Image).ToArray(), minibatchSize);
var labl = new BatchSource <float[]>((from x in train select x.Label.ToOneHot10(10).ToFloatArray()).ToArray(), minibatchSize);
// train the model
for (int ep = 0; ep < numMinibatchesToTrain; ep++)
{
Value ft, lb;
feat.MoveNext(); labl.MoveNext();
ft = Value.CreateBatchOfSequences <float>(inputShape, feat.Current, device);
lb = Value.CreateBatchOfSequences <float>(outputShape, labl.Current, device);
trainer.TrainMinibatch(
new Dictionary <Variable, Value>()
{
{ features, ft }, { label, lb }
}, device);
if (ep % 50 == 0)
{
var _loss = trainer.PreviousMinibatchLossAverage();
var _eval = trainer.PreviousMinibatchEvaluationAverage();
WriteLine($"Epoch={ep}, loss={_loss}, eval={_eval}");
}
}
int count = 0, correct = 0;
// Test the model
foreach (var x in test)
{
var imap = new Dictionary <Variable, Value> {
{ features, Value.CreateBatch(inputShape, x.Image, device) }
};
var omap = new Dictionary <Variable, Value> {
{ z, null }
};
z.Evaluate(imap, omap, device);
var o = omap[z].GetDenseData <float>(z).First();
var res = o.MaxIndex();
WriteLine("{0} => {1}", x.Label, res);
if (x.Label == res)
{
correct++;
}
count++;
}
WriteLine("Done, {0} of {1} correct ({2}%)", correct, count, (double)correct / (double)count * 100);
}
/// <summary>
/// RMSprop is an unpublished, adaptive learning rate method proposed by Geoff Hinton. This optimizer is usually a good choice for recurrent neural networks.
/// </summary>
/// <param name="modelOutput">The model output.</param>
/// <param name="learningRate">The learning rate.</param>
/// <param name="gamma">The gamma.</param>
/// <param name="inc">The incremental value</param>
/// <param name="dec">The decremental value.</param>
/// <param name="min">The minimum.</param>
/// <param name="max">The maximum.</param>
/// <param name="regulizer">The regulizer.</param>
/// <returns>Learner.</returns>
private Learner RMSprop(Function modelOutput, double learningRate = 0.001, double gamma = 0.9, double inc = 2, double dec = 0.01, double min = 0.01, double max = 1, Regulizers regulizer = null)
{
CNTK.TrainingParameterScheduleDouble learningRatePerSample = new CNTK.TrainingParameterScheduleDouble(learningRate, 1);
return(CNTKLib.RMSPropLearner(new ParameterVector(modelOutput.Parameters().ToList()), learningRatePerSample, gamma, inc, dec, max, min, false, GetAdditionalLearningOptions()));
}
public void Main()
{
WriteLine("Starting");
var G = new GraphLib();
WriteLine("Generating sample data");
// Создаём случайные точки двух типов
float cx1 = -1.0f, cy1 = -1.0f;
float cx2 = 1.0f, cy2 = 1.0f;
var Rnd = new Random();
var x = new List <float>();
var y = new List <float>();
var lab = new List <float>();
var lab_oh = new List <float[]>();
for (int i = 0; i < 200; i++)
{
x.Add(cx1 + Rnd.Next(-2.0, 2.0)); y.Add(cy1 + Rnd.Next(-2.0, 2.0));
lab.Add(-1.0f); lab_oh.Add(new float[] { 1.0f, 0.0f });
x.Add(cx2 + Rnd.Next(-2.0, 2.0)); y.Add(cy2 + Rnd.Next(-2.0, 2.0));
lab.Add(1.0f); lab_oh.Add(new float[] { 0.0f, 1.0f });
}
G.Plot(x, y);
WriteLine("Doing data split");
var x_train = x.Take(150).ToArray();
var y_train = y.Take(150).ToArray();
var l_train = lab.Take(150).ToArray();
var l_oh_train = lab_oh.Take(150).ToArray();
var x_test = x.Skip(150).ToArray();
var y_test = y.Skip(150).ToArray();
var l_test = lab.Skip(150).ToArray();
var l_oh_test = lab_oh.Skip(150).ToArray();
WriteLine("Creating network");
DeviceDescriptor device = DeviceDescriptor.CPUDevice;
// Create network model
int inputDim = 2;
int outputDim = 2;
Variable features = Variable.InputVariable(inputDim.AsArray(), DataType.Float);
Variable label = Variable.InputVariable(outputDim.AsArray(), DataType.Float);
var W = new Parameter(new int[] { outputDim, inputDim }, DataType.Float, 1, device, "w");
var b = new Parameter(new int[] { outputDim }, DataType.Float, 0, device, "b");
var z = CNTKLib.Times(W, features) + b;
var loss = CNTKLib.CrossEntropyWithSoftmax(z, label);
var evalError = CNTKLib.ClassificationError(z, label);
// prepare for training
CNTK.TrainingParameterScheduleDouble learningRatePerSample = new CNTK.TrainingParameterScheduleDouble(0.02, 1);
IList <Learner> parameterLearners =
new List <Learner>()
{
Learner.SGDLearner(z.Parameters(), learningRatePerSample)
};
var trainer = Trainer.CreateTrainer(z, loss, evalError, parameterLearners);
int minibatchSize = 64;
int numMinibatchesToTrain = 1000;
int k = 0; // current position in dataset
// train the model
for (int ep = 0; ep < numMinibatchesToTrain; ep++)
{
Value f, l;
var fa = new float[minibatchSize * inputDim];
var la = new float[minibatchSize * outputDim];
for (int j = 0; j < minibatchSize; j++)
{
fa[j * inputDim] = x_train[k];
fa[j * inputDim + 1] = y_train[k];
la[j * outputDim] = l_oh_train[k][0];
la[j * outputDim + 1] = l_oh_train[k][1];
k++;
if (k == x_train.Length)
{
k = 0;
}
}
f = Value.CreateBatch <float>(inputDim.AsArray(), fa, device);
l = Value.CreateBatch <float>(outputDim.AsArray(), la, device);
trainer.TrainMinibatch(
new Dictionary <Variable, Value>()
{
{ features, f }, { label, l }
}, device);
if (ep % 50 == 0)
{
var _loss = trainer.PreviousMinibatchLossAverage();
var _eval = trainer.PreviousMinibatchEvaluationAverage();
WriteLine($"Epoch={ep}, loss={_loss}, eval={_eval}");
}
}
WriteLine("Press any key to exit");
Console.ReadKey();
}
