private Function DefineModel_104A(Example_103_Util util, Variable inputVariables, DeviceDescriptor device)
{
var normalizedInput = CNTKLib.ElementTimes(Constant.Scalar <double>(1.0 / 255.0, device), inputVariables);
var encode3 = util.DenseNode(normalizedInput, 32, CNTKLib.ReLU);
var decoder = util.DenseNode(encode3, 28 * 28, CNTKLib.Sigmoid);
return(decoder);
}
private Function DefineModel_103C(Example_103_Util util, int nbLabels, Variable inputVariables)
{
Func <Variable, Function> activation = CNTKLib.ReLU;
var layer1 = util.DenseNode(inputVariables, 400, activation);
var layer2 = util.DenseNode(layer1, 200, activation);
var lastLayer = util.DenseNode(layer2, nbLabels, null); // linear layer
return(lastLayer);
}
private Function DefineModel_103D(Example_103_Util util, int nbLabels, Variable inputVariables)
{
Func <Variable, Function> activation = CNTKLib.ReLU;
var layer1 = util.Convolution(inputVariables, new[] { 5, 5 }, 8, new[] { 1, 1 }, activation);
var layer2 = util.Pooling(layer1, new[] { 2, 2 }, 8, new[] { 2, 2 });
var layer3 = util.Convolution(layer2, new[] { 5, 5 }, 16, new[] { 1, 1 }, activation);
var layer4 = util.Pooling(layer3, new[] { 3, 3 }, 16, new[] { 3, 3 });
var lastLayer = util.DenseNode(layer4, nbLabels, null); // linear layer
return(lastLayer);
}
private Function DefineModel_A(Variable input, int nbLabels, Example_103_Util util)
{
Func <Variable, Function> activation = CNTKLib.ReLU;
var layer1 = util.Convolution(input, new[] { 5, 5 }, 32, activation);
var layer2 = util.PoolingMax(layer1, new[] { 3, 3 }, new[] { 2, 2 });
var layer3 = util.Convolution(layer2, new[] { 5, 5 }, 32, activation);
var layer4 = util.PoolingMax(layer3, new[] { 3, 3 }, new[] { 2, 2 });
var layer5 = util.Convolution(layer4, new[] { 5, 5 }, 64, activation);
var layer6 = util.PoolingMax(layer5, new[] { 3, 3 }, new[] { 2, 2 });
var layer7 = util.DenseNode(layer6, 64, activation); // linear layer
var lastLayer = util.DenseNode(layer7, nbLabels, null); // linear layer
return(lastLayer);
}
private Function DefineModel_E(Variable input, int nbLabels, Example_103_Util util)
{
Func <Variable, Function> activation = CNTKLib.ReLU;
Variable net = input;
net = util.ConvolutionWithBatchNormalization(net, new[] { 3, 3 }, 16, activation);
net = util.ResNetBasicStack(net, 16, 3, activation);
net = util.ResNetInc(net, 32, activation);
net = util.ResNetBasicStack(net, 32, 2, activation);
net = util.ResNetInc(net, 64, activation);
net = util.ResNetBasicStack(net, 64, 2, activation);
net = util.PoolingMax(net, new[] { 8, 8 }, new[] { 1, 1 });
net = util.DenseNode(net, nbLabels, null);
return(net);
}
private Function DefineModel_D_Pourri(Variable input, int nbLabels, Example_103_Util util)
{
Func <Variable, Function> activation = CNTKLib.ReLU;
Variable net = input;
foreach (int dims in new[] { 64, 96, 128 })
{
net = util.Convolution(net, new[] { 3, 3 }, dims, activation);
net = util.Convolution(net, new[] { 3, 3 }, dims, activation);
net = util.PoolingMax(net, new[] { 3, 3 }, new int[] { 2, 2 });
}
for (int i = 0; i < 2; i++)
{
net = util.DenseNode(net, 1024, activation);
}
net = util.DenseNode(net, nbLabels, CNTKLib.Softmax);
return(net);
}
public void Run()
{
var device = DeviceDescriptor.UseDefaultDevice();
var util = new Example_103_Util();
Example_201_Data datasource = new Example_201_Data();
IEnumerable <Example_201_Item> trainingImages = datasource.LoadTrainingImages().ToList();
IEnumerable <Example_201_Item> testImages = datasource.LoadTestImages().ToList();
IDictionary <double, string> labelIndex = datasource.LoadLabelIndex().ToDictionary(x => (double)x.Key, x => x.Value);
int image_height = 32, image_width = 32, num_channels = 3, num_classes = 10;
Variable input = Variable.InputVariable(NDShape.CreateNDShape(new[] { image_height, image_width, num_channels }), DataType.Double, "input");
Variable expectedOutput = Variable.InputVariable(new int[] { num_classes }, DataType.Double, "expectedOutput");
Function normalizedInput = CNTKLib.ElementTimes(Constant.Scalar(1.0 / 255.0, device), input);
Function model = DefineModel_C(normalizedInput, num_classes, util);
Variable output = model.Output;
uint minibatchSize = 64;
Trainer trainer = MakeTrainer(expectedOutput, output, model, minibatchSize);
{ // train
int nbSamplesToUseForTraining = trainingImages.Count();
int numSweepsToTrainWith = 5;
int numMinibatchesToTrain = nbSamplesToUseForTraining * numSweepsToTrainWith / (int)minibatchSize;
var trainingInput = trainingImages.Select(x => x.Image.Select(y => (double)y).ToArray()).ToList();
var trainingOutput = trainingImages.Select(x => ToOneHotVector(x.Label, labelIndex.Count)).ToList();
var trainingMinibatchSource = new GenericMinibatchSource(input, trainingInput, expectedOutput, trainingOutput, nbSamplesToUseForTraining, numSweepsToTrainWith, minibatchSize, device);
RunTraining(trainer, trainingMinibatchSource, numMinibatchesToTrain, device);
}
// evaluate
Evaluate(model, testImages, input, device, labelIndex);
}
public void Run()
{
var device = DeviceDescriptor.UseDefaultDevice();
var util = new Example_103_Util();
// data
string trainImagesPath = "./Example_103/train-images-idx3-ubyte.gz";
//string trainLabelsPath = "./Example_103/train-labels-idx1-ubyte.gz";
List <double[]> trainImages = util.LoadImages(trainImagesPath).Select(x => x.Select(y => (double)y).ToArray()).ToList();
//List<int> trainLabels = util.LoadLabels(trainLabelsPath);
//List<double[]> trainLabels1Hot = trainLabels.Select(x => util.ConvertTo1Hot(x)).Select(x => x.Cast<double>().ToArray()).ToList();
string evelImagesPath = "./Example_103/t10k-images-idx3-ubyte.gz";
//string evalLabelsPath = "./Example_103/t10k-labels-idx1-ubyte.gz";
List <double[]> evalImages = util.LoadImages(evelImagesPath).Select(x => x.Select(y => (double)y).ToArray()).ToList();
//List<int> evalLabels = util.LoadLabels(evalLabelsPath);
//List<int[]> evalLabels1Hot = evalLabels.Select(x => util.ConvertTo1Hot(x)).ToList();
// model
int sampleSize = trainImages.Count;
int nbDimensionsInput = 28 * 28;
Variable inputVariables = Variable.InputVariable(NDShape.CreateNDShape(new [] { nbDimensionsInput }), DataType.Double, "input");
Variable expectedOutput = Variable.InputVariable(NDShape.CreateNDShape(new [] { nbDimensionsInput }), DataType.Double, "output");
Function encodeDecode = DefineModel_104B(util, inputVariables, device);
//var scaledModelOutput = CNTKLib.ElementTimes(Constant.Scalar<double>(1.0 / 255.0, device), encodeDecode);
//var scaledExpectedOutput = CNTKLib.ElementTimes(Constant.Scalar<double>(1.0 / 255.0, device), expectedOutput);
//{
// Function test = CNTKLib.ElementTimes(
// Constant.Scalar(-1.0d, device),
// inputVariables);
//}
//Function lossFunction = -scaledExpectedOutput * CNTKLib.Log(scaledModelOutput) - (Constant.Scalar(-1.0d, device) - scaledExpectedOutput) * CNTKLib.Log(1 - scaledModelOutput);
var scaledExpectedOutput = CNTKLib.ElementTimes(expectedOutput, Constant.Scalar(1 / 255.0, device));
//Function lossFunction = CNTKLib.CrossEntropyWithSoftmax(encodeDecode, expectedOutput);
// Function lossFunction = CNTKLib.CrossEntropyWithSoftmax(scaledModelOutput, scaledExpectedOutput);
Function lossFunction = CNTKLib.Square(CNTKLib.Minus(scaledExpectedOutput, encodeDecode));
Function evalErrorFunction = CNTKLib.ClassificationError(encodeDecode, scaledExpectedOutput);
// training
Trainer trainer;
{
// define training
//int epochSize = 30000;
uint minibatchSize = 64;
//double learningRate = 0.8;
int numSweepsToTrainWith = 2; // traduction de sweep ?
int nbSamplesToUseForTraining = 60000; // trainImages.Count;
double lr_per_sample = 0.2;
//double lr_per_sample = 0.2; // 0.00003;
//double lr_per_sample = 0.00003; // 0.00003;
uint epoch_size = 30000; // # 30000 samples is half the dataset size
TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(lr_per_sample, epoch_size);
TrainingParameterScheduleDouble momentumSchedule = new TrainingParameterScheduleDouble(0.9126265014311797, minibatchSize);
var parameters = new ParameterVector();
foreach (var p in encodeDecode.Parameters())
{
parameters.Add(p);
}
List <Learner> parameterLearners = new List <Learner>()
{
CNTKLib.FSAdaGradLearner(parameters, learningRatePerSample, momentumSchedule, true)
};
//IList<Learner> parameterLearners = new List<Learner>() { Learner.SGDLearner(encodeDecode.Parameters(), learningRatePerSample) };
trainer = Trainer.CreateTrainer(encodeDecode, lossFunction, evalErrorFunction, parameterLearners);
// run training
int numMinibatchesToTrain = nbSamplesToUseForTraining * numSweepsToTrainWith / (int)minibatchSize;
var minibatchSource = new GenericMinibatchSource(inputVariables, trainImages, expectedOutput, trainImages, nbSamplesToUseForTraining, numSweepsToTrainWith, minibatchSize, device);
double aggregate_metric = 0;
for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++)
{
IDictionary <Variable, MinibatchData> data = minibatchSource.GetNextRandomMinibatch();
trainer.TrainMinibatch(data, device);
double samples = trainer.PreviousMinibatchSampleCount();
double avg = trainer.PreviousMinibatchEvaluationAverage();
aggregate_metric += avg * samples;
double nbSampleSeen = trainer.TotalNumberOfSamplesSeen();
double train_error = aggregate_metric / nbSampleSeen;
Debug.WriteLine($"{minibatchCount} Average training error: {train_error:p2}");
}
}
// evaluate
{
uint testMinibatchSize = 32;
int nbSamplesToTest = 32;// evalImages.Count;
int numMinibatchesToTrain = nbSamplesToTest / (int)testMinibatchSize;
double metric_numer = 0;
double metric_denom = 0;
var minibatchSource = new GenericMinibatchSource(inputVariables, evalImages, expectedOutput, evalImages, nbSamplesToTest, 1, testMinibatchSize, device);
for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++)
{
IDictionary <Variable, MinibatchData> data = minibatchSource.GetNextRandomMinibatch();
////UnorderedMapVariableMinibatchData evalInput = new UnorderedMapVariableMinibatchData();
////foreach (var row in data)
//// evalInput[row.Key] = row.Value;
////double error = trainer.TestMinibatch(evalInput, device);
////metric_numer += Math.Abs(error * testMinibatchSize);
////metric_denom += testMinibatchSize;
////MinibatchData outputValue = evalInput[expectedOutput];
//IList<IList<double>> inputPixels = outputValue.data.GetDenseData<double>(inputVariables);
//IList<IList<double>> actualLabelSoftMax = outputValue.data.GetDenseData<double>(encodeDecode.Output);
//for (int i = 0; i < actualLabelSoftMax.Count; i++)
// PrintBitmap(inputPixels[i], actualLabelSoftMax[i], i);
// var normalizedInput = CNTKLib.ElementTimes(Constant.Scalar<double>(1.0 / 255.0, device), inputVariables);
Dictionary <Variable, Value> input = new Dictionary <Variable, Value>()
{
{ inputVariables, data[inputVariables].data }
};
Dictionary <Variable, Value> output = new Dictionary <Variable, Value>()
{
// { normalizedInput.Output, null }
{ encodeDecode.Output, null }
};
encodeDecode.Evaluate(input, output, device);
IList <IList <double> > inputPixels = input[inputVariables].GetDenseData <double>(inputVariables);
IList <IList <double> > outputPixels = output[encodeDecode.Output].GetDenseData <double>(encodeDecode.Output);
for (int i = 0; i < inputPixels.Count; i++)
{
PrintBitmap(inputPixels[i], outputPixels[i], i);
}
}
double test_error = (metric_numer * 100.0) / (metric_denom);
Debug.WriteLine($"Average test error: {test_error:p2}");
}
}
public void Run()
{
var device = DeviceDescriptor.UseDefaultDevice();
var util = new Example_103_Util();
// data
string trainImagesPath = "./Example_103/train-images-idx3-ubyte.gz";
string trainLabelsPath = "./Example_103/train-labels-idx1-ubyte.gz";
List <byte[]> trainImages = util.LoadImages(trainImagesPath);
List <int> trainLabels = util.LoadLabels(trainLabelsPath);
List <int[]> trainLabels1Hot = trainLabels.Select(x => util.ConvertTo1Hot(x)).ToList();
string evelImagesPath = "./Example_103/t10k-images-idx3-ubyte.gz";
string evalLabelsPath = "./Example_103/t10k-labels-idx1-ubyte.gz";
List <byte[]> evalImages = util.LoadImages(evelImagesPath);
List <int> evalLabels = util.LoadLabels(evalLabelsPath);
List <int[]> evalLabels1Hot = evalLabels.Select(x => util.ConvertTo1Hot(x)).ToList();
// model
int sampleSize = trainImages.Count;
int nbDimensionsInput = trainImages[0].Length;
int nbLabels = 10; // de 0 à 10
Variable inputVariables = Variable.InputVariable(NDShape.CreateNDShape(new[] { nbDimensionsInput }), DataType.Double, "input");
Variable expectedOutput = Variable.InputVariable(NDShape.CreateNDShape(new int[] { nbLabels }), DataType.Double, "output");
var scaledInput = CNTKLib.ElementTimes(Constant.Scalar <double>(1.0 / 255.0, device), inputVariables);
Function lastLayer = DefineModel_103C(util, nbLabels, scaledInput);
Function lossFunction = CNTKLib.CrossEntropyWithSoftmax(lastLayer, expectedOutput);
Function evalErrorFunction = CNTKLib.ClassificationError(lastLayer, expectedOutput);
// training
Trainer trainer;
{
// define training
uint minibatchSize = 64;
double learningRate = 0.2;
TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(learningRate, minibatchSize);
List <Learner> parameterLearners = new List <Learner>()
{
Learner.SGDLearner(lastLayer.Parameters(), learningRatePerSample)
};
trainer = Trainer.CreateTrainer(lastLayer, lossFunction, evalErrorFunction, parameterLearners);
// run training
int nbSamplesToUseForTraining = trainImages.Count;
int numSweepsToTrainWith = 10; // traduction de sweep ?
int numMinibatchesToTrain = nbSamplesToUseForTraining * numSweepsToTrainWith / (int)minibatchSize;
var minibatchSource = new Example_103_MinibatchSource(inputVariables, trainImages, expectedOutput, trainLabels1Hot, nbSamplesToUseForTraining, numSweepsToTrainWith, minibatchSize, device);
for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++)
{
IDictionary <Variable, MinibatchData> data = minibatchSource.GetNextRandomMinibatch();
trainer.TrainMinibatch(data, device);
util.PrintTrainingProgress(trainer, minibatchCount);
}
}
// evaluate
{
uint testMinibatchSize = 512;
int nbSamplesToTest = evalImages.Count;
int numMinibatchesToTrain = nbSamplesToTest / (int)testMinibatchSize;
double testResult = 0;
var minibatchSource = new Example_103_MinibatchSource(inputVariables, evalImages, expectedOutput, evalLabels1Hot, nbSamplesToTest, 1, testMinibatchSize, device);
for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++)
{
IDictionary <Variable, MinibatchData> data = minibatchSource.GetNextRandomMinibatch();
UnorderedMapVariableMinibatchData evalInput = new UnorderedMapVariableMinibatchData();
foreach (var row in data)
{
evalInput[row.Key] = row.Value;
}
double error = trainer.TestMinibatch(evalInput, device);
testResult += error;
//var z = CNTKLib.Softmax(lastLayer);
//var tOut = new Dictionary<Variable, Value>() { { z.Output, null } };
//z.Evaluate(
// new Dictionary<Variable, Value>() { { inputVariables, data[inputVariables].data } },
// tOut,
// device
// );
//Value outputValue = tOut[z.Output];
//IList<IList<double>> actualLabelSoftMax = outputValue.GetDenseData<double>(z.Output);
//var actualLabels = actualLabelSoftMax.Select((IList<double> l) => l.IndexOf(l.Max())).ToList();
//Value expectedOutputValue = data[expectedOutput].data;
//IList<IList<double>> expectedLabelsSoftmax = expectedOutputValue.GetDenseData<double>(z.Output);
//var expectedLabels = expectedLabelsSoftmax.Select((IList<double> l) => l.IndexOf(l.Max())).ToList();
//for(int i = 0; i < expectedLabels.Count; i++)
//{
// if (actualLabels[i] != expectedLabels[i])
// {
// Debug.WriteLine($"{actualLabels[i]} {expectedLabels[i]}");
// }
//}
//int misMatches = actualLabels.Zip(expectedLabels, (a, b) => a.Equals(b) ? 0 : 1).Sum();
Debug.WriteLine($"Average test error: {(testResult / (minibatchCount + 1)):p2}");
}
Debug.WriteLine($"Average test error: {(testResult / numMinibatchesToTrain):p2}");
}
}
private Function DefineModel_103B(Example_103_Util util, int nbLabels, Variable inputVariables)
{
return(util.DenseNode(inputVariables, nbLabels, null));
}
public Example_106()
{
_random = new Random(0);
_util = new Example_103_Util();
}
