private Function DefineModel_104A(Example_103_Util util, Variable inputVariables, DeviceDescriptor device) { var normalizedInput = CNTKLib.ElementTimes(Constant.Scalar <double>(1.0 / 255.0, device), inputVariables); var encode3 = util.DenseNode(normalizedInput, 32, CNTKLib.ReLU); var decoder = util.DenseNode(encode3, 28 * 28, CNTKLib.Sigmoid); return(decoder); }
private Function DefineModel_103C(Example_103_Util util, int nbLabels, Variable inputVariables) { Func <Variable, Function> activation = CNTKLib.ReLU; var layer1 = util.DenseNode(inputVariables, 400, activation); var layer2 = util.DenseNode(layer1, 200, activation); var lastLayer = util.DenseNode(layer2, nbLabels, null); // linear layer return(lastLayer); }
private Function DefineModel_103D(Example_103_Util util, int nbLabels, Variable inputVariables) { Func <Variable, Function> activation = CNTKLib.ReLU; var layer1 = util.Convolution(inputVariables, new[] { 5, 5 }, 8, new[] { 1, 1 }, activation); var layer2 = util.Pooling(layer1, new[] { 2, 2 }, 8, new[] { 2, 2 }); var layer3 = util.Convolution(layer2, new[] { 5, 5 }, 16, new[] { 1, 1 }, activation); var layer4 = util.Pooling(layer3, new[] { 3, 3 }, 16, new[] { 3, 3 }); var lastLayer = util.DenseNode(layer4, nbLabels, null); // linear layer return(lastLayer); }
private Function DefineModel_A(Variable input, int nbLabels, Example_103_Util util) { Func <Variable, Function> activation = CNTKLib.ReLU; var layer1 = util.Convolution(input, new[] { 5, 5 }, 32, activation); var layer2 = util.PoolingMax(layer1, new[] { 3, 3 }, new[] { 2, 2 }); var layer3 = util.Convolution(layer2, new[] { 5, 5 }, 32, activation); var layer4 = util.PoolingMax(layer3, new[] { 3, 3 }, new[] { 2, 2 }); var layer5 = util.Convolution(layer4, new[] { 5, 5 }, 64, activation); var layer6 = util.PoolingMax(layer5, new[] { 3, 3 }, new[] { 2, 2 }); var layer7 = util.DenseNode(layer6, 64, activation); // linear layer var lastLayer = util.DenseNode(layer7, nbLabels, null); // linear layer return(lastLayer); }
private Function DefineModel_E(Variable input, int nbLabels, Example_103_Util util) { Func <Variable, Function> activation = CNTKLib.ReLU; Variable net = input; net = util.ConvolutionWithBatchNormalization(net, new[] { 3, 3 }, 16, activation); net = util.ResNetBasicStack(net, 16, 3, activation); net = util.ResNetInc(net, 32, activation); net = util.ResNetBasicStack(net, 32, 2, activation); net = util.ResNetInc(net, 64, activation); net = util.ResNetBasicStack(net, 64, 2, activation); net = util.PoolingMax(net, new[] { 8, 8 }, new[] { 1, 1 }); net = util.DenseNode(net, nbLabels, null); return(net); }
private Function DefineModel_D_Pourri(Variable input, int nbLabels, Example_103_Util util) { Func <Variable, Function> activation = CNTKLib.ReLU; Variable net = input; foreach (int dims in new[] { 64, 96, 128 }) { net = util.Convolution(net, new[] { 3, 3 }, dims, activation); net = util.Convolution(net, new[] { 3, 3 }, dims, activation); net = util.PoolingMax(net, new[] { 3, 3 }, new int[] { 2, 2 }); } for (int i = 0; i < 2; i++) { net = util.DenseNode(net, 1024, activation); } net = util.DenseNode(net, nbLabels, CNTKLib.Softmax); return(net); }
public void Run() { var device = DeviceDescriptor.UseDefaultDevice(); var util = new Example_103_Util(); Example_201_Data datasource = new Example_201_Data(); IEnumerable <Example_201_Item> trainingImages = datasource.LoadTrainingImages().ToList(); IEnumerable <Example_201_Item> testImages = datasource.LoadTestImages().ToList(); IDictionary <double, string> labelIndex = datasource.LoadLabelIndex().ToDictionary(x => (double)x.Key, x => x.Value); int image_height = 32, image_width = 32, num_channels = 3, num_classes = 10; Variable input = Variable.InputVariable(NDShape.CreateNDShape(new[] { image_height, image_width, num_channels }), DataType.Double, "input"); Variable expectedOutput = Variable.InputVariable(new int[] { num_classes }, DataType.Double, "expectedOutput"); Function normalizedInput = CNTKLib.ElementTimes(Constant.Scalar(1.0 / 255.0, device), input); Function model = DefineModel_C(normalizedInput, num_classes, util); Variable output = model.Output; uint minibatchSize = 64; Trainer trainer = MakeTrainer(expectedOutput, output, model, minibatchSize); { // train int nbSamplesToUseForTraining = trainingImages.Count(); int numSweepsToTrainWith = 5; int numMinibatchesToTrain = nbSamplesToUseForTraining * numSweepsToTrainWith / (int)minibatchSize; var trainingInput = trainingImages.Select(x => x.Image.Select(y => (double)y).ToArray()).ToList(); var trainingOutput = trainingImages.Select(x => ToOneHotVector(x.Label, labelIndex.Count)).ToList(); var trainingMinibatchSource = new GenericMinibatchSource(input, trainingInput, expectedOutput, trainingOutput, nbSamplesToUseForTraining, numSweepsToTrainWith, minibatchSize, device); RunTraining(trainer, trainingMinibatchSource, numMinibatchesToTrain, device); } // evaluate Evaluate(model, testImages, input, device, labelIndex); }
public void Run() { var device = DeviceDescriptor.UseDefaultDevice(); var util = new Example_103_Util(); // data string trainImagesPath = "./Example_103/train-images-idx3-ubyte.gz"; //string trainLabelsPath = "./Example_103/train-labels-idx1-ubyte.gz"; List <double[]> trainImages = util.LoadImages(trainImagesPath).Select(x => x.Select(y => (double)y).ToArray()).ToList(); //List<int> trainLabels = util.LoadLabels(trainLabelsPath); //List<double[]> trainLabels1Hot = trainLabels.Select(x => util.ConvertTo1Hot(x)).Select(x => x.Cast<double>().ToArray()).ToList(); string evelImagesPath = "./Example_103/t10k-images-idx3-ubyte.gz"; //string evalLabelsPath = "./Example_103/t10k-labels-idx1-ubyte.gz"; List <double[]> evalImages = util.LoadImages(evelImagesPath).Select(x => x.Select(y => (double)y).ToArray()).ToList(); //List<int> evalLabels = util.LoadLabels(evalLabelsPath); //List<int[]> evalLabels1Hot = evalLabels.Select(x => util.ConvertTo1Hot(x)).ToList(); // model int sampleSize = trainImages.Count; int nbDimensionsInput = 28 * 28; Variable inputVariables = Variable.InputVariable(NDShape.CreateNDShape(new [] { nbDimensionsInput }), DataType.Double, "input"); Variable expectedOutput = Variable.InputVariable(NDShape.CreateNDShape(new [] { nbDimensionsInput }), DataType.Double, "output"); Function encodeDecode = DefineModel_104B(util, inputVariables, device); //var scaledModelOutput = CNTKLib.ElementTimes(Constant.Scalar<double>(1.0 / 255.0, device), encodeDecode); //var scaledExpectedOutput = CNTKLib.ElementTimes(Constant.Scalar<double>(1.0 / 255.0, device), expectedOutput); //{ // Function test = CNTKLib.ElementTimes( // Constant.Scalar(-1.0d, device), // inputVariables); //} //Function lossFunction = -scaledExpectedOutput * CNTKLib.Log(scaledModelOutput) - (Constant.Scalar(-1.0d, device) - scaledExpectedOutput) * CNTKLib.Log(1 - scaledModelOutput); var scaledExpectedOutput = CNTKLib.ElementTimes(expectedOutput, Constant.Scalar(1 / 255.0, device)); //Function lossFunction = CNTKLib.CrossEntropyWithSoftmax(encodeDecode, expectedOutput); // Function lossFunction = CNTKLib.CrossEntropyWithSoftmax(scaledModelOutput, scaledExpectedOutput); Function lossFunction = CNTKLib.Square(CNTKLib.Minus(scaledExpectedOutput, encodeDecode)); Function evalErrorFunction = CNTKLib.ClassificationError(encodeDecode, scaledExpectedOutput); // training Trainer trainer; { // define training //int epochSize = 30000; uint minibatchSize = 64; //double learningRate = 0.8; int numSweepsToTrainWith = 2; // traduction de sweep ? int nbSamplesToUseForTraining = 60000; // trainImages.Count; double lr_per_sample = 0.2; //double lr_per_sample = 0.2; // 0.00003; //double lr_per_sample = 0.00003; // 0.00003; uint epoch_size = 30000; // # 30000 samples is half the dataset size TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(lr_per_sample, epoch_size); TrainingParameterScheduleDouble momentumSchedule = new TrainingParameterScheduleDouble(0.9126265014311797, minibatchSize); var parameters = new ParameterVector(); foreach (var p in encodeDecode.Parameters()) { parameters.Add(p); } List <Learner> parameterLearners = new List <Learner>() { CNTKLib.FSAdaGradLearner(parameters, learningRatePerSample, momentumSchedule, true) }; //IList<Learner> parameterLearners = new List<Learner>() { Learner.SGDLearner(encodeDecode.Parameters(), learningRatePerSample) }; trainer = Trainer.CreateTrainer(encodeDecode, lossFunction, evalErrorFunction, parameterLearners); // run training int numMinibatchesToTrain = nbSamplesToUseForTraining * numSweepsToTrainWith / (int)minibatchSize; var minibatchSource = new GenericMinibatchSource(inputVariables, trainImages, expectedOutput, trainImages, nbSamplesToUseForTraining, numSweepsToTrainWith, minibatchSize, device); double aggregate_metric = 0; for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++) { IDictionary <Variable, MinibatchData> data = minibatchSource.GetNextRandomMinibatch(); trainer.TrainMinibatch(data, device); double samples = trainer.PreviousMinibatchSampleCount(); double avg = trainer.PreviousMinibatchEvaluationAverage(); aggregate_metric += avg * samples; double nbSampleSeen = trainer.TotalNumberOfSamplesSeen(); double train_error = aggregate_metric / nbSampleSeen; Debug.WriteLine($"{minibatchCount} Average training error: {train_error:p2}"); } } // evaluate { uint testMinibatchSize = 32; int nbSamplesToTest = 32;// evalImages.Count; int numMinibatchesToTrain = nbSamplesToTest / (int)testMinibatchSize; double metric_numer = 0; double metric_denom = 0; var minibatchSource = new GenericMinibatchSource(inputVariables, evalImages, expectedOutput, evalImages, nbSamplesToTest, 1, testMinibatchSize, device); for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++) { IDictionary <Variable, MinibatchData> data = minibatchSource.GetNextRandomMinibatch(); ////UnorderedMapVariableMinibatchData evalInput = new UnorderedMapVariableMinibatchData(); ////foreach (var row in data) //// evalInput[row.Key] = row.Value; ////double error = trainer.TestMinibatch(evalInput, device); ////metric_numer += Math.Abs(error * testMinibatchSize); ////metric_denom += testMinibatchSize; ////MinibatchData outputValue = evalInput[expectedOutput]; //IList<IList<double>> inputPixels = outputValue.data.GetDenseData<double>(inputVariables); //IList<IList<double>> actualLabelSoftMax = outputValue.data.GetDenseData<double>(encodeDecode.Output); //for (int i = 0; i < actualLabelSoftMax.Count; i++) // PrintBitmap(inputPixels[i], actualLabelSoftMax[i], i); // var normalizedInput = CNTKLib.ElementTimes(Constant.Scalar<double>(1.0 / 255.0, device), inputVariables); Dictionary <Variable, Value> input = new Dictionary <Variable, Value>() { { inputVariables, data[inputVariables].data } }; Dictionary <Variable, Value> output = new Dictionary <Variable, Value>() { // { normalizedInput.Output, null } { encodeDecode.Output, null } }; encodeDecode.Evaluate(input, output, device); IList <IList <double> > inputPixels = input[inputVariables].GetDenseData <double>(inputVariables); IList <IList <double> > outputPixels = output[encodeDecode.Output].GetDenseData <double>(encodeDecode.Output); for (int i = 0; i < inputPixels.Count; i++) { PrintBitmap(inputPixels[i], outputPixels[i], i); } } double test_error = (metric_numer * 100.0) / (metric_denom); Debug.WriteLine($"Average test error: {test_error:p2}"); } }
public void Run() { var device = DeviceDescriptor.UseDefaultDevice(); var util = new Example_103_Util(); // data string trainImagesPath = "./Example_103/train-images-idx3-ubyte.gz"; string trainLabelsPath = "./Example_103/train-labels-idx1-ubyte.gz"; List <byte[]> trainImages = util.LoadImages(trainImagesPath); List <int> trainLabels = util.LoadLabels(trainLabelsPath); List <int[]> trainLabels1Hot = trainLabels.Select(x => util.ConvertTo1Hot(x)).ToList(); string evelImagesPath = "./Example_103/t10k-images-idx3-ubyte.gz"; string evalLabelsPath = "./Example_103/t10k-labels-idx1-ubyte.gz"; List <byte[]> evalImages = util.LoadImages(evelImagesPath); List <int> evalLabels = util.LoadLabels(evalLabelsPath); List <int[]> evalLabels1Hot = evalLabels.Select(x => util.ConvertTo1Hot(x)).ToList(); // model int sampleSize = trainImages.Count; int nbDimensionsInput = trainImages[0].Length; int nbLabels = 10; // de 0 à 10 Variable inputVariables = Variable.InputVariable(NDShape.CreateNDShape(new[] { nbDimensionsInput }), DataType.Double, "input"); Variable expectedOutput = Variable.InputVariable(NDShape.CreateNDShape(new int[] { nbLabels }), DataType.Double, "output"); var scaledInput = CNTKLib.ElementTimes(Constant.Scalar <double>(1.0 / 255.0, device), inputVariables); Function lastLayer = DefineModel_103C(util, nbLabels, scaledInput); Function lossFunction = CNTKLib.CrossEntropyWithSoftmax(lastLayer, expectedOutput); Function evalErrorFunction = CNTKLib.ClassificationError(lastLayer, expectedOutput); // training Trainer trainer; { // define training uint minibatchSize = 64; double learningRate = 0.2; TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(learningRate, minibatchSize); List <Learner> parameterLearners = new List <Learner>() { Learner.SGDLearner(lastLayer.Parameters(), learningRatePerSample) }; trainer = Trainer.CreateTrainer(lastLayer, lossFunction, evalErrorFunction, parameterLearners); // run training int nbSamplesToUseForTraining = trainImages.Count; int numSweepsToTrainWith = 10; // traduction de sweep ? int numMinibatchesToTrain = nbSamplesToUseForTraining * numSweepsToTrainWith / (int)minibatchSize; var minibatchSource = new Example_103_MinibatchSource(inputVariables, trainImages, expectedOutput, trainLabels1Hot, nbSamplesToUseForTraining, numSweepsToTrainWith, minibatchSize, device); for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++) { IDictionary <Variable, MinibatchData> data = minibatchSource.GetNextRandomMinibatch(); trainer.TrainMinibatch(data, device); util.PrintTrainingProgress(trainer, minibatchCount); } } // evaluate { uint testMinibatchSize = 512; int nbSamplesToTest = evalImages.Count; int numMinibatchesToTrain = nbSamplesToTest / (int)testMinibatchSize; double testResult = 0; var minibatchSource = new Example_103_MinibatchSource(inputVariables, evalImages, expectedOutput, evalLabels1Hot, nbSamplesToTest, 1, testMinibatchSize, device); for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++) { IDictionary <Variable, MinibatchData> data = minibatchSource.GetNextRandomMinibatch(); UnorderedMapVariableMinibatchData evalInput = new UnorderedMapVariableMinibatchData(); foreach (var row in data) { evalInput[row.Key] = row.Value; } double error = trainer.TestMinibatch(evalInput, device); testResult += error; //var z = CNTKLib.Softmax(lastLayer); //var tOut = new Dictionary<Variable, Value>() { { z.Output, null } }; //z.Evaluate( // new Dictionary<Variable, Value>() { { inputVariables, data[inputVariables].data } }, // tOut, // device // ); //Value outputValue = tOut[z.Output]; //IList<IList<double>> actualLabelSoftMax = outputValue.GetDenseData<double>(z.Output); //var actualLabels = actualLabelSoftMax.Select((IList<double> l) => l.IndexOf(l.Max())).ToList(); //Value expectedOutputValue = data[expectedOutput].data; //IList<IList<double>> expectedLabelsSoftmax = expectedOutputValue.GetDenseData<double>(z.Output); //var expectedLabels = expectedLabelsSoftmax.Select((IList<double> l) => l.IndexOf(l.Max())).ToList(); //for(int i = 0; i < expectedLabels.Count; i++) //{ // if (actualLabels[i] != expectedLabels[i]) // { // Debug.WriteLine($"{actualLabels[i]} {expectedLabels[i]}"); // } //} //int misMatches = actualLabels.Zip(expectedLabels, (a, b) => a.Equals(b) ? 0 : 1).Sum(); Debug.WriteLine($"Average test error: {(testResult / (minibatchCount + 1)):p2}"); } Debug.WriteLine($"Average test error: {(testResult / numMinibatchesToTrain):p2}"); } }
private Function DefineModel_103B(Example_103_Util util, int nbLabels, Variable inputVariables) { return(util.DenseNode(inputVariables, nbLabels, null)); }
public Example_106() { _random = new Random(0); _util = new Example_103_Util(); }