public void Run() { var device = DeviceDescriptor.UseDefaultDevice(); var util = new Example_103_Util(); // data string trainImagesPath = "./Example_103/train-images-idx3-ubyte.gz"; //string trainLabelsPath = "./Example_103/train-labels-idx1-ubyte.gz"; List <double[]> trainImages = util.LoadImages(trainImagesPath).Select(x => x.Select(y => (double)y).ToArray()).ToList(); //List<int> trainLabels = util.LoadLabels(trainLabelsPath); //List<double[]> trainLabels1Hot = trainLabels.Select(x => util.ConvertTo1Hot(x)).Select(x => x.Cast<double>().ToArray()).ToList(); string evelImagesPath = "./Example_103/t10k-images-idx3-ubyte.gz"; //string evalLabelsPath = "./Example_103/t10k-labels-idx1-ubyte.gz"; List <double[]> evalImages = util.LoadImages(evelImagesPath).Select(x => x.Select(y => (double)y).ToArray()).ToList(); //List<int> evalLabels = util.LoadLabels(evalLabelsPath); //List<int[]> evalLabels1Hot = evalLabels.Select(x => util.ConvertTo1Hot(x)).ToList(); // model int sampleSize = trainImages.Count; int nbDimensionsInput = 28 * 28; Variable inputVariables = Variable.InputVariable(NDShape.CreateNDShape(new [] { nbDimensionsInput }), DataType.Double, "input"); Variable expectedOutput = Variable.InputVariable(NDShape.CreateNDShape(new [] { nbDimensionsInput }), DataType.Double, "output"); Function encodeDecode = DefineModel_104B(util, inputVariables, device); //var scaledModelOutput = CNTKLib.ElementTimes(Constant.Scalar<double>(1.0 / 255.0, device), encodeDecode); //var scaledExpectedOutput = CNTKLib.ElementTimes(Constant.Scalar<double>(1.0 / 255.0, device), expectedOutput); //{ // Function test = CNTKLib.ElementTimes( // Constant.Scalar(-1.0d, device), // inputVariables); //} //Function lossFunction = -scaledExpectedOutput * CNTKLib.Log(scaledModelOutput) - (Constant.Scalar(-1.0d, device) - scaledExpectedOutput) * CNTKLib.Log(1 - scaledModelOutput); var scaledExpectedOutput = CNTKLib.ElementTimes(expectedOutput, Constant.Scalar(1 / 255.0, device)); //Function lossFunction = CNTKLib.CrossEntropyWithSoftmax(encodeDecode, expectedOutput); // Function lossFunction = CNTKLib.CrossEntropyWithSoftmax(scaledModelOutput, scaledExpectedOutput); Function lossFunction = CNTKLib.Square(CNTKLib.Minus(scaledExpectedOutput, encodeDecode)); Function evalErrorFunction = CNTKLib.ClassificationError(encodeDecode, scaledExpectedOutput); // training Trainer trainer; { // define training //int epochSize = 30000; uint minibatchSize = 64; //double learningRate = 0.8; int numSweepsToTrainWith = 2; // traduction de sweep ? int nbSamplesToUseForTraining = 60000; // trainImages.Count; double lr_per_sample = 0.2; //double lr_per_sample = 0.2; // 0.00003; //double lr_per_sample = 0.00003; // 0.00003; uint epoch_size = 30000; // # 30000 samples is half the dataset size TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(lr_per_sample, epoch_size); TrainingParameterScheduleDouble momentumSchedule = new TrainingParameterScheduleDouble(0.9126265014311797, minibatchSize); var parameters = new ParameterVector(); foreach (var p in encodeDecode.Parameters()) { parameters.Add(p); } List <Learner> parameterLearners = new List <Learner>() { CNTKLib.FSAdaGradLearner(parameters, learningRatePerSample, momentumSchedule, true) }; //IList<Learner> parameterLearners = new List<Learner>() { Learner.SGDLearner(encodeDecode.Parameters(), learningRatePerSample) }; trainer = Trainer.CreateTrainer(encodeDecode, lossFunction, evalErrorFunction, parameterLearners); // run training int numMinibatchesToTrain = nbSamplesToUseForTraining * numSweepsToTrainWith / (int)minibatchSize; var minibatchSource = new GenericMinibatchSource(inputVariables, trainImages, expectedOutput, trainImages, nbSamplesToUseForTraining, numSweepsToTrainWith, minibatchSize, device); double aggregate_metric = 0; for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++) { IDictionary <Variable, MinibatchData> data = minibatchSource.GetNextRandomMinibatch(); trainer.TrainMinibatch(data, device); double samples = trainer.PreviousMinibatchSampleCount(); double avg = trainer.PreviousMinibatchEvaluationAverage(); aggregate_metric += avg * samples; double nbSampleSeen = trainer.TotalNumberOfSamplesSeen(); double train_error = aggregate_metric / nbSampleSeen; Debug.WriteLine($"{minibatchCount} Average training error: {train_error:p2}"); } } // evaluate { uint testMinibatchSize = 32; int nbSamplesToTest = 32;// evalImages.Count; int numMinibatchesToTrain = nbSamplesToTest / (int)testMinibatchSize; double metric_numer = 0; double metric_denom = 0; var minibatchSource = new GenericMinibatchSource(inputVariables, evalImages, expectedOutput, evalImages, nbSamplesToTest, 1, testMinibatchSize, device); for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++) { IDictionary <Variable, MinibatchData> data = minibatchSource.GetNextRandomMinibatch(); ////UnorderedMapVariableMinibatchData evalInput = new UnorderedMapVariableMinibatchData(); ////foreach (var row in data) //// evalInput[row.Key] = row.Value; ////double error = trainer.TestMinibatch(evalInput, device); ////metric_numer += Math.Abs(error * testMinibatchSize); ////metric_denom += testMinibatchSize; ////MinibatchData outputValue = evalInput[expectedOutput]; //IList<IList<double>> inputPixels = outputValue.data.GetDenseData<double>(inputVariables); //IList<IList<double>> actualLabelSoftMax = outputValue.data.GetDenseData<double>(encodeDecode.Output); //for (int i = 0; i < actualLabelSoftMax.Count; i++) // PrintBitmap(inputPixels[i], actualLabelSoftMax[i], i); // var normalizedInput = CNTKLib.ElementTimes(Constant.Scalar<double>(1.0 / 255.0, device), inputVariables); Dictionary <Variable, Value> input = new Dictionary <Variable, Value>() { { inputVariables, data[inputVariables].data } }; Dictionary <Variable, Value> output = new Dictionary <Variable, Value>() { // { normalizedInput.Output, null } { encodeDecode.Output, null } }; encodeDecode.Evaluate(input, output, device); IList <IList <double> > inputPixels = input[inputVariables].GetDenseData <double>(inputVariables); IList <IList <double> > outputPixels = output[encodeDecode.Output].GetDenseData <double>(encodeDecode.Output); for (int i = 0; i < inputPixels.Count; i++) { PrintBitmap(inputPixels[i], outputPixels[i], i); } } double test_error = (metric_numer * 100.0) / (metric_denom); Debug.WriteLine($"Average test error: {test_error:p2}"); } }
public void Run() { var device = DeviceDescriptor.UseDefaultDevice(); var util = new Example_103_Util(); // data string trainImagesPath = "./Example_103/train-images-idx3-ubyte.gz"; string trainLabelsPath = "./Example_103/train-labels-idx1-ubyte.gz"; List <byte[]> trainImages = util.LoadImages(trainImagesPath); List <int> trainLabels = util.LoadLabels(trainLabelsPath); List <int[]> trainLabels1Hot = trainLabels.Select(x => util.ConvertTo1Hot(x)).ToList(); string evelImagesPath = "./Example_103/t10k-images-idx3-ubyte.gz"; string evalLabelsPath = "./Example_103/t10k-labels-idx1-ubyte.gz"; List <byte[]> evalImages = util.LoadImages(evelImagesPath); List <int> evalLabels = util.LoadLabels(evalLabelsPath); List <int[]> evalLabels1Hot = evalLabels.Select(x => util.ConvertTo1Hot(x)).ToList(); // model int sampleSize = trainImages.Count; int nbDimensionsInput = trainImages[0].Length; int nbLabels = 10; // de 0 à 10 Variable inputVariables = Variable.InputVariable(NDShape.CreateNDShape(new[] { nbDimensionsInput }), DataType.Double, "input"); Variable expectedOutput = Variable.InputVariable(NDShape.CreateNDShape(new int[] { nbLabels }), DataType.Double, "output"); var scaledInput = CNTKLib.ElementTimes(Constant.Scalar <double>(1.0 / 255.0, device), inputVariables); Function lastLayer = DefineModel_103C(util, nbLabels, scaledInput); Function lossFunction = CNTKLib.CrossEntropyWithSoftmax(lastLayer, expectedOutput); Function evalErrorFunction = CNTKLib.ClassificationError(lastLayer, expectedOutput); // training Trainer trainer; { // define training uint minibatchSize = 64; double learningRate = 0.2; TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(learningRate, minibatchSize); List <Learner> parameterLearners = new List <Learner>() { Learner.SGDLearner(lastLayer.Parameters(), learningRatePerSample) }; trainer = Trainer.CreateTrainer(lastLayer, lossFunction, evalErrorFunction, parameterLearners); // run training int nbSamplesToUseForTraining = trainImages.Count; int numSweepsToTrainWith = 10; // traduction de sweep ? int numMinibatchesToTrain = nbSamplesToUseForTraining * numSweepsToTrainWith / (int)minibatchSize; var minibatchSource = new Example_103_MinibatchSource(inputVariables, trainImages, expectedOutput, trainLabels1Hot, nbSamplesToUseForTraining, numSweepsToTrainWith, minibatchSize, device); for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++) { IDictionary <Variable, MinibatchData> data = minibatchSource.GetNextRandomMinibatch(); trainer.TrainMinibatch(data, device); util.PrintTrainingProgress(trainer, minibatchCount); } } // evaluate { uint testMinibatchSize = 512; int nbSamplesToTest = evalImages.Count; int numMinibatchesToTrain = nbSamplesToTest / (int)testMinibatchSize; double testResult = 0; var minibatchSource = new Example_103_MinibatchSource(inputVariables, evalImages, expectedOutput, evalLabels1Hot, nbSamplesToTest, 1, testMinibatchSize, device); for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++) { IDictionary <Variable, MinibatchData> data = minibatchSource.GetNextRandomMinibatch(); UnorderedMapVariableMinibatchData evalInput = new UnorderedMapVariableMinibatchData(); foreach (var row in data) { evalInput[row.Key] = row.Value; } double error = trainer.TestMinibatch(evalInput, device); testResult += error; //var z = CNTKLib.Softmax(lastLayer); //var tOut = new Dictionary<Variable, Value>() { { z.Output, null } }; //z.Evaluate( // new Dictionary<Variable, Value>() { { inputVariables, data[inputVariables].data } }, // tOut, // device // ); //Value outputValue = tOut[z.Output]; //IList<IList<double>> actualLabelSoftMax = outputValue.GetDenseData<double>(z.Output); //var actualLabels = actualLabelSoftMax.Select((IList<double> l) => l.IndexOf(l.Max())).ToList(); //Value expectedOutputValue = data[expectedOutput].data; //IList<IList<double>> expectedLabelsSoftmax = expectedOutputValue.GetDenseData<double>(z.Output); //var expectedLabels = expectedLabelsSoftmax.Select((IList<double> l) => l.IndexOf(l.Max())).ToList(); //for(int i = 0; i < expectedLabels.Count; i++) //{ // if (actualLabels[i] != expectedLabels[i]) // { // Debug.WriteLine($"{actualLabels[i]} {expectedLabels[i]}"); // } //} //int misMatches = actualLabels.Zip(expectedLabels, (a, b) => a.Equals(b) ? 0 : 1).Sum(); Debug.WriteLine($"Average test error: {(testResult / (minibatchCount + 1)):p2}"); } Debug.WriteLine($"Average test error: {(testResult / numMinibatchesToTrain):p2}"); } }