Beispiel #1
0
        private void RunTraining(Trainer trainer, GenericMinibatchSource minibatchSource, int numMinibatchesToTrain, DeviceDescriptor device)
        {
            Debug.WriteLine($"Minibatch;CrossEntropyLoss;EvaluationCriterion;");
            double aggregate_metric = 0;

            for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++)
            {
                IDictionary <Variable, MinibatchData> data = minibatchSource.GetNextRandomMinibatch();
                trainer.TrainMinibatch(data, device);
                PrintTrainingProgress(trainer, minibatchCount);
            }
        }
Beispiel #2
0
        public void Run()
        {
            var device = DeviceDescriptor.UseDefaultDevice();
            var util   = new Example_103_Util();

            Example_201_Data datasource = new Example_201_Data();
            IEnumerable <Example_201_Item> trainingImages = datasource.LoadTrainingImages().ToList();
            IEnumerable <Example_201_Item> testImages     = datasource.LoadTestImages().ToList();
            IDictionary <double, string>   labelIndex     = datasource.LoadLabelIndex().ToDictionary(x => (double)x.Key, x => x.Value);

            int image_height = 32, image_width = 32, num_channels = 3, num_classes = 10;

            Variable input          = Variable.InputVariable(NDShape.CreateNDShape(new[] { image_height, image_width, num_channels }), DataType.Double, "input");
            Variable expectedOutput = Variable.InputVariable(new int[] { num_classes }, DataType.Double, "expectedOutput");

            Function normalizedInput = CNTKLib.ElementTimes(Constant.Scalar(1.0 / 255.0, device), input);
            Function model           = DefineModel_C(normalizedInput, num_classes, util);

            Variable output = model.Output;

            uint    minibatchSize = 64;
            Trainer trainer       = MakeTrainer(expectedOutput, output, model, minibatchSize);

            {   // train
                int nbSamplesToUseForTraining = trainingImages.Count();
                int numSweepsToTrainWith      = 5;
                int numMinibatchesToTrain     = nbSamplesToUseForTraining * numSweepsToTrainWith / (int)minibatchSize;
                var trainingInput             = trainingImages.Select(x => x.Image.Select(y => (double)y).ToArray()).ToList();
                var trainingOutput            = trainingImages.Select(x => ToOneHotVector(x.Label, labelIndex.Count)).ToList();
                var trainingMinibatchSource   = new GenericMinibatchSource(input, trainingInput, expectedOutput, trainingOutput, nbSamplesToUseForTraining, numSweepsToTrainWith, minibatchSize, device);
                RunTraining(trainer, trainingMinibatchSource, numMinibatchesToTrain, device);
            }

            // evaluate
            Evaluate(model, testImages, input, device, labelIndex);
        }
Beispiel #3
0
        public void Run()
        {
            var device = DeviceDescriptor.UseDefaultDevice();

            var util = new Example_103_Util();

            // data
            string trainImagesPath = "./Example_103/train-images-idx3-ubyte.gz";
            //string trainLabelsPath = "./Example_103/train-labels-idx1-ubyte.gz";
            List <double[]> trainImages = util.LoadImages(trainImagesPath).Select(x => x.Select(y => (double)y).ToArray()).ToList();
            //List<int> trainLabels = util.LoadLabels(trainLabelsPath);
            //List<double[]> trainLabels1Hot = trainLabels.Select(x => util.ConvertTo1Hot(x)).Select(x => x.Cast<double>().ToArray()).ToList();

            string evelImagesPath = "./Example_103/t10k-images-idx3-ubyte.gz";
            //string evalLabelsPath = "./Example_103/t10k-labels-idx1-ubyte.gz";
            List <double[]> evalImages = util.LoadImages(evelImagesPath).Select(x => x.Select(y => (double)y).ToArray()).ToList();
            //List<int> evalLabels = util.LoadLabels(evalLabelsPath);
            //List<int[]> evalLabels1Hot = evalLabels.Select(x => util.ConvertTo1Hot(x)).ToList();

            // model

            int sampleSize        = trainImages.Count;
            int nbDimensionsInput = 28 * 28;

            Variable inputVariables = Variable.InputVariable(NDShape.CreateNDShape(new [] { nbDimensionsInput }), DataType.Double, "input");
            Variable expectedOutput = Variable.InputVariable(NDShape.CreateNDShape(new [] { nbDimensionsInput }), DataType.Double, "output");

            Function encodeDecode = DefineModel_104B(util, inputVariables, device);

            //var scaledModelOutput = CNTKLib.ElementTimes(Constant.Scalar<double>(1.0 / 255.0, device), encodeDecode);
            //var scaledExpectedOutput = CNTKLib.ElementTimes(Constant.Scalar<double>(1.0 / 255.0, device), expectedOutput);

            //{

            //    Function test = CNTKLib.ElementTimes(
            //                Constant.Scalar(-1.0d, device),
            //                inputVariables);


            //}


            //Function lossFunction = -scaledExpectedOutput * CNTKLib.Log(scaledModelOutput) - (Constant.Scalar(-1.0d, device) - scaledExpectedOutput) * CNTKLib.Log(1 - scaledModelOutput);

            var scaledExpectedOutput = CNTKLib.ElementTimes(expectedOutput, Constant.Scalar(1 / 255.0, device));
            //Function lossFunction = CNTKLib.CrossEntropyWithSoftmax(encodeDecode, expectedOutput);

            // Function lossFunction = CNTKLib.CrossEntropyWithSoftmax(scaledModelOutput, scaledExpectedOutput);
            Function lossFunction = CNTKLib.Square(CNTKLib.Minus(scaledExpectedOutput, encodeDecode));

            Function evalErrorFunction = CNTKLib.ClassificationError(encodeDecode, scaledExpectedOutput);

            // training

            Trainer trainer;
            {
                // define training
                //int epochSize = 30000;
                uint minibatchSize = 64;
                //double learningRate = 0.8;
                int numSweepsToTrainWith      = 2;     // traduction de sweep ?
                int nbSamplesToUseForTraining = 60000; // trainImages.Count;

                double lr_per_sample = 0.2;
                //double lr_per_sample = 0.2; // 0.00003;
                //double lr_per_sample = 0.00003; // 0.00003;
                uint epoch_size = 30000; //        # 30000 samples is half the dataset size

                TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(lr_per_sample, epoch_size);
                TrainingParameterScheduleDouble momentumSchedule      = new TrainingParameterScheduleDouble(0.9126265014311797, minibatchSize);

                var parameters = new ParameterVector();
                foreach (var p in encodeDecode.Parameters())
                {
                    parameters.Add(p);
                }

                List <Learner> parameterLearners = new List <Learner>()
                {
                    CNTKLib.FSAdaGradLearner(parameters, learningRatePerSample, momentumSchedule, true)
                };
                //IList<Learner> parameterLearners = new List<Learner>() { Learner.SGDLearner(encodeDecode.Parameters(), learningRatePerSample) };
                trainer = Trainer.CreateTrainer(encodeDecode, lossFunction, evalErrorFunction, parameterLearners);

                // run training

                int numMinibatchesToTrain = nbSamplesToUseForTraining * numSweepsToTrainWith / (int)minibatchSize;

                var minibatchSource = new GenericMinibatchSource(inputVariables, trainImages, expectedOutput, trainImages, nbSamplesToUseForTraining, numSweepsToTrainWith, minibatchSize, device);

                double aggregate_metric = 0;
                for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++)
                {
                    IDictionary <Variable, MinibatchData> data = minibatchSource.GetNextRandomMinibatch();
                    trainer.TrainMinibatch(data, device);

                    double samples = trainer.PreviousMinibatchSampleCount();
                    double avg     = trainer.PreviousMinibatchEvaluationAverage();
                    aggregate_metric += avg * samples;
                    double nbSampleSeen = trainer.TotalNumberOfSamplesSeen();
                    double train_error  = aggregate_metric / nbSampleSeen;
                    Debug.WriteLine($"{minibatchCount} Average training error: {train_error:p2}");
                }
            }

            // evaluate
            {
                uint testMinibatchSize     = 32;
                int  nbSamplesToTest       = 32;// evalImages.Count;
                int  numMinibatchesToTrain = nbSamplesToTest / (int)testMinibatchSize;

                double metric_numer = 0;
                double metric_denom = 0;

                var minibatchSource = new GenericMinibatchSource(inputVariables, evalImages, expectedOutput, evalImages, nbSamplesToTest, 1, testMinibatchSize, device);
                for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++)
                {
                    IDictionary <Variable, MinibatchData> data = minibatchSource.GetNextRandomMinibatch();

                    ////UnorderedMapVariableMinibatchData evalInput = new UnorderedMapVariableMinibatchData();
                    ////foreach (var row in data)
                    ////    evalInput[row.Key] = row.Value;

                    ////double error = trainer.TestMinibatch(evalInput, device);

                    ////metric_numer += Math.Abs(error * testMinibatchSize);
                    ////metric_denom += testMinibatchSize;

                    ////MinibatchData outputValue = evalInput[expectedOutput];

                    //IList<IList<double>> inputPixels = outputValue.data.GetDenseData<double>(inputVariables);

                    //IList<IList<double>> actualLabelSoftMax = outputValue.data.GetDenseData<double>(encodeDecode.Output);

                    //for (int i = 0; i < actualLabelSoftMax.Count; i++)
                    //    PrintBitmap(inputPixels[i], actualLabelSoftMax[i], i);

                    // var normalizedInput = CNTKLib.ElementTimes(Constant.Scalar<double>(1.0 / 255.0, device), inputVariables);

                    Dictionary <Variable, Value> input = new Dictionary <Variable, Value>()
                    {
                        { inputVariables, data[inputVariables].data }
                    };
                    Dictionary <Variable, Value> output = new Dictionary <Variable, Value>()
                    {
                        // { normalizedInput.Output, null }
                        { encodeDecode.Output, null }
                    };

                    encodeDecode.Evaluate(input, output, device);

                    IList <IList <double> > inputPixels  = input[inputVariables].GetDenseData <double>(inputVariables);
                    IList <IList <double> > outputPixels = output[encodeDecode.Output].GetDenseData <double>(encodeDecode.Output);
                    for (int i = 0; i < inputPixels.Count; i++)
                    {
                        PrintBitmap(inputPixels[i], outputPixels[i], i);
                    }
                }

                double test_error = (metric_numer * 100.0) / (metric_denom);
                Debug.WriteLine($"Average test error: {test_error:p2}");
            }
        }