Exemple #1
0
        void run()
        {
            Console.Title = "Ch_05_Class_Activation_Heatmaps";
            var text         = System.IO.File.ReadAllText("imagenet_class_index.json");
            var imagenetInfo = Newtonsoft.Json.JsonConvert.DeserializeObject <Dictionary <int, List <string> > >(text);

            var imagePath        = System.IO.Path.Combine(System.IO.Directory.GetCurrentDirectory(), imageName);
            var pathToVGG16model = VGG16.download_model_if_needed();
            var image            = new float[224 * 224 * 3];

            CPPUtil.load_image(imagePath, image);

            int num_classes = 1000;
            var predictions = new float[num_classes];

            CPPUtil.evaluate_vgg16(pathToVGG16model, imagePath, predictions, num_classes);

            var indices       = Enumerable.Range(0, num_classes).ToArray <int>();
            var floatComparer = Comparer <float> .Default;

            Array.Sort(indices, (a, b) => floatComparer.Compare(predictions[b], predictions[a]));

            Console.WriteLine("Predictions:");
            for (int i = 0; i < 3; i++)
            {
                var imagenetClass     = imagenetInfo[indices[i]];
                var imagenetClassName = imagenetClass[1];
                var predicted_score   = predictions[indices[i]];
                Console.WriteLine($"\t({imagenetClassName} -> {predicted_score:f3})");
            }

            var imageWithHeatMap = new float[image.Length];

            CPPUtil.visualize_heatmap(pathToVGG16model, imagePath, "conv5_3", 386, imageWithHeatMap);

            var app    = new System.Windows.Application();
            var window = new PlotWindowBitMap("Original Image", image, 224, 224, 3);

            window.Show();
            var windowHeat = new PlotWindowBitMap("Class Activation Heatmap [386]", imageWithHeatMap, 224, 224, 3);

            windowHeat.Show();
            app.Run();
        }
Exemple #2
0
        public void Run()
        {
            // Prepare data
            var baseDataDirectoryPath = @"E:\DataSets\Mnist";
            var trainFilePath         = Path.Combine(baseDataDirectoryPath, "Train-28x28_cntk_text.txt");

            // Define data type and device for the model.
            var dataType = DataType.Float;
            var device   = DeviceDescriptor.UseDefaultDevice();

            // Setup initializers
            var random = new Random(232);
            Func <CNTKDictionary> weightInit = () => Initializers.Xavier(random.Next(), scale: 0.02);
            var biasInit = Initializers.Zero();

            // Ensure reproducible results with CNTK.
            CNTKLib.SetFixedRandomSeed((uint)random.Next());
            CNTKLib.ForceDeterministicAlgorithms();

            // Setup generator
            var ganeratorInputShape = NDShape.CreateNDShape(new int[] { 100 });
            var generatorInput      = Variable.InputVariable(ganeratorInputShape, dataType);
            var generatorNetwork    = Generator(generatorInput, weightInit, biasInit, device, dataType);

            // Setup discriminator
            var discriminatorInputShape = NDShape.CreateNDShape(new int[] { 784 }); // 28 * 28 * 1.
            var discriminatorInput      = Variable.InputVariable(discriminatorInputShape, dataType);
            // scale image input between -1.0 and 1.0.
            var discriminatorInputScaled = CNTKLib.Minus(
                CNTKLib.ElementTimes(Constant.Scalar(2 * 0.00390625f, device), discriminatorInput),
                Constant.Scalar(1.0f, device));

            var discriminatorNetwork = Discriminator(discriminatorInputScaled, weightInit, biasInit, device, dataType);

            // The discriminator must be used on both the real MNIST images and fake images generated by the generator function.
            // One way to represent this in the computational graph is to create a clone of the output of the discriminator function,
            // but with substituted inputs. Setting method = share in the clone function ensures that both paths through the discriminator model
            // use the same set of parameters.
            var discriminatorNetworkFake = discriminatorNetwork
                                           .Clone(ParameterCloningMethod.Share, replacements:
                                                  new Dictionary <Variable, Variable>
            {
                { discriminatorInputScaled.Output, generatorNetwork.Output },
            });

            // Create minibatch source for providing the real images.
            var imageNameToVariable = new Dictionary <string, Variable> {
                { "features", discriminatorInput }
            };
            var imageMinibatchSource = CreateMinibatchSource(trainFilePath, imageNameToVariable, randomize: true);

            // Create minibatch source for providing the noise.
            var noiseNameToVariable = new Dictionary <string, Variable> {
                { "noise", generatorInput }
            };
            var noiseMinibatchSource = new UniformNoiseMinibatchSource(noiseNameToVariable, min: -1.0f, max: 1.0f, seed: random.Next());

            // Combine both sources in the composite minibatch source.
            var compositeMinibatchSource = new CompositeMinibatchSource(imageMinibatchSource, noiseMinibatchSource);

            // Setup generator loss: 1.0 - C.log(D_fake)
            var generatorLossFunc = CNTKLib.Minus(Constant.Scalar(1.0f, device),
                                                  CNTKLib.Log(discriminatorNetworkFake));

            // Setup discriminator loss: -(C.log(D_real) + C.log(1.0 - D_fake))
            var discriminatorLossFunc = CNTKLib.Negate(CNTKLib.Plus(CNTKLib.Log(discriminatorNetwork),
                                                                    CNTKLib.Log(CNTKLib.Minus(Constant.Scalar(1.0f, device), discriminatorNetworkFake))));

            // Create fitters for the training loop.
            // Generator uses Adam and discriminator SGD.
            // Advice from: https://github.com/soumith/ganhacks
            var generatorLearner = Learners.Adam(generatorNetwork.Parameters(),
                                                 learningRate: 0.0002, momentum: 0.5, gradientClippingThresholdPerSample: 1.0);
            var generatorFitter = CreateFitter(generatorLearner, generatorNetwork, generatorLossFunc, device);

            var discriminatorLearner = Learners.SGD(discriminatorNetwork.Parameters(),
                                                    learningRate: 0.0002, gradientClippingThresholdPerSample: 1.0);
            var discriminatorFitter = CreateFitter(discriminatorLearner, discriminatorNetwork, discriminatorLossFunc, device);

            int epochs    = 30;
            int batchSize = 128;

            // Controls how many steps the discriminator takes,
            // each time the generator takes 1 step.
            // Default from the original paper is 1.
            int discriminatorSteps = 1;

            var isSweepEnd = false;

            for (int epoch = 0; epoch < epochs;)
            {
                for (int step = 0; step < discriminatorSteps; step++)
                {
                    // Discriminator needs both real images and noise,
                    // so uses the composite minibatch source.
                    var minibatchItems = compositeMinibatchSource.GetNextMinibatch(batchSize, device);
                    isSweepEnd = minibatchItems.isSweepEnd;

                    discriminatorFitter.FitNextStep(minibatchItems.minibatch, batchSize);
                    DisposeValues(minibatchItems.minibatch);
                }

                // Generator only needs noise images,
                // so uses the noise minibatch source separately.
                var noiseMinibatchItems = noiseMinibatchSource.GetNextMinibatch(batchSize, device);

                generatorFitter.FitNextStep(noiseMinibatchItems.minibatch, batchSize);
                DisposeValues(noiseMinibatchItems.minibatch);

                if (isSweepEnd)
                {
                    var generatorCurrentLoss = generatorFitter.CurrentLoss;
                    generatorFitter.ResetLossAndMetricAccumulators();

                    var discriminatorCurrentLoss = discriminatorFitter.CurrentLoss;
                    discriminatorFitter.ResetLossAndMetricAccumulators();

                    var traceOutput = $"Epoch: {epoch + 1:000} Generator Loss = {generatorCurrentLoss:F8}, Discriminator Loss = {discriminatorCurrentLoss:F8}";
                    Trace.WriteLine(traceOutput);

                    ++epoch;
                }
            }

            // Sample 6x6 images from generator.
            var samples = 6 * 6;
            var batch   = noiseMinibatchSource.GetNextMinibatch(samples, device);
            var noise   = batch.minibatch;

            var predictor  = new Predictor(generatorNetwork, device);
            var images     = predictor.PredictNextStep(noise);
            var imagesData = images.SelectMany(t => t).ToArray();

            // Show examples
            var app    = new Application();
            var window = new PlotWindowBitMap("Generated Images", imagesData, 28, 28, 1, true);

            window.Show();
            app.Run(window);
        }
        public void Run()
        {
            // Prepare data
            var baseDataDirectoryPath = @"E:\DataSets\Mnist";
            var trainFilePath         = Path.Combine(baseDataDirectoryPath, "Train-28x28_cntk_text.txt");

            // Define data type and device for the model.
            var dataType = DataType.Float;
            var device   = DeviceDescriptor.UseDefaultDevice();

            // Setup initializers
            var random = new Random(232);
            Func <CNTKDictionary> weightInit = () => Initializers.GlorotUniform(random.Next());
            var biasInit = Initializers.Zero();

            // Ensure reproducible results with CNTK.
            CNTKLib.SetFixedRandomSeed((uint)random.Next());
            CNTKLib.ForceDeterministicAlgorithms();

            // Setup input dimensions and variables.
            var inputShape          = NDShape.CreateNDShape(new int[] { 28, 28, 1 });
            var InputVariable       = Variable.InputVariable(inputShape, dataType);
            var scaledInputVariable = CNTKLib.ElementTimes(Constant.Scalar(0.00390625f, device), InputVariable);

            const int latentSpaceSize = 2;

            // Setup the encoder, this encodes the input into a mean and variance parameter.
            var(mean, logVariance) = Encoder(scaledInputVariable, latentSpaceSize, weightInit, biasInit, device, dataType);

            // add latent space sampling. This will draw a latent point using a small random epsilon.
            var epsilon            = CNTKLib.NormalRandom(new int[] { latentSpaceSize }, dataType);
            var latentSpaceSampler = CNTKLib.Plus(mean, CNTKLib.ElementTimes(CNTKLib.Exp(logVariance), epsilon));

            // add decoder, this decodes from latent space back to an image.
            var encoderDecoderNetwork = Decoder(latentSpaceSampler, weightInit, biasInit, device, dataType);

            // Create minibatch source for providing the input images.
            var nameToVariable = new Dictionary <string, Variable> {
                { "features", encoderDecoderNetwork.Arguments[0] }
            };
            var minibatchSource = CreateMinibatchSource(trainFilePath, nameToVariable, randomize: true);

            // Reconstruction loss. Forces the decoded samples to match the initial inputs.
            var reconstructionLoss = Losses.BinaryCrossEntropy(encoderDecoderNetwork.Output, scaledInputVariable);

            // Regularization loss. Helps in learning well-formed latent spaces and reducing overfitting to the training data.
            var regularizationLoss = RegularizationLoss(mean, logVariance, dataType);

            // Overall loss function. Sum of reconstruction- and regularization loss.
            var loss = CNTKLib.Plus(reconstructionLoss, regularizationLoss);

            // Setup trainer.
            var learner = Learners.Adam(encoderDecoderNetwork.Parameters(), learningRate: 0.001, momentum: 0.9);
            var trainer = Trainer.CreateTrainer(encoderDecoderNetwork, loss, loss, new List <Learner> {
                learner
            });
            // Create model.
            var model = new Model(trainer, encoderDecoderNetwork, dataType, device);

            Trace.WriteLine(model.Summary());

            // Train the model.
            model.Fit(minibatchSource, batchSize: 16, epochs: 10);

            //// Use the decoder to sample 15x15 images across the latent space.

            // Image generation only requires use of the decoder part of the network.
            // Clone and replace the latentSpaceSampler from training with a new decoder input variable.
            var decoderInputVariable = Variable.InputVariable(latentSpaceSampler.Output.Shape, dataType);
            var replacements         = new Dictionary <Variable, Variable>()
            {
                { latentSpaceSampler, decoderInputVariable }
            };
            var decoder = encoderDecoderNetwork.Clone(ParameterCloningMethod.Freeze, replacements);

            // Sample 15x15 samples from the latent space
            var minibatch = SampleMinibatchForGrid(device, decoderInputVariable, gridSize: 15);

            // Transform from points in latent space to images using the decoder.
            var predictor  = new Predictor(decoder, device);
            var images     = predictor.PredictNextStep(minibatch);
            var imagesData = images.SelectMany(t => t).ToArray();

            // Show examples.
            var app    = new Application();
            var window = new PlotWindowBitMap("Generated Images", imagesData, 28, 28, 1, true);

            window.Show();
            app.Run(window);
        }