void run()
{
Console.Title = "Ch_05_Class_Activation_Heatmaps";
var text = System.IO.File.ReadAllText("imagenet_class_index.json");
var imagenetInfo = Newtonsoft.Json.JsonConvert.DeserializeObject <Dictionary <int, List <string> > >(text);
var imagePath = System.IO.Path.Combine(System.IO.Directory.GetCurrentDirectory(), imageName);
var pathToVGG16model = VGG16.download_model_if_needed();
var image = new float[224 * 224 * 3];
CPPUtil.load_image(imagePath, image);
int num_classes = 1000;
var predictions = new float[num_classes];
CPPUtil.evaluate_vgg16(pathToVGG16model, imagePath, predictions, num_classes);
var indices = Enumerable.Range(0, num_classes).ToArray <int>();
var floatComparer = Comparer <float> .Default;
Array.Sort(indices, (a, b) => floatComparer.Compare(predictions[b], predictions[a]));
Console.WriteLine("Predictions:");
for (int i = 0; i < 3; i++)
{
var imagenetClass = imagenetInfo[indices[i]];
var imagenetClassName = imagenetClass[1];
var predicted_score = predictions[indices[i]];
Console.WriteLine($"\t({imagenetClassName} -> {predicted_score:f3})");
}
var imageWithHeatMap = new float[image.Length];
CPPUtil.visualize_heatmap(pathToVGG16model, imagePath, "conv5_3", 386, imageWithHeatMap);
var app = new System.Windows.Application();
var window = new PlotWindowBitMap("Original Image", image, 224, 224, 3);
window.Show();
var windowHeat = new PlotWindowBitMap("Class Activation Heatmap [386]", imageWithHeatMap, 224, 224, 3);
windowHeat.Show();
app.Run();
}
public void Run()
{
// Prepare data
var baseDataDirectoryPath = @"E:\DataSets\Mnist";
var trainFilePath = Path.Combine(baseDataDirectoryPath, "Train-28x28_cntk_text.txt");
// Define data type and device for the model.
var dataType = DataType.Float;
var device = DeviceDescriptor.UseDefaultDevice();
// Setup initializers
var random = new Random(232);
Func <CNTKDictionary> weightInit = () => Initializers.Xavier(random.Next(), scale: 0.02);
var biasInit = Initializers.Zero();
// Ensure reproducible results with CNTK.
CNTKLib.SetFixedRandomSeed((uint)random.Next());
CNTKLib.ForceDeterministicAlgorithms();
// Setup generator
var ganeratorInputShape = NDShape.CreateNDShape(new int[] { 100 });
var generatorInput = Variable.InputVariable(ganeratorInputShape, dataType);
var generatorNetwork = Generator(generatorInput, weightInit, biasInit, device, dataType);
// Setup discriminator
var discriminatorInputShape = NDShape.CreateNDShape(new int[] { 784 }); // 28 * 28 * 1.
var discriminatorInput = Variable.InputVariable(discriminatorInputShape, dataType);
// scale image input between -1.0 and 1.0.
var discriminatorInputScaled = CNTKLib.Minus(
CNTKLib.ElementTimes(Constant.Scalar(2 * 0.00390625f, device), discriminatorInput),
Constant.Scalar(1.0f, device));
var discriminatorNetwork = Discriminator(discriminatorInputScaled, weightInit, biasInit, device, dataType);
// The discriminator must be used on both the real MNIST images and fake images generated by the generator function.
// One way to represent this in the computational graph is to create a clone of the output of the discriminator function,
// but with substituted inputs. Setting method = share in the clone function ensures that both paths through the discriminator model
// use the same set of parameters.
var discriminatorNetworkFake = discriminatorNetwork
.Clone(ParameterCloningMethod.Share, replacements:
new Dictionary <Variable, Variable>
{
{ discriminatorInputScaled.Output, generatorNetwork.Output },
});
// Create minibatch source for providing the real images.
var imageNameToVariable = new Dictionary <string, Variable> {
{ "features", discriminatorInput }
};
var imageMinibatchSource = CreateMinibatchSource(trainFilePath, imageNameToVariable, randomize: true);
// Create minibatch source for providing the noise.
var noiseNameToVariable = new Dictionary <string, Variable> {
{ "noise", generatorInput }
};
var noiseMinibatchSource = new UniformNoiseMinibatchSource(noiseNameToVariable, min: -1.0f, max: 1.0f, seed: random.Next());
// Combine both sources in the composite minibatch source.
var compositeMinibatchSource = new CompositeMinibatchSource(imageMinibatchSource, noiseMinibatchSource);
// Setup generator loss: 1.0 - C.log(D_fake)
var generatorLossFunc = CNTKLib.Minus(Constant.Scalar(1.0f, device),
CNTKLib.Log(discriminatorNetworkFake));
// Setup discriminator loss: -(C.log(D_real) + C.log(1.0 - D_fake))
var discriminatorLossFunc = CNTKLib.Negate(CNTKLib.Plus(CNTKLib.Log(discriminatorNetwork),
CNTKLib.Log(CNTKLib.Minus(Constant.Scalar(1.0f, device), discriminatorNetworkFake))));
// Create fitters for the training loop.
// Generator uses Adam and discriminator SGD.
// Advice from: https://github.com/soumith/ganhacks
var generatorLearner = Learners.Adam(generatorNetwork.Parameters(),
learningRate: 0.0002, momentum: 0.5, gradientClippingThresholdPerSample: 1.0);
var generatorFitter = CreateFitter(generatorLearner, generatorNetwork, generatorLossFunc, device);
var discriminatorLearner = Learners.SGD(discriminatorNetwork.Parameters(),
learningRate: 0.0002, gradientClippingThresholdPerSample: 1.0);
var discriminatorFitter = CreateFitter(discriminatorLearner, discriminatorNetwork, discriminatorLossFunc, device);
int epochs = 30;
int batchSize = 128;
// Controls how many steps the discriminator takes,
// each time the generator takes 1 step.
// Default from the original paper is 1.
int discriminatorSteps = 1;
var isSweepEnd = false;
for (int epoch = 0; epoch < epochs;)
{
for (int step = 0; step < discriminatorSteps; step++)
{
// Discriminator needs both real images and noise,
// so uses the composite minibatch source.
var minibatchItems = compositeMinibatchSource.GetNextMinibatch(batchSize, device);
isSweepEnd = minibatchItems.isSweepEnd;
discriminatorFitter.FitNextStep(minibatchItems.minibatch, batchSize);
DisposeValues(minibatchItems.minibatch);
}
// Generator only needs noise images,
// so uses the noise minibatch source separately.
var noiseMinibatchItems = noiseMinibatchSource.GetNextMinibatch(batchSize, device);
generatorFitter.FitNextStep(noiseMinibatchItems.minibatch, batchSize);
DisposeValues(noiseMinibatchItems.minibatch);
if (isSweepEnd)
{
var generatorCurrentLoss = generatorFitter.CurrentLoss;
generatorFitter.ResetLossAndMetricAccumulators();
var discriminatorCurrentLoss = discriminatorFitter.CurrentLoss;
discriminatorFitter.ResetLossAndMetricAccumulators();
var traceOutput = $"Epoch: {epoch + 1:000} Generator Loss = {generatorCurrentLoss:F8}, Discriminator Loss = {discriminatorCurrentLoss:F8}";
Trace.WriteLine(traceOutput);
++epoch;
}
}
// Sample 6x6 images from generator.
var samples = 6 * 6;
var batch = noiseMinibatchSource.GetNextMinibatch(samples, device);
var noise = batch.minibatch;
var predictor = new Predictor(generatorNetwork, device);
var images = predictor.PredictNextStep(noise);
var imagesData = images.SelectMany(t => t).ToArray();
// Show examples
var app = new Application();
var window = new PlotWindowBitMap("Generated Images", imagesData, 28, 28, 1, true);
window.Show();
app.Run(window);
}
public void Run()
{
// Prepare data
var baseDataDirectoryPath = @"E:\DataSets\Mnist";
var trainFilePath = Path.Combine(baseDataDirectoryPath, "Train-28x28_cntk_text.txt");
// Define data type and device for the model.
var dataType = DataType.Float;
var device = DeviceDescriptor.UseDefaultDevice();
// Setup initializers
var random = new Random(232);
Func <CNTKDictionary> weightInit = () => Initializers.GlorotUniform(random.Next());
var biasInit = Initializers.Zero();
// Ensure reproducible results with CNTK.
CNTKLib.SetFixedRandomSeed((uint)random.Next());
CNTKLib.ForceDeterministicAlgorithms();
// Setup input dimensions and variables.
var inputShape = NDShape.CreateNDShape(new int[] { 28, 28, 1 });
var InputVariable = Variable.InputVariable(inputShape, dataType);
var scaledInputVariable = CNTKLib.ElementTimes(Constant.Scalar(0.00390625f, device), InputVariable);
const int latentSpaceSize = 2;
// Setup the encoder, this encodes the input into a mean and variance parameter.
var(mean, logVariance) = Encoder(scaledInputVariable, latentSpaceSize, weightInit, biasInit, device, dataType);
// add latent space sampling. This will draw a latent point using a small random epsilon.
var epsilon = CNTKLib.NormalRandom(new int[] { latentSpaceSize }, dataType);
var latentSpaceSampler = CNTKLib.Plus(mean, CNTKLib.ElementTimes(CNTKLib.Exp(logVariance), epsilon));
// add decoder, this decodes from latent space back to an image.
var encoderDecoderNetwork = Decoder(latentSpaceSampler, weightInit, biasInit, device, dataType);
// Create minibatch source for providing the input images.
var nameToVariable = new Dictionary <string, Variable> {
{ "features", encoderDecoderNetwork.Arguments[0] }
};
var minibatchSource = CreateMinibatchSource(trainFilePath, nameToVariable, randomize: true);
// Reconstruction loss. Forces the decoded samples to match the initial inputs.
var reconstructionLoss = Losses.BinaryCrossEntropy(encoderDecoderNetwork.Output, scaledInputVariable);
// Regularization loss. Helps in learning well-formed latent spaces and reducing overfitting to the training data.
var regularizationLoss = RegularizationLoss(mean, logVariance, dataType);
// Overall loss function. Sum of reconstruction- and regularization loss.
var loss = CNTKLib.Plus(reconstructionLoss, regularizationLoss);
// Setup trainer.
var learner = Learners.Adam(encoderDecoderNetwork.Parameters(), learningRate: 0.001, momentum: 0.9);
var trainer = Trainer.CreateTrainer(encoderDecoderNetwork, loss, loss, new List <Learner> {
learner
});
// Create model.
var model = new Model(trainer, encoderDecoderNetwork, dataType, device);
Trace.WriteLine(model.Summary());
// Train the model.
model.Fit(minibatchSource, batchSize: 16, epochs: 10);
//// Use the decoder to sample 15x15 images across the latent space.
// Image generation only requires use of the decoder part of the network.
// Clone and replace the latentSpaceSampler from training with a new decoder input variable.
var decoderInputVariable = Variable.InputVariable(latentSpaceSampler.Output.Shape, dataType);
var replacements = new Dictionary <Variable, Variable>()
{
{ latentSpaceSampler, decoderInputVariable }
};
var decoder = encoderDecoderNetwork.Clone(ParameterCloningMethod.Freeze, replacements);
// Sample 15x15 samples from the latent space
var minibatch = SampleMinibatchForGrid(device, decoderInputVariable, gridSize: 15);
// Transform from points in latent space to images using the decoder.
var predictor = new Predictor(decoder, device);
var images = predictor.PredictNextStep(minibatch);
var imagesData = images.SelectMany(t => t).ToArray();
// Show examples.
var app = new Application();
var window = new PlotWindowBitMap("Generated Images", imagesData, 28, 28, 1, true);
window.Show();
app.Run(window);
}