public void Dense_InputRank_And_MapRank_At_The_Same_Time_Throws()
{
var inputVariable = CNTKLib.InputVariable(new int[] { 2 }, m_dataType);
Layers.Dense(inputVariable, 2, Initializers.GlorotUniform(seed: 232), Initializers.Zero(),
m_device, m_dataType, inputRank: 1, mapRank: 1);
}
static void Main(string[] args)
{
Console.WriteLine("MNIST Test");
int seed;
using (var rng = new RNGCryptoServiceProvider())
{
var buffer = new byte[sizeof(int)];
rng.GetBytes(buffer);
seed = BitConverter.ToInt32(buffer, 0);
}
RandomProvider.SetSeed(seed);
var assembly = Assembly.GetExecutingAssembly();
var random = RandomProvider.GetRandom();
var trainingList = new List <Tuple <double[], double[]> >();
var testList = new List <Tuple <double[], double[]> >();
var logDictionary = new Dictionary <string, IEnumerable <double> >();
var logPath = "Log.csv";
var channels = 1;
var imageWidth = 28;
var imageHeight = 28;
var filters = 30;
var filterWidth = 5;
var filterHeight = 5;
var poolWidth = 2;
var poolHeight = 2;
using (Stream
imagesStream = assembly.GetManifestResourceStream("MNISTTest.train-images.idx3-ubyte"),
labelsStream = assembly.GetManifestResourceStream("MNISTTest.train-labels.idx1-ubyte"))
{
foreach (var image in MnistImage.Load(imagesStream, labelsStream).Take(1000))
{
var t = new double[10];
for (int i = 0; i < 10; i++)
{
if (i == image.Label)
{
t[i] = 1.0;
}
else
{
t[i] = 0.0;
}
}
trainingList.Add(Tuple.Create <double[], double[]>(image.Normalize(), t));
}
}
using (Stream
imagesStream = assembly.GetManifestResourceStream("MNISTTest.t10k-images.idx3-ubyte"),
labelsStream = assembly.GetManifestResourceStream("MNISTTest.t10k-labels.idx1-ubyte"))
{
foreach (var image in MnistImage.Load(imagesStream, labelsStream).Take(1000))
{
var t = new double[10];
for (int i = 0; i < 10; i++)
{
if (i == image.Label)
{
t[i] = 1.0;
}
else
{
t[i] = 0.0;
}
}
testList.Add(Tuple.Create <double[], double[]>(image.Normalize(), t));
}
}
var accuracyList = new List <double>();
var lossList = new List <double>();
var network = new Network(
new ConvolutionalPoolingLayer(channels, imageWidth, imageHeight, filters, filterWidth, filterHeight, poolWidth, poolHeight, new ReLU(), (index, fanIn, fanOut) => Initializers.HeNormal(fanIn),
new FullyConnectedLayer(filters * ConvolutionalPoolingLayer.GetOutputLength(imageWidth, filterWidth, poolWidth) * ConvolutionalPoolingLayer.GetOutputLength(imageHeight, filterHeight, poolHeight), new ReLU(), (index, fanIn, fanOut) => Initializers.GlorotUniform(fanIn, fanOut),
new SoftmaxLayer(100, 10, (index, fanIn, fanOut) => Initializers.GlorotUniform(fanIn, fanOut)))),
new Adam(), new SoftmaxCrossEntropy());
int epochs = 50;
int iterations = 1;
network.Stepped += (sender, e) =>
{
double tptn = 0;
trainingList.ForEach(x =>
{
var vector = network.Predicate(x.Item1);
var i = ArgMax(vector);
var j = ArgMax(x.Item2);
if (i == j && Math.Round(vector[i]) == x.Item2[j])
{
tptn += 1.0;
}
});
var accuracy = tptn / trainingList.Count;
accuracyList.Add(accuracy);
lossList.Add(network.Loss);
Console.WriteLine("Epoch {0}/{1}", iterations, epochs);
Console.WriteLine("Accuracy: {0}, Loss: {1}", accuracy, network.Loss);
iterations++;
};
Console.WriteLine("Training...");
var stopwatch = Stopwatch.StartNew();
network.Train(trainingList, epochs, 100);
stopwatch.Stop();
Console.WriteLine("Done ({0}).", stopwatch.Elapsed.ToString());
double testTptn = 0;
testList.ForEach(x =>
{
var vector = network.Predicate(x.Item1);
var i = ArgMax(vector);
var j = ArgMax(x.Item2);
if (i == j && Math.Round(vector[i]) == x.Item2[j])
{
testTptn += 1.0;
}
});
Console.WriteLine("Accuracy: {0}", testTptn / testList.Count);
logDictionary.Add("Accuracy", accuracyList);
logDictionary.Add("Loss", lossList);
ToCsv(logPath, logDictionary);
Console.Write("Saved log to {0}...", logPath);
}
public void Run()
{
// Prepare data
var baseDataDirectoryPath = @"E:\DataSets\Mnist";
var trainFilePath = Path.Combine(baseDataDirectoryPath, "Train-28x28_cntk_text.txt");
// Define data type and device for the model.
var dataType = DataType.Float;
var device = DeviceDescriptor.UseDefaultDevice();
// Setup initializers
var random = new Random(232);
Func <CNTKDictionary> weightInit = () => Initializers.GlorotUniform(random.Next());
var biasInit = Initializers.Zero();
// Ensure reproducible results with CNTK.
CNTKLib.SetFixedRandomSeed((uint)random.Next());
CNTKLib.ForceDeterministicAlgorithms();
// Setup input dimensions and variables.
var inputShape = NDShape.CreateNDShape(new int[] { 28, 28, 1 });
var InputVariable = Variable.InputVariable(inputShape, dataType);
var scaledInputVariable = CNTKLib.ElementTimes(Constant.Scalar(0.00390625f, device), InputVariable);
const int latentSpaceSize = 2;
// Setup the encoder, this encodes the input into a mean and variance parameter.
var(mean, logVariance) = Encoder(scaledInputVariable, latentSpaceSize, weightInit, biasInit, device, dataType);
// add latent space sampling. This will draw a latent point using a small random epsilon.
var epsilon = CNTKLib.NormalRandom(new int[] { latentSpaceSize }, dataType);
var latentSpaceSampler = CNTKLib.Plus(mean, CNTKLib.ElementTimes(CNTKLib.Exp(logVariance), epsilon));
// add decoder, this decodes from latent space back to an image.
var encoderDecoderNetwork = Decoder(latentSpaceSampler, weightInit, biasInit, device, dataType);
// Create minibatch source for providing the input images.
var nameToVariable = new Dictionary <string, Variable> {
{ "features", encoderDecoderNetwork.Arguments[0] }
};
var minibatchSource = CreateMinibatchSource(trainFilePath, nameToVariable, randomize: true);
// Reconstruction loss. Forces the decoded samples to match the initial inputs.
var reconstructionLoss = Losses.BinaryCrossEntropy(encoderDecoderNetwork.Output, scaledInputVariable);
// Regularization loss. Helps in learning well-formed latent spaces and reducing overfitting to the training data.
var regularizationLoss = RegularizationLoss(mean, logVariance, dataType);
// Overall loss function. Sum of reconstruction- and regularization loss.
var loss = CNTKLib.Plus(reconstructionLoss, regularizationLoss);
// Setup trainer.
var learner = Learners.Adam(encoderDecoderNetwork.Parameters(), learningRate: 0.001, momentum: 0.9);
var trainer = Trainer.CreateTrainer(encoderDecoderNetwork, loss, loss, new List <Learner> {
learner
});
// Create model.
var model = new Model(trainer, encoderDecoderNetwork, dataType, device);
Trace.WriteLine(model.Summary());
// Train the model.
model.Fit(minibatchSource, batchSize: 16, epochs: 10);
//// Use the decoder to sample 15x15 images across the latent space.
// Image generation only requires use of the decoder part of the network.
// Clone and replace the latentSpaceSampler from training with a new decoder input variable.
var decoderInputVariable = Variable.InputVariable(latentSpaceSampler.Output.Shape, dataType);
var replacements = new Dictionary <Variable, Variable>()
{
{ latentSpaceSampler, decoderInputVariable }
};
var decoder = encoderDecoderNetwork.Clone(ParameterCloningMethod.Freeze, replacements);
// Sample 15x15 samples from the latent space
var minibatch = SampleMinibatchForGrid(device, decoderInputVariable, gridSize: 15);
// Transform from points in latent space to images using the decoder.
var predictor = new Predictor(decoder, device);
var images = predictor.PredictNextStep(minibatch);
var imagesData = images.SelectMany(t => t).ToArray();
// Show examples.
var app = new Application();
var window = new PlotWindowBitMap("Generated Images", imagesData, 28, 28, 1, true);
window.Show();
app.Run(window);
}
