/// <summary>
/// Trains a neural net on the MNIST database (digit recognition)
/// The data files can be downloaded from http://yann.lecun.com/exdb/mnist/
/// </summary>
/// <param name="dataFilesPath">The path to a directory with the four extracted data files</param>
public static void MNIST(string dataFilesPath, string outputModelPath)
{
// neural network hyper parameters
const int HIDDEN_SIZE = 1024, BATCH_SIZE = 128, NUM_EPOCHS = 40;
const float TRAINING_RATE = 0.03f;
var errorMetric = ErrorMetricType.OneHot.Create();
var layerTemplate = new LayerDescriptor(0f)
{
WeightUpdate = WeightUpdateType.RMSprop,
Activation = ActivationType.LeakyRelu
};
Console.Write("Loading training data...");
var trainingData = Mnist.Load(dataFilesPath + "train-labels.idx1-ubyte", dataFilesPath + "train-images.idx3-ubyte");
var testData = Mnist.Load(dataFilesPath + "t10k-labels.idx1-ubyte", dataFilesPath + "t10k-images.idx3-ubyte");
Console.WriteLine("done");
Console.WriteLine("Starting training...");
using (var lap = GPUProvider.CreateLinearAlgebra()) {
var trainingSet = lap.NN.CreateTrainingDataProvider(trainingData.Select(d => d.Sample).ToList());
var testSet = lap.NN.CreateTrainingDataProvider(testData.Select(d => d.Sample).ToList());
using (var trainer = lap.NN.CreateBatchTrainer(layerTemplate, Mnist.INPUT_SIZE, HIDDEN_SIZE, Mnist.OUTPUT_SIZE)) {
var trainingManager = lap.NN.CreateFeedForwardManager(trainer, outputModelPath, testSet);
var trainingContext = lap.NN.CreateTrainingContext(errorMetric, TRAINING_RATE, BATCH_SIZE);
trainingContext.ScheduleTrainingRateChange(NUM_EPOCHS / 2, TRAINING_RATE / 3);
trainingManager.Train(trainingSet, NUM_EPOCHS, trainingContext);
}
}
}
/// <summary>
/// Classifies text into either positive or negative sentiment
/// The data files can be downloaded from https://archive.ics.uci.edu/ml/datasets/Sentiment+Labelled+Sentences
/// </summary>
/// <param name="dataFilesPath">Path to extracted data files</param>
public static void SentimentClassification(string dataFilesPath)
{
var files = new[] {
"amazon_cells_labelled.txt",
"imdb_labelled.txt",
"yelp_labelled.txt"
};
var LINE_SEPARATOR = "\n".ToCharArray();
var SEPARATOR = "\t".ToCharArray();
var stringTable = new StringTableBuilder();
var sentimentData = files.SelectMany(f => File.ReadAllText(dataFilesPath + f)
.Split(LINE_SEPARATOR)
.Where(l => !String.IsNullOrWhiteSpace(l))
.Select(l => l.Split(SEPARATOR))
.Select(s => Tuple.Create(_Tokenise(s[0]), s[1][0] == '1' ? "positive" : "negative"))
.Where(d => d.Item1.Any())
).Shuffle(0).ToList();
var splitSentimentData = sentimentData.Split();
// build training and test classification bag
var trainingClassificationBag = _BuildClassificationBag(splitSentimentData.Training, stringTable);
var testClassificationBag = _BuildClassificationBag(splitSentimentData.Test, stringTable);
// train a bernoulli naive bayes classifier
var bernoulli = trainingClassificationBag.TrainBernoulliNaiveBayes();
Console.WriteLine("Bernoulli accuracy: {0:P}", testClassificationBag
.Classify(bernoulli.CreateClassifier())
.Average(r => r.Score)
);
// train a multinomial naive bayes classifier
var multinomial = trainingClassificationBag.TrainMultinomialNaiveBayes();
Console.WriteLine("Multinomial accuracy: {0:P}", testClassificationBag
.Classify(multinomial.CreateClassifier())
.Average(r => r.Score)
);
// convert the bags to sparse vectors
var sentimentDataBag = _BuildClassificationBag(sentimentData, stringTable);
var sentimentDataSet = sentimentDataBag.ConvertToSparseVectors(false);
var sentimentDataTableSplit = sentimentDataSet.Split();
using (var lap = GPUProvider.CreateLinearAlgebra(false)) {
var maxIndex = sentimentDataSet.GetMaximumIndex() + 1;
var trainingData = sentimentDataTableSplit.Training.CreateTrainingDataProvider(lap, maxIndex);
var testData = sentimentDataTableSplit.Test.CreateTrainingDataProvider(lap, maxIndex);
var classificationTable = sentimentDataSet.GetClassifications().ToDictionary(d => (int)d.Value, d => d.Key);
// create the three classifiers
var bernoulliClassifier = bernoulli.CreateClassifier();
var multinomialClassifier = multinomial.CreateClassifier();
var neuralClassifier = lap.NN.CreateFeedForward(lap.NN.CreateTrainingContext(ErrorMetricType.OneHot, learningRate: 0.1f, batchSize: 128)
.TrainNeuralNetwork(lap, trainingData, testData, new LayerDescriptor(0.1f)
{
WeightUpdate = WeightUpdateType.Adam,
Activation = ActivationType.Relu,
WeightInitialisation = WeightInitialisationType.Xavier,
LayerTrainer = LayerTrainerType.Dropout
}, hiddenLayerSize: 512, numEpochs: 10)
);
// create the stacked training set
Console.WriteLine("Creating model stack data set...");
var modelStacker = new ModelStacker();
foreach (var item in sentimentDataSet.Classification)
{
var indexList = item.GetIndexList();
modelStacker.Add(new[] {
bernoulliClassifier.GetWeightedClassifications(indexList),
multinomialClassifier.GetWeightedClassifications(indexList),
neuralClassifier.GetWeightedClassifications(item.Vectorise(maxIndex), classificationTable)
}, item.Name);
}
// convert the stacked data to a data table and split it into training and test sets
var sentimentDataTable = modelStacker.GetTable();
var dataTableVectoriser = sentimentDataTable.GetVectoriser();
var split = sentimentDataTable.Split();
var trainingStack = lap.NN.CreateTrainingDataProvider(split.Training, dataTableVectoriser);
var testStack = lap.NN.CreateTrainingDataProvider(split.Test, dataTableVectoriser);
var targetColumnIndex = sentimentDataTable.TargetColumnIndex;
// train a neural network on the stacked data
var trainingContext = lap.NN.CreateTrainingContext(ErrorMetricType.OneHot, learningRate: 0.3f, batchSize: 8);
trainingContext.ScheduleTrainingRateChange(10, 0.1f);
var stackNN = lap.NN.CreateFeedForward(trainingContext.TrainNeuralNetwork(lap, trainingStack, testStack, new LayerDescriptor(0.1f)
{
WeightUpdate = WeightUpdateType.RMSprop,
Activation = ActivationType.LeakyRelu,
WeightInitialisation = WeightInitialisationType.Xavier
}, hiddenLayerSize: 32, numEpochs: 20));
uint stringIndex;
Console.WriteLine("Enter some text to test the classifiers...");
while (true)
{
Console.Write(">");
var line = Console.ReadLine();
if (String.IsNullOrWhiteSpace(line))
{
break;
}
var tokens = _Tokenise(line);
var indexList = new List <uint>();
foreach (var token in tokens)
{
if (stringTable.TryGetIndex(token, out stringIndex))
{
indexList.Add(stringIndex);
}
}
if (indexList.Any())
{
var queryTokens = indexList.GroupBy(d => d).Select(g => Tuple.Create(g.Key, (float)g.Count())).ToList();
var vector = new float[maxIndex];
foreach (var token in queryTokens)
{
vector[token.Item1] = token.Item2;
}
Console.WriteLine("Bernoulli classification: " + bernoulliClassifier.Classify(indexList).First());
Console.WriteLine("Multinomial classification: " + multinomialClassifier.Classify(indexList).First());
Console.WriteLine("Neural network classification: " + classificationTable[neuralClassifier.Execute(vector).MaximumIndex()]);
var stackInput = modelStacker.Vectorise(new[] {
bernoulliClassifier.GetWeightedClassifications(indexList),
multinomialClassifier.GetWeightedClassifications(indexList),
neuralClassifier.GetWeightedClassifications(vector, classificationTable)
});
Console.WriteLine("Stack classification: " + dataTableVectoriser.GetOutputLabel(targetColumnIndex, stackNN.Execute(stackInput).MaximumIndex()));
}
else
{
Console.WriteLine("Sorry, none of those words have been seen before.");
}
Console.WriteLine();
}
}
Console.WriteLine();
}
public static void ReducedMNIST(string dataFilesPath)
{
Console.Write("Loading training data...");
var trainingData = Mnist.Load(dataFilesPath + "train-labels.idx1-ubyte", dataFilesPath + "train-images.idx3-ubyte");
var testData = Mnist.Load(dataFilesPath + "t10k-labels.idx1-ubyte", dataFilesPath + "t10k-images.idx3-ubyte");
Console.WriteLine("done");
var onesAndZeroesTraining = trainingData.Where(s => s.Label == 0 || s.Label == 1).Shuffle(0).Take(1000).ToList();
var onesAndZeroesTest = testData.Where(s => s.Label == 0 || s.Label == 1).Shuffle(0).Take(100).ToList();
using (var lap = GPUProvider.CreateLinearAlgebra(false)) {
var convolutionDescriptor = new ConvolutionDescriptor(0.1f)
{
Stride = 1,
Padding = 1,
FilterDepth = 4,
FilterHeight = 3,
FilterWidth = 3,
WeightInitialisation = WeightInitialisationType.Xavier,
WeightUpdate = WeightUpdateType.RMSprop,
Activation = ActivationType.LeakyRelu
};
const int BATCH_SIZE = 128, NUM_EPOCHS = 2, IMAGE_WIDTH = 28;
const float TRAINING_RATE = 0.03f;
var errorMetric = ErrorMetricType.OneHot.Create();
var layerTemplate = new LayerDescriptor(0.1f)
{
WeightUpdate = WeightUpdateType.RMSprop,
Activation = ActivationType.LeakyRelu
};
var trainingSamples = onesAndZeroesTraining.Select(d => d.AsVolume).Select(d => Tuple.Create(d.AsTensor(lap), d.ExpectedOutput)).ToList();
var testSamples = onesAndZeroesTest.Select(d => d.AsVolume).Select(d => Tuple.Create(d.AsTensor(lap), d.ExpectedOutput)).ToList();
// create a network with a single convolutional layer followed by a max pooling layer
var convolutionalLayer = new IConvolutionalLayer [] {
lap.NN.CreateConvolutionalLayer(convolutionDescriptor, 1, IMAGE_WIDTH, false),
lap.NN.CreateMaxPoolingLayer(2, 2, 2)
};
var trainingDataProvider = lap.NN.CreateConvolutionalTrainingProvider(convolutionDescriptor, trainingSamples, convolutionalLayer, true);
var testDataProvider = lap.NN.CreateConvolutionalTrainingProvider(convolutionDescriptor, testSamples, convolutionalLayer, false);
ConvolutionalNetwork network;
using (var trainer = lap.NN.CreateBatchTrainer(layerTemplate, 784, trainingDataProvider.OutputSize)) {
var trainingContext = lap.NN.CreateTrainingContext(errorMetric, TRAINING_RATE, BATCH_SIZE);
trainingContext.EpochComplete += c => {
var output = trainer.Execute(testDataProvider.TrainingDataProvider);
var testError = output.Select(d => errorMetric.Compute(d.Output, d.ExpectedOutput)).Average();
trainingContext.WriteScore(testError, errorMetric.DisplayAsPercentage);
};
trainer.Train(trainingDataProvider.TrainingDataProvider, NUM_EPOCHS, trainingContext);
network = trainingDataProvider.GetCurrentNetwork(trainer);
}
foreach (var layer in convolutionalLayer)
{
layer.Dispose();
}
foreach (var item in trainingSamples)
{
item.Item1.Dispose();
}
foreach (var item in testSamples)
{
item.Item1.Dispose();
}
int correct = 0, total = 0;
using (var execution = lap.NN.CreateConvolutional(network)) {
foreach (var item in onesAndZeroesTest)
{
using (var tensor = item.AsVolume.AsTensor(lap)) {
using (var output = execution.Execute(tensor)) {
var maxIndex = output.MaximumIndex();
if (maxIndex == item.Label)
{
++correct;
}
++total;
}
}
}
}
Console.WriteLine($"Execution results: {(double)correct / total:P0} correct");
}
}