/// <summary> /// Trains a neural net on the MNIST database (digit recognition) /// The data files can be downloaded from http://yann.lecun.com/exdb/mnist/ /// </summary> /// <param name="dataFilesPath">The path to a directory with the four extracted data files</param> public static void MNIST(string dataFilesPath, string outputModelPath) { // neural network hyper parameters const int HIDDEN_SIZE = 1024, BATCH_SIZE = 128, NUM_EPOCHS = 40; const float TRAINING_RATE = 0.03f; var errorMetric = ErrorMetricType.OneHot.Create(); var layerTemplate = new LayerDescriptor(0f) { WeightUpdate = WeightUpdateType.RMSprop, Activation = ActivationType.LeakyRelu }; Console.Write("Loading training data..."); var trainingData = Mnist.Load(dataFilesPath + "train-labels.idx1-ubyte", dataFilesPath + "train-images.idx3-ubyte"); var testData = Mnist.Load(dataFilesPath + "t10k-labels.idx1-ubyte", dataFilesPath + "t10k-images.idx3-ubyte"); Console.WriteLine("done"); Console.WriteLine("Starting training..."); using (var lap = GPUProvider.CreateLinearAlgebra()) { var trainingSet = lap.NN.CreateTrainingDataProvider(trainingData.Select(d => d.Sample).ToList()); var testSet = lap.NN.CreateTrainingDataProvider(testData.Select(d => d.Sample).ToList()); using (var trainer = lap.NN.CreateBatchTrainer(layerTemplate, Mnist.INPUT_SIZE, HIDDEN_SIZE, Mnist.OUTPUT_SIZE)) { var trainingManager = lap.NN.CreateFeedForwardManager(trainer, outputModelPath, testSet); var trainingContext = lap.NN.CreateTrainingContext(errorMetric, TRAINING_RATE, BATCH_SIZE); trainingContext.ScheduleTrainingRateChange(NUM_EPOCHS / 2, TRAINING_RATE / 3); trainingManager.Train(trainingSet, NUM_EPOCHS, trainingContext); } } }
/// <summary> /// Classifies text into either positive or negative sentiment /// The data files can be downloaded from https://archive.ics.uci.edu/ml/datasets/Sentiment+Labelled+Sentences /// </summary> /// <param name="dataFilesPath">Path to extracted data files</param> public static void SentimentClassification(string dataFilesPath) { var files = new[] { "amazon_cells_labelled.txt", "imdb_labelled.txt", "yelp_labelled.txt" }; var LINE_SEPARATOR = "\n".ToCharArray(); var SEPARATOR = "\t".ToCharArray(); var stringTable = new StringTableBuilder(); var sentimentData = files.SelectMany(f => File.ReadAllText(dataFilesPath + f) .Split(LINE_SEPARATOR) .Where(l => !String.IsNullOrWhiteSpace(l)) .Select(l => l.Split(SEPARATOR)) .Select(s => Tuple.Create(_Tokenise(s[0]), s[1][0] == '1' ? "positive" : "negative")) .Where(d => d.Item1.Any()) ).Shuffle(0).ToList(); var splitSentimentData = sentimentData.Split(); // build training and test classification bag var trainingClassificationBag = _BuildClassificationBag(splitSentimentData.Training, stringTable); var testClassificationBag = _BuildClassificationBag(splitSentimentData.Test, stringTable); // train a bernoulli naive bayes classifier var bernoulli = trainingClassificationBag.TrainBernoulliNaiveBayes(); Console.WriteLine("Bernoulli accuracy: {0:P}", testClassificationBag .Classify(bernoulli.CreateClassifier()) .Average(r => r.Score) ); // train a multinomial naive bayes classifier var multinomial = trainingClassificationBag.TrainMultinomialNaiveBayes(); Console.WriteLine("Multinomial accuracy: {0:P}", testClassificationBag .Classify(multinomial.CreateClassifier()) .Average(r => r.Score) ); // convert the bags to sparse vectors var sentimentDataBag = _BuildClassificationBag(sentimentData, stringTable); var sentimentDataSet = sentimentDataBag.ConvertToSparseVectors(false); var sentimentDataTableSplit = sentimentDataSet.Split(); using (var lap = GPUProvider.CreateLinearAlgebra(false)) { var maxIndex = sentimentDataSet.GetMaximumIndex() + 1; var trainingData = sentimentDataTableSplit.Training.CreateTrainingDataProvider(lap, maxIndex); var testData = sentimentDataTableSplit.Test.CreateTrainingDataProvider(lap, maxIndex); var classificationTable = sentimentDataSet.GetClassifications().ToDictionary(d => (int)d.Value, d => d.Key); // create the three classifiers var bernoulliClassifier = bernoulli.CreateClassifier(); var multinomialClassifier = multinomial.CreateClassifier(); var neuralClassifier = lap.NN.CreateFeedForward(lap.NN.CreateTrainingContext(ErrorMetricType.OneHot, learningRate: 0.1f, batchSize: 128) .TrainNeuralNetwork(lap, trainingData, testData, new LayerDescriptor(0.1f) { WeightUpdate = WeightUpdateType.Adam, Activation = ActivationType.Relu, WeightInitialisation = WeightInitialisationType.Xavier, LayerTrainer = LayerTrainerType.Dropout }, hiddenLayerSize: 512, numEpochs: 10) ); // create the stacked training set Console.WriteLine("Creating model stack data set..."); var modelStacker = new ModelStacker(); foreach (var item in sentimentDataSet.Classification) { var indexList = item.GetIndexList(); modelStacker.Add(new[] { bernoulliClassifier.GetWeightedClassifications(indexList), multinomialClassifier.GetWeightedClassifications(indexList), neuralClassifier.GetWeightedClassifications(item.Vectorise(maxIndex), classificationTable) }, item.Name); } // convert the stacked data to a data table and split it into training and test sets var sentimentDataTable = modelStacker.GetTable(); var dataTableVectoriser = sentimentDataTable.GetVectoriser(); var split = sentimentDataTable.Split(); var trainingStack = lap.NN.CreateTrainingDataProvider(split.Training, dataTableVectoriser); var testStack = lap.NN.CreateTrainingDataProvider(split.Test, dataTableVectoriser); var targetColumnIndex = sentimentDataTable.TargetColumnIndex; // train a neural network on the stacked data var trainingContext = lap.NN.CreateTrainingContext(ErrorMetricType.OneHot, learningRate: 0.3f, batchSize: 8); trainingContext.ScheduleTrainingRateChange(10, 0.1f); var stackNN = lap.NN.CreateFeedForward(trainingContext.TrainNeuralNetwork(lap, trainingStack, testStack, new LayerDescriptor(0.1f) { WeightUpdate = WeightUpdateType.RMSprop, Activation = ActivationType.LeakyRelu, WeightInitialisation = WeightInitialisationType.Xavier }, hiddenLayerSize: 32, numEpochs: 20)); uint stringIndex; Console.WriteLine("Enter some text to test the classifiers..."); while (true) { Console.Write(">"); var line = Console.ReadLine(); if (String.IsNullOrWhiteSpace(line)) { break; } var tokens = _Tokenise(line); var indexList = new List <uint>(); foreach (var token in tokens) { if (stringTable.TryGetIndex(token, out stringIndex)) { indexList.Add(stringIndex); } } if (indexList.Any()) { var queryTokens = indexList.GroupBy(d => d).Select(g => Tuple.Create(g.Key, (float)g.Count())).ToList(); var vector = new float[maxIndex]; foreach (var token in queryTokens) { vector[token.Item1] = token.Item2; } Console.WriteLine("Bernoulli classification: " + bernoulliClassifier.Classify(indexList).First()); Console.WriteLine("Multinomial classification: " + multinomialClassifier.Classify(indexList).First()); Console.WriteLine("Neural network classification: " + classificationTable[neuralClassifier.Execute(vector).MaximumIndex()]); var stackInput = modelStacker.Vectorise(new[] { bernoulliClassifier.GetWeightedClassifications(indexList), multinomialClassifier.GetWeightedClassifications(indexList), neuralClassifier.GetWeightedClassifications(vector, classificationTable) }); Console.WriteLine("Stack classification: " + dataTableVectoriser.GetOutputLabel(targetColumnIndex, stackNN.Execute(stackInput).MaximumIndex())); } else { Console.WriteLine("Sorry, none of those words have been seen before."); } Console.WriteLine(); } } Console.WriteLine(); }
public static void ReducedMNIST(string dataFilesPath) { Console.Write("Loading training data..."); var trainingData = Mnist.Load(dataFilesPath + "train-labels.idx1-ubyte", dataFilesPath + "train-images.idx3-ubyte"); var testData = Mnist.Load(dataFilesPath + "t10k-labels.idx1-ubyte", dataFilesPath + "t10k-images.idx3-ubyte"); Console.WriteLine("done"); var onesAndZeroesTraining = trainingData.Where(s => s.Label == 0 || s.Label == 1).Shuffle(0).Take(1000).ToList(); var onesAndZeroesTest = testData.Where(s => s.Label == 0 || s.Label == 1).Shuffle(0).Take(100).ToList(); using (var lap = GPUProvider.CreateLinearAlgebra(false)) { var convolutionDescriptor = new ConvolutionDescriptor(0.1f) { Stride = 1, Padding = 1, FilterDepth = 4, FilterHeight = 3, FilterWidth = 3, WeightInitialisation = WeightInitialisationType.Xavier, WeightUpdate = WeightUpdateType.RMSprop, Activation = ActivationType.LeakyRelu }; const int BATCH_SIZE = 128, NUM_EPOCHS = 2, IMAGE_WIDTH = 28; const float TRAINING_RATE = 0.03f; var errorMetric = ErrorMetricType.OneHot.Create(); var layerTemplate = new LayerDescriptor(0.1f) { WeightUpdate = WeightUpdateType.RMSprop, Activation = ActivationType.LeakyRelu }; var trainingSamples = onesAndZeroesTraining.Select(d => d.AsVolume).Select(d => Tuple.Create(d.AsTensor(lap), d.ExpectedOutput)).ToList(); var testSamples = onesAndZeroesTest.Select(d => d.AsVolume).Select(d => Tuple.Create(d.AsTensor(lap), d.ExpectedOutput)).ToList(); // create a network with a single convolutional layer followed by a max pooling layer var convolutionalLayer = new IConvolutionalLayer [] { lap.NN.CreateConvolutionalLayer(convolutionDescriptor, 1, IMAGE_WIDTH, false), lap.NN.CreateMaxPoolingLayer(2, 2, 2) }; var trainingDataProvider = lap.NN.CreateConvolutionalTrainingProvider(convolutionDescriptor, trainingSamples, convolutionalLayer, true); var testDataProvider = lap.NN.CreateConvolutionalTrainingProvider(convolutionDescriptor, testSamples, convolutionalLayer, false); ConvolutionalNetwork network; using (var trainer = lap.NN.CreateBatchTrainer(layerTemplate, 784, trainingDataProvider.OutputSize)) { var trainingContext = lap.NN.CreateTrainingContext(errorMetric, TRAINING_RATE, BATCH_SIZE); trainingContext.EpochComplete += c => { var output = trainer.Execute(testDataProvider.TrainingDataProvider); var testError = output.Select(d => errorMetric.Compute(d.Output, d.ExpectedOutput)).Average(); trainingContext.WriteScore(testError, errorMetric.DisplayAsPercentage); }; trainer.Train(trainingDataProvider.TrainingDataProvider, NUM_EPOCHS, trainingContext); network = trainingDataProvider.GetCurrentNetwork(trainer); } foreach (var layer in convolutionalLayer) { layer.Dispose(); } foreach (var item in trainingSamples) { item.Item1.Dispose(); } foreach (var item in testSamples) { item.Item1.Dispose(); } int correct = 0, total = 0; using (var execution = lap.NN.CreateConvolutional(network)) { foreach (var item in onesAndZeroesTest) { using (var tensor = item.AsVolume.AsTensor(lap)) { using (var output = execution.Execute(tensor)) { var maxIndex = output.MaximumIndex(); if (maxIndex == item.Label) { ++correct; } ++total; } } } } Console.WriteLine($"Execution results: {(double)correct / total:P0} correct"); } }