public void MemoryLayerTest()
{
#if DEBUG
using var context = BrightWireGpuProvider.CreateLinearAlgebra();
var matrix = context.CreateZeroMatrix(10, 10);
context.PushLayer();
var matrix2 = context.CreateZeroMatrix(10, 10);
context.PopLayer();
Assert.IsFalse(matrix2.IsValid);
Assert.IsTrue(matrix.IsValid);
matrix.Dispose();
Assert.IsFalse(matrix.IsValid);
#endif
}
static void MNISTConvolutional(string dataFilesPath)
{
using (var lap = BrightWireGpuProvider.CreateLinearAlgebra()) {
var graph = new GraphFactory(lap);
Console.Write("Loading training data...");
var trainingData = _BuildTensors(graph, null, Mnist.Load(dataFilesPath + "train-labels.idx1-ubyte", dataFilesPath + "train-images.idx3-ubyte"));
var testData = _BuildTensors(graph, trainingData, Mnist.Load(dataFilesPath + "t10k-labels.idx1-ubyte", dataFilesPath + "t10k-images.idx3-ubyte"));
Console.WriteLine($"done - {trainingData.RowCount} training images and {testData.RowCount} test images loaded");
var errorMetric = graph.ErrorMetric.OneHotEncoding;
var propertySet = graph.CurrentPropertySet
.Use(graph.GradientDescent.RmsProp)
.Use(graph.WeightInitialisation.Xavier)
;
// create the network
const int HIDDEN_LAYER_SIZE = 1024;
const float LEARNING_RATE = 0.05f;
var engine = graph.CreateTrainingEngine(trainingData, LEARNING_RATE, 32);
engine.LearningContext.ScheduleLearningRate(15, LEARNING_RATE / 3);
graph.Connect(engine)
.AddConvolutional(8, 2, 5, 5, 1, false)
.Add(graph.ReluActivation())
.AddDropOut(0.5f)
.AddMaxPooling(2, 2, 2)
.AddConvolutional(16, 2, 5, 5, 1)
.Add(graph.ReluActivation())
.AddMaxPooling(2, 2, 2)
.Transpose()
.AddFeedForward(HIDDEN_LAYER_SIZE)
.Add(graph.ReluActivation())
.AddDropOut(0.5f)
.AddFeedForward(trainingData.OutputSize)
.Add(graph.SoftMaxActivation())
.AddBackpropagation(errorMetric)
;
engine.Train(20, testData, errorMetric);
// export the graph and verify that the error is the same
var networkGraph = engine.Graph;
var executionEngine = graph.CreateEngine(networkGraph);
var output = executionEngine.Execute(testData);
Console.WriteLine(output.Average(o => o.CalculateError(errorMetric)));
}
}
/// <summary>
/// Trains a feed forward neural net on the MNIST data set (handwritten digit recognition)
/// The data files can be downloaded from http://yann.lecun.com/exdb/mnist/
/// </summary>
/// <param name="dataFilesPath">The path to a directory with the four extracted data files</param>
public static void MNIST(string dataFilesPath)
{
using (var lap = BrightWireGpuProvider.CreateLinearAlgebra()) {
var graph = new GraphFactory(lap);
Console.Write("Loading training data...");
var trainingData = _BuildVectors(null, graph, Mnist.Load(dataFilesPath + "train-labels.idx1-ubyte", dataFilesPath + "train-images.idx3-ubyte"));
var testData = _BuildVectors(trainingData, graph, Mnist.Load(dataFilesPath + "t10k-labels.idx1-ubyte", dataFilesPath + "t10k-images.idx3-ubyte"));
Console.WriteLine($"done - {trainingData.RowCount} training images and {testData.RowCount} test images loaded");
// one hot encoding uses the index of the output vector's maximum value as the classification label
var errorMetric = graph.ErrorMetric.OneHotEncoding;
// configure the network properties
graph.CurrentPropertySet
.Use(graph.GradientDescent.RmsProp)
.Use(graph.WeightInitialisation.Xavier)
;
// create the training engine and schedule a training rate change
const float TRAINING_RATE = 0.1f;
var engine = graph.CreateTrainingEngine(trainingData, TRAINING_RATE, 128);
engine.LearningContext.ScheduleLearningRate(15, TRAINING_RATE / 3);
// create the network
graph.Connect(engine)
.AddFeedForward(outputSize: 1024)
.Add(graph.LeakyReluActivation())
.AddDropOut(dropOutPercentage: 0.5f)
.AddFeedForward(outputSize: trainingData.OutputSize)
.Add(graph.SigmoidActivation())
.AddBackpropagation(errorMetric)
;
// train the network for twenty iterations, saving the model on each improvement
Models.ExecutionGraph bestGraph = null;
engine.Train(20, testData, errorMetric, model => bestGraph = model.Graph);
// export the final model and execute it on the training set
var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph);
var output = executionEngine.Execute(testData);
Console.WriteLine($"Final accuracy: {output.Average(o => o.CalculateError(errorMetric)):P2}");
}
}
/// <summary>
/// Trains a neural net on the MNIST database (digit recognition)
/// The data files can be downloaded from http://yann.lecun.com/exdb/mnist/
/// </summary>
/// <param name="dataFilesPath">The path to a directory with the four extracted data files</param>
public static void MNIST(string dataFilesPath)
{
using (var lap = BrightWireGpuProvider.CreateLinearAlgebra()) {
var graph = new GraphFactory(lap);
Console.Write("Loading training data...");
var trainingData = _BuildVectors(null, graph, Mnist.Load(dataFilesPath + "train-labels.idx1-ubyte", dataFilesPath + "train-images.idx3-ubyte"));
var testData = _BuildVectors(trainingData, graph, Mnist.Load(dataFilesPath + "t10k-labels.idx1-ubyte", dataFilesPath + "t10k-images.idx3-ubyte"));
Console.WriteLine($"done - {trainingData.RowCount} training images and {testData.RowCount} test images loaded");
// use a one hot encoding error metric, rmsprop gradient descent and xavier weight initialisation
var errorMetric = graph.ErrorMetric.OneHotEncoding;
var propertySet = graph.CurrentPropertySet
//.Use(graph.Regularisation.L2)
.Use(graph.GradientDescent.RmsProp)
.Use(graph.WeightInitialisation.Xavier)
;
// create the training engine and schedule a training rate change
const float TRAINING_RATE = 0.1f;
var engine = graph.CreateTrainingEngine(trainingData, TRAINING_RATE, 128);
engine.LearningContext.ScheduleLearningRate(15, TRAINING_RATE / 3);
// create the network
graph.Connect(engine)
.AddFeedForward(1024)
.Add(graph.LeakyReluActivation())
.AddDropOut(0.5f)
.AddFeedForward(trainingData.OutputSize)
.Add(graph.SigmoidActivation())
.AddBackpropagation(errorMetric)
;
// train the network
engine.Train(20, testData, errorMetric);
// export the graph and verify that the error is the same
var networkGraph = engine.Graph;
var executionEngine = graph.CreateEngine(networkGraph);
var output = executionEngine.Execute(testData);
Console.WriteLine(output.Average(o => o.CalculateError(errorMetric)));
}
}
public static void TrainWithSelu(string dataFilesPath)
{
using (var lap = BrightWireGpuProvider.CreateLinearAlgebra()) {
var graph = new GraphFactory(lap);
// parse the iris CSV into a data table and normalise
var dataTable = new StreamReader(new MemoryStream(File.ReadAllBytes(dataFilesPath))).ParseCSV(',').Normalise(NormalisationType.Standard);
// split the data table into training and test tables
var split = dataTable.Split(0);
var trainingData = graph.CreateDataSource(split.Training);
var testData = graph.CreateDataSource(split.Test);
// one hot encoding uses the index of the output vector's maximum value as the classification label
var errorMetric = graph.ErrorMetric.OneHotEncoding;
// configure the network properties
graph.CurrentPropertySet
.Use(graph.GradientDescent.RmsProp)
.Use(graph.GaussianWeightInitialisation(true, 0.1f, GaussianVarianceCalibration.SquareRoot2N, GaussianVarianceCount.FanInFanOut))
;
// create the training engine and schedule a training rate change
const float TRAINING_RATE = 0.01f;
var engine = graph.CreateTrainingEngine(trainingData, TRAINING_RATE, batchSize: 128);
// create the network with the custom activation function
graph.Connect(engine)
.AddFeedForward(outputSize: 32)
.Add(new SeluActivation())
.AddFeedForward(trainingData.OutputSize)
.Add(graph.SigmoidActivation())
.AddBackpropagation(errorMetric)
;
// train the network, but only run the test set every 50 iterations
const int TRAINING_ITERATIONS = 1000;
engine.Train(TRAINING_ITERATIONS, testData, errorMetric, null, testCadence: 50);
}
}
public static void TrainWithSelu(string dataFilesPath)
{
using (var lap = BrightWireGpuProvider.CreateLinearAlgebra()) {
var graph = new GraphFactory(lap);
// parse the iris CSV into a data table and normalise
var dataTable = new StreamReader(new MemoryStream(File.ReadAllBytes(dataFilesPath))).ParseCSV(',').Normalise(NormalisationType.Standard);
// split the data table into training and test tables
var split = dataTable.Split(0);
var trainingData = graph.CreateDataSource(split.Training);
var testData = graph.CreateDataSource(split.Test);
// use a one hot encoding error metric, rmsprop gradient descent and xavier weight initialisation
var errorMetric = graph.ErrorMetric.OneHotEncoding;
var propertySet = graph.CurrentPropertySet
.Use(graph.GradientDescent.RmsProp)
.Use(graph.GaussianWeightInitialisation(true, 0.1f, GaussianVarianceCalibration.SquareRoot2N, GaussianVarianceCount.FanInFanOut))
;
// create the training engine and schedule a training rate change
const float TRAINING_RATE = 0.01f;
var engine = graph.CreateTrainingEngine(trainingData, TRAINING_RATE, 128);
// create the network
graph.Connect(engine)
.AddFeedForward(32)
.Add(new SeluActivation())
.AddFeedForward(trainingData.OutputSize)
.Add(graph.SigmoidActivation())
.AddBackpropagation(errorMetric)
;
// train the network
engine.Train(1000, testData, errorMetric, null, 50);
}
}
public static void Load(TestContext context)
{
_cuda = BrightWireGpuProvider.CreateLinearAlgebra(false);
_cpu = BrightWireProvider.CreateLinearAlgebra(false);
}
private static void Load()
{
_cuda = BrightWireGpuProvider.CreateLinearAlgebra(false);
_cpu = BrightWireProvider.CreateLinearAlgebra(false);
}
/// <summary>
/// Classifies text into either positive or negative sentiment
/// The data files can be downloaded from https://archive.ics.uci.edu/ml/datasets/Sentiment+Labelled+Sentences
/// </summary>
/// <param name="dataFilesPath">Path to extracted data files</param>
public static void SentimentClassification(string dataFilesPath)
{
var files = new[] {
"amazon_cells_labelled.txt",
"imdb_labelled.txt",
"yelp_labelled.txt"
};
var LINE_SEPARATOR = "\n".ToCharArray();
var SEPARATOR = "\t".ToCharArray();
var stringTable = new StringTableBuilder();
var sentimentData = files.SelectMany(f => File.ReadAllText(dataFilesPath + f)
.Split(LINE_SEPARATOR)
.Where(l => !String.IsNullOrWhiteSpace(l))
.Select(l => l.Split(SEPARATOR))
.Select(s => Tuple.Create(_Tokenise(s[0]), s[1][0] == '1' ? "positive" : "negative"))
.Where(d => d.Item1.Any())
).Shuffle(0).ToList();
var splitSentimentData = sentimentData.Split();
// build training and test classification bag
var trainingClassificationBag = _BuildIndexedClassifications(splitSentimentData.Training, stringTable);
var testClassificationBag = _BuildIndexedClassifications(splitSentimentData.Test, stringTable);
// train a bernoulli naive bayes classifier
var bernoulli = trainingClassificationBag.TrainBernoulliNaiveBayes();
Console.WriteLine("Bernoulli accuracy: {0:P}", testClassificationBag
.Classify(bernoulli.CreateClassifier())
.Average(r => r.Score)
);
// train a multinomial naive bayes classifier
var multinomial = trainingClassificationBag.TrainMultinomialNaiveBayes();
Console.WriteLine("Multinomial accuracy: {0:P}", testClassificationBag
.Classify(multinomial.CreateClassifier())
.Average(r => r.Score)
);
// convert the index lists to vectors and normalise along the way
var sentimentDataBag = _BuildIndexedClassifications(sentimentData, stringTable);
var sentimentDataTable = sentimentDataBag.ConvertToTable();
var normalisedDataTable = sentimentDataTable.Normalise(NormalisationType.Standard);
var vectoriser = normalisedDataTable.GetVectoriser();
var sentimentDataSet = normalisedDataTable.Split(0);
var dataTableAnalysis = normalisedDataTable.GetAnalysis();
using (var lap = BrightWireGpuProvider.CreateLinearAlgebra()) {
var graph = new GraphFactory(lap);
var trainingData = graph.CreateDataSource(sentimentDataSet.Training, vectoriser);
var testData = graph.CreateDataSource(sentimentDataSet.Test, vectoriser);
var indexListEncoder = (IIndexListEncoder)trainingData;
// use a one hot encoding error metric, rmsprop gradient descent and xavier weight initialisation
var errorMetric = graph.ErrorMetric.OneHotEncoding;
var propertySet = graph.CurrentPropertySet
.Use(graph.GradientDescent.RmsProp)
.Use(graph.WeightInitialisation.Xavier)
;
var engine = graph.CreateTrainingEngine(trainingData, 0.3f, 128);
engine.LearningContext.ScheduleLearningRate(5, 0.1f);
engine.LearningContext.ScheduleLearningRate(11, 1f);
engine.LearningContext.ScheduleLearningRate(15, 0.3f);
// train a neural network classifier
var neuralNetworkWire = graph.Connect(engine)
.AddFeedForward(512, "layer1")
//.AddBatchNormalisation()
.Add(graph.ReluActivation())
.AddDropOut(0.5f)
.AddFeedForward(trainingData.OutputSize, "layer2")
.Add(graph.ReluActivation())
.AddBackpropagation(errorMetric, "first-network")
;
// train the network
Console.WriteLine("Training neural network classifier...");
const int TRAINING_ITERATIONS = 10;
GraphModel bestNetwork = null;
engine.Train(TRAINING_ITERATIONS, testData, errorMetric, network => bestNetwork = network);
if (bestNetwork != null)
{
engine.LoadParametersFrom(bestNetwork.Graph);
}
var firstClassifier = graph.CreateEngine(engine.Graph);
// stop the backpropagation to the first neural network
engine.LearningContext.EnableNodeUpdates(neuralNetworkWire.Find("layer1"), false);
engine.LearningContext.EnableNodeUpdates(neuralNetworkWire.Find("layer2"), false);
// create the bernoulli classifier wire
var bernoulliClassifier = bernoulli.CreateClassifier();
var bernoulliWire = graph.Connect(engine)
.AddClassifier(bernoulliClassifier, sentimentDataSet.Training, dataTableAnalysis)
;
// create the multinomial classifier wire
var multinomialClassifier = multinomial.CreateClassifier();
var multinomialWire = graph.Connect(engine)
.AddClassifier(multinomialClassifier, sentimentDataSet.Training, dataTableAnalysis)
;
// join the bernoulli, multinomial and neural network classification outputs
var firstNetwork = neuralNetworkWire.Find("first-network");
var joined = graph.Join(multinomialWire, graph.Join(bernoulliWire, graph.Connect(trainingData.OutputSize, firstNetwork)));
// train an additional classifier on the output of the previous three classifiers
joined
.AddFeedForward(outputSize: 64)
//.AddBatchNormalisation()
.Add(graph.ReluActivation())
.AddDropOut(dropOutPercentage: 0.5f)
.AddFeedForward(trainingData.OutputSize)
//.AddBatchNormalisation()
.Add(graph.ReluActivation())
.AddBackpropagation(errorMetric)
;
// train the network again
Console.WriteLine("Training stacked neural network classifier...");
GraphModel bestStackedNetwork = null;
engine.Train(10, testData, errorMetric, network => bestStackedNetwork = network);
if (bestStackedNetwork != null)
{
engine.LoadParametersFrom(bestStackedNetwork.Graph);
}
Console.WriteLine("Enter some text to test the classifiers...");
while (true)
{
Console.Write(">");
var line = Console.ReadLine();
if (String.IsNullOrWhiteSpace(line))
{
break;
}
var tokens = _Tokenise(line);
var indexList = new List <uint>();
foreach (var token in tokens)
{
if (stringTable.TryGetIndex(token, out uint stringIndex))
{
indexList.Add(stringIndex);
}
}
if (indexList.Any())
{
var queryTokens = indexList.GroupBy(d => d).Select(g => Tuple.Create(g.Key, (float)g.Count())).ToList();
var vector = new float[trainingData.InputSize];
foreach (var token in queryTokens)
{
vector[token.Item1] = token.Item2;
}
var indexList2 = IndexList.Create(indexList.ToArray());
var encodedInput = indexListEncoder.Encode(indexList2);
Console.WriteLine("Bernoulli classification: " + bernoulliClassifier.Classify(indexList2).First().Label);
Console.WriteLine("Multinomial classification: " + multinomialClassifier.Classify(indexList2).First().Label);
var result = firstClassifier.Execute(encodedInput);
var classification = vectoriser.GetOutputLabel(1, (result.Output[0].Data[0] > result.Output[0].Data[1]) ? 0 : 1);
Console.WriteLine("Neural network classification: " + classification);
var stackedResult = engine.Execute(encodedInput);
var stackedClassification = vectoriser.GetOutputLabel(1, (stackedResult.Output[0].Data[0] > stackedResult.Output[0].Data[1]) ? 0 : 1);
Console.WriteLine("Stack classification: " + stackedClassification);
}
else
{
Console.WriteLine("Sorry, none of those words have been seen before.");
}
Console.WriteLine();
}
}
Console.WriteLine();
}
/// <summary>
/// Trains a feed forward neural net on the MNIST data set (handwritten digit recognition)
/// The data files can be downloaded from http://yann.lecun.com/exdb/mnist/
/// </summary>
/// <param name="dataFilesPath">The path to a directory with the four extracted data files</param>
/// <param name="outputModelPath">Optional path to save the best model to</param>
static void MNISTConvolutional(string dataFilesPath, string outputModelPath = null)
{
using var lap = BrightWireGpuProvider.CreateLinearAlgebra();
var graph = new GraphFactory(lap);
Console.Write("Loading training data...");
var mnistTraining = Mnist.Load(dataFilesPath + "train-labels.idx1-ubyte",
dataFilesPath + "train-images.idx3-ubyte");
var mnistTest = Mnist.Load(dataFilesPath + "t10k-labels.idx1-ubyte",
dataFilesPath + "t10k-images.idx3-ubyte");
var trainingData =
_BuildTensors(graph, null, mnistTraining /*.Where(d => d.Label < 2).ToList()*/);
var testData = _BuildTensors(graph, trainingData,
mnistTest /*.Where(d => d.Label < 2).ToList()*/);
Console.WriteLine(
$"done - {trainingData.RowCount} training images and {testData.RowCount} test images loaded");
// one hot encoding uses the index of the output vector's maximum value as the classification label
var errorMetric = graph.ErrorMetric.OneHotEncoding;
// configure the network properties
graph.CurrentPropertySet.Use(graph.GradientDescent.Adam).Use(
graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N));
// create the network
const int HIDDEN_LAYER_SIZE = 1024, TRAINING_ITERATIONS = 20;
const float LEARNING_RATE = 0.05f;
var engine = graph.CreateTrainingEngine(trainingData, LEARNING_RATE);
if (!string.IsNullOrWhiteSpace(outputModelPath) && File.Exists(outputModelPath))
{
Console.WriteLine("Loading existing model from: " + outputModelPath);
using var file = new FileStream(outputModelPath, FileMode.Open, FileAccess.Read);
var model = Serializer.Deserialize <GraphModel>(file);
engine = graph.CreateTrainingEngine(trainingData, model.Graph, LEARNING_RATE);
}
else
{
graph.Connect(engine).
AddConvolutional(filterCount: 16, padding: 2, filterWidth: 5, filterHeight: 5, xStride: 1,
yStride: 1, shouldBackpropagate: false).Add(graph.LeakyReluActivation()).
AddMaxPooling(filterWidth: 2, filterHeight: 2, xStride: 2, yStride: 2).
AddConvolutional(filterCount: 32, padding: 2, filterWidth: 5, filterHeight: 5, xStride: 1,
yStride: 1).Add(graph.LeakyReluActivation()).
AddMaxPooling(filterWidth: 2, filterHeight: 2, xStride: 2, yStride: 2).Transpose().
AddFeedForward(HIDDEN_LAYER_SIZE).Add(graph.LeakyReluActivation()).
AddDropOut(dropOutPercentage: 0.5f).AddFeedForward(trainingData.OutputSize).
Add(graph.SoftMaxActivation()).AddBackpropagation(errorMetric);
}
// lower the learning rate over time
engine.LearningContext.ScheduleLearningRate(15, LEARNING_RATE / 2);
// train the network for twenty iterations, saving the model on each improvement
Models.ExecutionGraph bestGraph = null;
engine.Train(TRAINING_ITERATIONS, testData, errorMetric, model =>
{
bestGraph = model.Graph;
if (!string.IsNullOrWhiteSpace(outputModelPath))
{
using var file = new FileStream(outputModelPath, FileMode.Create, FileAccess.Write);
Serializer.Serialize(file, model);
}
});
// export the final model and execute it on the training set
var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph);
var output = executionEngine.Execute(testData);
Console.WriteLine($"Final accuracy: {output.Average(o => o.CalculateError(errorMetric)):P2}");
// execute the model with a single image
var tensor = mnistTest.First().AsFloatTensor.Tensor;
var singleData = graph.CreateDataSource(new[] { tensor });
var result = executionEngine.Execute(singleData);
var prediction = result.Single().Output.Single().MaximumIndex();
}