/// <summary>
/// Creates a graph training engine
/// </summary>
/// <param name="dataSource">Data source with training data</param>
/// <param name="graph">The serialised graph to execute</param>
/// <param name="trainingRate">Initial learning rate</param>
/// <param name="batchSize">Mini batch size</param>
/// <param name="trainingErrorCalculation">How to calculate the training error</param>
/// <returns></returns>
public IGraphTrainingEngine CreateTrainingEngine(IDataSource dataSource, Models.ExecutionGraph graph, float trainingRate = 0.1f, int batchSize = 128, TrainingErrorCalculation trainingErrorCalculation = TrainingErrorCalculation.Fast)
{
var learningContext = new LearningContext(_lap, trainingRate, batchSize, trainingErrorCalculation, dataSource.IsSequential);
var input = this.CreateFrom(graph);
return(new TrainingEngine(_lap, dataSource, learningContext, input));
}
public void TestRecurrent()
{
var data = BinaryIntegers.Addition(100, false).Split(0);
var graph = new GraphFactory(_lap);
var errorMetric = graph.ErrorMetric.BinaryClassification;
graph.CurrentPropertySet.Use(graph.GradientDescent.Adam).Use(
graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N));
// create the engine
var trainingData = graph.CreateDataSource(data.Training);
var testData = trainingData.CloneWith(data.Test);
var engine = graph.CreateTrainingEngine(trainingData, learningRate: 0.01f, batchSize: 16);
// build the network
const int HIDDEN_LAYER_SIZE = 32, TRAINING_ITERATIONS = 5;
var memory = new float[HIDDEN_LAYER_SIZE];
var network = graph.Connect(engine).AddSimpleRecurrent(graph.ReluActivation(), memory).
AddFeedForward(engine.DataSource.OutputSize).Add(graph.ReluActivation()).
AddBackpropagationThroughTime(errorMetric);
// train the network for twenty iterations, saving the model on each improvement
Models.ExecutionGraph bestGraph = null;
engine.Train(TRAINING_ITERATIONS, testData, errorMetric, bn => bestGraph = bn.Graph);
// export the graph and verify it against some unseen integers on the best model
var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph);
var testData2 = graph.CreateDataSource(BinaryIntegers.Addition(8, true));
var results = executionEngine.Execute(testData2);
}
public void DeserialiseExecutionGraph()
{
var(graph, data) = MakeGraphAndData();
Models.ExecutionGraph executionGraphReloaded = null;
using (var file = new MemoryStream())
{
Serializer.Serialize(file, bestNetwork.Graph);
file.Position = 0;
executionGraphReloaded = Serializer.Deserialize <Models.ExecutionGraph>(file);
}
Assert.IsNotNull(executionGraphReloaded);
var engine = graph.CreateEngine(executionGraphReloaded);
AssertEngineGetsGoodResults(engine, data);
}
/// <summary>
/// Trains a feed forward neural net on the MNIST data set (handwritten digit recognition)
/// The data files can be downloaded from http://yann.lecun.com/exdb/mnist/
/// </summary>
/// <param name="dataFilesPath">The path to a directory with the four extracted data files</param>
public static void MNIST(string dataFilesPath)
{
using (var lap = BrightWireGpuProvider.CreateLinearAlgebra()) {
var graph = new GraphFactory(lap);
Console.Write("Loading training data...");
var trainingData = _BuildVectors(null, graph, Mnist.Load(dataFilesPath + "train-labels.idx1-ubyte", dataFilesPath + "train-images.idx3-ubyte"));
var testData = _BuildVectors(trainingData, graph, Mnist.Load(dataFilesPath + "t10k-labels.idx1-ubyte", dataFilesPath + "t10k-images.idx3-ubyte"));
Console.WriteLine($"done - {trainingData.RowCount} training images and {testData.RowCount} test images loaded");
// one hot encoding uses the index of the output vector's maximum value as the classification label
var errorMetric = graph.ErrorMetric.OneHotEncoding;
// configure the network properties
graph.CurrentPropertySet
.Use(graph.GradientDescent.RmsProp)
.Use(graph.WeightInitialisation.Xavier)
;
// create the training engine and schedule a training rate change
const float TRAINING_RATE = 0.1f;
var engine = graph.CreateTrainingEngine(trainingData, TRAINING_RATE, 128);
engine.LearningContext.ScheduleLearningRate(15, TRAINING_RATE / 3);
// create the network
graph.Connect(engine)
.AddFeedForward(outputSize: 1024)
.Add(graph.LeakyReluActivation())
.AddDropOut(dropOutPercentage: 0.5f)
.AddFeedForward(outputSize: trainingData.OutputSize)
.Add(graph.SigmoidActivation())
.AddBackpropagation(errorMetric)
;
// train the network for twenty iterations, saving the model on each improvement
Models.ExecutionGraph bestGraph = null;
engine.Train(20, testData, errorMetric, model => bestGraph = model.Graph);
// export the final model and execute it on the training set
var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph);
var output = executionEngine.Execute(testData);
Console.WriteLine($"Final accuracy: {output.Average(o => o.CalculateError(errorMetric)):P2}");
}
}
public static void IntegerAddition()
{
// generate 1000 random integer additions (split into training and test sets)
var data = BinaryIntegers.Addition(1000, false).Split(0);
using (var lap = BrightWireProvider.CreateLinearAlgebra(false)) {
var graph = new GraphFactory(lap);
// binary classification rounds each output to either 0 or 1
var errorMetric = graph.ErrorMetric.BinaryClassification;
// configure the network properties
graph.CurrentPropertySet
.Use(graph.GradientDescent.Adam)
.Use(graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N))
;
// create the engine
var trainingData = graph.CreateDataSource(data.Training);
var testData = trainingData.CloneWith(data.Test);
var engine = graph.CreateTrainingEngine(trainingData, learningRate: 0.01f, batchSize: 16);
// build the network
const int HIDDEN_LAYER_SIZE = 32, TRAINING_ITERATIONS = 30;
var memory = new float[HIDDEN_LAYER_SIZE];
var network = graph.Connect(engine)
.AddSimpleRecurrent(graph.ReluActivation(), memory)
.AddFeedForward(engine.DataSource.OutputSize)
.Add(graph.ReluActivation())
.AddBackpropagationThroughTime(errorMetric)
;
// train the network for twenty iterations, saving the model on each improvement
Models.ExecutionGraph bestGraph = null;
engine.Train(TRAINING_ITERATIONS, testData, errorMetric, bn => bestGraph = bn.Graph);
// export the graph and verify it against some unseen integers on the best model
var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph);
var testData2 = graph.CreateDataSource(BinaryIntegers.Addition(8, true));
var results = executionEngine.Execute(testData2);
// group the output
var groupedResults = new (FloatVector[] Input, FloatVector[] Target, FloatVector[] Output)[8];
/// <summary>
/// Creates a graph execution engine
/// </summary>
/// <param name="graph">The serialised graph to execute</param>
/// <returns></returns>
public IGraphEngine CreateEngine(Models.ExecutionGraph graph)
{
var input = this.CreateFrom(graph);
return(new ExecutionEngine(_lap, graph, input));
}
/// <summary>
/// Creates a graph training engine
/// </summary>
/// <param name="dataSource">Data source with training data</param>
/// <param name="learningContext">Previously created training context</param>
/// <param name="graph">The serialised graph to execute (optional)</param>
/// <returns></returns>
public IGraphTrainingEngine CreateTrainingEngine(IDataSource dataSource, ILearningContext learningContext, Models.ExecutionGraph graph = null)
{
var input = this.CreateFrom(graph);
return(new TrainingEngine(_lap, dataSource, learningContext, input));
}
/// <summary>
/// Trains a feed forward neural net on the MNIST data set (handwritten digit recognition)
/// The data files can be downloaded from http://yann.lecun.com/exdb/mnist/
/// </summary>
/// <param name="dataFilesPath">The path to a directory with the four extracted data files</param>
/// <param name="outputModelPath">Optional path to save the best model to</param>
static void MNISTConvolutional(string dataFilesPath, string outputModelPath = null)
{
using var lap = BrightWireGpuProvider.CreateLinearAlgebra();
var graph = new GraphFactory(lap);
Console.Write("Loading training data...");
var mnistTraining = Mnist.Load(dataFilesPath + "train-labels.idx1-ubyte",
dataFilesPath + "train-images.idx3-ubyte");
var mnistTest = Mnist.Load(dataFilesPath + "t10k-labels.idx1-ubyte",
dataFilesPath + "t10k-images.idx3-ubyte");
var trainingData =
_BuildTensors(graph, null, mnistTraining /*.Where(d => d.Label < 2).ToList()*/);
var testData = _BuildTensors(graph, trainingData,
mnistTest /*.Where(d => d.Label < 2).ToList()*/);
Console.WriteLine(
$"done - {trainingData.RowCount} training images and {testData.RowCount} test images loaded");
// one hot encoding uses the index of the output vector's maximum value as the classification label
var errorMetric = graph.ErrorMetric.OneHotEncoding;
// configure the network properties
graph.CurrentPropertySet.Use(graph.GradientDescent.Adam).Use(
graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N));
// create the network
const int HIDDEN_LAYER_SIZE = 1024, TRAINING_ITERATIONS = 20;
const float LEARNING_RATE = 0.05f;
var engine = graph.CreateTrainingEngine(trainingData, LEARNING_RATE);
if (!string.IsNullOrWhiteSpace(outputModelPath) && File.Exists(outputModelPath))
{
Console.WriteLine("Loading existing model from: " + outputModelPath);
using var file = new FileStream(outputModelPath, FileMode.Open, FileAccess.Read);
var model = Serializer.Deserialize <GraphModel>(file);
engine = graph.CreateTrainingEngine(trainingData, model.Graph, LEARNING_RATE);
}
else
{
graph.Connect(engine).
AddConvolutional(filterCount: 16, padding: 2, filterWidth: 5, filterHeight: 5, xStride: 1,
yStride: 1, shouldBackpropagate: false).Add(graph.LeakyReluActivation()).
AddMaxPooling(filterWidth: 2, filterHeight: 2, xStride: 2, yStride: 2).
AddConvolutional(filterCount: 32, padding: 2, filterWidth: 5, filterHeight: 5, xStride: 1,
yStride: 1).Add(graph.LeakyReluActivation()).
AddMaxPooling(filterWidth: 2, filterHeight: 2, xStride: 2, yStride: 2).Transpose().
AddFeedForward(HIDDEN_LAYER_SIZE).Add(graph.LeakyReluActivation()).
AddDropOut(dropOutPercentage: 0.5f).AddFeedForward(trainingData.OutputSize).
Add(graph.SoftMaxActivation()).AddBackpropagation(errorMetric);
}
// lower the learning rate over time
engine.LearningContext.ScheduleLearningRate(15, LEARNING_RATE / 2);
// train the network for twenty iterations, saving the model on each improvement
Models.ExecutionGraph bestGraph = null;
engine.Train(TRAINING_ITERATIONS, testData, errorMetric, model =>
{
bestGraph = model.Graph;
if (!string.IsNullOrWhiteSpace(outputModelPath))
{
using var file = new FileStream(outputModelPath, FileMode.Create, FileAccess.Write);
Serializer.Serialize(file, model);
}
});
// export the final model and execute it on the training set
var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph);
var output = executionEngine.Execute(testData);
Console.WriteLine($"Final accuracy: {output.Average(o => o.CalculateError(errorMetric)):P2}");
// execute the model with a single image
var tensor = mnistTest.First().AsFloatTensor.Tensor;
var singleData = graph.CreateDataSource(new[] { tensor });
var result = executionEngine.Execute(singleData);
var prediction = result.Single().Output.Single().MaximumIndex();
}
/// <summary>
/// Trains a feed forward neural net on the emotion dataset
/// http://lpis.csd.auth.gr/publications/tsoumakas-ismir08.pdf
/// The data files can be downloaded from https://downloads.sourceforge.net/project/mulan/datasets/emotions.rar
/// </summary>
/// <param name="dataFilePath"></param>
public static void MultiLabelSingleClassifier(string dataFilePath)
{
var emotionData = _LoadEmotionData(dataFilePath);
var attributeColumns = Enumerable.Range(0, emotionData.ColumnCount - CLASSIFICATION_COUNT).ToList();
var classificationColumns = Enumerable.Range(emotionData.ColumnCount - CLASSIFICATION_COUNT, CLASSIFICATION_COUNT).ToList();
// create a new data table with a vector input column and a vector output column
var dataTableBuilder = BrightWireProvider.CreateDataTableBuilder();
dataTableBuilder.AddColumn(ColumnType.Vector, "Attributes");
dataTableBuilder.AddColumn(ColumnType.Vector, "Target", isTarget: true);
emotionData.ForEach(row => {
var input = FloatVector.Create(row.GetFields <float>(attributeColumns).ToArray());
var target = FloatVector.Create(row.GetFields <float>(classificationColumns).ToArray());
dataTableBuilder.Add(input, target);
return(true);
});
var data = dataTableBuilder.Build().Split(0);
// train a neural network
using (var lap = BrightWireProvider.CreateLinearAlgebra(false)) {
var graph = new GraphFactory(lap);
// binary classification rounds each output to 0 or 1 and compares each output against the binary classification targets
var errorMetric = graph.ErrorMetric.BinaryClassification;
// configure the network properties
graph.CurrentPropertySet
.Use(graph.GradientDescent.Adam)
.Use(graph.WeightInitialisation.Xavier)
;
// create a training engine
const float TRAINING_RATE = 0.3f;
var trainingData = graph.CreateDataSource(data.Training);
var testData = trainingData.CloneWith(data.Test);
var engine = graph.CreateTrainingEngine(trainingData, TRAINING_RATE, 128);
// build the network
const int HIDDEN_LAYER_SIZE = 64, TRAINING_ITERATIONS = 2000;
var network = graph.Connect(engine)
.AddFeedForward(HIDDEN_LAYER_SIZE)
.Add(graph.SigmoidActivation())
.AddDropOut(dropOutPercentage: 0.5f)
.AddFeedForward(engine.DataSource.OutputSize)
.Add(graph.SigmoidActivation())
.AddBackpropagation(errorMetric)
;
// train the network
Models.ExecutionGraph bestGraph = null;
engine.Train(TRAINING_ITERATIONS, testData, errorMetric, model => bestGraph = model.Graph, 50);
// export the final model and execute it on the training set
var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph);
var output = executionEngine.Execute(testData);
// output the results
var rowIndex = 0;
foreach (var item in output)
{
var sb = new StringBuilder();
foreach (var classification in item.Output.Zip(item.Target, (o, t) => (Output: o, Target: t)))
{
var columnIndex = 0;
sb.AppendLine($"{rowIndex++}) ");
foreach (var column in classification.Output.Data.Zip(classification.Target.Data,
(o, t) => (Output: o, Target: t)))
{
var prediction = column.Output >= 0.5f ? "true" : "false";
var actual = column.Target >= 0.5f ? "true" : "false";
sb.AppendLine($"\t{columnIndex++}) predicted {prediction} (expected {actual})");
}
}
Console.WriteLine(sb.ToString());
}
}
}
/// <summary>
/// Creates a graph execution engine
/// </summary>
/// <param name="graph">The serialised graph to execute</param>
/// <returns></returns>
public IGraphEngine CreateEngine(Models.ExecutionGraph graph)
{
var input = this.CreateFrom(graph);
return(new ExecutionEngine(LinearAlgebraProvider, graph, input));
}
