/// <summary> /// Creates a graph training engine /// </summary> /// <param name="dataSource">Data source with training data</param> /// <param name="graph">The serialised graph to execute</param> /// <param name="trainingRate">Initial learning rate</param> /// <param name="batchSize">Mini batch size</param> /// <param name="trainingErrorCalculation">How to calculate the training error</param> /// <returns></returns> public IGraphTrainingEngine CreateTrainingEngine(IDataSource dataSource, Models.ExecutionGraph graph, float trainingRate = 0.1f, int batchSize = 128, TrainingErrorCalculation trainingErrorCalculation = TrainingErrorCalculation.Fast) { var learningContext = new LearningContext(_lap, trainingRate, batchSize, trainingErrorCalculation, dataSource.IsSequential); var input = this.CreateFrom(graph); return(new TrainingEngine(_lap, dataSource, learningContext, input)); }
public void TestRecurrent() { var data = BinaryIntegers.Addition(100, false).Split(0); var graph = new GraphFactory(_lap); var errorMetric = graph.ErrorMetric.BinaryClassification; graph.CurrentPropertySet.Use(graph.GradientDescent.Adam).Use( graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N)); // create the engine var trainingData = graph.CreateDataSource(data.Training); var testData = trainingData.CloneWith(data.Test); var engine = graph.CreateTrainingEngine(trainingData, learningRate: 0.01f, batchSize: 16); // build the network const int HIDDEN_LAYER_SIZE = 32, TRAINING_ITERATIONS = 5; var memory = new float[HIDDEN_LAYER_SIZE]; var network = graph.Connect(engine).AddSimpleRecurrent(graph.ReluActivation(), memory). AddFeedForward(engine.DataSource.OutputSize).Add(graph.ReluActivation()). AddBackpropagationThroughTime(errorMetric); // train the network for twenty iterations, saving the model on each improvement Models.ExecutionGraph bestGraph = null; engine.Train(TRAINING_ITERATIONS, testData, errorMetric, bn => bestGraph = bn.Graph); // export the graph and verify it against some unseen integers on the best model var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph); var testData2 = graph.CreateDataSource(BinaryIntegers.Addition(8, true)); var results = executionEngine.Execute(testData2); }
public void DeserialiseExecutionGraph() { var(graph, data) = MakeGraphAndData(); Models.ExecutionGraph executionGraphReloaded = null; using (var file = new MemoryStream()) { Serializer.Serialize(file, bestNetwork.Graph); file.Position = 0; executionGraphReloaded = Serializer.Deserialize <Models.ExecutionGraph>(file); } Assert.IsNotNull(executionGraphReloaded); var engine = graph.CreateEngine(executionGraphReloaded); AssertEngineGetsGoodResults(engine, data); }
/// <summary> /// Trains a feed forward neural net on the MNIST data set (handwritten digit recognition) /// The data files can be downloaded from http://yann.lecun.com/exdb/mnist/ /// </summary> /// <param name="dataFilesPath">The path to a directory with the four extracted data files</param> public static void MNIST(string dataFilesPath) { using (var lap = BrightWireGpuProvider.CreateLinearAlgebra()) { var graph = new GraphFactory(lap); Console.Write("Loading training data..."); var trainingData = _BuildVectors(null, graph, Mnist.Load(dataFilesPath + "train-labels.idx1-ubyte", dataFilesPath + "train-images.idx3-ubyte")); var testData = _BuildVectors(trainingData, graph, Mnist.Load(dataFilesPath + "t10k-labels.idx1-ubyte", dataFilesPath + "t10k-images.idx3-ubyte")); Console.WriteLine($"done - {trainingData.RowCount} training images and {testData.RowCount} test images loaded"); // one hot encoding uses the index of the output vector's maximum value as the classification label var errorMetric = graph.ErrorMetric.OneHotEncoding; // configure the network properties graph.CurrentPropertySet .Use(graph.GradientDescent.RmsProp) .Use(graph.WeightInitialisation.Xavier) ; // create the training engine and schedule a training rate change const float TRAINING_RATE = 0.1f; var engine = graph.CreateTrainingEngine(trainingData, TRAINING_RATE, 128); engine.LearningContext.ScheduleLearningRate(15, TRAINING_RATE / 3); // create the network graph.Connect(engine) .AddFeedForward(outputSize: 1024) .Add(graph.LeakyReluActivation()) .AddDropOut(dropOutPercentage: 0.5f) .AddFeedForward(outputSize: trainingData.OutputSize) .Add(graph.SigmoidActivation()) .AddBackpropagation(errorMetric) ; // train the network for twenty iterations, saving the model on each improvement Models.ExecutionGraph bestGraph = null; engine.Train(20, testData, errorMetric, model => bestGraph = model.Graph); // export the final model and execute it on the training set var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph); var output = executionEngine.Execute(testData); Console.WriteLine($"Final accuracy: {output.Average(o => o.CalculateError(errorMetric)):P2}"); } }
public static void IntegerAddition() { // generate 1000 random integer additions (split into training and test sets) var data = BinaryIntegers.Addition(1000, false).Split(0); using (var lap = BrightWireProvider.CreateLinearAlgebra(false)) { var graph = new GraphFactory(lap); // binary classification rounds each output to either 0 or 1 var errorMetric = graph.ErrorMetric.BinaryClassification; // configure the network properties graph.CurrentPropertySet .Use(graph.GradientDescent.Adam) .Use(graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N)) ; // create the engine var trainingData = graph.CreateDataSource(data.Training); var testData = trainingData.CloneWith(data.Test); var engine = graph.CreateTrainingEngine(trainingData, learningRate: 0.01f, batchSize: 16); // build the network const int HIDDEN_LAYER_SIZE = 32, TRAINING_ITERATIONS = 30; var memory = new float[HIDDEN_LAYER_SIZE]; var network = graph.Connect(engine) .AddSimpleRecurrent(graph.ReluActivation(), memory) .AddFeedForward(engine.DataSource.OutputSize) .Add(graph.ReluActivation()) .AddBackpropagationThroughTime(errorMetric) ; // train the network for twenty iterations, saving the model on each improvement Models.ExecutionGraph bestGraph = null; engine.Train(TRAINING_ITERATIONS, testData, errorMetric, bn => bestGraph = bn.Graph); // export the graph and verify it against some unseen integers on the best model var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph); var testData2 = graph.CreateDataSource(BinaryIntegers.Addition(8, true)); var results = executionEngine.Execute(testData2); // group the output var groupedResults = new (FloatVector[] Input, FloatVector[] Target, FloatVector[] Output)[8];
/// <summary> /// Creates a graph execution engine /// </summary> /// <param name="graph">The serialised graph to execute</param> /// <returns></returns> public IGraphEngine CreateEngine(Models.ExecutionGraph graph) { var input = this.CreateFrom(graph); return(new ExecutionEngine(_lap, graph, input)); }
/// <summary> /// Creates a graph training engine /// </summary> /// <param name="dataSource">Data source with training data</param> /// <param name="learningContext">Previously created training context</param> /// <param name="graph">The serialised graph to execute (optional)</param> /// <returns></returns> public IGraphTrainingEngine CreateTrainingEngine(IDataSource dataSource, ILearningContext learningContext, Models.ExecutionGraph graph = null) { var input = this.CreateFrom(graph); return(new TrainingEngine(_lap, dataSource, learningContext, input)); }
/// <summary> /// Trains a feed forward neural net on the MNIST data set (handwritten digit recognition) /// The data files can be downloaded from http://yann.lecun.com/exdb/mnist/ /// </summary> /// <param name="dataFilesPath">The path to a directory with the four extracted data files</param> /// <param name="outputModelPath">Optional path to save the best model to</param> static void MNISTConvolutional(string dataFilesPath, string outputModelPath = null) { using var lap = BrightWireGpuProvider.CreateLinearAlgebra(); var graph = new GraphFactory(lap); Console.Write("Loading training data..."); var mnistTraining = Mnist.Load(dataFilesPath + "train-labels.idx1-ubyte", dataFilesPath + "train-images.idx3-ubyte"); var mnistTest = Mnist.Load(dataFilesPath + "t10k-labels.idx1-ubyte", dataFilesPath + "t10k-images.idx3-ubyte"); var trainingData = _BuildTensors(graph, null, mnistTraining /*.Where(d => d.Label < 2).ToList()*/); var testData = _BuildTensors(graph, trainingData, mnistTest /*.Where(d => d.Label < 2).ToList()*/); Console.WriteLine( $"done - {trainingData.RowCount} training images and {testData.RowCount} test images loaded"); // one hot encoding uses the index of the output vector's maximum value as the classification label var errorMetric = graph.ErrorMetric.OneHotEncoding; // configure the network properties graph.CurrentPropertySet.Use(graph.GradientDescent.Adam).Use( graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N)); // create the network const int HIDDEN_LAYER_SIZE = 1024, TRAINING_ITERATIONS = 20; const float LEARNING_RATE = 0.05f; var engine = graph.CreateTrainingEngine(trainingData, LEARNING_RATE); if (!string.IsNullOrWhiteSpace(outputModelPath) && File.Exists(outputModelPath)) { Console.WriteLine("Loading existing model from: " + outputModelPath); using var file = new FileStream(outputModelPath, FileMode.Open, FileAccess.Read); var model = Serializer.Deserialize <GraphModel>(file); engine = graph.CreateTrainingEngine(trainingData, model.Graph, LEARNING_RATE); } else { graph.Connect(engine). AddConvolutional(filterCount: 16, padding: 2, filterWidth: 5, filterHeight: 5, xStride: 1, yStride: 1, shouldBackpropagate: false).Add(graph.LeakyReluActivation()). AddMaxPooling(filterWidth: 2, filterHeight: 2, xStride: 2, yStride: 2). AddConvolutional(filterCount: 32, padding: 2, filterWidth: 5, filterHeight: 5, xStride: 1, yStride: 1).Add(graph.LeakyReluActivation()). AddMaxPooling(filterWidth: 2, filterHeight: 2, xStride: 2, yStride: 2).Transpose(). AddFeedForward(HIDDEN_LAYER_SIZE).Add(graph.LeakyReluActivation()). AddDropOut(dropOutPercentage: 0.5f).AddFeedForward(trainingData.OutputSize). Add(graph.SoftMaxActivation()).AddBackpropagation(errorMetric); } // lower the learning rate over time engine.LearningContext.ScheduleLearningRate(15, LEARNING_RATE / 2); // train the network for twenty iterations, saving the model on each improvement Models.ExecutionGraph bestGraph = null; engine.Train(TRAINING_ITERATIONS, testData, errorMetric, model => { bestGraph = model.Graph; if (!string.IsNullOrWhiteSpace(outputModelPath)) { using var file = new FileStream(outputModelPath, FileMode.Create, FileAccess.Write); Serializer.Serialize(file, model); } }); // export the final model and execute it on the training set var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph); var output = executionEngine.Execute(testData); Console.WriteLine($"Final accuracy: {output.Average(o => o.CalculateError(errorMetric)):P2}"); // execute the model with a single image var tensor = mnistTest.First().AsFloatTensor.Tensor; var singleData = graph.CreateDataSource(new[] { tensor }); var result = executionEngine.Execute(singleData); var prediction = result.Single().Output.Single().MaximumIndex(); }
/// <summary> /// Trains a feed forward neural net on the emotion dataset /// http://lpis.csd.auth.gr/publications/tsoumakas-ismir08.pdf /// The data files can be downloaded from https://downloads.sourceforge.net/project/mulan/datasets/emotions.rar /// </summary> /// <param name="dataFilePath"></param> public static void MultiLabelSingleClassifier(string dataFilePath) { var emotionData = _LoadEmotionData(dataFilePath); var attributeColumns = Enumerable.Range(0, emotionData.ColumnCount - CLASSIFICATION_COUNT).ToList(); var classificationColumns = Enumerable.Range(emotionData.ColumnCount - CLASSIFICATION_COUNT, CLASSIFICATION_COUNT).ToList(); // create a new data table with a vector input column and a vector output column var dataTableBuilder = BrightWireProvider.CreateDataTableBuilder(); dataTableBuilder.AddColumn(ColumnType.Vector, "Attributes"); dataTableBuilder.AddColumn(ColumnType.Vector, "Target", isTarget: true); emotionData.ForEach(row => { var input = FloatVector.Create(row.GetFields <float>(attributeColumns).ToArray()); var target = FloatVector.Create(row.GetFields <float>(classificationColumns).ToArray()); dataTableBuilder.Add(input, target); return(true); }); var data = dataTableBuilder.Build().Split(0); // train a neural network using (var lap = BrightWireProvider.CreateLinearAlgebra(false)) { var graph = new GraphFactory(lap); // binary classification rounds each output to 0 or 1 and compares each output against the binary classification targets var errorMetric = graph.ErrorMetric.BinaryClassification; // configure the network properties graph.CurrentPropertySet .Use(graph.GradientDescent.Adam) .Use(graph.WeightInitialisation.Xavier) ; // create a training engine const float TRAINING_RATE = 0.3f; var trainingData = graph.CreateDataSource(data.Training); var testData = trainingData.CloneWith(data.Test); var engine = graph.CreateTrainingEngine(trainingData, TRAINING_RATE, 128); // build the network const int HIDDEN_LAYER_SIZE = 64, TRAINING_ITERATIONS = 2000; var network = graph.Connect(engine) .AddFeedForward(HIDDEN_LAYER_SIZE) .Add(graph.SigmoidActivation()) .AddDropOut(dropOutPercentage: 0.5f) .AddFeedForward(engine.DataSource.OutputSize) .Add(graph.SigmoidActivation()) .AddBackpropagation(errorMetric) ; // train the network Models.ExecutionGraph bestGraph = null; engine.Train(TRAINING_ITERATIONS, testData, errorMetric, model => bestGraph = model.Graph, 50); // export the final model and execute it on the training set var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph); var output = executionEngine.Execute(testData); // output the results var rowIndex = 0; foreach (var item in output) { var sb = new StringBuilder(); foreach (var classification in item.Output.Zip(item.Target, (o, t) => (Output: o, Target: t))) { var columnIndex = 0; sb.AppendLine($"{rowIndex++}) "); foreach (var column in classification.Output.Data.Zip(classification.Target.Data, (o, t) => (Output: o, Target: t))) { var prediction = column.Output >= 0.5f ? "true" : "false"; var actual = column.Target >= 0.5f ? "true" : "false"; sb.AppendLine($"\t{columnIndex++}) predicted {prediction} (expected {actual})"); } } Console.WriteLine(sb.ToString()); } } }
/// <summary> /// Creates a graph execution engine /// </summary> /// <param name="graph">The serialised graph to execute</param> /// <returns></returns> public IGraphEngine CreateEngine(Models.ExecutionGraph graph) { var input = this.CreateFrom(graph); return(new ExecutionEngine(LinearAlgebraProvider, graph, input)); }