public void TestRecurrent() { var data = BinaryIntegers.Addition(100, false).Split(0); var graph = new GraphFactory(_lap); var errorMetric = graph.ErrorMetric.BinaryClassification; graph.CurrentPropertySet .Use(graph.GradientDescent.Adam) .Use(graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N)); // create the engine var trainingData = graph.CreateDataSource(data.Training); var testData = trainingData.CloneWith(data.Test); var engine = graph.CreateTrainingEngine(trainingData, learningRate: 0.01f, batchSize: 16); // build the network const int HIDDEN_LAYER_SIZE = 32, TRAINING_ITERATIONS = 5; var memory = new float[HIDDEN_LAYER_SIZE]; var network = graph.Connect(engine) .AddSimpleRecurrent(graph.ReluActivation(), memory) .AddFeedForward(engine.DataSource.OutputSize) .Add(graph.ReluActivation()) .AddBackpropagationThroughTime(errorMetric) ; // train the network for twenty iterations, saving the model on each improvement BrightWire.Models.ExecutionGraph bestGraph = null; engine.Train(TRAINING_ITERATIONS, testData, errorMetric, bn => bestGraph = bn.Graph); // export the graph and verify it against some unseen integers on the best model var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph); var testData2 = graph.CreateDataSource(BinaryIntegers.Addition(8, true)); var results = executionEngine.Execute(testData2); }
public void BidirectionalAddition() { var trainingSet = BinaryIntegers.Addition(10, false).Select(l => l.ToArray()).ToList(); const int HIDDEN_SIZE = 16, NUM_EPOCHS = 100, BATCH_SIZE = 32; var errorMetric = ErrorMetricType.BinaryClassification.Create(); var layerTemplate = new LayerDescriptor(0.1f) { Activation = ActivationType.LeakyRelu, WeightInitialisation = WeightInitialisationType.Gaussian, DecayRate = 0.99f }; var recurrentTemplate = layerTemplate.Clone(); recurrentTemplate.WeightInitialisation = WeightInitialisationType.Gaussian; var trainingDataProvider = _lap.NN.CreateSequentialTrainingDataProvider(trainingSet); var layers = new INeuralNetworkBidirectionalLayer[] { _lap.NN.CreateBidirectionalLayer( _lap.NN.CreateSimpleRecurrentLayer(trainingDataProvider.InputSize, HIDDEN_SIZE, recurrentTemplate), _lap.NN.CreateSimpleRecurrentLayer(trainingDataProvider.InputSize, HIDDEN_SIZE, recurrentTemplate) ), _lap.NN.CreateBidirectionalLayer(_lap.NN.CreateFeedForwardRecurrentLayer(HIDDEN_SIZE * 2, trainingDataProvider.OutputSize, layerTemplate)) }; BidirectionalNetwork networkData = null; using (var trainer = _lap.NN.CreateBidirectionalBatchTrainer(layers)) { var forwardMemory = Enumerable.Range(0, HIDDEN_SIZE).Select(i => 0f).ToArray(); var backwardMemory = Enumerable.Range(0, HIDDEN_SIZE).Select(i => 0f).ToArray(); var trainingContext = _lap.NN.CreateTrainingContext(errorMetric, 0.1f, BATCH_SIZE); trainingContext.RecurrentEpochComplete += (tc, rtc) => { Debug.WriteLine(tc.LastTrainingError); }; trainer.Train(trainingDataProvider, forwardMemory, backwardMemory, NUM_EPOCHS, _lap.NN.CreateRecurrentTrainingContext(trainingContext)); networkData = trainer.NetworkInfo; networkData.ForwardMemory = new FloatArray { Data = forwardMemory }; networkData.BackwardMemory = new FloatArray { Data = backwardMemory }; } var network = _lap.NN.CreateBidirectional(networkData); foreach (var sequence in trainingSet) { var result = network.Execute(sequence.Select(d => d.Input).ToList()); } }
public static void IntegerAddition() { // generate 1000 random integer additions (split into training and test sets) var data = BinaryIntegers.Addition(1000, false).Split(0); using (var lap = BrightWireProvider.CreateLinearAlgebra(false)) { var graph = new GraphFactory(lap); // binary classification rounds each output to either 0 or 1 var errorMetric = graph.ErrorMetric.BinaryClassification; // configure the network properties graph.CurrentPropertySet .Use(graph.GradientDescent.Adam) .Use(graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N)) ; // create the engine var trainingData = graph.CreateDataSource(data.Training); var testData = trainingData.CloneWith(data.Test); var engine = graph.CreateTrainingEngine(trainingData, learningRate: 0.01f, batchSize: 16); // build the network const int HIDDEN_LAYER_SIZE = 32, TRAINING_ITERATIONS = 30; var memory = new float[HIDDEN_LAYER_SIZE]; var network = graph.Connect(engine) .AddSimpleRecurrent(graph.ReluActivation(), memory) .AddFeedForward(engine.DataSource.OutputSize) .Add(graph.ReluActivation()) .AddBackpropagationThroughTime(errorMetric) ; // train the network for twenty iterations, saving the model on each improvement Models.ExecutionGraph bestGraph = null; engine.Train(TRAINING_ITERATIONS, testData, errorMetric, bn => bestGraph = bn.Graph); // export the graph and verify it against some unseen integers on the best model var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph); var testData2 = graph.CreateDataSource(BinaryIntegers.Addition(8, true)); var results = executionEngine.Execute(testData2); // group the output var groupedResults = new (FloatVector[] Input, FloatVector[] Target, FloatVector[] Output)[8];
public static void IntegerAddition() { // generate 1000 random integer additions var dataSet = BinaryIntegers.Addition(1000, false) .Select(l => l.ToArray()) .ToList() ; // split the numbers into training and test sets int split = Convert.ToInt32(dataSet.Count * 0.8); var trainingData = dataSet.Take(split).ToList(); var testData = dataSet.Skip(split).ToList(); // neural network hyper parameters const int HIDDEN_SIZE = 32, NUM_EPOCHS = 25, BATCH_SIZE = 16; const float TRAINING_RATE = 0.001f; var errorMetric = ErrorMetricType.BinaryClassification.Create(); var layerTemplate = new LayerDescriptor(0.3f) { Activation = ActivationType.Relu, WeightInitialisation = WeightInitialisationType.Xavier, WeightUpdate = WeightUpdateType.RMSprop }; var recurrentTemplate = layerTemplate.Clone(); recurrentTemplate.WeightInitialisation = WeightInitialisationType.Gaussian; using (var lap = Provider.CreateLinearAlgebra()) { // create training data providers var trainingDataProvider = lap.NN.CreateSequentialTrainingDataProvider(trainingData); var testDataProvider = lap.NN.CreateSequentialTrainingDataProvider(testData); var layers = new INeuralNetworkRecurrentLayer[] { lap.NN.CreateSimpleRecurrentLayer(trainingDataProvider.InputSize, HIDDEN_SIZE, recurrentTemplate), lap.NN.CreateFeedForwardRecurrentLayer(HIDDEN_SIZE, trainingDataProvider.OutputSize, layerTemplate) }; // train the network RecurrentNetwork networkData = null; using (var trainer = lap.NN.CreateRecurrentBatchTrainer(layers)) { var memory = Enumerable.Range(0, HIDDEN_SIZE).Select(i => 0f).ToArray(); var trainingContext = lap.NN.CreateTrainingContext(errorMetric, TRAINING_RATE, BATCH_SIZE); trainingContext.RecurrentEpochComplete += (tc, rtc) => { var testError = trainer.Execute(testDataProvider, memory, rtc).SelectMany(s => s.Select(d => errorMetric.Compute(d.Output, d.ExpectedOutput))).Average(); Console.WriteLine($"Epoch {tc.CurrentEpoch} - score: {testError:P}"); }; trainer.Train(trainingDataProvider, memory, NUM_EPOCHS, lap.NN.CreateRecurrentTrainingContext(trainingContext)); networkData = trainer.NetworkInfo; networkData.Memory = new FloatArray { Data = memory }; } // evaluate the network on some freshly generated data var network = lap.NN.CreateRecurrent(networkData); foreach (var sequence in BinaryIntegers.Addition(8, true)) { var result = network.Execute(sequence.Select(d => d.Input).ToList()); Console.Write("First: "); foreach (var item in sequence) { _WriteBinary(item.Input[0]); } Console.WriteLine(); Console.Write("Second: "); foreach (var item in sequence) { _WriteBinary(item.Input[1]); } Console.WriteLine(); Console.WriteLine(" --------------------------------"); Console.Write("Expected: "); foreach (var item in sequence) { _WriteBinary(item.Output[0]); } Console.WriteLine(); Console.Write("Predicted: "); foreach (var item in result) { _WriteBinary(item.Output[0]); } Console.WriteLine(); Console.WriteLine(); } } }
public static void IntegerAddition() { // generate 1000 random integer additions (split into training and test sets) var data = BinaryIntegers.Addition(1000, false).Split(0); using (var lap = BrightWireProvider.CreateLinearAlgebra(false)) { var graph = new GraphFactory(lap); var errorMetric = graph.ErrorMetric.BinaryClassification; // modify the property set var propertySet = graph.CurrentPropertySet .Use(graph.GradientDescent.Adam) .Use(graph.WeightInitialisation.Xavier) ; // create the engine var trainingData = graph.CreateDataSource(data.Training); var testData = trainingData.CloneWith(data.Test); var engine = graph.CreateTrainingEngine(trainingData, 0.01f, 16); // build the network const int HIDDEN_LAYER_SIZE = 32; var memory = new float[HIDDEN_LAYER_SIZE]; var network = graph.Connect(engine) .AddSimpleRecurrent(graph.ReluActivation(), memory) .AddFeedForward(engine.DataSource.OutputSize) .Add(graph.ReluActivation()) .AddBackpropagationThroughTime(errorMetric) ; // train the network GraphModel bestNetwork = null; engine.Train(30, testData, errorMetric, bn => bestNetwork = bn); // export the graph and verify it against some unseen integers var executionEngine = graph.CreateEngine(bestNetwork.Graph); var testData2 = graph.CreateDataSource(BinaryIntegers.Addition(8, true)); var results = executionEngine.Execute(testData2); // group the output var groupedResults = new Tuple <FloatVector[], FloatVector[], FloatVector[]> [8]; for (var i = 0; i < 8; i++) { var input = new FloatVector[32]; var target = new FloatVector[32]; var output = new FloatVector[32]; for (var j = 0; j < 32; j++) { input[j] = results[j].Input[0][i]; target[j] = results[j].Target[i]; output[j] = results[j].Output[i]; } groupedResults[i] = Tuple.Create(input, target, output); } // write the results foreach (var result in groupedResults) { Console.Write("First: "); foreach (var item in result.Item1) { _WriteBinary(item.Data[0]); } Console.WriteLine(); Console.Write("Second: "); foreach (var item in result.Item1) { _WriteBinary(item.Data[1]); } Console.WriteLine(); Console.WriteLine(" --------------------------------"); Console.Write("Expected: "); foreach (var item in result.Item2) { _WriteBinary(item.Data[0]); } Console.WriteLine(); Console.Write("Predicted: "); foreach (var item in result.Item3) { _WriteBinary(item.Data[0]); } Console.WriteLine(); Console.WriteLine(); } } }