public void TestRecurrent()
{
var data = BinaryIntegers.Addition(100, false).Split(0);
var graph = new GraphFactory(_lap);
var errorMetric = graph.ErrorMetric.BinaryClassification;
graph.CurrentPropertySet
.Use(graph.GradientDescent.Adam)
.Use(graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N));
// create the engine
var trainingData = graph.CreateDataSource(data.Training);
var testData = trainingData.CloneWith(data.Test);
var engine = graph.CreateTrainingEngine(trainingData, learningRate: 0.01f, batchSize: 16);
// build the network
const int HIDDEN_LAYER_SIZE = 32, TRAINING_ITERATIONS = 5;
var memory = new float[HIDDEN_LAYER_SIZE];
var network = graph.Connect(engine)
.AddSimpleRecurrent(graph.ReluActivation(), memory)
.AddFeedForward(engine.DataSource.OutputSize)
.Add(graph.ReluActivation())
.AddBackpropagationThroughTime(errorMetric)
;
// train the network for twenty iterations, saving the model on each improvement
BrightWire.Models.ExecutionGraph bestGraph = null;
engine.Train(TRAINING_ITERATIONS, testData, errorMetric, bn => bestGraph = bn.Graph);
// export the graph and verify it against some unseen integers on the best model
var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph);
var testData2 = graph.CreateDataSource(BinaryIntegers.Addition(8, true));
var results = executionEngine.Execute(testData2);
}
public void BidirectionalAddition()
{
var trainingSet = BinaryIntegers.Addition(10, false).Select(l => l.ToArray()).ToList();
const int HIDDEN_SIZE = 16, NUM_EPOCHS = 100, BATCH_SIZE = 32;
var errorMetric = ErrorMetricType.BinaryClassification.Create();
var layerTemplate = new LayerDescriptor(0.1f)
{
Activation = ActivationType.LeakyRelu,
WeightInitialisation = WeightInitialisationType.Gaussian,
DecayRate = 0.99f
};
var recurrentTemplate = layerTemplate.Clone();
recurrentTemplate.WeightInitialisation = WeightInitialisationType.Gaussian;
var trainingDataProvider = _lap.NN.CreateSequentialTrainingDataProvider(trainingSet);
var layers = new INeuralNetworkBidirectionalLayer[] {
_lap.NN.CreateBidirectionalLayer(
_lap.NN.CreateSimpleRecurrentLayer(trainingDataProvider.InputSize, HIDDEN_SIZE, recurrentTemplate),
_lap.NN.CreateSimpleRecurrentLayer(trainingDataProvider.InputSize, HIDDEN_SIZE, recurrentTemplate)
),
_lap.NN.CreateBidirectionalLayer(_lap.NN.CreateFeedForwardRecurrentLayer(HIDDEN_SIZE * 2, trainingDataProvider.OutputSize, layerTemplate))
};
BidirectionalNetwork networkData = null;
using (var trainer = _lap.NN.CreateBidirectionalBatchTrainer(layers)) {
var forwardMemory = Enumerable.Range(0, HIDDEN_SIZE).Select(i => 0f).ToArray();
var backwardMemory = Enumerable.Range(0, HIDDEN_SIZE).Select(i => 0f).ToArray();
var trainingContext = _lap.NN.CreateTrainingContext(errorMetric, 0.1f, BATCH_SIZE);
trainingContext.RecurrentEpochComplete += (tc, rtc) => {
Debug.WriteLine(tc.LastTrainingError);
};
trainer.Train(trainingDataProvider, forwardMemory, backwardMemory, NUM_EPOCHS, _lap.NN.CreateRecurrentTrainingContext(trainingContext));
networkData = trainer.NetworkInfo;
networkData.ForwardMemory = new FloatArray {
Data = forwardMemory
};
networkData.BackwardMemory = new FloatArray {
Data = backwardMemory
};
}
var network = _lap.NN.CreateBidirectional(networkData);
foreach (var sequence in trainingSet)
{
var result = network.Execute(sequence.Select(d => d.Input).ToList());
}
}
public static void IntegerAddition()
{
// generate 1000 random integer additions (split into training and test sets)
var data = BinaryIntegers.Addition(1000, false).Split(0);
using (var lap = BrightWireProvider.CreateLinearAlgebra(false)) {
var graph = new GraphFactory(lap);
// binary classification rounds each output to either 0 or 1
var errorMetric = graph.ErrorMetric.BinaryClassification;
// configure the network properties
graph.CurrentPropertySet
.Use(graph.GradientDescent.Adam)
.Use(graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N))
;
// create the engine
var trainingData = graph.CreateDataSource(data.Training);
var testData = trainingData.CloneWith(data.Test);
var engine = graph.CreateTrainingEngine(trainingData, learningRate: 0.01f, batchSize: 16);
// build the network
const int HIDDEN_LAYER_SIZE = 32, TRAINING_ITERATIONS = 30;
var memory = new float[HIDDEN_LAYER_SIZE];
var network = graph.Connect(engine)
.AddSimpleRecurrent(graph.ReluActivation(), memory)
.AddFeedForward(engine.DataSource.OutputSize)
.Add(graph.ReluActivation())
.AddBackpropagationThroughTime(errorMetric)
;
// train the network for twenty iterations, saving the model on each improvement
Models.ExecutionGraph bestGraph = null;
engine.Train(TRAINING_ITERATIONS, testData, errorMetric, bn => bestGraph = bn.Graph);
// export the graph and verify it against some unseen integers on the best model
var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph);
var testData2 = graph.CreateDataSource(BinaryIntegers.Addition(8, true));
var results = executionEngine.Execute(testData2);
// group the output
var groupedResults = new (FloatVector[] Input, FloatVector[] Target, FloatVector[] Output)[8];
public static void IntegerAddition()
{
// generate 1000 random integer additions
var dataSet = BinaryIntegers.Addition(1000, false)
.Select(l => l.ToArray())
.ToList()
;
// split the numbers into training and test sets
int split = Convert.ToInt32(dataSet.Count * 0.8);
var trainingData = dataSet.Take(split).ToList();
var testData = dataSet.Skip(split).ToList();
// neural network hyper parameters
const int HIDDEN_SIZE = 32, NUM_EPOCHS = 25, BATCH_SIZE = 16;
const float TRAINING_RATE = 0.001f;
var errorMetric = ErrorMetricType.BinaryClassification.Create();
var layerTemplate = new LayerDescriptor(0.3f)
{
Activation = ActivationType.Relu,
WeightInitialisation = WeightInitialisationType.Xavier,
WeightUpdate = WeightUpdateType.RMSprop
};
var recurrentTemplate = layerTemplate.Clone();
recurrentTemplate.WeightInitialisation = WeightInitialisationType.Gaussian;
using (var lap = Provider.CreateLinearAlgebra()) {
// create training data providers
var trainingDataProvider = lap.NN.CreateSequentialTrainingDataProvider(trainingData);
var testDataProvider = lap.NN.CreateSequentialTrainingDataProvider(testData);
var layers = new INeuralNetworkRecurrentLayer[] {
lap.NN.CreateSimpleRecurrentLayer(trainingDataProvider.InputSize, HIDDEN_SIZE, recurrentTemplate),
lap.NN.CreateFeedForwardRecurrentLayer(HIDDEN_SIZE, trainingDataProvider.OutputSize, layerTemplate)
};
// train the network
RecurrentNetwork networkData = null;
using (var trainer = lap.NN.CreateRecurrentBatchTrainer(layers)) {
var memory = Enumerable.Range(0, HIDDEN_SIZE).Select(i => 0f).ToArray();
var trainingContext = lap.NN.CreateTrainingContext(errorMetric, TRAINING_RATE, BATCH_SIZE);
trainingContext.RecurrentEpochComplete += (tc, rtc) => {
var testError = trainer.Execute(testDataProvider, memory, rtc).SelectMany(s => s.Select(d => errorMetric.Compute(d.Output, d.ExpectedOutput))).Average();
Console.WriteLine($"Epoch {tc.CurrentEpoch} - score: {testError:P}");
};
trainer.Train(trainingDataProvider, memory, NUM_EPOCHS, lap.NN.CreateRecurrentTrainingContext(trainingContext));
networkData = trainer.NetworkInfo;
networkData.Memory = new FloatArray {
Data = memory
};
}
// evaluate the network on some freshly generated data
var network = lap.NN.CreateRecurrent(networkData);
foreach (var sequence in BinaryIntegers.Addition(8, true))
{
var result = network.Execute(sequence.Select(d => d.Input).ToList());
Console.Write("First: ");
foreach (var item in sequence)
{
_WriteBinary(item.Input[0]);
}
Console.WriteLine();
Console.Write("Second: ");
foreach (var item in sequence)
{
_WriteBinary(item.Input[1]);
}
Console.WriteLine();
Console.WriteLine(" --------------------------------");
Console.Write("Expected: ");
foreach (var item in sequence)
{
_WriteBinary(item.Output[0]);
}
Console.WriteLine();
Console.Write("Predicted: ");
foreach (var item in result)
{
_WriteBinary(item.Output[0]);
}
Console.WriteLine();
Console.WriteLine();
}
}
}
public static void IntegerAddition()
{
// generate 1000 random integer additions (split into training and test sets)
var data = BinaryIntegers.Addition(1000, false).Split(0);
using (var lap = BrightWireProvider.CreateLinearAlgebra(false)) {
var graph = new GraphFactory(lap);
var errorMetric = graph.ErrorMetric.BinaryClassification;
// modify the property set
var propertySet = graph.CurrentPropertySet
.Use(graph.GradientDescent.Adam)
.Use(graph.WeightInitialisation.Xavier)
;
// create the engine
var trainingData = graph.CreateDataSource(data.Training);
var testData = trainingData.CloneWith(data.Test);
var engine = graph.CreateTrainingEngine(trainingData, 0.01f, 16);
// build the network
const int HIDDEN_LAYER_SIZE = 32;
var memory = new float[HIDDEN_LAYER_SIZE];
var network = graph.Connect(engine)
.AddSimpleRecurrent(graph.ReluActivation(), memory)
.AddFeedForward(engine.DataSource.OutputSize)
.Add(graph.ReluActivation())
.AddBackpropagationThroughTime(errorMetric)
;
// train the network
GraphModel bestNetwork = null;
engine.Train(30, testData, errorMetric, bn => bestNetwork = bn);
// export the graph and verify it against some unseen integers
var executionEngine = graph.CreateEngine(bestNetwork.Graph);
var testData2 = graph.CreateDataSource(BinaryIntegers.Addition(8, true));
var results = executionEngine.Execute(testData2);
// group the output
var groupedResults = new Tuple <FloatVector[], FloatVector[], FloatVector[]> [8];
for (var i = 0; i < 8; i++)
{
var input = new FloatVector[32];
var target = new FloatVector[32];
var output = new FloatVector[32];
for (var j = 0; j < 32; j++)
{
input[j] = results[j].Input[0][i];
target[j] = results[j].Target[i];
output[j] = results[j].Output[i];
}
groupedResults[i] = Tuple.Create(input, target, output);
}
// write the results
foreach (var result in groupedResults)
{
Console.Write("First: ");
foreach (var item in result.Item1)
{
_WriteBinary(item.Data[0]);
}
Console.WriteLine();
Console.Write("Second: ");
foreach (var item in result.Item1)
{
_WriteBinary(item.Data[1]);
}
Console.WriteLine();
Console.WriteLine(" --------------------------------");
Console.Write("Expected: ");
foreach (var item in result.Item2)
{
_WriteBinary(item.Data[0]);
}
Console.WriteLine();
Console.Write("Predicted: ");
foreach (var item in result.Item3)
{
_WriteBinary(item.Data[0]);
}
Console.WriteLine();
Console.WriteLine();
}
}
}
