protected internal INeuralNetworkLayerUpdater CreateLayer(int inputSize, int outputSize, INeuralNetworkFactory factory, LayerDescriptor template)
{
var descriptor = template.Clone();
descriptor.Activation = ActivationType.None;
var layer = factory.CreateLayer(inputSize, outputSize, descriptor);
return(factory.CreateUpdater(layer, template));
}
public void BidirectionalAddition()
{
var trainingSet = BinaryIntegers.Addition(10, false).Select(l => l.ToArray()).ToList();
const int HIDDEN_SIZE = 16, NUM_EPOCHS = 100, BATCH_SIZE = 32;
var errorMetric = ErrorMetricType.BinaryClassification.Create();
var layerTemplate = new LayerDescriptor(0.1f)
{
Activation = ActivationType.LeakyRelu,
WeightInitialisation = WeightInitialisationType.Gaussian,
DecayRate = 0.99f
};
var recurrentTemplate = layerTemplate.Clone();
recurrentTemplate.WeightInitialisation = WeightInitialisationType.Gaussian;
var trainingDataProvider = _lap.NN.CreateSequentialTrainingDataProvider(trainingSet);
var layers = new INeuralNetworkBidirectionalLayer[] {
_lap.NN.CreateBidirectionalLayer(
_lap.NN.CreateSimpleRecurrentLayer(trainingDataProvider.InputSize, HIDDEN_SIZE, recurrentTemplate),
_lap.NN.CreateSimpleRecurrentLayer(trainingDataProvider.InputSize, HIDDEN_SIZE, recurrentTemplate)
),
_lap.NN.CreateBidirectionalLayer(_lap.NN.CreateFeedForwardRecurrentLayer(HIDDEN_SIZE * 2, trainingDataProvider.OutputSize, layerTemplate))
};
BidirectionalNetwork networkData = null;
using (var trainer = _lap.NN.CreateBidirectionalBatchTrainer(layers)) {
var forwardMemory = Enumerable.Range(0, HIDDEN_SIZE).Select(i => 0f).ToArray();
var backwardMemory = Enumerable.Range(0, HIDDEN_SIZE).Select(i => 0f).ToArray();
var trainingContext = _lap.NN.CreateTrainingContext(errorMetric, 0.1f, BATCH_SIZE);
trainingContext.RecurrentEpochComplete += (tc, rtc) => {
Debug.WriteLine(tc.LastTrainingError);
};
trainer.Train(trainingDataProvider, forwardMemory, backwardMemory, NUM_EPOCHS, _lap.NN.CreateRecurrentTrainingContext(trainingContext));
networkData = trainer.NetworkInfo;
networkData.ForwardMemory = new FloatArray {
Data = forwardMemory
};
networkData.BackwardMemory = new FloatArray {
Data = backwardMemory
};
}
var network = _lap.NN.CreateBidirectional(networkData);
foreach (var sequence in trainingSet)
{
var result = network.Execute(sequence.Select(d => d.Input).ToList());
}
}
public static void IntegerAddition()
{
// generate 1000 random integer additions
var dataSet = BinaryIntegers.Addition(1000, false)
.Select(l => l.ToArray())
.ToList()
;
// split the numbers into training and test sets
int split = Convert.ToInt32(dataSet.Count * 0.8);
var trainingData = dataSet.Take(split).ToList();
var testData = dataSet.Skip(split).ToList();
// neural network hyper parameters
const int HIDDEN_SIZE = 32, NUM_EPOCHS = 25, BATCH_SIZE = 16;
const float TRAINING_RATE = 0.001f;
var errorMetric = ErrorMetricType.BinaryClassification.Create();
var layerTemplate = new LayerDescriptor(0.3f)
{
Activation = ActivationType.Relu,
WeightInitialisation = WeightInitialisationType.Xavier,
WeightUpdate = WeightUpdateType.RMSprop
};
var recurrentTemplate = layerTemplate.Clone();
recurrentTemplate.WeightInitialisation = WeightInitialisationType.Gaussian;
using (var lap = Provider.CreateLinearAlgebra()) {
// create training data providers
var trainingDataProvider = lap.NN.CreateSequentialTrainingDataProvider(trainingData);
var testDataProvider = lap.NN.CreateSequentialTrainingDataProvider(testData);
var layers = new INeuralNetworkRecurrentLayer[] {
lap.NN.CreateSimpleRecurrentLayer(trainingDataProvider.InputSize, HIDDEN_SIZE, recurrentTemplate),
lap.NN.CreateFeedForwardRecurrentLayer(HIDDEN_SIZE, trainingDataProvider.OutputSize, layerTemplate)
};
// train the network
RecurrentNetwork networkData = null;
using (var trainer = lap.NN.CreateRecurrentBatchTrainer(layers)) {
var memory = Enumerable.Range(0, HIDDEN_SIZE).Select(i => 0f).ToArray();
var trainingContext = lap.NN.CreateTrainingContext(errorMetric, TRAINING_RATE, BATCH_SIZE);
trainingContext.RecurrentEpochComplete += (tc, rtc) => {
var testError = trainer.Execute(testDataProvider, memory, rtc).SelectMany(s => s.Select(d => errorMetric.Compute(d.Output, d.ExpectedOutput))).Average();
Console.WriteLine($"Epoch {tc.CurrentEpoch} - score: {testError:P}");
};
trainer.Train(trainingDataProvider, memory, NUM_EPOCHS, lap.NN.CreateRecurrentTrainingContext(trainingContext));
networkData = trainer.NetworkInfo;
networkData.Memory = new FloatArray {
Data = memory
};
}
// evaluate the network on some freshly generated data
var network = lap.NN.CreateRecurrent(networkData);
foreach (var sequence in BinaryIntegers.Addition(8, true))
{
var result = network.Execute(sequence.Select(d => d.Input).ToList());
Console.Write("First: ");
foreach (var item in sequence)
{
_WriteBinary(item.Input[0]);
}
Console.WriteLine();
Console.Write("Second: ");
foreach (var item in sequence)
{
_WriteBinary(item.Input[1]);
}
Console.WriteLine();
Console.WriteLine(" --------------------------------");
Console.Write("Expected: ");
foreach (var item in sequence)
{
_WriteBinary(item.Output[0]);
}
Console.WriteLine();
Console.Write("Predicted: ");
foreach (var item in result)
{
_WriteBinary(item.Output[0]);
}
Console.WriteLine();
Console.WriteLine();
}
}
}