public void When_querying_the_network_after_training_for_the_XOR_problem_it_should_return_the_correct_output()
{
// Arrange
const int numberOfEpochs = 10000;
const double learningRate = 0.3;
const double momentum = 0.1;
var network = new NeuralNetworkBuilder()
.Using(new XORNetworkLayout())
.Build();
network.Train(new[]
{
new TrainingExample(new[] { 0.0, 0.0 }, new[] { 0.0 }),
new TrainingExample(new[] { 0.0, 1.0 }, new[] { 1.0 }),
new TrainingExample(new[] { 1.0, 0.0 }, new[] { 1.0 }),
new TrainingExample(new[] { 1.0, 1.0 }, new[] { 0.0 })
}, numberOfEpochs, learningRate, momentum);
// Act
double output_0_0 = network.Query(new[] { 0.0, 0.0 }).Single();
double output_0_1 = network.Query(new[] { 0.0, 1.0 }).Single();
double output_1_0 = network.Query(new[] { 1.0, 0.0 }).Single();
double output_1_1 = network.Query(new[] { 1.0, 1.0 }).Single();
// Assert
output_0_0.Should().BeApproximately(0.0, 0.05);
output_0_1.Should().BeApproximately(1.0, 0.05);
output_1_0.Should().BeApproximately(1.0, 0.05);
output_1_1.Should().BeApproximately(0.0, 0.05);
}
public void When_querying_the_network_after_training_it_should_yield_the_correct_output()
{
// Arrange
var network = new NeuralNetworkBuilder()
.Using(new XORNetworkLayout())
.Build();
network.Train(new[]
{
new TrainingExample(new[] { 0.0, 0.0 }, new[] { 0.0 }),
new TrainingExample(new[] { 0.0, 1.0 }, new[] { 1.0 }),
new TrainingExample(new[] { 1.0, 0.0 }, new[] { 1.0 }),
new TrainingExample(new[] { 1.0, 1.0 }, new[] { 0.0 })
}, 10000, 0.3, 0.1);
// Act
double output_0_0 = network.Query(new[] { 0.0, 0.0 }).Single();
double output_0_1 = network.Query(new[] { 0.0, 1.0 }).Single();
double output_1_0 = network.Query(new[] { 1.0, 0.0 }).Single();
double output_1_1 = network.Query(new[] { 1.0, 1.0 }).Single();
// Assert
output_0_0.Should().BeApproximately(0.0, 0.05);
output_0_1.Should().BeApproximately(1.0, 0.05);
output_1_0.Should().BeApproximately(1.0, 0.05);
output_1_1.Should().BeApproximately(0.0, 0.05);
}
public void When_training_the_network_for_a_single_epoch_it_should_update_the_weights_and_biases_correctly()
{
// Arrange
const int numberOfEpochs = 1;
const double learningRate = 0.5;
const double momentum = 0.0;
var network = new NeuralNetworkBuilder()
.Using(new TwoLayerNetworkProvider())
.Build();
// Act
network.Train(new[]
{
new TrainingExample(new[] { 0.1, 0.2, 0.3 }, new[] { 1.0, 0.0 })
}, numberOfEpochs, learningRate, momentum);
// Assert
Layer hiddenLayer = network.GetLayers().First();
Layer outputLayer = network.GetLayers().Last();
var updatedOutputWeights = outputLayer.Weights.ToRowMajorArray();
updatedOutputWeights[0].Should().BeApproximately(0.18612817, 0.000000005);
updatedOutputWeights[1].Should().BeApproximately(0.28933822, 0.000000005);
updatedOutputWeights[2].Should().BeApproximately(0.39242441, 0.000000005);
updatedOutputWeights[3].Should().BeApproximately(0.49533644, 0.000000005);
updatedOutputWeights[4].Should().BeApproximately(0.49296848, 0.000000005);
updatedOutputWeights[5].Should().BeApproximately(0.58790113, 0.000000005);
updatedOutputWeights[6].Should().BeApproximately(0.68302931, 0.000000005);
updatedOutputWeights[7].Should().BeApproximately(0.77843241, 0.000000005);
var updatedOutputBiases = outputLayer.Biases.ToArray();
updatedOutputBiases[0].Should().BeApproximately(0.10570742, 0.000000005);
updatedOutputBiases[1].Should().BeApproximately(-0.05372498, 0.000000005);
var updatedHiddenWeights = hiddenLayer.Weights.ToRowMajorArray();
updatedHiddenWeights[0].Should().BeApproximately(0.09883190, 0.000000005);
updatedHiddenWeights[1].Should().BeApproximately(0.19766381, 0.000000005);
updatedHiddenWeights[2].Should().BeApproximately(0.29649571, 0.000000005);
updatedHiddenWeights[3].Should().BeApproximately(0.39877481, 0.000000005);
updatedHiddenWeights[4].Should().BeApproximately(0.49754961, 0.000000005);
updatedHiddenWeights[5].Should().BeApproximately(0.59632442, 0.000000005);
updatedHiddenWeights[6].Should().BeApproximately(0.69874635, 0.000000005);
updatedHiddenWeights[7].Should().BeApproximately(0.79749270, 0.000000005);
updatedHiddenWeights[8].Should().BeApproximately(0.89623905, 0.000000005);
updatedHiddenWeights[9].Should().BeApproximately(0.99874653, 0.000000005);
updatedHiddenWeights[10].Should().BeApproximately(1.09749307, 0.000000005);
updatedHiddenWeights[11].Should().BeApproximately(1.19623960, 0.000000005);
var updatedHiddenBiases = hiddenLayer.Biases.ToArray();
updatedHiddenBiases[0].Should().BeApproximately(0.04831904, 0.000000005);
updatedHiddenBiases[1].Should().BeApproximately(0.06774805, 0.000000005);
updatedHiddenBiases[2].Should().BeApproximately(0.08746351, 0.000000005);
updatedHiddenBiases[3].Should().BeApproximately(0.10746534, 0.000000005);
}
public void When_training_the_network_for_a_single_epoch_it_should_update_the_weights_and_biases_correctly()
{
// Arrange
const double learningRate = 0.5;
const double momentum = 0.0;
var network = new NeuralNetworkBuilder()
.Using(new TwoLayerNetworkProvider())
.Build();
// Act
network.Train(new[]
{
new TrainingExample(new[] { 1.0, -2.0, 3.0 }, new[] { 0.1234, 0.8766 })
}, 1, learningRate, momentum);
// Assert
Layer hiddenLayer = network.GetLayers().First();
Layer outputLayer = network.GetLayers().Last();
var updatedOutputWeights = outputLayer.Weights.ToColumnMajorArray();
updatedOutputWeights[0].Should().BeApproximately(-0.00791776, 0.000000005);
updatedOutputWeights[1].Should().BeApproximately(-0.00595456, 0.000000005);
updatedOutputWeights[2].Should().BeApproximately(-0.00399135, 0.000000005);
updatedOutputWeights[3].Should().BeApproximately(-0.00202815, 0.000000005);
updatedOutputWeights[4].Should().BeApproximately(0.04078488, 0.000000005);
updatedOutputWeights[5].Should().BeApproximately(0.04281853, 0.000000005);
updatedOutputWeights[6].Should().BeApproximately(0.04485217, 0.000000005);
updatedOutputWeights[7].Should().BeApproximately(0.04688582, 0.000000005);
var updatedOutputBiases = outputLayer.Biases.ToArray();
updatedOutputBiases[0].Should().BeApproximately(-0.02407493, 0.000000005);
updatedOutputBiases[1].Should().BeApproximately(0.07087427, 0.000000005);
var updatedHiddenWeights = hiddenLayer.Weights.ToColumnMajorArray();
updatedHiddenWeights[0].Should().BeApproximately(0.00099337, 0.000000005);
updatedHiddenWeights[1].Should().BeApproximately(0.00501327, 0.000000005);
updatedHiddenWeights[2].Should().BeApproximately(0.00898010, 0.000000005);
updatedHiddenWeights[3].Should().BeApproximately(0.00199127, 0.000000005);
updatedHiddenWeights[4].Should().BeApproximately(0.00601746, 0.000000005);
updatedHiddenWeights[5].Should().BeApproximately(0.00997380, 0.000000005);
updatedHiddenWeights[6].Should().BeApproximately(0.00298917, 0.000000005);
updatedHiddenWeights[7].Should().BeApproximately(0.00702166, 0.000000005);
updatedHiddenWeights[8].Should().BeApproximately(0.01096751, 0.000000005);
updatedHiddenWeights[9].Should().BeApproximately(0.00398707, 0.000000005);
updatedHiddenWeights[10].Should().BeApproximately(0.00802586, 0.000000005);
updatedHiddenWeights[11].Should().BeApproximately(0.01196121, 0.000000005);
var updatedHiddenBiases = hiddenLayer.Biases.ToArray();
updatedHiddenBiases[0].Should().BeApproximately(0.01299337, 0.000000005);
updatedHiddenBiases[1].Should().BeApproximately(0.01399127, 0.000000005);
updatedHiddenBiases[2].Should().BeApproximately(0.01498917, 0.000000005);
updatedHiddenBiases[3].Should().BeApproximately(0.01598707, 0.000000005);
}
public void When_training_the_network_for_a_single_epoch_it_should_return_the_cost()
{
// Arrange
const double learningRate = 0.5;
const double momentum = 0.0;
var network = new NeuralNetworkBuilder()
.Using(new TwoLayerNetworkProvider())
.Build();
// Act
double cost = network.Train(new[]
{
new TrainingExample(new[] { 1.0, -2.0, 3.0 }, new[] { 0.1234, 0.8766 })
}, 1, learningRate, momentum);
// Assert
cost.Should().BeApproximately(0.1418, 0.00005);
}
public void When_training_the_network_for_a_single_epoch_it_should_return_the_cost()
{
// Arrange
const int numberOfEpochs = 1;
const double learningRate = 0.5;
const double momentum = 0.0;
var network = new NeuralNetworkBuilder()
.Using(new TwoLayerNetworkProvider())
.Using(new QuadraticCost())
.Build();
// Act
double cost = network.Train(new[]
{
new TrainingExample(new[] { 0.1, 0.2, 0.3 }, new[] { 1.0, 0.0 })
}, numberOfEpochs, learningRate, momentum);
// Assert
cost.Should().BeApproximately(0.83395, 0.000005);
}
public void When_querying_the_network_after_training_it_should_return_the_correct_output()
{
// Arrange
const int numberOfEpochs = 20000;
const double learningRate = 0.3;
const double momentum = 0.1;
var network = new NeuralNetworkBuilder()
.Using(new TwoLayerNetworkProvider())
.Build();
network.Train(new[]
{
new TrainingExample(new[] { 0.1, 0.2, 0.3 }, new[] { 1.0, 0.0 })
}, numberOfEpochs, learningRate, momentum);
// Act
double[] output = network.Query(new[] { 0.1, 0.2, 0.3 });
// Assert
output[0].Should().BeApproximately(1.0, 0.005);
output[1].Should().BeApproximately(0.0, 0.005);
}
public void When_training_the_network_for_a_single_epoch_it_should_calculate_the_gradients()
{
// Arrange
const int numberOfEpochs = 1;
const double learningRate = 0.5;
const double momentum = 0.0;
var network = new NeuralNetworkBuilder()
.Using(new TwoLayerNetworkProvider())
.Build();
// Act
network.Train(new[]
{
new TrainingExample(new[] { 0.1, 0.2, 0.3 }, new[] { 1.0, 0.0 })
}, numberOfEpochs, learningRate, momentum);
// Assert
Layer hiddenLayer = network.GetLayers().First();
Layer outputLayer = network.GetLayers().Last();
double[] outputWeightGradients = outputLayer.WeightGradients.ToRowMajorArray();
outputWeightGradients[0].Should().BeApproximately(-0.07225635, 0.000000005);
outputWeightGradients[1].Should().BeApproximately(-0.07867644, 0.000000005);
outputWeightGradients[2].Should().BeApproximately(-0.08484882, 0.000000005);
outputWeightGradients[3].Should().BeApproximately(-0.09067289, 0.000000005);
outputWeightGradients[4].Should().BeApproximately(0.11406304, 0.000000005);
outputWeightGradients[5].Should().BeApproximately(0.12419773, 0.000000005);
outputWeightGradients[6].Should().BeApproximately(0.13394138, 0.000000005);
outputWeightGradients[7].Should().BeApproximately(0.14313518, 0.000000005);
var outputBiasGradients = outputLayer.BiasGradients.ToArray();
outputBiasGradients[0].Should().BeApproximately(-0.131415, 0.0000005);
outputBiasGradients[1].Should().BeApproximately(0.207450, 0.0000005);
var outputInputGradients = outputLayer.PreviousLayerActivationGradients.ToArray();
outputInputGradients[0].Should().BeApproximately(0.09438525, 0.000000005);
outputInputGradients[1].Should().BeApproximately(0.10198876, 0.000000005);
outputInputGradients[2].Should().BeApproximately(0.10959228, 0.000000005);
outputInputGradients[3].Should().BeApproximately(0.11719579, 0.000000005);
double[] hiddenWeightGradients = hiddenLayer.WeightGradients.ToRowMajorArray();
hiddenWeightGradients[0].Should().BeApproximately(0.00233619, 0.0000005);
hiddenWeightGradients[1].Should().BeApproximately(0.00467238, 0.0000005);
hiddenWeightGradients[2].Should().BeApproximately(0.00700857, 0.0000005);
hiddenWeightGradients[3].Should().BeApproximately(0.00245039, 0.0000005);
hiddenWeightGradients[4].Should().BeApproximately(0.00490078, 0.0000005);
hiddenWeightGradients[5].Should().BeApproximately(0.00735117, 0.0000005);
hiddenWeightGradients[6].Should().BeApproximately(0.00250730, 0.0000005);
hiddenWeightGradients[7].Should().BeApproximately(0.00501460, 0.0000005);
hiddenWeightGradients[8].Should().BeApproximately(0.00752190, 0.0000005);
hiddenWeightGradients[9].Should().BeApproximately(0.00250693, 0.0000005);
hiddenWeightGradients[10].Should().BeApproximately(0.00501386, 0.0000005);
hiddenWeightGradients[11].Should().BeApproximately(0.00752079, 0.0000005);
double[] hiddenBiasGradients = hiddenLayer.BiasGradients.ToArray();
hiddenBiasGradients[0].Should().BeApproximately(0.02336191, 0.000000005);
hiddenBiasGradients[1].Should().BeApproximately(0.02450390, 0.000000005);
hiddenBiasGradients[2].Should().BeApproximately(0.02507299, 0.000000005);
hiddenBiasGradients[3].Should().BeApproximately(0.02506932, 0.000000005);
hiddenLayer.PreviousLayerActivationGradients.Should().BeNull();
}
public void When_training_the_network_for_a_single_epoch_it_should_calculate_the_gradient_vectors()
{
// Arrange
const double learningRate = 0.5;
const double momentum = 0.0;
var network = new NeuralNetworkBuilder()
.Using(new TwoLayerNetworkProvider())
.Build();
// Act
network.Train(new[]
{
new TrainingExample(new[] { 1.0, -2.0, 3.0 }, new[] { 0.1234, 0.8766 })
}, 1, learningRate, momentum);
// Assert
Layer hiddenLayer = network.GetLayers().First();
Layer outputLayer = network.GetLayers().Last();
double[] outputWeightGradients = outputLayer.WeightGradients.ToColumnMajorArray();
outputWeightGradients[0].Should().BeApproximately(0.04983553, 0.000000005);
outputWeightGradients[1].Should().BeApproximately(0.04990912, 0.000000005);
outputWeightGradients[2].Should().BeApproximately(0.04998271, 0.000000005);
outputWeightGradients[3].Should().BeApproximately(0.05005629, 0.000000005);
outputWeightGradients[4].Should().BeApproximately(-0.04556976, 0.000000005);
outputWeightGradients[5].Should().BeApproximately(-0.04563706, 0.000000005);
outputWeightGradients[6].Should().BeApproximately(-0.04570435, 0.000000005);
outputWeightGradients[7].Should().BeApproximately(-0.04577163, 0.000000005);
var outputBiasGradients = outputLayer.BiasGradients.ToArray();
outputBiasGradients[0].Should().BeApproximately(0.098150, 0.0000005);
outputBiasGradients[1].Should().BeApproximately(-0.089749, 0.0000005);
var outputInputGradients = outputLayer.PreviousLayerActivationGradients.ToArray();
outputInputGradients[0].Should().BeApproximately(0.00005307, 0.000000005);
outputInputGradients[1].Should().BeApproximately(0.00006988, 0.000000005);
outputInputGradients[2].Should().BeApproximately(0.00008668, 0.000000005);
outputInputGradients[3].Should().BeApproximately(0.00010348, 0.000000005);
double[] hiddenWeightGradients = hiddenLayer.WeightGradients.ToColumnMajorArray();
hiddenWeightGradients[0].Should().BeApproximately(0.00001326, 0.0000005);
hiddenWeightGradients[1].Should().BeApproximately(-0.00002653, 0.0000005);
hiddenWeightGradients[2].Should().BeApproximately(0.00003980, 0.0000005);
hiddenWeightGradients[3].Should().BeApproximately(0.00001746, 0.0000005);
hiddenWeightGradients[4].Should().BeApproximately(-0.00003493, 0.0000005);
hiddenWeightGradients[5].Should().BeApproximately(0.00005239, 0.0000005);
hiddenWeightGradients[6].Should().BeApproximately(0.00002166, 0.0000005);
hiddenWeightGradients[7].Should().BeApproximately(-0.00004332, 0.0000005);
hiddenWeightGradients[8].Should().BeApproximately(0.00006499, 0.0000005);
hiddenWeightGradients[9].Should().BeApproximately(0.00002586, 0.0000005);
hiddenWeightGradients[10].Should().BeApproximately(-0.00005172, 0.0000005);
hiddenWeightGradients[11].Should().BeApproximately(0.00007758, 0.0000005);
double[] hiddenBiasGradients = hiddenLayer.BiasGradients.ToArray();
hiddenBiasGradients[0].Should().BeApproximately(0.000013265, 0.0000000005);
hiddenBiasGradients[1].Should().BeApproximately(0.000017464, 0.0000000005);
hiddenBiasGradients[2].Should().BeApproximately(0.000021662, 0.0000000005);
hiddenBiasGradients[3].Should().BeApproximately(0.000025860, 0.0000000005);
hiddenLayer.PreviousLayerActivationGradients.Should().BeNull();
}