public void Given__ASigmoidNetworkOfSuitableSize__AndSomeTrainingData(int inputLayerSize, int hiddenLayerSize, int outputLayerSize, int iterations, double trainingRate, int trainingSamplesCount)
{
var rawNet = new NeuralNet3LayerSigmoid(inputLayerSize, hiddenLayerSize, outputLayerSize).Randomize(2);
var netBeforeTraining = rawNet.ToString();
var interpetedNet = new InterpretedNet <string, int>(
rawNet,
s => s.Select(c => 0.1d * (c - 73)).ToArray(),
o => (int)(o.Single() * 11),
i => new [] { (ZeroToOne)(1 / 11d * i) },
(x, y) => x.Zip(y, (xx, yy) => Math.Abs(xx - yy))
);
var trainingData = GenerateRandomDataAndLabels(trainingSamplesCount);
var testData = GenerateRandomDataAndLabels(10 + trainingSamplesCount / 10);
var scoreBeforeTraining = CountHits(interpetedNet, testData);
new BackPropagationWithGradientDescent().ApplyToBatches(interpetedNet, trainingData, trainingSamplesCount / 10, trainingRate, iterations);
var scoreAfterTraining = CountHits(interpetedNet, testData);
//
Console.Write("Hits for NN size {3}, {4}, {5} Before / After training with {0} samples : {1} / {2}",
trainingSamplesCount * iterations,
scoreBeforeTraining, scoreAfterTraining,
inputLayerSize, hiddenLayerSize, outputLayerSize);
GenerateRandomDataAndLabels(10).Each(t => Console.WriteLine("{0} \t(should be \t{1}) \t: \t{2} ", t.Data, t.Label, interpetedNet.OutputFor(t.Data)));
Console.WriteLine(netBeforeTraining);
Console.WriteLine(rawNet);
//
scoreAfterTraining.Data.ShouldBeGreaterThan(scoreBeforeTraining.Data * 1.1);
}
public InterpretedNet(NeuralNet3LayerSigmoid net, Func <TData, double[]> inputEncoding, Func <IEnumerable <ZeroToOne>, TLabel> outputInterpretation, Func <TLabel, IEnumerable <ZeroToOne> > reverseInterpretation, Distances distancesFunction)
{
InputEncoding = inputEncoding;
OutputInterpretation = outputInterpretation;
ReverseInterpretation = reverseInterpretation;
Distances = distancesFunction;
Distance = PythagoreanDistance;
Net = net;
}
public void Given__ExampleWeightsFor221Net__HasExpectedFiringRatesAndOutput(double[] inputs, double[] inputToHidden, double[] hiddenToOutput, double expectedH0, double expectedH1, double expectedOutput)
{
var net = NeuralNet3LayerSigmoid.FromFlatWeightArrays(2, inputToHidden, hiddenToOutput);
//
net.ActivateInputs(inputs);
net.HiddenLayer[0].FiringRate.ShouldBe(expectedH0, "Hidden 0 Firing Rate");
net.HiddenLayer[1].FiringRate.ShouldBe(expectedH1, "Hidden 1 Firing Rate");
net.OutputLayer[0].FiringRate.ShouldBe(expectedOutput, "Output 0 Firing Rate");
net.OutputFor(inputs).ShouldBe(new ZeroToOne[] { expectedOutput });
}
public void Given__a221Network(double[] inputs, double[] inputToHidden, double[] hiddenToOutput)
{
var net = NeuralNet3LayerSigmoid.FromFlatWeightArrays(inputs.Length, inputToHidden, hiddenToOutput);
net.InputToHidden[0, 0].ShouldBe(inputToHidden[0], "inputToHidden[0,0]");
net.InputToHidden[0, 1].ShouldBe(inputToHidden[1], "inputToHidden[0,1]");
net.InputToHidden[1, 0].ShouldBe(inputToHidden[2 + 0], "inputToHidden[1,0]");
net.InputToHidden[1, 1].ShouldBe(inputToHidden[2 + 1], "inputToHidden[1,1]");
net.HiddenToOutput.ShouldEqualByValue(new MatrixD(new[, ] {
{ hiddenToOutput[0] }, { hiddenToOutput[1] }
}), "HiddenToOutput");
}
public void Given__BiasesHigh_WeightsHigh__Returns111__ForAnyInput(params double[] inputs)
{
const double h = Neuron.High;
var net = new NeuralNet3LayerSigmoid(new[, ] {
{ h }, { h }, { h }
}, new[, ] {
{ h, h, h }
}, new[] { h }, new[] { h, h, h });
//
Console.WriteLine(net.OutputFor(inputs));
net.OutputFor(inputs).ToArray().ShouldBe(new ZeroToOne[] { 1, 1, 1 });
}
public void Given__a221Network(double[] inputs, double[] inputToHidden, double[] hiddenToOutput, double[] hiddenBiases, double[] outputBiases)
{
var net = NeuralNet3LayerSigmoid.FromFlatArrays(inputs.Length, inputToHidden, hiddenToOutput, hiddenBiases, outputBiases);
net.InputToHidden[0, 0].ShouldBe(inputToHidden[0], "inputToHidden[0,0]");
net.InputToHidden[0, 1].ShouldBe(inputToHidden[1], "inputToHidden[0,1]");
net.InputToHidden[1, 0].ShouldBe(inputToHidden[2 + 0], "inputToHidden[1,0]");
net.InputToHidden[1, 1].ShouldBe(inputToHidden[2 + 1], "inputToHidden[1,1]");
net.HiddenToOutput.ShouldEqualByValue(new MatrixD(new[, ] {
{ hiddenToOutput[0] }, { hiddenToOutput[1] }
}), "HiddenToOutput");
net.HiddenLayer[0].Bias.ShouldBe(hiddenBiases[0], "hiddenBiases[0]");
net.HiddenLayer[1].Bias.ShouldBe(hiddenBiases[1], "hiddenBiases[1]");
net.OutputLayer[0].Bias.ShouldBe(outputBiases[0], "outputBiases[0]");
}
public void Given_Biases0_Weights0__ReturnsHalfHalfHalf__ForAnyInput(params double[] inputs)
{
var net = new NeuralNet3LayerSigmoid(3, 1, 3)
.SetBiases(new[] { 0d }, new [] { 0d, 0d, 0d })
.SetInputToHiddenWeights(new double[3, 1] {
{ 0 }, { 0 }, { 0 }
})
.SetHiddenToOutputWeights(new double[1, 3] {
{ 0, 0, 0 }
});
//
Console.WriteLine(net.OutputFor(inputs).ToArray());
//
net.OutputFor(inputs).ShouldEqualByValue(new ZeroToOne[] { 0.5, 0.5, 0.5 });
}
public void Given__BiasesLow_WeightsLow__Returns000__ForAnyInput(params double[] inputs)
{
const double low = Neuron.Low;
var net = new NeuralNet3LayerSigmoid(3, 1, 3)
.SetBiases(new[] { low }, new[] { low, low, low })
.SetInputToHiddenWeights(new double[3, 1] {
{ low }, { low }, { low }
})
.SetHiddenToOutputWeights(new double[1, 3] {
{ low, low, low }
});
//
Console.WriteLine(net.OutputFor(inputs));
net.OutputFor(inputs).ToArray().ShouldBe(new ZeroToOne[] { 0, 0, 0 });
}
public void Given__a323Network(double[] inputs, double[] inputToHidden, double[] hiddenToOutput)
{
var net = NeuralNet3LayerSigmoid.FromFlatWeightArrays(inputs.Length, inputToHidden, hiddenToOutput);
net.InputToHidden[0, 0].ShouldBe(inputToHidden[0]);
net.InputToHidden[0, 1].ShouldBe(inputToHidden[1]);
net.InputToHidden[1, 0].ShouldBe(inputToHidden[2 + 0]);
net.InputToHidden[1, 1].ShouldBe(inputToHidden[2 + 1]);
net.InputToHidden[2, 0].ShouldBe(inputToHidden[4 + 0]);
net.InputToHidden[2, 1].ShouldBe(inputToHidden[4 + 1]);
net.HiddenToOutput[0, 0].ShouldBe(hiddenToOutput[0]);
net.HiddenToOutput[0, 1].ShouldBe(hiddenToOutput[1]);
net.HiddenToOutput[0, 2].ShouldBe(hiddenToOutput[2]);
net.HiddenToOutput[1, 0].ShouldBe(hiddenToOutput[3 + 0]);
net.HiddenToOutput[1, 1].ShouldBe(hiddenToOutput[3 + 1]);
net.HiddenToOutput[1, 2].ShouldBe(hiddenToOutput[3 + 2]);
net.HiddenLayer.ShouldAll(n => n.Bias.ShouldBe(0));
net.OutputLayer.ShouldAll(n => n.Bias.ShouldBe(0));
}
public void Given__223_SigmoidNetwork(double[] inputs, double[] inputToHiddenWeights, double[] hiddenToOutputWeights, double target)
{
var net = NeuralNet3LayerSigmoid.FromFlatWeightArrays(inputs.Length, inputToHiddenWeights, hiddenToOutputWeights);
var targets = Enumerable.Range(0, net.OutputLayer.Length).Select(i => (ZeroToOne)target);
var calculatedDeltas = BackPropagationWithGradientDescent.DeltasFor(net, inputs, targets);
var exOutputDelta1 = -0.040681125112339026d;
var exOutputDelta2 = hiddenToOutputWeights[1].Equals(0) ? 0 : exOutputDelta1;
var exOutputDelta3 = hiddenToOutputWeights[2].Equals(0) ? 0 : exOutputDelta1;
var expected = new DeltasFor2LayersOfNet
{
OutputBiases = new[] { 0d, 0d, 0d },
OutputWeights = new MatrixD(
new[, ]
{
{ exOutputDelta1 *net.HiddenLayer[0].FiringRate, exOutputDelta2 *net.HiddenLayer[0].FiringRate, exOutputDelta3 *net.HiddenLayer[0].FiringRate },
{ exOutputDelta1 *net.HiddenLayer[1].FiringRate, exOutputDelta2 *net.HiddenLayer[1].FiringRate, exOutputDelta3 *net.HiddenLayer[1].FiringRate }
}),
};
calculatedDeltas.OutputWeights.ShouldEqualByValue(expected.OutputWeights);
calculatedDeltas.OutputBiases.ShouldEqualByValue(expected.OutputBiases);
}
public static DeltasFor2LayersOfNet DeltasFor(NeuralNet3LayerSigmoid net, IEnumerable <double> inputs, IEnumerable <ZeroToOne> targets)
{
var outputs = net.OutputFor(inputs).ToArray();
var outputDeltas = outputs
.Zip(targets, (o, target) => (target - o) * o * (1 - o)) /* see e.g. wikipedia https://en.wikipedia.org/wiki/Backpropagation#Derivation */
.ToArray();
var outputWeightDeltas = new MatrixD(net.HiddenToOutput.RowCount, net.HiddenToOutput.ColumnCount);
var hiddenDeltaParts = new MatrixD(net.HiddenToOutput.RowCount, net.HiddenToOutput.ColumnCount);
for (int i = 0; i < outputWeightDeltas.RowCount; i++)
{
for (int j = 0; j < outputWeightDeltas.ColumnCount; j++)
{
outputWeightDeltas[i, j] = net.HiddenLayer[i].FiringRate * outputDeltas[j];
hiddenDeltaParts[i, j] = outputDeltas[j] * (net.HiddenToOutput[i, j] + outputWeightDeltas[i, j]) * outputs[j] * (1 - outputs[j]);
}
}
var hiddenDeltas = hiddenDeltaParts.ByRows().Select(row => row.Sum()).ToArray();
var hiddenWeightDeltas = new MatrixD(net.InputToHidden.RowCount, net.InputToHidden.ColumnCount);
for (int i = 0; i < hiddenWeightDeltas.RowCount; i++)
{
for (int j = 0; j < hiddenWeightDeltas.ColumnCount; j++)
{
hiddenWeightDeltas[i, j] = net.InputLayer[i].FiringRate * hiddenDeltas[j];
}
}
return(new DeltasFor2LayersOfNet
{
OutputBiases = net.OutputLayer.Zip(outputDeltas, (neuron, delta) => neuron.Bias * delta).ToArray(),
OutputWeights = outputWeightDeltas,
HiddenBiases = net.HiddenLayer.Zip(hiddenDeltas, (neuron, delta) => neuron.Bias * delta).ToArray(),
HiddenWeights = hiddenWeightDeltas
});
}
public void Given__a323Network(double[] inputs, double[] inputToHidden, double[] hiddenToOutput, double[] hiddenBiases, double[] outputBiases)
{
var net = NeuralNet3LayerSigmoid.FromFlatArrays(inputs.Length, inputToHidden, hiddenToOutput, hiddenBiases, outputBiases);
net.InputToHidden[0, 0].ShouldBe(inputToHidden[0]);
net.InputToHidden[0, 1].ShouldBe(inputToHidden[1]);
net.InputToHidden[1, 0].ShouldBe(inputToHidden[2 + 0]);
net.InputToHidden[1, 1].ShouldBe(inputToHidden[2 + 1]);
net.InputToHidden[2, 0].ShouldBe(inputToHidden[4 + 0]);
net.InputToHidden[2, 1].ShouldBe(inputToHidden[4 + 1]);
net.HiddenToOutput[0, 0].ShouldBe(hiddenToOutput[0]);
net.HiddenToOutput[0, 1].ShouldBe(hiddenToOutput[1]);
net.HiddenToOutput[0, 2].ShouldBe(hiddenToOutput[2]);
net.HiddenToOutput[1, 0].ShouldBe(hiddenToOutput[3 + 0]);
net.HiddenToOutput[1, 1].ShouldBe(hiddenToOutput[3 + 1]);
net.HiddenToOutput[1, 2].ShouldBe(hiddenToOutput[3 + 2]);
net.HiddenLayer[0].Bias.ShouldBe(hiddenBiases[0], "hiddenBiases[0]");
net.HiddenLayer[1].Bias.ShouldBe(hiddenBiases[1], "hiddenBiases[1]");
net.OutputLayer[0].Bias.ShouldBe(outputBiases[0], "outputBiases[0]");
net.OutputLayer[1].Bias.ShouldBe(outputBiases[1], "outputBiases[1]");
net.OutputLayer[2].Bias.ShouldBe(outputBiases[2], "outputBiases[2]");
}
public void Given__221_SigmoidNetwork(double[] inputs, double[] inputToHiddenWeights, double[] hiddenToOutputWeights, double target)
{
var net = NeuralNet3LayerSigmoid.FromFlatWeightArrays(inputs.Length, inputToHiddenWeights, hiddenToOutputWeights);
var targets = Enumerable.Range(0, net.OutputLayer.Length).Select(i => (ZeroToOne)target);
var calculatedDeltas = BackPropagationWithGradientDescent.DeltasFor(net, inputs, targets);
/*
* 0.680267196698649d, 0.663738697404353d, 0.690283492907644d
*/
var exOutputDelta = -0.040681125112339026d;
var expected = new DeltasFor2LayersOfNet
{
OutputBiases = new[] { 0d },
OutputWeights = new MatrixD(
new[, ]
{
{ exOutputDelta *net.HiddenLayer[0].FiringRate },
{ exOutputDelta *net.HiddenLayer[1].FiringRate }
}),
};
calculatedDeltas.OutputWeights.ShouldEqualByValue(expected.OutputWeights);
calculatedDeltas.OutputBiases.ShouldEqualByValue(expected.OutputBiases);
}
public void Given__221_SigmoidNetwork(double[] inputs, double[] inputToHiddenWeights, double[] hiddenToOutputWeights, double target)
{
var net = NeuralNet3LayerSigmoid.FromFlatWeightArrays(inputs.Length, inputToHiddenWeights, hiddenToOutputWeights);
var targets = Enumerable.Range(0, net.OutputLayer.Length).Select(i => (ZeroToOne)target);
var calculatedDeltas = BackPropagationWithGradientDescent.DeltasFor(net, inputs, targets);
/*
* 0.680267196698649d, 0.663738697404353d, 0.690283492907644d
*/
var outputs = net.LastOutputs.ToArray();
var exOutputDelta = -0.040681125112339026d;
var exHiddenDelta0 = -0.0023685025015371172d;
var exHiddenDelta1 = -0.0075927347073177845d;
var expectedOutputDeltas = new DeltasFor2LayersOfNet
{
OutputBiases = new[] { 0d },
OutputWeights = new MatrixD(
new[, ]
{
{ exOutputDelta *net.HiddenLayer[0].FiringRate },
{ exOutputDelta *net.HiddenLayer[1].FiringRate }
}),
HiddenBiases = new [] { 0d, 0d },
HiddenWeights = new MatrixD(
new[, ]
{
{
exHiddenDelta0 *net.HiddenLayer[0].Inputs[0].Source.FiringRate,
exHiddenDelta1 *net.HiddenLayer[1].Inputs[0].Source.FiringRate
},
{
exHiddenDelta0 *net.HiddenLayer[0].Inputs[1].Source.FiringRate,
exHiddenDelta1 *net.HiddenLayer[1].Inputs[1].Source.FiringRate
}
}),
};
var expectedDeltaValues = new DeltasFor2LayersOfNet
{
OutputBiases = new[] { 0d },
OutputWeights = new MatrixD(
new[, ]
{
{ -0.027674034938717864 }, // -0.040681125112339026d * 0.680267196698649d +/- last two d.p.s
{ -0.027001636991007407 } // -0.040681125112339026d * 0.663738697404353d +/- last two d.p.s
}),
HiddenBiases = new[] { 0d, 0d },
HiddenWeights = new MatrixD(
new[, ]
{
{
-0.000828975875537991d,
-0.0026574571475612243d
},
{
-0.0021316522513834054d,
-0.0068334612365860059d
}
}),
};
calculatedDeltas.OutputWeights.ShouldEqualByValue(expectedOutputDeltas.OutputWeights, "OutputWeights");
calculatedDeltas.OutputBiases.ShouldEqualByValue(expectedOutputDeltas.OutputBiases, "OutputBiases");
calculatedDeltas.HiddenWeights.ShouldEqualByValue(expectedOutputDeltas.HiddenWeights, "HiddenWeights");
calculatedDeltas.HiddenBiases.ShouldEqualByValue(expectedOutputDeltas.HiddenBiases, "HiddenBiases");
//
expectedOutputDeltas.HiddenBiases.ShouldEqualByValue(expectedDeltaValues.HiddenBiases, "HiddenBiases");
expectedOutputDeltas.OutputBiases.ShouldEqualByValue(expectedDeltaValues.OutputBiases, "OutputBiases");
expectedOutputDeltas.HiddenWeights.ShouldEqualByValue(expectedDeltaValues.HiddenWeights, "HiddenWeights");
expectedOutputDeltas.OutputWeights.ShouldEqualByValue(expectedDeltaValues.OutputWeights, "OutputWeights");
}
