public void Given__ASigmoidNetworkOfSuitableSize__AndSomeTrainingData(int inputLayerSize, int hiddenLayerSize, int outputLayerSize, int iterations, double trainingRate, int trainingSamplesCount) { var rawNet = new NeuralNet3LayerSigmoid(inputLayerSize, hiddenLayerSize, outputLayerSize).Randomize(2); var netBeforeTraining = rawNet.ToString(); var interpetedNet = new InterpretedNet <string, int>( rawNet, s => s.Select(c => 0.1d * (c - 73)).ToArray(), o => (int)(o.Single() * 11), i => new [] { (ZeroToOne)(1 / 11d * i) }, (x, y) => x.Zip(y, (xx, yy) => Math.Abs(xx - yy)) ); var trainingData = GenerateRandomDataAndLabels(trainingSamplesCount); var testData = GenerateRandomDataAndLabels(10 + trainingSamplesCount / 10); var scoreBeforeTraining = CountHits(interpetedNet, testData); new BackPropagationWithGradientDescent().ApplyToBatches(interpetedNet, trainingData, trainingSamplesCount / 10, trainingRate, iterations); var scoreAfterTraining = CountHits(interpetedNet, testData); // Console.Write("Hits for NN size {3}, {4}, {5} Before / After training with {0} samples : {1} / {2}", trainingSamplesCount * iterations, scoreBeforeTraining, scoreAfterTraining, inputLayerSize, hiddenLayerSize, outputLayerSize); GenerateRandomDataAndLabels(10).Each(t => Console.WriteLine("{0} \t(should be \t{1}) \t: \t{2} ", t.Data, t.Label, interpetedNet.OutputFor(t.Data))); Console.WriteLine(netBeforeTraining); Console.WriteLine(rawNet); // scoreAfterTraining.Data.ShouldBeGreaterThan(scoreBeforeTraining.Data * 1.1); }
public InterpretedNet(NeuralNet3LayerSigmoid net, Func <TData, double[]> inputEncoding, Func <IEnumerable <ZeroToOne>, TLabel> outputInterpretation, Func <TLabel, IEnumerable <ZeroToOne> > reverseInterpretation, Distances distancesFunction) { InputEncoding = inputEncoding; OutputInterpretation = outputInterpretation; ReverseInterpretation = reverseInterpretation; Distances = distancesFunction; Distance = PythagoreanDistance; Net = net; }
public void Given__ExampleWeightsFor221Net__HasExpectedFiringRatesAndOutput(double[] inputs, double[] inputToHidden, double[] hiddenToOutput, double expectedH0, double expectedH1, double expectedOutput) { var net = NeuralNet3LayerSigmoid.FromFlatWeightArrays(2, inputToHidden, hiddenToOutput); // net.ActivateInputs(inputs); net.HiddenLayer[0].FiringRate.ShouldBe(expectedH0, "Hidden 0 Firing Rate"); net.HiddenLayer[1].FiringRate.ShouldBe(expectedH1, "Hidden 1 Firing Rate"); net.OutputLayer[0].FiringRate.ShouldBe(expectedOutput, "Output 0 Firing Rate"); net.OutputFor(inputs).ShouldBe(new ZeroToOne[] { expectedOutput }); }
public void Given__a221Network(double[] inputs, double[] inputToHidden, double[] hiddenToOutput) { var net = NeuralNet3LayerSigmoid.FromFlatWeightArrays(inputs.Length, inputToHidden, hiddenToOutput); net.InputToHidden[0, 0].ShouldBe(inputToHidden[0], "inputToHidden[0,0]"); net.InputToHidden[0, 1].ShouldBe(inputToHidden[1], "inputToHidden[0,1]"); net.InputToHidden[1, 0].ShouldBe(inputToHidden[2 + 0], "inputToHidden[1,0]"); net.InputToHidden[1, 1].ShouldBe(inputToHidden[2 + 1], "inputToHidden[1,1]"); net.HiddenToOutput.ShouldEqualByValue(new MatrixD(new[, ] { { hiddenToOutput[0] }, { hiddenToOutput[1] } }), "HiddenToOutput"); }
public void Given__BiasesHigh_WeightsHigh__Returns111__ForAnyInput(params double[] inputs) { const double h = Neuron.High; var net = new NeuralNet3LayerSigmoid(new[, ] { { h }, { h }, { h } }, new[, ] { { h, h, h } }, new[] { h }, new[] { h, h, h }); // Console.WriteLine(net.OutputFor(inputs)); net.OutputFor(inputs).ToArray().ShouldBe(new ZeroToOne[] { 1, 1, 1 }); }
public void Given__a221Network(double[] inputs, double[] inputToHidden, double[] hiddenToOutput, double[] hiddenBiases, double[] outputBiases) { var net = NeuralNet3LayerSigmoid.FromFlatArrays(inputs.Length, inputToHidden, hiddenToOutput, hiddenBiases, outputBiases); net.InputToHidden[0, 0].ShouldBe(inputToHidden[0], "inputToHidden[0,0]"); net.InputToHidden[0, 1].ShouldBe(inputToHidden[1], "inputToHidden[0,1]"); net.InputToHidden[1, 0].ShouldBe(inputToHidden[2 + 0], "inputToHidden[1,0]"); net.InputToHidden[1, 1].ShouldBe(inputToHidden[2 + 1], "inputToHidden[1,1]"); net.HiddenToOutput.ShouldEqualByValue(new MatrixD(new[, ] { { hiddenToOutput[0] }, { hiddenToOutput[1] } }), "HiddenToOutput"); net.HiddenLayer[0].Bias.ShouldBe(hiddenBiases[0], "hiddenBiases[0]"); net.HiddenLayer[1].Bias.ShouldBe(hiddenBiases[1], "hiddenBiases[1]"); net.OutputLayer[0].Bias.ShouldBe(outputBiases[0], "outputBiases[0]"); }
public void Given_Biases0_Weights0__ReturnsHalfHalfHalf__ForAnyInput(params double[] inputs) { var net = new NeuralNet3LayerSigmoid(3, 1, 3) .SetBiases(new[] { 0d }, new [] { 0d, 0d, 0d }) .SetInputToHiddenWeights(new double[3, 1] { { 0 }, { 0 }, { 0 } }) .SetHiddenToOutputWeights(new double[1, 3] { { 0, 0, 0 } }); // Console.WriteLine(net.OutputFor(inputs).ToArray()); // net.OutputFor(inputs).ShouldEqualByValue(new ZeroToOne[] { 0.5, 0.5, 0.5 }); }
public void Given__BiasesLow_WeightsLow__Returns000__ForAnyInput(params double[] inputs) { const double low = Neuron.Low; var net = new NeuralNet3LayerSigmoid(3, 1, 3) .SetBiases(new[] { low }, new[] { low, low, low }) .SetInputToHiddenWeights(new double[3, 1] { { low }, { low }, { low } }) .SetHiddenToOutputWeights(new double[1, 3] { { low, low, low } }); // Console.WriteLine(net.OutputFor(inputs)); net.OutputFor(inputs).ToArray().ShouldBe(new ZeroToOne[] { 0, 0, 0 }); }
public void Given__a323Network(double[] inputs, double[] inputToHidden, double[] hiddenToOutput) { var net = NeuralNet3LayerSigmoid.FromFlatWeightArrays(inputs.Length, inputToHidden, hiddenToOutput); net.InputToHidden[0, 0].ShouldBe(inputToHidden[0]); net.InputToHidden[0, 1].ShouldBe(inputToHidden[1]); net.InputToHidden[1, 0].ShouldBe(inputToHidden[2 + 0]); net.InputToHidden[1, 1].ShouldBe(inputToHidden[2 + 1]); net.InputToHidden[2, 0].ShouldBe(inputToHidden[4 + 0]); net.InputToHidden[2, 1].ShouldBe(inputToHidden[4 + 1]); net.HiddenToOutput[0, 0].ShouldBe(hiddenToOutput[0]); net.HiddenToOutput[0, 1].ShouldBe(hiddenToOutput[1]); net.HiddenToOutput[0, 2].ShouldBe(hiddenToOutput[2]); net.HiddenToOutput[1, 0].ShouldBe(hiddenToOutput[3 + 0]); net.HiddenToOutput[1, 1].ShouldBe(hiddenToOutput[3 + 1]); net.HiddenToOutput[1, 2].ShouldBe(hiddenToOutput[3 + 2]); net.HiddenLayer.ShouldAll(n => n.Bias.ShouldBe(0)); net.OutputLayer.ShouldAll(n => n.Bias.ShouldBe(0)); }
public void Given__223_SigmoidNetwork(double[] inputs, double[] inputToHiddenWeights, double[] hiddenToOutputWeights, double target) { var net = NeuralNet3LayerSigmoid.FromFlatWeightArrays(inputs.Length, inputToHiddenWeights, hiddenToOutputWeights); var targets = Enumerable.Range(0, net.OutputLayer.Length).Select(i => (ZeroToOne)target); var calculatedDeltas = BackPropagationWithGradientDescent.DeltasFor(net, inputs, targets); var exOutputDelta1 = -0.040681125112339026d; var exOutputDelta2 = hiddenToOutputWeights[1].Equals(0) ? 0 : exOutputDelta1; var exOutputDelta3 = hiddenToOutputWeights[2].Equals(0) ? 0 : exOutputDelta1; var expected = new DeltasFor2LayersOfNet { OutputBiases = new[] { 0d, 0d, 0d }, OutputWeights = new MatrixD( new[, ] { { exOutputDelta1 *net.HiddenLayer[0].FiringRate, exOutputDelta2 *net.HiddenLayer[0].FiringRate, exOutputDelta3 *net.HiddenLayer[0].FiringRate }, { exOutputDelta1 *net.HiddenLayer[1].FiringRate, exOutputDelta2 *net.HiddenLayer[1].FiringRate, exOutputDelta3 *net.HiddenLayer[1].FiringRate } }), }; calculatedDeltas.OutputWeights.ShouldEqualByValue(expected.OutputWeights); calculatedDeltas.OutputBiases.ShouldEqualByValue(expected.OutputBiases); }
public static DeltasFor2LayersOfNet DeltasFor(NeuralNet3LayerSigmoid net, IEnumerable <double> inputs, IEnumerable <ZeroToOne> targets) { var outputs = net.OutputFor(inputs).ToArray(); var outputDeltas = outputs .Zip(targets, (o, target) => (target - o) * o * (1 - o)) /* see e.g. wikipedia https://en.wikipedia.org/wiki/Backpropagation#Derivation */ .ToArray(); var outputWeightDeltas = new MatrixD(net.HiddenToOutput.RowCount, net.HiddenToOutput.ColumnCount); var hiddenDeltaParts = new MatrixD(net.HiddenToOutput.RowCount, net.HiddenToOutput.ColumnCount); for (int i = 0; i < outputWeightDeltas.RowCount; i++) { for (int j = 0; j < outputWeightDeltas.ColumnCount; j++) { outputWeightDeltas[i, j] = net.HiddenLayer[i].FiringRate * outputDeltas[j]; hiddenDeltaParts[i, j] = outputDeltas[j] * (net.HiddenToOutput[i, j] + outputWeightDeltas[i, j]) * outputs[j] * (1 - outputs[j]); } } var hiddenDeltas = hiddenDeltaParts.ByRows().Select(row => row.Sum()).ToArray(); var hiddenWeightDeltas = new MatrixD(net.InputToHidden.RowCount, net.InputToHidden.ColumnCount); for (int i = 0; i < hiddenWeightDeltas.RowCount; i++) { for (int j = 0; j < hiddenWeightDeltas.ColumnCount; j++) { hiddenWeightDeltas[i, j] = net.InputLayer[i].FiringRate * hiddenDeltas[j]; } } return(new DeltasFor2LayersOfNet { OutputBiases = net.OutputLayer.Zip(outputDeltas, (neuron, delta) => neuron.Bias * delta).ToArray(), OutputWeights = outputWeightDeltas, HiddenBiases = net.HiddenLayer.Zip(hiddenDeltas, (neuron, delta) => neuron.Bias * delta).ToArray(), HiddenWeights = hiddenWeightDeltas }); }
public void Given__a323Network(double[] inputs, double[] inputToHidden, double[] hiddenToOutput, double[] hiddenBiases, double[] outputBiases) { var net = NeuralNet3LayerSigmoid.FromFlatArrays(inputs.Length, inputToHidden, hiddenToOutput, hiddenBiases, outputBiases); net.InputToHidden[0, 0].ShouldBe(inputToHidden[0]); net.InputToHidden[0, 1].ShouldBe(inputToHidden[1]); net.InputToHidden[1, 0].ShouldBe(inputToHidden[2 + 0]); net.InputToHidden[1, 1].ShouldBe(inputToHidden[2 + 1]); net.InputToHidden[2, 0].ShouldBe(inputToHidden[4 + 0]); net.InputToHidden[2, 1].ShouldBe(inputToHidden[4 + 1]); net.HiddenToOutput[0, 0].ShouldBe(hiddenToOutput[0]); net.HiddenToOutput[0, 1].ShouldBe(hiddenToOutput[1]); net.HiddenToOutput[0, 2].ShouldBe(hiddenToOutput[2]); net.HiddenToOutput[1, 0].ShouldBe(hiddenToOutput[3 + 0]); net.HiddenToOutput[1, 1].ShouldBe(hiddenToOutput[3 + 1]); net.HiddenToOutput[1, 2].ShouldBe(hiddenToOutput[3 + 2]); net.HiddenLayer[0].Bias.ShouldBe(hiddenBiases[0], "hiddenBiases[0]"); net.HiddenLayer[1].Bias.ShouldBe(hiddenBiases[1], "hiddenBiases[1]"); net.OutputLayer[0].Bias.ShouldBe(outputBiases[0], "outputBiases[0]"); net.OutputLayer[1].Bias.ShouldBe(outputBiases[1], "outputBiases[1]"); net.OutputLayer[2].Bias.ShouldBe(outputBiases[2], "outputBiases[2]"); }
public void Given__221_SigmoidNetwork(double[] inputs, double[] inputToHiddenWeights, double[] hiddenToOutputWeights, double target) { var net = NeuralNet3LayerSigmoid.FromFlatWeightArrays(inputs.Length, inputToHiddenWeights, hiddenToOutputWeights); var targets = Enumerable.Range(0, net.OutputLayer.Length).Select(i => (ZeroToOne)target); var calculatedDeltas = BackPropagationWithGradientDescent.DeltasFor(net, inputs, targets); /* * 0.680267196698649d, 0.663738697404353d, 0.690283492907644d */ var exOutputDelta = -0.040681125112339026d; var expected = new DeltasFor2LayersOfNet { OutputBiases = new[] { 0d }, OutputWeights = new MatrixD( new[, ] { { exOutputDelta *net.HiddenLayer[0].FiringRate }, { exOutputDelta *net.HiddenLayer[1].FiringRate } }), }; calculatedDeltas.OutputWeights.ShouldEqualByValue(expected.OutputWeights); calculatedDeltas.OutputBiases.ShouldEqualByValue(expected.OutputBiases); }
public void Given__221_SigmoidNetwork(double[] inputs, double[] inputToHiddenWeights, double[] hiddenToOutputWeights, double target) { var net = NeuralNet3LayerSigmoid.FromFlatWeightArrays(inputs.Length, inputToHiddenWeights, hiddenToOutputWeights); var targets = Enumerable.Range(0, net.OutputLayer.Length).Select(i => (ZeroToOne)target); var calculatedDeltas = BackPropagationWithGradientDescent.DeltasFor(net, inputs, targets); /* * 0.680267196698649d, 0.663738697404353d, 0.690283492907644d */ var outputs = net.LastOutputs.ToArray(); var exOutputDelta = -0.040681125112339026d; var exHiddenDelta0 = -0.0023685025015371172d; var exHiddenDelta1 = -0.0075927347073177845d; var expectedOutputDeltas = new DeltasFor2LayersOfNet { OutputBiases = new[] { 0d }, OutputWeights = new MatrixD( new[, ] { { exOutputDelta *net.HiddenLayer[0].FiringRate }, { exOutputDelta *net.HiddenLayer[1].FiringRate } }), HiddenBiases = new [] { 0d, 0d }, HiddenWeights = new MatrixD( new[, ] { { exHiddenDelta0 *net.HiddenLayer[0].Inputs[0].Source.FiringRate, exHiddenDelta1 *net.HiddenLayer[1].Inputs[0].Source.FiringRate }, { exHiddenDelta0 *net.HiddenLayer[0].Inputs[1].Source.FiringRate, exHiddenDelta1 *net.HiddenLayer[1].Inputs[1].Source.FiringRate } }), }; var expectedDeltaValues = new DeltasFor2LayersOfNet { OutputBiases = new[] { 0d }, OutputWeights = new MatrixD( new[, ] { { -0.027674034938717864 }, // -0.040681125112339026d * 0.680267196698649d +/- last two d.p.s { -0.027001636991007407 } // -0.040681125112339026d * 0.663738697404353d +/- last two d.p.s }), HiddenBiases = new[] { 0d, 0d }, HiddenWeights = new MatrixD( new[, ] { { -0.000828975875537991d, -0.0026574571475612243d }, { -0.0021316522513834054d, -0.0068334612365860059d } }), }; calculatedDeltas.OutputWeights.ShouldEqualByValue(expectedOutputDeltas.OutputWeights, "OutputWeights"); calculatedDeltas.OutputBiases.ShouldEqualByValue(expectedOutputDeltas.OutputBiases, "OutputBiases"); calculatedDeltas.HiddenWeights.ShouldEqualByValue(expectedOutputDeltas.HiddenWeights, "HiddenWeights"); calculatedDeltas.HiddenBiases.ShouldEqualByValue(expectedOutputDeltas.HiddenBiases, "HiddenBiases"); // expectedOutputDeltas.HiddenBiases.ShouldEqualByValue(expectedDeltaValues.HiddenBiases, "HiddenBiases"); expectedOutputDeltas.OutputBiases.ShouldEqualByValue(expectedDeltaValues.OutputBiases, "OutputBiases"); expectedOutputDeltas.HiddenWeights.ShouldEqualByValue(expectedDeltaValues.HiddenWeights, "HiddenWeights"); expectedOutputDeltas.OutputWeights.ShouldEqualByValue(expectedDeltaValues.OutputWeights, "OutputWeights"); }