static void Main(string[] args) { double[][] inputs = { new double[] { 0, 0 }, new double[] { 0, 1 }, new double[] { 1, 0 }, new double[] { 1, 1 } }; double[] targets = { 0, 1, 1, 0 }; double error = 1; int epoch = 0; ActivationFunction f = new SigmoidFunction(); Layer layer1 = new Layer(2, 2, f); Layer layer2 = new Layer(1, 2, f); Network network = new Network(new Layer[] { layer1, layer2 }); LearningStrategy learning = new BackpropagationLearning(network, 0.25); while (error > 0.01) { error = learning.RunEpoch(inputs, targets); epoch++; Console.WriteLine("Iteration {0} error: {1}", epoch, error); } Console.WriteLine("Training complete after {0} epochs using the Backpropagation training regime.", epoch); Console.WriteLine("Testing"); network.Update(new double[] { 0, 0 }); Console.WriteLine("{0}", network.Output[0]); network.Update(new double[] { 0, 1 }); Console.WriteLine("{0}", network.Output[0]); network.Update(new double[] { 1, 0 }); Console.WriteLine("{0}", network.Output[0]); network.Update(new double[] { 1, 1 }); Console.WriteLine("{0}", network.Output[0]); }
public void NetworkUpdate_UpdatesAllLayers() { ActivationFunction sigmoidFunction = new SigmoidFunction(); Layer[] layers = new Layer[] { new Layer(2, 2, sigmoidFunction), new Layer(1, 2, sigmoidFunction) }; Network network = new Network(layers); double[] input = new double[] { 0.5, 0.6 }; double[] expected = new double[] { 0.626138674824 }; layers[0][0][0] = 0.1; layers[0][0][1] = 0.2; layers[0][1][0] = 0.3; layers[0][1][1] = 0.4; layers[1][0][0] = 0.5; layers[1][0][1] = 0.5; layers[0][0].Bias = -0.01; layers[0][1].Bias = -0.02; layers[1][0].Bias = -0.05; double[] actual = network.Update(input); double[] networkOutput = network.Output; Assert.AreEqual(expected[0], actual[0], 0.0001, "Invalid network output"); Assert.AreEqual(expected[0], networkOutput[0], 0.0001, "Invalid network output"); }
public void SparseAutoencoderOutputLayerDeltas_ReturnsDeltas() { ActivationFunction sigmoidFunction = new SigmoidFunction(); Layer[] layers = new Layer[] { new Layer(2, 2, sigmoidFunction), new Layer(1, 2, sigmoidFunction) }; Network network = new Network(layers); SparseAutoencoderLearning sparseAutoencoder = new SparseAutoencoderLearning(network); double[] input = new double[] { 0.5, 0.6 }; double[] target = new double[] { 1 }; double[] expected = new double[] { -0.0875168367272141 }; layers[0][0][0] = 0.1; layers[0][0][1] = 0.2; layers[0][1][0] = 0.3; layers[0][1][1] = 0.4; layers[1][0][0] = 0.5; layers[1][0][1] = 0.5; layers[0][0].Bias = -0.01; layers[0][1].Bias = -0.02; layers[1][0].Bias = -0.05; network.Update(input); double[] actual = sparseAutoencoder.OutputLayerDeltas(target); Assert.AreEqual(expected[0], actual[0], 0.0001, "Invalid output layer delta"); }
public void SparseAutoencoder_GradientChecking() { ActivationFunction sigmoidFunction = new SigmoidFunction(); Layer[] layers = new Layer[] { new Layer(2, 2, sigmoidFunction), new Layer(1, 2, sigmoidFunction) }; Network network = new Network(layers); SparseAutoencoderLearning sparseAutoencoder = new SparseAutoencoderLearning(network); double[][] input = new double[][] { new double[] { 0.5, 0.6 } }; double[][] target = new double[][] { new double[] { 1 } }; double[][][] gradient = new double[][][] { new double[][] { new double[] { 0, 0 }, new double[] { 0, 0 } }, new double[][] { new double[] { 0, 0 } } }; layers[0][0][0] = 0.1; layers[0][0][1] = 0.2; layers[0][1][0] = 0.3; layers[0][1][1] = 0.4; layers[1][0][0] = 0.5; layers[1][0][1] = 0.5; layers[0][0].Bias = 0.01; layers[0][1].Bias = 0.02; layers[1][0].Bias = 0.05; for (int i = 0; i < network.LayerCount; i++) { for (int j = 0; j < network[i].NeuronCount; j++) { for (int k = 0; k < network[i][j].InputCount; k++) { double tmp = network[i][j][k]; network[i][j][k] = tmp + 0.0001; gradient[i][j][k] += Math.Pow(network.Update(input[0])[0] - target[0][0], 2); network[i][j][k] = tmp - 0.0001; gradient[i][j][k] -= Math.Pow(network.Update(input[0])[0] - target[0][0], 2); network[i][j][k] = tmp; gradient[i][j][k] = 0.5 * gradient[i][j][k] / 0.0002; } } } sparseAutoencoder.UpdateCachedActivations(input); double[][] deltas = sparseAutoencoder.ComputeDeltas(0, target[0]); double[][][] partialDerivatives = sparseAutoencoder.ComputePartialDerivatives(0, deltas, input[0]); Assert.AreEqual(gradient[0][0][0], partialDerivatives[0][0][0], 0.0001, "Gradient checking failed"); Assert.AreEqual(gradient[0][0][1], partialDerivatives[0][0][1], 0.0001, "Gradient checking failed"); Assert.AreEqual(gradient[0][1][0], partialDerivatives[0][1][0], 0.0001, "Gradient checking failed"); Assert.AreEqual(gradient[0][1][1], partialDerivatives[0][1][1], 0.0001, "Gradient checking failed"); Assert.AreEqual(gradient[1][0][0], partialDerivatives[1][0][0], 0.0001, "Gradient checking failed"); Assert.AreEqual(gradient[1][0][1], partialDerivatives[1][0][1], 0.0001, "Gradient checking failed"); }
public void SparseAutoencoder_BatchGradientChecking() { ActivationFunction sigmoidFunction = new SigmoidFunction(); Layer[] layers = new Layer[] { new Layer(2, 2, sigmoidFunction), new Layer(1, 2, sigmoidFunction) }; Network network = new Network(layers); SparseAutoencoderLearning sparseAutoencoder = new SparseAutoencoderLearning(network, 3); double[][] input = new double[][] { new double[] { 0.5, 0.6 }, new double[] { 0.1, 0.2 }, new double[] { 0.3, 0.3 } }; double[][] target = new double[][] { new double[] { 1 }, new double[] { 0 }, new double[] { 0.5 } }; double[][][] gradientWeights = new double[][][] { new double[][] { new double[] { 0, 0 }, new double[] { 0, 0 } }, new double[][] { new double[] { 0, 0 } } }; double[][] gradientBias = new double[][] { new double[] { 0, 0 }, new double[] { 0 } }; double tmp; layers[0][0][0] = 0.1; layers[0][0][1] = 0.2; layers[0][1][0] = 0.3; layers[0][1][1] = 0.4; layers[1][0][0] = 0.5; layers[1][0][1] = 0.5; layers[0][0].Bias = 0.01; layers[0][1].Bias = 0.02; layers[1][0].Bias = 0.05; for (int i = 0; i < network.LayerCount; i++) { for (int j = 0; j < network[i].NeuronCount; j++) { for (int k = 0; k < network[i][j].InputCount; k++) { tmp = network[i][j][k]; network[i][j][k] = tmp + 0.0001; for (int l = 0; l < input.Length; l++) { gradientWeights[i][j][k] += ((double)1 / input.Length) * 0.5 * Math.Pow(network.Update(input[l])[0] - target[l][0], 2); } network[i][j][k] = tmp - 0.0001; for (int l = 0; l < input.Length; l++) { gradientWeights[i][j][k] -= ((double)1 / input.Length) * 0.5 * Math.Pow(network.Update(input[l])[0] - target[l][0], 2); } network[i][j][k] = tmp; gradientWeights[i][j][k] = gradientWeights[i][j][k] / 0.0002; } tmp = network[i][j].Bias; network[i][j].Bias = tmp + 0.0001; for(int l = 0; l < input.Length; l++) { gradientBias[i][j] += ((double)1 / input.Length) * 0.5 * Math.Pow(network.Update(input[l])[0] - target[l][0], 2); } network[i][j].Bias = tmp - 0.0001; for (int l = 0; l < input.Length; l++) { gradientBias[i][j] -= ((double)1 / input.Length) * 0.5 * Math.Pow(network.Update(input[l])[0] - target[l][0], 2); } network[i][j].Bias = tmp; gradientBias[i][j] = gradientBias[i][j] / 0.0002; } } Tuple<double[][][], double[][]> result = sparseAutoencoder.ComputeBatchPartialDerivatives(input, target); double[][][] partialDerivativesWeights = result.Item1; double[][] partialDerivativesBias = result.Item2; Assert.AreEqual(gradientWeights[0][0][0], ((double)1 / input.Length) * partialDerivativesWeights[0][0][0], 0.0001, "Gradient checking failed"); Assert.AreEqual(gradientWeights[0][0][1], ((double)1 / input.Length) * partialDerivativesWeights[0][0][1], 0.0001, "Gradient checking failed"); Assert.AreEqual(gradientWeights[0][1][0], ((double)1 / input.Length) * partialDerivativesWeights[0][1][0], 0.0001, "Gradient checking failed"); Assert.AreEqual(gradientWeights[0][1][1], ((double)1 / input.Length) * partialDerivativesWeights[0][1][1], 0.0001, "Gradient checking failed"); Assert.AreEqual(gradientWeights[1][0][0], ((double)1 / input.Length) * partialDerivativesWeights[1][0][0], 0.0001, "Gradient checking failed"); Assert.AreEqual(gradientWeights[1][0][1], ((double)1 / input.Length) * partialDerivativesWeights[1][0][1], 0.0001, "Gradient checking failed"); Assert.AreEqual(gradientBias[0][0], ((double)1 / input.Length) * partialDerivativesBias[0][0], 0.0001, "Gradient checking failed"); Assert.AreEqual(gradientBias[0][1], ((double)1 / input.Length) * partialDerivativesBias[0][1], 0.0001, "Gradient checking failed"); Assert.AreEqual(gradientBias[1][0], ((double)1 / input.Length) * partialDerivativesBias[1][0], 0.0001, "Gradient checking failed"); }