public void SoftmaxFunctionReturnsCorrectResult() { List<Node> nodes = new List<Node>() { new Node(new SoftmaxActivationFunction()), new Node(new SoftmaxActivationFunction()), new Node(new SoftmaxActivationFunction()) }; nodes[0].Input = 2.0; nodes[1].Input = -1.0; nodes[2].Input = 4.0; Layer layer = new Layer(LayerType.Hidden, new SoftmaxActivationFunction(), 1, 3); layer.Nodes = nodes; double scalingFactor = Math.Exp(2 - 4) + Math.Exp(-1 - 4) + Math.Exp(4 - 4); double result = Math.Exp(2 - 4) / scalingFactor; List<double> inputs = new List<double>(); foreach(Node node in nodes) { inputs.Add(node.Input); } layer.ActivateNodes(); //nodes[0].Softmax(inputs.ToArray()); Console.WriteLine("expected {0}, actual {1}", result, nodes[0].ActivatedSum); Assert.AreEqual(result, nodes[0].ActivatedSum); }
public Layer(Layer layer) { m_inputsCount = layer.m_inputsCount; m_neuronsCount = layer.m_neuronsCount; m_function = layer.m_function; m_neurons = new Neuron[m_neuronsCount]; for (int i = 0; i < m_neuronsCount; i++) m_neurons[i] = new Neuron(layer[i]); m_output = (double[])layer.m_output.Clone(); }
//public Network(int[] layersTopology, IActivationFunction[] functions) //{ // m_layers = new Layer[layersTopology.Length]; // int layer_input = layersTopology[0]; // for (int i = 0; i < layersTopology.Length; i++) // { // m_layers[i] = new Layer(layer_input, layersTopology[i], functions[i]); // layer_input = layersTopology[i]; // } //} public Network(int inputsCount, int[] layersTopology, double[][] thresholds, IActivationFunction[] functions) { m_layers = new Layer[layersTopology.Length]; int layer_input = inputsCount; for (int i = 0; i < layersTopology.Length; i++) { m_layers[i] = new Layer(layer_input, layersTopology[i], thresholds[i], functions[i]); layer_input = layersTopology[i]; } }
public void Activate(Layer layer = null) { if (this.ActivationFunction == null) { this.ActivatedSum = this.Input; } else { this.ActivatedSum = this.ActivationFunction.Activate(this, layer); } }
private static void ChangeWeights(double n, Layer[] layers, int l) { double dw; for (int i = 0; i < layers[l].cellCount; i++) { for (int j = 0; j < layers[l - 1].cellCount; j++) { dw = -n * layers[l].errors[i] * layers[l - 1].output[j]; layers[l].weights[j, i] += dw; } dw = -n * layers[l].errors[i]; layers[l].bias[i] += dw; } }
public void ActivateTanhSettingNodeValuesCorrectly() { Layer layer = new Layer(LayerType.Hidden, new HyperbolicTangentActivationFunction(), 1, 3); layer.Nodes[0].Input = 2.0; layer.Nodes[1].Input = -1.0; layer.Nodes[2].Input = 4.0; double result1 = Math.Tanh(layer.Nodes[0].Input); double result2 = Math.Tanh(layer.Nodes[1].Input); double result3 = Math.Tanh(layer.Nodes[2].Input); layer.ActivateNodes(); Assert.AreEqual(result1, layer.Nodes[0].ActivatedSum); Assert.AreEqual(result2, layer.Nodes[1].ActivatedSum); Assert.AreEqual(result3, layer.Nodes[2].ActivatedSum); }
public void ActivateSigmoidSettingNodeValuesCorrectly() { Layer layer = new Layer(LayerType.Hidden, new LogisticSigmoidActivationFunction(), 1, 3); layer.Nodes[0].Input = 2.0; layer.Nodes[1].Input = -1.0; layer.Nodes[2].Input = 4.0; double result1 = (1.0 / (1.0 + Math.Exp(-(layer.Nodes[0].Input)))); double result2 = (1.0 / (1.0 + Math.Exp(-(layer.Nodes[1].Input)))); double result3 = (1.0 / (1.0 + Math.Exp(-(layer.Nodes[2].Input)))); layer.ActivateNodes(); Assert.AreEqual(result1, layer.Nodes[0].ActivatedSum); Assert.AreEqual(result2, layer.Nodes[1].ActivatedSum); Assert.AreEqual(result3, layer.Nodes[2].ActivatedSum); }
public void ActivateSoftmaxSettingNodeValuesCorrectly() { Layer layer = new Layer(LayerType.Hidden, new SoftmaxActivationFunction(), 1, 3); layer.Nodes[0].Input = 2.0; layer.Nodes[1].Input = -1.0; layer.Nodes[2].Input = 4.0; double scalingFactor = Math.Exp(2 - 4) + Math.Exp(-1 - 4) + Math.Exp(4 - 4); double result1 = Math.Exp(2 - 4) / scalingFactor; double result2 = Math.Exp(-1 - 4) / scalingFactor; double result3 = Math.Exp(4 - 4) / scalingFactor; layer.ActivateNodes(); Assert.AreEqual(result1, layer.Nodes[0].ActivatedSum); Assert.AreEqual(result2, layer.Nodes[1].ActivatedSum); Assert.AreEqual(result3, layer.Nodes[2].ActivatedSum); }
public FeedForwardNeuralNetwork(int nrOfInputs, int nrOfOutputs, int nrHiddenLayers, int nrOfNeuronsPerHiddenLayer, float bias, float[] weights = null) { this.bias = bias; this.NrOfInputs = nrOfInputs; this.NrOfHiddenLayers = nrHiddenLayers; this.NrOfOutputs = nrOfOutputs; this.NrOfNeuronsPerHiddenLayer = nrOfNeuronsPerHiddenLayer; layers = new Layer[nrHiddenLayers + 1]; layers[0] = new Layer(nrOfInputs, nrOfNeuronsPerHiddenLayer); for (int i = 1; i < nrHiddenLayers; i++) layers[i] = new Layer(nrOfNeuronsPerHiddenLayer, nrOfNeuronsPerHiddenLayer); // last layer is the output layer layers[nrHiddenLayers] = new Layer(nrOfNeuronsPerHiddenLayer, nrOfOutputs); if (weights == null) weights = GetRandomWeights(weights); SetWeights(weights); }
public void LayerConstructorCreatesCorrectNumberOfNodes() { Layer layer = new Layer(LayerType.Input, null, 0, 4); Assert.AreEqual(4, layer.Nodes.Count); }
/// <summary> /// Create a new neural network /// with "inputs" inputs and size of "layers" /// layers of neurones. /// The layer i is made with layers_desc[i] neurones. /// The activation function of each neuron is set to default (Sigmoid with beta = 1). /// The lerning algorithm is set to default (Back Propagation). /// </summary> /// <param name="inputs">Number of inputs of the network</param> /// <param name="layers_desc">Number of neurons for each layer of the network</param> public NeuralNetwork(int inputs, int[] layers_desc) { if (layers_desc.Length<1) throw new Exception("PERCEPTRON : cannot build perceptron, it must have at least 1 layer of neurone"); if (inputs<1) throw new Exception("PERCEPTRON : cannot build perceptron, it must have at least 1 input"); la = new BackPropagationLearningAlgorithm(this); ni = inputs; ActivationFunction n_act = new SigmoidActivationFunction(); layers = new Layer[layers_desc.Length]; layers[0] = new Layer(layers_desc[0], ni); for(int i=1; i<layers_desc.Length; i++) layers[i] = new Layer(layers_desc[i],layers_desc[i-1],n_act); }
public void FeedForwardSetsNextLayerInputsCorrectly() { List<double> biases = new List<double>() { 1, 1, 1, 1 }; List<double> weights = new List<double>() { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; Layer input = new Layer(LayerType.Input, null, 0, 3); Layer hidden = new Layer(LayerType.Hidden, new LogisticSigmoidActivationFunction(), 1, 4); input.Weights = weights; input.Biases = biases; input.Nodes[0].Input = 1; input.Nodes[1].Input = 2; input.Nodes[2].Input = 3; input.Nodes[0].ActivatedSum = 1; input.Nodes[1].ActivatedSum = 2; input.Nodes[2].ActivatedSum = 3; input.ActivateNodes(); input.FeedForward(hidden); Assert.AreEqual(7, hidden.Nodes[0].Input); }
public void LayerConstructorThrowsIfNumberOfNodesLessThanOne() { Layer layer = new Layer(LayerType.Input, null, 0, 0); }
public InputNeuron(Layer l) : base(l) { }
static void createNeuralNetwork(String[] words) { Random r = Lib.NeuralNetwork.random; try { if (Char.IsDigit(words[1].ToCharArray()[0])) { int[] nodes = new int[words.Length - 1]; for (int i = 0; i < nodes.Length; i++) { nodes[i] = int.Parse(words[i + 1]); } nn = new Lib.NeuralNetwork(r, nodes); } else { Layer[] layers = new Layer[(words.Length - 1) / 2]; for (int i = 1; i < words.Length; i += 2) { if (words[i].Equals("relu")) { int nodes = int.Parse(words[i + 1]); layers[(i - 1) / 2] = new ReluLayer(nodes); } else if (words[i].Equals("do")) { int nodes = int.Parse(words[i + 1]); layers[(i - 1) / 2] = new DropoutLayer(nodes); } else if (words[i].Equals("lrelu")) { int nodes = int.Parse(words[i + 1]); layers[(i - 1) / 2] = new Lib.Layers.LeakyReluLayer(nodes); } else { Console.WriteLine("There is no layer type like \"" + words[i] + "\""); return; } } for (int i = 0; i < layers.Length; i++) { if (i != layers.Length - 1) { layers[i].weights = new Matrix(layers[i + 1].nodes, layers[i].nodes); layers[i].weights.randomize(r); } layers[i].bias = new Matrix(layers[i].nodes, 1); layers[i].bias.randomize(r); } nn = new Lib.NeuralNetwork(Lib.NeuralNetwork.random, layers); } } catch (Exception e) { printError(e); return; } Console.WriteLine("Neural Network was created succesfully!"); }
public void NeuralNetConstructorAssignsCorrectNumberOfWeightsToEachLayer() { Layer input = new Layer(LayerType.Input, null, 0, 3); Layer hidden = new Layer(LayerType.Hidden, new LogisticSigmoidActivationFunction(), 1, 4); Layer output = new Layer(LayerType.Output, new SoftmaxActivationFunction(), 2, 2); input.Nodes[0].Input = 1; input.Nodes[1].Input = 2; input.Nodes[2].Input = 3; input.Nodes[0].ActivatedSum = 1; input.Nodes[1].ActivatedSum = 2; input.Nodes[2].ActivatedSum = 3; List<Layer> layers = new List<Layer>(); layers.Add(input); layers.Add(hidden); layers.Add(output); //24 weights List<double> weights = new List<double>() { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; //6 biases List<double> biases = new List<double>() { 1, 1, 1, 1, 1, 1 }; NeuralNet nn = new NeuralNet(layers, weights, biases); Assert.AreEqual(nn.Layers[0].Nodes.Count * nn.Layers[1].Nodes.Count, nn.Layers[0].Weights.Count); Assert.AreEqual(nn.Layers[1].Nodes.Count * nn.Layers[2].Nodes.Count, nn.Layers[1].Weights.Count); }
public static void Main(string[] args) { //// inputs Byte[] inputByte = MyMath.GetRandomBytes(2); double n = 0.1; int layersCount = 2; Layer[] layers = new Layer[layersCount + 1]; layers[0] = new Layer(); layers[0].cellCount = inputByte.Length; layers[0].output = new double[inputByte.Length]; for (int i = 0; i < inputByte.Length; i++) layers[0].output[i] = inputByte[i] / 255.0; layers[1] = new Layer(); layers[1].cellCount = 3; layers[1].function = TransferFunction.Linear; layers[1].bias = MyMath.GetRandomDoubles(layers[1].cellCount); layers[1].weights = MyMath.GetRandom(layers[0].cellCount, layers[1].cellCount); layers[2] = new Layer(); layers[2].cellCount = 1; layers[2].function = TransferFunction.Linear; layers[2].bias = MyMath.GetRandomDoubles(layers[2].cellCount); layers[2].weights = MyMath.GetRandom(layers[1].cellCount, layers[2].cellCount); for (int z = 0; z < 50; z++) { //feed forward all for (int i = 1; i < layers.Length; i++) layers[i].FeedForward(layers[i - 1].output); //backpropagate last layer int l = layers.Length - 1; double[] targetOutput = new double[layers[l].cellCount]; //calculate errors layers[l].errors = new double[layers[l].cellCount]; for (int i = 0; i < targetOutput.Length; i++) layers[l].errors[i] = layers[l].output[i] - targetOutput[i]; Console.WriteLine("FeedForward error: " + Math.Round(MyMath.Sum(layers[l].errors),4)); //calculate weights ChangeWeights(n, layers, l); //backpropagate other layers for (l = layers.Length - 2; l > 0; l--) { //calculate errors layers[l].errors = new double[layers[l].cellCount]; for (int i = 0; i < layers[l].cellCount; i++) for (int j = 0; j < layers[l + 1].cellCount; j++) layers[l].errors[i] = layers[l + 1].errors[j] * layers[l + 1].weights[i, j]; //calculate weights ChangeWeights(n, layers, l); } } Console.ReadKey(); }
public void LayerConstructorAllowsLayersWithActivationTypeNone() { Layer input = new Layer(LayerType.Input, null, 0, 4); Layer hidden = new Layer(LayerType.Hidden, null, 1, 4); Layer output = new Layer(LayerType.Output, null, 2, 4); }
public void FeedForwardThrowsIfLayerActivatedSumsAreNull() { Layer hidden = new Layer(LayerType.Hidden, new LogisticSigmoidActivationFunction(), 1, 4); Layer output = new Layer(LayerType.Output, new SoftmaxActivationFunction(), 2, 3); hidden.Nodes[0].Input = 1; hidden.Nodes[1].Input = 2; hidden.Nodes[2].Input = 3; hidden.Nodes[3].Input = 4; hidden.Biases = new List<double>() { 1, 1, 1 }; hidden.Weights = new List<double>() {1,1,1,1,1,1,1,1,1,1,1,1}; hidden.FeedForward(output); }
public void FeedForwardThrowsIfBiasesDoesNotEqualNextLayerNodes() { Layer input = new Layer(LayerType.Input, null, 0, 5); Layer hidden = new Layer( LayerType.Hidden, new LogisticSigmoidActivationFunction(), 1, 4); input.Nodes[0].Input = 1; input.Nodes[1].Input = 2; input.Nodes[2].Input = 3; input.Nodes[3].Input = 4; input.Nodes[0].ActivatedSum = 1; input.Nodes[1].ActivatedSum = 2; input.Nodes[2].ActivatedSum = 3; input.Nodes[3].ActivatedSum = 4; input.ActivateNodes(); input.FeedForward(hidden); }
public void NeuralNetConstructorOrdersLayersAscendingByLayerOrder() { Layer input = new Layer(LayerType.Input, null, 0, 3); Layer output = new Layer(LayerType.Output, new SoftmaxActivationFunction(), 2, 2); Layer hidden = new Layer(LayerType.Hidden, new LogisticSigmoidActivationFunction(), 1, 4); input.Nodes[0].Input = 1; input.Nodes[1].Input = 2; input.Nodes[2].Input = 3; input.Nodes[0].ActivatedSum = 1; input.Nodes[1].ActivatedSum = 2; input.Nodes[2].ActivatedSum = 3; List<Layer> layers = new List<Layer>(); layers.Add(input); layers.Add(output); layers.Add(hidden); //24 weights List<double> weights = new List<double>() { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; //6 biases List<double> biases = new List<double>() { 1, 1, 1, 1, 1, 1 }; NeuralNet nn = new NeuralNet(layers, weights, biases); //nn.Run(); Assert.AreEqual(0, nn.Layers[0].LayerOrder); Assert.AreEqual(1, nn.Layers[1].LayerOrder); Assert.AreEqual(2, nn.Layers[2].LayerOrder); }
public void LayerConstructorThrowsIfLayerTypeInputAndActivationFunctionNotNull() { Layer layer = new Layer(LayerType.Input, new LogisticSigmoidActivationFunction(), 0, 5); }
public void LayerConstructorThrowsIfLayerTypeInputAndLayerOrderNotZero() { Layer layer = new Layer(LayerType.Input, null, 1, 5); }
public void LayerConstructorThrowsIfLayerTypeNotInputAndLayerOrderZero() { Layer layer = new Layer(LayerType.Hidden, new LogisticSigmoidActivationFunction(), 0, 5); }
public Network(Network network) { m_layers = new Layer[network.m_layers.Length]; for (int i = 0; i < m_layers.Length; i++) m_layers[i] = new Layer(network[i]); }
public void FeedForward(Layer nextLayer) { if (this.Biases.Count != nextLayer.Nodes.Count || this.Weights.Count != nextLayer.Nodes.Count * this.Nodes.Count) { throw new Exception("Wrong number of weights or biases."); } foreach(Node node in this.Nodes) { if (node.ActivatedSum == null) { throw new Exception("Nodes must be activated before FeedForward is called."); } } for(int i = 0; i < nextLayer.Nodes.Count; i++) { double input = 0; for(int p = 0; p < this.Nodes.Count; p++) { input += Convert.ToDouble(this.Nodes[p].ActivatedSum) * this.Weights[(i * this.Nodes.Count) + p]; } input += this.Biases[i]; nextLayer.Nodes[i].Input = input; } }