public void SoftmaxFunctionReturnsCorrectResult()
        {
            List<Node> nodes = new List<Node>()
            {
                new Node(new SoftmaxActivationFunction()),
                new Node(new SoftmaxActivationFunction()),
                new Node(new SoftmaxActivationFunction())
            };

            nodes[0].Input = 2.0;
            nodes[1].Input = -1.0;
            nodes[2].Input = 4.0;

            Layer layer = new Layer(LayerType.Hidden, new SoftmaxActivationFunction(), 1, 3);
            layer.Nodes = nodes;

            double scalingFactor = Math.Exp(2 - 4) + Math.Exp(-1 - 4) + Math.Exp(4 - 4);
            double result = Math.Exp(2 - 4) / scalingFactor;

            List<double> inputs = new List<double>();
            foreach(Node node in nodes)
            {
                inputs.Add(node.Input);
            }

            layer.ActivateNodes();

            //nodes[0].Softmax(inputs.ToArray());

            Console.WriteLine("expected {0}, actual {1}", result, nodes[0].ActivatedSum);

            Assert.AreEqual(result, nodes[0].ActivatedSum);
        }
Beispiel #2
0
 public Layer(Layer layer)
 {
     m_inputsCount = layer.m_inputsCount;
     m_neuronsCount = layer.m_neuronsCount;
     m_function = layer.m_function;
     m_neurons = new Neuron[m_neuronsCount];
     for (int i = 0; i < m_neuronsCount; i++)
         m_neurons[i] = new Neuron(layer[i]);
     m_output = (double[])layer.m_output.Clone();
 }
Beispiel #3
0
        //public Network(int[] layersTopology, IActivationFunction[] functions)
        //{
        //    m_layers = new Layer[layersTopology.Length];
        //    int layer_input = layersTopology[0];
        //    for (int i = 0; i < layersTopology.Length; i++)
        //    {
        //        m_layers[i] = new Layer(layer_input, layersTopology[i], functions[i]);
        //        layer_input = layersTopology[i];
        //    }
        //}

        public Network(int inputsCount, int[] layersTopology, double[][] thresholds, IActivationFunction[] functions)
        {
            m_layers = new Layer[layersTopology.Length];
            int layer_input = inputsCount;
            for (int i = 0; i < layersTopology.Length; i++)
            {
                m_layers[i] = new Layer(layer_input, layersTopology[i], thresholds[i], functions[i]);
                layer_input = layersTopology[i];
            }
        }
 public void Activate(Layer layer = null)
 {
     if (this.ActivationFunction == null)
     {
         this.ActivatedSum = this.Input;
     }
     else
     {
         this.ActivatedSum = this.ActivationFunction.Activate(this, layer);
     }
 }
Beispiel #5
0
 private static void ChangeWeights(double n, Layer[] layers, int l)
 {
     double dw;
     for (int i = 0; i < layers[l].cellCount; i++)
     {
         for (int j = 0; j < layers[l - 1].cellCount; j++)
         {
             dw = -n * layers[l].errors[i] * layers[l - 1].output[j];
             layers[l].weights[j, i] += dw;
         }
         dw = -n * layers[l].errors[i];
         layers[l].bias[i] += dw;
     }
 }
        public void ActivateTanhSettingNodeValuesCorrectly()
        {
            Layer layer = new Layer(LayerType.Hidden, new HyperbolicTangentActivationFunction(), 1, 3);

            layer.Nodes[0].Input = 2.0;
            layer.Nodes[1].Input = -1.0;
            layer.Nodes[2].Input = 4.0;

            double result1 = Math.Tanh(layer.Nodes[0].Input);
            double result2 = Math.Tanh(layer.Nodes[1].Input);
            double result3 = Math.Tanh(layer.Nodes[2].Input);

            layer.ActivateNodes();

            Assert.AreEqual(result1, layer.Nodes[0].ActivatedSum);
            Assert.AreEqual(result2, layer.Nodes[1].ActivatedSum);
            Assert.AreEqual(result3, layer.Nodes[2].ActivatedSum);
        }
        public void ActivateSigmoidSettingNodeValuesCorrectly()
        {
            Layer layer = new Layer(LayerType.Hidden, new LogisticSigmoidActivationFunction(), 1, 3);

            layer.Nodes[0].Input = 2.0;
            layer.Nodes[1].Input = -1.0;
            layer.Nodes[2].Input = 4.0;

            double result1 = (1.0 / (1.0 + Math.Exp(-(layer.Nodes[0].Input))));
            double result2 = (1.0 / (1.0 + Math.Exp(-(layer.Nodes[1].Input))));
            double result3 = (1.0 / (1.0 + Math.Exp(-(layer.Nodes[2].Input))));

            layer.ActivateNodes();

            Assert.AreEqual(result1, layer.Nodes[0].ActivatedSum);
            Assert.AreEqual(result2, layer.Nodes[1].ActivatedSum);
            Assert.AreEqual(result3, layer.Nodes[2].ActivatedSum);
        }
        public void ActivateSoftmaxSettingNodeValuesCorrectly()
        {
            Layer layer = new Layer(LayerType.Hidden, new SoftmaxActivationFunction(), 1, 3);

            layer.Nodes[0].Input = 2.0;
            layer.Nodes[1].Input = -1.0;
            layer.Nodes[2].Input = 4.0;

            double scalingFactor = Math.Exp(2 - 4) + Math.Exp(-1 - 4) + Math.Exp(4 - 4);
            double result1 = Math.Exp(2 - 4) / scalingFactor;
            double result2 = Math.Exp(-1 - 4) / scalingFactor;
            double result3 = Math.Exp(4 - 4) / scalingFactor;

            layer.ActivateNodes();

            Assert.AreEqual(result1, layer.Nodes[0].ActivatedSum);
            Assert.AreEqual(result2, layer.Nodes[1].ActivatedSum);
            Assert.AreEqual(result3, layer.Nodes[2].ActivatedSum);
        }
        public FeedForwardNeuralNetwork(int nrOfInputs, int nrOfOutputs, int nrHiddenLayers, int nrOfNeuronsPerHiddenLayer, float bias, float[] weights = null)
        {
            this.bias = bias;

            this.NrOfInputs = nrOfInputs;
            this.NrOfHiddenLayers = nrHiddenLayers;
            this.NrOfOutputs = nrOfOutputs;
            this.NrOfNeuronsPerHiddenLayer = nrOfNeuronsPerHiddenLayer;

            layers = new Layer[nrHiddenLayers + 1];

            layers[0] = new Layer(nrOfInputs, nrOfNeuronsPerHiddenLayer);
            for (int i = 1; i < nrHiddenLayers; i++)
                layers[i] = new Layer(nrOfNeuronsPerHiddenLayer, nrOfNeuronsPerHiddenLayer);

            // last layer is the output layer
            layers[nrHiddenLayers] = new Layer(nrOfNeuronsPerHiddenLayer, nrOfOutputs);

            if (weights == null)
                weights = GetRandomWeights(weights);

            SetWeights(weights);
        }
        public void LayerConstructorCreatesCorrectNumberOfNodes()
        {
            Layer layer = new Layer(LayerType.Input, null, 0, 4);

            Assert.AreEqual(4, layer.Nodes.Count);
        }
		/// <summary>
		/// Create a new neural network
		/// with "inputs" inputs and size of "layers"
		/// layers of neurones.
		/// The layer i is made with layers_desc[i] neurones.
		/// The activation function of each neuron is set to default (Sigmoid with beta = 1).
		/// The lerning algorithm is set to default (Back Propagation).
		/// </summary>
		/// <param name="inputs">Number of inputs of the network</param>
		/// <param name="layers_desc">Number of neurons for each layer of the network</param>
		public NeuralNetwork(int inputs, int[] layers_desc)
		{
			if (layers_desc.Length<1)
				throw new Exception("PERCEPTRON : cannot build perceptron, it must have at least 1 layer of neurone");
			if (inputs<1)
				throw new Exception("PERCEPTRON : cannot build perceptron, it must have at least 1 input");
			la = new BackPropagationLearningAlgorithm(this);
			ni = inputs;
			ActivationFunction n_act = new SigmoidActivationFunction();
			layers = new Layer[layers_desc.Length];
			layers[0] = new Layer(layers_desc[0], ni);
			for(int i=1; i<layers_desc.Length; i++) 
				layers[i] = new Layer(layers_desc[i],layers_desc[i-1],n_act);
		}
        public void FeedForwardSetsNextLayerInputsCorrectly()
        {
            List<double> biases = new List<double>()
            {
                1,
                1,
                1,
                1
            };

            List<double> weights = new List<double>()
            {
                1,
                1,
                1,

                1,
                1,
                1,

                1,
                1,
                1,

                1,
                1,
                1
            };

            Layer input = new Layer(LayerType.Input, null, 0, 3);
            Layer hidden = new Layer(LayerType.Hidden, new LogisticSigmoidActivationFunction(), 1, 4);

            input.Weights = weights;
            input.Biases = biases;

            input.Nodes[0].Input = 1;
            input.Nodes[1].Input = 2;
            input.Nodes[2].Input = 3;

            input.Nodes[0].ActivatedSum = 1;
            input.Nodes[1].ActivatedSum = 2;
            input.Nodes[2].ActivatedSum = 3;

            input.ActivateNodes();

            input.FeedForward(hidden);

            Assert.AreEqual(7, hidden.Nodes[0].Input);
        }
 public void LayerConstructorThrowsIfNumberOfNodesLessThanOne()
 {
     Layer layer = new Layer(LayerType.Input, null, 0, 0);
 }
Beispiel #14
0
 public InputNeuron(Layer l) : base(l)
 {
 }
Beispiel #15
0
        static void createNeuralNetwork(String[] words)
        {
            Random r = Lib.NeuralNetwork.random;

            try
            {
                if (Char.IsDigit(words[1].ToCharArray()[0]))
                {
                    int[] nodes = new int[words.Length - 1];
                    for (int i = 0; i < nodes.Length; i++)
                    {
                        nodes[i] = int.Parse(words[i + 1]);
                    }
                    nn = new Lib.NeuralNetwork(r, nodes);
                }
                else
                {
                    Layer[] layers = new Layer[(words.Length - 1) / 2];

                    for (int i = 1; i < words.Length; i += 2)
                    {
                        if (words[i].Equals("relu"))
                        {
                            int nodes = int.Parse(words[i + 1]);
                            layers[(i - 1) / 2] = new ReluLayer(nodes);
                        }
                        else if (words[i].Equals("do"))
                        {
                            int nodes = int.Parse(words[i + 1]);
                            layers[(i - 1) / 2] = new DropoutLayer(nodes);
                        }
                        else if (words[i].Equals("lrelu"))
                        {
                            int nodes = int.Parse(words[i + 1]);
                            layers[(i - 1) / 2] = new Lib.Layers.LeakyReluLayer(nodes);
                        }
                        else
                        {
                            Console.WriteLine("There is no layer type like \"" + words[i] + "\"");
                            return;
                        }
                    }

                    for (int i = 0; i < layers.Length; i++)
                    {
                        if (i != layers.Length - 1)
                        {
                            layers[i].weights = new Matrix(layers[i + 1].nodes, layers[i].nodes);
                            layers[i].weights.randomize(r);
                        }

                        layers[i].bias = new Matrix(layers[i].nodes, 1);
                        layers[i].bias.randomize(r);
                    }

                    nn = new Lib.NeuralNetwork(Lib.NeuralNetwork.random, layers);
                }
            } catch (Exception e)
            {
                printError(e);
                return;
            }
            Console.WriteLine("Neural Network was created succesfully!");
        }
        public void NeuralNetConstructorAssignsCorrectNumberOfWeightsToEachLayer()
        {
            Layer input = new Layer(LayerType.Input, null, 0, 3);
            Layer hidden = new Layer(LayerType.Hidden, new LogisticSigmoidActivationFunction(), 1, 4);
            Layer output = new Layer(LayerType.Output, new SoftmaxActivationFunction(), 2, 2);

            input.Nodes[0].Input = 1;
            input.Nodes[1].Input = 2;
            input.Nodes[2].Input = 3;

            input.Nodes[0].ActivatedSum = 1;
            input.Nodes[1].ActivatedSum = 2;
            input.Nodes[2].ActivatedSum = 3;

            List<Layer> layers = new List<Layer>();
            layers.Add(input);
            layers.Add(hidden);
            layers.Add(output);

            //24 weights
            List<double> weights = new List<double>()
            {
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1
            };

            //6 biases
            List<double> biases = new List<double>()
            {
                1,
                1,
                1,
                1,
                1,
                1
            };

            NeuralNet nn = new NeuralNet(layers, weights, biases);

            Assert.AreEqual(nn.Layers[0].Nodes.Count * nn.Layers[1].Nodes.Count, nn.Layers[0].Weights.Count);
            Assert.AreEqual(nn.Layers[1].Nodes.Count * nn.Layers[2].Nodes.Count, nn.Layers[1].Weights.Count);
        }
Beispiel #17
0
        public static void Main(string[] args)
        {
            //// inputs            
            Byte[] inputByte = MyMath.GetRandomBytes(2);

            double n = 0.1;
            int layersCount = 2;
            Layer[] layers = new Layer[layersCount + 1];

            layers[0] = new Layer();
            layers[0].cellCount = inputByte.Length;
            layers[0].output = new double[inputByte.Length];
            for (int i = 0; i < inputByte.Length; i++)
                layers[0].output[i] = inputByte[i] / 255.0;

            layers[1] = new Layer();
            layers[1].cellCount = 3;
            layers[1].function = TransferFunction.Linear;
            layers[1].bias = MyMath.GetRandomDoubles(layers[1].cellCount);
            layers[1].weights = MyMath.GetRandom(layers[0].cellCount, layers[1].cellCount);

            layers[2] = new Layer();
            layers[2].cellCount = 1;
            layers[2].function = TransferFunction.Linear;
            layers[2].bias = MyMath.GetRandomDoubles(layers[2].cellCount);
            layers[2].weights = MyMath.GetRandom(layers[1].cellCount, layers[2].cellCount);            

            for (int z = 0; z < 50; z++)
            {
                //feed forward all            
                for (int i = 1; i < layers.Length; i++)
                    layers[i].FeedForward(layers[i - 1].output);

                //backpropagate last layer
                int l = layers.Length - 1;

                double[] targetOutput = new double[layers[l].cellCount];

                //calculate errors
                layers[l].errors = new double[layers[l].cellCount];
                for (int i = 0; i < targetOutput.Length; i++)
                    layers[l].errors[i] = layers[l].output[i] - targetOutput[i];

                Console.WriteLine("FeedForward error: " + Math.Round(MyMath.Sum(layers[l].errors),4));

                //calculate weights
                ChangeWeights(n, layers, l);

                //backpropagate other layers
                for (l = layers.Length - 2; l > 0; l--)
                {
                    //calculate errors
                    layers[l].errors = new double[layers[l].cellCount];
                    for (int i = 0; i < layers[l].cellCount; i++)
                        for (int j = 0; j < layers[l + 1].cellCount; j++)
                            layers[l].errors[i] = layers[l + 1].errors[j] * layers[l + 1].weights[i, j];

                    //calculate weights
                    ChangeWeights(n, layers, l);
                }
            }

            Console.ReadKey();
        }
 public void LayerConstructorAllowsLayersWithActivationTypeNone()
 {
     Layer input = new Layer(LayerType.Input, null, 0, 4);
     Layer hidden = new Layer(LayerType.Hidden, null, 1, 4);
     Layer output = new Layer(LayerType.Output, null, 2, 4);
 }
        public void FeedForwardThrowsIfLayerActivatedSumsAreNull()
        {
            Layer hidden = new Layer(LayerType.Hidden, new LogisticSigmoidActivationFunction(), 1, 4);
            Layer output = new Layer(LayerType.Output, new SoftmaxActivationFunction(), 2, 3);

            hidden.Nodes[0].Input = 1;
            hidden.Nodes[1].Input = 2;
            hidden.Nodes[2].Input = 3;
            hidden.Nodes[3].Input = 4;

            hidden.Biases = new List<double>() { 1, 1, 1 };
            hidden.Weights = new List<double>() {1,1,1,1,1,1,1,1,1,1,1,1};

            hidden.FeedForward(output);
        }
        public void FeedForwardThrowsIfBiasesDoesNotEqualNextLayerNodes()
        {
            Layer input = new Layer(LayerType.Input, null, 0, 5);
            Layer hidden = new Layer( LayerType.Hidden, new LogisticSigmoidActivationFunction(), 1, 4);

            input.Nodes[0].Input = 1;
            input.Nodes[1].Input = 2;
            input.Nodes[2].Input = 3;
            input.Nodes[3].Input = 4;

            input.Nodes[0].ActivatedSum = 1;
            input.Nodes[1].ActivatedSum = 2;
            input.Nodes[2].ActivatedSum = 3;
            input.Nodes[3].ActivatedSum = 4;

            input.ActivateNodes();

            input.FeedForward(hidden);
        }
        public void NeuralNetConstructorOrdersLayersAscendingByLayerOrder()
        {
            Layer input = new Layer(LayerType.Input, null, 0, 3);
            Layer output = new Layer(LayerType.Output, new SoftmaxActivationFunction(), 2, 2);
            Layer hidden = new Layer(LayerType.Hidden, new LogisticSigmoidActivationFunction(), 1, 4);

            input.Nodes[0].Input = 1;
            input.Nodes[1].Input = 2;
            input.Nodes[2].Input = 3;

            input.Nodes[0].ActivatedSum = 1;
            input.Nodes[1].ActivatedSum = 2;
            input.Nodes[2].ActivatedSum = 3;

            List<Layer> layers = new List<Layer>();
            layers.Add(input);
            layers.Add(output);
            layers.Add(hidden);

            //24 weights
            List<double> weights = new List<double>()
            {
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1
            };

            //6 biases
            List<double> biases = new List<double>()
            {
                1,
                1,
                1,
                1,
                1,
                1
            };

            NeuralNet nn = new NeuralNet(layers, weights, biases);

            //nn.Run();

            Assert.AreEqual(0, nn.Layers[0].LayerOrder);
            Assert.AreEqual(1, nn.Layers[1].LayerOrder);
            Assert.AreEqual(2, nn.Layers[2].LayerOrder);
        }
 public void LayerConstructorThrowsIfLayerTypeInputAndActivationFunctionNotNull()
 {
     Layer layer = new Layer(LayerType.Input, new LogisticSigmoidActivationFunction(), 0, 5);
 }
 public void LayerConstructorThrowsIfLayerTypeInputAndLayerOrderNotZero()
 {
     Layer layer = new Layer(LayerType.Input, null, 1, 5);
 }
 public void LayerConstructorThrowsIfLayerTypeNotInputAndLayerOrderZero()
 {
     Layer layer = new Layer(LayerType.Hidden, new LogisticSigmoidActivationFunction(), 0, 5);
 }
Beispiel #25
0
 public Network(Network network)
 {
     m_layers = new Layer[network.m_layers.Length];
     for (int i = 0; i < m_layers.Length; i++)
         m_layers[i] = new Layer(network[i]);            
 }
        public void FeedForward(Layer nextLayer)
        {
            if (this.Biases.Count != nextLayer.Nodes.Count || this.Weights.Count != nextLayer.Nodes.Count * this.Nodes.Count)
            {
                throw new Exception("Wrong number of weights or biases.");
            }

            foreach(Node node in this.Nodes)
            {
                if (node.ActivatedSum == null)
                {
                    throw new Exception("Nodes must be activated before FeedForward is called.");
                }
            }

            for(int i = 0; i < nextLayer.Nodes.Count; i++)
            {
                double input = 0;

                for(int p = 0; p < this.Nodes.Count; p++)
                {
                    input += Convert.ToDouble(this.Nodes[p].ActivatedSum) * this.Weights[(i * this.Nodes.Count) + p];
                }

                input += this.Biases[i];

                nextLayer.Nodes[i].Input = input;
            }
        }