/// <summary>
        ///   Creates the initial weight vector w
        /// </summary>
        ///
        /// <returns>The sum of squared weights divided by 2.</returns>
        ///
        private double saveNetworkToArray()
        {
            double w, sumOfSquaredWeights = 0.0;

            // for each layer in the network
            for (int li = 0, cur = 0; li < network.Layers.Length; li++)
            {
                ActivationLayer layer = network.Layers[li] as ActivationLayer;

                // for each neuron in the layer
                for (int ni = 0; ni < network.Layers[li].Neurons.Length; ni++, cur++)
                {
                    ActivationNeuron neuron = layer.Neurons[ni] as ActivationNeuron;

                    // for each weight in the neuron
                    for (int wi = 0; wi < neuron.InputsCount; wi++, cur++)
                    {
                        // We copy it to the starting weights vector
                        w = weights[cur] = (float)neuron.Weights[wi];
                        sumOfSquaredWeights += w * w;
                    }

                    // and also for the threshold value (bias):
                    w = weights[cur] = (float)neuron.Threshold;
                    sumOfSquaredWeights += w * w;
                }
            }
            return(sumOfSquaredWeights / 2.0);
        }
        /// <summary>
        ///  Update network's weights.
        /// </summary>
        ///
        /// <returns>The sum of squared weights divided by 2.</returns>
        ///
        private double loadArrayIntoNetwork()
        {
            double w, sumOfSquaredWeights = 0.0;

            // For each layer in the network
            for (int li = 0, cur = 0; li < network.Layers.Length; li++)
            {
                ActivationLayer layer = network.Layers[li] as ActivationLayer;

                // for each neuron in the layer
                for (int ni = 0; ni < layer.Neurons.Length; ni++, cur++)
                {
                    ActivationNeuron neuron = layer.Neurons[ni] as ActivationNeuron;

                    // for each weight in the neuron
                    for (int wi = 0; wi < neuron.Weights.Length; wi++, cur++)
                    {
                        neuron.Weights[wi]   = w = weights[cur] + deltas[cur];
                        sumOfSquaredWeights += w * w;
                    }

                    // for each threshold value (bias):
                    neuron.Threshold     = w = weights[cur] + deltas[cur];
                    sumOfSquaredWeights += w * w;
                }
            }

            return(sumOfSquaredWeights / 2.0);
        }
示例#3
0
 /// <summary>
 /// Initializes a new instance of the <see cref="ActivationLayer"/> class.
 /// </summary>
 ///
 /// <param name="neuronsCount">Layer's neurons count.</param>
 /// <param name="inputsCount">Layer's inputs count.</param>
 /// <param name="function">Activation function of neurons of the layer.</param>
 ///
 /// <remarks>The new layer is randomized (see <see cref="ActivationNeuron.Randomize"/>
 /// method) after it is created.</remarks>
 ///
 public ActivationLayer(int neuronsCount, int inputsCount, IActivationFunction function)
     : base(neuronsCount, inputsCount)
 {
     // create each neuron
     for (int i = 0; i < neurons.Length; i++)
     {
         neurons[i] = new ActivationNeuron(inputsCount, function);
     }
 }
示例#4
0
        internal static ActivationNeuron DeserializeFromJson(JObject jneuron)
        {
            ActivationNeuron neuron = new ActivationNeuron();

            neuron.threshold   = jneuron["Threshold"].ToObject <double>();
            neuron.inputsCount = jneuron["InputsCount"].ToObject <int>();
            neuron.weights     = jneuron["Weights"].ToObject <double[]>();
            // neuron.output = jneuron["Output"].ToObject<double>();
            return(neuron);
        }
示例#5
0
        internal static JObject SerializeToJson(ActivationNeuron neuron)
        {
            JObject result = new JObject();

            result["Threshold"]   = neuron.threshold;
            result["InputsCount"] = neuron.inputsCount;
            result["Weights"]     = new JArray(neuron.weights);
            //   result["Output"] = neuron.output;
            return(result);
        }
        /// <summary>
        ///   Calculates the Jacobian Matrix using Finite Differences
        /// </summary>
        ///
        /// <returns>Returns the sum of squared errors of the network divided by 2.</returns>
        ///
        private double JacobianByFiniteDifference(double[][] input, double[][] desiredOutput)
        {
            double e, sumOfSquaredErrors = 0;

            // for each input training sample
            for (int i = 0, row = 0; i < input.Length; i++)
            {
                // Compute a forward pass
                double[] networkOutput = network.Compute(input[i]);

                // for each output respective to the input
                for (int j = 0; j < networkOutput.Length; j++, row++)
                {
                    // Calculate network error to build the residuals vector
                    e = errors[row] = desiredOutput[i][j] - networkOutput[j];
                    sumOfSquaredErrors += e * e;

                    // Computation of one of the Jacobian Matrix rows by numerical differentiation:
                    // for each weight w_j in the network, we have to compute its partial derivative
                    // to build the Jacobian matrix.

                    // So, for each layer:
                    for (int li = 0, col = 0; li < network.Layers.Length; li++)
                    {
                        ActivationLayer layer = network.Layers[li] as ActivationLayer;

                        // for each neuron:
                        for (int ni = 0; ni < layer.Neurons.Length; ni++, col++)
                        {
                            ActivationNeuron neuron = layer.Neurons[ni] as ActivationNeuron;

                            // for each weight:
                            for (int wi = 0; wi < neuron.InputsCount; wi++, col++)
                            {
                                // Compute its partial derivative
                                jacobian[col][row] = (float)ComputeDerivative(input[i], li, ni,
                                                                              wi, ref derivativeStepSize[col], networkOutput[j], j);
                            }

                            // and also for each threshold value (bias)
                            jacobian[col][row] = (float)ComputeDerivative(input[i], li, ni,
                                                                          -1, ref derivativeStepSize[col], networkOutput[j], j);
                        }
                    }
                }
            }

            // returns the sum of squared errors / 2
            return(sumOfSquaredErrors / 2.0);
        }
示例#7
0
        internal static ActivationLayer DeserializeFromJson(JObject jlayer)
        {
            ActivationLayer layer = new ActivationLayer();

            layer.inputsCount  = jlayer["InputsCount"].ToObject <int>();
            layer.neuronsCount = jlayer["NeuronsCount"].ToObject <int>();
            //   layer.output = jlayer["Outputs"].ToObject<double[]>();
            layer.neurons = new Neuron[layer.neuronsCount];
            int counter = 0;

            foreach (JObject jneuron in jlayer["Neurons"].Children <JObject>())
            {
                layer.neurons[counter++] = ActivationNeuron.DeserializeFromJson(jneuron);
            }
            return(layer);
        }
示例#8
0
        internal static JObject SerializeToJson(ActivationLayer layer)
        {
            JObject result = new JObject();

            result["InputsCount"]  = layer.inputsCount;
            result["NeuronsCount"] = layer.neuronsCount;
            //  result["Outputs"] = new JArray(layer.output);
            JArray neurons = new JArray();

            for (int i = 0; i < layer.neurons.Length; i++)
            {
                ActivationNeuron neuron = (ActivationNeuron)layer.neurons[i];
                neurons.Add(ActivationNeuron.SerializeToJson(neuron));
            }
            result["Neurons"] = neurons;

            return(result);
        }
        /// <summary>
        ///   Calculates partial derivatives for all weights of the network.
        /// </summary>
        ///
        /// <param name="input">The input vector.</param>
        /// <param name="desiredOutput">Desired output vector.</param>
        /// <param name="outputIndex">The current output location (index) in the desired output vector.</param>
        ///
        /// <returns>Returns summary squared error of the last layer.</returns>
        ///
        private double CalculateDerivatives(double[] input, double[] desiredOutput, int outputIndex)
        {
            // Start by the output layer first
            int             outputLayerIndex = network.Layers.Length - 1;
            ActivationLayer outputLayer      = network.Layers[outputLayerIndex] as ActivationLayer;

            double[] previousLayerOutput;

            // If we have only one single layer, the previous layer outputs is given by the input layer
            previousLayerOutput = (outputLayerIndex == 0) ? input : network.Layers[outputLayerIndex - 1].Output;

            // Clear derivatives for other output's neurons
            for (int i = 0; i < thresholdsDerivatives[outputLayerIndex].Length; i++)
            {
                thresholdsDerivatives[outputLayerIndex][i] = 0;
            }

            for (int i = 0; i < weightDerivatives[outputLayerIndex].Length; i++)
            {
                for (int j = 0; j < weightDerivatives[outputLayerIndex][i].Length; j++)
                {
                    weightDerivatives[outputLayerIndex][i][j] = 0;
                }
            }

            // Retrieve current desired output neuron
            ActivationNeuron outputNeuron = outputLayer.Neurons[outputIndex] as ActivationNeuron;

            float[] neuronWeightDerivatives = weightDerivatives[outputLayerIndex][outputIndex];

            double output     = outputNeuron.Output;
            double error      = desiredOutput[outputIndex] - output;
            double derivative = outputNeuron.ActivationFunction.Derivative2(output);

            // Set derivative for each weight in the neuron
            for (int i = 0; i < neuronWeightDerivatives.Length; i++)
            {
                neuronWeightDerivatives[i] = (float)(derivative * previousLayerOutput[i]);
            }

            // Set derivative for the current threshold (bias) term
            thresholdsDerivatives[outputLayerIndex][outputIndex] = (float)derivative;


            // Now, proceed to the next hidden layers
            for (int li = network.Layers.Length - 2; li >= 0; li--)
            {
                int nextLayerIndex = li + 1;

                ActivationLayer layer     = network.Layers[li] as ActivationLayer;
                ActivationLayer nextLayer = network.Layers[nextLayerIndex] as ActivationLayer;

                // If we are in the first layer, the previous layer is just the input layer
                previousLayerOutput = (li == 0) ? input : network.Layers[li - 1].Output;

                // Now, we will compute the derivatives for the current layer applying the chain
                //  rule. To apply the chain-rule, we will make use of the previous derivatives
                //  computed for the inner layers (forming a calculation chain, hence the name).

                // So, for each neuron in the current layer:
                for (int ni = 0; ni < layer.Neurons.Length; ni++)
                {
                    ActivationNeuron neuron = layer.Neurons[ni] as ActivationNeuron;

                    neuronWeightDerivatives = weightDerivatives[li][ni];

                    float[] layerDerivatives     = thresholdsDerivatives[li];
                    float[] nextLayerDerivatives = thresholdsDerivatives[li + 1];

                    double sum = 0;

                    // The chain-rule can be stated as (f(w*g(x))' = f'(w*g(x)) * w*g'(x)
                    //
                    // We will start computing the second part of the product. Since the g'
                    //  derivatives have already been computed in the previous computation,
                    //  we will be summing all previous function derivatives and weighting
                    //  them using their connection weight (synapses).
                    //
                    // So, for each neuron in the next layer:
                    for (int nj = 0; nj < nextLayerDerivatives.Length; nj++)
                    {
                        // retrieve the weight connecting the output of the current
                        //   neuron and the activation function of the next neuron.
                        double weight = nextLayer.Neurons[nj].Weights[ni];

                        // accumulate the synapse weight * next layer derivative
                        sum += weight * nextLayerDerivatives[nj];
                    }

                    // Continue forming the chain-rule statement
                    derivative = sum * neuron.ActivationFunction.Derivative2(neuron.Output);

                    // Set derivative for each weight in the neuron
                    for (int wi = 0; wi < neuronWeightDerivatives.Length; wi++)
                    {
                        neuronWeightDerivatives[wi] = (float)(derivative * previousLayerOutput[wi]);
                    }

                    // Set derivative for the current threshold
                    layerDerivatives[ni] = (float)(derivative);

                    // The threshold derivatives also gather the derivatives for
                    // the layer, and thus can be re-used in next calculations.
                }
            }

            // return error
            return(error);
        }