/// <summary>
/// Creates the initial weight vector w
/// </summary>
///
/// <returns>The sum of squared weights divided by 2.</returns>
///
private double saveNetworkToArray()
{
double w, sumOfSquaredWeights = 0.0;
// for each layer in the network
for (int li = 0, cur = 0; li < network.Layers.Length; li++)
{
ActivationLayer layer = network.Layers[li] as ActivationLayer;
// for each neuron in the layer
for (int ni = 0; ni < network.Layers[li].Neurons.Length; ni++, cur++)
{
ActivationNeuron neuron = layer.Neurons[ni] as ActivationNeuron;
// for each weight in the neuron
for (int wi = 0; wi < neuron.InputsCount; wi++, cur++)
{
// We copy it to the starting weights vector
w = weights[cur] = (float)neuron.Weights[wi];
sumOfSquaredWeights += w * w;
}
// and also for the threshold value (bias):
w = weights[cur] = (float)neuron.Threshold;
sumOfSquaredWeights += w * w;
}
}
return(sumOfSquaredWeights / 2.0);
}
/// <summary>
/// Update network's weights.
/// </summary>
///
/// <returns>The sum of squared weights divided by 2.</returns>
///
private double loadArrayIntoNetwork()
{
double w, sumOfSquaredWeights = 0.0;
// For each layer in the network
for (int li = 0, cur = 0; li < network.Layers.Length; li++)
{
ActivationLayer layer = network.Layers[li] as ActivationLayer;
// for each neuron in the layer
for (int ni = 0; ni < layer.Neurons.Length; ni++, cur++)
{
ActivationNeuron neuron = layer.Neurons[ni] as ActivationNeuron;
// for each weight in the neuron
for (int wi = 0; wi < neuron.Weights.Length; wi++, cur++)
{
neuron.Weights[wi] = w = weights[cur] + deltas[cur];
sumOfSquaredWeights += w * w;
}
// for each threshold value (bias):
neuron.Threshold = w = weights[cur] + deltas[cur];
sumOfSquaredWeights += w * w;
}
}
return(sumOfSquaredWeights / 2.0);
}
/// <summary>
/// Initializes a new instance of the <see cref="ActivationLayer"/> class.
/// </summary>
///
/// <param name="neuronsCount">Layer's neurons count.</param>
/// <param name="inputsCount">Layer's inputs count.</param>
/// <param name="function">Activation function of neurons of the layer.</param>
///
/// <remarks>The new layer is randomized (see <see cref="ActivationNeuron.Randomize"/>
/// method) after it is created.</remarks>
///
public ActivationLayer(int neuronsCount, int inputsCount, IActivationFunction function)
: base(neuronsCount, inputsCount)
{
// create each neuron
for (int i = 0; i < neurons.Length; i++)
{
neurons[i] = new ActivationNeuron(inputsCount, function);
}
}
internal static ActivationNeuron DeserializeFromJson(JObject jneuron)
{
ActivationNeuron neuron = new ActivationNeuron();
neuron.threshold = jneuron["Threshold"].ToObject <double>();
neuron.inputsCount = jneuron["InputsCount"].ToObject <int>();
neuron.weights = jneuron["Weights"].ToObject <double[]>();
// neuron.output = jneuron["Output"].ToObject<double>();
return(neuron);
}
internal static JObject SerializeToJson(ActivationNeuron neuron)
{
JObject result = new JObject();
result["Threshold"] = neuron.threshold;
result["InputsCount"] = neuron.inputsCount;
result["Weights"] = new JArray(neuron.weights);
// result["Output"] = neuron.output;
return(result);
}
/// <summary>
/// Calculates the Jacobian Matrix using Finite Differences
/// </summary>
///
/// <returns>Returns the sum of squared errors of the network divided by 2.</returns>
///
private double JacobianByFiniteDifference(double[][] input, double[][] desiredOutput)
{
double e, sumOfSquaredErrors = 0;
// for each input training sample
for (int i = 0, row = 0; i < input.Length; i++)
{
// Compute a forward pass
double[] networkOutput = network.Compute(input[i]);
// for each output respective to the input
for (int j = 0; j < networkOutput.Length; j++, row++)
{
// Calculate network error to build the residuals vector
e = errors[row] = desiredOutput[i][j] - networkOutput[j];
sumOfSquaredErrors += e * e;
// Computation of one of the Jacobian Matrix rows by numerical differentiation:
// for each weight w_j in the network, we have to compute its partial derivative
// to build the Jacobian matrix.
// So, for each layer:
for (int li = 0, col = 0; li < network.Layers.Length; li++)
{
ActivationLayer layer = network.Layers[li] as ActivationLayer;
// for each neuron:
for (int ni = 0; ni < layer.Neurons.Length; ni++, col++)
{
ActivationNeuron neuron = layer.Neurons[ni] as ActivationNeuron;
// for each weight:
for (int wi = 0; wi < neuron.InputsCount; wi++, col++)
{
// Compute its partial derivative
jacobian[col][row] = (float)ComputeDerivative(input[i], li, ni,
wi, ref derivativeStepSize[col], networkOutput[j], j);
}
// and also for each threshold value (bias)
jacobian[col][row] = (float)ComputeDerivative(input[i], li, ni,
-1, ref derivativeStepSize[col], networkOutput[j], j);
}
}
}
}
// returns the sum of squared errors / 2
return(sumOfSquaredErrors / 2.0);
}
internal static ActivationLayer DeserializeFromJson(JObject jlayer)
{
ActivationLayer layer = new ActivationLayer();
layer.inputsCount = jlayer["InputsCount"].ToObject <int>();
layer.neuronsCount = jlayer["NeuronsCount"].ToObject <int>();
// layer.output = jlayer["Outputs"].ToObject<double[]>();
layer.neurons = new Neuron[layer.neuronsCount];
int counter = 0;
foreach (JObject jneuron in jlayer["Neurons"].Children <JObject>())
{
layer.neurons[counter++] = ActivationNeuron.DeserializeFromJson(jneuron);
}
return(layer);
}
internal static JObject SerializeToJson(ActivationLayer layer)
{
JObject result = new JObject();
result["InputsCount"] = layer.inputsCount;
result["NeuronsCount"] = layer.neuronsCount;
// result["Outputs"] = new JArray(layer.output);
JArray neurons = new JArray();
for (int i = 0; i < layer.neurons.Length; i++)
{
ActivationNeuron neuron = (ActivationNeuron)layer.neurons[i];
neurons.Add(ActivationNeuron.SerializeToJson(neuron));
}
result["Neurons"] = neurons;
return(result);
}
/// <summary>
/// Calculates partial derivatives for all weights of the network.
/// </summary>
///
/// <param name="input">The input vector.</param>
/// <param name="desiredOutput">Desired output vector.</param>
/// <param name="outputIndex">The current output location (index) in the desired output vector.</param>
///
/// <returns>Returns summary squared error of the last layer.</returns>
///
private double CalculateDerivatives(double[] input, double[] desiredOutput, int outputIndex)
{
// Start by the output layer first
int outputLayerIndex = network.Layers.Length - 1;
ActivationLayer outputLayer = network.Layers[outputLayerIndex] as ActivationLayer;
double[] previousLayerOutput;
// If we have only one single layer, the previous layer outputs is given by the input layer
previousLayerOutput = (outputLayerIndex == 0) ? input : network.Layers[outputLayerIndex - 1].Output;
// Clear derivatives for other output's neurons
for (int i = 0; i < thresholdsDerivatives[outputLayerIndex].Length; i++)
{
thresholdsDerivatives[outputLayerIndex][i] = 0;
}
for (int i = 0; i < weightDerivatives[outputLayerIndex].Length; i++)
{
for (int j = 0; j < weightDerivatives[outputLayerIndex][i].Length; j++)
{
weightDerivatives[outputLayerIndex][i][j] = 0;
}
}
// Retrieve current desired output neuron
ActivationNeuron outputNeuron = outputLayer.Neurons[outputIndex] as ActivationNeuron;
float[] neuronWeightDerivatives = weightDerivatives[outputLayerIndex][outputIndex];
double output = outputNeuron.Output;
double error = desiredOutput[outputIndex] - output;
double derivative = outputNeuron.ActivationFunction.Derivative2(output);
// Set derivative for each weight in the neuron
for (int i = 0; i < neuronWeightDerivatives.Length; i++)
{
neuronWeightDerivatives[i] = (float)(derivative * previousLayerOutput[i]);
}
// Set derivative for the current threshold (bias) term
thresholdsDerivatives[outputLayerIndex][outputIndex] = (float)derivative;
// Now, proceed to the next hidden layers
for (int li = network.Layers.Length - 2; li >= 0; li--)
{
int nextLayerIndex = li + 1;
ActivationLayer layer = network.Layers[li] as ActivationLayer;
ActivationLayer nextLayer = network.Layers[nextLayerIndex] as ActivationLayer;
// If we are in the first layer, the previous layer is just the input layer
previousLayerOutput = (li == 0) ? input : network.Layers[li - 1].Output;
// Now, we will compute the derivatives for the current layer applying the chain
// rule. To apply the chain-rule, we will make use of the previous derivatives
// computed for the inner layers (forming a calculation chain, hence the name).
// So, for each neuron in the current layer:
for (int ni = 0; ni < layer.Neurons.Length; ni++)
{
ActivationNeuron neuron = layer.Neurons[ni] as ActivationNeuron;
neuronWeightDerivatives = weightDerivatives[li][ni];
float[] layerDerivatives = thresholdsDerivatives[li];
float[] nextLayerDerivatives = thresholdsDerivatives[li + 1];
double sum = 0;
// The chain-rule can be stated as (f(w*g(x))' = f'(w*g(x)) * w*g'(x)
//
// We will start computing the second part of the product. Since the g'
// derivatives have already been computed in the previous computation,
// we will be summing all previous function derivatives and weighting
// them using their connection weight (synapses).
//
// So, for each neuron in the next layer:
for (int nj = 0; nj < nextLayerDerivatives.Length; nj++)
{
// retrieve the weight connecting the output of the current
// neuron and the activation function of the next neuron.
double weight = nextLayer.Neurons[nj].Weights[ni];
// accumulate the synapse weight * next layer derivative
sum += weight * nextLayerDerivatives[nj];
}
// Continue forming the chain-rule statement
derivative = sum * neuron.ActivationFunction.Derivative2(neuron.Output);
// Set derivative for each weight in the neuron
for (int wi = 0; wi < neuronWeightDerivatives.Length; wi++)
{
neuronWeightDerivatives[wi] = (float)(derivative * previousLayerOutput[wi]);
}
// Set derivative for the current threshold
layerDerivatives[ni] = (float)(derivative);
// The threshold derivatives also gather the derivatives for
// the layer, and thus can be re-used in next calculations.
}
}
// return error
return(error);
}