/// <summary>
/// Update network's weights.
/// </summary>
///
/// <returns>The sum of squared weights divided by 2.</returns>
///
private double loadArrayIntoNetwork()
{
double w, sumOfSquaredWeights = 0.0;
// For each layer in the network
for (int li = 0, cur = 0; li < network.Layers.Length; li++)
{
ActivationLayer layer = network.Layers[li] as ActivationLayer;
// for each neuron in the layer
for (int ni = 0; ni < layer.Neurons.Length; ni++, cur++)
{
ActivationNeuron neuron = layer.Neurons[ni] as ActivationNeuron;
// for each weight in the neuron
for (int wi = 0; wi < neuron.Weights.Length; wi++, cur++)
{
neuron.Weights[wi] = w = weights[cur] + deltas[cur];
sumOfSquaredWeights += w * w;
}
// for each threshold value (bias):
neuron.Threshold = w = weights[cur] + deltas[cur];
sumOfSquaredWeights += w * w;
}
}
return(sumOfSquaredWeights / 2.0);
}
/// <summary>
/// Creates the initial weight vector w
/// </summary>
///
/// <returns>The sum of squared weights divided by 2.</returns>
///
private double saveNetworkToArray()
{
double w, sumOfSquaredWeights = 0.0;
// for each layer in the network
for (int li = 0, cur = 0; li < network.Layers.Length; li++)
{
ActivationLayer layer = network.Layers[li] as ActivationLayer;
// for each neuron in the layer
for (int ni = 0; ni < network.Layers[li].Neurons.Length; ni++, cur++)
{
ActivationNeuron neuron = layer.Neurons[ni] as ActivationNeuron;
// for each weight in the neuron
for (int wi = 0; wi < neuron.InputsCount; wi++, cur++)
{
// We copy it to the starting weights vector
w = weights[cur] = (float)neuron.Weights[wi];
sumOfSquaredWeights += w * w;
}
// and also for the threshold value (bias):
w = weights[cur] = (float)neuron.Threshold;
sumOfSquaredWeights += w * w;
}
}
return(sumOfSquaredWeights / 2.0);
}
/// <summary>
/// Calculates the Jacobian Matrix using Finite Differences
/// </summary>
///
/// <returns>Returns the sum of squared errors of the network divided by 2.</returns>
///
private double JacobianByFiniteDifference(double[][] input, double[][] desiredOutput)
{
double e, sumOfSquaredErrors = 0;
// for each input training sample
for (int i = 0, row = 0; i < input.Length; i++)
{
// Compute a forward pass
double[] networkOutput = network.Compute(input[i]);
// for each output respective to the input
for (int j = 0; j < networkOutput.Length; j++, row++)
{
// Calculate network error to build the residuals vector
e = errors[row] = desiredOutput[i][j] - networkOutput[j];
sumOfSquaredErrors += e * e;
// Computation of one of the Jacobian Matrix rows by numerical differentiation:
// for each weight w_j in the network, we have to compute its partial derivative
// to build the Jacobian matrix.
// So, for each layer:
for (int li = 0, col = 0; li < network.Layers.Length; li++)
{
ActivationLayer layer = network.Layers[li] as ActivationLayer;
// for each neuron:
for (int ni = 0; ni < layer.Neurons.Length; ni++, col++)
{
ActivationNeuron neuron = layer.Neurons[ni] as ActivationNeuron;
// for each weight:
for (int wi = 0; wi < neuron.InputsCount; wi++, col++)
{
// Compute its partial derivative
jacobian[col][row] = (float)ComputeDerivative(input[i], li, ni,
wi, ref derivativeStepSize[col], networkOutput[j], j);
}
// and also for each threshold value (bias)
jacobian[col][row] = (float)ComputeDerivative(input[i], li, ni,
-1, ref derivativeStepSize[col], networkOutput[j], j);
}
}
}
}
// returns the sum of squared errors / 2
return(sumOfSquaredErrors / 2.0);
}
/// <summary>
/// Initializes a new instance of the <see cref="ActivationNetwork"/> class.
/// </summary>
///
/// <param name="function">Activation function of neurons of the network.</param>
/// <param name="inputsCount">Network's inputs count.</param>
/// <param name="neuronsCount">Array, which specifies the amount of neurons in
/// each layer of the neural network.</param>
///
/// <remarks>The new network is randomized (see <see cref="ActivationNeuron.Randomize"/>
/// method) after it is created.</remarks>
///
public ActivationNetwork(IActivationFunction function, int inputsCount, params int[] neuronsCount)
: base(inputsCount, neuronsCount.Length)
{
// create each layer
for (int i = 0; i < layers.Length; i++)
{
layers[i] = new ActivationLayer(
// neurons count in the layer
neuronsCount[i],
// inputs count of the layer
(i == 0) ? inputsCount : neuronsCount[i - 1],
// activation function of the layer
function);
}
}
internal static ActivationLayer DeserializeFromJson(JObject jlayer)
{
ActivationLayer layer = new ActivationLayer();
layer.inputsCount = jlayer["InputsCount"].ToObject <int>();
layer.neuronsCount = jlayer["NeuronsCount"].ToObject <int>();
// layer.output = jlayer["Outputs"].ToObject<double[]>();
layer.neurons = new Neuron[layer.neuronsCount];
int counter = 0;
foreach (JObject jneuron in jlayer["Neurons"].Children <JObject>())
{
layer.neurons[counter++] = ActivationNeuron.DeserializeFromJson(jneuron);
}
return(layer);
}
/// <summary>
/// Initializes a new instance of the <see cref="LevenbergMarquardtLearning"/> class.
/// </summary>
///
/// <param name="network">Network to teach.</param>
/// <param name="useRegularization">True to use Bayesian regularization, false otherwise.</param>
/// <param name="method">The method by which the Jacobian matrix will be calculated.</param>
///
public LevenbergMarquardtLearning(ActivationNetwork network, bool useRegularization, JacobianMethod method)
{
this.ParallelOptions = new ParallelOptions();
this.network = network;
this.numberOfParameters = getNumberOfParameters(network);
this.outputCount = network.Layers[network.Layers.Length - 1].Neurons.Length;
this.useBayesianRegularization = useRegularization;
this.method = method;
this.weights = new float[numberOfParameters];
this.hessian = new float[numberOfParameters][];
for (int i = 0; i < hessian.Length; i++)
{
hessian[i] = new float[numberOfParameters];
}
this.diagonal = new float[numberOfParameters];
this.gradient = new float[numberOfParameters];
this.jacobian = new float[numberOfParameters][];
// Will use Backpropagation method for Jacobian computation
if (method == JacobianMethod.ByBackpropagation)
{
// create weight derivatives arrays
this.weightDerivatives = new float[network.Layers.Length][][];
this.thresholdsDerivatives = new float[network.Layers.Length][];
// initialize arrays
for (int i = 0; i < network.Layers.Length; i++)
{
ActivationLayer layer = (ActivationLayer)network.Layers[i];
this.weightDerivatives[i] = new float[layer.Neurons.Length][];
this.thresholdsDerivatives[i] = new float[layer.Neurons.Length];
for (int j = 0; j < layer.Neurons.Length; j++)
{
this.weightDerivatives[i][j] = new float[layer.InputsCount];
}
}
}
else // Will use finite difference method for Jacobian computation
{
throw new NotImplementedException("Finite difference method is not implemented");
}
}
internal static JObject SerializeToJson(ActivationLayer layer)
{
JObject result = new JObject();
result["InputsCount"] = layer.inputsCount;
result["NeuronsCount"] = layer.neuronsCount;
// result["Outputs"] = new JArray(layer.output);
JArray neurons = new JArray();
for (int i = 0; i < layer.neurons.Length; i++)
{
ActivationNeuron neuron = (ActivationNeuron)layer.neurons[i];
neurons.Add(ActivationNeuron.SerializeToJson(neuron));
}
result["Neurons"] = neurons;
return(result);
}
internal static JObject SerializeToJson(ActivationNetwork network)
{
JObject result = new JObject();
result["InputsCount"] = network.inputsCount;
result["layersCount"] = network.layersCount;
// result["Outputs"] = new JArray(network.output);
JArray layers = new JArray();
for (int i = 0; i < network.layers.Length; i++)
{
ActivationLayer layer = (ActivationLayer)network.layers[i];
layers.Add(ActivationLayer.SerializeToJson(layer));
}
result["Layers"] = layers;
return(result);
}
internal static ActivationNetwork DeserializeFromJson(JObject jnetwork)
{
ActivationNetwork network = new ActivationNetwork();
network.inputsCount = jnetwork["InputsCount"].ToObject <int>();
network.layersCount = jnetwork["layersCount"].ToObject <int>();
// network.output = jnetwork["Outputs"].ToObject<double[]>();
network.layers = new Layer[network.layersCount];
int counter = 0;
foreach (JObject jlayer in jnetwork["Layers"].Children <JObject>())
{
network.layers[counter++] = ActivationLayer.DeserializeFromJson(jlayer);
}
network.SetActivationFunction(new SigmoidFunction());
return(network);
}
/// <summary>
/// Calculates partial derivatives for all weights of the network.
/// </summary>
///
/// <param name="input">The input vector.</param>
/// <param name="desiredOutput">Desired output vector.</param>
/// <param name="outputIndex">The current output location (index) in the desired output vector.</param>
///
/// <returns>Returns summary squared error of the last layer.</returns>
///
private double CalculateDerivatives(double[] input, double[] desiredOutput, int outputIndex)
{
// Start by the output layer first
int outputLayerIndex = network.Layers.Length - 1;
ActivationLayer outputLayer = network.Layers[outputLayerIndex] as ActivationLayer;
double[] previousLayerOutput;
// If we have only one single layer, the previous layer outputs is given by the input layer
previousLayerOutput = (outputLayerIndex == 0) ? input : network.Layers[outputLayerIndex - 1].Output;
// Clear derivatives for other output's neurons
for (int i = 0; i < thresholdsDerivatives[outputLayerIndex].Length; i++)
{
thresholdsDerivatives[outputLayerIndex][i] = 0;
}
for (int i = 0; i < weightDerivatives[outputLayerIndex].Length; i++)
{
for (int j = 0; j < weightDerivatives[outputLayerIndex][i].Length; j++)
{
weightDerivatives[outputLayerIndex][i][j] = 0;
}
}
// Retrieve current desired output neuron
ActivationNeuron outputNeuron = outputLayer.Neurons[outputIndex] as ActivationNeuron;
float[] neuronWeightDerivatives = weightDerivatives[outputLayerIndex][outputIndex];
double output = outputNeuron.Output;
double error = desiredOutput[outputIndex] - output;
double derivative = outputNeuron.ActivationFunction.Derivative2(output);
// Set derivative for each weight in the neuron
for (int i = 0; i < neuronWeightDerivatives.Length; i++)
{
neuronWeightDerivatives[i] = (float)(derivative * previousLayerOutput[i]);
}
// Set derivative for the current threshold (bias) term
thresholdsDerivatives[outputLayerIndex][outputIndex] = (float)derivative;
// Now, proceed to the next hidden layers
for (int li = network.Layers.Length - 2; li >= 0; li--)
{
int nextLayerIndex = li + 1;
ActivationLayer layer = network.Layers[li] as ActivationLayer;
ActivationLayer nextLayer = network.Layers[nextLayerIndex] as ActivationLayer;
// If we are in the first layer, the previous layer is just the input layer
previousLayerOutput = (li == 0) ? input : network.Layers[li - 1].Output;
// Now, we will compute the derivatives for the current layer applying the chain
// rule. To apply the chain-rule, we will make use of the previous derivatives
// computed for the inner layers (forming a calculation chain, hence the name).
// So, for each neuron in the current layer:
for (int ni = 0; ni < layer.Neurons.Length; ni++)
{
ActivationNeuron neuron = layer.Neurons[ni] as ActivationNeuron;
neuronWeightDerivatives = weightDerivatives[li][ni];
float[] layerDerivatives = thresholdsDerivatives[li];
float[] nextLayerDerivatives = thresholdsDerivatives[li + 1];
double sum = 0;
// The chain-rule can be stated as (f(w*g(x))' = f'(w*g(x)) * w*g'(x)
//
// We will start computing the second part of the product. Since the g'
// derivatives have already been computed in the previous computation,
// we will be summing all previous function derivatives and weighting
// them using their connection weight (synapses).
//
// So, for each neuron in the next layer:
for (int nj = 0; nj < nextLayerDerivatives.Length; nj++)
{
// retrieve the weight connecting the output of the current
// neuron and the activation function of the next neuron.
double weight = nextLayer.Neurons[nj].Weights[ni];
// accumulate the synapse weight * next layer derivative
sum += weight * nextLayerDerivatives[nj];
}
// Continue forming the chain-rule statement
derivative = sum * neuron.ActivationFunction.Derivative2(neuron.Output);
// Set derivative for each weight in the neuron
for (int wi = 0; wi < neuronWeightDerivatives.Length; wi++)
{
neuronWeightDerivatives[wi] = (float)(derivative * previousLayerOutput[wi]);
}
// Set derivative for the current threshold
layerDerivatives[ni] = (float)(derivative);
// The threshold derivatives also gather the derivatives for
// the layer, and thus can be re-used in next calculations.
}
}
// return error
return(error);
}
