/// <summary>
/// Update all error derivatives, and accumulate the appropriate weight deltas,
/// given a reference to the loss function of the network and a target output,
/// assuming that this is the output layer of the network.
/// </summary>
/// <param name="target"></param>
/// <param name="lossFunction"></param>
public override void Backpropagate(Matrix target, ILossFunction lossFunction,
List <IRegulariser> regularisers)
{
// Rate of change of the loss with respect to each pre-activation
Vector preActivationErrorDerivatives = new Vector(thisLayer.Outputs);
for (int i = 0; i < thisLayer.Outputs; i++)
{
// Calculate output error derivatives; since this is an output layer, this
// is the error function derivative wrt. the ith output of this layer
outputErrorDerivatives[i] = lossFunction.ErrorDerivative(thisLayer.Output,
target, i);
}
denseReference.Activation.Peek(denseReference.PreActivation);
// Calculate the pre-activation partial derivatives
if (denseReference.Activation.IsInterdependent())
{
for (int i = 0; i < thisLayer.Outputs; i++) // i -> pre-activation index
{
for (int j = 0; j < thisLayer.Outputs; j++) // j -> output index
{
preActivationErrorDerivatives[i] =
preActivationErrorDerivatives[i] +
denseReference.Activation.Derivative(i, j) *
outputErrorDerivatives[j];
}
}
}
else
{
for (int i = 0; i < thisLayer.Outputs; i++)
{
preActivationErrorDerivatives[i] =
denseReference.Activation.Derivative(i, i) *
outputErrorDerivatives[i];
}
}
for (int i = 0; i < thisLayer.Inputs; i++) // i -> input neuron
{
// Calculate input error derivatives; the product of the derivative of the
// pre-activation wrt. the input and the derivative of the output wrt. the
// pre-activation, summed over all output neurons in this layer
double inputErrorDerivative = 0;
for (int j = 0; j < thisLayer.Outputs; j++) // j -> output neuron
{
// Add 1 to input index so that bias is skipped
inputErrorDerivative += denseReference.Weights[j, i + 1] *
preActivationErrorDerivatives[j];
}
inputErrorDerivatives[i] = inputErrorDerivative;
}
// Cycle through each weight
for (int i = 0; i < denseReference.Weights.Rows; i++) // i -> output neurons
{
for (int j = 0; j < denseReference.Weights.Columns; j++)
// j -> input neurons
{
double regulariserDelta = 0;
foreach (IRegulariser regulariser in regularisers)
{
regulariserDelta += regulariser.LossDerivative(
denseReference.Weights[i, j]);
}
weightDeltas[i, j] = weightDeltas[i, j] +
outputErrorDerivatives[i] *
preActivationErrorDerivatives[i] *
thisLayer.Input[j, 0] + regulariserDelta;
}
}
}
