protected void FinaliseOutputs(Step downstream)
{
for (int i = 0; i < downstream.WeightedInputs.Length; i++)
{
downstream.Output[i] = downstream.CalculateActivation(downstream.WeightedInputs[i]);
}
}
protected void FinaliseErrorSecondDerivatives(Step downstream)
{
for (int i = 0; i < downstream.ErrorDerivative.Length; i++)
{
double weightedInputs = downstream.WeightedInputs[i];
double activationDerivative = downstream.CalculateActivationDerivative(weightedInputs);
downstream.ErrorDerivative[i] *= activationDerivative * activationDerivative;
}
}
protected void EstimateBiasSecondDerivative(Step downstream)
{
for (int i = 0; i < downstream.Length; i++)
{
// Calculating the sum of: second derivatives of error with respect to the bias weight.
// Note that the bias is implemented as an always-on Neuron with a (the same) weight to the outputs neurons.
BiasStepSize += downstream.ErrorDerivative[i] * 1.0 * 1.0;
}
}
protected override void TrainCore(Step downstream)
{
RectangularStep step = (RectangularStep)downstream;
Debug.Assert(MapCount == downstream.Upstream.Count);
for (int i = 0; i < MapCount; i++)
{
PropogateError(step, i);
}
}
protected override void PreTrainCore(Step downstream)
{
RectangularStep step = (RectangularStep)downstream;
Debug.Assert(MapCount == downstream.Upstream.Count);
for (int i = 0; i < MapCount; i++)
{
PropogateUnitSecondDerivatives(step, i);
}
EstimateBiasSecondDerivative(step);
}
protected void FinaliseErrorFirstDerivatives(Step downstream)
{
// Calculating the dEj/dWij and dEi/dOi both requires a multiplication by the derivative of the activation function,
// it is done here once so it doesn't need to be done for each individual calculations.
// This turns dEk/dOk into dEk/dAk by multiplying it by dOk/dAk
for (int i = 0; i < downstream.ErrorDerivative.Length; i++)
{
double weightedInputs = downstream.WeightedInputs[i];
double activationDerivative = downstream.CalculateActivationDerivative(weightedInputs);
downstream.ErrorDerivative[i] *= activationDerivative;
}
}
protected override void PreTrainCore(Step downstream)
{
int inputIndex = 0;
foreach (Step upstream in downstream.Upstream)
{
Debug.Assert(inputIndex + upstream.Length <= InputLength);
for (int i = 0; i < upstream.Length; i++)
{
// Error second derivative relative to output is constant, as first derivative is 2.0 * (state - desiredState).
upstream.ErrorDerivative[inputIndex] = 2.0;
inputIndex += 1;
}
}
}
protected override void PreTrainCore(Step downstream)
{
Debug.Assert(InputNeurons % downstream.Upstream.Count == 0);
int neuronsPerUpstream = InputNeurons / downstream.Upstream.Count;
int inputNeuron = 0;
foreach (Step upstream in downstream.Upstream)
{
Debug.Assert(upstream.Length == neuronsPerUpstream);
for (int i = 0; i < neuronsPerUpstream; i++)
{
PropogateSecondDerivatives(downstream, upstream, i, inputNeuron++);
}
}
EstimateBiasSecondDerivative(downstream);
}
protected void PropogateForward(Step downstream, int output)
{
double sumSquaredError = 0;
int inputIndex = 0;
int definitionIndex = output * InputLength;
foreach (Step upstream in downstream.Upstream)
{
Debug.Assert(inputIndex + upstream.Length <= InputLength);
for (int i = 0; i < upstream.Length; i++)
{
double difference = upstream.Output[i] - ClassStateDefinitions[definitionIndex];
sumSquaredError += difference * difference;
inputIndex += 1;
definitionIndex += 1;
}
}
downstream.Output[output] = sumSquaredError;
}
protected override void TrainCore(Step downstream)
{
int inputIndex = 0;
int definitionIndex = correctClass * InputLength;
foreach (Step upstream in downstream.Upstream)
{
Debug.Assert(inputIndex + upstream.Length <= InputLength);
for (int i = 0; i < upstream.Length; i++)
{
double desiredState = ClassStateDefinitions[definitionIndex];
double firstDerivative = 2.0 * (upstream.Output[inputIndex] - desiredState);
upstream.ErrorDerivative[inputIndex] = firstDerivative;
inputIndex += 1;
definitionIndex += 1;
}
}
}
public override void ProprogateForward(Step downstream)
{
base.ProprogateForward(downstream);
FinaliseOutputs(downstream);
}
public override void PreTrain(Step downstream)
{
FinaliseErrorSecondDerivatives(downstream);
base.PreTrain(downstream);
}
public MarkingStep(Step upstream, LeNetConfiguration configuration)
: this(new[] { upstream }, configuration)
{
}
public virtual void ProprogateForward(Step downstream)
{
PropogateForwardCore(downstream);
}
public override void ProprogateForward(Step downstream)
{
base.ProprogateForward(downstream);
FinaliseOutputs(downstream);
}
public override void Train(Step downstream)
{
FinaliseErrorFirstDerivatives(downstream);
base.Train(downstream);
}
protected void PropogateError(Step downstream, Step upstream, int upstreamNeuron, int inputNeuron)
{
int weightIndex = inputNeuron * OutputNeurons;
double upstreamState = upstream.Output[upstreamNeuron];
double inputError = 0.0;
for (int output = 0; output < OutputNeurons; output++)
{
double downstreamErrorDerivative = downstream.ErrorDerivative[output];
// Calculate inputs error gradient by taking the sum, for all outputs of
// dEk/dAj multiplied by dAj/dOj (w/sum =dEj/dOj);
inputError += (downstreamErrorDerivative * Weight[weightIndex]);
// Calculate the Weight's first derivative with respect to the error
double weightErrorGradient = downstreamErrorDerivative * upstreamState;
double deltaWeight = WeightStepSize[weightIndex] * weightErrorGradient;
Weight[weightIndex] -= deltaWeight;
weightIndex += 1;
}
upstream.ErrorDerivative[upstreamNeuron] = inputError;
}
protected override void TrainCore(Step downstream)
{
Debug.Assert(InputNeurons % downstream.Upstream.Count == 0);
int neuronsPerUpstream = InputNeurons / downstream.Upstream.Count;
int inputNeuron = 0;
foreach (Step upstream in downstream.Upstream)
{
Debug.Assert(upstream.Length == neuronsPerUpstream);
for (int i = 0; i < neuronsPerUpstream; i++)
{
PropogateError(downstream, upstream, i, inputNeuron++);
}
}
}
protected abstract void PropogateForwardCore(Step step);
protected void PropogateSecondDerivatives(Step downstream, Step upstream, int upstreamNeuron, int inputNeuron)
{
int weightIndex = inputNeuron * OutputNeurons;
double upstreamState = upstream.Output[upstreamNeuron];
double upstreamErrorSecondDerivative = 0.0;
for (int output = 0; output < OutputNeurons; output++)
{
double downstreamErrorSecondDerivative = downstream.ErrorDerivative[output]; // (d^2)E/(dAj)^2, where Aj is the sum of inputs to this downstream unit.
// Here we calculate (d^2)Ej/(dWij)^2 by multiplying the 2nd derivative of E with respect to the sum of inputs, Aj
// by the state of Oi, the upstream unit, squared. Refer to Equation 25 in document.
// The summing happening here is described by equation 23.
double weight2ndDerivative = downstreamErrorSecondDerivative * upstreamState * upstreamState;
WeightStepSize[weightIndex] = weight2ndDerivative;
double weight = Weight[weightIndex];
// This is implementing the last sigma of Equation 27.
// This propogates error second derivatives back to previous layer, but will need to be multiplied by the second derivative
// of the activation function at the previous layer.
upstreamErrorSecondDerivative += weight * weight * downstreamErrorSecondDerivative;
weightIndex += 1;
}
upstream.ErrorDerivative[upstreamNeuron] += upstreamErrorSecondDerivative;
}
protected override void PropogateForwardCore(Step downstream)
{
Debug.Assert(InputNeurons % downstream.Upstream.Count == 0);
int neuronsPerUpstream = InputNeurons / downstream.Upstream.Count;
int inputIndex = 0;
foreach (Step upstream in downstream.Upstream)
{
Debug.Assert(inputIndex + upstream.Length <= InputNeurons);
for (int i = 0; i < neuronsPerUpstream; i++)
{
PropogateForward(downstream, upstream, i, inputIndex++);
}
}
}
protected void PropogateForward(Step downstream, Step upstream, int upstreamNeuron, int inputNeuron)
{
int weightIndex = inputNeuron * OutputNeurons;
double upstreamNeuronOutput = upstream.Output[upstreamNeuron];
double weightedSum = Bias;
for (int o = 0; o < OutputNeurons; o++)
{
downstream.WeightedInputs[o] += upstreamNeuronOutput * Weight[weightIndex++];
}
}
protected override void PropogateForwardCore(Step downstream)
{
Debug.Assert(downstream.Upstream.Count == 1);
for (int o = 0; o < ClassCount; o++)
{
PropogateForward(downstream, o);
}
}
public StepSnapshot(Step step, int width)
{
this.Step = step;
OutputSnapshot = new double[step.Output.Length];
Width = width;
}
public virtual void PreTrain(Step step)
{
PreTrainingSamples += 1;
PreTrainCore(step);
}
public MarkingStep(Step upstream, LeNetConfiguration configuration)
: this(new[] { upstream }, configuration)
{
}
public virtual void Train(Step downstream)
{
TrainCore(downstream);
}
public override void Train(Step downstream)
{
FinaliseErrorFirstDerivatives(downstream);
base.Train(downstream);
}
protected abstract void TrainCore(Step downstream);
public override void PreTrain(Step downstream)
{
FinaliseErrorSecondDerivatives(downstream);
base.PreTrain(downstream);
}