private void CalculateHiddenToOutputWeightChanges(Double[] idealOutput, Double learningRate)
{
m_totalOutputError = 0; // reset total error.
for (Int32 outputNeuronIndex = 0; outputNeuronIndex < m_numOutput; outputNeuronIndex++)
{
// Total Squared Error for each output neuron.
m_outputSquaredErrors[outputNeuronIndex] = BackpropFunctions.SquaredErrorFunction(idealOutput[outputNeuronIndex], m_outputLayerOutput[outputNeuronIndex]);
// Total Error of network
m_totalOutputError += m_outputSquaredErrors[outputNeuronIndex];
// Partial Derivative for total error with respect to output. (Derivative of total squared error results in this)
m_outputPartialDerivTotalErrorOverOutputNeuron[outputNeuronIndex] = m_outputLayerOutput[outputNeuronIndex] - idealOutput[outputNeuronIndex];
// Partial Derivative of output of a particular neuron respecting its total net input (derivative of activation function)
m_outputPartialDerivOutputNeuronOverTotalInput[outputNeuronIndex] = ActivationFunctions.TanhDerivative(m_outputLayerOutput[outputNeuronIndex]);
for (Int32 hiddenNeuronIndex = 0; hiddenNeuronIndex < m_numHidden + BiasNeuronCount; hiddenNeuronIndex++)
{
// Partial Derivative of total net input with respect to weight. Basically, the output of the hidden neuron.
m_outputPartialDerivTotalInputOverWeight[hiddenNeuronIndex, outputNeuronIndex] = m_hiddenLayerOutput[hiddenNeuronIndex];
// Multiply the partial derivatives together
Double deltaOutputNeuron = m_outputPartialDerivTotalErrorOverOutputNeuron[outputNeuronIndex] *
m_outputPartialDerivOutputNeuronOverTotalInput[outputNeuronIndex] *
m_outputPartialDerivTotalInputOverWeight[hiddenNeuronIndex, outputNeuronIndex];
Double changeInWeight = deltaOutputNeuron * learningRate;
m_hoWeightChanges[hiddenNeuronIndex, outputNeuronIndex] = changeInWeight;
}
}
}
private void CalculateInputToHiddenWeightChanges(Double learningRate)
{
// We need to figure out the partial derivative of the total error with respect to each weight.
// This is the same as:
// Partial Derivative of Total Error with respect to the output of the hidden neuron *
// (This is equal to the sum of partial derivatives of each output neuron's squared error with respect to the hidden neuron's output)
// Partial Derivative of the output of the hidden neuron with respect to its net input *
// Partial Derivative of the Net Input with respect to the weight from a input neuron to the hidden neuron.
for (Int32 hiddenNeuronIndex = 0; hiddenNeuronIndex < m_numHidden; hiddenNeuronIndex++)
{
for (Int32 inputNeuronIndex = 0; inputNeuronIndex < m_numInput + BiasNeuronCount; inputNeuronIndex++)
{
m_hiddenPartialDerivTotalErrorOverHiddenOutput[inputNeuronIndex, hiddenNeuronIndex] = 0.0;
Double runningTotal = 0.0;
for (Int32 outputNeuronIndex = 0; outputNeuronIndex < m_numOutput; outputNeuronIndex++)
{
runningTotal += m_outputPartialDerivTotalErrorOverOutputNeuron[outputNeuronIndex] *
m_outputPartialDerivOutputNeuronOverTotalInput[outputNeuronIndex] *
m_ho[hiddenNeuronIndex, outputNeuronIndex];
}
m_hiddenPartialDerivTotalErrorOverHiddenOutput[inputNeuronIndex, hiddenNeuronIndex] = runningTotal;
m_hiddenPartialDerivHiddenOutputOverNetInput[inputNeuronIndex, hiddenNeuronIndex] =
ActivationFunctions.TanhDerivative(m_hiddenLayerOutput[hiddenNeuronIndex]);
m_hiddenPartialDerivNetInputOverWeight[inputNeuronIndex, hiddenNeuronIndex] = m_lastInput[inputNeuronIndex];
m_ihWeightChanges[inputNeuronIndex, hiddenNeuronIndex] = learningRate *
m_hiddenPartialDerivTotalErrorOverHiddenOutput[inputNeuronIndex, hiddenNeuronIndex] *
m_hiddenPartialDerivHiddenOutputOverNetInput[inputNeuronIndex, hiddenNeuronIndex] *
m_hiddenPartialDerivNetInputOverWeight[inputNeuronIndex, hiddenNeuronIndex];
}
}
}
/// <summary>
/// Given the input (should be one entry for each input neuron, excluding the bias, whose input is always 1), calculate and store
/// the output of the hidden and output layers.
/// </summary>
/// <param name="input">An array of values corresponding to the input for the input layer.</param>
public void ComputeOutput(Double[] input)
{
if (input.Length != m_numInput)
{
throw new Exception("The input needs to be " + this.m_numInput.ToString() + " in length.");
}
for (Int32 i = 0; i < m_numInput; i++)
{
m_lastInput[i] = input[i];
}
// Step1: Compute output of Hidden Layer
for (Int32 i = 0; i < m_numHidden; i++)
{
m_hiddenLayerOutput[i] = 0; // Initialize
for (Int32 j = 0; j < m_numInput + BiasNeuronCount; j++)
{
if (j == m_numInput) // Bias
{
m_hiddenLayerOutput[i] += (m_ih[j, i] * 1.0);
}
else
{
m_hiddenLayerOutput[i] += (m_ih[j, i] * input[j]);
}
}
// Now that you summed weights * inputs, squash
m_hiddenLayerOutput[i] = ActivationFunctions.Tanh(m_hiddenLayerOutput[i]);
}
// Step2: Compute output of Output Layer
for (Int32 i = 0; i < this.m_numOutput; i++)
{
m_outputLayerOutput[i] = 0; // Initialize
for (Int32 j = 0; j < this.m_numHidden + BiasNeuronCount; j++)
{
if (j == NumHidden)
{
m_outputLayerOutput[i] += (m_ho[j, i] * 1.0);
}
else
{
m_outputLayerOutput[i] += (m_ho[j, i] * m_hiddenLayerOutput[j]);
}
}
// Now that you summed weights * inputs, squash
m_outputLayerOutput[i] = ActivationFunctions.Tanh(m_outputLayerOutput[i]);
}
}
