/// <summary>
/// <para>
/// Perform one pass of training. Adjust the weights based on the current state of the <see cref="Output"/> layer and the desired values.
/// Use <see cref="FeedForward"/> to calculate the output values.
/// </para>
///
/// <para>
/// Calculate the errors/losses of each layer (using <see cref="CalculateLoss(Vector)"/>)
/// and then adjust the weights accordingly (using <see cref="NNOperations.CalculateDeltas(Layer, Layer, Vector, ActivationFunction, float)"/>).
/// </para>
/// </summary>
/// <param name="desiredOutput">the desired output value of the network</param>
/// <returns>the results</returns>
public LayerResult[] AdjustWeights(Vector desiredOutput)
{
LayerResult[] results = CalculateLoss(desiredOutput);
for (int i = results.Length - 1; i >= 0; i--) // Iterate over results backwards
{
if (i == 0)
{
break;
}
LayerResult L1R = results[i];
LayerResult L0R = results[i - 1];
// Get the values to adjust weights and biases
Deltas L0deltas = NNOperations.CalculateDeltas(L0R.Layer, L1R.Layer, L1R.Loss, Properties.DerivativeActivation, Properties.LearningRate);
// create new adjusted weights and biases
Matrix nw = L0R.Layer.Weights + L0deltas.Weights;
Vector nb = L0R.Layer.Biases + L0deltas.Biases;
// Apply adjustments
L0R.Layer.Weights.Values = nw.Values;
L0R.Layer.Biases.Values = nb.Values;
results[i - 1].Deltas = L0deltas;
}
return(results);
}
/// <summary>
/// Get the deltas for the weights of a given layer L0.
/// </summary>
/// <param name="L0">first layer</param>
/// <param name="L1">second layer</param>
/// <param name="loss">the loss of L0</param>
/// <param name="daf">the (cool) derivative of the activation function</param>
/// <param name="learningRate">the learning rate</param>
/// <returns>the deltas for the weights</returns>
public static Deltas CalculateDeltas(Layer L0, Layer L1, Vector loss, ActivationFunction daf, float learningRate)
{
//Vector da = L1.Nodes.Copy().Map(v => daf(v));
Vector da = Vector.Map(L1.Nodes, v => daf(v)); // derivative of the values (a) of L1
Matrix ld = Matrix.ElementWiseOperation(loss.ToMatrix(), da.ToMatrix(), Operations.Multiply); // L1 loss derivative
Matrix db = ld * learningRate; // delta for biases
Matrix dw = ld * Matrix.Transpose(L0.Nodes.ToMatrix()) * learningRate; // delta for weights
// Put them into an object
Deltas deltas = new Deltas();
deltas.Weights = dw;
deltas.Biases = new Vector(Matrix.Transpose(db).Values[0]);
return(deltas);
}
/// <summary>
/// Create a result.
/// </summary>
/// <param name="layer">the layer</param>
/// <param name="loss">the loss</param>
public LayerResult(Layer layer, Vector loss)
{
Layer = layer;
Loss = loss;
Deltas = new Deltas();
}