/// <summary> /// <para> /// Perform one pass of training. Adjust the weights based on the current state of the <see cref="Output"/> layer and the desired values. /// Use <see cref="FeedForward"/> to calculate the output values. /// </para> /// /// <para> /// Calculate the errors/losses of each layer (using <see cref="CalculateLoss(Vector)"/>) /// and then adjust the weights accordingly (using <see cref="NNOperations.CalculateDeltas(Layer, Layer, Vector, ActivationFunction, float)"/>). /// </para> /// </summary> /// <param name="desiredOutput">the desired output value of the network</param> /// <returns>the results</returns> public LayerResult[] AdjustWeights(Vector desiredOutput) { LayerResult[] results = CalculateLoss(desiredOutput); for (int i = results.Length - 1; i >= 0; i--) // Iterate over results backwards { if (i == 0) { break; } LayerResult L1R = results[i]; LayerResult L0R = results[i - 1]; // Get the values to adjust weights and biases Deltas L0deltas = NNOperations.CalculateDeltas(L0R.Layer, L1R.Layer, L1R.Loss, Properties.DerivativeActivation, Properties.LearningRate); // create new adjusted weights and biases Matrix nw = L0R.Layer.Weights + L0deltas.Weights; Vector nb = L0R.Layer.Biases + L0deltas.Biases; // Apply adjustments L0R.Layer.Weights.Values = nw.Values; L0R.Layer.Biases.Values = nb.Values; results[i - 1].Deltas = L0deltas; } return(results); }
/// <summary> /// Get the deltas for the weights of a given layer L0. /// </summary> /// <param name="L0">first layer</param> /// <param name="L1">second layer</param> /// <param name="loss">the loss of L0</param> /// <param name="daf">the (cool) derivative of the activation function</param> /// <param name="learningRate">the learning rate</param> /// <returns>the deltas for the weights</returns> public static Deltas CalculateDeltas(Layer L0, Layer L1, Vector loss, ActivationFunction daf, float learningRate) { //Vector da = L1.Nodes.Copy().Map(v => daf(v)); Vector da = Vector.Map(L1.Nodes, v => daf(v)); // derivative of the values (a) of L1 Matrix ld = Matrix.ElementWiseOperation(loss.ToMatrix(), da.ToMatrix(), Operations.Multiply); // L1 loss derivative Matrix db = ld * learningRate; // delta for biases Matrix dw = ld * Matrix.Transpose(L0.Nodes.ToMatrix()) * learningRate; // delta for weights // Put them into an object Deltas deltas = new Deltas(); deltas.Weights = dw; deltas.Biases = new Vector(Matrix.Transpose(db).Values[0]); return(deltas); }
/// <summary> /// Create a result. /// </summary> /// <param name="layer">the layer</param> /// <param name="loss">the loss</param> public LayerResult(Layer layer, Vector loss) { Layer = layer; Loss = loss; Deltas = new Deltas(); }