private void PropogateError(TrainingData ts, int batchSize)
{
// Initialize the error from the future
List <Vector <double> > futureErrorCache = new List <Vector <double> >();
for (var i = 0; i < Layers.Count; i++)
{
futureErrorCache.Add(new DenseVector(Layers[i].OutputDimension)); // Only store the error relevant to that layer
}
// Step backwards through the memory
for (var t = OutputCache.Count - 1; t > 0; t--)
{
// Error on the output layer from the training data
Vector <double> error = CostFunc.Derivative(ts[t - 1].Response, OutputCache[t].Last());
// Step backwards through the net.
for (var i = Layers.Count - 1; i >= 0; i--)
{
error += futureErrorCache[i];
Vector <double> lastInput = Concatenate(Layers[i].InputDimension, OutputCache[t - 1][i + 1], OutputCache[t][i]);// [t-1][i+1] is the output of the current layer at a previous time
Vector <double> jointInputError = Layers[i].PropogateError(error, LearningRate / batchSize, lastInput);
Vector <double> pastStateError;
// If this is the first layer, error would be the error on the training input, so we can just ignore it.
Split(jointInputError, Layers[i].OutputDimension, out pastStateError, out error, OutputCache[t][i]?.Count ?? 1);
futureErrorCache[i] = pastStateError; // Store the most recent error from the future.
}
}
}
/// <summary>
/// Performs SGD on a set of training data using a mini-batch size provided.
/// </summary>
/// <param name="trainingSet"></param>
/// <param name="batchSize"></param>
/// <returns>The average cost function evaluation for each batch</returns>
internal override void Learn(HashSet <TrainingData> trainingSet, int batchSize)
{
batchSize = Math.Min(batchSize, trainingSet.Count);
Vector <double> output;
int count = 0;
double cost = 0;
int batchNumber = 0;
foreach (TrainingData td in trainingSet)
{
output = Process(td.Data);
cost += CostFunc.Of(td.Response, output) / batchSize;
PropogateError(CostFunc.Derivative(td.Response, output), batchSize);
count++;
if (Abort)
{
return;
}
if (count > 0 && count % batchSize == 0)
{
batchNumber++;
LastCost = cost;
ApplyError();
if (!Abort) // Trying to make this kind of threadsafe
{
Hook?.Invoke(batchNumber, this); // Trigger the batch level external control
}
count = 0;
cost = 0;
}
}
}