public IReadOnlyList <IRecurrentExecutionResults[]> Execute(ISequentialTrainingDataProvider trainingData, float[] memory, IRecurrentTrainingContext context)
{
List <IRecurrentExecutionResults> temp;
var sequenceOutput = new Dictionary <int, List <IRecurrentExecutionResults> >();
var batchSize = context.TrainingContext.MiniBatchSize;
foreach (var miniBatch in _GetMiniBatches(trainingData, false, batchSize))
{
_lap.PushLayer();
context.ExecuteForward(miniBatch, memory, (k, fc) => {
foreach (var action in _layer)
{
action.Execute(fc, false);
}
var memoryOutput = fc[1].AsIndexable().Rows.ToList();
// store the output
if (!sequenceOutput.TryGetValue(k, out temp))
{
sequenceOutput.Add(k, temp = new List <IRecurrentExecutionResults>());
}
var ret = fc[0].AsIndexable().Rows.Zip(miniBatch.GetExpectedOutput(fc, k).AsIndexable().Rows, (a, e) => Tuple.Create(a, e));
temp.AddRange(ret.Zip(memoryOutput, (t, d) => new RecurrentExecutionResults(t.Item1, t.Item2, d)));
});
// cleanup
context.TrainingContext.EndBatch();
_lap.PopLayer();
miniBatch.Dispose();
}
return(sequenceOutput.OrderBy(kv => kv.Key).Select(kv => kv.Value.ToArray()).ToList());
}
public void TrainOnMiniBatch(ISequentialMiniBatch miniBatch, float[] memory, IRecurrentTrainingContext context, Action <IMatrix> beforeBackProp, Action <IMatrix> afterBackProp)
{
var trainingContext = context.TrainingContext;
_lap.PushLayer();
var sequenceLength = miniBatch.SequenceLength;
var updateStack = new Stack <Tuple <Stack <INeuralNetworkRecurrentBackpropagation>, IMatrix, IMatrix, ISequentialMiniBatch, int> >();
context.ExecuteForward(miniBatch, memory, (k, fc) => {
var layerStack = new Stack <INeuralNetworkRecurrentBackpropagation>();
foreach (var action in _layer)
{
layerStack.Push(action.Execute(fc, true));
}
updateStack.Push(Tuple.Create(layerStack, miniBatch.GetExpectedOutput(fc, k), fc[0], miniBatch, k));
});
// backpropagate, accumulating errors across the sequence
using (var updateAccumulator = new UpdateAccumulator(trainingContext)) {
IMatrix curr = null;
while (updateStack.Any())
{
var update = updateStack.Pop();
var isT0 = !updateStack.Any();
var actionStack = update.Item1;
// calculate error
var expectedOutput = update.Item2;
if (expectedOutput != null)
{
curr = trainingContext.ErrorMetric.CalculateDelta(update.Item3, expectedOutput);
}
// backpropagate
beforeBackProp?.Invoke(curr);
while (actionStack.Any())
{
var backpropagationAction = actionStack.Pop();
var shouldCalculateOutput = actionStack.Any() || isT0;
curr = backpropagationAction.Execute(curr, trainingContext, true, updateAccumulator);
}
afterBackProp?.Invoke(curr);
// apply any filters
foreach (var filter in _filter)
{
filter.AfterBackPropagation(update.Item4, update.Item5, curr);
}
}
// adjust the initial memory against the error signal
if (curr != null)
{
using (var columnSums = curr.ColumnSums()) {
var initialDelta = columnSums.AsIndexable();
for (var j = 0; j < memory.Length; j++)
{
memory[j] += initialDelta[j] * trainingContext.TrainingRate;
}
}
}
}
// cleanup
trainingContext.EndBatch();
_lap.PopLayer();
}