public IReadOnlyList <IRecurrentExecutionResults[]> Execute(ISequentialTrainingDataProvider trainingData, float[] memory, IRecurrentTrainingContext context) { List <IRecurrentExecutionResults> temp; var sequenceOutput = new Dictionary <int, List <IRecurrentExecutionResults> >(); var batchSize = context.TrainingContext.MiniBatchSize; foreach (var miniBatch in _GetMiniBatches(trainingData, false, batchSize)) { _lap.PushLayer(); context.ExecuteForward(miniBatch, memory, (k, fc) => { foreach (var action in _layer) { action.Execute(fc, false); } var memoryOutput = fc[1].AsIndexable().Rows.ToList(); // store the output if (!sequenceOutput.TryGetValue(k, out temp)) { sequenceOutput.Add(k, temp = new List <IRecurrentExecutionResults>()); } var ret = fc[0].AsIndexable().Rows.Zip(miniBatch.GetExpectedOutput(fc, k).AsIndexable().Rows, (a, e) => Tuple.Create(a, e)); temp.AddRange(ret.Zip(memoryOutput, (t, d) => new RecurrentExecutionResults(t.Item1, t.Item2, d))); }); // cleanup context.TrainingContext.EndBatch(); _lap.PopLayer(); miniBatch.Dispose(); } return(sequenceOutput.OrderBy(kv => kv.Key).Select(kv => kv.Value.ToArray()).ToList()); }
public void TrainOnMiniBatch(ISequentialMiniBatch miniBatch, float[] memory, IRecurrentTrainingContext context, Action <IMatrix> beforeBackProp, Action <IMatrix> afterBackProp) { var trainingContext = context.TrainingContext; _lap.PushLayer(); var sequenceLength = miniBatch.SequenceLength; var updateStack = new Stack <Tuple <Stack <INeuralNetworkRecurrentBackpropagation>, IMatrix, IMatrix, ISequentialMiniBatch, int> >(); context.ExecuteForward(miniBatch, memory, (k, fc) => { var layerStack = new Stack <INeuralNetworkRecurrentBackpropagation>(); foreach (var action in _layer) { layerStack.Push(action.Execute(fc, true)); } updateStack.Push(Tuple.Create(layerStack, miniBatch.GetExpectedOutput(fc, k), fc[0], miniBatch, k)); }); // backpropagate, accumulating errors across the sequence using (var updateAccumulator = new UpdateAccumulator(trainingContext)) { IMatrix curr = null; while (updateStack.Any()) { var update = updateStack.Pop(); var isT0 = !updateStack.Any(); var actionStack = update.Item1; // calculate error var expectedOutput = update.Item2; if (expectedOutput != null) { curr = trainingContext.ErrorMetric.CalculateDelta(update.Item3, expectedOutput); } // backpropagate beforeBackProp?.Invoke(curr); while (actionStack.Any()) { var backpropagationAction = actionStack.Pop(); var shouldCalculateOutput = actionStack.Any() || isT0; curr = backpropagationAction.Execute(curr, trainingContext, true, updateAccumulator); } afterBackProp?.Invoke(curr); // apply any filters foreach (var filter in _filter) { filter.AfterBackPropagation(update.Item4, update.Item5, curr); } } // adjust the initial memory against the error signal if (curr != null) { using (var columnSums = curr.ColumnSums()) { var initialDelta = columnSums.AsIndexable(); for (var j = 0; j < memory.Length; j++) { memory[j] += initialDelta[j] * trainingContext.TrainingRate; } } } } // cleanup trainingContext.EndBatch(); _lap.PopLayer(); }