private void OnEpochComplete(ITrainingContext context, IRecurrentTrainingContext recurrentContext) { if (_CalculateTestScore(context, _memory, _testData, _trainer, recurrentContext, ref _bestScore, ref _bestOutput)) { using (var stream = new FileStream(_dataFile, FileMode.Create, FileAccess.Write)) Serializer.Serialize(stream, _bestOutput); } }
protected bool _CalculateTestScore(ITrainingContext context, float[] forwardMemory, float[] backwardMemory, ISequentialTrainingDataProvider data, INeuralNetworkBidirectionalBatchTrainer network, IRecurrentTrainingContext recurrentContext, ref double bestScore, ref BidirectionalNetwork output) { bool flag = false; var score = _GetScore(data, network, forwardMemory, backwardMemory, recurrentContext); var errorMetric = recurrentContext.TrainingContext.ErrorMetric; if ((errorMetric.HigherIsBetter && score > bestScore) || (!errorMetric.HigherIsBetter && score < bestScore)) { bestScore = score; output = network.NetworkInfo; output.ForwardMemory = new FloatArray { Data = forwardMemory }; output.BackwardMemory = new FloatArray { Data = backwardMemory }; flag = true; } context.WriteScore(score, errorMetric.DisplayAsPercentage, flag); return(flag); }
protected bool _CalculateTestScore(ITrainingContext context, float[] memory, ISequentialTrainingDataProvider data, INeuralNetworkRecurrentBatchTrainer network, IRecurrentTrainingContext recurrentContext, ref double bestScore, ref RecurrentNetwork output) { bool flag = false; var score = _GetScore(data, network, memory, recurrentContext); var errorMetric = recurrentContext.TrainingContext.ErrorMetric; if ((errorMetric.HigherIsBetter && score > bestScore) || (!errorMetric.HigherIsBetter && score < bestScore)) { bestScore = score; output = network.NetworkInfo; output.Memory = new FloatArray { Data = memory }; flag = true; } context.WriteScore(score, errorMetric.DisplayAsPercentage, flag); if (flag) { _noChange = 0; } else { ++_noChange; } if (_autoAdjustOnNoChangeCount.HasValue && _noChange >= _autoAdjustOnNoChangeCount.Value) { context.ReduceTrainingRate(); Console.WriteLine("Reducing training rate to " + context.TrainingRate); ApplyBestParams(); _noChange = 0; } return(flag); }
protected double _GetScore(ISequentialTrainingDataProvider data, INeuralNetworkBidirectionalBatchTrainer network, float[] forwardMemory, float[] backwardMemory, IRecurrentTrainingContext context) { return(Math.Abs(network.Execute(data, forwardMemory, backwardMemory, context).SelectMany(d => d).Select(d => context.TrainingContext.ErrorMetric.Compute(d.Output, d.ExpectedOutput)).Average())); }
public void Train(ISequentialTrainingDataProvider trainingData, int numEpochs, ITrainingContext context, IRecurrentTrainingContext recurrentContext = null) { if (recurrentContext == null) { recurrentContext = new RecurrentContext(_trainer.LinearAlgebraProvider, context); } _bestScore = _GetScore(_testData, _trainer, _memory, recurrentContext); Console.WriteLine(context.ErrorMetric.DisplayAsPercentage ? "Initial score: {0:P}" : "Initial score: {0}", _bestScore); _bestOutput = null; recurrentContext.TrainingContext.RecurrentEpochComplete += OnEpochComplete; _memory = _trainer.Train(trainingData, _memory, numEpochs, recurrentContext); recurrentContext.TrainingContext.RecurrentEpochComplete -= OnEpochComplete; // ensure best values are current ApplyBestParams(); }
public float CalculateCost(ISequentialTrainingDataProvider data, float[] memory, IRecurrentTrainingContext context) { return(Execute(data, memory, context).SelectMany(r => r).Select(r => context.TrainingContext.ErrorMetric.Compute(r.Output, r.ExpectedOutput)).Average()); }
public IReadOnlyList <IRecurrentExecutionResults[]> Execute(ISequentialTrainingDataProvider trainingData, float[] memory, IRecurrentTrainingContext context) { List <IRecurrentExecutionResults> temp; var sequenceOutput = new Dictionary <int, List <IRecurrentExecutionResults> >(); var batchSize = context.TrainingContext.MiniBatchSize; foreach (var miniBatch in _GetMiniBatches(trainingData, false, batchSize)) { _lap.PushLayer(); context.ExecuteForward(miniBatch, memory, (k, fc) => { foreach (var action in _layer) { action.Execute(fc, false); } var memoryOutput = fc[1].AsIndexable().Rows.ToList(); // store the output if (!sequenceOutput.TryGetValue(k, out temp)) { sequenceOutput.Add(k, temp = new List <IRecurrentExecutionResults>()); } var ret = fc[0].AsIndexable().Rows.Zip(miniBatch.GetExpectedOutput(fc, k).AsIndexable().Rows, (a, e) => Tuple.Create(a, e)); temp.AddRange(ret.Zip(memoryOutput, (t, d) => new RecurrentExecutionResults(t.Item1, t.Item2, d))); }); // cleanup context.TrainingContext.EndBatch(); _lap.PopLayer(); miniBatch.Dispose(); } return(sequenceOutput.OrderBy(kv => kv.Key).Select(kv => kv.Value.ToArray()).ToList()); }
public float[] Train(ISequentialTrainingDataProvider trainingData, float[] memory, int numEpochs, IRecurrentTrainingContext context) { var trainingContext = context.TrainingContext; for (int i = 0; i < numEpochs && context.TrainingContext.ShouldContinue; i++) { trainingContext.StartEpoch(trainingData.Count); var batchErrorList = new List <double>(); foreach (var miniBatch in _GetMiniBatches(trainingData, _stochastic, trainingContext.MiniBatchSize)) { TrainOnMiniBatch(miniBatch, memory, context, curr => { if (_collectTrainingError) // get a measure of the training error { batchErrorList.Add(curr.AsIndexable().Values.Select(v => Math.Pow(v, 2)).Average() / 2); } }, null); miniBatch.Dispose(); } trainingContext.EndRecurrentEpoch(_collectTrainingError ? batchErrorList.Average() : 0, context); } return(memory); }
public void TrainOnMiniBatch(ISequentialMiniBatch miniBatch, float[] memory, IRecurrentTrainingContext context, Action <IMatrix> beforeBackProp, Action <IMatrix> afterBackProp) { var trainingContext = context.TrainingContext; _lap.PushLayer(); var sequenceLength = miniBatch.SequenceLength; var updateStack = new Stack <Tuple <Stack <INeuralNetworkRecurrentBackpropagation>, IMatrix, IMatrix, ISequentialMiniBatch, int> >(); context.ExecuteForward(miniBatch, memory, (k, fc) => { var layerStack = new Stack <INeuralNetworkRecurrentBackpropagation>(); foreach (var action in _layer) { layerStack.Push(action.Execute(fc, true)); } updateStack.Push(Tuple.Create(layerStack, miniBatch.GetExpectedOutput(fc, k), fc[0], miniBatch, k)); }); // backpropagate, accumulating errors across the sequence using (var updateAccumulator = new UpdateAccumulator(trainingContext)) { IMatrix curr = null; while (updateStack.Any()) { var update = updateStack.Pop(); var isT0 = !updateStack.Any(); var actionStack = update.Item1; // calculate error var expectedOutput = update.Item2; if (expectedOutput != null) { curr = trainingContext.ErrorMetric.CalculateDelta(update.Item3, expectedOutput); } // backpropagate beforeBackProp?.Invoke(curr); while (actionStack.Any()) { var backpropagationAction = actionStack.Pop(); var shouldCalculateOutput = actionStack.Any() || isT0; curr = backpropagationAction.Execute(curr, trainingContext, true, updateAccumulator); } afterBackProp?.Invoke(curr); // apply any filters foreach (var filter in _filter) { filter.AfterBackPropagation(update.Item4, update.Item5, curr); } } // adjust the initial memory against the error signal if (curr != null) { using (var columnSums = curr.ColumnSums()) { var initialDelta = columnSums.AsIndexable(); for (var j = 0; j < memory.Length; j++) { memory[j] += initialDelta[j] * trainingContext.TrainingRate; } } } } // cleanup trainingContext.EndBatch(); _lap.PopLayer(); }
public IReadOnlyList <IRecurrentExecutionResults[]> Execute(ISequentialTrainingDataProvider trainingData, float[] forwardMemory, float[] backwardMemory, IRecurrentTrainingContext context) { List <IRecurrentExecutionResults> temp; var sequenceOutput = new Dictionary <int, List <IRecurrentExecutionResults> >(); var batchSize = context.TrainingContext.MiniBatchSize; foreach (var miniBatch in _GetMiniBatches(trainingData, false, batchSize)) { _lap.PushLayer(); var sequenceLength = miniBatch.SequenceLength; context.ExecuteBidirectional(miniBatch, _layer, forwardMemory, backwardMemory, _padding, null, (memoryOutput, output) => { // store the output for (var k = 0; k < sequenceLength; k++) { if (!sequenceOutput.TryGetValue(k, out temp)) { sequenceOutput.Add(k, temp = new List <IRecurrentExecutionResults>()); } var ret = output[k].AsIndexable().Rows.Zip(miniBatch.GetExpectedOutput(output, k).AsIndexable().Rows, (a, e) => Tuple.Create(a, e)); temp.AddRange(ret.Zip(memoryOutput[k], (t, d) => new RecurrentExecutionResults(t.Item1, t.Item2, d))); } }); // cleanup context.TrainingContext.EndBatch(); _lap.PopLayer(); miniBatch.Dispose(); } return(sequenceOutput.OrderBy(kv => kv.Key).Select(kv => kv.Value.ToArray()).ToList()); }
public BidirectionalMemory Train(ISequentialTrainingDataProvider trainingData, float[] forwardMemory, float[] backwardMemory, int numEpochs, IRecurrentTrainingContext context) { var trainingContext = context.TrainingContext; var logger = trainingContext.Logger; for (int i = 0; i < numEpochs && context.TrainingContext.ShouldContinue; i++) { trainingContext.StartEpoch(trainingData.Count); var batchErrorList = new List <double>(); foreach (var miniBatch in _GetMiniBatches(trainingData, _stochastic, trainingContext.MiniBatchSize)) { _lap.PushLayer(); var sequenceLength = miniBatch.SequenceLength; var updateStack = new Stack <Tuple <Stack <Tuple <INeuralNetworkRecurrentBackpropagation, INeuralNetworkRecurrentBackpropagation> >, IMatrix, IMatrix, ISequentialMiniBatch, int> >(); context.ExecuteBidirectional(miniBatch, _layer, forwardMemory, backwardMemory, _padding, updateStack, null); // backpropagate, accumulating errors across the sequence using (var updateAccumulator = new UpdateAccumulator(trainingContext)) { while (updateStack.Any()) { var update = updateStack.Pop(); var isT0 = !updateStack.Any(); var actionStack = update.Item1; // calculate error var expectedOutput = update.Item2; var curr = new List <IMatrix>(); curr.Add(trainingContext.ErrorMetric.CalculateDelta(update.Item3, expectedOutput)); // get a measure of the training error if (_collectTrainingError) { foreach (var item in curr) { batchErrorList.Add(item.AsIndexable().Values.Select(v => Math.Pow(v, 2)).Average() / 2); } } #region logging if (logger != null) { logger.WriteStartElement("initial-error"); foreach (var item in curr) { item.WriteTo(logger); } logger.WriteEndElement(); } #endregion // backpropagate while (actionStack.Any()) { var backpropagationAction = actionStack.Pop(); if (backpropagationAction.Item1 != null && backpropagationAction.Item2 != null && curr.Count == 1) { using (var m = curr[0]) { var split = m.SplitRows(forwardMemory.Length); curr[0] = split.Left; curr.Add(split.Right); } #region logging if (logger != null) { logger.WriteStartElement("post-split"); foreach (var item in curr) { item.WriteTo(logger); } logger.WriteEndElement(); } #endregion } if (backpropagationAction.Item1 != null) { using (var m = curr[0]) curr[0] = backpropagationAction.Item1.Execute(m, trainingContext, actionStack.Any() || isT0, updateAccumulator); } if (backpropagationAction.Item2 != null) { using (var m = curr[1]) curr[1] = backpropagationAction.Item2.Execute(m, trainingContext, actionStack.Any() || isT0, updateAccumulator); } #region logging if (logger != null) { logger.WriteStartElement("error"); foreach (var item in curr) { item.WriteTo(logger); } logger.WriteEndElement(); } #endregion } // apply any filters foreach (var filter in _filter) { foreach (var item in curr) { filter.AfterBackPropagation(update.Item4, update.Item5, item); } } // adjust the initial memory against the error signal if (isT0) { using (var columnSums0 = curr[0].ColumnSums()) using (var columnSums1 = curr[1].ColumnSums()) { var initialDelta = columnSums0.AsIndexable(); for (var j = 0; j < forwardMemory.Length; j++) { forwardMemory[j] += initialDelta[j] * trainingContext.TrainingRate; } initialDelta = columnSums1.AsIndexable(); for (var j = 0; j < backwardMemory.Length; j++) { backwardMemory[j] += initialDelta[j] * trainingContext.TrainingRate; } } } } } // cleanup trainingContext.EndBatch(); _lap.PopLayer(); miniBatch.Dispose(); } trainingContext.EndRecurrentEpoch(_collectTrainingError ? batchErrorList.Average() : 0, context); } return(new BidirectionalMemory(forwardMemory, backwardMemory)); }
public void EndRecurrentEpoch(double trainingError, IRecurrentTrainingContext context) { EndEpoch(trainingError); RecurrentEpochComplete?.Invoke(this, context); }