public override void Update(IMatrix biasDelta, IMatrix weightDelta, ITrainingContext context)
{
var t = context.CurrentEpoch;
using (var deltaSquared = weightDelta.PointwiseMultiply(weightDelta)) {
_cache.AddInPlace(weightDelta, _decay, 1 - _decay);
_cache2.AddInPlace(deltaSquared, _decay2, 1 - _decay2);
using (var mb = _cache.Clone())
using (var vb = _cache2.Clone()) {
mb.Multiply(1f / (1f - Convert.ToSingle(Math.Pow(_decay, t))));
vb.Multiply(1f / (1f - Convert.ToSingle(Math.Pow(_decay2, t))));
using (var vbSqrt = vb.Sqrt(1e-8f))
using (var delta = mb.PointwiseDivide(vbSqrt)) {
_layerUpdater.Update(biasDelta, delta, context);
}
}
}
}
public IMatrix Execute(IMatrix curr, ITrainingContext context, bool calculateOutput, INeuralNetworkUpdateAccumulator updateAccumulator)
{
using (var ad = _activation.Derivative(_output, curr)) {
// clip the gradient
ad.Constrain(-1f, 1f);
var delta = curr.PointwiseMultiply(ad);
var prevDelta = delta.TransposeAndMultiply(_memoryUpdater.Layer.Weight);
var memoryUpdate = _memory.TransposeThisAndMultiply(delta);
var inputUpdate = _input.TransposeThisAndMultiply(delta);
//_input.Dispose();
_memory.Dispose();
_output.Dispose();
updateAccumulator.Record(_memoryUpdater, delta, memoryUpdate);
updateAccumulator.Record(_inputUpdater, delta.Clone(), inputUpdate);
return(prevDelta);
}
}
protected bool _CalculateTestScore(ITrainingContext context, float[] forwardMemory, float[] backwardMemory, ISequentialTrainingDataProvider data, INeuralNetworkBidirectionalBatchTrainer network, IRecurrentTrainingContext recurrentContext, ref double bestScore, ref BidirectionalNetwork output)
{
bool flag = false;
var score = _GetScore(data, network, forwardMemory, backwardMemory, recurrentContext);
var errorMetric = recurrentContext.TrainingContext.ErrorMetric;
if ((errorMetric.HigherIsBetter && score > bestScore) || (!errorMetric.HigherIsBetter && score < bestScore))
{
bestScore = score;
output = network.NetworkInfo;
output.ForwardMemory = new FloatArray {
Data = forwardMemory
};
output.BackwardMemory = new FloatArray {
Data = backwardMemory
};
flag = true;
}
context.WriteScore(score, errorMetric.DisplayAsPercentage, flag);
return(flag);
}
public ParallelTreeTrainingOperation(
Random random,
ITrainingContext <F, S> trainingContext,
TrainingParameters parameters,
int maxThreads,
IDataPointCollection data,
ProgressWriter progress)
{
data_ = data;
trainingContext_ = trainingContext;
parameters_ = parameters;
maxThreads_ = maxThreads;
random_ = random;
progress_ = progress;
parentStatistics_ = trainingContext_.GetStatisticsAggregator();
leftChildStatistics_ = trainingContext_.GetStatisticsAggregator();
rightChildStatistics_ = trainingContext_.GetStatisticsAggregator();
responses_ = new float[data.Count()];
indices_ = new int[data.Count()];
for (int i = 0; i < indices_.Length; i++)
{
indices_[i] = i;
}
threadLocals_ = new ThreadLocalData[maxThreads_];
for (int threadIndex = 0; threadIndex < maxThreads_; threadIndex++)
{
threadLocals_[threadIndex] = new ThreadLocalData(random_, trainingContext_, parameters_, data_);
}
}
/// <summary>
/// Train a new decision tree given some training data and a training
/// problem described by an ITrainingContext instance.
/// </summary>
/// <param name="random">The single random number generator.</param>
/// <param name="progress">Progress reporting target.</param>
/// <param name="context">The ITrainingContext instance by which
/// the training framework interacts with the training data.
/// Implemented within client code.</param>
/// <param name="parameters">Training parameters.</param>
/// <param name="maxThreads">The maximum number of threads to use.</param>
/// <param name="data">The training data.</param>
/// <returns>A new decision tree.</returns>
static public Tree <F, S> TrainTree(
Random random,
ITrainingContext <F, S> context,
TrainingParameters parameters,
int maxThreads,
IDataPointCollection data,
ProgressWriter progress = null)
{
if (progress == null)
{
progress = new ProgressWriter(Verbosity.Interest, Console.Out);
}
ParallelTreeTrainingOperation <F, S> trainingOperation = new ParallelTreeTrainingOperation <F, S>(
random, context, parameters, maxThreads, data, progress);
Tree <F, S> tree = new Tree <F, S>(parameters.MaxDecisionLevels);
trainingOperation.TrainNodesRecurse(tree.nodes_, 0, 0, data.Count(), 0); // will recurse until termination criterion is met
tree.CheckValid();
return(tree);
}
private void OnEpochComplete(ITrainingContext context)
{
var score = _GetScore(context);
var flag = false;
var errorMetric = context.ErrorMetric;
if ((errorMetric.HigherIsBetter && score > _bestScore) || (!errorMetric.HigherIsBetter && score < _bestScore))
{
_bestScore = score;
_bestOutput = _trainer.NetworkInfo;
flag = true;
}
if ((context.CurrentEpoch % _reportCadence) == 0)
{
context.WriteScore(score, errorMetric.DisplayAsPercentage, flag);
}
if (flag)
{
using (var stream = new FileStream(_dataFile, FileMode.Create, FileAccess.Write))
Serializer.Serialize(stream, _bestOutput);
_noChange = 0;
}
else
{
++_noChange;
}
if (_autoAdjustOnNoChangeCount.HasValue && _noChange >= _autoAdjustOnNoChangeCount.Value)
{
_noChange = 0;
ApplyBestParams();
context.ReduceTrainingRate();
Console.WriteLine("Reducing training rate to " + context.TrainingRate);
}
}
public IMatrix Backpropagate(IMatrix input, IMatrix output, IMatrix errorSignal, ITrainingContext context, bool calculateOutput, INeuralNetworkUpdateAccumulator updates = null)
{
IMatrix ret = null;
// calculate the derivative to determine the gradients
using (var od = _layerUpdater.Layer.Activation?.Derivative(output, errorSignal) ?? output) {
if (_verifyDerivatives)
{
var dError = _VerifyDerivatives(od);
Debug.Assert(dError < 0.001f);
}
// use the gradients to determine the direction of each change
var delta = errorSignal.PointwiseMultiply(od);
if (calculateOutput)
{
ret = _layerUpdater.Layer.CalculateErrorSignal(delta);
}
// update the layer
_UpdateLayer(input, delta, context, updates);
}
return(ret);
}
public RecurrentContext(ILinearAlgebraProvider lap, ITrainingContext trainingContext)
{
_lap = lap;
_trainingContext = trainingContext;
}
public IMatrix Execute(IMatrix error, ITrainingContext context, bool calculateOutput, INeuralNetworkUpdateAccumulator updateAccumulator)
{
//using (var input = _input)
using (var output = _output)
return(_trainer.Backpropagate(_input, output, error, context, calculateOutput, updateAccumulator));
}
public override void Update(IMatrix biasDelta, IMatrix weightDelta, ITrainingContext context)
{
var l2 = 1.0f - (context.TrainingRate * _lambda / context.TrainingSamples);
Update(biasDelta, weightDelta, l2, context);
}
public UpdateAccumulator(ITrainingContext context)
{
_context = context;
}
public override void Update(IMatrix biasDelta, IMatrix weightDelta, ITrainingContext context)
{
_cache.AddInPlace(weightDelta, _momentum);
_layerUpdater.Update(biasDelta, _cache, context);
}
public virtual void Update(IMatrix biasDelta, IMatrix weightDelta, ITrainingContext context)
{
Update(biasDelta, weightDelta, 1, context);
}
public abstract void Update(IMatrix biasDelta, IMatrix weightDelta, ITrainingContext context);
public TrainingController(ITrainingContext context)
{
dbContext = context;
}
public float CalculateCost(ITrainingDataProvider data, ITrainingContext trainingContext)
{
return(Execute(data).Select(r => trainingContext.ErrorMetric.Compute(r.Output, r.ExpectedOutput)).Average());
}
public void Train(ISequentialTrainingDataProvider trainingData, int numEpochs, ITrainingContext context, IRecurrentTrainingContext recurrentContext = null)
{
if (recurrentContext == null)
{
recurrentContext = new RecurrentContext(_trainer.LinearAlgebraProvider, context);
}
_bestScore = _GetScore(_testData, _trainer, _memory, recurrentContext);
Console.WriteLine(context.ErrorMetric.DisplayAsPercentage ? "Initial score: {0:P}" : "Initial score: {0}", _bestScore);
_bestOutput = null;
recurrentContext.TrainingContext.RecurrentEpochComplete += OnEpochComplete;
_memory = _trainer.Train(trainingData, _memory, numEpochs, recurrentContext);
recurrentContext.TrainingContext.RecurrentEpochComplete -= OnEpochComplete;
// ensure best values are current
ApplyBestParams();
}
protected void Update(IMatrix biasDelta, IMatrix weightDelta, float weightCoefficient, ITrainingContext context)
{
_layer.Update(biasDelta, weightDelta, weightCoefficient, context.TrainingRate / context.MiniBatchSize);
}
double _GetScore(ITrainingContext context)
{
return(Math.Abs(_trainer.Execute(_testData).Average(d => context.ErrorMetric.Compute(d.Output, d.ExpectedOutput))));
}
public IRecurrentTrainingContext CreateRecurrentTrainingContext(ITrainingContext trainingContext)
{
return(new RecurrentContext(_lap, trainingContext));
}