void _Update(IMatrix error, INeuralNetworkLayerUpdater w, INeuralNetworkLayerUpdater u, INeuralNetworkUpdateAccumulator updateAccumulator)
{
var deltaW = _input.TransposeThisAndMultiply(error);
var deltaU = _pc.TransposeThisAndMultiply(error);
updateAccumulator.Record(w, error, deltaW);
updateAccumulator.Record(u, error.Clone(), deltaU);
}
public IMatrix Execute(IMatrix curr, ITrainingContext context, bool calculateOutput, INeuralNetworkUpdateAccumulator updateAccumulator)
{
using (var ad = _activation.Derivative(_output, curr)) {
// clip the gradient
ad.Constrain(-1f, 1f);
var delta = curr.PointwiseMultiply(ad);
var prevDelta = delta.TransposeAndMultiply(_memoryUpdater.Layer.Weight);
var memoryUpdate = _memory.TransposeThisAndMultiply(delta);
var inputUpdate = _input.TransposeThisAndMultiply(delta);
//_input.Dispose();
_memory.Dispose();
_output.Dispose();
updateAccumulator.Record(_memoryUpdater, delta, memoryUpdate);
updateAccumulator.Record(_inputUpdater, delta.Clone(), inputUpdate);
return(prevDelta);
}
}
protected virtual void _UpdateLayer(IMatrix input, IMatrix delta, ITrainingContext context, INeuralNetworkUpdateAccumulator updates)
{
var weightUpdate = input.TransposeThisAndMultiply(delta);
if (updates != null)
{
updates.Record(_layerUpdater, delta, weightUpdate);
}
else
{
_layerUpdater.Update(delta, weightUpdate, context);
weightUpdate.Dispose();
delta.Dispose();
}
}