public IMatrix Execute(IMatrix error, ITrainingContext context, bool calculateOutput, INeuralNetworkUpdateAccumulator updateAccumulator)
{
var matrixList = error.AsIndexable().Columns.Select(v => v.ToArray()).ToList();
var newMatrixList = new List <IMatrix>();
Tuple <int, int> newIndex;
for (var i = 0; i < matrixList.Count; i++)
{
var matrix = matrixList[i];
var table = _indexPosList[i];
newMatrixList.Add(_lap.Create(_rows, _columns, (x, y) => {
if (table.TryGetValue(Tuple.Create(x, y), out newIndex))
{
var newIndex2 = newIndex.Item1 * _newRows + newIndex.Item2;
return(matrix[newIndex2]);
}
return(0f);
}));
}
using (var tensor = _lap.CreateTensor(newMatrixList)) {
var ret = tensor.ConvertToMatrix();
foreach (var item in newMatrixList)
{
item.Dispose();
}
return(ret);
}
}
void _Update(IMatrix error, INeuralNetworkLayerUpdater w, INeuralNetworkLayerUpdater u, INeuralNetworkUpdateAccumulator updateAccumulator)
{
var deltaW = _input.TransposeThisAndMultiply(error);
var deltaU = _pc.TransposeThisAndMultiply(error);
updateAccumulator.Record(w, error, deltaW);
updateAccumulator.Record(u, error.Clone(), deltaU);
}
public IMatrix Execute(IMatrix error, ITrainingContext context, bool calculateOutput, INeuralNetworkUpdateAccumulator updateAccumulator)
{
const float NEG_MAX = -1f, POS_MAX = 1f;
using (var cd = _activation.Derivative(_c, null))
using (var ad = _activation.Derivative(_a, null)) {
using (var i2 = _ones.Subtract(_i))
using (var f2 = _ones.Subtract(_f))
using (var o2 = _ones.Subtract(_o))
using (var errorO = error.PointwiseMultiply(_o))
using (var dO = error.PointwiseMultiply(_ca))
using (var dC = errorO.PointwiseMultiply(cd)) {
// clip the gradients
dO.Constrain(NEG_MAX, POS_MAX);
dC.Constrain(NEG_MAX, POS_MAX);
using (var dI = dC.PointwiseMultiply(_a))
using (var dF = dC.PointwiseMultiply(_pc))
using (var dA = dC.PointwiseMultiply(_i))
using (var dCp = dC.PointwiseMultiply(_f))
using (var dIi = dI.PointwiseMultiply(_i))
using (var dFf = dF.PointwiseMultiply(_f))
using (var dOo = dO.PointwiseMultiply(_o)) {
var dA2 = dA.PointwiseMultiply(ad);
var dI2 = dIi.PointwiseMultiply(i2);
var dF2 = dFf.PointwiseMultiply(f2);
var dO2 = dOo.PointwiseMultiply(o2);
using (var ui = dI2.TransposeAndMultiply(_ui.Layer.Weight))
using (var uf = dF2.TransposeAndMultiply(_uf.Layer.Weight))
using (var uo = dO2.TransposeAndMultiply(_uo.Layer.Weight)) {
var uc = dA2.TransposeAndMultiply(_uc.Layer.Weight);
uc.AddInPlace(ui, 1f, 1f);
uc.AddInPlace(uf, 1f, 1f);
uc.AddInPlace(uo, 1f, 1f);
_Update(dA2, _wc, _uc, updateAccumulator);
_Update(dI2, _wi, _ui, updateAccumulator);
_Update(dF2, _wf, _uf, updateAccumulator);
_Update(dO2, _wo, _uo, updateAccumulator);
_Cleanup();
return(uc);
}
}
}
}
}
public IMatrix Backpropagate(IMatrix errorSignal, ITrainingContext context, bool calculateOutput, INeuralNetworkUpdateAccumulator updates = null)
{
Debug.Assert(_layerInput != null && _layerOutput != null);
IMatrix ret = Backpropagate(_layerInput, _layerOutput, errorSignal, context, calculateOutput, updates);
_Reset();
return(ret);
}
public IMatrix Backpropagate(IMatrix input, IMatrix output, IMatrix errorSignal, ITrainingContext context, bool calculateOutput, INeuralNetworkUpdateAccumulator updates = null)
{
IMatrix ret = null;
// calculate the derivative to determine the gradients
using (var od = _layerUpdater.Layer.Activation?.Derivative(output, errorSignal) ?? output) {
if (_verifyDerivatives)
{
var dError = _VerifyDerivatives(od);
Debug.Assert(dError < 0.001f);
}
// use the gradients to determine the direction of each change
var delta = errorSignal.PointwiseMultiply(od);
if (calculateOutput)
{
ret = _layerUpdater.Layer.CalculateErrorSignal(delta);
}
// update the layer
_UpdateLayer(input, delta, context, updates);
}
return(ret);
}
protected override void _UpdateLayer(IMatrix input, IMatrix delta, ITrainingContext context, INeuralNetworkUpdateAccumulator updates)
{
if (_filter != null && _filter2 != null)
{
// filter the updates to the bias against the filter
using (var columnSums = delta.ColumnSums())
using (var filteredColumnSums = columnSums.PointwiseMultiply(_filter2))
_layerUpdater.Layer.Bias.AddInPlace(filteredColumnSums, 1f / columnSums.Count);
// filter the weight updates against the filter
using (var weightUpdate = input.TransposeThisAndMultiply(delta))
using (var filteredWeightUpdate = weightUpdate.PointwiseMultiply(_filter))
_layerUpdater.Update(null, filteredWeightUpdate, context);
}
else
{
base._UpdateLayer(input, delta, context, updates);
}
}
protected virtual void _UpdateLayer(IMatrix input, IMatrix delta, ITrainingContext context, INeuralNetworkUpdateAccumulator updates)
{
var weightUpdate = input.TransposeThisAndMultiply(delta);
if (updates != null)
{
updates.Record(_layerUpdater, delta, weightUpdate);
}
else
{
_layerUpdater.Update(delta, weightUpdate, context);
weightUpdate.Dispose();
delta.Dispose();
}
}
public IMatrix Execute(IMatrix error, ITrainingContext context, bool calculateOutput, INeuralNetworkUpdateAccumulator updateAccumulator)
{
//using (var input = _input)
using (var output = _output)
return(_trainer.Backpropagate(_input, output, error, context, calculateOutput, updateAccumulator));
}
public IMatrix Execute(IMatrix curr, ITrainingContext context, bool calculateOutput, INeuralNetworkUpdateAccumulator updateAccumulator)
{
using (var ad = _activation.Derivative(_output, curr)) {
// clip the gradient
ad.Constrain(-1f, 1f);
var delta = curr.PointwiseMultiply(ad);
var prevDelta = delta.TransposeAndMultiply(_memoryUpdater.Layer.Weight);
var memoryUpdate = _memory.TransposeThisAndMultiply(delta);
var inputUpdate = _input.TransposeThisAndMultiply(delta);
//_input.Dispose();
_memory.Dispose();
_output.Dispose();
updateAccumulator.Record(_memoryUpdater, delta, memoryUpdate);
updateAccumulator.Record(_inputUpdater, delta.Clone(), inputUpdate);
return(prevDelta);
}
}
public IMatrix Backpropagate(IMatrix errorSignal, ITrainingContext context, bool calculateOutput, INeuralNetworkUpdateAccumulator updates)
{
if (_isTraining)
{
for (var i = 0; i < errorSignal.RowCount; i++)
{
using (var row = errorSignal.Row(i)) {
var backpropagationStack = _backpropagation[i];
var isFirst = true;
IMatrix lastError = null;
while (backpropagationStack.Any())
{
var backpropagation = backpropagationStack.Pop();
if (isFirst)
{
using (var rowError = row.ConvertInPlaceToMatrix(backpropagation.RowCount, backpropagation.ColumnCount))
lastError = backpropagation.Execute(rowError, context, backpropagationStack.Any(), updates);
isFirst = false;
}
else
{
var nextError = backpropagation.Execute(lastError, context, backpropagationStack.Any(), updates);
lastError.Dispose();
lastError = nextError;
}
}
}
}
}
return(null);
}