public override void _Backward(INode fromNode, IGraphData errorSignal, IContext context,
IReadOnlyList <INode> parents)
{
var matrix = errorSignal.GetMatrix();
(IMatrix left, IMatrix right) = matrix.SplitAtColumn(matrix.ColumnCount - _reverseSize);
context.AddBackward(errorSignal.ReplaceWith(left), parents[0], _source);
var batch = context.BatchSequence.MiniBatch;
var sequenceIndex = context.BatchSequence.SequenceIndex;
var reversedSequenceIndex = batch.SequenceCount - sequenceIndex - 1;
_source._reverseBackpropagation.Add(reversedSequenceIndex,
(parents[1], errorSignal.ReplaceWith(right)));
_source._contextTable.Add(sequenceIndex, context);
if (sequenceIndex == 0)
{
// process in order as we are pushing onto a stack (so will be read in reverse order)
for (var i = 0; i < batch.SequenceCount; i++)
{
var data = _source._reverseBackpropagation[i];
var reverseContext = _source._contextTable[i];
reverseContext.AddBackward(data.Item2, data.Item1, _source);
}
_source._reverseBackpropagation.Clear();
_source._contextTable.Clear();
}
}
public override void _Backward(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents)
{
var es = errorSignal.GetMatrix();
var delta1 = es.PointwiseMultiply(_input2);
var delta2 = es.PointwiseMultiply(_input1);
context.AddBackward(errorSignal.ReplaceWith(delta1), parents.First(), _source);
context.AddBackward(errorSignal.ReplaceWith(delta2), parents.Last(), _source);
}
public override void _Backward(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents)
{
IMatrix split, residual = errorSignal.GetMatrix();
int index = parents.Count - 1;
foreach (var item in _channels)
{
(residual, split) = residual.SplitAtColumn(residual.ColumnCount - item.Size);
context.AddBackward(errorSignal.ReplaceWith(split), parents[index--], _source);
}
context.AddBackward(errorSignal.ReplaceWith(residual), parents[index], _source);
}
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents)
{
using (var od = _input.ReluDerivative()) {
var delta = errorSignal.GetMatrix().PointwiseMultiply(od);
return(errorSignal.ReplaceWith(delta));
}
}
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal,
IContext context, IReadOnlyList <INode> parents)
{
var matrix = errorSignal.GetMatrix();
return(errorSignal.ReplaceWith(matrix.Transpose()));
}
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents)
{
var es = errorSignal.GetMatrix();
using (var ones = context.LinearAlgebraProvider.CreateMatrix(es.RowCount, es.ColumnCount, 1f / _rowCount))
return(errorSignal.ReplaceWith(ones.PointwiseMultiply(es)));
}
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal,
IContext context, IReadOnlyList <INode> parents)
{
var output = errorSignal.GetMatrix().PointwiseMultiply(_filter);
return(errorSignal.ReplaceWith(output));
}
public IGraphData Execute(IGraphData input, IContext context)
{
var output = input.GetMatrix();
if (context.IsTraining)
{
if (context.LearningContext.ErrorMetric == null)
{
context.LearningContext.ErrorMetric = _errorMetric;
}
var target = context.BatchSequence.Target?.GetMatrix();
if (target == null)
{
context.LearningContext.DeferBackpropagation(null,
signal => context.Backpropagate(signal));
}
else
{
var gradient = _errorMetric.CalculateGradient(context, output, target);
context.LearningContext.DeferBackpropagation(input.ReplaceWith(gradient),
signal => context.Backpropagate(signal));
}
}
return(input);
}
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents)
{
var es = errorSignal.GetMatrix();
es.Multiply(2f);
return(errorSignal.ReplaceWith(es));
}
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents)
{
var es = errorSignal.GetMatrix();
using (var oneHalf = context.LinearAlgebraProvider.CreateMatrix(es.RowCount, es.ColumnCount, 0.5f))
using (var sqrt = es.Sqrt(1e-8f))
using (var delta = oneHalf.PointwiseMultiply(sqrt))
return(errorSignal.ReplaceWith(delta));
}
public override void _Backward(INode fromNode, IGraphData errorSignal, IContext context,
IReadOnlyList <INode> parents)
{
var es = errorSignal.GetMatrix();
var negative = es.Clone();
negative.Multiply(-1f);
context.AddBackward(errorSignal, parents.First(), _source);
context.AddBackward(errorSignal.ReplaceWith(negative), parents.Last(), _source);
}
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents)
{
var es = errorSignal.GetMatrix();
using (var minusOne = context.LinearAlgebraProvider.CreateMatrix(es.RowCount, es.ColumnCount, -1f))
using (var inputSquared = _input.PointwiseMultiply(_input))
using (var delta = minusOne.PointwiseDivide(inputSquared)) {
return(errorSignal.ReplaceWith(delta.PointwiseMultiply(es)));
}
}
public IGraphData Execute(IGraphData input, IContext context)
{
var output = input.GetMatrix();
if (context.IsTraining)
{
var gradient = _errorMetric.CalculateGradient(context, output, context.BatchSequence.Target.GetMatrix());
context.Backpropagate(input.ReplaceWith(gradient));
}
return(input);
}
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents)
{
var error = errorSignal.GetMatrix();
var batchSize = (float)context.BatchSequence.MiniBatch.BatchSize;
var lap = context.LinearAlgebraProvider;
using (var dxHat = error.PointwiseMultiply(_gamma))
using (var temp = dxHat.PointwiseMultiply(_inputMinusMean))
using (var temp2 = _inverseVariance.PointwiseMultiply(_inverseVariance))
using (var inverseVarianceCubed = temp2.PointwiseMultiply(_inverseVariance))
using (var temp3 = temp.PointwiseMultiply(inverseVarianceCubed)) {
temp3.Multiply(-0.5f);
using (var dVar = temp3.ColumnSums())
using (var temp4 = dxHat.PointwiseMultiply(_inverseVariance)) {
temp4.Multiply(-1f);
using (var dmu = temp4.ColumnSums())
using (var temp5 = _inputMinusMean.ColumnSums()) {
temp5.Multiply(-2f / batchSize);
using (var dmu2 = temp5.PointwiseMultiply(dVar)) {
dmu.AddInPlace(dmu2);
using (var dVarMatrix = lap.CreateMatrix(Enumerable.Repeat(dVar, context.BatchSequence.MiniBatch.BatchSize).ToList()))
using (var dx1 = dxHat.PointwiseMultiply(_inverseVariance))
using (var dx2 = dVarMatrix.PointwiseMultiply(_inputMinusMean)) {
dx2.Multiply(-2f / batchSize);
using (var dx3 = lap.CreateMatrix(Enumerable.Repeat(dmu, context.BatchSequence.MiniBatch.BatchSize).ToList())) {
dx3.Multiply(1f / batchSize);
var dx = dx1.Add(dx2);
dx.AddInPlace(dx3);
var dBeta = dx.ColumnSums();
using (var temp6 = _xHat.PointwiseMultiply(error)) {
var dGamma = temp6.ColumnSums();
// store the updates
var learningContext = context.LearningContext;
learningContext.StoreUpdate(_source, dBeta, err => _source._beta.AddInPlace(err, 1f, learningContext.BatchLearningRate));
learningContext.StoreUpdate(_source, dGamma, err => _source._gamma.AddInPlace(err, 1f, learningContext.BatchLearningRate));
}
return(errorSignal.ReplaceWith(dx));
}
}
}
}
}
}
}
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents)
{
var matrix = errorSignal.GetMatrix().AsIndexable();
var delta = context.LinearAlgebraProvider.CreateMatrix(matrix.RowCount, matrix.ColumnCount, (i, j) => {
var x = matrix[i, j];
if (x >= 0)
{
return(SCALE);
}
return(SCALE * ALPHA * BoundMath.Exp(x));
});
return(errorSignal.ReplaceWith(delta));
}
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents)
{
var es = errorSignal.GetMatrix();
// work out the next error signal against the filtered weights
IMatrix ret = es.TransposeAndMultiply(_filteredWeights);
// calculate the update to the weights and filter out the dropped connections
var weightUpdate = _input.TransposeThisAndMultiply(es).PointwiseMultiply(_filter);
// store the updates
var learningContext = context.LearningContext;
learningContext.StoreUpdate(_source, es, err => _source.UpdateBias(err, learningContext));
learningContext.StoreUpdate(_source, weightUpdate, err => _source.UpdateWeights(err, learningContext));
return(errorSignal.ReplaceWith(ret));
}
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents)
{
var matrix = errorSignal.GetMatrix();
var lap = context.LinearAlgebraProvider;
var rowList = new List <IVector>();
for (var i = 0; i < matrix.RowCount; i++)
{
var rowMatrix = matrix.Row(i).ReshapeAsMatrix(_tensor.RowCount, _tensor.ColumnCount);
var matrixList = Enumerable.Repeat(rowMatrix, _tensor.Depth).ToList();
var tensor = lap.Create3DTensor(matrixList);
rowList.Add(tensor.ReshapeAsVector());
}
var errorMatrix = lap.CreateMatrixFromRows(rowList);
return(errorSignal.ReplaceWith(errorMatrix.Transpose()));
}
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents)
{
var es = errorSignal.GetMatrix();
// work out the next error signal
IMatrix ret = es.Multiply(_source._layer.Weight);
// calculate the update to the weights
var weightUpdate = _input.TransposeThisAndMultiply(es).Transpose();
// store the updates
var learningContext = context.LearningContext;
learningContext.StoreUpdate(_source, es, err => _source.UpdateBias(err, learningContext));
learningContext.StoreUpdate(_source, weightUpdate, err => _source._layer.UpdateWeights(err, learningContext));
return(errorSignal.ReplaceWith(ret));
}
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents)
{
var matrix = errorSignal.GetMatrix();
var rowList = new List <IVector>();
for (var i = 0; i < matrix.RowCount; i++)
{
using (var derivative = _rows[i].SoftmaxDerivative()) {
var sm = derivative.Multiply(matrix.Row(i));
rowList.Add(sm.ConvertInPlaceToVector());
}
}
var ret = context.LinearAlgebraProvider.CreateMatrix(rowList);
foreach (var item in rowList)
{
item.Dispose();
}
return(errorSignal.ReplaceWith(ret));
}
public IGraphData Execute(IGraphData input, IContext context)
{
var memory = context.ExecutionContext.GetMemory(_slotName);
return(input.ReplaceWith(input.GetMatrix().ConcatRows(memory)));
}
