public override void _Backward(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var matrix = errorSignal.GetMatrix(); (IMatrix left, IMatrix right) = matrix.SplitAtColumn(matrix.ColumnCount - _reverseSize); context.AddBackward(errorSignal.ReplaceWith(left), parents[0], _source); var batch = context.BatchSequence.MiniBatch; var sequenceIndex = context.BatchSequence.SequenceIndex; var reversedSequenceIndex = batch.SequenceCount - sequenceIndex - 1; _source._reverseBackpropagation.Add(reversedSequenceIndex, (parents[1], errorSignal.ReplaceWith(right))); _source._contextTable.Add(sequenceIndex, context); if (sequenceIndex == 0) { // process in order as we are pushing onto a stack (so will be read in reverse order) for (var i = 0; i < batch.SequenceCount; i++) { var data = _source._reverseBackpropagation[i]; var reverseContext = _source._contextTable[i]; reverseContext.AddBackward(data.Item2, data.Item1, _source); } _source._reverseBackpropagation.Clear(); _source._contextTable.Clear(); } }
public override void _Backward(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var es = errorSignal.GetMatrix(); var delta1 = es.PointwiseMultiply(_input2); var delta2 = es.PointwiseMultiply(_input1); context.AddBackward(errorSignal.ReplaceWith(delta1), parents.First(), _source); context.AddBackward(errorSignal.ReplaceWith(delta2), parents.Last(), _source); }
public override void _Backward(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { IMatrix split, residual = errorSignal.GetMatrix(); int index = parents.Count - 1; foreach (var item in _channels) { (residual, split) = residual.SplitAtColumn(residual.ColumnCount - item.Size); context.AddBackward(errorSignal.ReplaceWith(split), parents[index--], _source); } context.AddBackward(errorSignal.ReplaceWith(residual), parents[index], _source); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { using (var od = _input.ReluDerivative()) { var delta = errorSignal.GetMatrix().PointwiseMultiply(od); return(errorSignal.ReplaceWith(delta)); } }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var matrix = errorSignal.GetMatrix(); return(errorSignal.ReplaceWith(matrix.Transpose())); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var es = errorSignal.GetMatrix(); using (var ones = context.LinearAlgebraProvider.CreateMatrix(es.RowCount, es.ColumnCount, 1f / _rowCount)) return(errorSignal.ReplaceWith(ones.PointwiseMultiply(es))); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var output = errorSignal.GetMatrix().PointwiseMultiply(_filter); return(errorSignal.ReplaceWith(output)); }
public IGraphData Execute(IGraphData input, IContext context) { var output = input.GetMatrix(); if (context.IsTraining) { if (context.LearningContext.ErrorMetric == null) { context.LearningContext.ErrorMetric = _errorMetric; } var target = context.BatchSequence.Target?.GetMatrix(); if (target == null) { context.LearningContext.DeferBackpropagation(null, signal => context.Backpropagate(signal)); } else { var gradient = _errorMetric.CalculateGradient(context, output, target); context.LearningContext.DeferBackpropagation(input.ReplaceWith(gradient), signal => context.Backpropagate(signal)); } } return(input); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var es = errorSignal.GetMatrix(); es.Multiply(2f); return(errorSignal.ReplaceWith(es)); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var es = errorSignal.GetMatrix(); using (var oneHalf = context.LinearAlgebraProvider.CreateMatrix(es.RowCount, es.ColumnCount, 0.5f)) using (var sqrt = es.Sqrt(1e-8f)) using (var delta = oneHalf.PointwiseMultiply(sqrt)) return(errorSignal.ReplaceWith(delta)); }
public override void _Backward(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var es = errorSignal.GetMatrix(); var negative = es.Clone(); negative.Multiply(-1f); context.AddBackward(errorSignal, parents.First(), _source); context.AddBackward(errorSignal.ReplaceWith(negative), parents.Last(), _source); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var es = errorSignal.GetMatrix(); using (var minusOne = context.LinearAlgebraProvider.CreateMatrix(es.RowCount, es.ColumnCount, -1f)) using (var inputSquared = _input.PointwiseMultiply(_input)) using (var delta = minusOne.PointwiseDivide(inputSquared)) { return(errorSignal.ReplaceWith(delta.PointwiseMultiply(es))); } }
public IGraphData Execute(IGraphData input, IContext context) { var output = input.GetMatrix(); if (context.IsTraining) { var gradient = _errorMetric.CalculateGradient(context, output, context.BatchSequence.Target.GetMatrix()); context.Backpropagate(input.ReplaceWith(gradient)); } return(input); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var error = errorSignal.GetMatrix(); var batchSize = (float)context.BatchSequence.MiniBatch.BatchSize; var lap = context.LinearAlgebraProvider; using (var dxHat = error.PointwiseMultiply(_gamma)) using (var temp = dxHat.PointwiseMultiply(_inputMinusMean)) using (var temp2 = _inverseVariance.PointwiseMultiply(_inverseVariance)) using (var inverseVarianceCubed = temp2.PointwiseMultiply(_inverseVariance)) using (var temp3 = temp.PointwiseMultiply(inverseVarianceCubed)) { temp3.Multiply(-0.5f); using (var dVar = temp3.ColumnSums()) using (var temp4 = dxHat.PointwiseMultiply(_inverseVariance)) { temp4.Multiply(-1f); using (var dmu = temp4.ColumnSums()) using (var temp5 = _inputMinusMean.ColumnSums()) { temp5.Multiply(-2f / batchSize); using (var dmu2 = temp5.PointwiseMultiply(dVar)) { dmu.AddInPlace(dmu2); using (var dVarMatrix = lap.CreateMatrix(Enumerable.Repeat(dVar, context.BatchSequence.MiniBatch.BatchSize).ToList())) using (var dx1 = dxHat.PointwiseMultiply(_inverseVariance)) using (var dx2 = dVarMatrix.PointwiseMultiply(_inputMinusMean)) { dx2.Multiply(-2f / batchSize); using (var dx3 = lap.CreateMatrix(Enumerable.Repeat(dmu, context.BatchSequence.MiniBatch.BatchSize).ToList())) { dx3.Multiply(1f / batchSize); var dx = dx1.Add(dx2); dx.AddInPlace(dx3); var dBeta = dx.ColumnSums(); using (var temp6 = _xHat.PointwiseMultiply(error)) { var dGamma = temp6.ColumnSums(); // store the updates var learningContext = context.LearningContext; learningContext.StoreUpdate(_source, dBeta, err => _source._beta.AddInPlace(err, 1f, learningContext.BatchLearningRate)); learningContext.StoreUpdate(_source, dGamma, err => _source._gamma.AddInPlace(err, 1f, learningContext.BatchLearningRate)); } return(errorSignal.ReplaceWith(dx)); } } } } } } }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var matrix = errorSignal.GetMatrix().AsIndexable(); var delta = context.LinearAlgebraProvider.CreateMatrix(matrix.RowCount, matrix.ColumnCount, (i, j) => { var x = matrix[i, j]; if (x >= 0) { return(SCALE); } return(SCALE * ALPHA * BoundMath.Exp(x)); }); return(errorSignal.ReplaceWith(delta)); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var es = errorSignal.GetMatrix(); // work out the next error signal against the filtered weights IMatrix ret = es.TransposeAndMultiply(_filteredWeights); // calculate the update to the weights and filter out the dropped connections var weightUpdate = _input.TransposeThisAndMultiply(es).PointwiseMultiply(_filter); // store the updates var learningContext = context.LearningContext; learningContext.StoreUpdate(_source, es, err => _source.UpdateBias(err, learningContext)); learningContext.StoreUpdate(_source, weightUpdate, err => _source.UpdateWeights(err, learningContext)); return(errorSignal.ReplaceWith(ret)); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var matrix = errorSignal.GetMatrix(); var lap = context.LinearAlgebraProvider; var rowList = new List <IVector>(); for (var i = 0; i < matrix.RowCount; i++) { var rowMatrix = matrix.Row(i).ReshapeAsMatrix(_tensor.RowCount, _tensor.ColumnCount); var matrixList = Enumerable.Repeat(rowMatrix, _tensor.Depth).ToList(); var tensor = lap.Create3DTensor(matrixList); rowList.Add(tensor.ReshapeAsVector()); } var errorMatrix = lap.CreateMatrixFromRows(rowList); return(errorSignal.ReplaceWith(errorMatrix.Transpose())); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var es = errorSignal.GetMatrix(); // work out the next error signal IMatrix ret = es.Multiply(_source._layer.Weight); // calculate the update to the weights var weightUpdate = _input.TransposeThisAndMultiply(es).Transpose(); // store the updates var learningContext = context.LearningContext; learningContext.StoreUpdate(_source, es, err => _source.UpdateBias(err, learningContext)); learningContext.StoreUpdate(_source, weightUpdate, err => _source._layer.UpdateWeights(err, learningContext)); return(errorSignal.ReplaceWith(ret)); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var matrix = errorSignal.GetMatrix(); var rowList = new List <IVector>(); for (var i = 0; i < matrix.RowCount; i++) { using (var derivative = _rows[i].SoftmaxDerivative()) { var sm = derivative.Multiply(matrix.Row(i)); rowList.Add(sm.ConvertInPlaceToVector()); } } var ret = context.LinearAlgebraProvider.CreateMatrix(rowList); foreach (var item in rowList) { item.Dispose(); } return(errorSignal.ReplaceWith(ret)); }
public IGraphData Execute(IGraphData input, IContext context) { var memory = context.ExecutionContext.GetMemory(_slotName); return(input.ReplaceWith(input.GetMatrix().ConcatRows(memory))); }