protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var matrix = errorSignal.GetMatrix(); return(errorSignal.ReplaceWith(matrix.Transpose())); }
public override void _Backward(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var matrix = errorSignal.GetMatrix(); (IMatrix left, IMatrix right) = matrix.SplitAtColumn(matrix.ColumnCount - _reverseSize); context.AddBackward(errorSignal.ReplaceWith(left), parents[0], _source); var batch = context.BatchSequence.MiniBatch; var sequenceIndex = context.BatchSequence.SequenceIndex; var reversedSequenceIndex = batch.SequenceCount - sequenceIndex - 1; _source._reverseBackpropagation.Add(reversedSequenceIndex, (parents[1], errorSignal.ReplaceWith(right))); _source._contextTable.Add(sequenceIndex, context); if (sequenceIndex == 0) { // process in order as we are pushing onto a stack (so will be read in reverse order) for (var i = 0; i < batch.SequenceCount; i++) { var data = _source._reverseBackpropagation[i]; var reverseContext = _source._contextTable[i]; reverseContext.AddBackward(data.Item2, data.Item1, _source); } _source._reverseBackpropagation.Clear(); _source._contextTable.Clear(); } }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var es = errorSignal.GetMatrix(); es.Multiply(2f); return(errorSignal.ReplaceWith(es)); }
public IGraphData Execute(IGraphData input, IContext context) { var matrix = input.GetMatrix(); matrix.Constrain(_min, _max); return(input); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var output = errorSignal.GetMatrix().PointwiseMultiply(_filter); return(errorSignal.ReplaceWith(output)); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var es = errorSignal.GetMatrix(); using (var ones = context.LinearAlgebraProvider.CreateMatrix(es.RowCount, es.ColumnCount, 1f / _rowCount)) return(errorSignal.ReplaceWith(ones.PointwiseMultiply(es))); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { using (var od = _input.ReluDerivative()) { var delta = errorSignal.GetMatrix().PointwiseMultiply(od); return(errorSignal.ReplaceWith(delta)); } }
public IGraphData Execute(IGraphData input, IContext context) { var output = input.GetMatrix(); if (context.IsTraining) { if (context.LearningContext.ErrorMetric == null) { context.LearningContext.ErrorMetric = _errorMetric; } var target = context.BatchSequence.Target?.GetMatrix(); if (target == null) { context.LearningContext.DeferBackpropagation(null, signal => context.Backpropagate(signal)); } else { var gradient = _errorMetric.CalculateGradient(context, output, target); context.LearningContext.DeferBackpropagation(input.ReplaceWith(gradient), signal => context.Backpropagate(signal)); } } return(input); }
void _TestAction(IAction action, IGraphData input, IGraphData expectedOutput) { var context = new TestingContext(_cpu); var output = action.Execute(input, context); FloatingPointHelper.AssertEqual(output.GetMatrix().AsIndexable(), expectedOutput.GetMatrix().AsIndexable()); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var es = errorSignal.GetMatrix(); using (var oneHalf = context.LinearAlgebraProvider.CreateMatrix(es.RowCount, es.ColumnCount, 0.5f)) using (var sqrt = es.Sqrt(1e-8f)) using (var delta = oneHalf.PointwiseMultiply(sqrt)) return(errorSignal.ReplaceWith(delta)); }
public override void _Backward(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var es = errorSignal.GetMatrix(); var delta1 = es.PointwiseMultiply(_input2); var delta2 = es.PointwiseMultiply(_input1); context.AddBackward(errorSignal.ReplaceWith(delta1), parents.First(), _source); context.AddBackward(errorSignal.ReplaceWith(delta2), parents.Last(), _source); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var es = errorSignal.GetMatrix(); using (var minusOne = context.LinearAlgebraProvider.CreateMatrix(es.RowCount, es.ColumnCount, -1f)) using (var inputSquared = _input.PointwiseMultiply(_input)) using (var delta = minusOne.PointwiseDivide(inputSquared)) { return(errorSignal.ReplaceWith(delta.PointwiseMultiply(es))); } }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var error = errorSignal.GetMatrix(); using (var columnSums = error.ColumnSums()) { columnSums.Multiply(1f / error.RowCount); _source._param.AddInPlace(columnSums, 1f, context.LearningContext.BatchLearningRate); } return(errorSignal); }
public override void _Backward(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var es = errorSignal.GetMatrix(); var negative = es.Clone(); negative.Multiply(-1f); context.AddBackward(errorSignal, parents.First(), _source); context.AddBackward(errorSignal.ReplaceWith(negative), parents.Last(), _source); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var error = errorSignal.GetMatrix(); var delta = error.ColumnSums(); delta.Multiply(1f / error.RowCount); var delta2 = delta.Average(); _source._param += delta2 * context.LearningContext.BatchLearningRate; return(errorSignal); }
public IGraphData Execute(IGraphData input, IContext context) { var output = input.GetMatrix(); if (context.IsTraining) { var gradient = _errorMetric.CalculateGradient(context, output, context.BatchSequence.Target.GetMatrix()); context.Backpropagate(input.ReplaceWith(gradient)); } return(input); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var error = errorSignal.GetMatrix(); var batchSize = (float)context.BatchSequence.MiniBatch.BatchSize; var lap = context.LinearAlgebraProvider; using (var dxHat = error.PointwiseMultiply(_gamma)) using (var temp = dxHat.PointwiseMultiply(_inputMinusMean)) using (var temp2 = _inverseVariance.PointwiseMultiply(_inverseVariance)) using (var inverseVarianceCubed = temp2.PointwiseMultiply(_inverseVariance)) using (var temp3 = temp.PointwiseMultiply(inverseVarianceCubed)) { temp3.Multiply(-0.5f); using (var dVar = temp3.ColumnSums()) using (var temp4 = dxHat.PointwiseMultiply(_inverseVariance)) { temp4.Multiply(-1f); using (var dmu = temp4.ColumnSums()) using (var temp5 = _inputMinusMean.ColumnSums()) { temp5.Multiply(-2f / batchSize); using (var dmu2 = temp5.PointwiseMultiply(dVar)) { dmu.AddInPlace(dmu2); using (var dVarMatrix = lap.CreateMatrix(Enumerable.Repeat(dVar, context.BatchSequence.MiniBatch.BatchSize).ToList())) using (var dx1 = dxHat.PointwiseMultiply(_inverseVariance)) using (var dx2 = dVarMatrix.PointwiseMultiply(_inputMinusMean)) { dx2.Multiply(-2f / batchSize); using (var dx3 = lap.CreateMatrix(Enumerable.Repeat(dmu, context.BatchSequence.MiniBatch.BatchSize).ToList())) { dx3.Multiply(1f / batchSize); var dx = dx1.Add(dx2); dx.AddInPlace(dx3); var dBeta = dx.ColumnSums(); using (var temp6 = _xHat.PointwiseMultiply(error)) { var dGamma = temp6.ColumnSums(); // store the updates var learningContext = context.LearningContext; learningContext.StoreUpdate(_source, dBeta, err => _source._beta.AddInPlace(err, 1f, learningContext.BatchLearningRate)); learningContext.StoreUpdate(_source, dGamma, err => _source._gamma.AddInPlace(err, 1f, learningContext.BatchLearningRate)); } return(errorSignal.ReplaceWith(dx)); } } } } } } }
public override void _Backward(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { IMatrix split, residual = errorSignal.GetMatrix(); int index = parents.Count - 1; foreach (var item in _channels) { (residual, split) = residual.SplitAtColumn(residual.ColumnCount - item.Size); context.AddBackward(errorSignal.ReplaceWith(split), parents[index--], _source); } context.AddBackward(errorSignal.ReplaceWith(residual), parents[index], _source); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var errorMatrix = errorSignal.GetMatrix(); var tensor = errorMatrix.ReshapeAs4DTensor(_outputRows, _outputColumns, _depth); var output = tensor.ReverseMaxPool(_indices, _inputRows, _inputColumns, _filterWidth, _filterHeight, _xStride, _yStride); //output.AsMatrix().Constrain(-1f, 1f); //#if DEBUG // Debug.Assert(output.ReshapeAsVector().IsEntirelyFinite()); //#endif return(new Tensor4DGraphData(output.ReshapeAsMatrix(), output.RowCount, output.ColumnCount, output.Depth)); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var matrix = errorSignal.GetMatrix().AsIndexable(); var delta = context.LinearAlgebraProvider.CreateMatrix(matrix.RowCount, matrix.ColumnCount, (i, j) => { var x = matrix[i, j]; if (x >= 0) { return(SCALE); } return(SCALE * ALPHA * BoundMath.Exp(x)); }); return(errorSignal.ReplaceWith(delta)); }
public override void _Backward(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { if (context.BatchSequence.Type == MiniBatchSequenceType.SequenceStart) { var es = errorSignal.GetMatrix(); using (var columnSums = es.ColumnSums()) { var initialDelta = columnSums.AsIndexable(); for (var j = 0; j < _source._data.Length; j++) { _source._data[j] += initialDelta[j] * context.LearningContext.BatchLearningRate; } } } }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var tensor = errorSignal.GetMatrix(). ReshapeAs4DTensor(_newHeight, _newWidth, _source._filter.ColumnCount); var padding = _source._padding; // calculate the weight and bias updates using (var update = _im2Col.TransposeThisAndMultiply(tensor)) { var weightUpdate = update.CombineDepthSlices(); var biasUpdate = tensor.ColumnSums(); context.LearningContext.StoreUpdate(_source, weightUpdate, err => _source.Update(err, context.LearningContext)); context.LearningContext.StoreUpdate(_source, biasUpdate, bu => _UpdateBias(bu, context.LearningContext)); } if (_source._shouldBackpropagate) { var filters = _source._filter; var inputDepth = _source._inputDepth; var filterWidth = _source._filterWidth; var filterHeight = _source._filterHeight; var xStride = _source._xStride; var yStride = _source._yStride; var outputRows = _inputHeight + padding * 2; var outputColumns = _inputWidth + padding * 2; var outputDepth = _inputDepth; var reverseIm2Col = tensor.ReverseIm2Col(filters, outputRows, outputColumns, outputDepth, filterWidth, filterHeight, xStride, yStride); var delta = reverseIm2Col; if (padding > 0) { var delta2 = delta.RemovePadding(padding); delta.Dispose(); delta = delta2; } return(new Tensor4DGraphData(delta.ReshapeAsMatrix(), _inputHeight, _inputWidth, inputDepth)); } return(errorSignal); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var matrix = errorSignal.GetMatrix(); var lap = context.LinearAlgebraProvider; var rowList = new List <IVector>(); for (var i = 0; i < matrix.RowCount; i++) { var rowMatrix = matrix.Row(i).ReshapeAsMatrix(_tensor.RowCount, _tensor.ColumnCount); var matrixList = Enumerable.Repeat(rowMatrix, _tensor.Depth).ToList(); var tensor = lap.Create3DTensor(matrixList); rowList.Add(tensor.ReshapeAsVector()); } var errorMatrix = lap.CreateMatrixFromRows(rowList); return(errorSignal.ReplaceWith(errorMatrix.Transpose())); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var es = errorSignal.GetMatrix(); // work out the next error signal IMatrix ret = es.Multiply(_source._layer.Weight); // calculate the update to the weights var weightUpdate = _input.TransposeThisAndMultiply(es).Transpose(); // store the updates var learningContext = context.LearningContext; learningContext.StoreUpdate(_source, es, err => _source.UpdateBias(err, learningContext)); learningContext.StoreUpdate(_source, weightUpdate, err => _source._layer.UpdateWeights(err, learningContext)); return(errorSignal.ReplaceWith(ret)); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var es = errorSignal.GetMatrix(); // work out the next error signal against the filtered weights IMatrix ret = es.TransposeAndMultiply(_filteredWeights); // calculate the update to the weights and filter out the dropped connections var weightUpdate = _input.TransposeThisAndMultiply(es).PointwiseMultiply(_filter); // store the updates var learningContext = context.LearningContext; learningContext.StoreUpdate(_source, es, err => _source.UpdateBias(err, learningContext)); learningContext.StoreUpdate(_source, weightUpdate, err => _source.UpdateWeights(err, learningContext)); return(errorSignal.ReplaceWith(ret)); }
public override void _Backward(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var es = errorSignal.GetMatrix(); using (var columnSums = es.ColumnSums()) { columnSums.Multiply(1f / es.RowCount); // store the updates var learningContext = context.LearningContext; learningContext.StoreUpdate(_source, columnSums, err => { var delta = err.AsIndexable(); for (var j = 0; j < _source._data.Length; j++) { _source._data[j] += delta[j] * context.LearningContext.BatchLearningRate; } }); } }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var matrix = errorSignal.GetMatrix(); var rowList = new List <IVector>(); for (var i = 0; i < matrix.RowCount; i++) { using (var derivative = _rows[i].SoftmaxDerivative()) { var sm = derivative.Multiply(matrix.Row(i)); rowList.Add(sm.ConvertInPlaceToVector()); } } var ret = context.LinearAlgebraProvider.CreateMatrix(rowList); foreach (var item in rowList) { item.Dispose(); } return(errorSignal.ReplaceWith(ret)); }
protected override IGraphData _Backpropagate(INode fromNode, IGraphData errorSignal, IContext context, IReadOnlyList <INode> parents) { var lap = context.LinearAlgebraProvider; var tensor = errorSignal.GetMatrix().ConvertTo4DTensor(_newHeight, _newWidth, _source._filter.ColumnCount); var padding = _source._padding; // calculate the weight and bias updates var weightUpdate = _im2Col.TransposeThisAndMultiply(tensor).CombineDepthSlices(); var biasUpdate = tensor.ColumnSums(); context.LearningContext.StoreUpdate(_source, weightUpdate, err => _source.Update(err, context.LearningContext)); context.LearningContext.StoreUpdate(_source, biasUpdate, bu => _UpdateBias(bu, context.LearningContext)); if (_source._shouldBackpropagate) { var filters = _source._filter; var inputDepth = _source._inputDepth; var filterWidth = _source._filterWidth; var filterHeight = _source._filterHeight; var stride = _source._stride; var filterList = new List <IReadOnlyList <IVector> >(); for (var i = 0; i < filters.ColumnCount; i++) { filterList.Add(filters.Column(i).Split(inputDepth).Select(v => v.Rotate(v.Count / filterWidth)).ToList()); } using (var reverseIm2Col = tensor.ReverseIm2Col(filterList, _inputHeight, _inputWidth, inputDepth, padding, filterWidth, filterHeight, stride)) { var delta = reverseIm2Col.ConvertTo4DTensor(_inputHeight + padding * 2, _inputWidth + padding * 2); if (padding > 0) { delta = delta.RemovePadding(padding); } return(new Tensor4DGraphData(delta.ConvertToMatrix(), _inputHeight, _inputWidth, inputDepth)); } } return(errorSignal); }
public IGraphData Execute(IGraphData input, IContext context) { context.ExecutionContext.SetMemory(_id, input.GetMatrix()); return(input); }
public IGraphData Execute(IGraphData input, IContext context) { var memory = context.ExecutionContext.GetMemory(_slotName); return(input.ReplaceWith(input.GetMatrix().ConcatRows(memory))); }