public override void ExecuteForward(IContext context) { if (context.BatchSequence.Type == MiniBatchSequenceType.SequenceStart) { _lastBackpropagation = null; } _start.ExecuteForward(context); }
public override void ExecuteForward(IContext context) { _start.ExecuteForward(context); }
public override void ExecuteForward(IContext context) { var data = context.Data; IMatrix input; IReadOnlyList <IVector> samples; var lap = context.LinearAlgebraProvider; var shouldDispose = false; if (data.Depth > 1) { // reshape the tensor by depth slice var tensor4D = data.Get4DTensor(); var slots = new List <IVector> [data.Depth]; for (var i = 0; i < data.Depth; i++) { slots[i] = new List <IVector>(); } foreach (var tensor in tensor4D.ReshapeAsMatrix().ColumnVectors()) { var depthSlices = tensor.ReshapeAsColumnMatrix().ReshapeAsVector().Split(data.Depth); for (var i = 0; i < data.Depth; i++) { slots[i].Add(depthSlices[i]); } } // TODO: create row vectors from slots and matrix from rows //input = lap.CreateMatrixFromRows(list); //samples = list; shouldDispose = true; throw new NotImplementedException(); } else { input = context.Data.GetMatrix(); samples = input.RowVectors(); } // collect statistics if (context.IsTraining && _statistics != null) { foreach (var vector in samples) { _statistics.Update(vector); vector.Dispose(); } } if (context.IsTraining) { _start.ExecuteForward(context); // invalidate the cache if (_gammaCached != null) { _gammaCached.Dispose(); _gammaCached = null; _betaCached?.Dispose(); _betaCached = null; _meanCached?.Dispose(); _meanCached = null; _stdDevCached?.Dispose(); _stdDevCached = null; } } else if (_statistics != null) { if (_gammaCached?.IsValid != true || _gammaCached?.RowCount != input.RowCount || _gammaCached?.ColumnCount != input.ColumnCount) { _gammaCached = context.LinearAlgebraProvider.CreateMatrix(input.RowCount, input.ColumnCount, (x, y) => _gamma.Data[y]); } if (_betaCached?.IsValid != true || _betaCached?.RowCount != input.RowCount || _betaCached?.ColumnCount != input.ColumnCount) { _betaCached = context.LinearAlgebraProvider.CreateMatrix(input.RowCount, input.ColumnCount, (x, y) => _beta.Data[y]); } if (_meanCached?.IsValid != true || _meanCached?.RowCount != input.RowCount || _meanCached?.ColumnCount != input.ColumnCount) { var mean = _statistics.Mean; _meanCached = context.LinearAlgebraProvider.CreateMatrixFromRows(Enumerable.Repeat(mean, input.RowCount).ToList()); } if (_stdDevCached?.IsValid != true || _stdDevCached?.RowCount != input.RowCount || _stdDevCached?.ColumnCount != input.ColumnCount) { using (var variance = _statistics.GetSampleVariance()) using (var stdDev = variance.Sqrt()) { _stdDevCached = context.LinearAlgebraProvider.CreateMatrixFromRows(Enumerable.Repeat(stdDev, input.RowCount).ToList()); } } input.SubtractInPlace(_meanCached); using (var xHat = input.PointwiseDivide(_stdDevCached)) { using (var ret = xHat.PointwiseMultiply(_gammaCached)) { ret.AddInPlace(_betaCached); _AddNextGraphAction(context, context.Data.ReplaceWith(ret), null); } } } if (shouldDispose) { input.Dispose(); } foreach (var item in samples) { item.Dispose(); } }