public override void ExecuteForward(IContext context)
{
if (context.BatchSequence.Type == MiniBatchSequenceType.SequenceStart)
{
_lastBackpropagation = null;
}
_start.ExecuteForward(context);
}
public override void ExecuteForward(IContext context)
{
_start.ExecuteForward(context);
}
public override void ExecuteForward(IContext context)
{
var data = context.Data;
IMatrix input;
IReadOnlyList <IVector> samples;
var lap = context.LinearAlgebraProvider;
var shouldDispose = false;
if (data.Depth > 1)
{
// reshape the tensor by depth slice
var tensor4D = data.Get4DTensor();
var slots = new List <IVector> [data.Depth];
for (var i = 0; i < data.Depth; i++)
{
slots[i] = new List <IVector>();
}
foreach (var tensor in tensor4D.ReshapeAsMatrix().ColumnVectors())
{
var depthSlices = tensor.ReshapeAsColumnMatrix().ReshapeAsVector().Split(data.Depth);
for (var i = 0; i < data.Depth; i++)
{
slots[i].Add(depthSlices[i]);
}
}
// TODO: create row vectors from slots and matrix from rows
//input = lap.CreateMatrixFromRows(list);
//samples = list;
shouldDispose = true;
throw new NotImplementedException();
}
else
{
input = context.Data.GetMatrix();
samples = input.RowVectors();
}
// collect statistics
if (context.IsTraining && _statistics != null)
{
foreach (var vector in samples)
{
_statistics.Update(vector);
vector.Dispose();
}
}
if (context.IsTraining)
{
_start.ExecuteForward(context);
// invalidate the cache
if (_gammaCached != null)
{
_gammaCached.Dispose();
_gammaCached = null;
_betaCached?.Dispose();
_betaCached = null;
_meanCached?.Dispose();
_meanCached = null;
_stdDevCached?.Dispose();
_stdDevCached = null;
}
}
else if (_statistics != null)
{
if (_gammaCached?.IsValid != true || _gammaCached?.RowCount != input.RowCount || _gammaCached?.ColumnCount != input.ColumnCount)
{
_gammaCached = context.LinearAlgebraProvider.CreateMatrix(input.RowCount, input.ColumnCount, (x, y) => _gamma.Data[y]);
}
if (_betaCached?.IsValid != true || _betaCached?.RowCount != input.RowCount || _betaCached?.ColumnCount != input.ColumnCount)
{
_betaCached = context.LinearAlgebraProvider.CreateMatrix(input.RowCount, input.ColumnCount, (x, y) => _beta.Data[y]);
}
if (_meanCached?.IsValid != true || _meanCached?.RowCount != input.RowCount || _meanCached?.ColumnCount != input.ColumnCount)
{
var mean = _statistics.Mean;
_meanCached = context.LinearAlgebraProvider.CreateMatrixFromRows(Enumerable.Repeat(mean, input.RowCount).ToList());
}
if (_stdDevCached?.IsValid != true || _stdDevCached?.RowCount != input.RowCount || _stdDevCached?.ColumnCount != input.ColumnCount)
{
using (var variance = _statistics.GetSampleVariance())
using (var stdDev = variance.Sqrt()) {
_stdDevCached = context.LinearAlgebraProvider.CreateMatrixFromRows(Enumerable.Repeat(stdDev, input.RowCount).ToList());
}
}
input.SubtractInPlace(_meanCached);
using (var xHat = input.PointwiseDivide(_stdDevCached)) {
using (var ret = xHat.PointwiseMultiply(_gammaCached)) {
ret.AddInPlace(_betaCached);
_AddNextGraphAction(context, context.Data.ReplaceWith(ret), null);
}
}
}
if (shouldDispose)
{
input.Dispose();
}
foreach (var item in samples)
{
item.Dispose();
}
}
