public static Tensor AddBias(this Session session, Tensor x, Tensor b)
{
int mb = x.Axes[0]; // number of items in a mini-batch
if (mb == 1)
{
return(session.Add(x, b));
}
Tensor y = session.Allocate("addBias", x.Axes);
int stride = x.Strides[0]; // item length
Debug.Assert(stride == b.Length, "Biases tensor has invalid size.");
// repeat for each item in mini-batch
for (int i = 0, off = 0; i < mb; i++, off += stride)
{
Mathematics.Add(stride, x.Weights, off, b.Weights, 0, y.Weights, off);
}
if (session.CalculateGradients && (x.CalculateGradient || b.CalculateGradient))
{
session.Push(() =>
{
Tensor dy = session.GetGradient(y);
// dx += dy
if (x.CalculateGradient)
{
Tensor dx = session.GetGradient(x);
lock (dx)
{
dx.Add(dy);
}
}
// db += sum(dy)
if (b.CalculateGradient)
{
float[] ones = new float[mb];
MKL.Set(mb, 1.0f, ones, 0);
Tensor db = session.GetGradient(b);
lock (db)
{
MKL.MxV(MatrixLayout.ColumnMajor, stride, mb, y.Gradient, 0, false, ones, 0, db.Weights, 0, false);
/*for (int i = 0, off = 0; i < mb; i++, off += stride)
* {
* Mathematics.Add(stride, y.Gradient, off, db.Weights, 0);
* }*/
}
}
});
}
return(y);
}