public override Signal Backward(ComputeShader compute, Signal dout)
{
dbeta = Refresh(1, dout.Columns, dbeta);
MatOperations.SumMV(compute, dout, dbeta);
dgamma = Refresh(1, dout.Columns, dgamma);
MatOperations.MulMM(compute, dout, xn);
MatOperations.SumMV(compute, xn, dgamma);
var dxn = new Signal(dout);
MatOperations.MulMVM(compute, dout, gamma, dxn);
var dxc = new Signal(dxn);
MatOperations.DivMVM(compute, dxn, std, dxc);
var dxn_x_xc = new Signal(dxn);
MatOperations.MulMMM(compute, dxn, xc, dxn_x_xc);
dxn.Dispose();
var std_x_std = new Signal(std);
MatOperations.MulMMM(compute, std, std, std_x_std);
var dxn_x_xc_div_std_x_std = new Signal(dxn_x_xc);
MatOperations.DivMVM(compute, dxn_x_xc, std_x_std, dxn_x_xc_div_std_x_std);
dxn_x_xc.Dispose();
std_x_std.Dispose();
var dstd = new Signal(std);
MatOperations.SumMV(compute, dxn_x_xc_div_std_x_std, dstd);
dxn_x_xc_div_std_x_std.Dispose();
var dvar = new Signal(dstd);
DVar(compute, dstd, std, dvar);
dstd.Dispose();
DXc(compute, xc, dvar, dxc, 2f / batchSize);
dvar.Dispose();
var dmu = new Signal(1, dxc.Columns);
MatOperations.SumMV(compute, dxc, dmu);
var dx = new Signal(dout);
DX(compute, dxc, dmu, dx, 1f / batchSize);
dxc.Dispose();
dmu.Dispose();
return(dx);
}
public override Signal Forward(ComputeShader compute, Signal x, bool train)
{
if (train)
{
var mu = new Signal(1, x.Columns);
MatOperations.MeanMV(compute, x, mu);
// x.Log("x");
// mu.Log("mu");
xc = Refresh(x, xc); // xc = x - mu
MatOperations.SubMVM(compute, x, mu, xc);
// xc.Log("xc");
var variance = new Signal(1, xc.Columns);
MatOperations.VarianceMV(compute, xc, variance);
// variance.Log("variance");
std = Refresh(variance, std);
MatOperations.SqrtMM(compute, variance, std);
// std.Log("std");
xn = Refresh(xc, xn); // xn = xc / std
MatOperations.DivMVM(compute, xc, std, xn);
batchSize = x.Rows;
Momentum(compute, mu, runningMean);
Momentum(compute, variance, runningVar);
mu.Dispose();
variance.Dispose();
}
else
{
xc = Refresh(x, xc); // xc = x - runningMean
MatOperations.SubMVM(compute, x, runningMean, xc);
// x.Log("x");
// runningMean.Log("runningMean");
// xn.Log("xn");
xn = Refresh(xc, xn); // xn = xc / sqrt(runningVar + epsilon)
Xn(compute, xc, runningVar, xn);
}
var output = new Signal(xn);
var kernel = compute.FindKernel("BNForward");
compute.SetBuffer(kernel, "_X", xn.Buffer);
compute.SetBuffer(kernel, "_Gamma", gamma.Buffer);
compute.SetBuffer(kernel, "_Beta", beta.Buffer);
compute.SetBuffer(kernel, "_Y", output.Buffer);
Dispatch(compute, kernel, output.Rows, output.Columns);
return(output);
}
public override Signal Forward(ComputeShader compute, Signal x, bool train)
{
mask = Refresh(x, mask);
var kernel = compute.FindKernel("ReLU");
compute.SetBuffer(kernel, "_X", x.Buffer);
compute.SetBuffer(kernel, "_Y", mask.Buffer);
Dispatch(compute, kernel, mask.Rows, mask.Columns);
var output = new Signal(mask);
MatOperations.CopyMM(compute, mask, output);
return(output);
}
public override Signal Forward(ComputeShader compute, Signal x, bool train)
{
this.x = Refresh(x, this.x);
MatOperations.CopyMM(compute, x, this.x);
var output = new Signal(x.Rows, weights.Columns);
// matmul M = input * weights
MatOperations.Multiply(compute, x, weights, output);
// matplus M´ = M + biases
MatOperations.AddVM(compute, biases, output);
// output.Log();
return(output);
}
public override Signal Backward(ComputeShader compute, Signal dout)
{
var dx = new Signal(dout.Rows, weights.Rows);
MatOperations.MultiplyMT(compute, dout, weights, dx);
// dx.Log();
dW = Refresh(x.Columns, dout.Columns, dW);
MatOperations.MultiplyTM(compute, x, dout, dW);
dB = Refresh(1, dout.Columns, dB);
MatOperations.SumMV(compute, dout, dB);
// dx.Log();
// dw.Log();
// db.Log();
return(dx);
}
