public NDarray <double> Backward(NDarray <double> accumGrad)
{
var gamma0 = new NDarray <double>(gamma);
var s0 = ND.SumAxis(accumGrad, 0);
var s1 = ND.SumAxis(ND.MulNDarray(accumGrad, xCentered), 0);
if (IsTraining)
{
var gGamma = ND.SumAxis(ND.MulNDarray(accumGrad, xNorm), 0);
var gBeta = s0;
gOpt.Update(gamma, gGamma);
bOpt.Update(beta, gBeta);
}
double batchSize = accumGrad.Shape[0];
var m0 = ND.MulNDarray(gamma0, stdDevInv, 1.0 / batchSize, 1);
var m1 = ND.MulNDarray(xNorm, ND.MulNDarray(stdDevInv, s1));
var diff = ND.SubNDarray(accumGrad, ND.AddNDarray(s0, m1), batchSize, 1);
accumGrad = ND.MulNDarray(m0, diff);
return(accumGrad);
}
public NDarray <U> Backward(NDarray <U> accumGrad)
{
for (int i = 0; i < wTmp.Count; ++i)
{
wTmp.Data[i] = weight.Data[i];
}
if (IsTraining)
{
var gW = ND.GemmTAB(LastInput, accumGrad);
var gB = ND.SumAxis(accumGrad, 0, true);
wOpt.Update(weight, gW);
bOpt.Update(biases, gB);
}
return(ND.GemmATB(accumGrad, wTmp));
}
public override NDArray <Type> Backward(NDArray <Type> accumGrad)
{
var Wtmp = new NDArray <Type>(W);
if (trainable)
{
var gW = NDArray <Type> .Dot(layerInput.T, accumGrad);
var gw0 = (new NDArray <Type>(accumGrad.Shape)) + NumDN.Sum(accumGrad);
W = WOpt.Update(W, gW);
w0 = w0Opt.Update(w0, gw0);
}
var accumGrad0 = NDArray <Type> .Dot(accumGrad, W.T);
return(accumGrad0);
}
/*
* def backward_pass(self, accum_grad):
# Reshape accumulated gradient into column shape
# accum_grad = accum_grad.transpose(1, 2, 3, 0).reshape(self.n_filters, -1)
#
# if self.trainable:
# Take dot product between column shaped accum. gradient and column shape
# layer input to determine the gradient at the layer with respect to layer weights
# grad_w = accum_grad.dot(self.X_col.T).reshape(self.W.shape)
# The gradient with respect to bias terms is the sum similarly to in Dense layer
# grad_w0 = np.sum(accum_grad, axis=1, keepdims=True)
#
# Update the layers weights
# self.W = self.W_opt.update(self.W, grad_w)
# self.w0 = self.w0_opt.update(self.w0, grad_w0)
#
# Recalculate the gradient which will be propogated back to prev. layer
# accum_grad = self.W_col.T.dot(accum_grad)
# Reshape from column shape to image shape
# accum_grad = column_to_image(accum_grad,
# self.layer_input.shape,
# self.filter_shape,
# stride=self.stride,
# output_shape=self.padding)
#
# return accum_grad
*/
public override NDArray <Type> Backward(NDArray <Type> accumGrad)
{
var accumGrad0 = accumGrad.transpose(1, 2, 3, 0).ReShape(numFilters, -1);
if (trainable)
{
var gradW = NDArray <Type> .Dot(accumGrad0, X_col.T).ReShape(W.Shape);
var gradw0 = NumDN.Sum(accumGrad0, true, 1);
W = WOpt.Update(W, gradW);
w0 = w0Opt.Update(w0, gradw0);
}
accumGrad0 = NDArray <Type> .Dot(W_col.T, accumGrad0);
accumGrad0 = col2img(accumGrad0, layerInput.Shape, filterShape, stride, padding);
return(accumGrad0);
}
public NDarray Backward(NDarray accumGrad)
{
accumGrad.MulFbInplace(lastXact, activation.Deriv);
for (int i = 0; i < wTmp.Count; ++i)
{
wTmp.Data[i] = weights.Data[i];
}
if (IsTraining)
{
var gw = NDarray.GemmTABC(lastX, accumGrad);
wOptm.Update(weights, gw);
var gb = accumGrad.SumAxis0();
bOptm.Update(biases, gb);
}
return(NDarray.GemmATBC(accumGrad, wTmp));
}
public NDarray <Type> Backward(NDarray <Type> accumGrad)
{
if (activationLayer != null)
{
accumGrad = activationLayer.Backward(accumGrad);
}
NDarray <Type> W = weights.T;
if (IsTraining)
{
var gW = ND.TensorDot(LayerInput.T, accumGrad);
weights = weightsOptmz.Update(weights, gW);
if (useBiases)
{
var gw0 = accumGrad.Sum(0, true);
biases = biasesOptmz.Update(biases, gw0);
}
}
return(ND.TensorDot <Type>(accumGrad, W));
}
public override NDArray <Type> Backward(NDArray <Type> accumGrad)
{
var gamma0 = new NDArray <Type>(gamma);
if (trainable)
{
var X_norm = X_centered * stddev_inv;
var grad_gamma = NumDN.Sum(accumGrad * X_norm, 0);
var grad_beta = NumDN.Sum(accumGrad, 0);
gamma = gOpt.Update(gamma, grad_gamma);
beta = bOpt.Update(beta, grad_beta);
}
double batchSize = accumGrad.Shape[0];
accumGrad = (1.0 / batchSize) * gamma0 * stddev_inv * (
batchSize * accumGrad
- NumDN.Sum(accumGrad, 0)
- X_centered * stddev_inv * stddev_inv * NumDN.Sum(accumGrad * X_centered, 0)
);
return(accumGrad);
}
