void ReshapeFit(int[] args)
{
int pos = -1, nb = 0;
for (int k = 0; k < args.Length; ++k)
{
if (args[k] == -1)
{
++nb;
pos = k;
}
}
if (nb > 1)
{
throw new ArgumentException();
}
if (nb == 1)
{
args[pos] = 1;
int dim0 = NumDN.ShapeLength(Shape);
int dim1 = NumDN.ShapeLength(args);
if (dim0 % dim1 != 0)
{
throw new ArgumentException();
}
args[pos] = dim0 / dim1;
}
}
public override NDArray <Type> Forward(NDArray <Type> X, bool isTraining)
{
if (runningMean == null)
{
runningMean = NumDN.Mean(X, 0);
runningVar = NumDN.Var(X, 0);
}
NDArray <Type> mean, varr;
if (isTraining)
{
mean = NumDN.Mean(X, 0);
varr = NumDN.Var(X, 0);
runningMean = momentum * runningMean + (1 - momentum) * mean;
runningVar = momentum * runningVar + (1 - momentum) * varr;
}
else
{
mean = runningMean;
varr = runningVar;
}
X_centered = X - mean;
stddev_inv = 1.0 / NumDN.Sq(varr + eps);
var X_norm = X_centered * stddev_inv;
var output = gamma * X_norm + beta;
return(output);
}
public override void Initialize(IOptimizer <Type> optimizer)
{
double lim = 1.0 / Math.Sqrt(Inputs[0]);
W = NumDN.Uniform <Type>(-lim, lim, Inputs[0], Outputs[0]);
w0 = NDArray <Type> .Zeros(1, Outputs[0]);
WOpt = optimizer.Clone();
w0Opt = optimizer.Clone();
}
public override void Initialize(IOptimizer <Type> optimizer = null)
{
fHeight = filterShape[0];
fWidth = filterShape[1];
channel = Inputs[0];
double lim = 1.0 / Math.Sqrt(NumDN.ShapeLength(filterShape));
W = NumDN.Uniform <Type>(-lim, lim, numFilters, channel, fWidth, fHeight);
w0 = NDArray <Type> .Zeros(numFilters, 1);
WOpt = optimizer.Clone();
w0Opt = optimizer.Clone();
}
public NDArray(Type v, int[] shape)
{
int dim = NumDN.ShapeLength(shape);
if (dim <= 0)
{
throw new ArgumentException();
}
Shape = shape.ToArray();
tmpArr = new int[Shape.Length];
items = Enumerable.Repeat(v, dim).ToArray();
}
public NDArray(Type[] nd, int[] shape)
{
int dim = NumDN.ShapeLength(shape);
if (dim <= 0)
{
throw new ArgumentException();
}
Shape = shape.ToArray();
tmpArr = new int[Shape.Length];
items = nd.ToArray();
}
public double Acc(NDArray <Type> y, NDArray <Type> p)
{
int n = y.Shape[0];
double s = 0;
var v = NumDN.Abs(y - p).Apply(x => Convert.ToDouble(x) < 0.5 ? 1 : 0);
for (int k = 0; k < n; ++k)
{
s += Enumerable.Range(k * 10, 10).Select(i => v.items[i]).Sum() == 10 ? 1.0 : 0.0;
}
//var m = (y - p).items.Select(x => Math.Abs(Convert.ToDouble(x)) < 0.7 ? 1.0 : 0.0).Average();
return(s);
}
public void ReShapeInplace(params int[] args)
{
ReshapeFit(args);
int oldDim = NumDN.ShapeLength(Shape);
int newDim = NumDN.ShapeLength(args);
if (oldDim != newDim)
{
throw new ArgumentException();
}
Shape = args.Length != 0 ? args.ToArray() : new int[] { 1 };
tmpArr = new int[Shape.Length];
}
static void Bench(int batchSize, int loops)
{
NumDN.DebugNumPy = false;
var sw = Stopwatch.StartNew();
for (int k = 0; k < loops; ++k)
{
var x = NumDN.UniformInt(0, 16, batchSize, 1, 8, 8).Cast <float>();
var cols = Conv2d <float> .img2col(x, new int[] { 3, 3 }, 1);
var x0 = Conv2d <float> .col2img(cols, x.Shape, new int[] { 3, 3 }, 1);
}
Console.WriteLine($"{0.001 * sw.ElapsedMilliseconds} s");
}
public NDArray <Type> ReShape(params int[] args)
{
ReshapeFit(args);
int oldDim = NumDN.ShapeLength(Shape);
int newDim = NumDN.ShapeLength(args);
if (oldDim != newDim)
{
throw new ArgumentException();
}
var ndarr = new NDArray <Type>(this);
ndarr.Shape = args.Length != 0 ? args.ToArray() : new int[] { 1 };
ndarr.tmpArr = new int[ndarr.Shape.Length];
return(ndarr);
}
public override NDArray <Type> Backward(NDArray <Type> accumGrad)
{
var Wtmp = new NDArray <Type>(W);
if (trainable)
{
var gW = NDArray <Type> .Dot(layerInput.T, accumGrad);
var gw0 = (new NDArray <Type>(accumGrad.Shape)) + NumDN.SumDouble(accumGrad);
W = WOpt.Update(W, gW);
w0 = w0Opt.Update(w0, gw0);
}
var accumGrad0 = NDArray <Type> .Dot(accumGrad, W.T);
return(accumGrad0);
}
/*
* def backward_pass(self, accum_grad):
# Reshape accumulated gradient into column shape
# accum_grad = accum_grad.transpose(1, 2, 3, 0).reshape(self.n_filters, -1)
#
# if self.trainable:
# Take dot product between column shaped accum. gradient and column shape
# layer input to determine the gradient at the layer with respect to layer weights
# grad_w = accum_grad.dot(self.X_col.T).reshape(self.W.shape)
# The gradient with respect to bias terms is the sum similarly to in Dense layer
# grad_w0 = np.sum(accum_grad, axis=1, keepdims=True)
#
# Update the layers weights
# self.W = self.W_opt.update(self.W, grad_w)
# self.w0 = self.w0_opt.update(self.w0, grad_w0)
#
# Recalculate the gradient which will be propogated back to prev. layer
# accum_grad = self.W_col.T.dot(accum_grad)
# Reshape from column shape to image shape
# accum_grad = column_to_image(accum_grad,
# self.layer_input.shape,
# self.filter_shape,
# stride=self.stride,
# output_shape=self.padding)
#
# return accum_grad
*/
public override NDArray <Type> Backward(NDArray <Type> accumGrad)
{
var accumGrad0 = accumGrad.transpose(1, 2, 3, 0).ReShape(numFilters, -1);
if (trainable)
{
var gradW = NDArray <Type> .Dot(accumGrad0, X_col.T).ReShape(W.Shape);
var gradw0 = NumDN.Sum(accumGrad0, true, 1);
W = WOpt.Update(W, gradW);
w0 = w0Opt.Update(w0, gradw0);
}
accumGrad0 = NDArray <Type> .Dot(W_col.T, accumGrad0);
accumGrad0 = col2img(accumGrad0, layerInput.Shape, filterShape, stride, padding);
return(accumGrad0);
}
public NDArray <Type> Update(NDArray <Type> w, NDArray <Type> g)
{
if (m == null)
{
m = NDArray <Type> .Zeros(g.Shape);
v = NDArray <Type> .Zeros(g.Shape);
}
m = b1 * m + (1 - b1) * g;
v = b2 * v + (1 - b2) * NumDN.Sq(g);
var mh = m / (1.0 - b1);
var vh = v / (1.0 - b2);
weightsUpdate = learning_rate * mh / (NumDN.Sqrt(vh) + eps);
return(w - weightsUpdate);
}
public NDArray <Type> this[int i]
{
get
{
if (i >= Shape[0] || i < -1)
{
throw new ArgumentException();
}
var dim0 = NumDN.ShapeLength(Shape);
var dim1 = dim0 / Shape[0];
var nshape = Shape.Skip(1).ToArray();
var start = i * dim1;
var nd = new NDArray <Type>(nshape);
for (int j = 0, k = start; j < dim1; ++j, ++k)
{
nd.items[j] = items[k];
}
return(nd);
}
set
{
var dim0 = NumDN.ShapeLength(Shape);
var dim1 = NumDN.ShapeLength(value.Shape);
if (dim0 / dim1 != Shape[0] || i >= Shape[0] || i < 0)
{
throw new ArgumentException();
}
var start = i * dim1;
for (int j = 0, k = start; j < dim1; ++j, ++k)
{
items[k] = value.items[j];
}
}
}
public override NDArray <Type> Backward(NDArray <Type> accumGrad)
{
var gamma0 = new NDArray <Type>(gamma);
if (trainable)
{
var X_norm = X_centered * stddev_inv;
var grad_gamma = NumDN.Sum(accumGrad * X_norm, 0);
var grad_beta = NumDN.Sum(accumGrad, 0);
gamma = gOpt.Update(gamma, grad_gamma);
beta = bOpt.Update(beta, grad_beta);
}
double batchSize = accumGrad.Shape[0];
accumGrad = (1.0 / batchSize) * gamma0 * stddev_inv * (
batchSize * accumGrad
- NumDN.Sum(accumGrad, 0)
- X_centered * stddev_inv * stddev_inv * NumDN.Sum(accumGrad * X_centered, 0)
);
return(accumGrad);
}
public override int[] GetOutputShape() => new int[]
{
NumDN.ShapeLength(Inputs)
};
public NDArray <Type> Function(NDArray <Type> x) => NumDN.Sigmoid(x);
public static NDArray <Type> Dot(NDArray <Type> nD0, NDArray <Type> nD1)
{
if (nD0.Shape.Length == 1)
{
nD0 = nD0.ReShape(1, nD0.Shape[0]);
}
if (nD1.Shape.Length == 1)
{
nD1 = nD1.ReShape(nD1.Shape[0], 1);
}
int length0 = nD0.Shape.Length;
int length1 = nD1.Shape.Length;
int piv = nD0.Shape.Last();
if (piv != nD1.Shape[length1 - 2])
{
throw new ArgumentException($"Cannot multiply {nD0.Shape.Glue("x")} and {nD1.Shape.Glue("x")}");
}
int[] nshape = new int[length0 + length1 - 2];
int[] idxInfos = new int[length0 + length1 - 2];
for (int k = 0, k0 = 0; k < length0 + length1; ++k)
{
if (k == length0 - 1 || k == length0 + length1 - 2)
{
continue;
}
if (k < length0 - 1)
{
nshape[k] = nD0.Shape[idxInfos[k] = k];
}
else
{
nshape[k0] = nD1.Shape[idxInfos[k0] = k - length0];
}
++k0;
}
int dim = NumDN.ShapeLength(nshape);
var nd = new NDArray <Type>(nshape);
for (int m = 0; m < nd.items.Length; ++m)
{
Type sum = OpsT.Zero;
nd.Index2Array(m);
for (int k = 0; k < nshape.Length; ++k)
{
if (k < length0 - 1)
{
nD0.tmpArr[idxInfos[k]] = nd.tmpArr[k];
}
else
{
nD1.tmpArr[idxInfos[k]] = nd.tmpArr[k];
}
}
for (int i = 0; i < piv; ++i)
{
nD0.tmpArr[length0 - 1] = nD1.tmpArr[length1 - 2] = i;
int idx0 = nD0.Array2Index();
int idx1 = nD1.Array2Index();
sum = OpsT.Add(sum, OpsT.Mul(nD0.items[idx0], nD1.items[idx1]));
}
nd.items[m] = sum;
}
return(nd);
}
public NDArray <Type> Loss(NDArray <Type> y, NDArray <Type> p) => 0.5 * NumDN.Sq(y - p);
public NDArray <Type> Function(NDArray <Type> x)
{
var ex = NumDN.Exp(x - NumDN.Max(x));
return(ex / NumDN.SumDouble(ex));
}
public NDArray <Type> Loss(NDArray <Type> y, NDArray <Type> p)
{
var p0 = NumDN.Clamp(p, 1e-15, 1 - 1e-15);
return(-y *NumDN.Log(p0) - (1 - y) * NumDN.Log(1 - p0));
}
public NDArray <Type> Function(NDArray <Type> x) => NumDN.Tanh(x);
public NDArray <Type> Gradient(NDArray <Type> x) => NumDN.Sigmoid(x) * (1 - NumDN.Sigmoid(x));
public NDArray <Type> Gradient(NDArray <Type> x) => 1 - NumDN.Sq(NumDN.Tanh(x));
public NDArray <Type> Grad(NDArray <Type> y, NDArray <Type> p)
{
var p0 = NumDN.Clamp(p, 1e-15, 1 - 1e-15);
return(-y / p0 + (1 - y) / (1 - p0));
}
