public override NdArray SingleInputForward(NdArray input)
{
int outputHeight = (input.Shape[1] - 1) * this.StrideY + this.KernelHeight - this.PadY * 2;
int outputWidth = (input.Shape[2] - 1) * this.StrideX + this.KernelWidth - this.PadX * 2;
Real[] result = new Real[input.BatchCount * this.OutputCount * outputWidth * outputHeight];
int outSizeOffset = outputWidth * outputHeight;
int inputSizeOffset = input.Shape[1] * input.Shape[2];
int kSizeOffset = this.Weight.Shape[2] * this.Weight.Shape[3];
for (int batchCount = 0; batchCount < input.BatchCount; batchCount++)
{
for (int och = 0; och < this.OutputCount; och++)
{
for (int oy = this.PadY; oy < outputHeight + this.PadY; oy++)
{
int iyLimit = oy / this.StrideY + 1 < input.Shape[1] ? oy / this.StrideY + 1 : input.Shape[1];
int iyStart = oy - this.Weight.Shape[2] < 0 ? 0 : (oy - this.Weight.Shape[2]) / this.StrideY + 1;
for (int ox = this.PadX; ox < outputWidth + this.PadX; ox++)
{
int ixLimit = ox / this.StrideX + 1 < input.Shape[2] ? ox / this.StrideX + 1 : input.Shape[2];
int ixStart = ox - this.Weight.Shape[3] < 0 ? 0 : (ox - this.Weight.Shape[3]) / this.StrideX + 1;
int outputIndex = batchCount * this.OutputCount * outSizeOffset + och * outSizeOffset + (oy - this.PadY) * outputWidth + ox - this.PadX;
for (int ich = 0; ich < input.Shape[0]; ich++)
{
int inputIndexOffset = batchCount * input.Length + ich * inputSizeOffset;
int kernelIndexOffset = ich * this.Weight.Shape[1] * kSizeOffset + och * kSizeOffset;
for (int iy = iyStart; iy < iyLimit; iy++)
{
for (int ix = ixStart; ix < ixLimit; ix++)
{
int inputIndex = inputIndexOffset + iy * input.Shape[2] + ix;
int kernelIndex = kernelIndexOffset + (oy - iy * this.StrideY) * this.Weight.Shape[3] + (ox - ix * this.StrideX);
result[outputIndex] += input.Data[inputIndex] * this.Weight.Data[kernelIndex];
}
}
}
}
}
}
}
if (this.Activation != null && !NoBias)
{
for (int batchCount = 0; batchCount < input.BatchCount; batchCount++)
{
for (int och = 0; och < this.OutputCount; och++)
{
for (int oy = this.PadY; oy < outputHeight + this.PadY; oy++)
{
for (int ox = this.PadX; ox < outputWidth + this.PadX; ox++)
{
int outputIndex = batchCount * this.OutputCount * outSizeOffset + och * outSizeOffset + (oy - this.PadY) * outputWidth + ox - this.PadX;
result[outputIndex] += this.Bias.Data[och];
result[outputIndex] = this.Activation.ForwardActivate(result[outputIndex]);
}
}
}
}
}
else if (!NoBias)
{
for (int batchCount = 0; batchCount < input.BatchCount; batchCount++)
{
for (int och = 0; och < this.OutputCount; och++)
{
for (int oy = this.PadY; oy < outputHeight + this.PadY; oy++)
{
for (int ox = this.PadX; ox < outputWidth + this.PadX; ox++)
{
int outputIndex = batchCount * this.OutputCount * outSizeOffset + och * outSizeOffset + (oy - this.PadY) * outputWidth + ox - this.PadX;
result[outputIndex] += this.Bias.Data[och];
}
}
}
}
}
else if (this.Activation != null)
{
for (int batchCount = 0; batchCount < input.BatchCount; batchCount++)
{
for (int och = 0; och < this.OutputCount; och++)
{
for (int oy = this.PadY; oy < outputHeight + this.PadY; oy++)
{
for (int ox = this.PadX; ox < outputWidth + this.PadX; ox++)
{
int outputIndex = batchCount * this.OutputCount * outSizeOffset + och * outSizeOffset + (oy - this.PadY) * outputWidth + ox - this.PadX;
result[outputIndex] = this.Activation.ForwardActivate(result[outputIndex]);
}
}
}
}
}
return(NdArray.Convert(result, new[] { this.OutputCount, outputHeight, outputWidth }, input.BatchCount, this));
}
protected override NdArray NeedPreviousForwardCpu([NotNull] NdArray input)
{
int outputHeight = (input.Shape[1] - 1) * _subSampleY + _kHeight - _trimY * 2;
int outputWidth = (input.Shape[2] - 1) * _subSampleX + _kWidth - _trimX * 2;
Real[] result = new Real[input.BatchCount * OutputCount * outputWidth * outputHeight];
int outSizeOffset = outputWidth * outputHeight;
int inputSizeOffset = input.Shape[1] * input.Shape[2];
int kSizeOffset = Weight.Shape[2] * Weight.Shape[3];
for (int batchCount = 0; batchCount < input.BatchCount; batchCount++)
{
for (int och = 0; och < OutputCount; och++)
{
for (int oy = _trimY; oy < outputHeight + _trimY; oy++)
{
int iyLimit = oy / _subSampleY + 1 < input.Shape[1] ? oy / _subSampleY + 1 : input.Shape[1];
int iyStart = oy - Weight.Shape[2] < 0 ? 0 : (oy - Weight.Shape[2]) / _subSampleY + 1;
for (int ox = _trimX; ox < outputWidth + _trimX; ox++)
{
int ixLimit = ox / _subSampleX + 1 < input.Shape[2] ? ox / _subSampleX + 1 : input.Shape[2];
int ixStart = ox - Weight.Shape[3] < 0 ? 0 : (ox - Weight.Shape[3]) / _subSampleX + 1;
int outputIndex = batchCount * OutputCount * outSizeOffset + och * outSizeOffset + (oy - _trimY) * outputWidth + ox - _trimX;
for (int ich = 0; ich < input.Shape[0]; ich++)
{
int inputIndexOffset = batchCount * input.Length + ich * inputSizeOffset;
int kernelIndexOffset = och * Weight.Shape[1] * kSizeOffset + ich * kSizeOffset;
for (int iy = iyStart; iy < iyLimit; iy++)
{
for (int ix = ixStart; ix < ixLimit; ix++)
{
int inputIndex = inputIndexOffset + iy * input.Shape[2] + ix;
int kernelIndex = kernelIndexOffset + (oy - iy * _subSampleY) * Weight.Shape[3] + (ox - ix * _subSampleX);
result[outputIndex] += input.Data[inputIndex] * Weight.Data[kernelIndex];
}
}
}
}
}
}
}
if (Activator != null && !NoBias)
{
for (int batchCount = 0; batchCount < input.BatchCount; batchCount++)
{
for (int och = 0; och < OutputCount; och++)
{
for (int oy = _trimY; oy < outputHeight + _trimY; oy++)
{
for (int ox = _trimX; ox < outputWidth + _trimX; ox++)
{
int outputIndex = batchCount * OutputCount * outSizeOffset + och * outSizeOffset + (oy - _trimY) * outputWidth + ox - _trimX;
result[outputIndex] += Bias.Data[och];
result[outputIndex] = Activator.ForwardActivate(result[outputIndex]);
}
}
}
}
}
else if (!NoBias)
{
for (int batchCount = 0; batchCount < input.BatchCount; batchCount++)
{
for (int och = 0; och < OutputCount; och++)
{
for (int oy = _trimY; oy < outputHeight + _trimY; oy++)
{
for (int ox = _trimX; ox < outputWidth + _trimX; ox++)
{
int outputIndex = batchCount * OutputCount * outSizeOffset + och * outSizeOffset + (oy - _trimY) * outputWidth + ox - _trimX;
result[outputIndex] += Bias.Data[och];
}
}
}
}
}
else if (Activator != null)
{
for (int batchCount = 0; batchCount < input.BatchCount; batchCount++)
{
for (int och = 0; och < OutputCount; och++)
{
for (int oy = _trimY; oy < outputHeight + _trimY; oy++)
{
for (int ox = _trimX; ox < outputWidth + _trimX; ox++)
{
int outputIndex = batchCount * OutputCount * outSizeOffset + och * outSizeOffset + (oy - _trimY) * outputWidth + ox - _trimX;
result[outputIndex] = Activator.ForwardActivate(result[outputIndex]);
}
}
}
}
}
return(NdArray.Convert(result, new[] { OutputCount, outputHeight, outputWidth }, input.BatchCount, this));
}
public NdArray ForwardCpu(params NdArray[] xs)
{
Real[] result = new Real[xs[0].Data.Length];
switch (_operation)
{
case EltwiseParameter.EltwiseOp.Prod:
Array.Copy(xs[0].Data, result, result.Length);
for (int i = 1; i < xs.Length; i++)
{
for (int j = 0; j < result.Length; j++)
{
result[j] *= xs[i].Data[j];
}
}
break;
case EltwiseParameter.EltwiseOp.Sum:
if (this._coeffs != null)
{
for (int i = 0; i < xs.Length; i++)
{
for (int j = 0; j < result.Length; j++)
{
result[j] += xs[i].Data[j] * _coeffs[i];
}
}
}
else
{
for (int i = 0; i < xs.Length; i++)
{
for (int j = 0; j < result.Length; j++)
{
result[j] += xs[i].Data[j];
}
}
}
break;
case EltwiseParameter.EltwiseOp.Max:
Array.Copy(xs[0].Data, result, result.Length);
int[] outputIndex = new int[result.Length];
for (int i = 1; i < xs.Length; i++)
{
for (int j = 0; j < result.Length; j++)
{
if (result[j] < xs[i].Data[j])
{
outputIndex[j] = i;
result[j] = xs[i].Data[j];
}
}
}
PrevOutputIndex.Add(outputIndex);
break;
}
return(NdArray.Convert(result, xs[0].Shape, xs[0].BatchCount, this));
}
private NdArray ForwardCpu(NdArray x)
{
//Acquisition of calculation parameters
if (this.IsTrain)
{
//Set Mean and Variance of members
this.Variance = new Real[this.ChannelSize];
for (int i = 0; i < this.Variance.Length; i++)
{
this.Variance[i] = 0;
}
this.Mean = new Real[this.ChannelSize];
for (int i = 0; i < this.Mean.Length; i++)
{
for (int index = 0; index < x.BatchCount; index++)
{
this.Mean[i] += x.Data[i + index * x.Length];
}
this.Mean[i] /= x.BatchCount;
}
for (int i = 0; i < this.Mean.Length; i++)
{
for (int index = 0; index < x.BatchCount; index++)
{
this.Variance[i] += (x.Data[i + index * x.Length] - this.Mean[i]) * (x.Data[i + index * x.Length] - this.Mean[i]);
}
this.Variance[i] /= x.BatchCount;
}
for (int i = 0; i < this.Variance.Length; i++)
{
this.Variance[i] += this.Eps;
}
}
else
{
this.Mean = this.AvgMean.Data;
this.Variance = this.AvgVar.Data;
}
this.Std = new Real[this.Variance.Length];
for (int i = 0; i < this.Variance.Length; i++)
{
this.Std[i] = Math.Sqrt(this.Variance[i]);
}
//Calculate result
this.Xhat = new Real[x.Data.Length];
Real[] y = new Real[x.Data.Length];
int dataSize = 1;
for (int i = 1; i < x.Shape.Length; i++)
{
dataSize *= x.Shape[i];
}
for (int batchCount = 0; batchCount < x.BatchCount; batchCount++)
{
for (int i = 0; i < this.ChannelSize; i++)
{
for (int location = 0; location < dataSize; location++)
{
int index = batchCount * this.ChannelSize * dataSize + i * dataSize + location;
this.Xhat[index] = (x.Data[index] - this.Mean[i]) / this.Std[i];
y[index] = this.Gamma.Data[i] * this.Xhat[index] + this.Beta.Data[i];
}
}
}
//Update parameters
if (this.IsTrain)
{
int m = x.BatchCount;
Real adjust = m / Math.Max(m - 1.0, 1.0); //unbiased estimation
for (int i = 0; i < this.AvgMean.Data.Length; i++)
{
this.AvgMean.Data[i] *= this.Decay;
this.Mean[i] *= 1 - this.Decay; //reuse buffer as a temporary
this.AvgMean.Data[i] += this.Mean[i];
this.AvgVar.Data[i] *= this.Decay;
this.Variance[i] *= (1 - this.Decay) * adjust; //reuse buffer as a temporary
this.AvgVar.Data[i] += this.Variance[i];
}
}
return(NdArray.Convert(y, x.Shape, x.BatchCount, this));
}
protected override NdArray NeedPreviousForwardCpu(NdArray input)
{
int outputHeight = (int)Math.Floor((input.Shape[1] - _kHeight + _padY * 2.0) / _strideY) + 1;
int outputWidth = (int)Math.Floor((input.Shape[2] - _kWidth + _padX * 2.0) / _strideX) + 1;
Real[] result = new Real[OutputCount * outputHeight * outputWidth * input.BatchCount];
for (int batchCounter = 0; batchCounter < input.BatchCount; batchCounter++)
{
int resultIndex = batchCounter * OutputCount * outputHeight * outputWidth;
for (int och = 0; och < OutputCount; och++)
{
//For W index
int outChOffset = och * InputCount * _kHeight * _kWidth;
for (int oy = 0; oy < outputHeight * _strideY; oy += _strideY)
{
int kyStartIndex = oy - _padY < 0 ? 0 : oy - _padY;
int kyLimit = _kHeight + oy - _padY < input.Shape[1] ? _kHeight + oy - _padY : input.Shape[1];
for (int ox = 0; ox < outputWidth * _strideX; ox += _strideX)
{
int kxStartIndex = ox - _padX < 0 ? 0 : ox - _padX;
int kxLimit = _kWidth + ox - _padX < input.Shape[2] ? _kWidth + ox - _padX : input.Shape[2];
for (int ich = 0; ich < InputCount; ich++)
{
//For W index
int inChOffset = ich * _kHeight * _kWidth;
//For input index
int inputOffset = ich * input.Shape[1] * input.Shape[2];
for (int ky = kyStartIndex; ky < kyLimit; ky++)
{
for (int kx = kxStartIndex; kx < kxLimit; kx++)
{
int wIndex = outChOffset + inChOffset + (ky - oy + _padY) * _kWidth + kx - ox + _padX;
int inputIndex = inputOffset + ky * input.Shape[2] + kx + batchCounter * input.Length;
result[resultIndex] += input.Data[inputIndex] * Weight.Data[wIndex];
}
}
}
resultIndex++;
}
}
}
}
if (Activator != null && !NoBias)
{
for (int batchCounter = 0; batchCounter < input.BatchCount; batchCounter++)
{
int resultIndex = batchCounter * OutputCount * outputHeight * outputWidth;
for (int och = 0; och < OutputCount; och++)
{
for (int location = 0; location < outputHeight * outputWidth; location++)
{
result[resultIndex] += Bias.Data[och];
result[resultIndex] = Activator.ForwardActivate(result[resultIndex]);
resultIndex++;
}
}
}
}
else if (!NoBias)
{
for (int batchCounter = 0; batchCounter < input.BatchCount; batchCounter++)
{
int resultIndex = batchCounter * OutputCount * outputHeight * outputWidth;
for (int och = 0; och < OutputCount; och++)
{
for (int location = 0; location < outputHeight * outputWidth; location++)
{
result[resultIndex] += Bias.Data[och];
resultIndex++;
}
}
}
}
else if (Activator != null)
{
for (int batchCounter = 0; batchCounter < input.BatchCount; batchCounter++)
{
int resultIndex = batchCounter * OutputCount * outputHeight * outputWidth;
for (int och = 0; och < OutputCount; och++)
{
for (int location = 0; location < outputHeight * outputWidth; location++)
{
result[resultIndex] = Activator.ForwardActivate(result[resultIndex]);
resultIndex++;
}
}
}
}
return(NdArray.Convert(result, new[] { OutputCount, outputHeight, outputWidth }, input.BatchCount, this));
}
public static NdArray <Real> SingleInputForward(NdArray <Real> x, NdArray <Real> weight, NdArray <Real> bias, int strideX, int strideY, int padX, int padY, ICompressibleActivation <Real> activation, IFunction <Real> conv2d)
{
int outputCount = weight.Shape[0];
int inputCount = weight.Shape[1];
int kernelHeight = weight.Shape[2];
int kernelWidth = weight.Shape[3];
int outputHeight = (int)Math.Floor((x.Shape[1] - kernelHeight + padY * 2.0f) / strideY) + 1;
int outputWidth = (int)Math.Floor((x.Shape[2] - kernelWidth + padX * 2.0f) / strideX) + 1;
Real[] y = new Real[x.BatchCount * outputCount * outputHeight * outputWidth];
for (int batchCounter = 0; batchCounter < x.BatchCount; batchCounter++)
{
int yBatchOffset = batchCounter * outputCount * outputHeight * outputWidth;
int xBatchOffset = batchCounter * x.Length;
for (int och = 0; och < outputCount; och++)
{
int kOchOffset = och * inputCount * kernelHeight * kernelWidth;
int yChOffset = yBatchOffset + och * outputHeight * outputWidth;
for (int oy = 0; oy < outputHeight * strideY; oy += strideY)
{
int iyStart = oy - padY < 0 ? 0 : oy - padY;
int iyLimit = kernelHeight + oy - padY < x.Shape[1] ? kernelHeight + oy - padY : x.Shape[1];
for (int ox = 0; ox < outputWidth * strideX; ox += strideX)
{
int ixStart = ox - padX < 0 ? 0 : ox - padX;
int ixLimit = kernelWidth + ox - padX < x.Shape[2] ? kernelWidth + ox - padX : x.Shape[2];
int yIndex = yChOffset + oy / strideY * outputWidth + ox / strideX;
for (int ich = 0; ich < inputCount; ich++)
{
int kIchOffset = kOchOffset + ich * kernelHeight * kernelWidth;
int xChOffset = xBatchOffset + ich * x.Shape[1] * x.Shape[2];
for (int iy = iyStart; iy < iyLimit; iy++)
{
for (int ix = ixStart; ix < ixLimit; ix++)
{
int wIndex = kIchOffset + (iy - oy + padY) * kernelWidth + ix - ox + padX;
int xIndex = xChOffset + iy * x.Shape[2] + ix;
y[yIndex] += x.Data[xIndex] * weight.Data[wIndex];
}
}
}
}
}
}
}
if (activation != null && bias != null)
{
for (int batchCounter = 0; batchCounter < x.BatchCount; batchCounter++)
{
int resultIndex = batchCounter * outputCount * outputHeight * outputWidth;
for (int och = 0; och < outputCount; och++)
{
for (int location = 0; location < outputHeight * outputWidth; location++)
{
y[resultIndex] += bias.Data[och];
y[resultIndex] = activation.ForwardActivate(y[resultIndex]);
resultIndex++;
}
}
}
}
else if (bias != null)
{
for (int batchCounter = 0; batchCounter < x.BatchCount; batchCounter++)
{
int resultIndex = batchCounter * outputCount * outputHeight * outputWidth;
for (int och = 0; och < outputCount; och++)
{
for (int location = 0; location < outputHeight * outputWidth; location++)
{
y[resultIndex] += bias.Data[och];
resultIndex++;
}
}
}
}
else if (activation != null)
{
for (int i = 0; i < y.Length; i++)
{
y[i] = activation.ForwardActivate(y[i]);
}
}
return(NdArray.Convert(y, new[] { outputCount, outputHeight, outputWidth }, x.BatchCount, conv2d));
}
public static NdArray <Real> SingleInputForward(NdArray <Real> input, NdArray <Real> weight, NdArray <Real> bias, int strideX, int strideY, int padX, int padY, ICompressibleActivation <Real> activation, IFunction <Real> deconv2d)
{
int inputCount = weight.Shape[0];
int outputCount = weight.Shape[1];
int kernelHeight = weight.Shape[2];
int kernelWidth = weight.Shape[3];
int outputHeight = (input.Shape[1] - 1) * strideY + kernelHeight - padY * 2;
int outputWidth = (input.Shape[2] - 1) * strideX + kernelWidth - padX * 2;
Real[] result = new Real[input.BatchCount * outputCount * outputWidth * outputHeight];
int outSizeOffset = outputWidth * outputHeight;
int inputSizeOffset = input.Shape[1] * input.Shape[2];
int kSizeOffset = weight.Shape[2] * weight.Shape[3];
for (int batchCount = 0; batchCount < input.BatchCount; batchCount++)
{
for (int och = 0; och < outputCount; och++)
{
for (int oy = padY; oy < outputHeight + padY; oy++)
{
int iyLimit = oy / strideY + 1 < input.Shape[1] ? oy / strideY + 1 : input.Shape[1];
int iyStart = oy - weight.Shape[2] < 0 ? 0 : (oy - weight.Shape[2]) / strideY + 1;
for (int ox = padX; ox < outputWidth + padX; ox++)
{
int ixLimit = ox / strideX + 1 < input.Shape[2] ? ox / strideX + 1 : input.Shape[2];
int ixStart = ox - weight.Shape[3] < 0 ? 0 : (ox - weight.Shape[3]) / strideX + 1;
int outputIndex = batchCount * outputCount * outSizeOffset + och * outSizeOffset + (oy - padY) * outputWidth + ox - padX;
for (int ich = 0; ich < input.Shape[0]; ich++)
{
int inputIndexOffset = batchCount * input.Length + ich * inputSizeOffset;
int kernelIndexOffset = ich * weight.Shape[1] * kSizeOffset + och * kSizeOffset;
for (int iy = iyStart; iy < iyLimit; iy++)
{
for (int ix = ixStart; ix < ixLimit; ix++)
{
int inputIndex = inputIndexOffset + iy * input.Shape[2] + ix;
int kernelIndex = kernelIndexOffset + (oy - iy * strideY) * weight.Shape[3] + (ox - ix * strideX);
result[outputIndex] += input.Data[inputIndex] * weight.Data[kernelIndex];
}
}
}
}
}
}
}
if (activation != null && bias != null)
{
for (int batchCount = 0; batchCount < input.BatchCount; batchCount++)
{
for (int och = 0; och < outputCount; och++)
{
for (int oy = padY; oy < outputHeight + padY; oy++)
{
for (int ox = padX; ox < outputWidth + padX; ox++)
{
int outputIndex = batchCount * outputCount * outSizeOffset + och * outSizeOffset + (oy - padY) * outputWidth + ox - padX;
result[outputIndex] += bias.Data[och];
result[outputIndex] = activation.ForwardActivate(result[outputIndex]);
}
}
}
}
}
else if (bias != null)
{
for (int batchCount = 0; batchCount < input.BatchCount; batchCount++)
{
for (int och = 0; och < outputCount; och++)
{
for (int oy = padY; oy < outputHeight + padY; oy++)
{
for (int ox = padX; ox < outputWidth + padX; ox++)
{
int outputIndex = batchCount * outputCount * outSizeOffset + och * outSizeOffset + (oy - padY) * outputWidth + ox - padX;
result[outputIndex] += bias.Data[och];
}
}
}
}
}
else if (activation != null)
{
for (int batchCount = 0; batchCount < input.BatchCount; batchCount++)
{
for (int och = 0; och < outputCount; och++)
{
for (int oy = padY; oy < outputHeight + padY; oy++)
{
for (int ox = padX; ox < outputWidth + padX; ox++)
{
int outputIndex = batchCount * outputCount * outSizeOffset + och * outSizeOffset + (oy - padY) * outputWidth + ox - padX;
result[outputIndex] = activation.ForwardActivate(result[outputIndex]);
}
}
}
}
}
return(NdArray.Convert(result, new[] { outputCount, outputHeight, outputWidth }, input.BatchCount, deconv2d));
}
public static NdArray <Real>[] MultiInputForward(NdArray <Real>[] xs, EltwiseParameter.EltwiseOp operation, float[] coeffs, List <int[]> PrevOutputIndex, IFunction <Real> eltwise)
{
Real[] result = new Real[xs[0].Data.Length];
switch (operation)
{
case EltwiseParameter.EltwiseOp.Prod:
Array.Copy(xs[0].Data, result, result.Length);
for (int i = 1; i < xs.Length; i++)
{
for (int j = 0; j < result.Length; j++)
{
result[j] *= xs[i].Data[j];
}
}
break;
case EltwiseParameter.EltwiseOp.Sum:
if (coeffs != null)
{
for (int i = 0; i < xs.Length; i++)
{
for (int j = 0; j < result.Length; j++)
{
result[j] += xs[i].Data[j] * coeffs[i];
}
}
}
else
{
for (int i = 0; i < xs.Length; i++)
{
for (int j = 0; j < result.Length; j++)
{
result[j] += xs[i].Data[j];
}
}
}
break;
case EltwiseParameter.EltwiseOp.Max:
Array.Copy(xs[0].Data, result, result.Length);
int[] outputIndex = new int[result.Length];
for (int i = 1; i < xs.Length; i++)
{
for (int j = 0; j < result.Length; j++)
{
if (result[j] < xs[i].Data[j])
{
outputIndex[j] = i;
result[j] = xs[i].Data[j];
}
}
}
PrevOutputIndex.Add(outputIndex);
break;
}
return(new [] { NdArray.Convert(result, xs[0].Shape, xs[0].BatchCount, eltwise) });
}
private NdArray ForwardCpu([NotNull] NdArray x)
{
// Acquire parameters for calculation
if (IsTrain)
{
// Set Mean and Variance of member
Variance = new Real[ChannelSize];
for (int i = 0; i < Variance.Length; i++)
{
Variance[i] = 0;
}
Mean = new Real[ChannelSize];
for (int i = 0; i < Mean.Length; i++)
{
for (int index = 0; index < x.BatchCount; index++)
{
Mean[i] += x.Data[i + index * x.Length];
}
Mean[i] /= x.BatchCount;
}
for (int i = 0; i < Mean.Length; i++)
{
for (int index = 0; index < x.BatchCount; index++)
{
Variance[i] += (x.Data[i + index * x.Length] - Mean[i]) * (x.Data[i + index * x.Length] - Mean[i]);
}
Variance[i] /= x.BatchCount;
}
for (int i = 0; i < Variance.Length; i++)
{
Variance[i] += Eps;
}
}
else
{
Mean = AvgMean.Data;
Variance = AvgVar.Data;
}
Std = new Real[Variance.Length];
for (int i = 0; i < Variance.Length; i++)
{
Std[i] = Math.Sqrt(Variance[i]);
}
// Calculate result
Xhat = new Real[x.Data.Length];
Real[] y = new Real[x.Data.Length];
int dataSize = 1;
for (int i = 1; i < x.Shape.Length; i++)
{
dataSize *= x.Shape[i];
}
for (int batchCount = 0; batchCount < x.BatchCount; batchCount++)
{
for (int i = 0; i < ChannelSize; i++)
{
for (int location = 0; location < dataSize; location++)
{
int index = batchCount * ChannelSize * dataSize + i * dataSize + location;
Xhat[index] = (x.Data[index] - Mean[i]) / Std[i];
y[index] = Gamma.Data[i] * Xhat[index] + Beta.Data[i];
}
}
}
// Update parameters
if (IsTrain)
{
int m = x.BatchCount;
Real adjust = m / Math.Max(m - 1.0, 1.0); // unbiased estimation
if (Verbose)
{
RILogManager.Default?.ViewerSendWatch("Unbiased Estimation", adjust);
}
for (int i = 0; i < AvgMean.Data.Length; i++)
{
AvgMean.Data[i] *= Decay;
Mean[i] *= 1 - Decay; // reuse buffer as a temporary
AvgMean.Data[i] += Mean[i];
AvgVar.Data[i] *= Decay;
Variance[i] *= (1 - Decay) * adjust; // reuse buffer as a temporary
AvgVar.Data[i] += Variance[i];
}
}
return(NdArray.Convert(y, x.Shape, x.BatchCount, this));
}
public override NdArray SingleInputForward(NdArray x)
{
int outputHeight = (int)Math.Floor((x.Shape[1] - this.KernelHeight + this.PadY * 2.0) / this.StrideY) + 1;
int outputWidth = (int)Math.Floor((x.Shape[2] - this.KernelWidth + this.PadX * 2.0) / this.StrideX) + 1;
Real[] y = new Real[x.BatchCount * this.OutputCount * outputHeight * outputWidth];
for (int batchCounter = 0; batchCounter < x.BatchCount; batchCounter++)
{
int yBatchOffset = batchCounter * this.OutputCount * outputHeight * outputWidth;
int xBatchOffset = batchCounter * x.Length;
for (int och = 0; och < this.OutputCount; och++)
{
int kOchOffset = och * this.InputCount * this.KernelHeight * this.KernelWidth;
int yChOffset = yBatchOffset + och * outputHeight * outputWidth;
for (int oy = 0; oy < outputHeight * this.StrideY; oy += this.StrideY)
{
int iyStart = oy - this.PadY < 0 ? 0 : oy - this.PadY;
int iyLimit = this.KernelHeight + oy - this.PadY < x.Shape[1] ? this.KernelHeight + oy - this.PadY : x.Shape[1];
for (int ox = 0; ox < outputWidth * this.StrideX; ox += this.StrideX)
{
int ixStart = ox - this.PadX < 0 ? 0 : ox - this.PadX;
int ixLimit = this.KernelWidth + ox - this.PadX < x.Shape[2] ? this.KernelWidth + ox - this.PadX : x.Shape[2];
int yIndex = yChOffset + oy / this.StrideY * outputWidth + ox / this.StrideX;
for (int ich = 0; ich < this.InputCount; ich++)
{
int kIchOffset = kOchOffset + ich * this.KernelHeight * this.KernelWidth;
int xChOffset = xBatchOffset + ich * x.Shape[1] * x.Shape[2];
for (int iy = iyStart; iy < iyLimit; iy++)
{
for (int ix = ixStart; ix < ixLimit; ix++)
{
int wIndex = kIchOffset + (iy - oy + this.PadY) * this.KernelWidth + ix - ox + this.PadX;
int xIndex = xChOffset + iy * x.Shape[2] + ix;
y[yIndex] += x.Data[xIndex] * this.Weight.Data[wIndex];
}
}
}
}
}
}
}
if (this.Activation != null && !NoBias)
{
for (int batchCounter = 0; batchCounter < x.BatchCount; batchCounter++)
{
int resultIndex = batchCounter * this.OutputCount * outputHeight * outputWidth;
for (int och = 0; och < this.OutputCount; och++)
{
for (int location = 0; location < outputHeight * outputWidth; location++)
{
y[resultIndex] += this.Bias.Data[och];
y[resultIndex] = this.Activation.ForwardActivate(y[resultIndex]);
resultIndex++;
}
}
}
}
else if (!NoBias)
{
for (int batchCounter = 0; batchCounter < x.BatchCount; batchCounter++)
{
int resultIndex = batchCounter * this.OutputCount * outputHeight * outputWidth;
for (int och = 0; och < this.OutputCount; och++)
{
for (int location = 0; location < outputHeight * outputWidth; location++)
{
y[resultIndex] += this.Bias.Data[och];
resultIndex++;
}
}
}
}
else if (this.Activation != null)
{
for (int i = 0; i < y.Length; i++)
{
y[i] = this.Activation.ForwardActivate(y[i]);
}
}
return(NdArray.Convert(y, new[] { this.OutputCount, outputHeight, outputWidth }, x.BatchCount, this));
}
