public int CalcSubRegionHash(InternalArray ar, int sx, int sy, int ww, int hout, int wout)
{
int pointer = 0;
int hash = 0;
for (int ch = 0; ch < ar.Shape[0]; ch++)
{
var xx = sx * ww;
var yy = sy * ww;
pointer = xx + yy * ar.offsets[1];
for (int i2 = 0; i2 < ww; i2++)
{
for (int j2 = 0; j2 < ww; j2++)
{
hash ^= (int)(ar.Data[pointer] * 1000);
pointer++;
}
pointer += ar.offsets[1];
pointer -= ww;
}
pointer += ar.offsets[0];
}
return(hash);
}
public void Sub(InternalArray ar)
{
for (int i = 0; i < Data.Length; i++)
{
Data[i] -= ar.Data[i];
}
}
public void CalcRegionsHashes(InternalArray ar)
{
int ww = RegionSize;//region size
int rw = ar.Shape[1] / ww;
int rh = ar.Shape[2] / ww;
for (int i = 0; i < rw; i++)
{
for (int j = 0; j < rh; j++)
{
int pointer = 0;
pointer = 0;//zero channel
var xx = i * ww;
var yy = j * ww;
pointer = xx + yy * ar.offsets[1];
int hash = 0;
for (int i2 = 0; i2 < ww; i2++)
{
for (int j2 = 0; j2 < ww; j2++)
{
hash ^= (int)(ar.Data[pointer] * 1000);
pointer++;
}
pointer += ar.offsets[1];
pointer -= ww;
}
}
}
}
internal InternalArray Unsqueeze(int v)
{
InternalArray ret = new InternalArray(new int[] { 1, Shape[0], Shape[1], Shape[2] });
ret.Data = Data.ToArray();
return(ret);
}
public static InternalArray ParseFromString(string str)
{
List <int> dims = new List <int>();
int dim = 0;
int maxdim = 0;
StringBuilder sb = new StringBuilder();
List <int> cnt = new List <int>();
List <double> data = new List <double>();
str = str.Substring(str.IndexOf('['));
for (int i = 0; i < str.Length; i++)
{
if (str[i] == '[')
{
dim++; maxdim = Math.Max(dim, maxdim);
if (dims.Count < maxdim)
{
dims.Add(0);
cnt.Add(0);
}
cnt[dim - 1] = 1;
sb.Clear();
continue;
}
if (str[i] == ']')
{
if (dim == maxdim)
{
data.Add(double.Parse(sb.ToString()));
}
dims[dim - 1] = Math.Max(dims[dim - 1], cnt[dim - 1]);
dim--;
sb.Clear();
if (dim == 0)
{
break;
}
continue;
}
if (str[i] == ',')
{
if (dim == maxdim)
{
data.Add(double.Parse(sb.ToString()));
}
sb.Clear();
cnt[dim - 1]++;
continue;
}
sb.Append(str[i]);
}
InternalArray ret = new InternalArray(dims.ToArray());
ret.Data = data.ToArray();
return(ret);
}
public static InternalArray Get1DImageFrom3DArray(this InternalArray array, int ind1, int ind2)
{
var pos = ind1 * array.offsets[0] + ind2 * array.offsets[1];
InternalArray ret = new InternalArray(new int[] { array.Shape[2] });
Array.Copy(array.Data, pos, ret.Data, 0, ret.Data.Length);
return(ret);
}
public static InternalArray GetNext2dImageFrom4dArray(this InternalArray array, ref int pos0)
{
InternalArray ret = new InternalArray(new int[] { array.Shape[2], array.Shape[3] });
Array.Copy(array.Data, pos0, ret.Data, 0, ret.Data.Length);
pos0 += array.offsets[1];
return(ret);
}
public static InternalArray Get3DImageFrom4DArray(this InternalArray array, int ind1)
{
int pos = ind1 * array.offsets[0];
InternalArray ret = new InternalArray(new int[] { array.Shape[1], array.Shape[2], array.Shape[3] });
Array.Copy(array.Data, pos, ret.Data, 0, ret.Data.Length);
return(ret);
}
public void ProcessSubRegion(InternalArray ar, InternalArray[,] filters, InternalArray ret, int sx, int sy, int ww, int hout, int wout)
{
var hin = ar.Shape[1];
var win = ar.Shape[2];
var c = ar.Shape[0];
int shiftx = padding[0] - kernelSize[0] / 2;
int shifty = padding[1] - kernelSize[1] / 2;
var maxw = Math.Min(wout, (sy + 1) * ww);
var maxh = Math.Min(hout, (sx + 1) * ww);
var starth = sx * ww;
var startw = sy * ww;
for (int i = starth; i < maxh; i++)
{
for (int j = startw; j < maxw; j++)
{
for (int ch = 0; ch < outChannels; ch++)
{
double val = 0;
for (int zz = 0; zz < c; zz++)
{
var kernel = filters[ch, zz];
for (int i1 = 0; i1 < kernelSize[0]; i1++)
{
for (int j1 = 0; j1 < kernelSize[0]; j1++)
{
//var x = i * stride[0] - kernelSize[0] / 2 + i1;
//var y = j * stride[1] - kernelSize[1] / 2 + j1;
//outspace
var xout = (i) * stride[0] - kernelSize[0] / 2 + i1 * dilation[0];
var yout = (j) * stride[1] - kernelSize[1] / 2 + j1 * dilation[1];
//inspace
var xin = xout - shiftx;
var yin = yout - shifty;
if (!ar.WithIn(zz, xin, yin))
{
continue;
}
//var y=jmul+j1
var ii1 = i1 * kernel.Shape[1] + j1;
var ii2 = zz * ar.offsets[0] + xin * ar.offsets[1] + yin;
val += kernel.Get2D(i1, j1) * ar.Get3D(zz, xin, yin);
var ii3 = j + ch * ret.offsets[0] + i * ret.offsets[1];
}
}
}
ret.Set3D(ch, i, j, val);
}
}
}
}
public InternalArray Clone()
{
InternalArray ret = new InternalArray(Shape);
ret.Shape = (int[])ret.Shape.Clone();
ret.Data = new double[Data.Length];
Array.Copy(Data, ret.Data, Data.Length);
return(ret);
}
public void ProcessSubRegionOptimized(InternalArray ar, InternalArray[,] filters, InternalArray ret, int sx, int sy, int ww, int hout, int wout)
{
var hin = ar.Shape[1];
var win = ar.Shape[2];
var c = ar.Shape[0];
int shiftx = padding[0] - kernelSize[0] / 2;
int shifty = padding[1] - kernelSize[1] / 2;
var maxw = Math.Min(wout, (sy + 1) * ww);
var maxh = Math.Min(hout, (sx + 1) * ww);
var starth = sx * ww;
var startw = sy * ww;
for (int i = starth; i < maxh; i++)
{
var imul = (i) * stride[0] - kernelSize[0] / 2 - shiftx;
var maxi1 = Math.Min((ar.Shape[1] - imul) / dilation[0], kernelSize[0]);
var mini1 = Math.Max((int)Math.Ceiling(-(double)imul / dilation[0]), 0);
for (int j = startw; j < maxw; j++)
{
var jmul = (j) * stride[1] - kernelSize[1] / 2 - shifty;
var minj1 = Math.Max((int)Math.Ceiling(-(double)jmul / dilation[1]), 0);
var maxj1 = Math.Min((ar.Shape[2] - jmul) / dilation[1], kernelSize[1]);
for (int ch = 0; ch < outChannels; ch++)
{
double val = 0;
for (int zz = 0; zz < c; zz++)
{
var kernel = filters[ch, zz];
var offset1 = zz * ar.offsets[0];
int kindex = 0;
for (int i1 = mini1; i1 < maxi1; i1++)
{
var x = imul + i1 * dilation[0];
for (int j1 = minj1; j1 < maxj1; j1++)
{
var y = jmul + j1 * dilation[1];
var index = offset1 + x * ar.offsets[1] + y;
val += kernel.Data[kindex] * ar.Data[index];
kindex++;
}
}
}
ret.Set3D(ch, i, j, val);
}
}
}
}
public static InternalArray Randn(this Random r, int[] dims)
{
var ar = new InternalArray(dims);
for (int i = 0; i < ar.Data.Length; i++)
{
ar.Data[i] = r.NextGaussian();
}
return(ar);
}
public Conv2d(int inChannels, int outChannels, int kSize, int stride, int padding, bool bias = false, int dilation = 1)
{
this.inChannels = inChannels;
this.outChannels = outChannels;
Weight = new InternalArray(new int[] { outChannels, inChannels, kSize, kSize });
this.padding = new int[] { padding, padding };
this.stride = new[] { stride, stride };
this.kernelSize = new[] { kSize, kSize };
this.dilation = new[] { dilation, dilation };
}
public static InternalArray FromXml(string path)
{
XDocument doc = XDocument.Load(path);
var item = doc.Descendants("data").First().Value;
var sz = doc.Descendants("size").First().Value;
var dims = sz.Split(new char[] { ',', '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries).ToArray();
InternalArray ret = new InternalArray(dims.Select(int.Parse).ToArray());
var data = item.Split(new char[] { ',', '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries).ToArray();
ret.Data = data.Select(z => double.Parse(z.Replace(",", "."), CultureInfo.InvariantCulture)).ToArray();
return(ret);
}
public static InternalArray Unpad2d(InternalArray ar)
{
InternalArray ret = new InternalArray(new int[] { ar.Shape[0] - 2, ar.Shape[1] - 2 });
int pos = 0;
int target = ar.Shape[1];
for (int i = 0; i < ret.Shape[0]; i++)
{
//copy row
target++;
Array.Copy(ar.Data, target, ret.Data, pos, ret.Shape[1]);
target += ret.Shape[1];
pos += ret.Shape[1];
target++;
}
return(ret);
}
public static InternalArray[] ParallelProcess(NeuralItem[] items, InternalArray input)
{
InternalArray[] res = new InternalArray[items.Length];
if (!AllowParallelProcessing)
{
for (int i = 0; i < items.Length; i++)
{
res[i] = items[i].Forward(input);
}
}
else
{
Parallel.For(0, items.Length, (i) =>
{
res[i] = items[i].Forward(input);
});
}
return(res);
}
internal InternalArray Transpose(int[] v)
{
//3d only!
InternalArray ret = new InternalArray(v.Select(z => Shape[z]).ToArray());
for (int i = 0; i < Shape[0]; i++)
{
for (int j = 0; j < Shape[1]; j++)
{
for (int k = 0; k < Shape[2]; k++)
{
var val = Get3D(i, j, k);
var ar1 = new int[] { i, j, k };
ret.Set3D(ar1[v[0]], ar1[v[1]], ar1[v[2]], val);
}
}
}
return(ret);
}
/// <summary>
/// stride=1, kernel=1,padding=0
/// </summary>
/// <param name="ar"></param>
/// <param name="hout"></param>
/// <param name="wout"></param>
/// <param name="c"></param>
/// <param name="hin"></param>
/// <param name="win"></param>
/// <returns></returns>
public InternalArray ProcessImageOptimizedNoPaddingKernel1(InternalArray ar, int hout, int wout, int c, int hin, int win)
{
InternalArray ret = new InternalArray(new int[] { outChannels, hout, wout });
InternalArray[,] filters = new InternalArray[outChannels, c];
for (int ch = 0; ch < outChannels; ch++)
{
for (int zz = 0; zz < c; zz++)
{
var kernel = Weight.Get2DImageFrom4DArray(ch, zz);
filters[ch, zz] = kernel;
}
}
Parallel.For(0, hout, (i) =>
{
for (int j = 0; j < wout; j++)
{
var index2 = i * ret.offsets[1] + j;
for (int ch = 0; ch < outChannels; ch++)
{
double val = 0;
var index = i * ar.offsets[1] + j;
for (int zz = 0; zz < c; zz++)
{
var kernel = filters[ch, zz];
val += kernel.Data[0] * ar.Data[index];
index += ar.offsets[0];
}
//ret.Data[ch * ret.offsets[0] + i * ret.offsets[1] + j] = val;
ret.Data[index2] = val;
index2 += ret.offsets[0];
}
}
});
return(ret);
}
public bool IsEqual(InternalArray resss)
{
if (Shape.Length != resss.Shape.Length)
{
return(false);
}
for (int i = 0; i < Shape.Length; i++)
{
if (Shape[i] != resss.Shape[i])
{
return(false);
}
}
for (int i = 0; i < Data.Length; i++)
{
if (Math.Abs(Data[i] - resss.Data[i]) > tolerance)
{
return(false);
}
}
return(true);
}
public static InternalArray Cat(InternalArray[] items, int dim = 0)
{
var n = items[0].Shape[0];
var sumch = items.Sum(z => z.Shape[1]);
var h = items[0].Shape[2];
var w = items[0].Shape[3];
InternalArray ret = new InternalArray(new int[] { n, sumch, h, w });
List <double> data = new List <double>();
foreach (var item in items)
{
for (int i = 0; i < n; i++)
{
var img = Get3DImageFrom4DArray(item, i);
data.AddRange(img.Data);
}
}
ret.Data = data.ToArray();
return(ret);
}
public static InternalArray ParseFromString2(string str)
{
var ar = str.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries).ToArray();
var a1 = ar[0].Split(new char[] { ' ', ',', '(', ')' }, StringSplitOptions.RemoveEmptyEntries).ToArray();
var dims = a1.Select(int.Parse).ToArray();
InternalArray ret = new InternalArray(dims.ToArray());
foreach (var ss in ar.Skip(1))
{
var s1 = ss.Split(new char[] { ',', '(', ')' }, StringSplitOptions.RemoveEmptyEntries).ToArray();
int[] index = new int[dims.Length];
for (int i = 0; i < dims.Length; i++)
{
index[i] = int.Parse(s1[i]);
}
var val = double.Parse(s1.Last());
ret.Set4D(index[0], index[1], index[2], index[3], val);
}
return(ret);
}
public static InternalArray Pad3d(InternalArray ar, int padCnt = 1)
{
InternalArray ret = new InternalArray(new int[] { ar.Shape[0], ar.Shape[1] + padCnt * 2, ar.Shape[2] + padCnt * 2 });
int pos = 0;
int target = 0;
for (int j = 0; j < ar.Shape[0]; j++)
{
target += ret.Shape[1] * padCnt;
for (int i = 0; i < ar.Shape[1]; i++)
{
//copy row
target += padCnt;
Array.Copy(ar.Data, pos, ret.Data, target, ar.Shape[2]);
target += ar.Shape[2];
pos += ar.Shape[2];
target += padCnt;
}
target += ret.Shape[1] * padCnt;
}
return(ret);
}
public override InternalArray Forward(InternalArray ar)
{
#if PROFILER
LogItem = new CalcLogItem(this, "avgPool2D");
var sw = Stopwatch.StartNew();
#endif
var hin = ar.Shape[2];
var win = ar.Shape[3];
var n = ar.Shape[0];
var c = ar.Shape[1];
var hout = ((hin + 2 * padding[0] - kernelSize[0]) / stride[0]) + 1;
var wout = ((win + 2 * padding[1] - kernelSize[1]) / stride[1]) + 1;
InternalArray ret = new InternalArray(new int[] { n, c, hout, wout });
//get all 2d images
//append them together to [n,c]
List <double> data = new List <double>();
for (int i = 0; i < n; i++)
{
for (int j = 0; j < c; j++)
{
var img = Helpers.Get2DImageFrom4DArray(ar, i, j);
var img2 = Process2DImage(img);
data.AddRange(img2.Data);
}
}
ret.Data = data.ToArray();
#if PROFILER
sw.Stop();
Profiler.AddLog(LogItem, this, "exec", sw.ElapsedMilliseconds, true);
#endif
return(ret);
}
public InternalArray Process2DImage(InternalArray ar)
{
var hin = ar.Shape[0];
var win = ar.Shape[1];
var hout = ((hin + 2 * padding[0] - kernelSize[0]) / stride[0]) + 1;
var wout = ((win + 2 * padding[1] - kernelSize[1]) / stride[1]) + 1;
InternalArray ret = new InternalArray(new int[] { hout, wout });
for (int i = 0; i < hout; i++)
{
for (int j = 0; j < wout; j++)
{
double avg = 0;
for (int i1 = 0; i1 < kernelSize[0]; i1++)
{
for (int j1 = 0; j1 < kernelSize[1]; j1++)
{
var x = i * stride[0] + i1 - kernelSize[0] / 2;
var y = j * stride[1] + j1 - kernelSize[1] / 2;
if (ar.WithIn(x, y))
{
avg += ar.Get2D(x, y);
}
}
}
avg /= kernelSize[0] * kernelSize[1];
ret.Set2D(i, j, avg);
}
}
return(ret);
}
public override InternalArray Forward(InternalArray ar)
{
#if PROFILER
LogItem = new CalcLogItem(this, "batchNorm2d");
var sw = Stopwatch.StartNew();
#endif
InternalArray ret = new InternalArray(ar.Shape);
var n = ar.Shape[0];
var c = ar.Shape[1];
List <double> data = new List <double>();
for (int i = 0; i < n; i++)
{
for (int j = 0; j < c; j++)
{
var img = Helpers.Get2DImageFrom4DArray(ar, i, j);
for (int zi = 0; zi < img.Data.Length; zi++)
{
img.Data[zi] = ((img.Data[zi] - RunningMean.Data[j]) / Math.Sqrt(RunningVar.Data[j] + eps)) * Weight.Data[j] + Bias.Data[j];
}
data.AddRange(img.Data);
}
}
ret.Data = data.ToArray();
#if PROFILER
sw.Stop();
if (Parent != null)
{
Profiler.AddLog(Parent.LogItem, LogItem);
}
#endif
return(ret);
}
public override InternalArray Forward(InternalArray ar1)
{
InternalArray ar = ar1.Clone();
var n = ar1.Shape[0];
var c = ar1.Shape[1];
List <double> data = new List <double>();
int pos0 = 0;
for (int i = 0; i < n; i++)
{
for (int j = 0; j < c; j++)
{
var img = ar.GetNext2dImageFrom4dArray(ref pos0);
for (int z = 0; z < img.Data.Length; z++)
{
img.Data[z] = img.Data[z] < 0 ? (img.Data[z] * Weight.Data[j]) : img.Data[z];
}
data.AddRange(img.Data);
}
}
ar.Data = data.ToArray();
return(ar);
}
public static InternalArray Add(InternalArray a1, InternalArray a2)
{
if (a1.Shape.Length != a2.Shape.Length)
{
throw new ArgumentException("dim1.len!=dim2.len");
}
int[] outs = new int[a1.Shape.Length];
bool rightBigger = false;
for (int i = 0; i < a1.Shape.Length; i++)
{
outs[i] = Math.Max(a1.Shape[i], a2.Shape[i]);
if (a2.Shape[i] > a1.Shape[i])
{
rightBigger = true;
}
}
if (rightBigger)
{
var temp = a2;
a2 = a1;
a1 = temp;
}
InternalArray ret = new InternalArray(outs);
if (outs.Length == 4)
{
//iterate over all values of bigger one and add smallest one.
int index = 0;
int index2 = 0;
for (int i = 0; i < a1.Shape[0]; i++)
{
if (!a2.WithIn(i, 0, 0, 0))
{
continue;
}
for (int i1 = 0; i1 < a1.Shape[1]; i1++)
{
if (!a2.WithIn(i, i1, 0, 0))
{
continue;
}
for (int i2 = 0; i2 < a1.Shape[2]; i2++)
{
if (!a2.WithIn(i, i1, i2, 0))
{
continue;
}
for (int i3 = 0; i3 < a1.Shape[3]; i3++)
{
if (!a2.WithIn(i, i1, i2, i3))
{
continue;
}
//a1.Data[index++] = a2.Get4D(i, i1, i2, i3);
ret.Data[index] = a1.Data[index] + a2.Data[index2];
index++;
index2++;
}
}
}
}
}
else
{
throw new NotImplementedException();
}
return(ret);
}
public PReLU(int nOut)
{
Weight = new InternalArray(new[] { nOut });
}
public override int SetData(List <InternalArray> arrays)
{
Weight = arrays[0];
return(1);
}
public override InternalArray Forward(InternalArray ar)
{
//Profiler.PushCurrent(new CalcLogItem(this, "conv2d"));
#if PROFILER
LogItem = new CalcLogItem(this, "conv2d");
Profiler.AddLog(LogItem);
var sw = Stopwatch.StartNew();
#endif
var hin = ar.Shape[2];
var win = ar.Shape[3];
var n = ar.Shape[0];
var c = ar.Shape[1];
var hout = ((hin + 2 * padding[0] - dilation[0] * (kernelSize[0] - 1) - 1) / stride[0]) + 1;
var wout = ((win + 2 * padding[1] - dilation[1] * (kernelSize[1] - 1) - 1) / stride[1]) + 1;
var cout = Weight.Shape[0];
InternalArray ret = new InternalArray(new int[] { n, cout, hout, wout });
//get all 3d images
int pos = 0;
int pos0 = 0;
for (int i = 0; i < n; i++)
{
var img = ar.GetNext3DImageFrom4DArray(ref pos0);
InternalArray img2 = null;
if (AllowOptimized)
{
if (AllowOptimizedViaDot)
{
img2 = ProcessImageViaDot(img);
#if PROFILER
method = "dot";
#endif
}
else
{
/*if (dilation[0] == 1 && kernelSize[0] == 3 && padding[0] > 0)
* {
#if PROFILER
*
* method = "+pad";
#endif
* img2 = ProcessImageOptimized2(img, hout, wout, c, hin, win);
* var img4 = Helpers.Pad3d(img, padding[0]);
* var img3 = ProcessImageOptimizedNoPadding(img4, hout + padding[0] * 2, wout + padding[0] * 2, c, img.Shape[1], img.Shape[2]);
* var img5 = Helpers.Unpad3d(img, padding[0]);
*
* }
* else*/
{
if (padding[0] == 0 && dilation[1] == 1 && kernelSize[0] == 1 && stride[0] == 1)
{
#if PROFILER
method = "nopad.k1";
#endif
img2 = ProcessImageOptimizedNoPaddingKernel1(img, hout, wout, c, hin, win);
/*var img3 = ProcessImageOptimized2(img, hout, wout, c, hin, win);
* if (!img2.IsEqual(img3))
* {
* throw new Exception("11");
* }*/
}
else
if (padding[0] == 0 && dilation[0] == 1 && kernelSize[0] == 3)
{
#if PROFILER
method = "no pad";
//Helpers.Pad2d()
#endif
img2 = ProcessImageOptimizedNoPadding(img, hout, wout, c, hin, win);
/*var img3 = ProcessImageOptimized2(img, hout, wout, c, hin, win);
* if (!img2.IsEqual(img3))
* {
* throw new Exception("11");
* }
*/
}
else
{
#if PROFILER
method = "opt2";
#endif
img2 = ProcessImageOptimized2(img, hout, wout, c, hin, win);
}
}
}
}
else
{
#if PROFILER
method = "native";
#endif
//img2 = ProcessImage(img);
img2 = ProcessImageBySubRegions(img);
/*var res1 = ProcessImageBySubRegions(img);
* if (!img2.IsEqual(res1))
* {
* throw new Exception("11");
* }*/
}
Array.Copy(img2.Data, 0, ret.Data, pos, img2.Data.Length);
pos += img2.Data.Length;
}
//sw.Stop();
// LastMs = sw.ElapsedMilliseconds;
#if PROFILER
Profiler.AddLog(LogItem, this, method + "; hin: " + hin + "; win: ;" + win + "; padding: " + padding[0] + "; dilation: " + dilation[0] + "; ksize: " + kernelSize[0] + "; stride: " + stride[0] + "; nIn: " + inChannels + "; nOut: " + outChannels, LastMs);
Profiler.AddLog(LogItem, this, $"ar: {ar}; w: {Weight}");
Profiler.AddLog(LogItem, this, "out shape: " + ret.Shape.Aggregate("", (x, y) => x + y + "; "));
//Profiler.PopCurrent();
if (Parent != null)
{
Profiler.AddLog(Parent.LogItem, LogItem);
}
#endif
return(ret);
}