public static void Load2D4D(string filename, ManagedArray A, int i, int j, char delimiter = ',')
{
if (File.Exists(filename))
{
var temp = new ManagedArray(A.x, A.y);
var lines = File.ReadAllLines(filename);
for (int y = 0; y < A.y; y++)
{
if (y < lines.Length)
{
var tokens = lines[y].Split(delimiter);
for (int x = 0; x < A.x; x++)
{
temp[x, y] = Convert.ToDouble(tokens[x], ci);
}
}
}
ManagedOps.Copy2D4DIJ(A, temp, i, j);
ManagedOps.Free(temp);
}
}
public static ManagedArray Set(double[,,,] A)
{
var ii = A.GetLength(0);
var jj = A.GetLength(1);
var yy = A.GetLength(2);
var xx = A.GetLength(3);
var model = new ManagedArray(xx, yy, 1, ii, jj);
var temp = new ManagedArray(xx, yy);
for (var i = 0; i < ii; i++)
{
for (var j = 0; j < jj; j++)
{
for (var y = 0; y < yy; y++)
{
for (var x = 0; x < xx; x++)
{
temp[x, y] = A[i, j, y, x];
}
}
ManagedOps.Copy2D4DIJ(model, temp, i, j);
}
}
ManagedOps.Free(temp);
return(model);
}
public void Setup(ManagedArray input, int classes)
{
var random = new Random(Guid.NewGuid().GetHashCode());
var InputMaps = 1;
var MapSizeX = input.x;
var MapSizeY = input.y;
for (var l = 0; l < Layers.Count; l++)
{
if (Layers[l].Type == LayerTypes.Subsampling)
{
MapSizeX = MapSizeX / Layers[l].Scale;
MapSizeY = MapSizeY / Layers[l].Scale;
}
if (Layers[l].Type == LayerTypes.Convolution)
{
MapSizeX = MapSizeX - Layers[l].KernelSize + 1;
MapSizeY = MapSizeY - Layers[l].KernelSize + 1;
Layers[l].FeatureMap = new ManagedArray(Layers[l].KernelSize, Layers[l].KernelSize, 1, InputMaps, Layers[l].OutputMaps);
var fan_out = Layers[l].OutputMaps * Layers[l].KernelSize * Layers[l].KernelSize;
for (var j = 0; j < Layers[l].OutputMaps; j++)
{
var fan_in = InputMaps * Layers[l].KernelSize * Layers[l].KernelSize;
for (var i = 0; i < InputMaps; i++)
{
var rand = new ManagedArray(Layers[l].KernelSize, Layers[l].KernelSize);
Rand(rand, random, fan_in, fan_out);
ManagedOps.Copy2D4DIJ(Layers[l].FeatureMap, rand, i, j);
ManagedOps.Free(rand);
}
}
Layers[l].Bias = new ManagedArray(Layers[l].OutputMaps);
InputMaps = Layers[l].OutputMaps;
}
}
// 'onum' is the number of labels, that's why it is calculated using size(y, 1). If you have 20 labels so the output of the network will be 20 neurons.
// 'fvnum' is the number of output neurons at the last layer, the layer just before the output layer.
// 'Bias' is the biases of the output neurons.
// 'Weights' is the weights between the last layer and the output neurons. Note that the last layer is fully connected to the output layer, that's why the size of the weights is (onum * fvnum)
var fvnum = MapSizeX * MapSizeY * InputMaps;
var onum = classes;
Bias = new ManagedArray(1, onum);
Weights = new ManagedArray(fvnum, onum);
Rand(Weights, random, fvnum, onum);
}
// Update Network Weights based on computed errors
public void BackPropagation(ManagedArray batch)
{
var n = Layers.Count;
var last = n - 1;
var batchz = Layers[last].Activation.z;
// backprop deltas
ManagedOps.Free(OutputDelta, OutputError);
OutputDelta = new ManagedArray(Output, false);
OutputError = new ManagedArray(Output, false);
for (var x = 0; x < Output.Length(); x++)
{
// error
OutputError[x] = Output[x] - batch[x];
// output delta
OutputDelta[x] = OutputError[x] * (Output[x] * (1 - Output[x]));
}
// Loss Function
L = 0.5 * ManagedMatrix.SquareSum(OutputError) / batch.x;
ManagedOps.Free(WeightsTransposed, FeatureVectorDelta);
FeatureVectorDelta = new ManagedArray(FeatureVector, false);
WeightsTransposed = new ManagedArray(Weights, false);
// feature vector delta
ManagedMatrix.Transpose(WeightsTransposed, Weights);
ManagedMatrix.Multiply(FeatureVectorDelta, WeightsTransposed, OutputDelta);
// only conv layers has sigm function
if (Layers[last].Type == LayerTypes.Convolution)
{
for (var x = 0; x < FeatureVectorDelta.Length(); x++)
{
FeatureVectorDelta[x] = FeatureVectorDelta[x] * FeatureVector[x] * (1 - FeatureVector[x]);
}
}
// reshape feature vector deltas into output map style
var MapSize = Layers[last].Activation.x * Layers[last].Activation.y;
var temp1D = new ManagedArray(1, MapSize, false);
var temp2D = new ManagedArray(Layers[last].Activation.x, Layers[last].Activation.y, false);
ManagedOps.Free(Layers[last].Delta);
Layers[last].Delta = new ManagedArray(Layers[last].Activation, false);
for (var j = 0; j < Layers[last].Activation.i; j++)
{
for (var ii = 0; ii < batchz; ii++)
{
ManagedOps.Copy2D(temp1D, FeatureVectorDelta, ii, j * MapSize);
temp1D.Reshape(Layers[last].Activation.x, Layers[last].Activation.y);
ManagedMatrix.Transpose(temp2D, temp1D);
ManagedOps.Copy2D4D(Layers[last].Delta, temp2D, ii, j);
temp1D.Reshape(1, MapSize);
}
}
ManagedOps.Free(temp1D, temp2D);
for (var l = n - 2; l >= 0; l--)
{
var next = l + 1;
if (Layers[l].Type == LayerTypes.Convolution)
{
ManagedOps.Free(Layers[l].Delta);
Layers[l].Delta = new ManagedArray(Layers[l].Activation, false);
var xx = Layers[next].Scale * Layers[next].Activation.x;
var yy = Layers[next].Scale * Layers[next].Activation.y;
var FeatureMap = new ManagedArray(Layers[next].Activation.x, Layers[next].Activation.y, false);
var FeatureMapExpanded = new ManagedArray(xx, yy, false);
var Activation = new ManagedArray(xx, yy, false);
var Delta = new ManagedArray(xx, yy, false);
var Scale = (1.0 / (Layers[next].Scale * Layers[next].Scale));
for (var j = 0; j < Layers[l].Activation.i; j++)
{
for (var z = 0; z < batchz; z++)
{
ManagedOps.Copy4D2D(FeatureMap, Layers[next].Delta, z, j);
ManagedMatrix.Expand(FeatureMap, Layers[next].Scale, Layers[next].Scale, FeatureMapExpanded);
ManagedOps.Copy4D2D(Activation, Layers[l].Activation, z, j);
for (var x = 0; x < Delta.Length(); x++)
{
Delta[x] = Activation[x] * (1 - Activation[x]) * FeatureMapExpanded[x] * Scale;
}
ManagedOps.Copy2D4D(Layers[l].Delta, Delta, z, j);
}
}
ManagedOps.Free(FeatureMap, FeatureMapExpanded, Activation, Delta);
}
else if (Layers[l].Type == LayerTypes.Subsampling)
{
ManagedOps.Free(Layers[l].Delta);
Layers[l].Delta = new ManagedArray(Layers[l].Activation, false);
var Delta = new ManagedArray(Layers[next].Activation.x, Layers[next].Activation.y, batchz);
var FeatureMap = new ManagedArray(Layers[next].KernelSize, Layers[next].KernelSize, false);
var rot180 = new ManagedArray(Layers[next].KernelSize, Layers[next].KernelSize, false);
var z = new ManagedArray(Layers[l].Activation.x, Layers[l].Activation.y, batchz);
var ztemp = new ManagedArray(Layers[l].Activation.x, Layers[l].Activation.y, batchz, false);
for (var i = 0; i < Layers[l].Activation.i; i++)
{
ManagedOps.Set(z, 0.0);
for (var j = 0; j < Layers[next].Activation.i; j++)
{
ManagedOps.Copy4DIJ2D(FeatureMap, Layers[next].FeatureMap, i, j);
ManagedMatrix.Rotate180(rot180, FeatureMap);
ManagedOps.Copy4D3D(Delta, Layers[next].Delta, j);
ManagedConvolution.Full(Delta, rot180, ztemp);
ManagedMatrix.Add(z, ztemp);
}
ManagedOps.Copy3D4D(Layers[l].Delta, z, i);
}
ManagedOps.Free(Delta, FeatureMap, rot180, z, ztemp);
}
}
// calc gradients
for (var l = 1; l < n; l++)
{
var prev = l - 1;
if (Layers[l].Type == LayerTypes.Convolution)
{
ManagedOps.Free(Layers[l].DeltaFeatureMap, Layers[l].DeltaBias);
Layers[l].DeltaFeatureMap = new ManagedArray(Layers[l].FeatureMap, false);
Layers[l].DeltaBias = new ManagedArray(Layers[l].OutputMaps, false);
var FeatureMapDelta = new ManagedArray(Layers[l].FeatureMap.x, Layers[l].FeatureMap.y, Layers[l].FeatureMap.z, false);
// d[j]
var dtemp = new ManagedArray(Layers[l].Activation.x, Layers[l].Activation.y, batchz, false);
// a[i] and flipped
var atemp = new ManagedArray(Layers[prev].Activation.x, Layers[prev].Activation.y, batchz, false);
var ftemp = new ManagedArray(Layers[prev].Activation.x, Layers[prev].Activation.y, batchz, false);
for (var j = 0; j < Layers[l].Activation.i; j++)
{
ManagedOps.Copy4D3D(dtemp, Layers[l].Delta, j);
for (var i = 0; i < Layers[prev].Activation.i; i++)
{
ManagedOps.Copy4D3D(atemp, Layers[prev].Activation, i);
ManagedMatrix.FlipAll(ftemp, atemp);
ManagedConvolution.Valid(ftemp, dtemp, FeatureMapDelta);
ManagedMatrix.Multiply(FeatureMapDelta, 1.0 / batchz);
ManagedOps.Copy2D4DIJ(Layers[l].DeltaFeatureMap, FeatureMapDelta, i, j);
}
Layers[l].DeltaBias[j] = ManagedMatrix.Sum(dtemp) / batchz;
}
ManagedOps.Free(FeatureMapDelta, dtemp, atemp, ftemp);
}
}
var FeatureVectorTransposed = new ManagedArray(FeatureVector, false);
ManagedMatrix.Transpose(FeatureVectorTransposed, FeatureVector);
ManagedOps.Free(WeightsDelta, BiasDelta);
WeightsDelta = new ManagedArray(Weights, false);
BiasDelta = new ManagedArray(Bias, false);
ManagedMatrix.Multiply(WeightsDelta, OutputDelta, FeatureVectorTransposed);
ManagedMatrix.Multiply(WeightsDelta, 1.0 / batchz);
ManagedMatrix.Mean(BiasDelta, OutputDelta, 0);
ManagedOps.Free(FeatureVectorTransposed);
}
