// Matrix * Constant Multiplication public static void Multiply(ManagedArray A, double B) { for (var x = 0; x < A.Length(); x++) { A[x] = A[x] * B; } }
public void Rand(ManagedArray rand, Random random) { for (var x = 0; x < rand.Length(); x++) { rand[x] = (random.NextDouble() - 0.5) * 2.0; } }
// Matrix Addition with Scaling public static void Add(ManagedArray A, ManagedArray B, double Scale = 1) { for (var x = 0; x < A.Length(); x++) { A[x] = A[x] + Scale * B[x]; } }
public void Rand(ManagedArray rand, Random random, int fan_in, int fan_out) { for (var x = 0; x < rand.Length(); x++) { rand[x] = (random.NextDouble() - 0.5) * 2.0 * Math.Sqrt(6.0 / (fan_in + fan_out)); } }
// Element by element multiplication public static void Product(ManagedArray result, ManagedArray A, ManagedArray B) { for (var x = 0; x < A.Length(); x++) { result[x] = A[x] * B[x]; } }
// Matrix + Constant Addition public static void Add(ManagedArray A, double B) { for (var x = 0; x < A.Length(); x++) { A[x] = A[x] + B; } }
// Backward propagation public void BackPropagation(ManagedArray input) { var last = Weights.GetLength(0) - 1; D[0] = ManagedMatrix.Diff(Y, Y_true); var current = 1; for (var layer = last - 1; layer >= 0; layer--) { var prev = current - 1; var W = new ManagedArray(Weights[layer + 1].x - 1, Weights[layer + 1].y, false); var DZ = ManagedMatrix.DSigm(Z[layer]); D[current] = (new ManagedArray(W.x, D[prev].y, false)); ManagedOps.Copy2D(W, Weights[layer + 1], 1, 0); ManagedMatrix.Multiply(D[current], D[prev], W); ManagedMatrix.Product(D[current], DZ); ManagedOps.Free(W, DZ); current++; } for (var layer = 0; layer < Weights.GetLength(0); layer++) { var tD = ManagedMatrix.Transpose(D[Weights.GetLength(0) - layer - 1]); Deltas[layer] = (new ManagedArray(Weights[layer].x, Weights[layer].y, false)); ManagedMatrix.Multiply(Deltas[layer], tD, X[layer]); ManagedMatrix.Multiply(Deltas[layer], 1.0 / input.y); ManagedOps.Free(tD); } Cost = 0.0; L2 = 0.0; for (var i = 0; i < Y_true.Length(); i++) { L2 += 0.5 * (D[0][i] * D[0][i]); Cost += (-Y_true[i] * Math.Log(Y[i]) - (1 - Y_true[i]) * Math.Log(1 - Y[i])); } Cost /= input.y; L2 /= input.y; // Cleanup for (var layer = 0; layer < Weights.GetLength(0); layer++) { ManagedOps.Free(D[layer], X[layer], Z[layer]); } }
public static void Save1DY(string filename, ManagedArray A) { using (var file = new StreamWriter(filename, false)) { for (int y = 0; y < A.Length(); y++) { file.WriteLine(A[y].ToString(ci)); } } }
// Get element per element difference between arrays public static ManagedArray Diff(ManagedArray A, ManagedArray B) { var result = new ManagedArray(A, false); for (int x = 0; x < A.Length(); x++) { result[x] = A[x] - B[x]; } return(result); }
// get sum of squares of each element public static double SquareSum(ManagedArray A) { var sum = 0.0; for (int x = 0; x < A.Length(); x++) { sum += A[x] * A[x]; } return(sum); }
public static ManagedArray Pow(double A, ManagedArray powers) { var result = new ManagedArray(powers); for (var i = 0; i < powers.Length(); i++) { result[i] = Math.Pow(A, powers[i]); } return(result); }
public static ManagedArray Pow(ManagedArray A, double power) { var result = new ManagedArray(A); for (var x = 0; x < A.Length(); x++) { result[x] = Math.Pow(A[x], power); } return(result); }
// Element by element addition public static ManagedArray BSXADD(ManagedArray A, ManagedArray B) { var result = new ManagedArray(A); for (var x = 0; x < A.Length(); x++) { result[x] = A[x] + B[x]; } return(result); }
// Apply sigmoid function to matrix public static ManagedArray Sigm(ManagedArray A) { var result = new ManagedArray(A, false); for (int x = 0; x < A.Length(); x++) { result[x] = Sigmoid(A[x]); } return(result); }
public static void Load1DY(string filename, ManagedArray A) { if (File.Exists(filename)) { var lines = File.ReadAllLines(filename); for (int y = 0; y < A.Length(); y++) { A[y] = Convert.ToDouble(lines[y], ci); } } }
// Apply delta sigmoid function to matrix public static ManagedArray DSigm(ManagedArray A) { var result = new ManagedArray(A, false); for (var x = 0; x < A.Length(); x++) { var sigmoid = Sigmoid(A[x]); result[x] = sigmoid * (1.0 - sigmoid); } return(result); }
public static void Load1D(string filename, ManagedArray A, char delimiter = ',') { if (File.Exists(filename)) { var lines = File.ReadAllLines(filename); if (lines.Length > 0) { var tokens = lines[0].Split(delimiter); for (int x = 0; x < A.Length(); x++) { A[x] = Convert.ToDouble(tokens[x], ci); } } } }
// Update Network Weights based on computed errors public void BackPropagation(ManagedArray batch) { var n = Layers.Count; var last = n - 1; var batchz = Layers[last].Activation.z; // backprop deltas ManagedOps.Free(OutputDelta, OutputError); OutputDelta = new ManagedArray(Output, false); OutputError = new ManagedArray(Output, false); for (var x = 0; x < Output.Length(); x++) { // error OutputError[x] = Output[x] - batch[x]; // output delta OutputDelta[x] = OutputError[x] * (Output[x] * (1 - Output[x])); } // Loss Function L = 0.5 * ManagedMatrix.SquareSum(OutputError) / batch.x; ManagedOps.Free(WeightsTransposed, FeatureVectorDelta); FeatureVectorDelta = new ManagedArray(FeatureVector, false); WeightsTransposed = new ManagedArray(Weights, false); // feature vector delta ManagedMatrix.Transpose(WeightsTransposed, Weights); ManagedMatrix.Multiply(FeatureVectorDelta, WeightsTransposed, OutputDelta); // only conv layers has sigm function if (Layers[last].Type == LayerTypes.Convolution) { for (var x = 0; x < FeatureVectorDelta.Length(); x++) { FeatureVectorDelta[x] = FeatureVectorDelta[x] * FeatureVector[x] * (1 - FeatureVector[x]); } } // reshape feature vector deltas into output map style var MapSize = Layers[last].Activation.x * Layers[last].Activation.y; var temp1D = new ManagedArray(1, MapSize, false); var temp2D = new ManagedArray(Layers[last].Activation.x, Layers[last].Activation.y, false); ManagedOps.Free(Layers[last].Delta); Layers[last].Delta = new ManagedArray(Layers[last].Activation, false); for (var j = 0; j < Layers[last].Activation.i; j++) { for (var ii = 0; ii < batchz; ii++) { ManagedOps.Copy2D(temp1D, FeatureVectorDelta, ii, j * MapSize); temp1D.Reshape(Layers[last].Activation.x, Layers[last].Activation.y); ManagedMatrix.Transpose(temp2D, temp1D); ManagedOps.Copy2D4D(Layers[last].Delta, temp2D, ii, j); temp1D.Reshape(1, MapSize); } } ManagedOps.Free(temp1D, temp2D); for (var l = n - 2; l >= 0; l--) { var next = l + 1; if (Layers[l].Type == LayerTypes.Convolution) { ManagedOps.Free(Layers[l].Delta); Layers[l].Delta = new ManagedArray(Layers[l].Activation, false); var xx = Layers[next].Scale * Layers[next].Activation.x; var yy = Layers[next].Scale * Layers[next].Activation.y; var FeatureMap = new ManagedArray(Layers[next].Activation.x, Layers[next].Activation.y, false); var FeatureMapExpanded = new ManagedArray(xx, yy, false); var Activation = new ManagedArray(xx, yy, false); var Delta = new ManagedArray(xx, yy, false); var Scale = (1.0 / (Layers[next].Scale * Layers[next].Scale)); for (var j = 0; j < Layers[l].Activation.i; j++) { for (var z = 0; z < batchz; z++) { ManagedOps.Copy4D2D(FeatureMap, Layers[next].Delta, z, j); ManagedMatrix.Expand(FeatureMap, Layers[next].Scale, Layers[next].Scale, FeatureMapExpanded); ManagedOps.Copy4D2D(Activation, Layers[l].Activation, z, j); for (var x = 0; x < Delta.Length(); x++) { Delta[x] = Activation[x] * (1 - Activation[x]) * FeatureMapExpanded[x] * Scale; } ManagedOps.Copy2D4D(Layers[l].Delta, Delta, z, j); } } ManagedOps.Free(FeatureMap, FeatureMapExpanded, Activation, Delta); } else if (Layers[l].Type == LayerTypes.Subsampling) { ManagedOps.Free(Layers[l].Delta); Layers[l].Delta = new ManagedArray(Layers[l].Activation, false); var Delta = new ManagedArray(Layers[next].Activation.x, Layers[next].Activation.y, batchz); var FeatureMap = new ManagedArray(Layers[next].KernelSize, Layers[next].KernelSize, false); var rot180 = new ManagedArray(Layers[next].KernelSize, Layers[next].KernelSize, false); var z = new ManagedArray(Layers[l].Activation.x, Layers[l].Activation.y, batchz); var ztemp = new ManagedArray(Layers[l].Activation.x, Layers[l].Activation.y, batchz, false); for (var i = 0; i < Layers[l].Activation.i; i++) { ManagedOps.Set(z, 0.0); for (var j = 0; j < Layers[next].Activation.i; j++) { ManagedOps.Copy4DIJ2D(FeatureMap, Layers[next].FeatureMap, i, j); ManagedMatrix.Rotate180(rot180, FeatureMap); ManagedOps.Copy4D3D(Delta, Layers[next].Delta, j); ManagedConvolution.Full(Delta, rot180, ztemp); ManagedMatrix.Add(z, ztemp); } ManagedOps.Copy3D4D(Layers[l].Delta, z, i); } ManagedOps.Free(Delta, FeatureMap, rot180, z, ztemp); } } // calc gradients for (var l = 1; l < n; l++) { var prev = l - 1; if (Layers[l].Type == LayerTypes.Convolution) { ManagedOps.Free(Layers[l].DeltaFeatureMap, Layers[l].DeltaBias); Layers[l].DeltaFeatureMap = new ManagedArray(Layers[l].FeatureMap, false); Layers[l].DeltaBias = new ManagedArray(Layers[l].OutputMaps, false); var FeatureMapDelta = new ManagedArray(Layers[l].FeatureMap.x, Layers[l].FeatureMap.y, Layers[l].FeatureMap.z, false); // d[j] var dtemp = new ManagedArray(Layers[l].Activation.x, Layers[l].Activation.y, batchz, false); // a[i] and flipped var atemp = new ManagedArray(Layers[prev].Activation.x, Layers[prev].Activation.y, batchz, false); var ftemp = new ManagedArray(Layers[prev].Activation.x, Layers[prev].Activation.y, batchz, false); for (var j = 0; j < Layers[l].Activation.i; j++) { ManagedOps.Copy4D3D(dtemp, Layers[l].Delta, j); for (var i = 0; i < Layers[prev].Activation.i; i++) { ManagedOps.Copy4D3D(atemp, Layers[prev].Activation, i); ManagedMatrix.FlipAll(ftemp, atemp); ManagedConvolution.Valid(ftemp, dtemp, FeatureMapDelta); ManagedMatrix.Multiply(FeatureMapDelta, 1.0 / batchz); ManagedOps.Copy2D4DIJ(Layers[l].DeltaFeatureMap, FeatureMapDelta, i, j); } Layers[l].DeltaBias[j] = ManagedMatrix.Sum(dtemp) / batchz; } ManagedOps.Free(FeatureMapDelta, dtemp, atemp, ftemp); } } var FeatureVectorTransposed = new ManagedArray(FeatureVector, false); ManagedMatrix.Transpose(FeatureVectorTransposed, FeatureVector); ManagedOps.Free(WeightsDelta, BiasDelta); WeightsDelta = new ManagedArray(Weights, false); BiasDelta = new ManagedArray(Bias, false); ManagedMatrix.Multiply(WeightsDelta, OutputDelta, FeatureVectorTransposed); ManagedMatrix.Multiply(WeightsDelta, 1.0 / batchz); ManagedMatrix.Mean(BiasDelta, OutputDelta, 0); ManagedOps.Free(FeatureVectorTransposed); }
// Backward propagation public void BackPropagation(ManagedArray training) { // add bias column to input layer var InputBias = new ManagedArray(1, training.y); ManagedOps.Set(InputBias, 1.0); // x = cbind(array(1, c(nrow(training_set), 1)), training_set) var x = ManagedMatrix.CBind(InputBias, training); // compute intermediate delta values per layer // d3 = y_k - y_matrix var D3 = ManagedMatrix.Diff(Yk, Y_output); // d2 = d3 %*% w_kj[, 2:ncol(w_kj)] * nnet_dsigmoid(z_2) var sWkj = new ManagedArray(Wkj.x - 1, Wkj.y); ManagedOps.Copy2D(sWkj, Wkj, 1, 0); var D2 = new ManagedArray(sWkj.x, D3.y); ManagedMatrix.Multiply(D2, D3, sWkj); var DZ2 = ManagedMatrix.DSigm(Z2); ManagedMatrix.Product(D2, DZ2); // dWji = (t(d2) %*% x) // dWkj = (t(d3) % *% a_2) var tD2 = new ManagedArray(D2.y, D2.x); var tD3 = new ManagedArray(D3.y, D3.x); ManagedMatrix.Transpose(tD2, D2); ManagedMatrix.Transpose(tD3, D3); DeltaWji = new ManagedArray(Wji.x, Wji.y); DeltaWkj = new ManagedArray(Wkj.x, Wkj.y); ManagedMatrix.Multiply(DeltaWji, tD2, x); ManagedMatrix.Multiply(DeltaWkj, tD3, A2); // cost = sum(-y_matrix * log(y_k) - (1 - y_matrix) * log(1 - y_k)) Cost = 0.0; L2 = 0.0; for (var i = 0; i < Y_output.Length(); i++) { L2 += 0.5 * (D3[i] * D3[i]); Cost += (-Y_output[i] * Math.Log(Yk[i]) - (1 - Y_output[i]) * Math.Log(1 - Yk[i])); } // cost = cost / m // dWji = dWji / m // dWkj = dWkj / m Cost /= training.y; L2 /= training.y; ManagedMatrix.Multiply(DeltaWji, 1.0 / training.y); ManagedMatrix.Multiply(DeltaWkj, 1.0 / training.y); // cleanup ManagedOps.Free(D2, D3, DZ2, InputBias); ManagedOps.Free(sWkj, tD2, tD3, x); // cleanup of arrays allocated in Forward ManagedOps.Free(A2, Yk, Z2); }
// Compute Forward Transform on 3D Input public void FeedForward(ManagedArray batch, bool pool = false) { var n = Layers.Count; var last = n - 1; var InputMaps = 1; ManagedOps.Free(Layers[0].Activation); Layers[0].Activation = new ManagedArray(batch, false); ManagedOps.Copy4D3D(Layers[0].Activation, batch, 0); for (var l = 1; l < n; l++) { var prev = l - 1; if (Layers[l].Type == LayerTypes.Convolution) { var zx = Layers[prev].Activation.x - Layers[l].KernelSize + 1; var zy = Layers[prev].Activation.y - Layers[l].KernelSize + 1; var zz = batch.z; ManagedOps.Free(Layers[l].Activation); Layers[l].Activation = new ManagedArray(zx, zy, zz, Layers[l].OutputMaps, 1, false); var Activation = new ManagedArray(Layers[prev].Activation.x, Layers[prev].Activation.y, batch.z, false); var FeatureMap = new ManagedArray(Layers[l].KernelSize, Layers[l].KernelSize, false); // create temp output map var z = new ManagedArray(zx, zy, zz); var ztemp = new ManagedArray(zx, zy, zz, false); // !!below can probably be handled by insane matrix operations for (var j = 0; j < Layers[l].OutputMaps; j++) // for each output map { ManagedOps.Set(z, 0.0); for (var i = 0; i < InputMaps; i++) { // copy Layers ManagedOps.Copy4D3D(Activation, Layers[prev].Activation, i); ManagedOps.Copy4DIJ2D(FeatureMap, Layers[l].FeatureMap, i, j); // convolve with corresponding kernel and add to temp output map ManagedConvolution.Valid(Activation, FeatureMap, ztemp); ManagedMatrix.Add(z, ztemp); } // add bias, pass through nonlinearity ManagedMatrix.Add(z, Layers[l].Bias[j]); var sigm = ManagedMatrix.Sigm(z); ManagedOps.Copy3D4D(Layers[l].Activation, sigm, j); ManagedOps.Free(sigm); } ManagedOps.Free(Activation, FeatureMap, z, ztemp); InputMaps = Layers[l].OutputMaps; } else if (Layers[l].Type == LayerTypes.Subsampling) { // downsample // generate downsampling kernel var scale = (double)(Layers[l].Scale * Layers[l].Scale); var FeatureMap = new ManagedArray(Layers[l].Scale, Layers[l].Scale, false); ManagedOps.Set(FeatureMap, 1.0 / scale); ManagedOps.Free(Layers[l].Activation); Layers[l].Activation = new ManagedArray(Layers[prev].Activation.x / Layers[l].Scale, Layers[prev].Activation.y / Layers[l].Scale, batch.z, InputMaps, 1); var Activation = new ManagedArray(Layers[prev].Activation.x, Layers[prev].Activation.y, batch.z, false); var z = new ManagedArray(Layers[prev].Activation.x - Layers[l].Scale + 1, Layers[prev].Activation.y - Layers[l].Scale + 1, batch.z, false); for (var j = 0; j < InputMaps; j++) { // copy Layers ManagedOps.Copy4D3D(Activation, Layers[prev].Activation, j); // Subsample ManagedConvolution.Valid(Activation, FeatureMap, z); if (pool) { ManagedOps.Pool3D4D(Layers[l].Activation, z, j, Layers[l].Scale); } else { ManagedOps.Copy3D4D(Layers[l].Activation, z, j, Layers[l].Scale); } } ManagedOps.Free(Activation, FeatureMap, z); } } var MapSize = Layers[last].Activation.x * Layers[last].Activation.y; ManagedOps.Free(FeatureVector); FeatureVector = new ManagedArray(batch.z, MapSize * Layers[last].Activation.i); var temp1D = new ManagedArray(Layers[last].Activation.y, Layers[last].Activation.x, false); var temp2D = new ManagedArray(Layers[last].Activation.x, Layers[last].Activation.y, false); // concatenate all end layer feature maps into vector for (var j = 0; j < Layers[last].Activation.i; j++) { for (var ii = 0; ii < batch.z; ii++) { // Use Row-major in flattening the feature map ManagedOps.Copy4D2D(temp2D, Layers[last].Activation, ii, j); ManagedMatrix.Transpose(temp1D, temp2D); temp1D.Reshape(1, MapSize); ManagedOps.Copy2DOffset(FeatureVector, temp1D, ii, j * MapSize); } } var WeightsFeatureVector = new ManagedArray(FeatureVector.x, Weights.y, false); ManagedMatrix.Multiply(WeightsFeatureVector, Weights, FeatureVector); var repmat = new ManagedArray(batch.z, Bias.Length(), false); ManagedMatrix.Expand(Bias, batch.z, 1, repmat); ManagedMatrix.Add(WeightsFeatureVector, repmat); // feedforward into output perceptrons ManagedOps.Free(Output); Output = ManagedMatrix.Sigm(WeightsFeatureVector); ManagedOps.Free(WeightsFeatureVector, repmat, temp1D, temp2D); }