// Forward Propagation public void Forward(ManagedArray training) { // add bias column to input layer var InputBias = new ManagedArray(1, training.y); ManagedOps.Set(InputBias, 1.0); // x = cbind(array(1, c(nrow(training_set), 1)), training_set) var x = ManagedMatrix.CBind(InputBias, training); // compute hidden layer activation // z_2 = x %*% t(w_ji) var tWji = new ManagedArray(Wji.y, Wji.x); ManagedMatrix.Transpose(tWji, Wji); Z2 = new ManagedArray(tWji.x, x.y); ManagedMatrix.Multiply(Z2, x, tWji); // z_j = nnet_sigmoid(z_2) var Zj = ManagedMatrix.Sigm(Z2); // add bias column to hidden layer output var HiddenBias = new ManagedArray(1, Zj.y); ManagedOps.Set(HiddenBias, 1.0); // a_2 = cbind(array(1, c(nrow(z_j), 1)), z_j) A2 = ManagedMatrix.CBind(HiddenBias, Zj); // compute output layer var tWkj = new ManagedArray(Wkj.y, Wkj.x); ManagedMatrix.Transpose(tWkj, Wkj); // y_k = nnet_sigmoid(a_2 %*% t(w_kj)) var A2Wkj = new ManagedArray(tWkj.x, A2.y); ManagedMatrix.Multiply(A2Wkj, A2, tWkj); Yk = ManagedMatrix.Sigm(A2Wkj); // cleanup ManagedOps.Free(A2Wkj, HiddenBias, InputBias); ManagedOps.Free(tWkj, tWji, x, Zj); }
// Forward Propagation public void Forward(ManagedArray input) { // create bias column var InputBias = new ManagedArray(1, input.y, false); ManagedOps.Set(InputBias, 1.0); // Compute input activations var last = Weights.GetLength(0) - 1; for (var layer = 0; layer < Weights.GetLength(0); layer++) { var XX = layer == 0 ? ManagedMatrix.CBind(InputBias, input) : ManagedMatrix.CBind(InputBias, Activations[layer - 1]); var tW = ManagedMatrix.Transpose(Weights[layer]); var ZZ = ManagedMatrix.Multiply(XX, tW); X[layer] = XX; Z[layer] = ZZ; if (layer != last) { var SS = ManagedMatrix.Sigm(ZZ); Activations[layer] = SS; } else { ManagedOps.Free(Y); Y = ManagedMatrix.Sigm(ZZ); } ManagedOps.Free(tW); } // Cleanup for (var layer = 0; layer < Activations.GetLength(0); layer++) { ManagedOps.Free(Activations[layer]); } ManagedOps.Free(InputBias); }
// Backward propagation public void BackPropagation(ManagedArray training) { // add bias column to input layer var InputBias = new ManagedArray(1, training.y); ManagedOps.Set(InputBias, 1.0); // x = cbind(array(1, c(nrow(training_set), 1)), training_set) var x = ManagedMatrix.CBind(InputBias, training); // compute intermediate delta values per layer // d3 = y_k - y_matrix var D3 = ManagedMatrix.Diff(Yk, Y_output); // d2 = d3 %*% w_kj[, 2:ncol(w_kj)] * nnet_dsigmoid(z_2) var sWkj = new ManagedArray(Wkj.x - 1, Wkj.y); ManagedOps.Copy2D(sWkj, Wkj, 1, 0); var D2 = new ManagedArray(sWkj.x, D3.y); ManagedMatrix.Multiply(D2, D3, sWkj); var DZ2 = ManagedMatrix.DSigm(Z2); ManagedMatrix.Product(D2, DZ2); // dWji = (t(d2) %*% x) // dWkj = (t(d3) % *% a_2) var tD2 = new ManagedArray(D2.y, D2.x); var tD3 = new ManagedArray(D3.y, D3.x); ManagedMatrix.Transpose(tD2, D2); ManagedMatrix.Transpose(tD3, D3); DeltaWji = new ManagedArray(Wji.x, Wji.y); DeltaWkj = new ManagedArray(Wkj.x, Wkj.y); ManagedMatrix.Multiply(DeltaWji, tD2, x); ManagedMatrix.Multiply(DeltaWkj, tD3, A2); // cost = sum(-y_matrix * log(y_k) - (1 - y_matrix) * log(1 - y_k)) Cost = 0.0; L2 = 0.0; for (var i = 0; i < Y_output.Length(); i++) { L2 += 0.5 * (D3[i] * D3[i]); Cost += (-Y_output[i] * Math.Log(Yk[i]) - (1 - Y_output[i]) * Math.Log(1 - Yk[i])); } // cost = cost / m // dWji = dWji / m // dWkj = dWkj / m Cost /= training.y; L2 /= training.y; ManagedMatrix.Multiply(DeltaWji, 1.0 / training.y); ManagedMatrix.Multiply(DeltaWkj, 1.0 / training.y); // cleanup ManagedOps.Free(D2, D3, DZ2, InputBias); ManagedOps.Free(sWkj, tD2, tD3, x); // cleanup of arrays allocated in Forward ManagedOps.Free(A2, Yk, Z2); }