// Forward Propagation
public void Forward(ManagedArray training)
{
// add bias column to input layer
var InputBias = new ManagedArray(1, training.y);
ManagedOps.Set(InputBias, 1.0);
// x = cbind(array(1, c(nrow(training_set), 1)), training_set)
var x = ManagedMatrix.CBind(InputBias, training);
// compute hidden layer activation
// z_2 = x %*% t(w_ji)
var tWji = new ManagedArray(Wji.y, Wji.x);
ManagedMatrix.Transpose(tWji, Wji);
Z2 = new ManagedArray(tWji.x, x.y);
ManagedMatrix.Multiply(Z2, x, tWji);
// z_j = nnet_sigmoid(z_2)
var Zj = ManagedMatrix.Sigm(Z2);
// add bias column to hidden layer output
var HiddenBias = new ManagedArray(1, Zj.y);
ManagedOps.Set(HiddenBias, 1.0);
// a_2 = cbind(array(1, c(nrow(z_j), 1)), z_j)
A2 = ManagedMatrix.CBind(HiddenBias, Zj);
// compute output layer
var tWkj = new ManagedArray(Wkj.y, Wkj.x);
ManagedMatrix.Transpose(tWkj, Wkj);
// y_k = nnet_sigmoid(a_2 %*% t(w_kj))
var A2Wkj = new ManagedArray(tWkj.x, A2.y);
ManagedMatrix.Multiply(A2Wkj, A2, tWkj);
Yk = ManagedMatrix.Sigm(A2Wkj);
// cleanup
ManagedOps.Free(A2Wkj, HiddenBias, InputBias);
ManagedOps.Free(tWkj, tWji, x, Zj);
}
// Forward Propagation
public void Forward(ManagedArray input)
{
// create bias column
var InputBias = new ManagedArray(1, input.y, false);
ManagedOps.Set(InputBias, 1.0);
// Compute input activations
var last = Weights.GetLength(0) - 1;
for (var layer = 0; layer < Weights.GetLength(0); layer++)
{
var XX = layer == 0 ? ManagedMatrix.CBind(InputBias, input) : ManagedMatrix.CBind(InputBias, Activations[layer - 1]);
var tW = ManagedMatrix.Transpose(Weights[layer]);
var ZZ = ManagedMatrix.Multiply(XX, tW);
X[layer] = XX;
Z[layer] = ZZ;
if (layer != last)
{
var SS = ManagedMatrix.Sigm(ZZ);
Activations[layer] = SS;
}
else
{
ManagedOps.Free(Y);
Y = ManagedMatrix.Sigm(ZZ);
}
ManagedOps.Free(tW);
}
// Cleanup
for (var layer = 0; layer < Activations.GetLength(0); layer++)
{
ManagedOps.Free(Activations[layer]);
}
ManagedOps.Free(InputBias);
}
// Backward propagation
public void BackPropagation(ManagedArray training)
{
// add bias column to input layer
var InputBias = new ManagedArray(1, training.y);
ManagedOps.Set(InputBias, 1.0);
// x = cbind(array(1, c(nrow(training_set), 1)), training_set)
var x = ManagedMatrix.CBind(InputBias, training);
// compute intermediate delta values per layer
// d3 = y_k - y_matrix
var D3 = ManagedMatrix.Diff(Yk, Y_output);
// d2 = d3 %*% w_kj[, 2:ncol(w_kj)] * nnet_dsigmoid(z_2)
var sWkj = new ManagedArray(Wkj.x - 1, Wkj.y);
ManagedOps.Copy2D(sWkj, Wkj, 1, 0);
var D2 = new ManagedArray(sWkj.x, D3.y);
ManagedMatrix.Multiply(D2, D3, sWkj);
var DZ2 = ManagedMatrix.DSigm(Z2);
ManagedMatrix.Product(D2, DZ2);
// dWji = (t(d2) %*% x)
// dWkj = (t(d3) % *% a_2)
var tD2 = new ManagedArray(D2.y, D2.x);
var tD3 = new ManagedArray(D3.y, D3.x);
ManagedMatrix.Transpose(tD2, D2);
ManagedMatrix.Transpose(tD3, D3);
DeltaWji = new ManagedArray(Wji.x, Wji.y);
DeltaWkj = new ManagedArray(Wkj.x, Wkj.y);
ManagedMatrix.Multiply(DeltaWji, tD2, x);
ManagedMatrix.Multiply(DeltaWkj, tD3, A2);
// cost = sum(-y_matrix * log(y_k) - (1 - y_matrix) * log(1 - y_k))
Cost = 0.0;
L2 = 0.0;
for (var i = 0; i < Y_output.Length(); i++)
{
L2 += 0.5 * (D3[i] * D3[i]);
Cost += (-Y_output[i] * Math.Log(Yk[i]) - (1 - Y_output[i]) * Math.Log(1 - Yk[i]));
}
// cost = cost / m
// dWji = dWji / m
// dWkj = dWkj / m
Cost /= training.y;
L2 /= training.y;
ManagedMatrix.Multiply(DeltaWji, 1.0 / training.y);
ManagedMatrix.Multiply(DeltaWkj, 1.0 / training.y);
// cleanup
ManagedOps.Free(D2, D3, DZ2, InputBias);
ManagedOps.Free(sWkj, tD2, tD3, x);
// cleanup of arrays allocated in Forward
ManagedOps.Free(A2, Yk, Z2);
}