const double Epsilon = 0.01; // Epsilon is a value near to 0
static void Main(string[] args)
{
//Parameters
int inputCount = 2;
int hiddenCount = 5;
int outputCount = 4;
int examplesCount = 4;
double learningRate = 0.1;
Random r = new Random();
//Training data //INPUT
Matrix x = new double[, ] {
{ 0, 0 },
{ 0, 1 },
{ 1, 0 },
{ 1, 1 }
};
//DESIRED OUTPUT
//XNOR AND OR XOR
Matrix y = new double[, ] {
{ 1, 0, 0, 0 },
{ 0, 0, 1, 1 },
{ 0, 1, 1, 1 },
{ 1, 1, 1, 0 }
};
//Weight init
Matrix w1 = (Matrix.Random(inputCount + 1, hiddenCount, r) - 0.5) * 2.0;
Matrix w2 = (Matrix.Random(hiddenCount + 1, outputCount, r) - 0.5) * 2.0;
// TO USE RELU
// 0. Change all sigmoid function, for relu function
// 1. a3 must have no Nonlinear function Matrix a3 = z3;
// 2. because of that Delta3 has not derivated Matrix Delta3 = a3Error * 1;
// 3. The learning rate must be smaller, like 0.001
for (int l = 0; l < 5001; l++) //epoch
{
//Forward pass
Matrix z1 = x.AddColumn(Matrix.Ones(examplesCount, 1));
Matrix a1 = z1; //(Examples, input + 1)
Matrix z2 = (a1 * w1).AddColumn(Matrix.Ones(examplesCount, 1));
Matrix a2 = sigmoid(z2); //(examples, hidden neurons + 1) // APPLY NON LINEAR
Matrix z3 = a2 * w2;
Matrix a3 = sigmoid(z3); //(examples, output)
//Bacpropagation
Matrix a3Error = a3 - y; //(examples, output) //LOSS
Matrix Delta3 = a3Error * sigmoid(z3, true);
Matrix a2Error = Delta3 * w2.T;
Matrix Delta2 = a2Error * sigmoid(z2, true);
Delta2 = Delta2.Slice(0, 1, Delta2.x, Delta2.y); //Slicing Extra delta (from biass neuron)
w2 -= (a2.T * Delta3) * learningRate;
w1 -= (a1.T * Delta2) * learningRate;
double loss = a3Error.abs.average * examplesCount;
Console.WriteLine("Loss: " + loss);
if (l % 1000 == 0)
{
Console.WriteLine("---------" + l + "----------------------------------------------------");
Console.WriteLine("X: " + x.size.ToString());
Console.WriteLine(x.ToString());
Console.WriteLine("Prediction: " + a3.size.ToString());
Console.WriteLine(a3);
}
}
Console.ReadKey();
}
static void Main(string[] args)
{
int entrada = 2;
int pesos = 5;
int salida = 4;
int umbral = 4;
double recalcular = 0.1;
Random r = new Random();
Matrix x = new double[, ] {
{ 0, 0 },
{ 0, 1 },
{ 1, 0 },
{ 1, 1 }
};
Matrix y = new double[, ] {
{ 1, 0, 0, 0 },
{ 0, 0, 1, 1 },
{ 0, 1, 1, 1 },
{ 1, 1, 1, 0 }
};
Matrix w1 = (Matrix.Random(entrada + 1, pesos, r) - 0.5) * 2.0;
Matrix w2 = (Matrix.Random(pesos + 1, salida, r) - 0.5) * 2.0;
Matrix w3 = (Matrix.Random(pesos + 1, salida, r) - 0.5) * 2.0;
Matrix w4 = (Matrix.Random(pesos + 1, salida, r) - 0.5) * 2.0;
for (int l = 0; l < 5001; l++)
{
Matrix z1 = x.AddColumn(Matrix.Ones(umbral, 1));
Matrix a1 = z1;
Matrix z2 = (a1 * w1).AddColumn(Matrix.Ones(umbral, 1));
Matrix a2 = siguiente(z2);
Matrix z3 = a2 * w2;
Matrix a3 = siguiente(z3);
Matrix a3error = a3 - y;
Matrix Delta3 = a3error * siguiente(z3, true);
Matrix a2error = Delta3 * w2.T;
Matrix Delta2 = a2error * siguiente(z2, true);
Delta2 = Delta2.Slice(0, 1, Delta2.x, Delta2.y);
w2 -= (a2.T * Delta3) * recalcular;
w1 -= (a1.T * Delta2) * recalcular;
double perdidas = a3error.abs.average * umbral;
Console.WriteLine("Entrenando: " + perdidas);
if (l % 1000 == 0)
{
Console.WriteLine("---" + l + "---");
Console.WriteLine("X: " + x.size.ToString());
Console.WriteLine(x.ToString());
Console.WriteLine("Repesos: " + a3.size.ToString());
Console.WriteLine(a3);
}
}
Console.ReadKey();
}