//-------------------------------------------------------------------------
#region Konstruktoren
public NeuralNetwork(int amountIN, int amountHN, int amountON, float[] weights, IActivationFunction ActFct = null)
{ //Keine Zufallsgewichte
int amountWeights = ComputeNumOfWeights(amountIN, amountHN, amountON);
if (ActFct == null)
{
ActFct = new Sigmoid();
}
if (amountWeights != weights.GetLength(0))
{
throw new ArgumentException("weights-Array hat falsche anzahl an argumenten!");
}
for (int i = 0; i < amountIN; i++)
{
inputNeurons.Add(new InputNeuron());
}
for (int i = 0; i < amountON; i++)
{
outputNeurons.Add(new WorkingNeuron(ActFct));
}
createNewHidden(amountHN);
createFullMesh(weights);
refreshData();
}
public NeuralNetwork(int amountIN, int amountHN, int amountON, float RangeRNDweights = 1, IActivationFunction ActFct = null)
{
if (ActFct == null)
{
ActFct = new Sigmoid();
}
int amountWeights = 0;
amountWeights = ComputeNumOfWeights(amountIN, amountHN, amountON);
for (int i = 0; i < amountIN; i++)
{
inputNeurons.Add(new InputNeuron());
}
for (int i = 0; i < amountON; i++)
{
outputNeurons.Add(new WorkingNeuron(ActFct));
}
createNewHidden(amountHN);
float[] weights = new float[ComputeNumOfWeights(amountIN, amountHN, amountON)];
weights = CreateRandomArray(weights.GetLength(0), RangeRNDweights);
createFullMesh(weights);
}
public double CalculateSigmoidError(double?target = null)
{
if (target == null)
{
return(Error = OutputSynapses.Sum(a => a.OutputNeuron.Error * a.Weight) * Sigmoid.Derivative(Output));
}
return(Error = Sigmoid.Derivative(Output) * CalculateError(target.Value));
}
public double CalculateGradient(double?target = null) //This function is used in back propagation and uses many techniques such as direvative of sigmoid function
{
if (target == null)
{
return(Gradient = OutputSynapses.Sum(a => a.OutputNeuron.Gradient * a.Weight) * Sigmoid.Derivative(Value)); //Calculate the error for each neuron in hidden layer
}
return(Gradient = CalculateError(target.Value) * Sigmoid.Derivative(Value)); //Calculate the error in the output layer
}
public double CalculateGradient(double?target = null)
{
if (target == null)
{
return(Gradient = OutputSynapses.Sum(a => a.OutputNeuron.Gradient * a.Weight) * Sigmoid.Derivative(Value));
}
return(Gradient = CalculateError(target.Value) * Sigmoid.Derivative(Value));
}
public double NeuronOutput()
{
value = 0.0f;
foreach (Connection i in InputConnections)
{
value += (i.weight * i.prevLayerNeuron.value) + bias;
}
value = Sigmoid.Output(value);
return(value);
}
public double GetOuputValue()
{
double sum = 0;
for (int i = 0; i < InputSynapses.Count; i++)
{
sum += InputSynapses[i].Weight * InputSynapses[i].InputNeuron.Output;
}
return(Output = Sigmoid.Output(InputSynapses.Sum(a => a.Weight * a.InputNeuron.Output) + BiasWeight));
}
public virtual float CalculateValue()
{
//return Value = Sigmoid.Output(InputSynapses.Sum(a => a.Weight * a.InputNeuron.Value) + Bias);
float sum = 0f;
for (int i = 0; i < InputSynapses.Count; i++)
{
var synapse = InputSynapses[i];
sum += synapse.Weight * synapse.InputNeuron.Value;
}
return(Value = Sigmoid.Output(sum + Bias));
}
//Takes an input vector and an expected output vector
//Uses the discrepancy between the two to train the network via backpropagation
//regFactor is the regularization constant 1 - (lambda * learningRate/(2 * size of training set))
//A regFactor of 1 is taken to indicate the absence of regularization
public void TrainIteration(Matrix inputs, Matrix expectedResult, float regFactor)
{
if (regFactor <= 0.0F || regFactor > 1.0F)
{
throw new ArgumentException("Regularization factor must be on the range (0, 1].");
}
Matrix[] weightedLayerSums = new Matrix[numLayers];
Matrix[] layerOutputs = new Matrix[numLayers];
Matrix[] newWeightMatrices = new Matrix[numLayers];
newWeightMatrices[0] = null;
layerOutputs[0] = inputs;
weightedLayerSums[0] = null;
//First we evaulate the network (feedforward) and track all output vectors and weighted sums as we go
for (int i = 1; i < numLayers; ++i)
{
weightedLayerSums[i] = weightMatrices[i] * layerOutputs[i - 1] + biases[i];
layerOutputs[i] = Sigmoid.Sigma(weightedLayerSums[i]);
}
Matrix[] layerDeltas = new Matrix[numLayers];
//Calculate the gradient (quadratic cost)
/*Matrix gradient = QuadraticGradient(layerOutputs[numLayers - 1], expectedResult);
* layerDeltas[numLayers - 1] = Matrix.HadamardProduct(gradient, Sigmoid.SigmaPrime(weightedLayerSums[numLayers - 1]));
* //Then, make a backwards pass through the network, calculating the weight changes and applying them
* for (int i = numLayers - 2; i > 0; --i) {
* layerDeltas[i] = Matrix.HadamardProduct(Matrix.Transpose(weightMatrices[i + 1]) * layerDeltas[i + 1], Sigmoid.SigmaPrime(weightedLayerSums[i]));
* }
* //Gradient descent
* for (int i = numLayers - 1; i > 0; --i) {
* weightMatrices[i] = weightMatrices[i] - (learningRate * layerDeltas[i]) * Matrix.Transpose(layerOutputs[i - 1]);
* biases[i] = biases[i] - learningRate * (layerDeltas[i]);
* }*/
for (int i = numLayers - 1; i > 0; --i)
{
if (i == numLayers - 1)
{
Matrix gradient = QuadraticGradient(layerOutputs[i], expectedResult);
layerDeltas[i] = Matrix.HadamardProduct(gradient, Sigmoid.SigmaPrime(weightedLayerSums[i]));
}
else
{
layerDeltas[i] = Matrix.HadamardProduct(Matrix.Transpose(weightMatrices[i + 1]) * layerDeltas[i + 1], Sigmoid.SigmaPrime(weightedLayerSums[i]));
}
newWeightMatrices[i] = (regFactor * weightMatrices[i]) - (learningRate * layerDeltas[i]) * Matrix.Transpose(layerOutputs[i - 1]);
biases[i] = biases[i] - learningRate * (layerDeltas[i]);
}
weightMatrices = newWeightMatrices;
}
public float CalculateGradient(float?target = null)
{
if (target == null)
{
//return Gradient = OutputSynapses.Sum(a => a.OutputNeuron.Gradient * a.Weight) * Sigmoid.Derivative(Value);
float sum = 0f;
for (int i = 0; i < OutputSynapses.Count; i++)
{
var synapse = OutputSynapses[i];
sum += synapse.OutputNeuron.Gradient * synapse.Weight;
}
return(Gradient = sum * Sigmoid.Derivative(Value));
}
return(Gradient = CalculateError(target.Value) * Sigmoid.Derivative(Value));
}
public NeuralNetwork(int[] layers)
{
var lsl = layers.Length - 1;
this.nn = new Matrix[lsl];
var afn = new Sigmoid();
for (int i = 0; i < lsl; i++)
{
this.nn[i] = Matrix.RandomMatrix(layers[i] + 1, layers[i + 1], 1000);
// this.nn[i] = new Matrix(layers[i] + 1, layers[i + 1], 100);
}
// nn[lsl-1].afn = new ReLU();
}
public double LocalGradient(double?desOut = null)
{
if (desOut != null)
{
gradient = LocalError(desOut.Value) * Sigmoid.FirstDerivative(value);
}
else
{
gradient = 0.0f;
foreach (Connection o in OutputConnections)
{
gradient = o.weight * o.nextLayerNeuron.gradient * Sigmoid.FirstDerivative(value);
}
}
return(gradient);
}
public void BackPropagate(Pair pair)
{
var input = pair.input.ToArray();
var output = pair.output.ToArray();
ForwardPass(input);
for (var i = 0; i < outputLayer.neurons.Count; i++)
{
var neuron = outputLayer.neurons[i];
var outputA = neuron.outputA;
var expectedOutput = output[i];
neuron.error = expectedOutput - outputA;
}
for (var layer = outputLayer.previous; layer != null; layer = layer.previous)
{
foreach (var neuron in layer.neurons)
{
neuron.error = 0;
foreach (var connection in neuron.outputs)
{
var error = connection.output.error;
var weight = connection.weight;
neuron.error += error * weight;
}
}
}
foreach (var connection in connections)
{
var error = connection.output.error;
var gradient = Sigmoid.Derivative(connection.output.outputA);
var previousOutputA = connection.input.outputA;
connection.weight += learning_rate * error * gradient * previousOutputA;
}
for (var layer = inputLayer; layer != null; layer = layer.next)
{
foreach (var neuron in layer.neurons)
{
var error = neuron.error;
var gradient = Sigmoid.Derivative(neuron.outputA);
neuron.bias += learning_rate * error * gradient;
}
}
}
//Takes the given input vector and passes it through the network
//Returns the network's output vector
public Matrix Evaluate(Matrix inputs)
{
if (inputs == null)
{
throw new NullReferenceException("Cannot evaluate a null input vector.");
}
if (inputs.NumColumns != 1 || inputs.NumRows != layerSizes[0])
{
throw new ArgumentException("Input vector must be a nx1 matrix and n must equal the number of nodes in the first layer of the network.");
}
Matrix layerOutput = inputs;
for (int i = 1; i < numLayers; ++i)
{
layerOutput = Sigmoid.Sigma((weightMatrices[i] * layerOutput) + biases[i]);
}
return(layerOutput);
}
public void ForwardPass(double[] inputs)
{
SetInputs(inputs);
foreach (var inputNeuron in inputLayer.neurons)
{
inputNeuron.outputA = Sigmoid.Value(inputNeuron.inputZ + inputNeuron.bias);
}
for (var layer = inputLayer.next; layer != null; layer = layer.next)
{
foreach (var neuron in layer.neurons)
{
neuron.inputZ = 0;
foreach (var connection in neuron.inputs)
{
neuron.inputZ += connection.input.outputA * connection.weight;
}
neuron.outputA = Sigmoid.Value(neuron.inputZ + neuron.bias);
}
}
}
public virtual double CalculateValue() //This function is used in forward propogation process.
{
return(Value = Sigmoid.Output(InputSynapses.Sum(a => a.Weight * a.InputNeuron.Value) + Bias)); //Calculate by ably activation function to (weight*input)+bias
}
static void Main(string[] args)
{
Random random = new Random();
BinaryStep binaryStep = new BinaryStep();
Sigmoid sigmoid = new Sigmoid();
// XOR
// 3 ? 1
// bit bit AND OR XOR NOT
double[][] inputs = new double[][]
{
//Nand
new double[] { 0, 0, 1, 0, 0, 1 },
new double[] { 0, 1, 1, 0, 0, 1 },
new double[] { 1, 0, 1, 0, 0, 1 },
new double[] { 1, 1, 1, 0, 0, 1 },
//Or
new double[] { 0, 0, 0, 1, 0, 0 },
new double[] { 0, 1, 0, 1, 0, 0 },
new double[] { 1, 0, 0, 1, 0, 0 },
new double[] { 1, 1, 0, 1, 0, 0 },
//And
new double[] { 0, 0, 1, 0, 0, 0 },
new double[] { 0, 1, 1, 0, 0, 0 },
new double[] { 1, 0, 1, 0, 0, 0 },
new double[] { 1, 1, 1, 0, 0, 0 },
//Nor
new double[] { 0, 0, 0, 1, 0, 1 },
new double[] { 0, 1, 0, 1, 0, 1 },
new double[] { 1, 0, 0, 1, 0, 1 },
new double[] { 1, 1, 0, 1, 0, 1 },
//XOR
new double[] { 0, 0, 0, 0, 1, 0 },
new double[] { 0, 1, 0, 0, 1, 0 },
new double[] { 1, 0, 0, 0, 1, 0 },
new double[] { 1, 1, 0, 0, 1, 0 },
//NXOR
new double[] { 0, 0, 0, 0, 1, 1 },
new double[] { 0, 1, 0, 0, 1, 1 },
new double[] { 1, 0, 0, 0, 1, 1 },
new double[] { 1, 1, 0, 0, 1, 1 },
};
double[] outputs = new double[]
{
1, 1, 1, 0,
0, 1, 1, 1,
0, 0, 0, 1,
1, 0, 0, 0,
0, 1, 1, 0,
1, 0, 0, 1,
};
int epochs = 0;
#region Genetic
/*
* // population: 1000 nn
* (NeuralNetwork network, double mae)[] population = new(NeuralNetwork network, double mae)[1000];
*
* for (int i = 0; i < population.Length; i++)
* {
* population[i] = (new NeuralNetwork(6, new IActivation[][]
* {
* Enumerable.Repeat(binaryStep, 10).ToArray(),
* new IActivation[] { binaryStep }
* }), 1);
* population[i].network.Randomize(random);
* }
*
* double top = double.MaxValue;
*
* do
* {
* // run the population
* //double[] mae = new double[population.Length];
* for (int i = 0; i < population.Length; i++)
* {
* population[i].mae = population[i].network.MAE(inputs, outputs);
* }
*
* Array.Sort(population, (a, b) => a.mae.CompareTo(b.mae));
*
* // sort them by their error (MAE)
* // keep the top guy the same, mutate the rest (mutation rate should be const) (scale is affected by error)
* top = population[0].mae;
* StringBuilder topValues = new StringBuilder("{ ");
* for (int i = 0; i < inputs.Length; i++)
* {
* topValues.Append($"{population[0].network.Compute(inputs[i])[0]:0.00}");
* topValues.Append(' ');
* }
*
* topValues.Append($"}} : {top:0.00}");
* Console.SetCursorPosition(0, 0);
* Console.WriteLine("{ 1.00 1.00 1.00 0.00 0.00 1.00 1.00 1.00 0.00 0.00 0.00 1.00 1.00 0.00 0.00 0.00 0.00 1.00 1.00 0.00 1.00 0.00 0.00 1.00 }");
* Console.WriteLine(topValues.ToString());
*
* //10% keep
* //80% crossover with a random net from the top 10% AND mutate
* //10% randomize
* int crossStart = (int) (population.Length * .05);
* int randomStart = (int)(population.Length * .7);
* for (int i = crossStart; i < randomStart; i++)
* {
* NeuralNetwork toCross = population[random.Next(0, crossStart)].network;
* population[i].network.Crossover(random, toCross);
* population[i].network.Mutate(random, .2, population[i].mae);
* }
* for(int i = randomStart; i < population.Length; i++)
* {
* population[i].network.Randomize(random);
* }
* } while (top != 0);
*/
#endregion
#region Backprop
Chx chx = new Chx(Path.Combine(@"\\GMRDC1\Folder Redirection\Ryan.Alameddine\Documents\Visual Studio 2017\Projects\NeuralNet\MiniMaxTree\MiniMaxTree\bin\Debug\netcoreapp2.1", "CHX.txt"));
NeuralNetwork neuralNetwork = new NeuralNetwork(33, new IActivation[][]
{
Enumerable.Repeat(sigmoid, 15).ToArray(),
Enumerable.Repeat(sigmoid, 15).ToArray(),
new IActivation[] { sigmoid }
});
neuralNetwork.Randomize(random);
StringBuilder topValues = new StringBuilder();
while (true)
{
epochs++;
if (Console.KeyAvailable)
{
char keyChar = Console.ReadKey().KeyChar;
if (keyChar == 's')
{
var desktop = Environment.GetFolderPath(Environment.SpecialFolder.Desktop);
File.WriteAllLines(Path.Combine(desktop, "trained.json"), new string[] { JsonConvert.SerializeObject(neuralNetwork, Formatting.Indented, new JsonSerializerSettings()
{
ContractResolver = new Resolver()
}) });
}
else if (keyChar == 'x')
{
var desktop = Environment.GetFolderPath(Environment.SpecialFolder.Desktop);
var lines = File.ReadAllText(Path.Combine(desktop, "trained.json"));
neuralNetwork = JsonConvert.DeserializeObject <NeuralNetwork>(lines, new JsonSerializerSettings()
{
ContractResolver = new Resolver()
});
}
}
double MAE = neuralNetwork.Train(chx.Spaces, chx.Outputs, 1, .0125, 0.2) / chx.Spaces.Length;
if (epochs % 1000 == 0)
{
/*double sumError = 0;
* for (int i = 0; i < chx.TestSpaces.Length; i++)
* {
* neuralNetwork.Compute(chx.TestSpaces[i]);
* sumError += neuralNetwork.CalculateError(chx.TestOutputs[i]);
* }*/
Console.WriteLine(MAE);
Console.WriteLine(neuralNetwork.MAE(chx.TestSpaces, chx.TestOutputs));
///Console.WriteLine(sumError / chx.TestSpaces.Length);
Console.WriteLine($"Epochs: {epochs}");
}
}
Console.ReadKey();
#endregion
#region MNIST
/*MNIST mnist = new MNIST();
*
* // 784,400,10
* NeuralNetwork neuralNetwork = new NeuralNetwork(784, new IActivation[][]
* {
* Enumerable.Repeat(new Sigmoid(), 600).ToArray(),
* Enumerable.Repeat(new Sigmoid(), 300).ToArray(),
* Enumerable.Repeat(new Sigmoid(), 10).ToArray(),
* });
* neuralNetwork.Randomize(random);
*
* double mae = 0;
* do
* {
* mae = neuralNetwork.Train(mnist.Pixels, mnist.Outputs, 1, .3, .9);
* Console.WriteLine($"{mae}");
* }
* while (mae > 0);
* ;*/
#endregion MNIST
}
public void BackPropagate(Batch batch)
{
var learning_rate = 0.01;
foreach (var neuron in neurons)
{
neuron.error = 0;
}
foreach (var connection in connections)
{
connection.costCorrection = 0;
}
foreach (var pair in batch)
{
var input = pair.input.ToArray();
var output = pair.output.ToArray();
ForwardPass(input);
for (var i = 0; i < outputLayer.neurons.Count; i++)
{
var neuron = outputLayer.neurons[i];
var outputA = neuron.outputA;
var expectedOutput = output[i];
var error = expectedOutput - outputA;
var dCdA = 2 * error;
neuron.error += dCdA / batch.Count;
}
for (var layer = outputLayer.previous; layer != null; layer = layer.previous)
{
foreach (var neuron in layer.neurons)
{
foreach (var connection in neuron.outputs)
{
var dCdA = connection.output.error;
var dAdZ = Sigmoid.Derivative(connection.output.outputA);
var dZdA = connection.weight;
var dZdW = connection.input.outputA;
neuron.error += dCdA * dAdZ * dZdA / batch.Count;
connection.costCorrection += dCdA * dAdZ * dZdW / batch.Count;
}
}
}
}
foreach (var connection in connections)
{
connection.weight += connection.costCorrection * learning_rate;
}
for (var layer = inputLayer; layer != null; layer = layer.next)
{
foreach (var neuron in layer.neurons)
{
var dCdA = neuron.error;
var dAdZ = Sigmoid.Derivative(neuron.outputA);
var dZdB = 1;
neuron.bias += dCdA * dAdZ * dZdB * learning_rate;
}
}
}
public virtual double CalculateValue()
{
return(Value = Sigmoid.Output(InputSynapses.Sum(a => a.Weight * a.InputNeuron.Value) + Bias));
}
public void CalculateValueRecurrent()
{
OutputValue = Sigmoid.Output(InputSynapses.Sum(syn => syn.Weight * syn.InputNeuron.LastValue) + Bias);
}