private void InitializeFirstLayer() { //Put first sample into the nodes of the first layer (X1,X2,X3,X4 into 1st Layer Nodes) for (int i = 0; i < nodesPerLayer[0]; i++) { List<double> temp = new List<double>(); for (int j = 0; j < list_of_features.Count; j++) { temp.Add(list_of_features[j][0]); } Neuron t = new Neuron(); t.setInputs(temp); hiddenlayers[0].Add(t); } //Initialize weights of the nodes of the first layer to 0 List<double> tempWeights = new List<double>(); for (int i = 0; i < list_of_features.Count; i++) { tempWeights.Add(0); } for (int i = 0; i < nodesPerLayer[0]; i++) { hiddenlayers[0][i].setWeights(tempWeights); } }
public double Backpropagation(double[] expectedValues) { double totalNetworkCost = 0.0; // Compute error for the output neurons to get the ball rolling. // See https://github.com/kwende/CSharpNeuralNetworkExplorations/blob/master/Explorations/SimpleMLP/Documentation/OutputNeuronErrors.png for (int d = 0; d < expectedValues.Length; d++) { Neuron outputNeuronBeingExamined = OutputLayer.Neurons[d]; double expectedOutput = expectedValues[d]; double actualOutput = outputNeuronBeingExamined.Activation; double actualInput = outputNeuronBeingExamined.TotalInput; double cost = _costFunction.Compute(expectedOutput, actualOutput); totalNetworkCost += cost; double errorRelativeToActivation = (_costFunction.ComputeDerivativeWRTActivation(actualOutput, expectedOutput)); double errorWrtToNeuron = errorRelativeToActivation * Math.Sigmoid.ComputeDerivative(actualInput); outputNeuronBeingExamined.AddError(errorWrtToNeuron); for (int e = 0; e < outputNeuronBeingExamined.UpstreamDendrites.Count; e++) { Dendrite dendrite = outputNeuronBeingExamined.UpstreamDendrites[e]; Neuron upstreamNeuron = (Neuron)dendrite.UpStreamNeuron; double errorRelativeToWeight = (errorWrtToNeuron * upstreamNeuron.Activation); dendrite.AddError(errorRelativeToWeight); } } // Compute error for each neuron in each layer moving backwards (backprop). for (int d = HiddenLayers.Count - 1; d >= 0; d--) { HiddenLayer hiddenLayer = HiddenLayers[d]; for (int e = 0; e < hiddenLayer.Neurons.Count; e++) { Neuron thisNeuron = (Neuron)hiddenLayer.Neurons[e]; double dropoutBit = hiddenLayer.DropOutMask[e]; double input = thisNeuron.TotalInput; double errorSum = 0.0; List <Dendrite> downStreamDendrites = thisNeuron.DownstreamDendrites; for (int f = 0; f < downStreamDendrites.Count; f++) { Dendrite currentDendrite = downStreamDendrites[f]; Neuron downStreamNeuron = currentDendrite.DownStreamNeuron; double delta = downStreamNeuron.CurrentNeuronError; double weight = currentDendrite.Weight; errorSum += delta * weight; } double errorWrtToThisNeuron = errorSum * Math.Sigmoid.ComputeDerivative(input) * dropoutBit; thisNeuron.AddError(errorWrtToThisNeuron); for (int f = 0; f < thisNeuron.UpstreamDendrites.Count; f++) { Dendrite dendrite = thisNeuron.UpstreamDendrites[f]; Neuron upstreamNeuron = (Neuron)dendrite.UpStreamNeuron; double errorRelativeToWeight = (errorWrtToThisNeuron * upstreamNeuron.Activation); dendrite.AddError(errorRelativeToWeight); } } } return(totalNetworkCost); }
static int trainingSetNumber = 2; //Applies to aproximation variant #endregion public static void Main(String[] args) { Random gen = new Random(); if (variant == Variant.transformation) { double succesfulOutputsCount = 0; double totalOutputsCount = 4 * executionsCount; for (int counter = 1; counter <= executionsCount; counter++) { string fileName = variant.ToString() + "_" + bias.ToString() + "_Execution" + counter.ToString() + "EpochsDiffrences.xml"; StreamWriter sw = new StreamWriter(fileName); XmlSerializer xs = new XmlSerializer(typeof(List <double>)); List <double> EpochsMSEs = new List <double>(); double[][] testSamples = LoadTrainingDataFromFileTransformation(); double[][] finalInputOutput = null; List <double[]> trainingSet = new List <double[]>(); RefillTrainingSet(trainingSet, testSamples); Neuron[] hiddenLayer = null; Neuron[] outputLayer = null; InitalizeLayers(ref hiddenLayer, ref outputLayer); for (int i = 1; i <= epochsCount; i++) { double EpochMSE = 0; double IterationError = 0; EpochMSE = 0; for (int j = trainingSet.Count; j > 0; j--) { IterationError = 0; int randomIndex = gen.Next(j); double[] inputs1 = trainingSet[randomIndex]; double[] inputs2 = new double[hiddenLayerCount]; foreach (Neuron n in hiddenLayer) { n.Inputs = inputs1; } for (int k = 0; k < hiddenLayer.Length; k++) { inputs2[k] = hiddenLayer[k].Output(); } foreach (Neuron n in outputLayer) { n.Inputs = inputs2; } double[] outputsErrors = new double[4]; for (int k = 0; k < outputLayer.Length; k++) { outputsErrors[k] = (inputs1[k] - outputLayer[k].Output()); IterationError += Pow(outputsErrors[k], 2); } for (int k = 0; k < outputLayer.Length; k++) { outputLayer[k].Error = Sigm.FunctionDerivative(outputLayer[k].Output()) * (outputsErrors[k]); } for (int k = 0; k < hiddenLayer.Length; k++) { double value = 0; for (int l = 0; l < hiddenLayer[k].Weights.Length; l++) { value += Sigm.FunctionDerivative(hiddenLayer[k].Output()) * outputLayer[l].Error * outputLayer[l].Weights[k]; } hiddenLayer[k].Error = value; } for (int k = 0; k < outputLayer.Length; k++) { outputLayer[k].UpdateWeights(); } for (int k = 0; k < hiddenLayer.Length; k++) { hiddenLayer[k].UpdateWeights(); } trainingSet.RemoveAt(randomIndex); EpochMSE += IterationError; } EpochMSE /= 4; RefillTrainingSet(trainingSet, testSamples); if (i % 20 == 1) { EpochsMSEs.Add(EpochMSE); } } for (int i = 0; i < 4; i++) { int maxIndex = 0; double[] inputs1 = trainingSet[i]; double[] inputs2 = new double[hiddenLayerCount]; foreach (Neuron n in hiddenLayer) { n.Inputs = inputs1; } for (int j = 0; j < hiddenLayer.Length; j++) { inputs2[j] = hiddenLayer[j].Output(); } foreach (Neuron n in outputLayer) { n.Inputs = inputs2; } for (int j = 0; j < outputLayer.Length; j++) { if (outputLayer[j].Output() > outputLayer[maxIndex].Output()) { maxIndex = j; } } List <int> indexes = GetNumbers(4); indexes.Remove(maxIndex); for (int j = 0; j < 4; j++) { WriteLine($"Input: {trainingSet[i][j]} Output: {outputLayer[j].Output()}"); } WriteLine(); if (outputLayer[indexes[0]].Output() < 0.5 && outputLayer[indexes[1]].Output() < 0.5 && outputLayer[indexes[2]].Output() < 0.5 && outputLayer[maxIndex].Output() > 0.5) { succesfulOutputsCount++; } } WriteLine("================================================"); ReadKey(); xs.Serialize(sw, EpochsMSEs); } WriteLine($"Successful: {succesfulOutputsCount} Total: {totalOutputsCount}"); XmlSerializer xs1 = new XmlSerializer(typeof(double[])); using (StreamWriter sw1 = new StreamWriter(variant.ToString() + "_" + bias.ToString() + "_Execution_stats.xml")) { xs1.Serialize(sw1, new double[] { succesfulOutputsCount, totalOutputsCount }); } ReadKey(); } if (variant == Variant.aproximation) { for (int counter = 1; counter <= executionsCount; counter++) { StreamWriter sw = new StreamWriter(variant.ToString() + "_" + bias.ToString() + "_Execution" + counter.ToString() + "EpochsDiffrences.xml"); XmlSerializer xs = new XmlSerializer(typeof(List <ApproximationData>)); List <ApproximationData> toSerialize = new List <ApproximationData>(); List <double> trainingDataInputs = new List <double>(); List <double> trainingDataOutputs = new List <double>(); List <double> testingDataInputs = new List <double>(); List <double> testingDataOutputs = new List <double>(); LoadTrainingDataFromFileAproximation(trainingDataInputs, trainingDataOutputs, testingDataInputs, testingDataOutputs); Neuron[] hiddenLayer = new Neuron[hiddenLayerCount]; Neuron[] outputLayer = new Neuron[1]; for (int i = 0; i < hiddenLayer.Length; i++) { hiddenLayer[i] = new Neuron(1, 1); hiddenLayer[i].RandomizeValues(); } outputLayer[0] = new Neuron(hiddenLayerCount, 2); outputLayer[0].RandomizeValues(); double TrainingMSE = 0; for (int i = 1; i <= epochsCount; i++) { List <int> numbers = GetNumbers(trainingDataInputs.Count); List <double> finalOutput = new List <double>(); TrainingMSE = 0; for (int j = 0; j < trainingDataInputs.Count; j++) { int randomIndex = gen.Next(numbers.Count); numbers.RemoveAt(randomIndex); double[] hiddenLayerInputs = new double[] { trainingDataInputs[randomIndex] }; double[] outputLayerInputs = new double[hiddenLayerCount]; foreach (Neuron n in hiddenLayer) { n.Inputs = hiddenLayerInputs; } for (int k = 0; k < hiddenLayer.Length; k++) { outputLayerInputs[k] = hiddenLayer[k].Output(); } outputLayer[0].Inputs = outputLayerInputs; double diffrence = 0; diffrence = trainingDataOutputs[randomIndex] - outputLayer[0].Output(); TrainingMSE += Pow(diffrence, 2); outputLayer[0].Error = Linear.FunctionDerivative(outputLayer[0].Output()) * diffrence; for (int k = 0; k < hiddenLayer.Length; k++) { hiddenLayer[k].Error = Sigm.FunctionDerivative(hiddenLayer[k].Output()) * outputLayer[0].Error * outputLayer[0].Weights[k]; hiddenLayer[k].UpdateWeights(); } outputLayer[0].UpdateWeights(); } TrainingMSE /= trainingDataInputs.Count; double TestingMSE = 0; for (int j = 0; j < testingDataInputs.Count; j++) { double[] hiddenLayerInputs = new double[] { testingDataInputs[j] }; double[] outputLayerInputs = new double[hiddenLayerCount]; foreach (Neuron n in hiddenLayer) { n.Inputs = hiddenLayerInputs; } for (int k = 0; k < hiddenLayer.Length; k++) { outputLayerInputs[k] = hiddenLayer[k].Output(); } outputLayer[0].Inputs = outputLayerInputs; TestingMSE += Pow(testingDataOutputs[j] - outputLayer[0].Output(), 2); if (i == epochsCount) { finalOutput.Add(outputLayer[0].Output()); } } if (i == epochsCount) { XmlSerializer xs1 = new XmlSerializer(typeof(List <double>)); using (StreamWriter sw1 = new StreamWriter(variant.ToString() + "_" + bias.ToString() + "_Execution" + counter.ToString() + "FinalOuput.xml")) { xs1.Serialize(sw1, finalOutput); } } TestingMSE /= testingDataInputs.Count; ApproximationData approximationData; approximationData.MSETrening = TrainingMSE; approximationData.MSETest = TestingMSE; if (i % 20 == 1) { toSerialize.Add(approximationData); } } xs.Serialize(sw, toSerialize); } } }