public static Dendrite BuildDendrite(Neuron upstreamNeuron, Neuron downStreamNeuron, IWeightBuilder weightBuilder) { Dendrite toReturn = new Dendrite(); toReturn.DownStreamNeuron = downStreamNeuron; toReturn.UpStreamNeuron = upstreamNeuron; toReturn.Weight = weightBuilder.BuildWeight(); return(toReturn); }
public void UpdateNetwork(double stepSize, int sizeOfTrainingData, int batchSize) { // Generate the list of layers to update, starting from the // beginnign to the end (the output layer). List <Layer> layersToUpdate = new List <Layer>(); foreach (HiddenLayer hiddenLayer in HiddenLayers) { layersToUpdate.Add(hiddenLayer); } layersToUpdate.Add(OutputLayer); // start from the last layer and work my way backward. for (int c = layersToUpdate.Count - 1; c >= 0; c--) { // take each of the neurons in the layer and update the bias // take each of the dendrites attached to the neuron and update the weight. for (int n = 0; n < layersToUpdate[c].Neurons.Count; n++) { // the current neuron being examined. Neuron thisNeuron = layersToUpdate[c].Neurons[n]; bool isNeuronDropped = layersToUpdate[c].DropOutMask[n] == 0; // get the sum of all the errors of this neuron across the batch. // divide by the batch size. This is the average error for the neuron. double averageNeuronError = thisNeuron.SumOfErrorsOfNeuron /= (batchSize * 1.0); thisNeuron.ClearError(); // updating the weight is just subtracting the bias (the delta is the gradient // of the bias). dC/db = delta. Make sure to weight the delta by the step size. thisNeuron.Bias = thisNeuron.Bias - (stepSize * averageNeuronError); for (int d = 0; d < thisNeuron.UpstreamDendrites.Count; d++) { Dendrite dendrite = thisNeuron.UpstreamDendrites[d]; double averageErrorWrtWeight = dendrite.SumOfErrorsWrtWeights / (batchSize * 1.0); dendrite.ClearError(); double regularization = 0.0; if (_regularizationFunction != null && !isNeuronDropped) { regularization = _regularizationFunction.Compute(dendrite.Weight, sizeOfTrainingData); } dendrite.Weight = dendrite.Weight - (stepSize * (averageErrorWrtWeight + regularization)); } } } }
public static Neuron BuildNeuron(IWeightBuilder weightBuilder, Layer previousLayer) { Neuron toReturn = new Neuron(); toReturn.Bias = 0; if (previousLayer != null) { for (int c = 0; c < previousLayer.Neurons.Count; c++) { Neuron previousNeuron = previousLayer.Neurons[c]; Dendrite dendrite = Dendrite.BuildDendrite(previousNeuron, toReturn, weightBuilder); toReturn.UpstreamDendrites.Add(dendrite); previousNeuron.DownstreamDendrites.Add(dendrite); } } return(toReturn); }
public double Backpropagation(double[] expectedValues) { double totalNetworkCost = 0.0; // Compute error for the output neurons to get the ball rolling. // See https://github.com/kwende/CSharpNeuralNetworkExplorations/blob/master/Explorations/SimpleMLP/Documentation/OutputNeuronErrors.png for (int d = 0; d < expectedValues.Length; d++) { Neuron outputNeuronBeingExamined = OutputLayer.Neurons[d]; double expectedOutput = expectedValues[d]; double actualOutput = outputNeuronBeingExamined.Activation; double actualInput = outputNeuronBeingExamined.TotalInput; double cost = _costFunction.Compute(expectedOutput, actualOutput); totalNetworkCost += cost; double errorRelativeToActivation = (_costFunction.ComputeDerivativeWRTActivation(actualOutput, expectedOutput)); double errorWrtToNeuron = errorRelativeToActivation * Math.Sigmoid.ComputeDerivative(actualInput); outputNeuronBeingExamined.AddError(errorWrtToNeuron); for (int e = 0; e < outputNeuronBeingExamined.UpstreamDendrites.Count; e++) { Dendrite dendrite = outputNeuronBeingExamined.UpstreamDendrites[e]; Neuron upstreamNeuron = (Neuron)dendrite.UpStreamNeuron; double errorRelativeToWeight = (errorWrtToNeuron * upstreamNeuron.Activation); dendrite.AddError(errorRelativeToWeight); } } // Compute error for each neuron in each layer moving backwards (backprop). for (int d = HiddenLayers.Count - 1; d >= 0; d--) { HiddenLayer hiddenLayer = HiddenLayers[d]; for (int e = 0; e < hiddenLayer.Neurons.Count; e++) { Neuron thisNeuron = (Neuron)hiddenLayer.Neurons[e]; double dropoutBit = hiddenLayer.DropOutMask[e]; double input = thisNeuron.TotalInput; double errorSum = 0.0; List <Dendrite> downStreamDendrites = thisNeuron.DownstreamDendrites; for (int f = 0; f < downStreamDendrites.Count; f++) { Dendrite currentDendrite = downStreamDendrites[f]; Neuron downStreamNeuron = currentDendrite.DownStreamNeuron; double delta = downStreamNeuron.CurrentNeuronError; double weight = currentDendrite.Weight; errorSum += delta * weight; } double errorWrtToThisNeuron = errorSum * Math.Sigmoid.ComputeDerivative(input) * dropoutBit; thisNeuron.AddError(errorWrtToThisNeuron); for (int f = 0; f < thisNeuron.UpstreamDendrites.Count; f++) { Dendrite dendrite = thisNeuron.UpstreamDendrites[f]; Neuron upstreamNeuron = (Neuron)dendrite.UpStreamNeuron; double errorRelativeToWeight = (errorWrtToThisNeuron * upstreamNeuron.Activation); dendrite.AddError(errorRelativeToWeight); } } } return(totalNetworkCost); }