private double CalculateError(double[] desiredOutput) { // current and the next layers ActivationLayer layer, layerNext; // current and the next errors arrays double[] errors, errorsNext; // error values double error = 0, e, sum; // neuron's output value double output; // layers count int layersCount = network.LayersCount; // assume, that all neurons of the network have the same activation function IActivationFunction function = network.Layers[0].Neurons[0].ActivationFunction; // calculate error values for the last layer first layer = network.Layers[layersCount - 1]; errors = neuronErrors[layersCount - 1]; for (int i = 0, n = layer.NeuronsCount; i < n; i++) { output = layer.Neurons[i].Output; // error of the neuron e = desiredOutput[i] - output; // error multiplied with activation function's derivative errors[i] = e * function.Derivative(layer.Neurons[i].Z); // squre the error and sum it error += (e * e); } // calculate error values for other layers for (int j = layersCount - 2; j >= 0; j--) { layer = network.Layers[j]; layerNext = network.Layers[j + 1]; errors = neuronErrors[j]; errorsNext = neuronErrors[j + 1]; // for all neurons of the layer for (int i = 0, n = layer.NeuronsCount; i < n; i++) { sum = 0.0; // for all neurons of the next layer for (int k = 0, m = layerNext.NeuronsCount; k < m; k++) { sum += errorsNext[k] * layerNext.Neurons[k].Weights[i]; } errors[i] = sum * function.Derivative(layer.Neurons[i].Z); } } // return squared error of the last layer divided by 2 return(error / 2.0); }
public IMatrix Execute(IMatrix error, ITrainingContext context, bool calculateOutput, INeuralNetworkUpdateAccumulator updateAccumulator) { const float NEG_MAX = -1f, POS_MAX = 1f; using (var cd = _activation.Derivative(_c, null)) using (var ad = _activation.Derivative(_a, null)) { using (var i2 = _ones.Subtract(_i)) using (var f2 = _ones.Subtract(_f)) using (var o2 = _ones.Subtract(_o)) using (var errorO = error.PointwiseMultiply(_o)) using (var dO = error.PointwiseMultiply(_ca)) using (var dC = errorO.PointwiseMultiply(cd)) { // clip the gradients dO.Constrain(NEG_MAX, POS_MAX); dC.Constrain(NEG_MAX, POS_MAX); using (var dI = dC.PointwiseMultiply(_a)) using (var dF = dC.PointwiseMultiply(_pc)) using (var dA = dC.PointwiseMultiply(_i)) using (var dCp = dC.PointwiseMultiply(_f)) using (var dIi = dI.PointwiseMultiply(_i)) using (var dFf = dF.PointwiseMultiply(_f)) using (var dOo = dO.PointwiseMultiply(_o)) { var dA2 = dA.PointwiseMultiply(ad); var dI2 = dIi.PointwiseMultiply(i2); var dF2 = dFf.PointwiseMultiply(f2); var dO2 = dOo.PointwiseMultiply(o2); using (var ui = dI2.TransposeAndMultiply(_ui.Layer.Weight)) using (var uf = dF2.TransposeAndMultiply(_uf.Layer.Weight)) using (var uo = dO2.TransposeAndMultiply(_uo.Layer.Weight)) { var uc = dA2.TransposeAndMultiply(_uc.Layer.Weight); uc.AddInPlace(ui, 1f, 1f); uc.AddInPlace(uf, 1f, 1f); uc.AddInPlace(uo, 1f, 1f); _Update(dA2, _wc, _uc, updateAccumulator); _Update(dI2, _wi, _ui, updateAccumulator); _Update(dF2, _wf, _uf, updateAccumulator); _Update(dO2, _wo, _uo, updateAccumulator); _Cleanup(); return(uc); } } } } }
public void Train(NeuralNet neuralNet, double[] input, double[] output) { Propagate(neuralNet, input); var layers = neuralNet.Layers.OrderBy(l => l.Order).ToList(); var outputLayer = layers.LastOrDefault(); int i = 0; foreach (var outputNeuron in outputLayer.Neurons) { outputNeuron.Delta = (outputNeuron.Value - output[i]) * _activationFunction.Derivative(outputNeuron.Value); foreach (var previousDendrite in outputNeuron.PreviousDendrites) { previousDendrite.Delta = outputNeuron.Delta * previousDendrite.PreviousNeuron.Value; } i++; } for (i = layers.Count - 2; i >= 0; i--) { var layer = layers[i]; foreach (var neuron in layer.Neurons) { neuron.Delta = 0; foreach (var dendrite in neuron.NextDendrites) { neuron.Delta += dendrite.NextNeuron.Delta * dendrite.Weight; } neuron.Delta *= _activationFunction.Derivative(neuron.Value); foreach (var dendrite in neuron.PreviousDendrites) { dendrite.Delta += neuron.Delta * dendrite.PreviousNeuron.Value; } } } foreach (var layer in layers) { foreach (var neuron in layer.Neurons) { foreach (var dendrite in neuron.NextDendrites) { dendrite.Weight -= dendrite.Delta * neuralNet.TrainingRate; } } } }
private List <List <Matrix> > Backprop(Matrix pInput, Matrix pExpected) { var deltaWeights = new List <Matrix>(); var deltaBiases = new List <Matrix>(); foreach (var layer in _layers) { deltaBiases.Add(new Matrix(layer.Bias.Rows, layer.Bias.Columns)); deltaWeights.Add(new Matrix(layer.Weights.Rows, layer.Weights.Columns)); } ComputeOutputs(pInput); var m1 = CostFunction.MeanSquaredErrorDerivative(_layers.Last().ActivationsMatrix, pExpected); var m2 = ActivationFunction.Derivative(_layers.Last().WeightedInputMatrix); var delta = Matrix.HadamardProduct(m1, m2); deltaBiases[deltaBiases.Count - 1] = delta; if (_layers.Count > 1) { deltaWeights[deltaWeights.Count - 1] = delta * _layers[_layers.Count - 2].ActivationsMatrix.Transpose(); } else { deltaWeights[deltaWeights.Count - 1] = delta * inputLayer.Weights.Transpose(); } for (var i = deltaWeights.Count - 2; i >= 0; i--) { var activationMatrix = _layers[i].ActivationsMatrix; var derivedMatrix = ActivationFunction.Derivative(activationMatrix); delta = Matrix.HadamardProduct(_layers[i + 1].Weights.Transpose() * delta, derivedMatrix); deltaBiases[i] = delta; if (i == 0) { deltaWeights[i] = delta * inputLayer.Weights.Transpose(); } else { deltaWeights[i] = delta * _layers[i - 1].ActivationsMatrix.Transpose(); } } return(new List <List <Matrix> > { deltaWeights, deltaBiases }); }
public void Learn(double error, double learningRate) { Delta = error * _activationFunction.Derivative(OutputSignal); for (var i = 0; i < Weights.Length; i++) { var newWeight = Weights[i] - InputSignals[i] * Delta * learningRate; Weights[i] = newWeight; } }
public double Train(double[] input, double output) { double error = 0.5 * Enumerable .Range(0, inputs) .Select(x => Math.Pow(output - Compute(input), 2)) .Sum(); WeightsHidden = Enumerable .Range(0, hidden) .Select(x => alpha * activate.Derivative(error * input[x])) .ToArray(); double errorHidden = Enumerable .Range(0, hidden) .Select(y => WeightsHidden[y] + activate.Derivative(WeightsHidden[y] * error * alpha)) .Sum(); WeightsInput = Enumerable .Range(0, inputs) .Select(x => errorHidden * activate.Function(errorHidden) * WeightsInput[x]) .ToArray(); return(error); }
public double CalculateGradient(double?target = null) { var derivative = ActivationFunction.Derivative(Value); if (target != null) { Gradient = derivative * CalculateError(target.Value); } else { Gradient = derivative * OutputSynapses.Sum(synapse => synapse.OutputNeuron.Gradient * synapse.Weight); } return(Gradient); }
/// <summary> /// Calcualates the derivative of the weighted sum with respect to the specified <see cref="IActivationFunction"/>. /// </summary> /// <param name="activationFunction">The activation function.</param> /// <param name="nodeDepth">The node depth.</param> /// <param name="oneHots">The one hot values.</param> /// <returns>The derivatives.</returns> public virtual double[][] WeightedSumDerivative(IActivationFunction activationFunction, int nodeDepth, int[] oneHots) { double[][] cachedWeightedSum = _neuralNetwork.WeightedSumCache[nodeDepth]; int rowCount = cachedWeightedSum.Length; int colCount = cachedWeightedSum[0].Length; double[][] result = VectorUtilities.CreateMatrix(rowCount, colCount); for (var i = 0; i < rowCount; i++) { result[i] = activationFunction.Derivative(cachedWeightedSum[i], oneHots[i].OneHot(colCount)); } return(result); }
public IMatrix Execute(IMatrix curr, ITrainingContext context, bool calculateOutput, INeuralNetworkUpdateAccumulator updateAccumulator) { using (var ad = _activation.Derivative(_output, curr)) { // clip the gradient ad.Constrain(-1f, 1f); var delta = curr.PointwiseMultiply(ad); var prevDelta = delta.TransposeAndMultiply(_memoryUpdater.Layer.Weight); var memoryUpdate = _memory.TransposeThisAndMultiply(delta); var inputUpdate = _input.TransposeThisAndMultiply(delta); //_input.Dispose(); _memory.Dispose(); _output.Dispose(); updateAccumulator.Record(_memoryUpdater, delta, memoryUpdate); updateAccumulator.Record(_inputUpdater, delta.Clone(), inputUpdate); return(prevDelta); } }
public override void Derivative(double error, Indexer inputs, Indexer weights, Indexer inputsDerivative, Indexer weightsDerivative) { var sum = weights[inputsCount]; for (int i = 0; i < inputsCount; i++) { sum += inputs[i] * weights[i]; } var d = Function.Derivative(sum); if (double.IsNaN(d)) { throw new Exception(); } weightsDerivative[inputsCount] += error * d; for (int i = 0; i < inputsCount; i++) { weightsDerivative[i] += error * d * inputs[i]; inputsDerivative[i] += error * d * weights[i]; } }
public void CalculateError(float desiredOutput) { _derivativeOutput = _transfer.Derivative(_output); _error = (desiredOutput - _output) * _derivativeOutput; }
public void Train(double errorContribution) { Error = errorContribution * _activationFunction.Derivative(GetValue()); Inputs.Select(i => i.Value).ToList().ForEach(a => a.UpdateWeight(Error)); _cachedOutput = null; }