private double CalculateError(double[] desiredOutput)
{
// current and the next layers
ActivationLayer layer, layerNext;
// current and the next errors arrays
double[] errors, errorsNext;
// error values
double error = 0, e, sum;
// neuron's output value
double output;
// layers count
int layersCount = network.LayersCount;
// assume, that all neurons of the network have the same activation function
IActivationFunction function = network.Layers[0].Neurons[0].ActivationFunction;
// calculate error values for the last layer first
layer = network.Layers[layersCount - 1];
errors = neuronErrors[layersCount - 1];
for (int i = 0, n = layer.NeuronsCount; i < n; i++)
{
output = layer.Neurons[i].Output;
// error of the neuron
e = desiredOutput[i] - output;
// error multiplied with activation function's derivative
errors[i] = e * function.Derivative(layer.Neurons[i].Z);
// squre the error and sum it
error += (e * e);
}
// calculate error values for other layers
for (int j = layersCount - 2; j >= 0; j--)
{
layer = network.Layers[j];
layerNext = network.Layers[j + 1];
errors = neuronErrors[j];
errorsNext = neuronErrors[j + 1];
// for all neurons of the layer
for (int i = 0, n = layer.NeuronsCount; i < n; i++)
{
sum = 0.0;
// for all neurons of the next layer
for (int k = 0, m = layerNext.NeuronsCount; k < m; k++)
{
sum += errorsNext[k] * layerNext.Neurons[k].Weights[i];
}
errors[i] = sum * function.Derivative(layer.Neurons[i].Z);
}
}
// return squared error of the last layer divided by 2
return(error / 2.0);
}
public IMatrix Execute(IMatrix error, ITrainingContext context, bool calculateOutput, INeuralNetworkUpdateAccumulator updateAccumulator)
{
const float NEG_MAX = -1f, POS_MAX = 1f;
using (var cd = _activation.Derivative(_c, null))
using (var ad = _activation.Derivative(_a, null)) {
using (var i2 = _ones.Subtract(_i))
using (var f2 = _ones.Subtract(_f))
using (var o2 = _ones.Subtract(_o))
using (var errorO = error.PointwiseMultiply(_o))
using (var dO = error.PointwiseMultiply(_ca))
using (var dC = errorO.PointwiseMultiply(cd)) {
// clip the gradients
dO.Constrain(NEG_MAX, POS_MAX);
dC.Constrain(NEG_MAX, POS_MAX);
using (var dI = dC.PointwiseMultiply(_a))
using (var dF = dC.PointwiseMultiply(_pc))
using (var dA = dC.PointwiseMultiply(_i))
using (var dCp = dC.PointwiseMultiply(_f))
using (var dIi = dI.PointwiseMultiply(_i))
using (var dFf = dF.PointwiseMultiply(_f))
using (var dOo = dO.PointwiseMultiply(_o)) {
var dA2 = dA.PointwiseMultiply(ad);
var dI2 = dIi.PointwiseMultiply(i2);
var dF2 = dFf.PointwiseMultiply(f2);
var dO2 = dOo.PointwiseMultiply(o2);
using (var ui = dI2.TransposeAndMultiply(_ui.Layer.Weight))
using (var uf = dF2.TransposeAndMultiply(_uf.Layer.Weight))
using (var uo = dO2.TransposeAndMultiply(_uo.Layer.Weight)) {
var uc = dA2.TransposeAndMultiply(_uc.Layer.Weight);
uc.AddInPlace(ui, 1f, 1f);
uc.AddInPlace(uf, 1f, 1f);
uc.AddInPlace(uo, 1f, 1f);
_Update(dA2, _wc, _uc, updateAccumulator);
_Update(dI2, _wi, _ui, updateAccumulator);
_Update(dF2, _wf, _uf, updateAccumulator);
_Update(dO2, _wo, _uo, updateAccumulator);
_Cleanup();
return(uc);
}
}
}
}
}
public void Train(NeuralNet neuralNet, double[] input, double[] output)
{
Propagate(neuralNet, input);
var layers = neuralNet.Layers.OrderBy(l => l.Order).ToList();
var outputLayer = layers.LastOrDefault();
int i = 0;
foreach (var outputNeuron in outputLayer.Neurons)
{
outputNeuron.Delta = (outputNeuron.Value - output[i]) * _activationFunction.Derivative(outputNeuron.Value);
foreach (var previousDendrite in outputNeuron.PreviousDendrites)
{
previousDendrite.Delta = outputNeuron.Delta * previousDendrite.PreviousNeuron.Value;
}
i++;
}
for (i = layers.Count - 2; i >= 0; i--)
{
var layer = layers[i];
foreach (var neuron in layer.Neurons)
{
neuron.Delta = 0;
foreach (var dendrite in neuron.NextDendrites)
{
neuron.Delta += dendrite.NextNeuron.Delta * dendrite.Weight;
}
neuron.Delta *= _activationFunction.Derivative(neuron.Value);
foreach (var dendrite in neuron.PreviousDendrites)
{
dendrite.Delta += neuron.Delta * dendrite.PreviousNeuron.Value;
}
}
}
foreach (var layer in layers)
{
foreach (var neuron in layer.Neurons)
{
foreach (var dendrite in neuron.NextDendrites)
{
dendrite.Weight -= dendrite.Delta * neuralNet.TrainingRate;
}
}
}
}
private List <List <Matrix> > Backprop(Matrix pInput, Matrix pExpected)
{
var deltaWeights = new List <Matrix>();
var deltaBiases = new List <Matrix>();
foreach (var layer in _layers)
{
deltaBiases.Add(new Matrix(layer.Bias.Rows, layer.Bias.Columns));
deltaWeights.Add(new Matrix(layer.Weights.Rows, layer.Weights.Columns));
}
ComputeOutputs(pInput);
var m1 = CostFunction.MeanSquaredErrorDerivative(_layers.Last().ActivationsMatrix, pExpected);
var m2 = ActivationFunction.Derivative(_layers.Last().WeightedInputMatrix);
var delta = Matrix.HadamardProduct(m1, m2);
deltaBiases[deltaBiases.Count - 1] = delta;
if (_layers.Count > 1)
{
deltaWeights[deltaWeights.Count - 1] = delta * _layers[_layers.Count - 2].ActivationsMatrix.Transpose();
}
else
{
deltaWeights[deltaWeights.Count - 1] = delta * inputLayer.Weights.Transpose();
}
for (var i = deltaWeights.Count - 2; i >= 0; i--)
{
var activationMatrix = _layers[i].ActivationsMatrix;
var derivedMatrix = ActivationFunction.Derivative(activationMatrix);
delta = Matrix.HadamardProduct(_layers[i + 1].Weights.Transpose() * delta, derivedMatrix);
deltaBiases[i] = delta;
if (i == 0)
{
deltaWeights[i] = delta * inputLayer.Weights.Transpose();
}
else
{
deltaWeights[i] = delta * _layers[i - 1].ActivationsMatrix.Transpose();
}
}
return(new List <List <Matrix> > {
deltaWeights, deltaBiases
});
}
public void Learn(double error, double learningRate)
{
Delta = error * _activationFunction.Derivative(OutputSignal);
for (var i = 0; i < Weights.Length; i++)
{
var newWeight = Weights[i] - InputSignals[i] * Delta * learningRate;
Weights[i] = newWeight;
}
}
public double Train(double[] input, double output)
{
double error = 0.5 * Enumerable
.Range(0, inputs)
.Select(x => Math.Pow(output - Compute(input), 2))
.Sum();
WeightsHidden = Enumerable
.Range(0, hidden)
.Select(x => alpha * activate.Derivative(error * input[x]))
.ToArray();
double errorHidden = Enumerable
.Range(0, hidden)
.Select(y => WeightsHidden[y] + activate.Derivative(WeightsHidden[y] * error * alpha))
.Sum();
WeightsInput = Enumerable
.Range(0, inputs)
.Select(x => errorHidden * activate.Function(errorHidden) * WeightsInput[x])
.ToArray();
return(error);
}
public double CalculateGradient(double?target = null)
{
var derivative = ActivationFunction.Derivative(Value);
if (target != null)
{
Gradient = derivative * CalculateError(target.Value);
}
else
{
Gradient = derivative * OutputSynapses.Sum(synapse => synapse.OutputNeuron.Gradient * synapse.Weight);
}
return(Gradient);
}
/// <summary>
/// Calcualates the derivative of the weighted sum with respect to the specified <see cref="IActivationFunction"/>.
/// </summary>
/// <param name="activationFunction">The activation function.</param>
/// <param name="nodeDepth">The node depth.</param>
/// <param name="oneHots">The one hot values.</param>
/// <returns>The derivatives.</returns>
public virtual double[][] WeightedSumDerivative(IActivationFunction activationFunction, int nodeDepth, int[] oneHots)
{
double[][] cachedWeightedSum = _neuralNetwork.WeightedSumCache[nodeDepth];
int rowCount = cachedWeightedSum.Length;
int colCount = cachedWeightedSum[0].Length;
double[][] result = VectorUtilities.CreateMatrix(rowCount, colCount);
for (var i = 0; i < rowCount; i++)
{
result[i] = activationFunction.Derivative(cachedWeightedSum[i], oneHots[i].OneHot(colCount));
}
return(result);
}
public IMatrix Execute(IMatrix curr, ITrainingContext context, bool calculateOutput, INeuralNetworkUpdateAccumulator updateAccumulator)
{
using (var ad = _activation.Derivative(_output, curr)) {
// clip the gradient
ad.Constrain(-1f, 1f);
var delta = curr.PointwiseMultiply(ad);
var prevDelta = delta.TransposeAndMultiply(_memoryUpdater.Layer.Weight);
var memoryUpdate = _memory.TransposeThisAndMultiply(delta);
var inputUpdate = _input.TransposeThisAndMultiply(delta);
//_input.Dispose();
_memory.Dispose();
_output.Dispose();
updateAccumulator.Record(_memoryUpdater, delta, memoryUpdate);
updateAccumulator.Record(_inputUpdater, delta.Clone(), inputUpdate);
return(prevDelta);
}
}
public override void Derivative(double error, Indexer inputs, Indexer weights, Indexer inputsDerivative, Indexer weightsDerivative)
{
var sum = weights[inputsCount];
for (int i = 0; i < inputsCount; i++)
{
sum += inputs[i] * weights[i];
}
var d = Function.Derivative(sum);
if (double.IsNaN(d))
{
throw new Exception();
}
weightsDerivative[inputsCount] += error * d;
for (int i = 0; i < inputsCount; i++)
{
weightsDerivative[i] += error * d * inputs[i];
inputsDerivative[i] += error * d * weights[i];
}
}
public void CalculateError(float desiredOutput)
{
_derivativeOutput = _transfer.Derivative(_output);
_error = (desiredOutput - _output) * _derivativeOutput;
}
public void Train(double errorContribution)
{
Error = errorContribution * _activationFunction.Derivative(GetValue());
Inputs.Select(i => i.Value).ToList().ForEach(a => a.UpdateWeight(Error));
_cachedOutput = null;
}
