public void GetGradients(double costGradient, double[] previousLayerActivations, ActivationFunctions activation, double bias, out List <double> weigthsGradients, out List <double> previousLayerActivationGradients, out double biasGradient) { weigthsGradients = previousLayerActivationGradients = new List <double>(); double neuronLinearFunction = LinearFunction(previousLayerActivations, Weigths, bias); double ActivationFuncGradient = Derivatives.DerivativeOf(neuronLinearFunction, activation) * costGradient; for (int weigthIndex = 0; weigthIndex < Weigths.Count; weigthIndex++) { weigthsGradients.Add(ActivationFuncGradient * previousLayerActivations[weigthIndex]); previousLayerActivationGradients.Add(ActivationFuncGradient * Weigths[weigthIndex]); } biasGradient = costGradient * ActivationFuncGradient; }
public static List <double> GetCostGradients(double[] output, double[] expected, CostFunctions costFunction, out double cost) { cost = GetCostOf(output, expected, costFunction); List <double> Gradients = new List <double>(); for (int i = 0; i < Math.Min(output.Length, expected.Length); i++) { if (double.IsNaN(expected[i])) { Gradients.Add(0); } else { Gradients.Add(Derivatives.DerivativeOf(output[i], expected[i], costFunction)); } } return(Gradients); }
/// <summary> /// /// </summary> /// <param name="costGradient"></param> /// <param name="bias"></param> /// <param name="initialValues">Remember to add hiddenState at lstmWeigths</param> /// <param name="recurrentWeigth"></param> /// <param name="prevHiddenGrad">prev inital cost gradient</param> /// <param name="preActivations"></param> /// <param name="weigths"></param> /// <param name="weigthGrads"></param> /// <param name="recWeigthGrad"></param> /// <param name="biasGrad"></param> public void GetGradients(double costGradient, double bias, double hiddenState, double prevHiddenGrad, double[] prevActivations, ActivationFunctions activationFunction , out List <double> prevActivationsGrads, out List <double> weigthGrads, out double recWeigthGrad, out double biasGrad) { //execution double linearFunc = LinearFunction(prevActivations, Weigths, bias); double linearWithRecurrent = linearFunc + recurrentWeigth * hiddenState; //end execution double currentGrad = costGradient; currentGrad *= Derivatives.DerivativeOf(linearWithRecurrent, activationFunction); double recurrentMultiplicationDerivative = Derivatives.MultiplicationDerivative(hiddenState, prevHiddenGrad, recurrentWeigth, 0); double linearFuncDerivative = 0; for (int i = 0; i < prevActivations.Length; i++) { linearFuncDerivative += prevActivations[i]; //Short for multiplication derivative, bias obviated ´cause it has to be added and derivative of a constant is 0 } currentGrad *= linearFuncDerivative + recurrentMultiplicationDerivative; biasGrad = currentGrad; recWeigthGrad = currentGrad * hiddenState; weigthGrads = new List <double>(); foreach (var input in prevActivations) { weigthGrads.Add(input * currentGrad); } prevActivationsGrads = new List <double>(); foreach (var weigth in Weigths) { prevActivationsGrads.Add(weigth * currentGrad); } }
/// <summary> /// if expected has NaN it has not expected output /// </summary> public void SupervisedBackProp(List <double[]> input, List <double[]> expectedOutput, CostFunctions costFunction, List <List <NeuronValues> > startingStates, out double cost, out List <List <List <NeuronValues> > > grads, out List <double[]> biasGrads) { cost = 0; if (input.Count != expectedOutput.Count) { throw new ArgumentOutOfRangeException(); } var copy = new TemporalNetwork(ActivationFunction, TemporalLayers); copy.SetTemporalStates(startingStates); List <double[]> costs = new List <double[]>(); for (int t = 0; t < input.Count; t++) { double[] outputT = copy.ExecuteNetwork(input[t]); costs.Add(new double[expectedOutput[t].Length]); cost += Cost.GetCostOf(outputT, expectedOutput[t], costFunction);//REPAIR for (int i = 0; i < expectedOutput[i].Length; i++) { costs[t][i] = Derivatives.DerivativeOf(outputT[i], expectedOutput[t][i], costFunction); } } GetGradients(costs, input, startingStates, out grads, out biasGrads); }