public void GetGradients(double costGradient, double[] previousLayerActivations, ActivationFunctions activation, double bias, out List <double> weigthsGradients, out List <double> previousLayerActivationGradients, out double biasGradient)
{
weigthsGradients = previousLayerActivationGradients = new List <double>();
double neuronLinearFunction = LinearFunction(previousLayerActivations, Weigths, bias);
double ActivationFuncGradient = Derivatives.DerivativeOf(neuronLinearFunction, activation) * costGradient;
for (int weigthIndex = 0; weigthIndex < Weigths.Count; weigthIndex++)
{
weigthsGradients.Add(ActivationFuncGradient * previousLayerActivations[weigthIndex]);
previousLayerActivationGradients.Add(ActivationFuncGradient * Weigths[weigthIndex]);
}
biasGradient = costGradient * ActivationFuncGradient;
}
public static List <double> GetCostGradients(double[] output, double[] expected, CostFunctions costFunction, out double cost)
{
cost = GetCostOf(output, expected, costFunction);
List <double> Gradients = new List <double>();
for (int i = 0; i < Math.Min(output.Length, expected.Length); i++)
{
if (double.IsNaN(expected[i]))
{
Gradients.Add(0);
}
else
{
Gradients.Add(Derivatives.DerivativeOf(output[i], expected[i], costFunction));
}
}
return(Gradients);
}
/// <summary>
///
/// </summary>
/// <param name="costGradient"></param>
/// <param name="bias"></param>
/// <param name="initialValues">Remember to add hiddenState at lstmWeigths</param>
/// <param name="recurrentWeigth"></param>
/// <param name="prevHiddenGrad">prev inital cost gradient</param>
/// <param name="preActivations"></param>
/// <param name="weigths"></param>
/// <param name="weigthGrads"></param>
/// <param name="recWeigthGrad"></param>
/// <param name="biasGrad"></param>
public void GetGradients(double costGradient, double bias, double hiddenState, double prevHiddenGrad, double[] prevActivations, ActivationFunctions activationFunction
, out List <double> prevActivationsGrads, out List <double> weigthGrads, out double recWeigthGrad, out double biasGrad)
{
//execution
double linearFunc = LinearFunction(prevActivations, Weigths, bias);
double linearWithRecurrent = linearFunc + recurrentWeigth * hiddenState;
//end execution
double currentGrad = costGradient;
currentGrad *= Derivatives.DerivativeOf(linearWithRecurrent, activationFunction);
double recurrentMultiplicationDerivative = Derivatives.MultiplicationDerivative(hiddenState, prevHiddenGrad, recurrentWeigth, 0);
double linearFuncDerivative = 0;
for (int i = 0; i < prevActivations.Length; i++)
{
linearFuncDerivative += prevActivations[i]; //Short for multiplication derivative, bias obviated ´cause it has to be added and derivative of a constant is 0
}
currentGrad *= linearFuncDerivative + recurrentMultiplicationDerivative;
biasGrad = currentGrad;
recWeigthGrad = currentGrad * hiddenState;
weigthGrads = new List <double>();
foreach (var input in prevActivations)
{
weigthGrads.Add(input * currentGrad);
}
prevActivationsGrads = new List <double>();
foreach (var weigth in Weigths)
{
prevActivationsGrads.Add(weigth * currentGrad);
}
}
/// <summary>
/// if expected has NaN it has not expected output
/// </summary>
public void SupervisedBackProp(List <double[]> input, List <double[]> expectedOutput, CostFunctions costFunction, List <List <NeuronValues> > startingStates, out double cost, out List <List <List <NeuronValues> > > grads, out List <double[]> biasGrads)
{
cost = 0;
if (input.Count != expectedOutput.Count)
{
throw new ArgumentOutOfRangeException();
}
var copy = new TemporalNetwork(ActivationFunction, TemporalLayers);
copy.SetTemporalStates(startingStates);
List <double[]> costs = new List <double[]>();
for (int t = 0; t < input.Count; t++)
{
double[] outputT = copy.ExecuteNetwork(input[t]);
costs.Add(new double[expectedOutput[t].Length]);
cost += Cost.GetCostOf(outputT, expectedOutput[t], costFunction);//REPAIR
for (int i = 0; i < expectedOutput[i].Length; i++)
{
costs[t][i] = Derivatives.DerivativeOf(outputT[i], expectedOutput[t][i], costFunction);
}
}
GetGradients(costs, input, startingStates, out grads, out biasGrads);
}
