public static NeuronWeigths operator +(NeuronWeigths a, NeuronWeigths b)
{
bool hasLSTM = a.LSTMWeigths != null || b.LSTMWeigths != null;
LSTMWeigths lSTMWeigths = null;
if (hasLSTM)
{
if (a.LSTMWeigths != null && b.LSTMWeigths != null)
{
a.LSTMWeigths.cellState += b.LSTMWeigths.cellState;
a.LSTMWeigths.hiddenState += b.LSTMWeigths.hiddenState;
a.LSTMWeigths.forgetWeigth += b.LSTMWeigths.forgetWeigth;
a.LSTMWeigths.storeWeigth += b.LSTMWeigths.storeWeigth;
a.LSTMWeigths.outputWeigth += b.LSTMWeigths.outputWeigth;
lSTMWeigths = a.LSTMWeigths;
}
else
{
if (a.LSTMWeigths != null)
{
lSTMWeigths = a.LSTMWeigths;
}
else
{
lSTMWeigths = b.LSTMWeigths;
}
}
}
return(new NeuronWeigths(new List <double>(AddArrays(a.weigths.ToArray(), b.weigths.ToArray())), lSTMWeigths));
}
public LSTMCell(int previousLayerLenght, List <double> weigths = null, LSTMWeigths LSTMWeigths = null) : base(previousLayerLenght, weigths)
{
recurrent = LSTMWeigths;
if (LSTMWeigths == null)
{
recurrent = new LSTMWeigths();
recurrent.cellState = 0;
recurrent.hiddenState = 0;
}
}
//TODO: Change out arrays to lists
public void GetGrads(double[] prevActivations, double bias, LSTMWeigths cellhiddenStates, double cellStateGrad, double prevCellStateGrad, double prevHiddenGrad, double outputWeigthGrad
, out List <double> prevActivationsGrads, out List <double> weigthsGrads, out double biasGrad, out LSTMWeigths LSTMWeigthsGrads)
{
LSTMWeigthsGrads = new LSTMWeigths(0, 0, outputWeigthGrad);
LSTMWeigths initialState = cellhiddenStates;
cellhiddenStates.hiddenState += bias;
for (int i = 0; i < prevActivations.Length; i++)
{
cellhiddenStates.hiddenState += prevActivations[i] * Weigths[i];
}
double linearFunc = cellhiddenStates.hiddenState;
double hiddenStateSigmoid = SigmoidActivation(cellhiddenStates.hiddenState);
//forget gate
double forgetMultiplication;
cellhiddenStates.cellState *= forgetMultiplication = hiddenStateSigmoid * recurrent.forgetWeigth;
//store gate
cellhiddenStates.cellState += SigmoidActivation(cellhiddenStates.hiddenState) * recurrent.storeWeigth * TanhActivation(cellhiddenStates.hiddenState);
//output gate
cellhiddenStates.hiddenState = hiddenStateSigmoid * recurrent.outputWeigth * TanhActivation(cellhiddenStates.cellState);
double currentGrad = cellStateGrad;
double storeWeigthMultiplicationDerivative = hiddenStateSigmoid * Derivatives.SigmoidDerivative(linearFunc);
double storeGateMultiplicationDerivative = Derivatives.MultiplicationDerivative(hiddenStateSigmoid * recurrent.storeWeigth, storeWeigthMultiplicationDerivative
, TanhActivation(linearFunc), Derivatives.TanhDerivative(linearFunc));
double sigmoidDerivative = Derivatives.SigmoidDerivative(linearFunc);
double forgetWeigthMultiplicationDerivative = sigmoidDerivative * hiddenStateSigmoid;
double forgetGateMultiplicationDerivative = Derivatives.MultiplicationDerivative(initialState.cellState, prevCellStateGrad, forgetMultiplication, forgetWeigthMultiplicationDerivative);
//To store addition
currentGrad *= forgetGateMultiplicationDerivative + storeGateMultiplicationDerivative;
double gradToStoreWeigth = currentGrad;
gradToStoreWeigth *= storeGateMultiplicationDerivative;
gradToStoreWeigth *= storeWeigthMultiplicationDerivative;
LSTMWeigthsGrads.storeWeigth = gradToStoreWeigth;
//To Forget multiplication
currentGrad *= forgetGateMultiplicationDerivative;
LSTMWeigthsGrads.forgetWeigth = currentGrad * forgetWeigthMultiplicationDerivative;
biasGrad = prevHiddenGrad * bias;
prevActivationsGrads = weigthsGrads = new List <double>();
for (int i = 0; i < prevActivations.Length; i++)
{
prevActivationsGrads.Add(prevHiddenGrad * Weigths[i]);
weigthsGrads.Add(prevHiddenGrad * prevActivations[i]);
}
}
public static LSTMCell FromString(string str)
{
string[] recurrentS = str.Remove(0, str.IndexOf("(")).Replace(")", "").Split(new string[] { "(" }, StringSplitOptions.RemoveEmptyEntries);//separateParenthesis and forgetLinearWeigths
str = str.Remove(str.IndexOf("("));
List <double> weigths = StringParser.StringToDoubleList(str);
LSTMWeigths lSTMweigths = LSTMWeigths.FromString(recurrentS[0]);
if (recurrentS.Length > 1)
{
List <double> states = StringParser.StringToDoubleList(recurrentS[1]);
lSTMweigths.hiddenState = states[0];
lSTMweigths.cellState = states[1];
}
return(new LSTMCell(weigths.Count, weigths, lSTMweigths));
}
public void GetInitialGrads(double initialCost, double[] prevActivations, double bias, LSTMWeigths cellhiddenStates, out double cellStateGrad, out double outputWeigthGrad)
{
cellhiddenStates.hiddenState += bias;
double linearFunction;
for (int i = 0; i < prevActivations.Length; i++)
{
cellhiddenStates.hiddenState += prevActivations[i] * Weigths[i];
}
linearFunction = cellhiddenStates.hiddenState;
double hiddenStateSigmoid = SigmoidActivation(cellhiddenStates.hiddenState);
//forget gate
cellhiddenStates.cellState *= hiddenStateSigmoid * recurrent.forgetWeigth;
//store gate
cellhiddenStates.cellState += SigmoidActivation(cellhiddenStates.hiddenState) * recurrent.storeWeigth * TanhActivation(cellhiddenStates.hiddenState);
//output gate
cellhiddenStates.hiddenState = hiddenStateSigmoid * recurrent.outputWeigth * TanhActivation(cellhiddenStates.cellState);
//derivative of output weigth
double outputWeigthDerivative = Derivatives.MultiplicationDerivative(cellhiddenStates.hiddenState, Derivatives.SigmoidDerivative(linearFunction), recurrent.outputWeigth, 0);
//* output gate derivative
initialCost *= Derivatives.MultiplicationDerivative(hiddenStateSigmoid * recurrent.outputWeigth, outputWeigthDerivative, TanhActivation(cellhiddenStates.cellState), Derivatives.TanhDerivative(cellhiddenStates.cellState));
outputWeigthGrad = initialCost * outputWeigthDerivative;
initialCost *= Derivatives.TanhDerivative(cellhiddenStates.cellState);
cellStateGrad = initialCost;
}
public NeuronWeigths(List <double> weigths, LSTMWeigths LSTMWeigths)
{
this.weigths = new List <double>();
this.weigths.AddRange(weigths);
this.LSTMWeigths = LSTMWeigths;
}
