public Matrix Activate(Matrix input, Graph g)
{
Matrix concat = g.ConcatVectors(input, _context);
Matrix sum = g.Mul(_w, concat); sum = g.Add(sum, _b);
Matrix output = g.Nonlin(_f, sum);
//rollover activations for next iteration
_context = output;
return output;
}
public Matrix Activate(Matrix input, Graph g)
{
//input gate
Matrix sum0 = g.Mul(_wix, input);
Matrix sum1 = g.Mul(_wih, _hiddenContext);
Matrix inputGate = g.Nonlin(_inputGateActivation, g.Add(g.Add(sum0, sum1), _inputBias));
//forget gate
Matrix sum2 = g.Mul(_wfx, input);
Matrix sum3 = g.Mul(_wfh, _hiddenContext);
Matrix forgetGate = g.Nonlin(_forgetGateActivation, g.Add(g.Add(sum2, sum3), _forgetBias));
//output gate
Matrix sum4 = g.Mul(_wox, input);
Matrix sum5 = g.Mul(_woh, _hiddenContext);
Matrix outputGate = g.Nonlin(_outputGateActivation, g.Add(g.Add(sum4, sum5), _outputBias));
//write operation on cells
Matrix sum6 = g.Mul(_wcx, input);
Matrix sum7 = g.Mul(_wch, _hiddenContext);
Matrix cellInput = g.Nonlin(_cellInputActivation, g.Add(g.Add(sum6, sum7), _cellWriteBias));
//compute new cell activation
Matrix retainCell = g.Elmul(forgetGate, _cellContext);
Matrix writeCell = g.Elmul(inputGate, cellInput);
Matrix cellAct = g.Add(retainCell, writeCell);
//compute hidden state as gated, saturated cell activations
Matrix output = g.Elmul(outputGate, g.Nonlin(_cellOutputActivation, cellAct));
//rollover activations for next iteration
_hiddenContext = output;
_cellContext = cellAct;
return output;
}
public Matrix Activate(Matrix input, Graph g)
{
Matrix sum0 = g.Mul(_hmix, input);
Matrix sum1 = g.Mul(_hHmix, _context);
Matrix actMix = g.Nonlin(_fMix, g.Add(g.Add(sum0, sum1), _bmix));
Matrix sum2 = g.Mul(_hreset, input);
Matrix sum3 = g.Mul(_hHreset, _context);
Matrix actReset = g.Nonlin(_fReset, g.Add(g.Add(sum2, sum3), _breset));
Matrix sum4 = g.Mul(_hnew, input);
Matrix gatedContext = g.Elmul(actReset, _context);
Matrix sum5 = g.Mul(_hHnew, gatedContext);
Matrix actNewPlusGatedContext = g.Nonlin(_fNew, g.Add(g.Add(sum4, sum5), _bnew));
Matrix memvals = g.Elmul(actMix, _context);
Matrix newvals = g.Elmul(g.OneMinus(actMix), actNewPlusGatedContext);
Matrix output = g.Add(memvals, newvals);
//rollover activations for next iteration
_context = output;
return output;
}