/// <summary>
/// Execute Backpropagation through time
/// </summary>
/// <returns></returns>
public (float loss, Array <FloatArray2D> dwy, Array <FloatArray> dby, Array <Array <FloatArray2D> > dwh, Array <FloatArray> dbh, Array <FloatArray> hs) BPTT(int[] inputs, int[] targets, Array <FloatArray> hprev)
{
float loss = 0f;
//Feedforward
var xs = new Array <FloatArray>(inputs.Length); // Inputs
var hs = new Array <Array <FloatArray> >(inputs.Length); // Hidden Result
var ps = new Array <Array <FloatArray> >(inputs.Length); // Softmax probabilit
var tg = new Array <FloatArray>(inputs.Length); // Targets
hs[-1] = hprev;
//Backward
var dwy = new Array <FloatArray2D>(1);
var dby = new Array <FloatArray>(1);
var dwh = new Array <Array <FloatArray2D> >(recurrentUnits);
var dbh = new Array <FloatArray>(recurrentUnits);
var dhnext = new Array <FloatArray>(recurrentUnits);
for (int t = 0; t < inputs.Length; t++)
{
xs[t] = new FloatArray(input_size);
xs[t][inputs[t]] = 1;
tg[t] = new FloatArray(output_size);
tg[t][targets[t]] = 1;
(hs[t], ps[t]) = FeedForward(xs[t], hs[t - 1]);
}
for (int t = inputs.Length - 1; t >= 0; t--)
{
// Sequencial
(var l, var dy) = decoder.ComputeErrorNBackward(tg[t], ps[t]);
(dhnext, _) = encoder.Backward(dy, dhnext, hs[t]);
// Parallel
(var wy, var by) = decoder.ComputeGradient(hs[t][-1]);
(var wh, var bh) = encoder.ComputeGradient(xs[t], hs[t - 1]);
// Parallel
dwy = dwy.Sum(wy);
dby = dby.Sum(by);
dwh = dwh.Sum(wh);
dbh = dbh.Sum(bh);
loss += l;
}
// Parallel
dwy = NumMath.Normalize(-5, 5, dwy);
dby = NumMath.Normalize(-5, 5, dby);
dwh = NumMath.Normalize(-5, 5, dwh);
dbh = NumMath.Normalize(-5, 5, dbh);
return(loss, dwy, dby, dwh, dbh, hs[-1]);
