public override Vol forward(Vol V, bool is_training)
{
this.in_act = V;
var V2 = V.cloneAndZero();
var N = V.w.Length;
for (var i = 0; i < N; i++)
{
V2.w[i] = Math.Tanh(V.w[i]);
}
this.out_act = V2;
return(this.out_act);
}
public override Vol forward(Vol V, bool is_training)
{
this.in_act = V;
var V2 = V.cloneAndZero();
var N = V.w.Length;
var V2w = V2.w;
var Vw = V.w;
for (var i = 0; i < N; i++)
{
V2w[i] = 1.0 / (1.0 + Math.Exp(-Vw[i]));
}
this.out_act = V2;
return(this.out_act);
}
public override Vol forward(Vol V, bool is_training)
{
this.in_act = V;
var A = V.cloneAndZero();
this.S_cache_ = V.cloneAndZero();
var n2 = Math.Floor((double)this.n / 2);
for (var x = 0; x < V.sx; x++)
{
for (var y = 0; y < V.sy; y++)
{
for (var i = 0; i < V.depth; i++)
{
var ai = V.get(x, y, i);
// normalize in a window of size n
var den = 0.0;
for (var j = Math.Max(0, i - n2); j <= Math.Min(i + n2, V.depth - 1); j++)
{
var aa = V.get(x, y, (int)j);
den += aa * aa;
}
den *= this.alpha / this.n;
den += this.k;
this.S_cache_.set(x, y, i, den); // will be useful for backprop
den = Math.Pow(den, this.beta);
A.set(x, y, i, ai / den);
}
}
}
this.out_act = A;
return(this.out_act); // dummy identity function for now
}