private static void InitializeReLUs(NetworkGraph graph)
{
// relus like a bit of positive bias to get gradients early
// otherwise it's technically possible that a relu unit will never turn on (by chance)
// and will never get any gradient and never contribute any computation. Dead relu.
foreach (Layer layer in graph.Vertices.Where(x => x is ReLULayer).ToList())
{
Layer target = layer;
if (graph.InDegree(target) == 1)
{
Edge <Layer> edge = graph.InEdges(target)[0];
if (edge.Target is MaxPoolingLayer)
{
target = edge.Source;
}
}
if (graph.InDegree(target) == 1)
{
Edge <Layer> edge = graph.InEdges(target)[0];
if (edge.Source is StochasticLayer stochasticLayer)
{
stochasticLayer.B.Set(0.1f);
}
}
}
}
