/// <summary>
/// Gets a bias input vector for the specified layer. Each item is the bias weight on the connecting node in the next layer.
/// </summary>
/// <param name="network">Current network.</param>
/// <param name="layer">Forward layer of biases and their weights. The layer should be between 0 (first hidden layer) and the last hidden layer.</param>
/// <returns>Vector.</returns>
public static Vector GetBiases(this Network network, int layer = 0)
{
if (layer > network.Layers - 1)
{
throw new ArgumentOutOfRangeException(nameof(layer), $"There are no bias nodes from the output layer [{layer}].");
}
var nodes = network.GetNodes(layer + 1).ToArray();
Vector biases = Vector.Zeros(nodes.Where(w => !w.IsBias).Count());
for (int i = 0; i < nodes.Length; i++)
{
if (!nodes[i].IsBias)
{
var bias = nodes[i].In.FirstOrDefault(f => f.Source.IsBias);
biases[i] = bias.Weight;
}
}
return(biases);
}
/// <summary>
/// Stacks the given networks in order, on top of the current network, to create a fully connected deep neural network.
/// <para>This is useful in building pretrained multi-layered neural networks, where each layer is partially trained prior to stacking.</para>
/// </summary>
/// <param name="network">Current network.</param>
/// <param name="removeInputs">If true, the input nodes in additional layers are removed prior to stacking.
/// <para>This will link the previous network's output layer with the hidden units of the next layer.</para>
/// </param>
/// <param name="removeOutputs">If true, output nodes in the input and middle layers are removed prior to stacking.
/// <para>This will link the previous network's hidden or output layer with the input or hidden units (when <paramref name="removeInputs"/> is true) in the next layer.</para>
/// </param>
/// <param name="addBiases">If true, missing bias nodes are automatically added within new hidden layers.</param>
/// <param name="constrain">If true, the weights within each network are constrained leaving the new interconnecting network weights for training.</param>
/// <param name="networks">Network objects to stack on top of the current network. Each network is added downstream from the input nodes.</param>
public static Network Stack(this Network network, bool removeInputs = false, bool removeOutputs = false, bool addBiases = true,
bool constrain = true, params Network[] networks)
{
IFunction ident = new Ident();
// prune output layer on first (if pruning)
Network deep = (removeOutputs ? network.Prune(true, 1) : network);
if (constrain)
{
deep.Constrain();
}
// get the current network's output layer
List <Neuron> prevOutput = deep.Out.ToList();
for (int x = 0; x < networks.Length; x++)
{
Network net = networks[x];
if (constrain)
{
net.Constrain();
}
// remove input layer on next network (if pruning)
if (removeInputs)
{
net = net.Prune(false, 1);
}
// remove output layer on next (middle) network (if pruning)
if (removeOutputs && x < networks.Length - 1)
{
net = net.Prune(true, 1);
}
// add biases (for hidden network layers)
if (addBiases)
{
if (!prevOutput.Any(a => a.IsBias == true))
{
int layerId = prevOutput.First().LayerId;
var bias = new Neuron(true)
{
Label = $"B{layerId}",
ActivationFunction = ident,
NodeId = 0,
LayerId = layerId
};
// add to graph
deep.AddNode(bias);
// copy to previous network's output layer (for reference)
prevOutput.Insert(0, bias);
}
}
int deepLayers = deep.Layers;
Neuron[] prevLayer = null;
var layers = net.GetVertices().OfType <Neuron>()
.GroupBy(g => g.LayerId)
.ToArray();
for (int layer = 0; layer < layers.Count(); layer++)
{
// get nodes in current layer of current network
var nodes = layers.ElementAt(layer).ToArray();
// set new layer ID (relative to pos. in resulting graph)
int layerId = layer + deepLayers;
foreach (var node in nodes)
{
// set the new layer ID
node.LayerId = layerId;
// add to graph
deep.AddNode(node);
if (!node.IsBias)
{
// if not input layer in current network
if (layer > 0)
{
// add afferent edges to graph for current node
deep.AddEdges(net.GetInEdges(node).ToArray());
}
else
{
// add connections from previous network output layer to next input layer
foreach (var onode in prevOutput)
{
deep.AddEdge(Edge.Create(onode, node));
}
}
}
}
// nodes in last layer
if (prevLayer != null)
{
// add outgoing connections for each node in previous layer
foreach (var inode in prevLayer)
{
deep.AddEdges(net.GetOutEdges(inode).ToArray());
}
}
// remember last layer
prevLayer = nodes.ToArray();
}
// remember last network output nodes
prevOutput = deep.GetNodes(deep.Layers - 1).ToList();
}
deep.Reindex();
deep.In = deep.GetNodes(0).ToArray();
deep.Out = deep.GetNodes(deep.Layers - 1).ToArray();
return(deep);
}
private static int VerifyNetwork(Network network)
{
int errors = 0;
var edges = network.GetEdges();
for (int layer = 0; layer < network.Layers; layer++)
{
var nodes = network.GetNodes(layer);
foreach (var node in nodes)
{
foreach (var iedge in node.In)
{
if (!edges.Any(a => a.ParentId == iedge.ParentId && a.ChildId == iedge.ChildId))
errors++;
}
foreach (var oedge in node.Out)
{
if (!edges.Any(a => a.ParentId == oedge.ParentId && a.ChildId == oedge.ChildId))
errors++;
}
if (layer == 0 && node.In.Count > 0) errors++;
if (layer == network.Layers - 1 && node.Out.Count > 0) errors++;
}
}
network.Forward(Vector.Ones(network.In.Length - 1));
if (network.Out.Any(a => double.IsNaN(a.Output) || double.IsInfinity(a.Output))) errors++;
return errors;
}