/// <summary>
/// Stacks the given networks in order, on top of the current network, to create a fully connected deep neural network.
/// <para>This is useful in building pretrained multi-layered neural networks, where each layer is partially trained prior to stacking.</para>
/// </summary>
/// <param name="network">Current network.</param>
/// <param name="removeInputs">If true, the input nodes in additional layers are removed prior to stacking.
/// <para>This will link the previous network's output layer with the hidden units of the next layer.</para>
/// </param>
/// <param name="removeOutputs">If true, output nodes in the input and middle layers are removed prior to stacking.
/// <para>This will link the previous network's hidden or output layer with the input or hidden units (when <paramref name="removeInputs"/> is true) in the next layer.</para>
/// </param>
/// <param name="addBiases">If true, missing bias nodes are automatically added within new hidden layers.</param>
/// <param name="constrain">If true, the weights within each network are constrained leaving the new interconnecting network weights for training.</param>
/// <param name="networks">Network objects to stack on top of the current network. Each network is added downstream from the input nodes.</param>
public static Network Stack(this Network network, bool removeInputs = false, bool removeOutputs = false, bool addBiases = true,
bool constrain = true, params Network[] networks)
{
IFunction ident = new Ident();
// prune output layer on first (if pruning)
Network deep = (removeOutputs ? network.Prune(true, 1) : network);
if (constrain)
{
deep.Constrain();
}
// get the current network's output layer
List <Neuron> prevOutput = deep.Out.ToList();
for (int x = 0; x < networks.Length; x++)
{
Network net = networks[x];
if (constrain)
{
net.Constrain();
}
// remove input layer on next network (if pruning)
if (removeInputs)
{
net = net.Prune(false, 1);
}
// remove output layer on next (middle) network (if pruning)
if (removeOutputs && x < networks.Length - 1)
{
net = net.Prune(true, 1);
}
// add biases (for hidden network layers)
if (addBiases)
{
if (!prevOutput.Any(a => a.IsBias == true))
{
int layerId = prevOutput.First().LayerId;
var bias = new Neuron(true)
{
Label = $"B{layerId}",
ActivationFunction = ident,
NodeId = 0,
LayerId = layerId
};
// add to graph
deep.AddNode(bias);
// copy to previous network's output layer (for reference)
prevOutput.Insert(0, bias);
}
}
int deepLayers = deep.Layers;
Neuron[] prevLayer = null;
var layers = net.GetVertices().OfType <Neuron>()
.GroupBy(g => g.LayerId)
.ToArray();
for (int layer = 0; layer < layers.Count(); layer++)
{
// get nodes in current layer of current network
var nodes = layers.ElementAt(layer).ToArray();
// set new layer ID (relative to pos. in resulting graph)
int layerId = layer + deepLayers;
foreach (var node in nodes)
{
// set the new layer ID
node.LayerId = layerId;
// add to graph
deep.AddNode(node);
if (!node.IsBias)
{
// if not input layer in current network
if (layer > 0)
{
// add afferent edges to graph for current node
deep.AddEdges(net.GetInEdges(node).ToArray());
}
else
{
// add connections from previous network output layer to next input layer
foreach (var onode in prevOutput)
{
deep.AddEdge(Edge.Create(onode, node));
}
}
}
}
// nodes in last layer
if (prevLayer != null)
{
// add outgoing connections for each node in previous layer
foreach (var inode in prevLayer)
{
deep.AddEdges(net.GetOutEdges(inode).ToArray());
}
}
// remember last layer
prevLayer = nodes.ToArray();
}
// remember last network output nodes
prevOutput = deep.GetNodes(deep.Layers - 1).ToList();
}
deep.Reindex();
deep.In = deep.GetNodes(0).ToArray();
deep.Out = deep.GetNodes(deep.Layers - 1).ToArray();
return(deep);
}