/// <summary>Defaults.</summary>
/// <param name="d">The Descriptor to process.</param>
/// <param name="x">The Vector to process.</param>
/// <param name="y">The Vector to process.</param>
/// <param name="activation">The activation.</param>
/// <returns>A Network.</returns>
public static Network Default(Descriptor d, Matrix x, Vector y, IFunction activation)
{
var nn = new Network();
// set output to number of choices of available
// 1 if only two choices
var distinct = y.Distinct().Count();
var output = distinct > 2 ? distinct : 1;
// identity funciton for bias nodes
IFunction ident = new Ident();
// set number of hidden units to (Input + Hidden) * 2/3 as basic best guess.
var hidden = (int)Math.Ceiling((decimal)(x.Cols + output) * 2m / 3m);
// creating input nodes
nn.In = new Node[x.Cols + 1];
nn.In[0] = new Node { Label = "B0", Activation = ident };
for (var i = 1; i < x.Cols + 1; i++)
{
nn.In[i] = new Node { Label = d.ColumnAt(i - 1), Activation = ident };
}
// creating hidden nodes
var h = new Node[hidden + 1];
h[0] = new Node { Label = "B1", Activation = ident };
for (var i = 1; i < hidden + 1; i++)
{
h[i] = new Node { Label = string.Format("H{0}", i), Activation = activation };
}
// creating output nodes
nn.Out = new Node[output];
for (var i = 0; i < output; i++)
{
nn.Out[i] = new Node { Label = GetLabel(i, d), Activation = activation };
}
// link input to hidden. Note: there are
// no inputs to the hidden bias node
for (var i = 1; i < h.Length; i++)
{
for (var j = 0; j < nn.In.Length; j++)
{
Edge.Create(nn.In[j], h[i]);
}
}
// link from hidden to output (full)
for (var i = 0; i < nn.Out.Length; i++)
{
for (var j = 0; j < h.Length; j++)
{
Edge.Create(h[j], nn.Out[i]);
}
}
return nn;
}
/// <summary>Defaults.</summary>
/// <param name="d">The Descriptor to process.</param>
/// <param name="x">The Vector to process.</param>
/// <param name="y">The Vector to process.</param>
/// <param name="activationFunction">The activation.</param>
/// <param name="outputFunction">The ouput function for hidden nodes (Optional).</param>
/// <param name="epsilon">epsilon</param>
/// <returns>A Network.</returns>
public static Network Create(this Network network, Descriptor d, Matrix x, Vector y, IFunction activationFunction, IFunction outputFunction = null, double epsilon = double.NaN)
{
// set output to number of choices of available
// 1 if only two choices
int distinct = y.Distinct().Count();
int output = distinct > 2 ? distinct : 1;
// identity funciton for bias nodes
IFunction ident = new Ident();
// set number of hidden units to (Input + Hidden) * 2/3 as basic best guess.
int hidden = (int)System.Math.Ceiling((double)(x.Cols + output) * 2.0 / 3.0);
return network.Create(x.Cols, output, activationFunction, outputFunction,
fnNodeInitializer: new Func<int, int, Neuron>((l, i) =>
{
if (l == 0) return new Neuron(false) { Label = d.ColumnAt(i - 1), ActivationFunction = activationFunction, NodeId = i, LayerId = l };
else if (l == 2) return new Neuron(false) { Label = Network.GetLabel(i, d), ActivationFunction = activationFunction, NodeId = i, LayerId = l };
else return new Neuron(false) { ActivationFunction = activationFunction, NodeId = i, LayerId = l };
}), hiddenLayers: hidden);
}
/// <summary>
/// Stacks the given networks in order, on top of the current network, to create a fully connected deep neural network.
/// <para>This is useful in building pretrained multi-layered neural networks, where each layer is partially trained prior to stacking.</para>
/// </summary>
/// <param name="network">Current network.</param>
/// <param name="removeInputs">If true, the input nodes in additional layers are removed prior to stacking.
/// <para>This will link the previous network's output layer with the hidden units of the next layer.</para>
/// </param>
/// <param name="removeOutputs">If true, output nodes in the input and middle layers are removed prior to stacking.
/// <para>This will link the previous network's hidden or output layer with the input or hidden units (when <paramref name="removeInputs"/> is true) in the next layer.</para>
/// </param>
/// <param name="addBiases">If true, missing bias nodes are automatically added within new hidden layers.</param>
/// <param name="constrain">If true, the weights within each network are constrained leaving the new interconnecting network weights for training.</param>
/// <param name="networks">Network objects to stack on top of the current network. Each network is added downstream from the input nodes.</param>
public static Network Stack(this Network network, bool removeInputs = false, bool removeOutputs = false, bool addBiases = true,
bool constrain = true, params Network[] networks)
{
IFunction ident = new Ident();
// prune output layer on first (if pruning)
Network deep = (removeOutputs ? network.Prune(true, 1) : network);
if (constrain) deep.Constrain();
// get the current network's output layer
List<Neuron> prevOutput = deep.Out.ToList();
for (int x = 0; x < networks.Length; x++)
{
Network net = networks[x];
if (constrain) net.Constrain();
// remove input layer on next network (if pruning)
if (removeInputs) net = net.Prune(false, 1);
// remove output layer on next (middle) network (if pruning)
if (removeOutputs && x < networks.Length - 1) net = net.Prune(true, 1);
// add biases (for hidden network layers)
if (addBiases)
{
if (!prevOutput.Any(a => a.IsBias == true))
{
int layerId = prevOutput.First().LayerId;
var bias = new Neuron(true)
{
Label = $"B{layerId}",
ActivationFunction = ident,
NodeId = 0,
LayerId = layerId
};
// add to graph
deep.AddNode(bias);
// copy to previous network's output layer (for reference)
prevOutput.Insert(0, bias);
}
}
int deepLayers = deep.Layers;
Neuron[] prevLayer = null;
var layers = net.GetVertices().OfType<Neuron>()
.GroupBy(g => g.LayerId)
.ToArray();
for (int layer = 0; layer < layers.Count(); layer++)
{
// get nodes in current layer of current network
var nodes = layers.ElementAt(layer).ToArray();
// set new layer ID (relative to pos. in resulting graph)
int layerId = layer + deepLayers;
foreach (var node in nodes)
{
// set the new layer ID
node.LayerId = layerId;
// add to graph
deep.AddNode(node);
if (!node.IsBias)
{
// if not input layer in current network
if (layer > 0)
{
// add afferent edges to graph for current node
deep.AddEdges(net.GetInEdges(node).ToArray());
}
else
{
// add connections from previous network output layer to next input layer
foreach (var onode in prevOutput)
deep.AddEdge(Edge.Create(onode, node));
}
}
}
// nodes in last layer
if (prevLayer != null)
{
// add outgoing connections for each node in previous layer
foreach (var inode in prevLayer)
{
deep.AddEdges(net.GetOutEdges(inode).ToArray());
}
}
// remember last layer
prevLayer = nodes.ToArray();
}
// remember last network output nodes
prevOutput = deep.GetNodes(deep.Layers - 1).ToList();
}
deep.Reindex();
deep.In = deep.GetNodes(0).ToArray();
deep.Out = deep.GetNodes(deep.Layers - 1).ToArray();
return deep;
}
/// <summary>
/// Creates a new deep neural network based on the supplied inputs and layers.
/// </summary>
/// <param name="d">Descriptor object.</param>
/// <param name="X">Training examples</param>
/// <param name="y">Training labels</param>
/// <param name="activationFunction">Activation Function for each output layer.</param>
/// <param name="outputFunction">Ouput Function for each output layer.</param>
/// <param name="hiddenLayers">The intermediary (hidden) layers / ensembles in the network.</param>
/// <returns>A Deep Neural Network</returns>
public static Network Create(this Network network, Descriptor d, Matrix X, Vector y, IFunction activationFunction, IFunction outputFunction = null, params NetworkLayer[] hiddenLayers)
{
// set output to number of choices of available
// 1 if only two choices
int distinct = y.Distinct().Count();
int output = distinct > 2 ? distinct : 1;
// identity function for bias nodes
IFunction ident = new Ident();
// creating input nodes
network.In = new Neuron[X.Cols + 1];
network.In[0] = new Neuron { Label = "B0", ActivationFunction = ident };
for (int i = 1; i < X.Cols + 1; i++)
network.In[i] = new Neuron { Label = d.ColumnAt(i - 1), ActivationFunction = ident };
// creating output nodes
network.Out = new Neuron[output];
for (int i = 0; i < output; i++)
network.Out[i] = new Neuron { Label = Network.GetLabel(i, d), ActivationFunction = activationFunction, OutputFunction = outputFunction };
for (int layer = 0; layer < hiddenLayers.Count(); layer++)
{
if (layer == 0 && hiddenLayers[layer].IsAutoencoder)
{
// init and train it.
}
// connect input with previous layer or input layer
// connect last layer with output layer
}
// link input to hidden. Note: there are
// no inputs to the hidden bias node
//for (int i = 1; i < h.Length; i++)
// for (int j = 0; j < nn.In.Length; j++)
// Edge.Create(nn.In[j], h[i]);
//// link from hidden to output (full)
//for (int i = 0; i < nn.Out.Length; i++)
// for (int j = 0; j < h.Length; j++)
// Edge.Create(h[j], nn.Out[i]);
return network;
}
/// <summary>
/// Creates a new fully connected deep neural network based on the supplied size and depth parameters.
/// </summary>
/// <param name="inputLayer">Neurons in the input layer.</param>
/// <param name="outputLayer">Neurons in the output layer.</param>
/// <param name="activationFunction">Activation function for the hidden and output layers.</param>
/// <param name="outputFunction">(Optional) Output function of the the Nodes in the output layer (overrides the Activation function).</param>
/// <param name="fnNodeInitializer">(Optional) Function to call for initializing new Nodes - supplying parameters for the layer and node index.</param>
/// <param name="fnWeightInitializer">(Optional) Function to call for initializing the weights of each connection (including bias nodes).
/// <para>Where int1 = Source layer (0 is input layer), int2 = Source Node, int3 = Target node in the next layer.</para></param>
/// <param name="epsilon">Weight initialization parameter for random weight selection. Weight will be in the range of: -epsilon to +epsilon.</param>
/// <param name="hiddenLayers">An array of hidden neuron dimensions, where each element is the size of each layer (excluding bias nodes).</param>
/// <returns>Returns an untrained neural network model.</returns>
public static Network Create(this Network network, int inputLayer, int outputLayer, IFunction activationFunction, IFunction outputFunction = null, Func<int, int, Neuron> fnNodeInitializer = null,
Func<int, int, int, double> fnWeightInitializer = null, double epsilon = double.NaN, params int[] hiddenLayers)
{
IFunction ident = new Ident();
if (hiddenLayers == null || hiddenLayers.Length == 0)
hiddenLayers = new int[] { (int) System.Math.Ceiling((inputLayer + outputLayer + 1) * (2.0 / 3.0)) };
List<double> layers = new List<double>();
layers.Add(inputLayer);
foreach (int l in hiddenLayers)
layers.Add(l + 1);
layers.Add(outputLayer);
if (fnNodeInitializer == null)
fnNodeInitializer = new Func<int, int, Neuron>((i, j) => new Neuron());
if (fnWeightInitializer == null)
fnWeightInitializer = new Func<int, int, int, double>((l, i, j) => {
double inputs = (l > 0 ? layers[l - 1] : 0);
double outputs = (l < layers.Count - 1 ? layers[l + 1] : 0);
double eps = (double.IsNaN(epsilon) ? Edge.GetEpsilon(activationFunction.Minimum, activationFunction.Maximum, inputs, outputs) : epsilon);
return Edge.GetWeight(eps);
});
// creating input nodes
network.In = new Neuron[inputLayer + 1];
network.In[0] = network.AddNode(new Neuron(true) { Label = "B0", ActivationFunction = ident, NodeId = 0, LayerId = 0 });
for (int i = 1; i < inputLayer + 1; i++)
{
network.In[i] = fnNodeInitializer(0, i);
network.In[i].Label = (network.In[i].Label ?? string.Format("I{0}", i));
network.In[i].ActivationFunction = (network.In[i].ActivationFunction ?? ident);
network.In[i].LayerId = 0;
network.In[i].NodeId = i;
network.AddNode(network.In[i]);
}
Neuron[] last = null;
for (int layerIdx = 0; layerIdx < hiddenLayers.Length; layerIdx++)
{
// creating hidden nodes
Neuron[] layer = new Neuron[hiddenLayers[layerIdx] + 1];
layer[0] = network.AddNode(new Neuron(true) { Label = $"B{layerIdx + 1}", ActivationFunction = ident, LayerId = layerIdx + 1, NodeId = 0 });
for (int i = 1; i < layer.Length; i++)
{
layer[i] = fnNodeInitializer(layerIdx + 1, i);
layer[i].Label = (layer[i].Label ?? String.Format("H{0}.{1}", layerIdx + 1, i));
layer[i].ActivationFunction = (layer[i].ActivationFunction ?? activationFunction);
layer[i].OutputFunction = layer[i].OutputFunction;
layer[i].LayerId = layerIdx + 1;
layer[i].NodeId = i;
network.AddNode(layer[i]);
}
if (layerIdx > 0 && layerIdx < hiddenLayers.Length)
{
// create hidden to hidden (full)
for (int i = 0; i < last.Length; i++)
for (int x = 1; x < layer.Length; x++)
network.AddEdge(Edge.Create(last[i], layer[x], weight: fnWeightInitializer(layerIdx, i, x), epsilon: epsilon));
}
else if (layerIdx == 0)
{
// create input to hidden (full)
for (int i = 0; i < network.In.Length; i++)
for (int j = 1; j < layer.Length; j++)
network.AddEdge(Edge.Create(network.In[i], layer[j], weight: fnWeightInitializer(layerIdx, i, j), epsilon: epsilon));
}
last = layer;
}
// creating output nodes
network.Out = new Neuron[outputLayer];
for (int i = 0; i < outputLayer; i++)
{
network.Out[i] = fnNodeInitializer(hiddenLayers.Length + 1, i);
network.Out[i].Label = (network.Out[i].Label ?? String.Format("O{0}", i));
network.Out[i].ActivationFunction = (network.Out[i].ActivationFunction ?? activationFunction);
network.Out[i].OutputFunction = (network.Out[i].OutputFunction ?? outputFunction);
network.Out[i].LayerId = hiddenLayers.Length + 1;
network.Out[i].NodeId = i;
network.AddNode(network.Out[i]);
}
// link from (last) hidden to output (full)
for (int i = 0; i < network.Out.Length; i++)
for (int j = 0; j < last.Length; j++)
network.AddEdge(Edge.Create(last[j], network.Out[i], weight: fnWeightInitializer(hiddenLayers.Length, j, i), epsilon: epsilon));
return network;
}
/// <summary>
/// Initializes a new Network Layer. A network layer is sub neural network made up of inputs, hidden layer(s) and an output layer.
/// </summary>
/// <param name="inputNodes">Number of Nodes at the input layer (includes bias Node).</param>
/// <param name="outputNodes">Number of Nodes at the output layer.</param>
/// <param name="isFullyConnected">Indicates whether the output layer of this network is fully connected with the next Network Layer / Network.</param>
/// <param name="hiddenLayers">Number of Nodes in each layer and the number of layers, where the array length is the number of
/// layers and each value is the number of Nodes in that layer.</param>
/// <param name="fnNodeInitializer">Function for creating a new Node at each layer (zero-based) and Node index (zero-based). The 0 index layer corresponds to the input layer.</param>
/// <param name="isAutoencoder">Determines whether this Network layer is an auto-encoding layer.</param>
/// <param name="layerConnections">(Optional) Connection properties for this Network where the first dimension is the layer, second the Node index in that layer,
/// and third the Node indexes in the next layer to pair with.<para>For example: 1 => 2 => [2, 3] will link Node 2 in layer 1 to Nodes 2 + 3 in layer 2.</para></param>
/// <param name="createBiasNodes">(Optional) Indicates whether bias nodes are automatically created (thus bypassing the <paramref name="fnNodeInitializer"/> function).</param>
/// <param name="linkBiasNodes">(Optional) Indicates whether bias nodes in hidden layers are automatically linked to their respective hidden nodes.
/// <para>If this is set to True, it will override any bias node connections specified in parameter <paramref name="layerConnections"/></para></param>
public NetworkLayer(int inputNodes, int outputNodes, bool isFullyConnected, int[] hiddenLayers, Func<int, int, Neuron> fnNodeInitializer,
bool isAutoencoder = false, int[][][] layerConnections = null, bool createBiasNodes = true, bool linkBiasNodes = true)
: base()
{
this.IsFullyConnected = isFullyConnected;
this.IsAutoencoder = isAutoencoder;
if (!this.IsFullyConnected && (layerConnections == null || layerConnections.Length != (2 + hiddenLayers.Length)))
throw new ArgumentException("Connections must be supplied when the output layer is not fully connected with the next Network Layer.", nameof(layerConnections));
IFunction ident = new Ident();
this.In = new Neuron[inputNodes];
this.In[0] = new Neuron { Label = "B0", ActivationFunction = ident };
// create input nodes
for (int i = 1; i < this.In.Length; i++)
this.In[i] = fnNodeInitializer(0, i);
// create hidden layers
Neuron[][] hiddenNodes = new Neuron[hiddenLayers.Length][];
for (int hiddenLayer = 0; hiddenLayer < hiddenLayers.Count(); hiddenLayer++)
{
hiddenNodes[hiddenLayer] = new Neuron[hiddenLayers[hiddenLayer]];
for (int h = 0; h < hiddenLayers[hiddenLayer]; h++)
{
if (h == 0)
{
// create the bias node in this layer
if (createBiasNodes)
hiddenNodes[hiddenLayer][0] = new Neuron { Label = $"B{hiddenLayer + 1}", ActivationFunction = ident };
else
hiddenNodes[hiddenLayer][0] = fnNodeInitializer(hiddenLayer + 1, h);
}
else
{
// create the hidden node in this layer
hiddenNodes[hiddenLayer][h] = fnNodeInitializer(hiddenLayer + 1, h);
}
}
// do hidden to hidden node connections
if (hiddenLayer > 0 && (hiddenLayer != hiddenLayers.Length - 1))
{
bool useConnection = (layerConnections != null && layerConnections[hiddenLayer + 1] != null);
if (linkBiasNodes)
{
for (int node = 1; node < hiddenNodes[hiddenLayer].Length; node++)
Edge.Create(hiddenNodes[hiddenLayer - 1][0], hiddenNodes[hiddenLayer][node]);
}
// connect nodes in previous layer with this layer
for (int h = 0; h < hiddenLayers[hiddenLayer - 1]; h++)
{
if (linkBiasNodes && h == 0) continue;
// check for connection properties
if (useConnection && layerConnections[hiddenLayer + 1].Length > h && layerConnections[hiddenLayer + 1][h] != null)
{
// use connection properties
foreach (int connection in layerConnections[hiddenLayer + 1][h])
{
Edge.Create(hiddenNodes[hiddenLayer - 1][h], hiddenNodes[hiddenLayer][connection]);
}
}
}
}
}
// creating output nodes
this.Out = new Neuron[outputNodes];
for (int i = 0; i < outputNodes; i++)
this.Out[i] = fnNodeInitializer(hiddenLayers.Length + 1, i);
// link last hidden layer with output nodes
for (int i = 0; i < this.Out.Length; i++)
for (int j = 0; j < hiddenNodes.Last().Length; j++)
Edge.Create(hiddenNodes.Last().ElementAt(j), this.Out[i]);
}
