public void HideNode(TreeNode node)
{
if (hiddenNodes.ContainsKey(node.Tag))
{
//the node is already hidden
return;
}
HiddenNode hn = new HiddenNode(node);
hiddenNodes.Add(node.Tag, hn);
TreeNodeCollection parentNodesCollection = getParentNodesCollection(node);
if (InvokeRequired)
{
Invoke(new HideNodeDelegate(parentNodesCollection.Remove), new object[] {node});
}
else
{
if (SelectedNode == node)
{
SelectedNode = null;
}
parentNodesCollection.Remove(node);
}
}
private void CreateSeparationPlaneNodesAndLinks(int distance, Sample key, OutputNode outputNode)
{
var threshold = distance + 0.5 - key.InputVector.Count(x => x == 1);
var hiddenNode = new HiddenNode {
Threshold = -threshold
};
model.HiddenNodes.Add(hiddenNode);
for (int i = 0; i < key.InputVector.Count; i++)
{
model.InputToHiddenSynapticLinks.Add(new InputToHiddenSynapticLink
{
InputNode = model.InputNodes[i],
HiddenNode = hiddenNode,
Weight = key.InputVector[i] == 1 ? 1 : -1
});
}
model.HiddenToOutputSynapticLinks.Add(new HiddenToOutputSynapticLink
{
HiddenNode = hiddenNode,
OutputNode = outputNode
});
}
public static IList <T> CreateGenericNode <T>(int numberOfNodes)
{
var t = typeof(T);
int index = _registeredTypes.IndexOf(t);
var typeToCreate = _registeredTypes[index];
IList <T> list = new List <T>();
if (typeToCreate == typeof(InputNode))
{
for (int i = 0; i < numberOfNodes; i++)
{
InputNode node = new InputNode();
list.Add((T)(object)node);
}
}
else if (typeToCreate == typeof(HiddenNode))
{
for (int i = 0; i < numberOfNodes; i++)
{
HiddenNode node = new HiddenNode();
list.Add((T)(object)node);
}
}
else if (typeToCreate == typeof(OutputNode))
{
for (int i = 0; i < numberOfNodes; i++)
{
OutputNode node = new OutputNode();
list.Add((T)(object)node);
}
}
else if (typeToCreate == typeof(BiasNode))
{
for (int i = 0; i < numberOfNodes; i++)
{
BiasNode node = new BiasNode();
list.Add((T)(object)node);
}
}
return(list);
}
public override void VTrain(VMatrix features, VMatrix labels)
{
if (m_hidden.Length < 1)
{
m_hidden = new int[1] {
features.Cols() * 2
};
}
// add the input nodes
List <Node> iNodes = new List <Node>();
for (var i = 0; i < features.Cols(); i++)
{
iNodes.Add(new InputNode(i, i, m_rand));
}
m_layers.Add(iNodes);
int prevNodes = iNodes.Count + 1;
// add the hidden nodes
for (var layer = 0; layer < m_hidden.Length; layer++)
{
List <Node> hNodes = new List <Node>();
for (var n = 0; n < m_hidden[layer]; n++)
{
var node = new HiddenNode(n, prevNodes, m_rand);
if (m_activation == "relu")
{
if (m_actRandom)
{
node.alpha = m_actAlpha * m_rand.NextDouble();
node.threshold = m_actThreshold * m_rand.NextDouble();
node.beta = ((m_actBeta - 1.0) * m_rand.NextDouble()) + 1.0;
}
else
{
node.alpha = m_actAlpha;
node.threshold = m_actThreshold;
node.beta = m_actBeta;
}
}
hNodes.Add(node);
}
m_layers.Add(hNodes);
prevNodes = hNodes.Count + 1;
}
// add the output nodes
List <Node> oNodes = new List <Node>();
for (var col = 0; col < labels.Cols(); col++)
{
var labelValueCount = labels.ValueCount(col);
if (labelValueCount < 2)
{
// continuous
var node = new OutputNode(oNodes.Count, prevNodes, true, col, -1, m_rand);
if (m_activation == "relu")
{
if (m_actRandom)
{
node.alpha = m_actAlpha * m_rand.NextDouble();
node.threshold = m_actThreshold * m_rand.NextDouble();
node.beta = ((m_actBeta - 1.0) * m_rand.NextDouble()) + 1.0;
}
else
{
node.alpha = m_actAlpha;
node.threshold = m_actThreshold;
node.beta = m_actBeta;
}
}
oNodes.Add(node);
}
else
{
for (var n = 0; n < labelValueCount; n++)
{
var node = new OutputNode(oNodes.Count, prevNodes, false, col, n, m_rand);
if (m_activation == "relu")
{
if (m_actRandom)
{
node.alpha = m_actAlpha * m_rand.NextDouble();
node.threshold = m_actThreshold * m_rand.NextDouble();
node.beta = ((m_actBeta - 1.0) * m_rand.NextDouble()) + 1.0;
}
else
{
node.alpha = m_actAlpha;
node.threshold = m_actThreshold;
node.beta = m_actBeta;
}
}
oNodes.Add(node);
}
}
}
m_layers.Add(oNodes);
int trainSize = (int)(0.75 * features.Rows());
VMatrix trainFeatures = new VMatrix(features, 0, 0, trainSize, features.Cols());
VMatrix trainLabels = new VMatrix(labels, 0, 0, trainSize, labels.Cols());
VMatrix validationFeatures = new VMatrix(features, trainSize, 0, features.Rows() - trainSize, features.Cols());
VMatrix validationLabels = new VMatrix(labels, trainSize, 0, labels.Rows() - trainSize, labels.Cols());
Console.Write("Layers: ");
Console.Write(iNodes.Count);
Console.Write('x');
for (var l = 0; l < m_hidden.Length; l++)
{
Console.Write(m_hidden[l]);
Console.Write('x');
}
Console.WriteLine(oNodes.Count);
Console.WriteLine("AF: " + m_activation);
Console.WriteLine(string.Format("AParam: {0},{1},{2},{3}", m_actAlpha, m_actThreshold, m_actBeta, m_actRandom));
Console.WriteLine("Boost: " + m_boost);
Console.WriteLine("Epoch\tMSE (validation)");
if (m_outputFile != null)
{
m_outputFile.Write("Layers: ");
m_outputFile.Write(iNodes.Count);
m_outputFile.Write('x');
for (var l = 0; l < m_hidden.Length; l++)
{
m_outputFile.Write(m_hidden[l]);
m_outputFile.Write('x');
}
m_outputFile.WriteLine(oNodes.Count);
m_outputFile.WriteLine("Momentum: " + m_momentum);
m_outputFile.WriteLine("AF: " + m_activation);
m_outputFile.WriteLine(string.Format("AParam: {0},{1},{2},{3}", m_actAlpha, m_actThreshold, m_actBeta, m_actRandom));
m_outputFile.WriteLine("Boost: " + m_boost);
m_outputFile.WriteLine();
m_outputFile.WriteLine("Weights");
PrintWeights();
m_outputFile.WriteLine("Epoch\tMSE (validation)");
}
for (int round = 1; round < m_layers.Count; round++)
{
int epoch = 0; // current epoch number
int bestEpoch = 0; // epoch number of best MSE
int eCount = 0; // number of epochs since the best MSE
bool checkDone = false; // if true, check to see if we're done
double initialMSE = double.MaxValue; // MSE for first epoch
double bestMSE = double.MaxValue; // best validation MSE so far
for (; ;)
{
// shuffle the training set
trainFeatures.Shuffle(m_rand, trainLabels);
TrainEpoch(++epoch, trainFeatures, trainLabels, round);
// check the MSE after this epoch
double mse = VGetMSE(validationFeatures, validationLabels);
Console.WriteLine(string.Format("{0}:{1}-{2}\t{3}", round, epoch, eCount, mse));
if (m_outputFile != null)
{
m_outputFile.WriteLine(string.Format("{0}:{1}-{2}\t{3}", round, epoch, eCount, mse));
m_outputFile.Flush();
}
if ((mse == 0.0) || (epoch > 5000))
{
break;
}
else if ((epoch == 1) || (mse < bestMSE))
{
if (epoch == 1)
{
// save the initial MSE
initialMSE = mse;
}
else if (!checkDone && (mse < initialMSE * 0.9))
{
checkDone = true;
}
eCount = 0;
// save the best for later
bestMSE = mse;
bestEpoch = epoch;
for (var layer = 1; layer < m_layers.Count; layer++)
{
foreach (var node in m_layers[layer])
{
node.SaveBestWeights();
}
}
}
else if (checkDone)
{
// check to see if we're done
eCount++;
if (eCount >= 20)
{
break;
}
}
if ((bestEpoch > 0) && (bestEpoch != epoch))
{
for (var layer = round; layer < m_layers.Count; layer++)
{
foreach (var node in m_layers[layer])
{
node.RestoreBestWeights();
node.InitDeltas();
}
}
if (m_outputFile != null)
{
m_outputFile.WriteLine();
m_outputFile.WriteLine(string.Format("Best Weights (from Epoch {0}, valMSE={1})", bestEpoch, bestMSE));
PrintWeights();
}
}
}
}
if (m_outputFile != null)
{
m_outputFile.WriteLine();
m_outputFile.WriteLine("Weights");
PrintWeights();
m_outputFile.Close();
}
}
public override void VTrain(VMatrix features, VMatrix labels)
{
if (Layers.Count < 1)
{
// create the layers
if (Parameters.Hidden.Length < 1)
{
Parameters.Hidden = new[] { features.Cols() * 2 };
}
// add the input nodes
var iNodes = new List <Node>();
for (var i = 0; i < features.Cols(); i++)
{
iNodes.Add(new InputNode(i, i, _rand));
}
var iLayer = new Layer()
{
Type = LayerType.Input,
Nodes = iNodes,
Previous = null,
Next = null
};
Layers.Add(iLayer);
var prevLayer = iLayer;
// add the hidden nodes
foreach (var t in Parameters.Hidden)
{
var hNodes = new List <Node>();
for (var n = 0; n < t; n++)
{
var node = new HiddenNode(n, prevLayer.Nodes.Count + 1, t, _rand);
hNodes.Add(node);
}
var hLayer = new Layer()
{
Type = LayerType.Hidden,
Nodes = hNodes,
Previous = prevLayer,
Next = null
};
Layers.Add(hLayer);
prevLayer.Next = hLayer;
prevLayer = hLayer;
}
// add the output nodes
var oNodes = new List <Node>();
var oCount = 0;
for (var col = 0; col < labels.Cols(); col++)
{
var labelValueCount = labels.ValueCount(col);
if (labelValueCount < 2)
{
// continuous
oCount++;
}
else
{
oCount += labelValueCount;
}
}
for (var col = 0; col < labels.Cols(); col++)
{
var labelValueCount = labels.ValueCount(col);
if (labelValueCount < 2)
{
// continuous
var node = new OutputNode(oNodes.Count, true, col, -1, prevLayer.Nodes.Count + 1, oCount, _rand);
oNodes.Add(node);
}
else
{
for (var n = 0; n < labelValueCount; n++)
{
var node = new OutputNode(oNodes.Count, false, col, n, prevLayer.Nodes.Count + 1, oCount, _rand);
oNodes.Add(node);
}
}
}
var oLayer = new Layer()
{
Type = LayerType.Output,
Nodes = oNodes,
Previous = prevLayer
};
Layers.Add(oLayer);
prevLayer.Next = oLayer;
}
var trainSize = (int)(0.75 * features.Rows());
var trainFeatures = new VMatrix(features, 0, 0, trainSize, features.Cols());
var trainLabels = new VMatrix(labels, 0, 0, trainSize, labels.Cols());
var validationFeatures = new VMatrix(features, trainSize, 0, features.Rows() - trainSize, features.Cols());
var validationLabels = new VMatrix(labels, trainSize, 0, labels.Rows() - trainSize, labels.Cols());
Console.Write("Layers: ");
foreach (var layer in Layers)
{
Console.Write(layer.Nodes.Count);
if (layer.Type == LayerType.Output)
{
Console.WriteLine();
}
else
{
Console.Write('x');
}
}
Console.WriteLine("AF: " + Parameters.Activation);
Console.WriteLine("Epoch\tMSE (validation)");
int epoch; // current epoch number
var bestEpoch = 0; // epoch number of best MSE
var eCount = 0; // number of epochs since the best MSE
var checkDone = false; // if true, check to see if we're done
var initialMse = Parameters.InitialMse; // MSE for first epoch
var bestMse = Parameters.StartMse; // best validation MSE so far
double bestAccuracy = 0;
var batchCount = (trainFeatures.Rows() + Parameters.BatchSize - 1) / Parameters.BatchSize;
int countInterval = batchCount / 10;
if (countInterval < 1)
{
countInterval = 1;
}
var startEpoch = Parameters.StartEpoch + 1;
for (epoch = startEpoch;; epoch++)
{
// shuffle the training set
trainFeatures.Shuffle(_rand, trainLabels);
var cl = Console.CursorLeft;
for (var batch = 0; batch < batchCount; batch++)
{
var startIdx = batch * Parameters.BatchSize;
var count = Parameters.BatchSize;
if ((startIdx + count) > trainFeatures.Rows())
{
count = trainFeatures.Rows() - startIdx;
}
TrainBatch(trainFeatures, trainLabels, startIdx, count);
if ((((batch + 1) % countInterval) == 0) || (batch == (batchCount - 1)))
{
Console.SetCursorPosition(cl, Console.CursorTop);
Console.Write(batch + 1);
}
}
Console.WriteLine();
// check the MSE
var mse = VGetMSE(validationFeatures, validationLabels);
if ((epoch == startEpoch) && (initialMse == 0))
{
// save the initial MSE
initialMse = mse;
}
var accuracy = VMeasureAccuracy(validationFeatures, validationLabels, null);
if ((epoch % Parameters.SnapshotInterval) == 0)
{
SaveSnapshot(epoch, mse, initialMse, accuracy);
}
Console.WriteLine($"{epoch}-{eCount}\t{mse}");
if ((mse == 0) || (epoch > 5000))
{
break;
}
if ((epoch == startEpoch) || (mse < bestMse))
{
if ((epoch != startEpoch) && !checkDone && (mse < initialMse * 0.9))
{
checkDone = true;
}
eCount = 0;
// save the best for later
bestMse = mse;
bestEpoch = epoch;
bestAccuracy = accuracy;
foreach (var layer in Layers)
{
foreach (var node in layer.Nodes)
{
node.SaveBestWeights();
}
}
}
else if (checkDone)
{
// check to see if we're done
eCount++;
if (eCount >= 20)
{
break;
}
}
}
if ((bestEpoch > 0) && (bestEpoch != epoch))
{
foreach (var layer in Layers)
{
foreach (var node in layer.Nodes)
{
node.RestoreBestWeights();
}
}
}
SaveSnapshot(bestEpoch, bestMse, initialMse, bestAccuracy, true);
}
/// <summary>
/// Rebuild structure so we get layer information to make sure there are not circular dependencies.
/// </summary>
private void RebuildStructure()
{
// Clear the temporary node storage.
_tempNodes.Clear();
// Load all nodes for each connection gene.
ConnectionGenes.ForEach(LoadNodes);
// Find all the node in the connection genes.
_tempNodes.AddRange(ConnectionGenes.SelectMany(p => new[] { p.InNode, p.OutNode }).Distinct());
// Add all input nodes
_tempNodes.AddRange(Inputs.Select(p => p.InputNode));
// Add all output nodes.
_tempNodes.AddRange(Outputs.Select(p => p.OutputNode));
// For each node of type OutputNode set the layer to 0, otherwise set the layer to the minimum value and force to true.
foreach (Node node in _tempNodes)
{
if (node is OutputNode)
{
SetLayer(node, 0);
}
else
{
SetLayer(node, uint.MinValue, true);
}
}
// For each node of type InputNode set the layer to minimum value and force to true.
foreach (InputNode node in _tempNodes.OfType <InputNode>())
{
SetLayer(node, uint.MaxValue, true);
}
void SetLayer(Node node, uint layer, bool force = false)
{
// Set the layer depending on force and if the current layer is higher than the layer given.
node.Layer = force ? layer : (layer > node.Layer ? layer : node.Layer);
// If force is true, return early.
if (force)
{
return;
}
// For each connection gene where the out node identifier is the same as the given node identifier
// set the layer to the node's layer plus one.
foreach (ConnectionGene connectionGene in ConnectionGenes)
{
if (connectionGene.OutNodeIdentifier.Equals(node.NodeIdentifier))
{
SetLayer(connectionGene.InNode, node.Layer + 1);
}
}
}
void LoadNodes(ConnectionGene connectionGene)
{
// Get or create the In and Out nodes based on the given node identifiers.
connectionGene.InNode = GetOrCreateNodeForNodeId(connectionGene.InNodeIdentifier);
connectionGene.OutNode = GetOrCreateNodeForNodeId(connectionGene.OutNodeIdentifier);
}
Node GetOrCreateNodeForNodeId(uint nodeIdentifier)
{
// Tries to get a node for the given id, if not found returns a default.
Node node = GetNodeForId(nodeIdentifier);
// If the node is not equal to default return the given node.
if (node != default)
{
return(node);
}
// Tries to find the node in the hidden nodes list, if not found returns a default.
node = _hiddenNodes.FirstOrDefault(n => n.NodeIdentifier == nodeIdentifier);
// If the node is not equal to default return the given node.
if (node != default)
{
return(node);
}
// If no node was found then create a hidden node.
node = new HiddenNode(nodeIdentifier);
// Add it to the hidden nodes list.
_hiddenNodes.Add((HiddenNode)node);
// Return the hidden node.
return(node);
}
Node GetNodeForId(uint nodeIdentifier)
{
// Tries to find the first relation where the input node has the given identifier or return default.
OrganismInputNode organismInputNode = Inputs.FirstOrDefault(n => n.InputNode.NodeIdentifier == nodeIdentifier);
// if the organism input node is not default return the input node.
if (organismInputNode != default)
{
return(organismInputNode.InputNode);
}
// Tries to find the first relation where the output node has the given identifier, else return default.
return(Outputs.FirstOrDefault(n => n.OutputNode.NodeIdentifier == nodeIdentifier)?.OutputNode);
}
}
private void InitializeHiddenNodes(int hiddenNodeCount, double minInitialWeight, double maxInitialWeight)
{
_hiddenNodes = new List<HiddenNode>(hiddenNodeCount);
// add the threshold hidden node as 'input' for the output layer
_hiddenNodes.Add(new HiddenNode(0, -1.0) { IsThresholdUnit = true });
// add the true hidden nodes
for (int i = 1; i <= hiddenNodeCount; i++)
{
HiddenNode hiddenNode = new HiddenNode(i);
_hiddenNodes.Add(hiddenNode);
// initialize inbound weights
foreach (InputNode inputNode in _inputNodes)
{
double initialValue = Weight.GetInitialWeight(minInitialWeight, maxInitialWeight);
_weights.Add(new Weight(inputNode, hiddenNode, initialValue));
}
}
}
