public override void VTrain(VMatrix features, VMatrix labels)
{
if (m_hidden.Length < 1)
{
m_hidden = new int[1] {
features.Cols() * 2
};
}
// add the input nodes
List <Node> iNodes = new List <Node>();
for (var i = 0; i < features.Cols(); i++)
{
iNodes.Add(new InputNode(i, i, m_rand));
}
m_layers.Add(iNodes);
int prevNodes = iNodes.Count + 1;
// add the hidden nodes
for (var layer = 0; layer < m_hidden.Length; layer++)
{
List <Node> hNodes = new List <Node>();
for (var n = 0; n < m_hidden[layer]; n++)
{
var node = new HiddenNode(n, prevNodes, m_rand);
if (m_activation == "relu")
{
if (m_actRandom)
{
node.alpha = m_actAlpha * m_rand.NextDouble();
node.threshold = m_actThreshold * m_rand.NextDouble();
node.beta = ((m_actBeta - 1.0) * m_rand.NextDouble()) + 1.0;
}
else
{
node.alpha = m_actAlpha;
node.threshold = m_actThreshold;
node.beta = m_actBeta;
}
}
hNodes.Add(node);
}
m_layers.Add(hNodes);
prevNodes = hNodes.Count + 1;
}
// add the output nodes
List <Node> oNodes = new List <Node>();
for (var col = 0; col < labels.Cols(); col++)
{
var labelValueCount = labels.ValueCount(col);
if (labelValueCount < 2)
{
// continuous
var node = new OutputNode(oNodes.Count, prevNodes, true, col, -1, m_rand);
if (m_activation == "relu")
{
if (m_actRandom)
{
node.alpha = m_actAlpha * m_rand.NextDouble();
node.threshold = m_actThreshold * m_rand.NextDouble();
node.beta = ((m_actBeta - 1.0) * m_rand.NextDouble()) + 1.0;
}
else
{
node.alpha = m_actAlpha;
node.threshold = m_actThreshold;
node.beta = m_actBeta;
}
}
oNodes.Add(node);
}
else
{
for (var n = 0; n < labelValueCount; n++)
{
var node = new OutputNode(oNodes.Count, prevNodes, false, col, n, m_rand);
if (m_activation == "relu")
{
if (m_actRandom)
{
node.alpha = m_actAlpha * m_rand.NextDouble();
node.threshold = m_actThreshold * m_rand.NextDouble();
node.beta = ((m_actBeta - 1.0) * m_rand.NextDouble()) + 1.0;
}
else
{
node.alpha = m_actAlpha;
node.threshold = m_actThreshold;
node.beta = m_actBeta;
}
}
oNodes.Add(node);
}
}
}
m_layers.Add(oNodes);
int trainSize = (int)(0.75 * features.Rows());
VMatrix trainFeatures = new VMatrix(features, 0, 0, trainSize, features.Cols());
VMatrix trainLabels = new VMatrix(labels, 0, 0, trainSize, labels.Cols());
VMatrix validationFeatures = new VMatrix(features, trainSize, 0, features.Rows() - trainSize, features.Cols());
VMatrix validationLabels = new VMatrix(labels, trainSize, 0, labels.Rows() - trainSize, labels.Cols());
Console.Write("Layers: ");
Console.Write(iNodes.Count);
Console.Write('x');
for (var l = 0; l < m_hidden.Length; l++)
{
Console.Write(m_hidden[l]);
Console.Write('x');
}
Console.WriteLine(oNodes.Count);
Console.WriteLine("AF: " + m_activation);
Console.WriteLine(string.Format("AParam: {0},{1},{2},{3}", m_actAlpha, m_actThreshold, m_actBeta, m_actRandom));
Console.WriteLine("Boost: " + m_boost);
Console.WriteLine("Epoch\tMSE (validation)");
if (m_outputFile != null)
{
m_outputFile.Write("Layers: ");
m_outputFile.Write(iNodes.Count);
m_outputFile.Write('x');
for (var l = 0; l < m_hidden.Length; l++)
{
m_outputFile.Write(m_hidden[l]);
m_outputFile.Write('x');
}
m_outputFile.WriteLine(oNodes.Count);
m_outputFile.WriteLine("Momentum: " + m_momentum);
m_outputFile.WriteLine("AF: " + m_activation);
m_outputFile.WriteLine(string.Format("AParam: {0},{1},{2},{3}", m_actAlpha, m_actThreshold, m_actBeta, m_actRandom));
m_outputFile.WriteLine("Boost: " + m_boost);
m_outputFile.WriteLine();
m_outputFile.WriteLine("Weights");
PrintWeights();
m_outputFile.WriteLine("Epoch\tMSE (validation)");
}
for (int round = 1; round < m_layers.Count; round++)
{
int epoch = 0; // current epoch number
int bestEpoch = 0; // epoch number of best MSE
int eCount = 0; // number of epochs since the best MSE
bool checkDone = false; // if true, check to see if we're done
double initialMSE = double.MaxValue; // MSE for first epoch
double bestMSE = double.MaxValue; // best validation MSE so far
for (; ;)
{
// shuffle the training set
trainFeatures.Shuffle(m_rand, trainLabels);
TrainEpoch(++epoch, trainFeatures, trainLabels, round);
// check the MSE after this epoch
double mse = VGetMSE(validationFeatures, validationLabels);
Console.WriteLine(string.Format("{0}:{1}-{2}\t{3}", round, epoch, eCount, mse));
if (m_outputFile != null)
{
m_outputFile.WriteLine(string.Format("{0}:{1}-{2}\t{3}", round, epoch, eCount, mse));
m_outputFile.Flush();
}
if ((mse == 0.0) || (epoch > 5000))
{
break;
}
else if ((epoch == 1) || (mse < bestMSE))
{
if (epoch == 1)
{
// save the initial MSE
initialMSE = mse;
}
else if (!checkDone && (mse < initialMSE * 0.9))
{
checkDone = true;
}
eCount = 0;
// save the best for later
bestMSE = mse;
bestEpoch = epoch;
for (var layer = 1; layer < m_layers.Count; layer++)
{
foreach (var node in m_layers[layer])
{
node.SaveBestWeights();
}
}
}
else if (checkDone)
{
// check to see if we're done
eCount++;
if (eCount >= 20)
{
break;
}
}
if ((bestEpoch > 0) && (bestEpoch != epoch))
{
for (var layer = round; layer < m_layers.Count; layer++)
{
foreach (var node in m_layers[layer])
{
node.RestoreBestWeights();
node.InitDeltas();
}
}
if (m_outputFile != null)
{
m_outputFile.WriteLine();
m_outputFile.WriteLine(string.Format("Best Weights (from Epoch {0}, valMSE={1})", bestEpoch, bestMSE));
PrintWeights();
}
}
}
}
if (m_outputFile != null)
{
m_outputFile.WriteLine();
m_outputFile.WriteLine("Weights");
PrintWeights();
m_outputFile.Close();
}
}
public override void VTrain(VMatrix features, VMatrix labels)
{
if (m_hidden.Length < 1)
{
m_hidden = new int[1] {
features.Cols() * 2
};
}
int wIdx = 0; // index into the weights array
// add the input nodes
List <Node> iNodes = new List <Node>();
for (var i = 0; i < features.Cols(); i++)
{
iNodes.Add(new InputNode(i, 0, m_rand));
}
m_layers.Add(iNodes);
int trainSize = (int)(0.75 * features.Rows());
VMatrix trainFeatures = new VMatrix(features, 0, 0, trainSize, features.Cols());
VMatrix trainLabels = new VMatrix(labels, 0, 0, trainSize, labels.Cols());
VMatrix validationFeatures = new VMatrix(features, trainSize, 0, features.Rows() - trainSize, features.Cols());
VMatrix validationLabels = new VMatrix(labels, trainSize, 0, labels.Rows() - trainSize, labels.Cols());
Console.WriteLine("R-E-C\tMSE (training)\t\tMSE (validation)\taccuracy (validation)");
if (m_outputFile != null)
{
m_outputFile.WriteLine("Momentum: " + m_momentum);
m_outputFile.WriteLine();
m_outputFile.WriteLine("Weights");
PrintWeights();
m_outputFile.WriteLine("R-E-C\tMSE (training)\t\tMSE (validation)\taccuracy (validation)");
}
if (m_weights != null)
{
// not training
double trainMSE = VGetMSE(trainFeatures, trainLabels);
double mse = VGetMSE(validationFeatures, validationLabels);
double accuracy = VMeasureAccuracy(validationFeatures, validationLabels, null);
Console.WriteLine($"1\t{trainMSE}\t{mse}\t{accuracy}");
if (m_outputFile != null)
{
m_outputFile.WriteLine($"1\t{trainMSE}\t{mse}\t{accuracy}");
}
}
else
{
for (int round = 1; round <= m_hidden.Length + 1; round++)
{
if (round <= m_hidden.Length)
{
// add hidden nodes
int prevNodes = m_layers[m_layers.Count - 1].Count + 1;
List <Node> hNodes = new List <Node>();
for (var n = 0; n < m_hidden[m_layers.Count - 1]; n++)
{
if (m_weights != null)
{
hNodes.Add(new HiddenNode(prevNodes, m_rand, m_weights[wIdx++]));
}
else
{
hNodes.Add(new HiddenNode(prevNodes, m_rand, null));
}
}
m_layers.Add(hNodes);
prevNodes = hNodes.Count + 1;
// add output nodes
List <Node> oNodes = new List <Node>();
if (round < m_hidden.Length)
{
// dae layer - add output nodes to match inputs
for (var col = 0; col < m_layers[m_layers.Count - 2].Count; col++)
{
oNodes.Add(new OutputNode(prevNodes, true, col, -1, m_rand, null));
}
}
else
{
// final layer - figure out how many outputs we need
for (var col = 0; col < labels.Cols(); col++)
{
var labelValueCount = labels.ValueCount(col);
if (labelValueCount < 2)
{
// continuous
if (m_weights != null)
{
oNodes.Add(new OutputNode(prevNodes, true, col, -1, m_rand, m_weights[wIdx++]));
}
else
{
oNodes.Add(new OutputNode(prevNodes, true, col, -1, m_rand, null));
}
}
else
{
for (var n = 0; n < labelValueCount; n++)
{
if (m_weights != null)
{
oNodes.Add(new OutputNode(prevNodes, false, col, n, m_rand, m_weights[wIdx++]));
}
else
{
oNodes.Add(new OutputNode(prevNodes, false, col, n, m_rand, null));
}
}
}
}
}
m_layers.Add(oNodes);
InitNodes();
}
int epoch = 0; // current epoch number
int bestEpoch = 0; // epoch number of best MSE
int eCount = 0; // number of epochs since the best MSE
bool checkDone = false; // if true, check to see if we're done
double mse = 0; // validation MSE
double bestTrainMSE = double.MaxValue; // best training MSE so far
double bestMSE = double.MaxValue; // best validation MSE so far
double accuracy = 0; // validation accuracy
double bestAccuracy = double.MaxValue; // best validationa accuracy so far
for (; ;)
{
// shuffle the training set
trainFeatures.Shuffle(m_rand, trainLabels);
double trainMSE = TrainEpoch(++epoch, trainFeatures, trainLabels, round <m_hidden.Length, round> m_hidden.Length);
// check the MSE after this epoch
if (round < m_hidden.Length)
{
mse = IGetMSE(validationFeatures);
accuracy = 0;
}
else
{
mse = VGetMSE(validationFeatures, validationLabels);
// check the validation accuracy
accuracy = VMeasureAccuracy(validationFeatures, validationLabels, null);
}
Console.WriteLine($"{round}-{epoch}-{eCount}\t{trainMSE}\t{mse}\t{accuracy}");
if (m_outputFile != null)
{
m_outputFile.WriteLine($"{round}-{epoch}-{eCount}\t{trainMSE}\t{mse}\t{accuracy}");
m_outputFile.Flush();
}
if ((mse == 0.0) || (epoch > 10000))
{
break;
}
else if ((epoch == 1) || (mse < bestMSE))
{
if (epoch == 1)
{
// save the initial MSE
bestMSE = mse;
}
else if ((mse / bestMSE) > 0.99)
{
if (!checkDone)
{
checkDone = true;
eCount = 0;
}
}
else
{
checkDone = false;
eCount = 0;
}
// save the best for later
bestTrainMSE = trainMSE;
bestMSE = mse;
bestAccuracy = accuracy;
bestEpoch = epoch;
for (var layer = 0; layer < m_layers.Count - 1; layer++)
{
foreach (var node in m_layers[layer])
{
node.SaveBestWeights();
}
}
}
else if (!checkDone)
{
checkDone = true;
eCount = 0;
}
if (checkDone)
{
// check to see if we're done
eCount++;
if (eCount >= 20)
{
break;
}
}
}
if (round < m_hidden.Length)
{
// delete the output layer
m_layers.RemoveAt(m_layers.Count - 1);
}
if ((bestEpoch > 0) && (bestEpoch != epoch))
{
for (var layer = 0; layer < m_layers.Count - 1; layer++)
{
foreach (var node in m_layers[layer])
{
node.RestoreBestWeights();
}
}
Console.WriteLine($"Best Weights (from Epoch {bestEpoch}, trainMSE={bestTrainMSE}, valMSE={bestMSE})");
if (m_outputFile != null)
{
m_outputFile.WriteLine();
m_outputFile.WriteLine($"Best Weights (from Epoch {bestEpoch}, trainMSE={bestTrainMSE}, valMSE={bestMSE})");
m_outputFile.Flush();
}
}
}
}
if (m_outputFile != null)
{
m_outputFile.WriteLine();
m_outputFile.WriteLine("Weights");
PrintWeights();
}
if (m_outputFile != null)
{
m_outputFile.Close();
}
}
public override void VTrain(VMatrix features, VMatrix labels)
{
if (Layers.Count < 1)
{
// create the layers
if (Parameters.Hidden.Length < 1)
{
Parameters.Hidden = new[] { features.Cols() * 2 };
}
// add the input nodes
var iNodes = new List <Node>();
for (var i = 0; i < features.Cols(); i++)
{
iNodes.Add(new InputNode(i, i, _rand));
}
var iLayer = new Layer()
{
Type = LayerType.Input,
Nodes = iNodes,
Previous = null,
Next = null
};
Layers.Add(iLayer);
var prevLayer = iLayer;
// add the hidden nodes
foreach (var t in Parameters.Hidden)
{
var hNodes = new List <Node>();
for (var n = 0; n < t; n++)
{
var node = new HiddenNode(n, prevLayer.Nodes.Count + 1, t, _rand);
hNodes.Add(node);
}
var hLayer = new Layer()
{
Type = LayerType.Hidden,
Nodes = hNodes,
Previous = prevLayer,
Next = null
};
Layers.Add(hLayer);
prevLayer.Next = hLayer;
prevLayer = hLayer;
}
// add the output nodes
var oNodes = new List <Node>();
var oCount = 0;
for (var col = 0; col < labels.Cols(); col++)
{
var labelValueCount = labels.ValueCount(col);
if (labelValueCount < 2)
{
// continuous
oCount++;
}
else
{
oCount += labelValueCount;
}
}
for (var col = 0; col < labels.Cols(); col++)
{
var labelValueCount = labels.ValueCount(col);
if (labelValueCount < 2)
{
// continuous
var node = new OutputNode(oNodes.Count, true, col, -1, prevLayer.Nodes.Count + 1, oCount, _rand);
oNodes.Add(node);
}
else
{
for (var n = 0; n < labelValueCount; n++)
{
var node = new OutputNode(oNodes.Count, false, col, n, prevLayer.Nodes.Count + 1, oCount, _rand);
oNodes.Add(node);
}
}
}
var oLayer = new Layer()
{
Type = LayerType.Output,
Nodes = oNodes,
Previous = prevLayer
};
Layers.Add(oLayer);
prevLayer.Next = oLayer;
}
var trainSize = (int)(0.75 * features.Rows());
var trainFeatures = new VMatrix(features, 0, 0, trainSize, features.Cols());
var trainLabels = new VMatrix(labels, 0, 0, trainSize, labels.Cols());
var validationFeatures = new VMatrix(features, trainSize, 0, features.Rows() - trainSize, features.Cols());
var validationLabels = new VMatrix(labels, trainSize, 0, labels.Rows() - trainSize, labels.Cols());
Console.Write("Layers: ");
foreach (var layer in Layers)
{
Console.Write(layer.Nodes.Count);
if (layer.Type == LayerType.Output)
{
Console.WriteLine();
}
else
{
Console.Write('x');
}
}
Console.WriteLine("AF: " + Parameters.Activation);
Console.WriteLine("Epoch\tMSE (validation)");
int epoch; // current epoch number
var bestEpoch = 0; // epoch number of best MSE
var eCount = 0; // number of epochs since the best MSE
var checkDone = false; // if true, check to see if we're done
var initialMse = Parameters.InitialMse; // MSE for first epoch
var bestMse = Parameters.StartMse; // best validation MSE so far
double bestAccuracy = 0;
var batchCount = (trainFeatures.Rows() + Parameters.BatchSize - 1) / Parameters.BatchSize;
int countInterval = batchCount / 10;
if (countInterval < 1)
{
countInterval = 1;
}
var startEpoch = Parameters.StartEpoch + 1;
for (epoch = startEpoch;; epoch++)
{
// shuffle the training set
trainFeatures.Shuffle(_rand, trainLabels);
var cl = Console.CursorLeft;
for (var batch = 0; batch < batchCount; batch++)
{
var startIdx = batch * Parameters.BatchSize;
var count = Parameters.BatchSize;
if ((startIdx + count) > trainFeatures.Rows())
{
count = trainFeatures.Rows() - startIdx;
}
TrainBatch(trainFeatures, trainLabels, startIdx, count);
if ((((batch + 1) % countInterval) == 0) || (batch == (batchCount - 1)))
{
Console.SetCursorPosition(cl, Console.CursorTop);
Console.Write(batch + 1);
}
}
Console.WriteLine();
// check the MSE
var mse = VGetMSE(validationFeatures, validationLabels);
if ((epoch == startEpoch) && (initialMse == 0))
{
// save the initial MSE
initialMse = mse;
}
var accuracy = VMeasureAccuracy(validationFeatures, validationLabels, null);
if ((epoch % Parameters.SnapshotInterval) == 0)
{
SaveSnapshot(epoch, mse, initialMse, accuracy);
}
Console.WriteLine($"{epoch}-{eCount}\t{mse}");
if ((mse == 0) || (epoch > 5000))
{
break;
}
if ((epoch == startEpoch) || (mse < bestMse))
{
if ((epoch != startEpoch) && !checkDone && (mse < initialMse * 0.9))
{
checkDone = true;
}
eCount = 0;
// save the best for later
bestMse = mse;
bestEpoch = epoch;
bestAccuracy = accuracy;
foreach (var layer in Layers)
{
foreach (var node in layer.Nodes)
{
node.SaveBestWeights();
}
}
}
else if (checkDone)
{
// check to see if we're done
eCount++;
if (eCount >= 20)
{
break;
}
}
}
if ((bestEpoch > 0) && (bestEpoch != epoch))
{
foreach (var layer in Layers)
{
foreach (var node in layer.Nodes)
{
node.RestoreBestWeights();
}
}
}
SaveSnapshot(bestEpoch, bestMse, initialMse, bestAccuracy, true);
}
public override void VTrain(VMatrix features, VMatrix labels)
{
if (m_hidden.Length < 1)
{
m_hidden = new int[1] {
features.Cols() * 2
};
}
// create output nodes
List <Node> oNodes = new List <Node>();
// figure out how many outputs we need
for (var col = 0; col < labels.Cols(); col++)
{
var labelValueCount = labels.ValueCount(col);
if (labelValueCount < 2)
{
// continuous
oNodes.Add(new OutputNode(m_hidden[m_hidden.Length - 1] + 1, true, col, -1, m_rand));
}
else
{
for (var n = 0; n < labelValueCount; n++)
{
oNodes.Add(new OutputNode(m_hidden[m_hidden.Length - 1] + 1, false, col, n, m_rand));
}
}
}
int oCount = oNodes.Count;
for (var plane = 0; plane < oCount; plane++)
{
m_layers.Add(new List <List <Node> >());
// add the input nodes
List <Node> iNodes = new List <Node>();
for (var i = 0; i < features.Cols(); i++)
{
iNodes.Add(new InputNode(i, 0, m_rand));
}
m_layers[plane].Add(iNodes);
int prevNodes = iNodes.Count + 1;
for (var layer = 0; layer <= m_hidden.Length; layer++)
{
if (layer < m_hidden.Length)
{
// add hidden nodes
List <Node> hNodes = new List <Node>();
for (var n = 0; n < m_hidden[layer]; n++)
{
hNodes.Add(new HiddenNode(prevNodes, m_rand));
}
m_layers[plane].Add(hNodes);
prevNodes = hNodes.Count + 1;
}
else
{
// add output node
m_layers[plane].Add(new List <Node>()
{
oNodes[plane]
});
}
}
}
InitNodes();
int trainSize = (int)(0.75 * features.Rows());
VMatrix trainFeatures = new VMatrix(features, 0, 0, trainSize, features.Cols());
VMatrix trainLabels = new VMatrix(labels, 0, 0, trainSize, labels.Cols());
VMatrix validationFeatures = new VMatrix(features, trainSize, 0, features.Rows() - trainSize, features.Cols());
VMatrix validationLabels = new VMatrix(labels, trainSize, 0, labels.Rows() - trainSize, labels.Cols());
Console.WriteLine(string.Format("Planes: {0}", oCount));
Console.Write(string.Format("Layers: {0}x", features.Cols()));
for (var l = 0; l < m_hidden.Length; l++)
{
Console.Write(m_hidden[l]);
Console.Write('x');
}
Console.WriteLine("1");
Console.WriteLine("Momentum: " + m_momentum);
Console.WriteLine("AF: " + m_activation);
Console.WriteLine(string.Format("AParam: {0},{1},{2},{3}", m_actLeak, m_actThreshold, m_actSlope, m_actRandom));
Console.WriteLine("P-R-C\tMSE (validation)");
if (m_outputFile != null)
{
m_outputFile.WriteLine(string.Format("Planes: {0}", oCount));
m_outputFile.Write(string.Format("Layers: {0}x", features.Cols()));
for (var l = 0; l < m_hidden.Length; l++)
{
m_outputFile.Write(m_hidden[l]);
m_outputFile.Write('x');
}
m_outputFile.WriteLine("1");
m_outputFile.WriteLine("Momentum: " + m_momentum);
m_outputFile.WriteLine("AF: " + m_activation);
m_outputFile.WriteLine(string.Format("AParam: {0},{1},{2},{3}", m_actLeak, m_actThreshold, m_actSlope, m_actRandom));
m_outputFile.WriteLine();
m_outputFile.WriteLine("P-R-C\tMSE (validation)");
}
// train the net, one plane at a time
for (var plane = 0; plane < oCount; plane++)
{
int epoch = 0; // current epoch number
int bestEpoch = 0; // epoch number of best MSE
int eCount = 0; // number of epochs since the best MSE
bool checkDone = false; // if true, check to see if we're done
double initialMSE = double.MaxValue; // MSE for first epoch
double bestMSE = double.MaxValue; // best validation MSE so far
for (; ;)
{
// shuffle the training set
trainFeatures.Shuffle(m_rand, trainLabels);
TrainEpoch(plane, ++epoch, trainFeatures, trainLabels);
// check the MSE after this epoch
double mse = PGetMSE(plane, validationFeatures, validationLabels);
Console.WriteLine(string.Format("{0}-{1}-{2}\t{3}", plane, epoch, eCount, mse));
if (m_outputFile != null)
{
m_outputFile.WriteLine(string.Format("{0}-{1}-{3}\t{3}", plane, epoch, eCount, mse));
m_outputFile.Flush();
}
if ((mse == 0.0) || (epoch > 10000))
{
break;
}
else if ((epoch == 1) || (mse < bestMSE))
{
if (epoch == 1)
{
// save the initial MSE
initialMSE = mse;
}
else if (!checkDone && (mse < initialMSE * 0.9))
{
checkDone = true;
}
eCount = 0;
// save the best for later
bestMSE = mse;
bestEpoch = epoch;
for (var layer = 0; layer < m_layers[plane].Count - 1; layer++)
{
foreach (var node in m_layers[plane][layer])
{
node.SaveBestWeights();
}
}
}
else if (checkDone)
{
// check to see if we're done
eCount++;
if (eCount >= 20)
{
break;
}
}
else if ((epoch > 100) && /*(mse < initialMSE) &&*/ (mse > ((bestMSE + initialMSE) / 2)))
{
checkDone = true;
}
}
if ((bestEpoch > 0) && (bestEpoch != epoch))
{
for (var layer = 0; layer < m_layers[plane].Count - 1; layer++)
{
foreach (var node in m_layers[plane][layer])
{
node.RestoreBestWeights();
}
}
Console.WriteLine(string.Format("Best Weights (from Epoch {0}, valMSE={1})", bestEpoch, bestMSE));
if (m_outputFile != null)
{
m_outputFile.WriteLine();
m_outputFile.WriteLine(string.Format("Best Weights (from Epoch {0}, valMSE={1})", bestEpoch, bestMSE));
m_outputFile.Flush();
}
}
}
if (m_outputFile != null)
{
m_outputFile.WriteLine();
m_outputFile.WriteLine("Weights");
PrintWeights();
}
if (m_outputFile != null)
{
m_outputFile.Close();
}
}
public override void VTrain(VMatrix features, VMatrix labels)
{
if (m_hidden.Length < 1)
{
m_hidden = new int[1] {
features.Cols() * 2
};
}
int prevNodes = features.Cols() + 1;
int wIdx = 0; // index into the weights array
for (var layer = 0; layer <= m_hidden.Length; layer++)
{
// add the nodes for this layer
List <Node> nodes = new List <Node>();
if (layer < m_hidden.Length)
{
// hidden layer
for (var n = 0; n < m_hidden[layer]; n++)
{
if (m_weights != null)
{
nodes.Add(new Node(prevNodes, false, 0, 0, m_rand, m_weights[wIdx++]));
}
else
{
nodes.Add(new Node(prevNodes, false, 0, 0, m_rand, null));
}
}
}
else
{
// output layer - figure out how many outputs we need
for (var col = 0; col < labels.Cols(); col++)
{
var labelValueCount = labels.ValueCount(col);
if (labelValueCount < 2)
{
// continuous
if (m_weights != null)
{
nodes.Add(new Node(prevNodes, true, col, -1, m_rand, m_weights[wIdx++]));
}
else
{
nodes.Add(new Node(prevNodes, true, col, -1, m_rand, null));
}
}
else
{
for (var n = 0; n < labelValueCount; n++)
{
if (m_weights != null)
{
nodes.Add(new Node(prevNodes, false, col, n, m_rand, m_weights[wIdx++]));
}
else
{
nodes.Add(new Node(prevNodes, false, col, n, m_rand, null));
}
}
}
}
}
prevNodes = nodes.Count + 1;
m_layers.Add(nodes);
}
InitNodes();
int trainSize = (int)(0.75 * features.Rows());
VMatrix trainFeatures = new VMatrix(features, 0, 0, trainSize, features.Cols());
VMatrix trainLabels = new VMatrix(labels, 0, 0, trainSize, labels.Cols());
VMatrix validationFeatures = new VMatrix(features, trainSize, 0, features.Rows() - trainSize, features.Cols());
VMatrix validationLabels = new VMatrix(labels, trainSize, 0, labels.Rows() - trainSize, labels.Cols());
int epoch = 0; // current epoch number
double bestTrainMSE = double.MaxValue; // best training MSE so far
double bestMSE = double.MaxValue; // best validation MSE so far
double bestAccuracy = double.MaxValue; // best validationa accuracy so far
double initialMSE = double.MaxValue; // MSE for first epoch
int eCount = 0; // number of epochs since the best MSE
int bestEpoch = 0; // epoch number of best MSE
bool done = false;
bool checkDone = false; // if true, check to see if we're done
Console.WriteLine("Epoch\tMSE (training)\t\tMSE (validation)\taccuracy (validation)");
if (m_outputFile != null)
{
m_outputFile.WriteLine(string.Format("{0} layers, {1} output nodes", m_layers.Count, m_layers[m_layers.Count - 1].Count));
m_outputFile.WriteLine("Momentum: " + m_momentum);
m_outputFile.WriteLine();
m_outputFile.WriteLine("Weights");
PrintWeights();
m_outputFile.WriteLine("Epoch\tMSE (training)\t\tMSE (validation)\taccuracy (validation)");
}
do
{
// shuffle the training set
trainFeatures.Shuffle(m_rand, trainLabels);
double trainMSE;
if (m_weights != null)
{
// not training
trainMSE = VGetMSE(trainFeatures, trainLabels);
epoch++;
}
else
{
trainMSE = TrainEpoch(++epoch, trainFeatures, trainLabels);
}
// check the MSE after this epoch
double mse = VGetMSE(validationFeatures, validationLabels);
// check the validation accuracy after this epoch
double accuracy = VMeasureAccuracy(validationFeatures, validationLabels, null);
Console.WriteLine(string.Format("{0}\t{1}\t{2}\t{3}", epoch, trainMSE, mse, accuracy));
if (m_outputFile != null)
{
m_outputFile.WriteLine(string.Format("{0}\t{1}\t{2}\t{3}", epoch, trainMSE, mse, accuracy));
}
if (m_weights != null)
{
// not really training
done = true;
}
else if (mse == 0.0)
{
// can't get better than this
done = true;
}
else if ((epoch == 1) || (mse <= bestMSE))
{
if (epoch == 1)
{
// save the initial MSE
initialMSE = mse;
}
else if (!checkDone && (mse < initialMSE * 0.9))
{
checkDone = true;
}
// save the best for later
bestTrainMSE = trainMSE;
bestMSE = mse;
bestAccuracy = accuracy;
bestEpoch = epoch;
eCount = 0;
for (var layer = 0; layer < m_layers.Count - 1; layer++)
{
foreach (var node in m_layers[layer])
{
node.SaveBestWeights();
}
}
}
else if (checkDone)
{
// check to see if we're done
eCount++;
if (eCount >= 20)
{
done = true;
}
}
else if (mse > initialMSE * 1.1)
{
// are we getting really worse?
checkDone = true;
}
else if (epoch >= 10000)
{
// time to stop
done = true;
}
} while (!done);
if (m_outputFile != null)
{
m_outputFile.WriteLine();
m_outputFile.WriteLine("Weights");
PrintWeights();
}
if ((bestEpoch > 0) && (bestEpoch != epoch))
{
for (var layer = 0; layer < m_layers.Count - 1; layer++)
{
foreach (var node in m_layers[layer])
{
node.RestoreBestWeights();
}
}
if (m_outputFile != null)
{
m_outputFile.WriteLine();
m_outputFile.WriteLine(string.Format("Best Weights (from Epoch {0}, trainMSE={1}, valMSE={2}, valAcc={3})", bestEpoch, bestTrainMSE, bestMSE, bestAccuracy));
PrintWeights();
}
}
if (m_outputFile != null)
{
m_outputFile.Close();
}
}
public override void VTrain(VMatrix features, VMatrix labels)
{
if (m_hidden.Length < 1)
{
m_hidden = new int[1] {
features.Cols() * 2
};
}
// add the input nodes
var iNodes = new List <Node>();
for (var i = 0; i < features.Cols(); i++)
{
iNodes.Add(new InputNode(i, 0, m_rand));
}
m_layers.Add(iNodes);
// figure out how many outputs we need
var oCount = 0;
for (var col = 0; col < labels.Cols(); col++)
{
var labelValueCount = labels.ValueCount(col);
if (labelValueCount < 2)
{
// continuous
oCount++;
}
else
{
oCount += labelValueCount;
}
}
var trainSize = (int)(0.75 * features.Rows());
var trainFeatures = new VMatrix(features, 0, 0, trainSize, features.Cols());
var trainLabels = new VMatrix(labels, 0, 0, trainSize, labels.Cols());
var validationFeatures = new VMatrix(features, trainSize, 0, features.Rows() - trainSize, features.Cols());
var validationLabels = new VMatrix(labels, trainSize, 0, labels.Rows() - trainSize, labels.Cols());
Console.Write("Layers: ");
Console.Write(features.Cols());
Console.Write('x');
for (var l = 0; l < m_hidden.Length; l++)
{
Console.Write(m_hidden[l]);
Console.Write('x');
}
Console.WriteLine(oCount);
Console.WriteLine("Momentum: " + m_momentum);
Console.WriteLine("C: " + m_corruptLevel);
Console.WriteLine("AF: " + m_activation);
Console.WriteLine($"AParam: {m_actLeak},{m_actThreshold},{m_actSlope},{m_actRandom}");
Console.WriteLine("TrainAll: " + m_trainAll);
Console.WriteLine("R-E-C\tMSE (validation)");
if (m_outputFile != null)
{
m_outputFile.Write("Layers: ");
m_outputFile.Write(features.Cols());
m_outputFile.Write('x');
for (var l = 0; l < m_hidden.Length; l++)
{
m_outputFile.Write(m_hidden[l]);
m_outputFile.Write('x');
}
m_outputFile.WriteLine(oCount);
m_outputFile.WriteLine("Momentum: " + m_momentum);
m_outputFile.WriteLine("C: " + m_corruptLevel);
m_outputFile.WriteLine("AF: " + m_activation);
m_outputFile.WriteLine($"AParam: {m_actLeak},{m_actThreshold},{m_actSlope},{m_actRandom}");
m_outputFile.WriteLine("TrainAll: " + m_trainAll);
m_outputFile.WriteLine();
m_outputFile.WriteLine("R-E-C\tMSE (validation)");
}
var maxRounds = (m_trainAll ? m_hidden.Length : m_hidden.Length + 1);
for (var round = 1; round <= maxRounds; round++)
{
if (round <= m_hidden.Length)
{
// add hidden nodes
var prevNodes = m_layers[m_layers.Count - 1].Count + 1;
var hNodes = new List <Node>();
for (var n = 0; n < m_hidden[m_layers.Count - 1]; n++)
{
hNodes.Add(new HiddenNode(prevNodes, m_rand));
}
m_layers.Add(hNodes);
prevNodes = hNodes.Count + 1;
// add output nodes
var oNodes = new List <Node>();
// figure out how many outputs we need
for (var col = 0; col < labels.Cols(); col++)
{
var labelValueCount = labels.ValueCount(col);
if (labelValueCount < 2)
{
// continuous
oNodes.Add(new OutputNode(prevNodes, true, col, -1, m_rand));
}
else
{
for (var n = 0; n < labelValueCount; n++)
{
oNodes.Add(new OutputNode(prevNodes, false, col, n, m_rand));
}
}
}
m_layers.Add(oNodes);
InitNodes();
}
var epoch = 0; // current epoch number
var bestEpoch = 0; // epoch number of best MSE
var eCount = 0; // number of epochs since the best MSE
var checkDone = false; // if true, check to see if we're done
var initialMSE = double.MaxValue; // MSE for first epoch
var bestMSE = double.MaxValue; // best validation MSE so far
for (; ;)
{
// shuffle the training set
trainFeatures.Shuffle(m_rand, trainLabels);
TrainEpoch(++epoch, trainFeatures, trainLabels, round < m_hidden.Length, m_trainAll || (round > m_hidden.Length));
// check the MSE after this epoch
var mse = VGetMSE(validationFeatures, validationLabels);
Console.WriteLine($"{round}-{epoch}-{eCount}\t{mse}");
if (m_outputFile != null)
{
m_outputFile.WriteLine($"{round}-{epoch}-{eCount}\t{mse}");
m_outputFile.Flush();
}
if ((mse == 0.0) || (epoch > 10000))
{
break;
}
else if ((epoch == 1) || (mse < bestMSE))
{
if (epoch == 1)
{
// save the initial MSE
initialMSE = mse;
}
else if (!checkDone && (mse < initialMSE * 0.9))
{
checkDone = true;
}
eCount = 0;
// save the best for later
bestMSE = mse;
bestEpoch = epoch;
for (var layer = 0; layer < m_layers.Count - 1; layer++)
{
foreach (var node in m_layers[layer])
{
node.SaveBestWeights();
}
}
}
else if (checkDone)
{
// check to see if we're done
eCount++;
if (eCount >= 20)
{
break;
}
}
else if ((epoch > 100) && /*(mse < initialMSE) &&*/ (mse > ((bestMSE + initialMSE) / 2)))
{
checkDone = true;
}
}
if (round < m_hidden.Length)
{
// delete the output layer
m_layers.RemoveAt(m_layers.Count - 1);
}
if ((bestEpoch > 0) && (bestEpoch != epoch))
{
for (var layer = 0; layer < m_layers.Count - 1; layer++)
{
foreach (var node in m_layers[layer])
{
node.RestoreBestWeights();
}
}
Console.WriteLine($"Best Weights (from Epoch {bestEpoch}, valMSE={bestMSE})");
if (m_outputFile != null)
{
m_outputFile.WriteLine();
m_outputFile.WriteLine($"Best Weights (from Epoch {bestEpoch}, valMSE={bestMSE})");
m_outputFile.Flush();
}
}
}
if (m_outputFile != null)
{
m_outputFile.WriteLine();
m_outputFile.WriteLine("Weights");
PrintWeights();
}
if (m_outputFile != null)
{
m_outputFile.Close();
}
}
public override void VTrain(VMatrix features, VMatrix labels)
{
if (m_hidden.Length < 1)
{
m_hidden = new int[1] {
features.Cols() * 2
};
}
// add the input nodes
List <Node> iNodes = new List <Node>();
for (var i = 0; i < features.Cols(); i++)
{
iNodes.Add(new InputNode(i, 0, m_rand));
}
m_layers.Add(iNodes);
int prevNodes = iNodes.Count + 1;
int wIdx = 0; // index into the weights array
// add the hidden nodes
for (var layer = 0; layer < m_hidden.Length; layer++)
{
// add the nodes for this layer
List <Node> hNodes = new List <Node>();
// if not the last 2 hidden layers, add c bias weight
if (layer < m_hidden.Length - 2)
{
prevNodes++;
}
for (var n = 0; n < m_hidden[layer]; n++)
{
if (m_weights != null)
{
hNodes.Add(new HiddenNode(prevNodes, m_rand, m_weights[wIdx++]));
}
else
{
hNodes.Add(new HiddenNode(prevNodes, m_rand, null));
}
}
prevNodes = hNodes.Count + 1;
m_layers.Add(hNodes);
}
// add the output nodes - figure out how many outputs we need
List <Node> oNodes = new List <Node>();
for (var col = 0; col < labels.Cols(); col++)
{
var labelValueCount = labels.ValueCount(col);
if (labelValueCount < 2)
{
// continuous
if (m_weights != null)
{
oNodes.Add(new OutputNode(prevNodes, true, col, -1, m_rand, m_weights[wIdx++]));
}
else
{
oNodes.Add(new OutputNode(prevNodes, true, col, -1, m_rand, null));
}
}
else
{
for (var n = 0; n < labelValueCount; n++)
{
if (m_weights != null)
{
oNodes.Add(new OutputNode(prevNodes, false, col, n, m_rand, m_weights[wIdx++]));
}
else
{
oNodes.Add(new OutputNode(prevNodes, false, col, n, m_rand, null));
}
}
}
}
m_layers.Add(oNodes);
InitNodes();
int trainSize = (int)(0.75 * features.Rows());
VMatrix trainFeatures = new VMatrix(features, 0, 0, trainSize, features.Cols());
VMatrix trainLabels = new VMatrix(labels, 0, 0, trainSize, labels.Cols());
VMatrix validationFeatures = new VMatrix(features, trainSize, 0, features.Rows() - trainSize, features.Cols());
VMatrix validationLabels = new VMatrix(labels, trainSize, 0, labels.Rows() - trainSize, labels.Cols());
int epoch = 0; // current epoch number
int bestEpoch = 0; // epoch number of best MSE
int eCount; // number of epochs since the best MSE
bool checkDone; // if true, check to see if we're done
double bestTrainMSE = double.MaxValue; // best training MSE so far
double bestMSE = double.MaxValue; // best validation MSE so far
double bestAccuracy = double.MaxValue; // best validationa accuracy so far
Console.WriteLine("Epoch\tMSE (training)\t\tMSE (validation)\taccuracy (validation)");
if (m_outputFile != null)
{
m_outputFile.WriteLine(string.Format("{0} layers, {1} output nodes", m_layers.Count, m_layers[m_layers.Count - 1].Count));
m_outputFile.WriteLine("Momentum: " + m_momentum);
m_outputFile.WriteLine();
m_outputFile.WriteLine("Weights");
PrintWeights();
m_outputFile.WriteLine("Epoch\tMSE (training)\t\tMSE (validation)\taccuracy (validation)");
}
if (m_weights != null)
{
// not training
double trainMSE = VGetMSE(trainFeatures, trainLabels);
epoch = 1;
double mse = VGetMSE(validationFeatures, validationLabels);
double accuracy = VMeasureAccuracy(validationFeatures, validationLabels, null);
Console.WriteLine(string.Format("{0}\t{1}\t{2}\t{3}", epoch, trainMSE, mse, accuracy));
if (m_outputFile != null)
{
m_outputFile.WriteLine(string.Format("{0}\t{1}\t{2}\t{3}", epoch, trainMSE, mse, accuracy));
}
}
else
{
for (int hLayer = 1; hLayer <= m_hidden.Length; hLayer++)
{
if (hLayer < m_hidden.Length)
{
// dbn layer
epoch = 0;
eCount = 0;
checkDone = false;
double wDelta = 0;
double lastDelta = 0;
double bestDelta = double.MaxValue;
int maxEpochs = 500000 / trainFeatures.Rows();
if (maxEpochs < 10)
{
maxEpochs = 10;
}
for (; ;)
{
// shuffle the training set
trainFeatures.Shuffle(m_rand, trainLabels);
wDelta = TrainDBN(hLayer, ++epoch, trainFeatures, trainLabels);
Console.WriteLine(string.Format("{0}\t{1}", epoch, wDelta));
if (m_outputFile != null)
{
m_outputFile.WriteLine(string.Format("{0}\t{1}", epoch, wDelta));
}
if (epoch > maxEpochs)
{
break;
}
else if (epoch == 1)
{
bestDelta = wDelta;
}
else if ((wDelta / lastDelta) >= 0.99)
{
if (!checkDone)
{
checkDone = true;
eCount = 0;
}
}
else if (wDelta < bestDelta)
{
checkDone = false;
}
else if (!checkDone)
{
checkDone = true;
eCount = 0;
}
if (checkDone)
{
// check to see if we're done
eCount++;
if (eCount >= 5)
{
break;
}
}
if (wDelta < bestDelta)
{
bestDelta = wDelta;
}
lastDelta = wDelta;
}
}
else
{
// final hidden layer
epoch = 0;
eCount = 0;
checkDone = false;
double initialMSE = double.MaxValue; // MSE for first epoch
for (; ;)
{
// shuffle the training set
trainFeatures.Shuffle(m_rand, trainLabels);
double trainMSE = TrainEpoch(++epoch, trainFeatures, trainLabels);
// check the MSE after this epoch
double mse = VGetMSE(validationFeatures, validationLabels);
// check the validation accuracy after this epoch
double accuracy = VMeasureAccuracy(validationFeatures, validationLabels, null);
Console.WriteLine(string.Format("{0}\t{1}\t{2}\t{3}", epoch, trainMSE, mse, accuracy));
if (m_outputFile != null)
{
m_outputFile.WriteLine(string.Format("{0}\t{1}\t{2}\t{3}", epoch, trainMSE, mse, accuracy));
}
if (mse == 0.0)
{
// can't get better than this
break;
}
else if ((epoch == 1) || (mse <= bestMSE))
{
if (epoch == 1)
{
// save the initial MSE
initialMSE = mse;
}
else if (!checkDone && (mse < initialMSE * 0.9))
{
checkDone = true;
}
// save the best for later
bestTrainMSE = trainMSE;
bestMSE = mse;
bestAccuracy = accuracy;
bestEpoch = epoch;
eCount = 0;
for (var layer = 0; layer < m_layers.Count - 1; layer++)
{
foreach (var node in m_layers[layer])
{
node.SaveBestWeights();
}
}
}
else if (checkDone)
{
// check to see if we're done
eCount++;
if (eCount >= 20)
{
break;
}
}
else if (mse > initialMSE * 1.1)
{
// are we getting really worse?
checkDone = true;
}
else if (epoch >= 10000)
{
// time to stop
break;
}
}
}
}
}
if (m_outputFile != null)
{
m_outputFile.WriteLine();
m_outputFile.WriteLine("Weights");
PrintWeights();
}
if ((bestEpoch > 0) && (bestEpoch != epoch))
{
for (var layer = 0; layer < m_layers.Count - 1; layer++)
{
foreach (var node in m_layers[layer])
{
node.RestoreBestWeights();
}
}
if (m_outputFile != null)
{
m_outputFile.WriteLine();
m_outputFile.WriteLine(string.Format("Best Weights (from Epoch {0}, trainMSE={1}, valMSE={2}, valAcc={3})", bestEpoch, bestTrainMSE, bestMSE, bestAccuracy));
PrintWeights();
}
}
if (m_outputFile != null)
{
m_outputFile.Close();
}
}
public override void VTrain(VMatrix features, VMatrix labels)
{
if (labels.ValueCount(0) > 2)
{
m_count = labels.ValueCount(0);
}
// create one set of weights for each output
m_weights = new List <double[]>();
for (var p = 0; p < m_count; p++)
{
var weights = new double[features.Cols() + 1];
for (var i = 0; i < weights.Length; i++)
{
weights[i] = 1.0 - (m_rand.NextDouble() * 2.0);
}
m_weights.Add(weights);
}
// iterate through each of the instances
for (var instance = 0; instance < m_count; instance++)
{
double error; // error rate for the current epoch
var bestError = 1.0; // best (smallest) error rate so far
var eCount = 0; // number of epochs since the best error
var epoch = 0; // current epoch number
var done = false;
double bestAccuracy = 0; // best accuracy so far
var bestEpoch = 0; // epoch number of best accuracy
var bestWeights = new double[features.Cols() + 1]; // best weights
if (m_outputFile != null)
{
m_outputFile.WriteLine("Instance " + instance);
m_outputFile.WriteLine("Epoch\tError Rate");
}
do
{
// shuffle the training set
features.Shuffle(m_rand, labels);
error = TrainEpoch(instance, ++epoch, features, labels);
// check the accuracy after this epoch
var accuracy = GetAccuracy(instance, features, labels);
if (accuracy > bestAccuracy)
{
// save the best for later
bestAccuracy = accuracy;
bestEpoch = epoch;
for (var i = 0; i < bestWeights.Length; i++)
{
bestWeights[i] = m_weights[instance][i];
}
}
if (error == 0.0)
{
// can't get better than this
done = true;
}
else if ((epoch == 1) || (error <= bestError))
{
// save the best error so far
bestError = error;
eCount = 0;
}
else
{
// check to see if we're done
eCount++;
if (eCount >= 10)
{
done = true;
}
}
} while (!done);
if (m_outputFile != null)
{
m_outputFile.WriteLine();
m_outputFile.WriteLine("Weights");
for (var i = 0; i < m_weights[instance].Length - 1; i++)
{
m_outputFile.Write(string.Format("{0}\t", m_weights[instance][i]));
}
m_outputFile.WriteLine(string.Format("{0}\t", m_weights[instance][m_weights[instance].Length - 1]));
m_outputFile.WriteLine();
}
if (bestEpoch != epoch)
{
for (var i = 0; i < bestWeights.Length; i++)
{
m_weights[instance][i] = bestWeights[i];
}
if (m_outputFile != null)
{
m_outputFile.WriteLine();
m_outputFile.WriteLine(string.Format("Best Weights (from Epoch {0}, accuracy={1})", bestEpoch, bestAccuracy));
for (var i = 0; i < m_weights[instance].Length - 1; i++)
{
m_outputFile.Write(string.Format("{0}\t", m_weights[instance][i]));
}
m_outputFile.WriteLine(string.Format("{0}\t", m_weights[instance][m_weights[instance].Length - 1]));
m_outputFile.WriteLine();
}
}
}
if (m_outputFile != null)
{
m_outputFile.Close();
}
}