public override void CalculateResult()
{
Console.WriteLine(@"SVM Results:");
var testSet = new BasicNeuralDataSet(TestSet, IdealTestOutput);
var values = new List<double[]>();
var result = new List<double[]>();
foreach (var input in testSet)
{
var output = Svm.Compute(input.Input);
Console.WriteLine(input.Input[0] + @", actual=" + output[0] + @", ideal=" + input.Ideal[0]);
var tmp = new double[input.Input.Count];
input.Input.CopyTo(tmp,0,input.Input.Count);
values.Add(tmp);
var tmp2 = new double[output.Count];
output.CopyTo(tmp2, 0, output.Count);
result.Add(tmp2);
}
ResultTestSet = new BasicNeuralDataSet(values.ToArray(), result.ToArray());
Console.WriteLine("Done");
}
/// <summary>
/// Measure the performance of the network
/// </summary>
/// <param name = "network">Network to analyze</param>
/// <param name = "dataset">Dataset with input and ideal data</param>
/// <returns>Error % of correct bits, returned by the network.</returns>
public static double MeasurePerformance(BasicNetwork network, BasicNeuralDataSet dataset)
{
int correctBits = 0;
float threshold = 0.0f;
IActivationFunction activationFunction = network.GetActivation(network.LayerCount - 1); //get the activation function of the output layer
if (activationFunction is ActivationSigmoid)
{
threshold = 0.5f; /* > 0.5, range of sigmoid [0..1]*/
}
else if (activationFunction is ActivationTANH)
{
threshold = 0.0f; /*> 0, range of bipolar sigmoid is [-1..1]*/
}
else
throw new ArgumentException("Bad activation function");
int n = (int) dataset.Count;
Parallel.For(0, n, (i) =>
{
IMLData actualOutputs = network.Compute(dataset.Data[i].Input);
lock (LockObject)
{
for (int j = 0, k = actualOutputs.Count; j < k; j++)
if ((actualOutputs[j] > threshold && dataset.Data[i].Ideal[j] > threshold)
|| (actualOutputs[j] < threshold && dataset.Data[i].Ideal[j] < threshold))
correctBits++;
}
});
long totalOutputBitsCount = dataset.Count*dataset.Data[0].Ideal.Count;
return (double) correctBits/totalOutputBitsCount;
}
/// <summary>
/// Generate a random training set.
/// </summary>
/// <param name="seed">The seed value to use, the same seed value will always produce
/// the same results.</param>
/// <param name="count">How many training items to generate.</param>
/// <param name="inputCount">How many input numbers.</param>
/// <param name="idealCount">How many ideal numbers.</param>
/// <param name="min">The minimum random number.</param>
/// <param name="max">The maximum random number.</param>
/// <returns>The random training set.</returns>
public static BasicNeuralDataSet Generate(long seed,
int count, int inputCount,
int idealCount, double min, double max)
{
LinearCongruentialGenerator rand =
new LinearCongruentialGenerator(seed);
BasicNeuralDataSet result = new BasicNeuralDataSet();
for (int i = 0; i < count; i++)
{
INeuralData inputData = new BasicNeuralData(inputCount);
for (int j = 0; j < inputCount; j++)
{
inputData.Data[j] = rand.Range(min, max);
}
INeuralData idealData = new BasicNeuralData(idealCount);
for (int j = 0; j < idealCount; j++)
{
idealData[j] = rand.Range(min, max);
}
BasicNeuralDataPair pair = new BasicNeuralDataPair(inputData,
idealData);
result.Add(pair);
}
return result;
}
/// <summary>
/// Clone this object.
/// </summary>
/// <returns>A clone of this object.</returns>
public override object Clone()
{
BasicNeuralDataSet result = new BasicNeuralDataSet();
foreach (INeuralDataPair pair in this.Data)
{
result.Add((INeuralDataPair)pair.Clone());
}
return(result);
}
public void LoadTrainingData(ref BasicNeuralDataSet trainingSet, ref BasicNeuralDataSet validationSet)
{
ParseTrainingModel(TrainingModel);
var trainSetCount = (int)(Input.Count() * ((100.0 - ValidationSetSize) / 100));
Input.Shuffle();
Output.Shuffle();
MyExtensions.ResetStableShuffle();
trainingSet = new BasicNeuralDataSet(Input.Take(trainSetCount).ToArray(), Output.Take(trainSetCount).ToArray());
validationSet = new BasicNeuralDataSet(Input.Skip(trainSetCount).ToArray(), Output.Skip(trainSetCount).ToArray());
}
public void Execute(IExampleInterface app)
{
//Specify the number of dimensions and the number of neurons per dimension
int dimensions = 2;
int numNeuronsPerDimension = 7;
//Set the standard RBF neuron width.
//Literature seems to suggest this is a good default value.
double volumeNeuronWidth = 2.0 / numNeuronsPerDimension;
//RBF can struggle when it comes to flats at the edge of the sample space.
//We have added the ability to include wider neurons on the sample space boundary which greatly
//improves fitting to flats
bool includeEdgeRBFs = true;
#region Setup
//General setup is the same as before
RadialBasisPattern pattern = new RadialBasisPattern();
pattern.InputNeurons = dimensions;
pattern.OutputNeurons = 1;
//Total number of neurons required.
//Total number of Edges is calculated possibly for future use but not used any further here
int numNeurons = (int)Math.Pow(numNeuronsPerDimension, dimensions);
int numEdges = (int)(dimensions * Math.Pow(2, dimensions - 1));
pattern.AddHiddenLayer(numNeurons);
BasicNetwork network = pattern.Generate();
RadialBasisFunctionLayer rbfLayer = (RadialBasisFunctionLayer)network.GetLayer(RadialBasisPattern.RBF_LAYER);
network.Reset();
//Position the multidimensional RBF neurons, with equal spacing, within the provided sample space from 0 to 1.
rbfLayer.SetRBFCentersAndWidthsEqualSpacing(0, 1, RBFEnum.Gaussian, dimensions, volumeNeuronWidth, includeEdgeRBFs);
#endregion
//Create some training data that can not easily be represented by gaussians
//There are other training examples for both 1D and 2D
//Degenerate training data only provides outputs as 1 or 0 (averaging over all outputs for a given set of inputs would produce something approaching the smooth training data).
//Smooth training data provides true values for the provided input dimensions.
Create2DSmoothTainingDataGit();
//Create the training set and train.
INeuralDataSet trainingSet = new BasicNeuralDataSet(INPUT, IDEAL);
ITrain train = new SVDTraining(network, trainingSet);
//SVD is a single step solve
int epoch = 1;
do
{
train.Iteration();
Console.WriteLine("Epoch #" + epoch + " Error:" + train.Error);
epoch++;
} while ((epoch < 1) && (train.Error > 0.001));
// test the neural network
Console.WriteLine("Neural Network Results:");
//Create a testing array which may be to a higher resoltion than the original training data
Set2DTestingArrays(100);
trainingSet = new BasicNeuralDataSet(INPUT, IDEAL);
//Write out the results data
using (var sw = new System.IO.StreamWriter("results.csv", false))
{
foreach (INeuralDataPair pair in trainingSet)
{
INeuralData output = network.Compute(pair.Input);
//1D//sw.WriteLine(InverseScale(pair.Input[0]) + ", " + Chop(InverseScale(output[0])));// + ", " + pair.Ideal[0]);
sw.WriteLine(InverseScale(pair.Input[0]) + ", " + InverseScale(pair.Input[1]) + ", " + Chop(InverseScale(output[0])));// + ", " + pair.Ideal[0]);// + ",ideal=" + pair.Ideal[0]);
//3D//sw.WriteLine(InverseScale(pair.Input[0]) + ", " + InverseScale(pair.Input[1]) + ", " + InverseScale(pair.Input[2]) + ", " + Chop(InverseScale(output[0])));// + ", " + pair.Ideal[0]);// + ",ideal=" + pair.Ideal[0]);
//Console.WriteLine(pair.Input[0] + ", actual=" + output[0] + ",ideal=" + pair.Ideal[0]);
}
}
Console.WriteLine("\nFit output saved to results.csv");
Console.WriteLine("\nComplete - Please press the 'any' key to close.");
Console.ReadKey();
}
public virtual void CalculateResult()
{
var values = new List<double[]>();
var result = new List<double[]>();
foreach (var input in TestSet)
{
var output = new double[Network.OutputSize];
Network.Model.Compute(input, output);
values.Add(input);
result.Add(output);
}
ResultTestSet = new BasicNeuralDataSet(values.ToArray(), result.ToArray());
}
/// <summary>
/// Mencari solusi model neural network
/// </summary>
private void searchSolution()
{
// Normalize Data
switch (this.selectedActivationFunction)
{
case ActivationFunctionEnumeration.SemiLinearFunction:
this.activationFunction = new SemiLinearFunction();
this.normalizeData(0.1, 0.9);
break;
case ActivationFunctionEnumeration.SigmoidFunction:
this.activationFunction = new SigmoidFunction();
this.normalizeData(0.1, 0.9);
break;
case ActivationFunctionEnumeration.BipolarSigmoidFunction:
this.activationFunction = new BipolarSigmoidFunction();
this.normalizeData(-0.9, 0.9);
break;
case ActivationFunctionEnumeration.HyperbolicTangentFunction:
this.activationFunction = new HyperbolicTangentFunction();
this.normalizeData(-0.9, 0.9);
break;
default:
this.activationFunction = new BipolarSigmoidFunction();
this.normalizeData(-0.9, 0.9);
break;
}
//create network
this.network = new BasicNetwork();
this.network.AddLayer(new FeedforwardLayer(this.activationFunction, this.inputLayerNeurons));
this.network.AddLayer(new FeedforwardLayer(this.activationFunction, this.hiddenLayerNeurons));
this.network.AddLayer(new FeedforwardLayer(this.activationFunction, this.outputLayerNeurons));
this.network.Reset();
//variable for looping
//needToStop = false;
double mse = 0.0, error = 0.0, mae=0.0;
int iteration = 1;
// parameters
double msle = 0.0, mspe = 0.0, generalizationLoss = 0.0, pq = 0.0;
double[] trainingErrors = new double[this.strip];
for (int i = 0; i < this.strip; i++) trainingErrors[i] = double.MaxValue / strip;
double lastMSE = double.MaxValue;
// advanced early stopping
int n = this.data.Length - this.network.InputLayer.NeuronCount;
int validationSet = (int)Math.Round(this.validationSetRatio * n);
int trainingSet = n - validationSet;
double[][] networkTrainingInput = new double[trainingSet][];
double[][] networkTrainingOutput = new double[trainingSet][];
for (int i = 0; i < trainingSet; i++)
{
networkTrainingInput[i] = new double[this.network.InputLayer.NeuronCount];
networkTrainingOutput[i] = new double[1];
}
for (int i = 0; i < trainingSet; i++)
{
for (int j = 0; j < this.network.InputLayer.NeuronCount; j++)
{
networkTrainingInput[i][j] = this.networkInput[i][j];
}
networkTrainingOutput[i][0] = this.networkOutput[i][0];
}
// validation set
double[] solutionValidation = new double[validationSet];
double[] inputForValidation = new double[this.network.InputLayer.NeuronCount];
double[] inputForValidationNetwork = new double[this.network.InputLayer.NeuronCount];
// array for saving neural weights and parameters
this.bestValidationError = double.MaxValue;
this.bestWeightMatrix = new double[this.network.Layers.Count -1][,];
this.bestSolution = new double[n];
for (int i = 0; i < this.network.Layers.Count - 1; i++)
{
this.bestWeightMatrix[i] = new double[this.network.Layers[i].WeightMatrix.Rows, this.network.Layers[i].WeightMatrix.Cols];
}
//best network criterion
double bestNetworkError = double.MaxValue, bestNetworkMSE = double.MaxValue, bestNetworkMAE = double.MaxValue;
// build array for graph
this.solutionData = new double[n];
this.predictedPoint = new cPoint[n];
this.validationPoint = new cPoint[validationSet];
//initialize point for graph
predictedDS.Samples = predictedPoint;
validationDS.Samples = validationPoint;
this.predictedDS.Active = true;
// prepare training data
INeuralDataSet dataset;
if (this.useAdvanceEarlyStopping)
dataset = new BasicNeuralDataSet(networkTrainingInput, networkTrainingOutput);
else
dataset = new BasicNeuralDataSet(this.networkInput, this.networkOutput);
// initialize trainer
this.learning = new Backpropagation(this.network, dataset, this.learningRate, this.momentum);
//training
while (!needToStop)
{
double sse = 0.0;
double sae = 0.0;
double ssle = 0.0;
double sspe = 0.0;
this.learning.Iteration();
error = learning.Error;
if (this.useAdvanceEarlyStopping)
{
this.validationDS.Active = true;
}
else
{
this.validationDS.Active = false;
}
for (int i = 0; i < n; i++)
{
INeuralData neuraldata = new BasicNeuralData(this.networkInput[i]);
this.solutionData[i] = (this.network.Compute(neuraldata)[0]
- this.minNormalizedData) / this.factor + this.minData;
this.predictedPoint[i].x = i + this.network.InputLayer.NeuronCount;
this.predictedPoint[i].y = (float)this.solutionData[i];
sse += Math.Pow(this.solutionData[i] - this.data[i + this.network.InputLayer.NeuronCount], 2);
sae += Math.Abs(this.solutionData[i] - this.data[i + this.network.InputLayer.NeuronCount]);
//calculate advance early stopping
if (this.useAdvanceEarlyStopping)
{
if (i < n - validationSet)
{
ssle += Math.Pow(this.solutionData[i] - this.data[i + this.network.InputLayer.NeuronCount], 2);
}
else
{
// initialize the first validation set input
if (i == n - validationSet)
{
for (int j = 0; j < this.network.InputLayer.NeuronCount; j++)
{
inputForValidation[this.network.InputLayer.NeuronCount - 1 - j] = this.data[this.data.Length - (n - i) - 1 - j];
}
}
for (int j = 0; j < this.network.InputLayer.NeuronCount; j++)
{
inputForValidationNetwork[j] = (inputForValidation[j] - this.minData) * this.factor + this.minNormalizedData;
}
INeuralData neuraldataval = new BasicNeuralData(inputForValidationNetwork);
solutionValidation[i - n + validationSet] = (this.network.Compute(neuraldataval)[0] - this.minNormalizedData) / this.factor + this.minData;
this.validationPoint[i - n + validationSet].x = i + this.network.InputLayer.NeuronCount;
this.validationPoint[i - n + validationSet].y = (float)solutionValidation[i - n + validationSet];
sspe += Math.Pow(this.data[i + this.network.InputLayer.NeuronCount] - solutionValidation[i - n + validationSet], 2);
// initialize the next validation set input from the current validation set input
for (int j = 0; j < this.network.InputLayer.NeuronCount - 1; j++)
{
inputForValidation[j] = inputForValidation[j + 1];
}
inputForValidation[this.network.InputLayer.NeuronCount - 1] = solutionValidation[i - n + validationSet];
}
}
}
mse = sse / this.solutionData.Length;
mae = sae / this.solutionData.Length;
//Console.WriteLine(error.ToString());
//Display it
this.iterationBox.Text = iteration.ToString();
this.maeBox.Text = mae.ToString("F5");
this.mseBox.Text = mse.ToString("F5");
this.errorBox.Text = error.ToString("F5");
seriesGraph.Refresh();
if (this.useAdvanceEarlyStopping)
{
//calculate advance early stopping 2
mspe = sspe / validationSet;
msle = ssle / (this.solutionData.Length - validationSet);
//save best weight
if (this.bestValidationError > mspe)
{
this.bestValidationError = mspe;
this.bestSolution = this.solutionData;
// weight matrix
for (int i = 0; i < this.network.Layers.Count - 1; i++)
for (int j = 0; j < this.network.Layers[i].WeightMatrix.Rows; j++)
for (int k = 0; k < this.network.Layers[i].WeightMatrix.Cols; k++)
this.bestWeightMatrix[i][j,k] = this.network.Layers[i].WeightMatrix[j, k];
bestNetworkError = error;
bestNetworkMAE = mae;
bestNetworkMSE = mse;
}
//calculate generalization loss &pq
generalizationLoss = 100 * (mspe / this.bestValidationError - 1);
trainingErrors[(iteration - 1) % this.strip] = msle;
double minStripTrainingError = double.MaxValue, sumStripTrainingError = 0.0;
for (int i = 0; i < this.strip; i++)
{
sumStripTrainingError += trainingErrors[i];
if (trainingErrors[i] < minStripTrainingError) minStripTrainingError = trainingErrors[i];
}
double trainingProgress = 1000 * ((sumStripTrainingError / (this.strip * minStripTrainingError)) - 1);
pq = generalizationLoss / trainingProgress;
//display advance early stopping
this.learningErrorBox.Text = msle.ToString("F5");
this.validationErrorBox.Text = mspe.ToString("F5");
this.generalizationLossBox.Text = generalizationLoss.ToString("F5");
this.pqBox.Text = pq.ToString("F5");
this.seriesGraph.Refresh();
//stopping
switch (this.advanceStoppingMethod)
{
case AdvanceStoppingMethodEnumeration.GeneralizationLoss:
if (generalizationLoss > this.generalizationLossTreshold) needToStop = true;
break;
case AdvanceStoppingMethodEnumeration.ProgressQuotient:
if (pq > this.pqTreshold) needToStop = true;
break;
}
}
if (this.withCheckingCycle && iteration % this.checkingCycle == 0)
{
switch (this.checkingMethod)
{
case CheckingMethodEnumeration.byMSEValue:
if (mse <= this.byMSEValueStopping) needToStop = true;
break;
case CheckingMethodEnumeration.byMSEChange:
if (lastMSE - mse <= this.byMSEChangeStopping) needToStop = true;
break;
}
lastMSE = mse;
}
if (iteration >= this.maxIteration)
{
needToStop = true;
}
iteration++;
}
//restore weight
if (this.useAdvanceEarlyStopping)
{
this.solutionData = this.bestSolution;
// weight matrix
for (int i = 0; i < this.network.Layers.Count - 1; i++)
for (int j = 0; j < this.network.Layers[i].WeightMatrix.Rows; j++)
for (int k = 0; k < this.network.Layers[i].WeightMatrix.Cols; k++)
this.network.Layers[i].WeightMatrix[j, k] = this.bestWeightMatrix[i][j, k];
//best network criterion
this.error = bestNetworkError;
this.mse = bestNetworkMSE;
this.mae = bestNetworkMAE;
}
else
{
this.error = error;
this.mse = mse;
this.mae = mae;
}
this.enableControls(true);
}
/// <summary>
/// This is based off of this article:
/// http://www.codeproject.com/Articles/54575/An-Introduction-to-Encog-Neural-Networks-for-C
/// </summary>
/// <remarks>
/// Go here for documentation of encog:
/// http://www.heatonresearch.com/wiki
///
/// Download link:
/// https://github.com/encog/encog-dotnet-core/releases
/// </remarks>
private void btnXOR_Click(object sender, RoutedEventArgs e)
{
try
{
_trainingData = null;
_results = null;
BasicNetwork network = new BasicNetwork();
#region Create nodes
// Create the network's nodes
//NOTE: Using ActivationSigmoid, because there are no negative values. If there were negative, use ActivationTANH
//http://www.heatonresearch.com/wiki/Activation_Function
//NOTE: ActivationSigmoid (0 to 1) and ActivationTANH (-1 to 1) are pure but slower. A cruder but faster function is ActivationElliott (0 to 1) and ActivationElliottSymmetric (-1 to 1)
//http://www.heatonresearch.com/wiki/Elliott_Activation_Function
network.AddLayer(new BasicLayer(new ActivationSigmoid(), true, 2)); // input layer
network.AddLayer(new BasicLayer(new ActivationSigmoid(), true, 6)); // hidden layer
network.AddLayer(new BasicLayer(new ActivationSigmoid(), true, 1)); // output layer
network.Structure.FinalizeStructure();
// Randomize the links
network.Reset();
#endregion
#region Training data
// Neural networks must be trained before they are of any use. To train this neural network, we must provide training
// data. The training data is the truth table for the XOR operator. The XOR has the following inputs:
double[][] xor_input = new[]
{
new[] { 0d, 0d },
new[] { 1d, 0d },
new[] { 0d, 1d },
new[] { 1d, 1d },
};
// And the expected outputs
double[][] xor_ideal_output = new[]
{
new[] { 0d },
new[] { 1d },
new[] { 1d },
new[] { 0d },
};
_trainingData = GetDrawDataFromTrainingData(xor_input, xor_ideal_output);
#endregion
#region Train network
INeuralDataSet trainingSet = new BasicNeuralDataSet(xor_input, xor_ideal_output);
// This is a good general purpose training algorithm
//http://www.heatonresearch.com/wiki/Training
ITrain train = new ResilientPropagation(network, trainingSet);
List<double> log = new List<double>();
int trainingIteration = 1;
do
{
train.Iteration();
log.Add(train.Error);
trainingIteration++;
} while ((trainingIteration < 2000) && (train.Error > 0.001));
// Paste this into excel and chart it to see the error trend
string logExcel = string.Join("\r\n", log);
#endregion
#region Test
//NOTE: I initially ran a bunch of tests, but the network always returns exactly the same result when given the same inputs
//var test = Enumerable.Range(0, 1000).
// Select(o => new { In1 = _rand.Next(2), In2 = _rand.Next(2) }).
var test = xor_input.
Select(o => new { In1 = Convert.ToInt32(o[0]), In2 = Convert.ToInt32(o[1]) }).
Select(o => new
{
o.In1,
o.In2,
Expected = XOR(o.In1, o.In2),
NN = CallNN(network, o.In1, o.In2),
}).
Select(o => new { o.In1, o.In2, o.Expected, o.NN, Error = Math.Abs(o.Expected - o.NN) }).
OrderByDescending(o => o.Error).
ToArray();
#endregion
#region Test intermediate values
// It was only trained with inputs of 0 and 1. Let's see what it does with values in between
var intermediates = Enumerable.Range(0, 1000).
Select(o => new { In1 = _rand.NextDouble(), In2 = _rand.NextDouble() }).
Select(o => new
{
o.In1,
o.In2,
NN = CallNN(network, o.In1, o.In2),
}).
OrderBy(o => o.In1).
ThenBy(o => o.In2).
//OrderBy(o => o.NN).
ToArray();
#endregion
#region Serialize/Deserialize
// Serialize it
string weightDump = network.DumpWeights();
double[] dumpArray = weightDump.Split(',').
Select(o => double.Parse(o)).
ToArray();
//TODO: Shoot through the layers, and store in some custom structure that can be serialized, then walked through to rebuild on deserialize
//string[] layerDump = network.Structure.Layers.
// Select(o => o.ToString()).
// ToArray();
// Create a clone
BasicNetwork clone = new BasicNetwork();
clone.AddLayer(new BasicLayer(new ActivationSigmoid(), true, 2));
clone.AddLayer(new BasicLayer(new ActivationSigmoid(), true, 6));
clone.AddLayer(new BasicLayer(new ActivationSigmoid(), true, 1));
clone.Structure.FinalizeStructure();
clone.DecodeFromArray(dumpArray);
// Test the clone
string cloneDump = clone.DumpWeights();
bool isSame = weightDump == cloneDump;
var cloneTests = xor_input.
Select(o => new
{
Input = o,
NN = CallNN(clone, o[0], o[1]),
}).ToArray();
#endregion
#region Store results
double[] matchValues = new[] { 0d, 1d };
double matchRange = .03; //+- 5% of target value would be considered a match
_results = intermediates.
Select(o => Tuple.Create(new Point(o.In1, o.In2), o.NN, IsMatch(o.NN, matchValues, matchRange))).
ToArray();
#endregion
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString(), this.Title, MessageBoxButton.OK, MessageBoxImage.Error);
}
finally
{
RedrawResults();
}
}
protected void ReorderOutput(Network network, BasicNeuralDataSet dataset, Dictionary<int, List<BasicMLData>> trackIdFingerprints, double[][] binaryCodes)
{
int outputNeurons = network.GetLayerNeuronCount(network.LayerCount - 1);
int trackCount = trackIdFingerprints.Count;
// For each song, compute Am
double[][] am = new double[trackCount][];
int counter = 0;
foreach (KeyValuePair<int, List<BasicMLData>> pair in trackIdFingerprints)
{
List<BasicMLData> sxSnippet = pair.Value;
if (sxSnippet.Count < trainingSongSnippets)
{
throw new NetTrainerException("Not enough snippets for a song");
}
am[counter] = new double[outputNeurons];
foreach (BasicMLData snippet in sxSnippet)
{
IMLData actualOutput = network.Compute(snippet);
for (int k = 0; k < outputNeurons; k++)
{
actualOutput[k] /= outputNeurons;
am[counter][k] += actualOutput[k];
}
}
counter++;
}
// Get a collection of tracks (shallow copy)
int[] unassignedTracks = new int[trackCount];
int countTrack = 0;
foreach (KeyValuePair<int, List<BasicMLData>> item in trackIdFingerprints)
{
unassignedTracks[countTrack++] = item.Key;
}
int currItteration = 0;
// Find binary code - track pair that has min l2 norm across all binary codes
List<Tuple<int, int>> binCodeTrackPair = BinaryOutputUtil.FindMinL2Norm(binaryCodes, am);
foreach (Tuple<int, int> pair in binCodeTrackPair)
{
// Set the input-output for all fingerprints of that song
List<BasicMLData> songFingerprints = trackIdFingerprints[unassignedTracks[pair.Item2]];
foreach (BasicMLData songFingerprint in songFingerprints)
{
for (int i = 0, n = songFingerprint.Count; i < n; i++)
{
dataset.Data[currItteration].Input[i] = songFingerprint[i];
}
for (int i = 0, n = binaryCodes[pair.Item1].Length; i < n; i++)
{
dataset.Data[currItteration].Ideal[i] = binaryCodes[pair.Item1][i];
}
currItteration++;
}
}
}
public void Train(Network network, TrainingCallback callback)
{
IActivationFunction activationFunctionInput = network.GetActivation(0);
int outputNeurons = network.GetLayerNeuronCount(network.LayerCount - 1);
double error = 0;
callback.Invoke(TrainingStatus.FillingStandardInputs, 0, 0, 0); /*First operation is filling standard input/outputs*/
Dictionary<int, List<BasicMLData>> trackIdFingerprints = GetNormalizedTrackFingerprints(activationFunctionInput, trainingSongSnippets, outputNeurons);
workingThread = Thread.CurrentThread;
IActivationFunction activationFunctionOutput = network.GetActivation(network.LayerCount - 1);
double[][] normalizedBinaryCodes = GetNormalizedBinaryCodes(activationFunctionOutput, outputNeurons);
Tuple<double[][], double[][]> tuple = FillStandardInputsOutputs(trackIdFingerprints, normalizedBinaryCodes); /*Fill standard input output*/
double[][] inputs = tuple.Item1;
double[][] outputs = tuple.Item2;
if (inputs == null || outputs == null)
{
callback.Invoke(TrainingStatus.Exception, 0, 0, 0);
return;
}
int currentIterration = 0;
double correctOutputs = 0.0;
BasicNeuralDataSet dataset = new BasicNeuralDataSet(inputs, outputs);
ITrain learner = new ResilientPropagation(network, dataset);
try
{
// Dynamic output reordering cycle
/*Idyn = 50*/
for (int i = 0; i < Idyn; i++)
{
if (paused)
{
pauseSem.WaitOne();
}
correctOutputs = NetworkPerformanceMeter.MeasurePerformance(network, dataset);
callback.Invoke(TrainingStatus.OutputReordering, correctOutputs, error, currentIterration);
ReorderOutput(network, dataset, trackIdFingerprints, normalizedBinaryCodes);
/*Edyn = 10*/
for (int j = 0; j < Edyn; j++)
{
if (paused)
{
pauseSem.WaitOne();
}
correctOutputs = NetworkPerformanceMeter.MeasurePerformance(network, dataset);
callback.Invoke(TrainingStatus.RunningDynamicEpoch, correctOutputs, error, currentIterration);
learner.Iteration();
error = learner.Error;
currentIterration++;
}
}
for (int i = 0; i < Efixed; i++)
{
if (paused)
{
pauseSem.WaitOne();
}
correctOutputs = NetworkPerformanceMeter.MeasurePerformance(network, dataset);
callback.Invoke(TrainingStatus.FixedTraining, correctOutputs, error, currentIterration);
learner.Iteration();
error = learner.Error;
currentIterration++;
}
network.ComputeMedianResponses(inputs, trainingSongSnippets);
callback.Invoke(TrainingStatus.Finished, correctOutputs, error, currentIterration);
}
catch (ThreadAbortException)
{
callback.Invoke(TrainingStatus.Aborted, correctOutputs, error, currentIterration);
paused = false;
}
}
/// <summary>
/// Load the specified Encog object from an XML reader.
/// </summary>
/// <param name="xmlIn">The XML reader to use.</param>
/// <returns>The loaded object.</returns>
public IEncogPersistedObject Load(ReadXML xmlIn)
{
String name = xmlIn.LastTag.Attributes[
EncogPersistedCollection.ATTRIBUTE_NAME];
String description = xmlIn.LastTag.GetAttributeValue(
EncogPersistedCollection.ATTRIBUTE_DESCRIPTION);
this.currentDataSet = new BasicNeuralDataSet();
this.currentDataSet.Name = name;
this.currentDataSet.Description = description;
while (xmlIn.ReadToTag())
{
if (xmlIn.IsIt(BasicNeuralDataSetPersistor.TAG_ITEM, true))
{
HandleItem(xmlIn);
}
else if (xmlIn.IsIt(EncogPersistedCollection.TYPE_TRAINING, false))
{
break;
}
}
return this.currentDataSet;
}
/// <summary>
/// Load the binary dataset to memory. Memory access is faster.
/// </summary>
/// <returns>A memory dataset.</returns>
public INeuralDataSet LoadToMemory()
{
BasicNeuralDataSet result = new BasicNeuralDataSet();
foreach (INeuralDataPair pair in this)
{
result.Add(pair);
}
return result;
}
/// <summary>
/// Construct an enumerator.
/// </summary>
/// <param name="owner">The owner of the enumerator.</param>
public BasicNeuralEnumerator(BasicNeuralDataSet owner)
{
this.current = -1;
this.owner = owner;
}
public void PerLineClassification(object sender, DoWorkEventArgs e)
{
worker = sender as BackgroundWorker;
//inputtemplate = new double[DataContainer.model.timetable.rosterings.Count];
//idealtemplate = new double[DataContainer.model.timetable.rosterings.Count * 4];
inputtemplate = new double[23];
idealtemplate = new double[23 * 4];
Dictionary<string, int> inputmap = new Dictionary<string, int>()
{
{"Belfast - Connolly", 0},
{"DART",1},
{"IWT",2},
{"DFDS",3},
{"Timber",4},
{"Tara Mines",5},
{"Northern Commuter",6},
{"Cork - Heuston",7},
{"Heuston Commuter",8},
{"Tralee",9 },
{"Limerick - Heuston",10},
{"Limerick Junction - Limerick",11},
{"Ballybrophy - Limerick",12},
{"Waterford - Heuston",13},
{"Waterford - Limerick Junction",14},
{"Rosslare - Waterford",15},
{"Rosslare - Connolly",16},
{"Galway - Heuston",17},
{"Westport - Heuston",18},
{"Sligo - Connolly",19},
{"Maynooth Commuter",20}
};
inputtemplate = new double[inputmap.Count];
idealtemplate = new double[inputmap.Count * 4];
List<double[]> inputlist = new List<double[]>();
List<double[]> outputlist = new List<double[]>();
foreach(List<DelayCombination> day in DataContainer.NeuralNetwork.DelayCombinations.dict.Values.Where(x => x != null))
{
foreach(DelayCombination delaycombination in day)
{
worker.ReportProgress(0, "Preprocessing Data... Date : " + delaycombination.primarydelays[0].date.ToString("dd/MM/yyyy"));
double[] inputline = new double[inputtemplate.Length];
double[] outputline = new double[idealtemplate.Length];
double[] secondarydelaysize = new double[inputtemplate.Length];
foreach (Delay d in delaycombination.primarydelays)
{
//string line = DataContainer.model.timetable.trains.Single(x => x.id == d.traincode).description;
if (GetLine(d) == "None") { goto skip; }
inputline[inputmap[GetLine(d)]] += d.destinationdelay;
}
foreach (Delay d in delaycombination.secondarydelays)
{
//string line = DataContainer.model.timetable.trains.Single(x => x.id == d.traincode).description;
if (GetLine(d) == "None") { goto skip; }
secondarydelaysize[inputmap[GetLine(d)]] += d.destinationdelay;
}
for (int i = 0; i < secondarydelaysize.Length; i++)
{
if (secondarydelaysize[i] == 0)
{
outputline[i * 4] = 1;
}
else if (secondarydelaysize[i] < 300)
{
outputline[i * 4 + 1] = 1;
}
else if (secondarydelaysize[i] < 600)
{
outputline[i * 4 + 2] = 1;
}
else
{
outputline[i * 4 + 3] = 1;
}
}
inputlist.Add(inputline);
outputlist.Add(outputline);
skip:
continue;
}
}
double[][] input = inputlist.ToArray();
double[][] output = outputlist.ToArray();
INeuralDataSet dataset = new BasicNeuralDataSet(input, output);
DataContainer.NeuralNetwork.Data = dataset;
}
private static EncogTrainingResponse TrainNetwork2(BasicNetwork network, TrainingData training, double maxError, CancellationToken cancelToken, double? maxSeconds = null)
{
//TODO: When the final layer is softmax, the error seems to be higher. Probably because the training outputs need to be run through softmax
const int MAXITERATIONS = 5000;
INeuralDataSet trainingSet = new BasicNeuralDataSet(training.Input, training.Output);
ITrain train = new ResilientPropagation(network, trainingSet);
DateTime startTime = DateTime.UtcNow;
TimeSpan? maxTime = maxSeconds != null ? TimeSpan.FromSeconds(maxSeconds.Value) : (TimeSpan?)null;
bool success = false;
//List<double> log = new List<double>();
int iteration = 1;
double error = double.MaxValue;
while (true)
{
if (cancelToken.IsCancellationRequested)
{
break;
}
train.Iteration();
error = train.Error;
//log.Add(error);
iteration++;
if (double.IsNaN(error))
{
break;
}
else if (error < maxError)
{
success = true;
break;
}
else if (iteration >= MAXITERATIONS)
{
break;
}
else if (maxTime != null && DateTime.UtcNow - startTime > maxTime)
{
break;
}
}
//string logExcel = string.Join("\r\n", log); // paste this into excel and chart it to see the error trend
train.FinishTraining();
return new EncogTrainingResponse(network, success, error, iteration, (DateTime.UtcNow - startTime).TotalSeconds);
}
