public override ConfusionMatrix Execute()
{
//Create an network with one layer and one neuron in that layer
var network = new ActivationNetwork(new ThresholdFunction(), 3, 1);
//Bind the reference of the neuron
var neuron = network.Layers[0].Neurons[0] as ActivationNeuron;
//Create the Perceptron learning algorithm
//Library perceptron implements a single layer linear classifier
var teacher = new PerceptronLearning(network);
teacher.LearningRate = 0.1;
//Enrich the dimensions of the vectors, padding 1 to the end
var richTraining = AlgorithmHelpers.PaddDimension(trainingSet);
var richTesting = AlgorithmHelpers.PaddDimension(testSet);
//Training the network until the error is small enough
//or 500 hundred iterations have been computed
int epochs = 0;
while (true)
{
double error = teacher.RunEpoch(richTraining, trainingOutput);/// trainingSet.Length;
++epochs;
if (error < 0.025 * trainingSet.Length || epochs == 500) break;
}
var predicted = richTesting
.Select(x => neuron.Compute(x))
.Select(x => Convert.ToInt32(x))
.ToArray();
//Create a confusion matrix with the calculated parameters
ConfusionMatrix cmatrix = new ConfusionMatrix(predicted, expected, POSITIVE, NEGATIVE);
OnAlgorithmEnded(Enumerable.Repeat(neuron, 1), cmatrix);
return cmatrix;
}
public AdditionalNeuralNetwork(int numberOfTags)
{
this.network = new ActivationNetwork(new SigmoidFunction(1.0d),numberOfTags,numberOfTags);
this.teacher = new ISupervisedLearning[] {new DeltaRuleLearning(this.network),new PerceptronLearning(this.network),new BackPropagationLearning(this.network)};
//this.teacher.LearningRate = 0.10d;
//this.teacher.Momentum = 0.10d;
}
public void test() {
// initialize input and output values
var input = new double[4][] {
new double[] { 0, 0 }, new double[] { 0, 1 },
new double[] { 1, 0 }, new double[] { 1, 1 }
};
var output = new double[4][] {
new double[] { 0 }, new double[] { 1 },
new double[] { 1 }, new double[] { 1 }
};
// create neural network
var network = new ActivationNetwork(
new SigmoidFunction(2),
2, // two inputs in the network
//2, // two neurons in the first layer
1); // one neuron in the second layer
// create teacher
var teacher =
new BackPropagationLearning(network);
// loop
while (true) {
// run epoch of learning procedure
var error = teacher.RunEpoch(input, output);
// check error value to see if we need to stop
// ...
if (error < 0.001) {
break;
}
}
Console.WriteLine(network.Compute(new double[] { 0, 0 })[0] + ","
+ network.Compute(new double[] { 0, 1 })[0] + ","
+ network.Compute(new double[] { 1, 0 })[0] + ","
+ network.Compute(new double[] { 1, 1 })[0]);
}
public IA(Game game, int players)
: base(game)
{
rndSeedGen = new Random();
rndControl = new Random();
rndMovControl = new Random();
this.comidas = null;
this.jugadores = null;
this.numWeights = HIDDEN_UNITS0 * (INPUT_UNITS + 1) + HIDDEN_UNITS1 * (HIDDEN_UNITS0 + 1) + OUTPUT_UNITS * (HIDDEN_UNITS1 + 1);
redes = new ActivationNetwork[players];
for (int i = 0; i < redes.Length; i++)
{
redes[i] = new ActivationNetwork(new SigmoidFunction(400), INPUT_UNITS, HIDDEN_UNITS0, HIDDEN_UNITS1, OUTPUT_UNITS);
}
inputVector = new double[INPUT_UNITS];
outputVector = new double[OUTPUT_UNITS];
doneEvents = new ManualResetEvent[players];
for (int i = 0; i < players; i++) doneEvents[i] = new ManualResetEvent(false);
//Se puede jugar con los parametros de los rangos para modificar la evolucion de las redes
//Tambien se puede modificar el metodo de seleccion.
chromosomeGenerator = new UniformGenerator(new Range(-10f, 10f), rndSeedGen.Next(-100, 100));
mutationAdditionGenerator = new UniformGenerator(new Range(-8f, 8f), rndSeedGen.Next(-100, 100));
mutationMultiplierGenerator = new UniformGenerator(new Range(-8f, 8f), rndSeedGen.Next(-100, 100));
fitnessFunction = new GameFitnessFunction();
selectionMethod = new EliteSelection();
padre = new gameChromosome(chromosomeGenerator, mutationMultiplierGenerator, mutationAdditionGenerator, numWeights);
poblacion = new Population(WorldGame.JUGADORES, padre, fitnessFunction, selectionMethod);
}
public virtual void Prepare()
{
PrepareData();
PrepareCharts();
network = new ActivationNetwork(new Tanh(0.2),
Sizes[0],
Sizes.Skip(1).ToArray());
network.ForEachWeight(z => rnd.NextDouble() * 2 - 1);
teacher = new BackPropagationLearning(network);
teacher.LearningRate = 1;
Form = new Form()
{
Text = GetType().Name,
Size = new Size(800, 600),
FormBorderStyle = FormBorderStyle.FixedDialog,
Controls =
{
AreaChart,
HistoryChart
}
};
}
static void ForEachWeight(ActivationNetwork network, Func<double, double> modifier)
{
foreach (var l in network.Layers)
foreach (var n in l.Neurons)
for (int i = 0; i < n.Weights.Length; i++)
n.Weights[i] = modifier(n.Weights[i]);
}
public void Test()
{
ActivationNetwork network = new ActivationNetwork(
new SigmoidFunction(),
2, // two inputs in the network
2, // two neurons in the first layer
1); // one neuron in the second layer
BackPropagationLearning teacher = new BackPropagationLearning(network);
double lastError = double.MaxValue;
int counter = 0;
while (true)
{
counter++;
var error = teacher.RunEpoch(input, output);
if (lastError - error < 0.0000001 && error < 0.001)
break;
lastError = error;
}
//var bla = network.Compute(input[0])[0];
//var round = Math.Round(network.Compute(input[0])[0], 2);
//var result = output[0][0];
//Assert.IsTrue(Math.Abs(round - result) < double.Epsilon);
Assert.IsTrue(Math.Abs(network.Compute(input[0])[0] - output[0][0]) < 0.03);
Assert.IsTrue(Math.Abs(network.Compute(input[1])[0] - output[1][0]) < 0.03);
Assert.IsTrue(Math.Abs(network.Compute(input[2])[0] - output[2][0]) < 0.03);
Assert.IsTrue(Math.Abs(network.Compute(input[3])[0] - output[3][0]) < 0.03);
Console.WriteLine($"Loop counter = {counter}.");
}
public void Learn()
{
var network = new ActivationNetwork(new BipolarSigmoidFunction(), Constants.StoneCount, 1);
var teacher = new BackPropagationLearning(network);//new PerceptronLearning(network);
var data = LoadData("4-6-2012-04-24.know");
double error = 1.0;
int index = 0;
while (error > 0.001 && index < 100000) {
error = teacher.RunEpoch(data.Item1, data.Item2);
index++;
}
network.Save("4-6-2012-04-24.bp.net");
var text = "□○○○●○○□○●●□□●□□";
var i = ToDouble(text);//-2
var o = network.Compute(i);
var eval = o[0] * 2 * Constants.StoneCount - Constants.StoneCount;
Console.WriteLine("{0} {1}", text, eval);
}
protected virtual void CreateNetwork()
{
network = new ActivationNetwork(new Tanh(1), 1, 5, 1);
network.ForEachWeight(z => rnd.NextDouble() * 2 - 1);
teacher = new BackPropagationLearning(network);
teacher.LearningRate = 1;
}
protected override void CreateNetwork()
{
network = new ActivationNetwork(new Tanh(0.1), 1, 5, 1);
network.ForEachWeight(z => rnd.NextDouble() * 2 - 1);
teacher = new BackPropagationLearning(network);
teacher.LearningRate = 1;
teacher.Momentum = 0.3;
}
public NeuralNetworkBot1()
: base(
"Neural Network Bot I",
"Runs a neural network neural network to compute a score out of different simulated moves",
true)
{
_network = new ActivationNetwork(new SigmoidFunction(), 4*4, 4*4, 1);
//_network.Randomize();
//ActivationNetwork.Load();
//_network.Save();
}
public EstimationResult Estimate(IEnumerable<IDateValue> dateValues)
{
var data = dateValues.ToArray();
var samplesCount = data.Length - LayerWidth;
var factor = 1.7 / data.Length;
var yMin = data.Min(x => x.Value);
var input = new double[samplesCount][];
var output = new double[samplesCount][];
for (var i = 0; i < samplesCount; i++)
{
input[i] = new double[LayerWidth];
output[i] = new double[1];
for (var j = 0; j < LayerWidth; j++)
input[i][j] = (data[i + j].Value - yMin) * factor - 0.85;
output[i][0] = (data[i + LayerWidth].Value - yMin) * factor - 0.85;
}
var network = new ActivationNetwork(
new BipolarSigmoidFunction(SigmoidAlphaValue),
LayerWidth, LayerWidth * 2, 1);
var teacher = new BackPropagationLearning(network)
{
LearningRate = LearningRate,
Momentum = Momentum
};
var solutionSize = data.Length - LayerWidth;
var solution = new double[solutionSize, 2];
var networkInput = new double[LayerWidth];
for (var j = 0; j < solutionSize; j++)
solution[j, 0] = j + LayerWidth;
TimesLoop.Do(Iterations, () =>
{
teacher.RunEpoch(input, output);
for (int i = 0, n = data.Length - LayerWidth; i < n; i++)
{
for (var j = 0; j < LayerWidth; j++)
networkInput[j] = (data[i + j].Value - yMin) * factor - 0.85;
solution[i, 1] = (network.Compute(networkInput)[0] + 0.85) / factor + yMin;
}
});
return EstimationResult.Create(solution[0, 1], this);
}
public NeuralNetworkBot2()
: base(
"Neural Network Bot II",
"Runs a neural network neural network to directly calculate outputs for each move",
true)
{
_network = new ActivationNetwork(new SigmoidFunction(), 4*4, 4*4, 2);
//_network.Randomize();
Load();
_teacher = new BackPropagationLearning(_network);
_teacher.LearningRate = 0.05;
_teacher.Momentum = 0.05;
}
public static void ChromosomeToNetwork(gameChromosome chromosome, ActivationNetwork network)
{
double[] values = chromosome.Value;
int l = 0;
for (int i = 0; i < network.LayersCount; i++)
for (int j = 0; j < network[i].NeuronsCount; j++)
for (int k = 0; k <= network[i][j].InputsCount; k++)
{
if (k == 0)
network[i][j].Threshold = values[l];
else
network[i][j][k - 1] = values[l];
l++;
}
}
static void Main(string[] args)
{
// initialize input and output values
double[][] input = new double[4][] {
new double[] {0, 0}, new double[] {0, 1},
new double[] {1, 0}, new double[] {1, 1}
};
double[][] output = new double[4][] {
new double[] {0}, new double[] {1},
new double[] {1}, new double[] {0}
};
// create neural network
ActivationNetwork network = new ActivationNetwork(
new SigmoidFunction(1),
2, // two inputs in the network
2, // two neurons in the first layer
1); // one neuron in the second layer
// create teacher
BackPropagationLearning teacher =
new BackPropagationLearning(network);
// loop
for (int i = 0; i < 10000; i++)
{
// run epoch of learning procedure
double error = teacher.RunEpoch(input, output);
// check error value to see if we need to stop
// ...
Console.Out.WriteLine("#" + i + "\t" + error);
}
double[] ret1 = network.Compute(new double[] { 0, 0 });
double[] ret2 = network.Compute(new double[] { 1, 0 });
double[] ret3 = network.Compute(new double[] { 0, 1 });
double[] ret4 = network.Compute(new double[] { 1, 1 });
Console.Out.WriteLine();
Console.Out.WriteLine("Eval(0, 0) = " + ret1[0]);
Console.Out.WriteLine("Eval(1, 0) = " + ret2[0]);
Console.Out.WriteLine("Eval(0, 1) = " + ret3[0]);
Console.Out.WriteLine("Eval(1, 1) = " + ret4[0]);
Console.ReadLine();
}
static void Main()
{
weights = new double[3];
network = new ActivationNetwork(new SignumActivationFunction(), 2, 1);
weights[0] = ((ActivationNeuron)network.Layers[0].Neurons[0]).Threshold = 0;
weights[1] = network.Layers[0].Neurons[0].Weights[0] = 0.9;
weights[2] = network.Layers[0].Neurons[0].Weights[1] = 0.2;
learning = new PerceptronLearning(network);
learning.LearningRate = 0.005;
form = new MyForm() { WindowState = FormWindowState.Maximized };
form.Paint += (s, a) => Redraw(a.Graphics);
var timer = new System.Windows.Forms.Timer();
timer.Interval = 10;
timer.Tick += NextSample;
timer.Start();
Application.Run(form);
}
public double Evaluate(IChromosome chromosome)
{
// Конструираме невронна мрежа и изчисляваме резултата
DoubleArrayChromosome dac = (DoubleArrayChromosome)chromosome;
ActivationNetwork Network = new ActivationNetwork(
new BipolarSigmoidFunction(sigmoidAlphaValue),
mArchitecture[0], mArchitecture[1], mArchitecture[2]);
int current = 0;
int i = 0;
// Тегла на скрит слой
for (i = 0; i < mArchitecture[1]; i++)
{
for(int j=0; j < mArchitecture[0]; j++){
Network[0][i][j] = dac.Value[current++];
}
}
// Тегла на изходен слой
for (i = 0; i < mArchitecture[2]; i++)
{
for (int j = 0; j < mArchitecture[1]; j++)
{
Network[1][i][j] = dac.Value[current++];
}
}
double Sum = 0.0;
for (int cnt = 0; cnt < mInput.Length; cnt++)
{
double[] predicted_output = Network.Compute(mInput[cnt]);
for (int l = 0; l < predicted_output.Length; l++)
{
Sum += (predicted_output[l] - mOutput[cnt][l]) * (predicted_output[l] - mOutput[cnt][l]);
}
}
return 100-Sum;
}
public void TestGenetic()
{
ActivationNetwork network = new ActivationNetwork(new SigmoidFunction(), 2, 2, 1);
EvolutionaryLearning superTeacher = new EvolutionaryLearning(network, 10);
double lastError = double.MaxValue;
int counter = 0;
while (true)
{
counter++;
var error = superTeacher.RunEpoch(input, output);
if (lastError - error < 0.0000001 && error < 0.0001)
break;
lastError = error;
}
Assert.IsTrue(Math.Abs(network.Compute(input[0])[0] - output[0][0]) < 0.03);
Assert.IsTrue(Math.Abs(network.Compute(input[1])[0] - output[1][0]) < 0.03);
Assert.IsTrue(Math.Abs(network.Compute(input[2])[0] - output[2][0]) < 0.03);
Assert.IsTrue(Math.Abs(network.Compute(input[3])[0] - output[3][0]) < 0.03);
Console.WriteLine($"Loop counter = {counter}.");
}
public void RunEpochTest1()
{
Accord.Math.Tools.SetupGenerator(0);
double[][] input =
{
new double[] { -1, -1 },
new double[] { -1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
double[][] output =
{
new double[] { -1 },
new double[] { 1 },
new double[] { 1 },
new double[] { -1 }
};
Neuron.RandGenerator = new ThreadSafeRandom(0);
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(2), 2, 2, 1);
var teacher = new ParallelResilientBackpropagationLearning(network);
double error = 1.0;
while (error > 1e-5)
error = teacher.RunEpoch(input, output);
for (int i = 0; i < input.Length; i++)
{
double actual = network.Compute(input[i])[0];
double expected = output[i][0];
Assert.AreEqual(expected, actual, 0.01);
Assert.IsFalse(Double.IsNaN(actual));
}
}
public DiscreteNeuralNetworkByChord(List<NGram<Chord>[]> bad, List<NGram<Chord>[]> okay, List<NGram<Chord>[]> good, IActivationFunction function)
{
bad.NullCheck();
okay.NullCheck();
good.NullCheck();
bad.Any().AssertTrue();
okay.Any().AssertTrue();
good.Any().AssertTrue();
List<Tuple<double[], double[]>> input = new List<Tuple<double[], double[]>>(bad.Count + okay.Count + good.Count);
input.AddRange(
bad.Select(x => new Tuple<double[], double[]>(
x.SelectMany(y => y.SelectMany(p => ConvertChordIntoTrainingInput(p))).ToArray(),
Enumerable.Repeat<double>(DiscreteNeuralNetworkByChord.BADWEIGHT, bad.Count).ToArray())));
input.AddRange(
okay.Select(x => new Tuple<double[], double[]>(
x.SelectMany(y => y.SelectMany(p => ConvertChordIntoTrainingInput(p))).ToArray(),
Enumerable.Repeat<double>(OkayWeight, okay.Count).ToArray())));
input.AddRange(
good.Select(x => new Tuple<double[], double[]>(
x.SelectMany(y => y.SelectMany(p => ConvertChordIntoTrainingInput(p))).ToArray(),
Enumerable.Repeat<double>(DiscreteNeuralNetworkByChord.GOODWEIGHT, good.Count).ToArray())));
this.Max = input.Max(x => x.Item1.Max());
int minIndex = input.Min(x => x.Item1.Length);
var normalized = input.Select(item => Tuple.Create(item.Item1.Take(minIndex).Select(x => x / this.Max).ToArray(), item.Item2.Take(minIndex).ToArray())).ToArray();
this.trainingData = normalized.ToArray();
this.ActivationNetwork = new ActivationNetwork(function, this.trainingData.Max(y => y.Item1.Length), (HiddenLayerSize == 0) ? 23 : HiddenLayerSize, 1);
this.LearningMethod = new ResilientBackpropagationLearning(this.ActivationNetwork);
this.ActivationNetwork.Randomize();
}
public NeuralNetworkOperations(int characterSize)
{
neuralNet = new ActivationNetwork(new BipolarSigmoidFunction(2.0f), characterSize, 400, characterCount);
neuralNet.Randomize();
teacher = new AForge.Neuro.Learning.BackPropagationLearning(neuralNet);
teacher.LearningRate = 0.5f;
teacher.Momentum = 0.1f;
prepareDataForTeacher();
//var letters = treningLetterListInput.Zip(treningLetterListOutput, (i,o) => new { treningLetterListInput = i, treningLetterListOutput = o });
double err = 400.0f;
int count = 0;
while(err >= 30.0f)
{
err = teacher.RunEpoch(treningLetterListInput.ToArray(), treningLetterListOutput.ToArray());
count++;
}
}
public string Learn(string knowFile)
{
var network = new ActivationNetwork(new BipolarSigmoidFunction(), Constants.StoneCount, 1);
var teacher = new BackPropagationLearning(network); //new PerceptronLearning(network);
var data = LoadData(knowFile);
double error = int.MaxValue;
int index = 0;
while (error > 1.0 && index++ < 5000) {
error = teacher.RunEpoch(data.Item1, data.Item2);
}
var networkFile = knowFile + ".net";
network.Save(networkFile);
Console.WriteLine("Learn: {0}, Gen: {1}", knowFile, networkFile);
return networkFile;
}
public void Test()
{
var network = new ActivationNetwork(new SigmoidFunction(), inputCount, firstLayerNeurons, secondLayerNeurons, thirdLayerNeurons, lastLayerNeurons);
var teacher = new BackPropagationLearning(network);
var lastError = double.MaxValue;
var counter = 0;
while (true)
{
counter++;
var error = teacher.RunEpoch(input, output);
if ((lastError - error < 0.00001 && error < 0.01) || counter > 1200000)
break;
lastError = error;
}
var result1 = network.Compute(new double[] {1, 0, 1, 0, 1, 0, 1, 0});
Console.WriteLine($"2 + 2, 2 * 2 = {result1[0]}, {result1[1]}");
var result2 = network.Compute(new double[] {0, 1, 0, 1, 1, 0, 0, 1});
Console.WriteLine($"1 + 1, 2 * 1 = {result2[0]}, {result2[1]}");
var result3 = network.Compute(new double[] {1, 0, 1, 0, 0, 1, 0, 0});
Console.WriteLine($"2 + 2, 1 * 0 = {result3[0]}, {result3[1]}");
var result4 = network.Compute(new double[] {0, 1, 0, 0, 0, 1, 1, 0});
Console.WriteLine($"1 + 0, 1 * 2 = {result4[0]}, {result4[1]}");
}
static void NeuralNetworkAccompanimentTest()
{
// initialize input and output values
double[][] input = new double[4][] {
new double[] {0, 0}, new double[] {0, 1},
new double[] {1, 0}, new double[] {1, 1}
};
double[][] output = new double[4][] {
new double[] {0}, new double[] {1},
new double[] {1}, new double[] {0}
};
SigmoidFunction sig = new SigmoidFunction();
Accord.Neuro.Networks.RestrictedBoltzmannMachine boltz = new Accord.Neuro.Networks.RestrictedBoltzmannMachine(200, 200);
// create neural network
ActivationNetwork network = new ActivationNetwork(
new SigmoidFunction(2),
200,
20,
200);
//BackPropagationLearning teacher = new BackPropagationLearning(network);
//LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(network);
Accord.Neuro.Learning.ParallelResilientBackpropagationLearning teacher = new ParallelResilientBackpropagationLearning(network);
Accord.Neuro.Networks.DeepBeliefNetwork dpn = new Accord.Neuro.Networks.DeepBeliefNetwork(200, 20);
// teacher.IncreaseFactor = 1.01;
Composition c = Composition.LoadFromMIDI("test/other/ff7tifa.mid");
MusicPlayer player = new MusicPlayer();
//player.Play(c.Tracks[0]);
List<double[]> inputs = new List<double[]>(); List<double[]> outputs = new List<double[]>();
inputs.Add(GetDoublesFromNotes((c.Tracks[0].GetMainSequence() as MelodySequence).ToArray()));
outputs.Add(GetDoublesFromNotes((c.Tracks[1].GetMainSequence() as MelodySequence).ToArray()));
// inputs.Add(GetDoublesFromNotes((c.Tracks[1].GetMainSequence() as MelodySequence).ToArray()));
// outputs.Add(GetDoublesFromNotes((c.Tracks[2].GetMainSequence() as MelodySequence).ToArray()));
// inputs.Add(GetDoublesFromNotes((c.Tracks[0].GetMainSequence() as MelodySequence).ToArray()));
// outputs.Add(GetDoublesFromNotes((c.Tracks[3].GetMainSequence() as MelodySequence).ToArray()));
int its = 0;
while (its++ < 10000)
{
double error = teacher.RunEpoch(inputs.ToArray(), outputs.ToArray());
Console.WriteLine("{0}: Error - {1}", its, error);
}
var input_melody = (c.Tracks[0].GetMainSequence() as MelodySequence);
var new_notes = network.Compute(GetDoublesFromNotes(input_melody.ToArray()));
var new_mel = GetMelodyFromDoubles(new_notes);
player.Play(new_mel);
Console.ReadLine();
}
/// <summary>
/// Mencari solusi model neural network
/// </summary>
private void searchSolution()
{
// Normalize Data
switch (this.selectedActivationFunction)
{
case ActivationFunctionEnumeration.SemiLinearFunction:
this.activationFunction = new SemiLinearFunction();
this.normalizeData(0.1, 0.9);
break;
case ActivationFunctionEnumeration.SigmoidFunction:
this.activationFunction = new SigmoidFunction();
this.normalizeData(0.1, 0.9);
break;
case ActivationFunctionEnumeration.BipolarSigmoidFunction:
this.activationFunction = new BipolarSigmoidFunction();
this.normalizeData(-0.9, 0.9);
break;
case ActivationFunctionEnumeration.HyperbolicTangentFunction:
this.activationFunction = new HyperbolicTangentFunction();
this.normalizeData(-0.9, 0.9);
break;
default:
this.activationFunction = new BipolarSigmoidFunction();
this.normalizeData(-0.9, 0.9);
break;
}
//create network
this.network = new ActivationNetwork
(this.activationFunction, this.inputLayerNeurons, this.hiddenLayerNeurons, this.outputLayerNeurons);
this.network.Randomize();
this.learning = new BackPropagationLearning(this.network);
this.learning.LearningRate = this.learningRate;
this.learning.Momentum = this.momentum;
//variable for looping
//needToStop = false;
double mse = 0.0, error = 0.0, mae=0.0;
int iteration = 1;
double msle = 0.0, mspe = 0.0, generalizationLoss = 0.0, pq = 0.0;
double[] trainingErrors = new double[this.strip];
for (int i = 0; i < this.strip; i++) trainingErrors[i] = double.MaxValue / strip;
double lastMSE = double.MaxValue;
int n = this.data.Length - this.network.InputsCount;
int validationSet = (int)Math.Round(this.validationSetRatio * n);
int trainingSet = n - validationSet;
double[][] networkTrainingInput = new double[trainingSet][];
double[][] networkTrainingOutput = new double[trainingSet][];
for (int i = 0; i < trainingSet; i++)
{
networkTrainingInput[i] = new double[this.network.InputsCount];
networkTrainingOutput[i] = new double[1];
}
for (int i = 0; i < trainingSet; i++)
{
for (int j = 0; j < this.network.InputsCount; j++)
{
networkTrainingInput[i][j] = this.networkInput[i][j];
}
networkTrainingOutput[i][0] = this.networkOutput[i][0];
}
double[] solutionValidation = new double[validationSet];
double[] inputForValidation = new double[this.network.InputsCount];
double[] inputForValidationNetwork = new double[this.network.InputsCount];
this.bestValidationError = double.MaxValue;
this.bestNetworkWeight = new double[this.network.LayersCount][][];
this.bestNetworkBias = new double[this.network.LayersCount][];
this.bestSolution = new double[n];
for (int i = 0; i < this.network.LayersCount; i++)
{
this.bestNetworkWeight[i] = new double[this.network[i].NeuronsCount][];
this.bestNetworkBias[i] = new double[this.network[i].NeuronsCount];
for (int j = 0; j < this.network[i].NeuronsCount; j++)
{
this.bestNetworkWeight[i][j] = new double[this.network[i][j].InputsCount];
}
}
//best network criterion
double bestNetworkError = double.MaxValue, bestNetworkMSE = double.MaxValue, bestNetworkMAE = double.MaxValue;
//training
while (!needToStop)
{
double sse = 0.0;
double sae = 0.0;
double ssle = 0.0;
double sspe = 0.0;
if (this.useAdvanceEarlyStopping)
error = this.learning.RunEpoch(networkTrainingInput, networkTrainingOutput);
else
error = this.learning.RunEpoch(this.networkInput, this.networkOutput);
this.solutionData = new double[n];
this.solutionToShow = new double[n, 2];
if (this.useAdvanceEarlyStopping)
{
//validation
//for (int j = 0; j < this.network.InputsCount; j++)
//{
// inputForValidation[this.network.InputsCount - 1 - j] = this.data[this.data.Length - validationSet - 1 - j];
//}
}
else
{
seriesChart.UpdateDataSeries("Validation", null);
}
this.validationSolutionToShow = new double[validationSet, 2];
for (int i = 0; i < n; i++)
{
this.solutionData[i] = (this.network.Compute(this.networkInput[i])[0]
- this.minNormalizedData) / this.factor + this.minData;
this.solutionToShow[i, 0] = i + this.network.InputsCount;
this.solutionToShow[i,1] = this.solutionData[i];
sse += Math.Pow(this.solutionData[i] - this.data[i + this.network.InputsCount], 2);
sae += Math.Abs(this.solutionData[i] - this.data[i + this.network.InputsCount]);
//calculate advance early stopping
if (this.useAdvanceEarlyStopping)
{
if (i < n - validationSet)
{
ssle += Math.Pow(this.solutionData[i] - this.data[i + this.network.InputsCount], 2);
}
else
{
if (i == n - validationSet)
{
for (int j = 0; j < this.network.InputsCount; j++)
{
inputForValidation[this.network.InputsCount - 1 - j] = this.data[this.data.Length - (n - i) - 1 - j];
}
}
for (int j = 0; j < this.network.InputsCount; j++)
{
inputForValidationNetwork[j] = (inputForValidation[j] - this.minData) * this.factor + this.minNormalizedData;
}
solutionValidation[i - n + validationSet] = (this.network.Compute(inputForValidationNetwork)[0] - this.minNormalizedData) / this.factor + this.minData;
this.validationSolutionToShow[i - n + validationSet, 0] = i + this.network.InputsCount;
this.validationSolutionToShow[i - n + validationSet, 1] = solutionValidation[i - n + validationSet];
sspe += Math.Pow(this.data[i + this.network.InputsCount] - solutionValidation[i - n + validationSet], 2);
for (int j = 0; j < this.network.InputsCount - 1; j++)
{
inputForValidation[j] = inputForValidation[j + 1];
}
inputForValidation[this.network.InputsCount - 1] = solutionValidation[i - n + validationSet];
}
}
}
mse = sse / this.solutionData.Length;
mae = sae / this.solutionData.Length;
//Display it
this.iterationBox.Text = iteration.ToString();
this.maeBox.Text = mae.ToString("F5");
this.mseBox.Text = mse.ToString("F5");
this.errorBox.Text = error.ToString("F5");
if (this.previewGrapicRadio.Checked)
{
seriesChart.UpdateDataSeries("Predicted", this.solutionToShow);
}
else if (this.previewTextRadio.Checked)
{
if (!this.useAdvanceEarlyStopping)
{
txtProgress.AppendText("Iteration: " + iteration.ToString() + "\tError: " + error.ToString("F5") + "\tMAE: " + mae.ToString("F5") + "\tMSE: " + mse.ToString("F5") + "\n");
txtProgress.ScrollToCaret();
}
}
if (this.useAdvanceEarlyStopping)
{
//calculate advance early stopping 2
mspe = sspe / validationSet;
msle = ssle / (this.solutionData.Length - validationSet);
//save best weight
if (this.bestValidationError > mspe)
{
this.bestValidationError = mspe;
this.bestSolution = this.solutionData;
for (int i = 0; i < this.network.LayersCount; i++)
for (int j = 0; j < this.network[i].NeuronsCount; j++)
for (int k = 0; k < this.network[i][j].InputsCount; k++)
this.bestNetworkWeight[i][j][k] = this.network[i][j][k];
for (int i = 0; i < this.network.LayersCount; i++)
for (int j = 0; j < this.network[i].NeuronsCount; j++)
this.bestNetworkBias[i][j] = this.network[i][j].Threshold;
bestNetworkError = error;
bestNetworkMAE = mae;
bestNetworkMSE = mse;
}
//calculate generalization loss &pq
generalizationLoss = 100 * (mspe / this.bestValidationError - 1);
trainingErrors[(iteration - 1) % this.strip] = msle;
double minStripTrainingError = double.MaxValue, sumStripTrainingError = 0.0;
for (int i = 0; i < this.strip; i++)
{
sumStripTrainingError += trainingErrors[i];
if (trainingErrors[i] < minStripTrainingError) minStripTrainingError = trainingErrors[i];
}
double trainingProgress = 1000 * ((sumStripTrainingError / (this.strip * minStripTrainingError)) - 1);
pq = generalizationLoss / trainingProgress;
//display advance early stopping
this.learningErrorBox.Text = msle.ToString("F5");
this.validationErrorBox.Text = mspe.ToString("F5");
this.generalizationLossBox.Text = generalizationLoss.ToString("F5");
this.pqBox.Text = pq.ToString("F5");
seriesChart.UpdateDataSeries("Validation", this.validationSolutionToShow);
if (this.previewTextRadio.Checked)
{
txtProgress.AppendText("Iteration: " + iteration.ToString() + "\tError: " + error.ToString("F5") + "\tMAE: " + mae.ToString("F5") + "\tMSE: " + mse.ToString("F5") + "\tLearning Error: " + msle.ToString("F5") + "\tValidation Error: " + mspe.ToString("F5") + "\tGeneralization Loss: " + generalizationLoss.ToString("F5") + "\tPQ: " + pq.ToString("F5") + "\n");
txtProgress.ScrollToCaret();
}
//stopping
switch (this.advanceStoppingMethod)
{
case AdvanceStoppingMethodEnumeration.GeneralizationLoss:
if (generalizationLoss > this.generalizationLossTreshold) needToStop = true;
break;
case AdvanceStoppingMethodEnumeration.ProgressQuotient:
if (pq > this.pqTreshold) needToStop = true;
break;
}
}
if (this.withCheckingCycle && iteration % this.checkingCycle == 0)
{
switch (this.checkingMethod)
{
case CheckingMethodEnumeration.byMSEValue:
if (mse <= this.byMSEValueStopping) needToStop = true;
break;
case CheckingMethodEnumeration.byMSEChange:
if (lastMSE - mse <= this.byMSEChangeStopping) needToStop = true;
break;
}
lastMSE = mse;
}
if (iteration >= this.maxIteration)
{
needToStop = true;
}
iteration++;
}
//restore weight
if (this.useAdvanceEarlyStopping)
{
this.solutionData = this.bestSolution;
for (int i = 0; i < this.network.LayersCount; i++)
for (int j = 0; j < this.network[i].NeuronsCount; j++)
for (int k = 0; k < this.network[i][j].InputsCount; k++)
this.network[i][j][k] = this.bestNetworkWeight[i][j][k];
for (int i = 0; i < this.network.LayersCount; i++)
for (int j = 0; j < this.network[i].NeuronsCount; j++)
this.network[i][j].Threshold = this.bestNetworkBias[i][j];
//best network criterion
this.error = bestNetworkError;
this.mse = bestNetworkMSE;
this.mae = bestNetworkMAE;
}
else
{
this.error = error;
this.mse = mse;
this.mae = mae;
}
this.enableControls(true);
}
// Worker thread
void SearchSolution()
{
// number of learning samples
int samples = data.Length - predictionSize - windowSize;
// data transformation factor
double factor = 1.7 / chart.RangeY.Length;
double yMin = chart.RangeY.Min;
// prepare learning data
double[][] input = new double[samples][];
double[][] output = new double[samples][];
for (int i = 0; i < samples; i++)
{
input[i] = new double[windowSize];
output[i] = new double[1];
// set input
for (int j = 0; j < windowSize; j++)
{
input[i][j] = (data[i + j] - yMin) * factor - 0.85;
}
// set output
output[i][0] = (data[i + windowSize] - yMin) * factor - 0.85;
}
// create multi-layer neural network
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(sigmoidAlphaValue),
windowSize, windowSize * 2, 1);
// create teacher
var teacher = new ParallelResilientBackpropagationLearning(network);
teacher.Reset(initialStep);
// run at least one backpropagation epoch
//teacher2.RunEpoch(input, output);
// iterations
int iteration = 1;
// solution array
int solutionSize = data.Length - windowSize;
double[,] solution = new double[solutionSize, 2];
double[] networkInput = new double[windowSize];
// calculate X values to be used with solution function
for (int j = 0; j < solutionSize; j++)
{
solution[j, 0] = j + windowSize;
}
// loop
while (!needToStop)
{
// run epoch of learning procedure
double error = teacher.RunEpoch(input, output) / samples;
// calculate solution and learning and prediction errors
double learningError = 0.0;
double predictionError = 0.0;
// go through all the data
for (int i = 0, n = data.Length - windowSize; i < n; i++)
{
// put values from current window as network's input
for (int j = 0; j < windowSize; j++)
{
networkInput[j] = (data[i + j] - yMin) * factor - 0.85;
}
// evalue the function
solution[i, 1] = (network.Compute(networkInput)[0] + 0.85) / factor + yMin;
// calculate prediction error
if (i >= n - predictionSize)
{
predictionError += Math.Abs(solution[i, 1] - data[windowSize + i]);
}
else
{
learningError += Math.Abs(solution[i, 1] - data[windowSize + i]);
}
}
// update solution on the chart
chart.UpdateDataSeries("solution", solution);
// set current iteration's info
SetText(currentIterationBox, iteration.ToString());
SetText(currentLearningErrorBox, learningError.ToString("F3"));
SetText(currentPredictionErrorBox, predictionError.ToString("F3"));
// increase current iteration
iteration++;
// check if we need to stop
if ((iterations != 0) && (iteration > iterations))
break;
}
// show new solution
for (int j = windowSize, k = 0, n = data.Length; j < n; j++, k++)
{
AddSubItem(dataList, j, solution[k, 1].ToString());
}
// enable settings controls
EnableControls(true);
}
// Worker thread
void SearchSolution()
{
// number of learning samples
int samples = data.GetLength(0);
// data transformation factor
double yFactor = 1.7 / chart.RangeY.Length;
double yMin = chart.RangeY.Min;
double xFactor = 2.0 / chart.RangeX.Length;
double xMin = chart.RangeX.Min;
// prepare learning data
double[][] input = new double[samples][];
double[][] output = new double[samples][];
for (int i = 0; i < samples; i++)
{
input[i] = new double[1];
output[i] = new double[1];
// set input
input[i][0] = (data[i, 0] - xMin) * xFactor - 1.0;
// set output
output[i][0] = (data[i, 1] - yMin) * yFactor - 0.85;
}
// create multi-layer neural network
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(sigmoidAlphaValue),
1, neuronsInFirstLayer, 1);
// create teacher
BackPropagationLearning teacher = new BackPropagationLearning(network);
// set learning rate and momentum
teacher.LearningRate = learningRate;
teacher.Momentum = momentum;
// iterations
int iteration = 1;
// solution array
double[,] solution = new double[50, 2];
double[] networkInput = new double[1];
// calculate X values to be used with solution function
for (int j = 0; j < 50; j++)
{
solution[j, 0] = chart.RangeX.Min + (double)j * chart.RangeX.Length / 49;
}
// loop
while (!needToStop)
{
// run epoch of learning procedure
double error = teacher.RunEpoch(input, output) / samples;
// calculate solution
for (int j = 0; j < 50; j++)
{
networkInput[0] = (solution[j, 0] - xMin) * xFactor - 1.0;
solution[j, 1] = (network.Compute(networkInput)[0] + 0.85) / yFactor + yMin;
}
chart.UpdateDataSeries("solution", solution);
// calculate error
double learningError = 0.0;
for (int j = 0, k = data.GetLength(0); j < k; j++)
{
networkInput[0] = input[j][0];
learningError += Math.Abs(data[j, 1] - ((network.Compute(networkInput)[0] + 0.85) / yFactor + yMin));
}
// set current iteration's info
UpdateTextbox(currentIterationBox, iteration.ToString());
//currentIterationBox.Text = iteration.ToString();
UpdateTextbox(currentErrorBox, learningError.ToString("F3"));
//currentErrorBox.Text = learningError.ToString("F3");
// increase current iteration
iteration++;
// check if we need to stop
if ((iterations != 0) && (iteration > iterations))
break;
}
// enable settings controls
EnableControls(true);
}
// Worker thread
void SearchSolution()
{
// initialize input and output values
double[][] input = null;
double[][] output = null;
if (sigmoidType == 0)
{
// unipolar data
input = new double[4][] {
new double[] {0, 0},
new double[] {0, 1},
new double[] {1, 0},
new double[] {1, 1}
};
output = new double[4][] {
new double[] {0},
new double[] {1},
new double[] {1},
new double[] {0}
};
}
else
{
// biipolar data
input = new double[4][] {
new double[] {-1, -1},
new double[] {-1, 1},
new double[] { 1, -1},
new double[] { 1, 1}
};
output = new double[4][] {
new double[] {-1},
new double[] { 1},
new double[] { 1},
new double[] {-1}
};
}
// create perceptron
ActivationNetwork network = new ActivationNetwork(
(sigmoidType == 0) ?
(IActivationFunction)new SigmoidFunction(sigmoidAlphaValue) :
(IActivationFunction)new BipolarSigmoidFunction(sigmoidAlphaValue),
2, 2, 1);
// create teacher
BackPropagationLearning teacher = new BackPropagationLearning(network);
// set learning rate and momentum
teacher.LearningRate = learningRate;
teacher.Momentum = momentum;
// iterations
int iteration = 1;
// statistic files
StreamWriter errorsFile = null;
try
{
// check if we need to save statistics to files
if (saveStatisticsToFiles)
{
// open files
errorsFile = File.CreateText("errors.csv");
}
// erros list
List<double> errorsList = new List<double>();
// loop
while (!needToStop)
{
// run epoch of learning procedure
double error = teacher.RunEpoch(input, output);
errorsList.Add(error);
// save current error
if (errorsFile != null)
{
errorsFile.WriteLine(error);
}
// show current iteration & error
UpdateTextbox(currentIterationBox, iteration.ToString());
//currentIterationBox.Text = iteration.ToString();
UpdateTextbox(currentErrorBox, error.ToString());
//currentErrorBox.Text = error.ToString();
iteration++;
// check if we need to stop
if (error <= learningErrorLimit)
break;
}
// show error's dynamics
double[,] errors = new double[errorsList.Count, 2];
for (int i = 0, n = errorsList.Count; i < n; i++)
{
errors[i, 0] = i;
errors[i, 1] = errorsList[i];
}
errorChart.RangeX = new DoubleRange(0, errorsList.Count - 1);
errorChart.UpdateDataSeries("error", errors);
}
catch (IOException)
{
MessageBox.Show("Failed writing file", "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
finally
{
// close files
if (errorsFile != null)
errorsFile.Close();
}
// enable settings controls
EnableControls(true);
}
/// <summary>
/// Gets the number of parameters in a network.
/// </summary>
private static int getNumberOfParameters(ActivationNetwork network)
{
int sum = 0;
for (int i = 0; i < network.Layers.Length; i++)
{
for (int j = 0; j < network.Layers[i].Neurons.Length; j++)
{
// number of weights plus the bias value
sum += network.Layers[i].Neurons[j].InputsCount + 1;
}
}
return sum;
}
/// <summary>
/// Initializes a new instance of the <see cref="LevenbergMarquardtLearning"/> class.
/// </summary>
///
/// <param name="network">Network to teach.</param>
/// <param name="useRegularization">True to use bayesian regularization, false otherwise.</param>
/// <param name="method">The method by which the Jacobian matrix will be calculated.</param>
///
public LevenbergMarquardtLearning(ActivationNetwork network, bool useRegularization, JacobianMethod method)
{
this.network = network;
this.numberOfParameters = getNumberOfParameters(network);
this.outputCount = network.Layers[network.Layers.Length - 1].Neurons.Length;
this.useBayesianRegularization = useRegularization;
this.method = method;
this.weights = new float[numberOfParameters];
this.hessian = new float[numberOfParameters][];
for (int i = 0; i < hessian.Length; i++)
hessian[i] = new float[numberOfParameters];
this.diagonal = new float[numberOfParameters];
this.gradient = new float[numberOfParameters];
this.jacobian = new float[numberOfParameters][];
// Will use backpropagation method for Jacobian computation
if (method == JacobianMethod.ByBackpropagation)
{
// create weight derivatives arrays
this.weightDerivatives = new float[network.Layers.Length][][];
this.thresholdsDerivatives = new float[network.Layers.Length][];
// initialize arrays
for (int i = 0; i < network.Layers.Length; i++)
{
ActivationLayer layer = (ActivationLayer)network.Layers[i];
this.weightDerivatives[i] = new float[layer.Neurons.Length][];
this.thresholdsDerivatives[i] = new float[layer.Neurons.Length];
for (int j = 0; j < layer.Neurons.Length; j++)
this.weightDerivatives[i][j] = new float[layer.InputsCount];
}
}
else // Will use finite difference method for Jacobian computation
{
// create differential coefficient arrays
this.differentialCoefficients = createCoefficients(3);
this.derivativeStepSize = new double[numberOfParameters];
// initialize arrays
for (int i = 0; i < numberOfParameters; i++)
this.derivativeStepSize[i] = derivativeStep;
}
}
