public void RunEpochTest1()
{
Accord.Math.Tools.SetupGenerator(0);
double[][] input =
{
new double[] { -1, -1 },
new double[] { -1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
double[][] output =
{
new double[] { -1 },
new double[] { 1 },
new double[] { 1 },
new double[] { -1 }
};
Neuron.RandGenerator = new ThreadSafeRandom(0);
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(2), 2, 2, 1);
var teacher = new LevenbergMarquardtLearning(network,
false, JacobianMethod.ByFiniteDifferences);
double error = 1.0;
while (error > 1e-5)
error = teacher.RunEpoch(input, output);
for (int i = 0; i < input.Length; i++)
Assert.AreEqual(network.Compute(input[i])[0], output[i][0], 0.1);
}
/// <summary>
/// Initializes a new instance of the <see cref="BackPropagationLearning"/> class
/// </summary>
///
/// <param name="network">Network to teach</param>
///
public BackPropagationLearning( ActivationNetwork network )
{
this.network = network;
// create error and deltas arrays
neuronErrors = new double[network.LayersCount][];
weightsUpdates = new double[network.LayersCount][][];
thresholdsUpdates = new double[network.LayersCount][];
// initialize errors and deltas arrays for each layer
for ( int i = 0, n = network.LayersCount; i < n; i++ )
{
Layer layer = network[i];
neuronErrors[i] = new double[layer.NeuronsCount];
weightsUpdates[i] = new double[layer.NeuronsCount][];
thresholdsUpdates[i] = new double[layer.NeuronsCount];
// for each neuron
for ( int j = 0; j < layer.NeuronsCount; j++ )
{
weightsUpdates[i][j] = new double[layer.InputsCount];
}
}
}
public void MulticlassTest1()
{
Accord.Math.Tools.SetupGenerator(0);
Neuron.RandGenerator = new ThreadSafeRandom(0);
int numberOfInputs = 3;
int numberOfClasses = 4;
int hiddenNeurons = 5;
double[][] input =
{
new double[] { -1, -1, -1 }, // 0
new double[] { -1, 1, -1 }, // 1
new double[] { 1, -1, -1 }, // 1
new double[] { 1, 1, -1 }, // 0
new double[] { -1, -1, 1 }, // 2
new double[] { -1, 1, 1 }, // 3
new double[] { 1, -1, 1 }, // 3
new double[] { 1, 1, 1 } // 2
};
int[] labels =
{
0,
1,
1,
0,
2,
3,
3,
2,
};
double[][] outputs = Accord.Statistics.Tools
.Expand(labels, numberOfClasses, -1, 1);
var function = new BipolarSigmoidFunction(2);
var network = new ActivationNetwork(function,
numberOfInputs, hiddenNeurons, numberOfClasses);
new NguyenWidrow(network).Randomize();
var teacher = new LevenbergMarquardtLearning(network);
double error = Double.PositiveInfinity;
for (int i = 0; i < 10; i++)
error = teacher.RunEpoch(input, outputs);
for (int i = 0; i < input.Length; i++)
{
int answer;
double[] output = network.Compute(input[i]);
double response = output.Max(out answer);
int expected = labels[i];
Assert.AreEqual(expected, answer);
}
}
/// <summary>
/// Constructs a new Gaussian Weight initialization.
/// </summary>
///
/// <param name="network">The activation network whose weights will be initialized.</param>
/// <param name="stdDev">The standard deviation to be used. Common values lie in the 0.001-
/// 0.1 range. Default is 0.1.</param>
///
public GaussianWeights(ActivationNetwork network, double stdDev = 0.1)
{
this.network = network;
this.random = new GaussianGenerator(0f, (float)stdDev, Accord.Math.Tools.Random.Next());
this.UpdateThresholds = false;
}
/// <summary>
/// Generates points trained points from learning.
/// </summary>
private async Task <double[][]> ComputeTrainedPoints(double[][] correctPoints)
{
return(await Task.Run(() =>
{
lock (_threadLocker)
{
try
{
_network = (ActivationNetwork)Network.Load(ConfigurationPath());
}
catch (Exception ex)
{
throw ex;
}
Scaler scaler = new Scaler(correctPoints);
var inputScaledData = correctPoints.Select(a => a.ToArray()).ToArray();
scaler.Scale(ref inputScaledData);
var solution = inputScaledData.Select(a => a.ToArray()).ToArray();
var networkInput = new double[2 * _inputPoints];
for (int j = _inputPoints; j < solution.GetLength(0); j++)
{
for (int k = _inputPoints, l = 0; k > 0; k--, l = l + 2)
{
networkInput[l] = solution[j - k][0];
networkInput[l + 1] = solution[j - k][1];
}
try
{
var result = _network.Compute(networkInput);
solution[j][0] = result[0];
solution[j][1] = result[1];
}
catch
{
solution[j][0] = double.NaN;
solution[j][1] = double.NaN;
}
}
for (int i = 0; i < solution.GetLength(0); i++)
{
if (double.IsNaN(solution[i][0]) || double.IsNaN(solution[i][1]))
{
continue;
}
solution[i][0] = scaler.Rescale(solution[i][0], XYEnum.X);
solution[i][1] = scaler.Rescale(solution[i][1], XYEnum.Y);
}
_network.Save(ConfigurationPath());
_network = null;
return solution;
}
}));
}
public void JacobianByChainRuleTest3()
{
// Network with 3 hidden layers: 2-4-3-4-1
Accord.Math.Tools.SetupGenerator(0);
double[][] input =
{
new double[] { -1, -1 },
new double[] { -1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
double[][] output =
{
new double[] { -1 },
new double[] { 1 },
new double[] { 1 },
new double[] { -1 }
};
Neuron.RandGenerator = new ThreadSafeRandom(0);
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(2), 2, 4, 3, 4, 1);
var teacher1 = new LevenbergMarquardtLearning(network,
false, JacobianMethod.ByFiniteDifferences);
var teacher2 = new LevenbergMarquardtLearning(network,
false, JacobianMethod.ByBackpropagation);
// Set lambda to lambda max so no iterations are performed
teacher1.LearningRate = 1e30f;
teacher2.LearningRate = 1e30f;
teacher1.RunEpoch(input, output);
teacher2.RunEpoch(input, output);
var jacobian1 = teacher1.Jacobian;
var jacobian2 = teacher2.Jacobian;
for (int i = 0; i < jacobian1.Length; i++)
{
for (int j = 0; j < jacobian1[i].Length; j++)
{
double j1 = jacobian1[i][j];
double j2 = jacobian2[i][j];
Assert.AreEqual(j1, j2, 1e-4);
Assert.IsFalse(Double.IsNaN(j1));
Assert.IsFalse(Double.IsNaN(j2));
}
}
}
static void Main(string[] args)
{
string fn = @"C:\Users\semio\Desktop\nefile.csv";
var f = File.ReadLines(fn);
var data = from z in f.Skip(1)
let zz = z.Split(';')
select new Digit
{
Label = int.Parse(zz.Last()),
Width = int.Parse(zz[1]),
Height = int.Parse(zz[2]),
Image = zz.FirstOrDefault().Split(' ').Select(double.Parse).ToArray()
};
Digit[] train = data.Take(9).ToArray();
Digit[] test = data.Skip(9).ToArray();
double prErr = 10000000;
double error = 100;
var network = new ActivationNetwork(new SigmoidFunction(), 50 * 50, 100, 100, 2);
var learner = new BackPropagationLearning(network)
{
LearningRate = 0.005
};
Console.WriteLine("Start Learning");
new GaussianWeights(network, 0.1).Randomize();
double err = 1;
int i = 1;
while (err > 0.004)
{
err = learner.RunEpoch((from x in train select x.Image.Select(t => (double)t).ToArray()).ToArray(),
(from x in train select ToOneHotEnc(x.Label, 2)).ToArray());
Console.WriteLine($"EpochNum = {i} --- Error={err}");
i++;
if (Math.Abs(error - prErr) < 0.0001)
{
// Уменьшаем коэффициент скорости обучения на 2
//learner.LearningRate /= 2;
//if (learner.LearningRate < 0.001)
// learner.LearningRate = 0.001;
}
prErr = error;
}
foreach (Digit v in test)
{
var n = network.Compute(v.Image);
//ImageBox.Show(v.Image.Select(x => (double)x / 2).ToArray(), 50, 50);
var z = getMaxInex(n);
Console.WriteLine("{0} => {1}; 2 - {2}; 3 - {3}", v.Label, z + 2, n[0], n[1]);
}
Console.ReadKey();
}
public NeuralNetwork(double[][] inputData, double[][] outputData)
{
_inputData = inputData;
_outputData = outputData;
Network = new ActivationNetwork(new SigmoidFunction(),
inputData[0].Length, // Input neuron count
outputData[0].Length); // Output neuron count
Network.Randomize();
}
public AttackStateNN()
{
var activationFunction = new Accord.Neuro.ActivationFunctions.GaussianFunction();
network = new ActivationNetwork(new SigmoidFunction(0.01), 4 + 4 + 4 * 5 * 5 + 25, new[] { 100, 100, 1 });
new GaussianWeights(network, 0.3).Randomize();
teacher = new ResilientBackpropagationLearning(network);
teacher.LearningRate = 0.1;
}
public static ActivationNetwork LoadNetwork(string file)
{
IFormatter formatter = new BinaryFormatter();
Stream stream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read);
ActivationNetwork network = (ActivationNetwork)formatter.Deserialize(stream);
stream.Close();
return(network);
}
/// <summary>
/// Реиницализация сети - создаём заново сеть с указанной структурой
/// </summary>
/// <param name="structure">Массив с указанием нейронов на каждом слое (включая сенсорный)</param>
/// <param name="initialLearningRate">Начальная скорость обучения</param>
public override void ReInit(int[] structure, double initialLearningRate = 0.25)
{
// Создаём сеть - вроде того
network = new ActivationNetwork((IActivationFunction) new SigmoidFunction(2.0), structure[0],
structure.Skip(1).ToArray <int>());
// Встряска "мозгов" сети - рандомизируем веса связей
new NguyenWidrow(network).Randomize();
}
public void CreateNetwork()
{
// create neural network
network = new ActivationNetwork(
new BipolarSigmoidFunction(),
1, // inputs in the network
10, // neurons in the first (hidden) layer
1); // neuron in the second (output) layer
}
public void Accept(INet sourceNet)
{
_network = Network.Load(sourceNet.Persistence()) as ActivationNetwork;
_teacher = new BackPropagationLearning(_network)
{
LearningRate = _learningRate,
Momentum = 0.9
};
}
private void button2_Click_1(object sender, EventArgs e)
{
button2.Enabled = false;
network2 = null;
reprop = null;
network2 = new ActivationNetwork(new BipolarSigmoidFunction(param1), 100, 100, 90, 80, 70, 60, 50, 40, 30, 20, 10, 1);
reprop = new ResilientBackpropagationLearning(network3);
network2.Randomize();
}
/// <summary>
/// Load a network
/// </summary>
/// <param name="FilePath">The path to the binary network file</param>
/// <returns></returns>
private void LoadNet(string FilePath)
{
FileStream fs = new FileStream(FilePath, FileMode.Open);
BinaryFormatter formatter = new BinaryFormatter();
ActivationNetwork net = (ActivationNetwork)formatter.Deserialize(fs);
fs.Close();
network = net;
}
private void button3_Click_1(object sender, EventArgs e)
{
button3.Enabled = false;
network1 = null;
teacher = null;
network1 = new ActivationNetwork(new BipolarSigmoidFunction(param2), 100, 100, 90, 80, 70, 60, 50, 40, 30, 20, 10, 1);
teacher = new BackPropagationLearning(network1);
network1.Randomize();
}
private void button4_Click_1(object sender, EventArgs e)
{
button4.Enabled = false;
network3 = null;
evteacher = null;
network3 = new ActivationNetwork(new BipolarSigmoidFunction(param3), 100, 100, 90, 80, 70, 60, 50, 40, 30, 20, 10, 1);
evteacher = new EvolutionaryLearning(network2, 100);
network3.Randomize();
}
internal override void Think(EvoEngine engine)
{
if (Network == null)
{
Network = new ActivationNetwork(new BipolarSigmoidFunction(), WorldScene.NeuralNetworkShape[0], WorldScene.NeuralNetworkShape.Skip(1).ToArray());
// Nw = (I+1)*H1 +(H1+1)*H2 +(H2+1)*O
// I = inputs
// H1 = neurons in hidden layer 1
// H2 = neurons in hidden layer 2
// O = Number of outputs
// Nw = (5+1)*0 + (0+1)*0 + (5+1)*4 = 24 // 5in 4out
// Nw = (5+1)*4 + (4+1)*0 + (4+1)*4 = 44 // 5in 1hl4 4out
// Nw = (11+1)*4 + (4+1)*0 + (4+1)*4 = 68 // 11in 1hl4 4out
var dna = GetCurrentGene();
SetNetworkWeights(dna);
//float scale = (float)dna.Genes[scene.NeuralNetworkWeightsCount - 1 + 0];
//if (scale < 0.4) scale = 0.4f;
//if (scale > 0.6) scale = 0.6f;
//Snake.SetScale(scale);
//int div = scene.NeuralNetworkWeightsCount / 3;
//Color c = new Color(
// (int)Math.Max((dna.Genes[div*0] * 255), 128),
// (int)Math.Max((dna.Genes[div*1] * 255), 128),
// (int)Math.Max((dna.Genes[div*2] * 255), 128)
// );
//Snake.Color = c;
}
if (engine.KeyState.GetPressedKeys().Length > 0)
{
return;
}
double[] netout = Network.Compute(convertToDouble(Snake.GetFoodSensorsActivation()));
//if (Math.Max(netout[0], netout[1]) > 0.5)
if (netout[0] > netout[1])
{
Snake.Accelerate();
}
else
{
Snake.Brake();
}
//if (Math.Max(netout[2], netout[3]) > 0.5)
if (netout[2] > netout[3])
{
Snake.RotateLeft();
}
else
{
Snake.RotateRight();
}
}
public void ExecuteFold(int k)
{
int LengthOfInput = this.FormattedData[0].Input.Count();
int LengthOfOutput = this.FormattedData[0].Output.Count();
ActivationNetwork NeuralNetwork = new ActivationNetwork(
new SigmoidFunction(2),
LengthOfInput,
this.NumberOfHiddenLayerNeurons(LengthOfInput, LengthOfOutput),
LengthOfOutput);
NguyenWidrow weights = new NguyenWidrow(NeuralNetwork);
weights.Randomize();
ResilientBackpropagationLearning BackProp = new ResilientBackpropagationLearning(NeuralNetwork);
BackProp.LearningRate = this.LearningRate;
//BackProp.Momentum = 0.5;
List <NetIO> TrainingData = new List <NetIO>();
List <NetIO> ValidationData = new List <NetIO>();
ReadWrite.RemoveKFold(this.FormattedData, ref TrainingData, ref ValidationData, k);
// for each epoch
int epoch = 0;
int maxEpochs = int.MaxValue;
EarlyStoppingTools netError = new EarlyStoppingTools(this.patience);
do
{
++epoch;
double internalError = BackProp.RunEpoch(TrainingData.Select(l => l.Input.ToArray()).ToArray(),
TrainingData.Select(l => l.Output.ToArray()).ToArray());
this.RssError = EarlyStoppingTools.RssError(NeuralNetwork,
ValidationData.Select(l => l.Input.ToArray()).ToArray(),
ValidationData.Select(l => l.Output.ToArray()).ToArray());
//Console.WriteLine("Epochs: " + epoch);
//Console.WriteLine("Training error: " + internalError);
//Console.WriteLine("CV Error: " + this.RssError);
} while (!netError.ExceedsPatience(RssError) && epoch < maxEpochs);
Console.Write("Target: ");
ValidationData[0].Output.ForEach(i => Console.Write(i));
Console.WriteLine();
Console.WriteLine("Result: " + string.Join(",", NeuralNetwork.Compute(ValidationData[0].Input.ToArray())));
this.NumberOfEpochs = epoch;
Console.WriteLine("Epochs required: " + epoch);
Console.WriteLine("Error: " + RssError);
}
static void Main(string[] args)
{
//vybrat excel s hodnotami
//vyparsovat ho, ulozit si hodnoty
//rozbehat neuronku
//naucit neuronku
//testovat neuronku
//skusit pustit data on-the-fly
OpenFileDialog ofd = new OpenFileDialog()
{
Multiselect = false
};
string excelFileName;
if (ofd.ShowDialog() == DialogResult.OK)
{
excelFileName = ofd.FileName;
}
else
{
throw new Exception("Je nutne vybrat subor");
}
var excelManager = new ExcelParser(excelFileName);
excelManager.ParseSourceData();
double[][] input = NeuronData.GetInputMatrix();
double[][] output = NeuronData.GetOutputMatrix();
// create neural network
ActivationNetwork network = new ActivationNetwork(
new SigmoidFunction(3),
3, // two inputs in the network
3, // two neurons in the first layer
2); // one neuron in the second layer
// create teacher
BackPropagationLearning teacher =
new BackPropagationLearning(network);
bool needToStop = false;
// loop
while (!needToStop)
{
// run epoch of learning procedure
System.Threading.Thread.Sleep(1000);
double error = teacher.RunEpoch(input, output);
Console.WriteLine(error);
// check error value to see if we need to stop
// ...
}
}
private void BtnLoadData_Click(object sender, EventArgs e)
{
network = new ActivationNetwork(new SigmoidFunction(2), 1024, 256, 128, 10);
new NguyenWidrow(network).Randomize();
StringReader reader = new StringReader(File.ReadAllText("Data\\optdigits-tra.txt"));
int trainingStart = 0;
int trainingCount = 1000;
int testingStart = 1000;
int testingCount = 1000;
int c1 = 0;
int c2 = 1;
int trainingSet = 0;
int testingSet = 0;
dgvLearnSet.Rows.Clear();
dgvLearnSet.RowTemplate.Height = 40;
dgvVerifySet.Rows.Clear();
dgvVerifySet.RowTemplate.Height = 40;
while (true)
{
char[] buffer = new char[33 * 32];
int read = reader.ReadBlock(buffer, 0, buffer.Length);
string label = reader.ReadLine();
if (read < buffer.Length || label == null)
{
break;
}
if (c1 > trainingStart && c1 <= trainingStart + trainingCount)
{
Bitmap bitmap = Ultilities.Extract(new string(buffer));
double[] features = Ultilities.Extract(bitmap);
int clabel1 = Int32.Parse(label);
dgvLearnSet.Rows.Add(c1, bitmap, clabel1, features);
trainingSet++;
}
else if (c1 > testingStart && c1 <= testingStart + testingCount)
{
Bitmap bitmap = Ultilities.Extract(new string(buffer));
double[] features = Ultilities.Extract(bitmap);
int clabel1 = Int32.Parse(label);
dgvVerifySet.Rows.Add(c2, bitmap, "", features, clabel1);
testingSet++;
c2++;
}
c1++;
}
btnLearn.Enabled = true;
btnVerify.Enabled = true;
}
private static void Training()
{
var csv = new CsvReader(new StreamReader("C:\\Users\\jantk_000\\Documents\\SearchResultRelevance\\trainextracted.csv"));
List <LabeledData> data = csv.GetRecords <LabeledData>().ToList();
List <LabeledData> train = new List <LabeledData>();
List <LabeledData> validation = new List <LabeledData>();
for (int i = 0; i < data.Count; i++)
{
if (i % 5 == 0)
{
validation.Add(data[i]);
}
else
{
train.Add(data[i]);
}
}
double[][] trainInput = train.Select(labeledData => labeledData.GetInputVector()).ToArray();
double[][] trainOutput = train.Select(labeledData => new[] { labeledData.GetOutput() }).ToArray();
double[][] validationInput = validation.Select(labeledData => labeledData.GetInputVector()).ToArray();
double[][] validationOutput = validation.Select(labeledData => new[] { labeledData.GetOutput() }).ToArray();
ActivationNetwork network = new ActivationNetwork(new SigmoidFunction(), 3, 4, 1);
BackPropagationLearning rprop = new BackPropagationLearning(network)
{
LearningRate = 0.01, Momentum = 0.01
};
double lastError = double.MaxValue;
for (int i = 0;; i++)
{
rprop.RunEpoch(trainInput, trainOutput);
double err = 0;
for (int j = 0; j < validationInput.Length; j++)
{
double[] input = validationInput[j];
double[] output = network.Compute(input);
err += Math.Sqrt(Math.Pow(validationOutput[j][0] - output[0], 2));
}
err = err / validationInput.Length;
var improvement = lastError - err;
lastError = err;
Console.WriteLine("Iteration {0} RMSE {1} Improvement {2}", i, err, improvement);
if (improvement < 0.000001)
{
break;
}
}
network.Save("Network.bin");
}
private ActivationNetwork Copy(ActivationNetwork network)
{
using (var memoryStream = new MemoryStream())
{
network.Save(memoryStream);
memoryStream.Position = 0;
return((ActivationNetwork)Network.Load(memoryStream));
}
}
public Network(int input_sz, int output_sz = 10)
{
input_size = input_sz;
output_size = output_sz;
net = new ActivationNetwork(new Accord.Neuro.BipolarSigmoidFunction(),
input_size, input_size * 3, input_size * 2, input_size, 100, output_size);
backprop = new ParallelResilientBackpropagationLearning(net);
nguen = new NguyenWidrow(net);
nguen.Randomize();
}
/// <summary>
/// Deserialization Constructor
/// </summary>
/// <param name="info"></param>
/// <param name="ctxt"></param>
public BackPropagationLearning(SerializationInfo info, StreamingContext ctxt)
{
//Get the values from info and assign them to the appropriate properties
network = (ActivationNetwork)info.GetValue("network", typeof(ActivationNetwork));
learningRate = (double)info.GetValue("learningRate", typeof(double));
momentum = (double)info.GetValue("momentum", typeof(double));
neuronErrors = (double[][])info.GetValue("neuronErrors", typeof(double[][]));
weightsUpdates = (double[][][])info.GetValue("weightsUpdates", typeof(double[][][]));
thresholdsUpdates = (double[][])info.GetValue("thresholdsUpdates", typeof(double[][]));
}
/// <summary>
/// Initializes a new instance of the <see cref="DeltaRuleLearning"/> class.
/// </summary>
///
/// <param name="network">Network to teach.</param>
///
/// <exception cref="ArgumentException">Invalid nuaral network. It should have one layer only.</exception>
///
public DeltaRuleLearning( ActivationNetwork network )
{
// check layers count
if ( network.Layers.Length != 1 )
{
throw new ArgumentException( "Invalid nuaral network. It should have one layer only." );
}
this.network = network;
}
protected override void CreateNetwork()
{
network = new ActivationNetwork(new Tanh(0.1), 1, 7, 7, 6, 1);
network.ForEachWeight(z => rnd.NextDouble() * 2 - 1);
teacher = new BackPropagationLearning(network);
teacher.LearningRate = 1;
teacher.Momentum = 0.3;
}
/// <summary>
/// Constructs a new Nguyen-Widrow Weight initialization.
/// </summary>
///
/// <param name="network">The activation network whose weights will be initialized.</param>
///
public NguyenWidrow(ActivationNetwork network)
{
this.network = network;
int hiddenNodes = network.Layers[0].Neurons.Length;
int inputNodes = network.Layers[0].InputsCount;
randRange = new Range(-0.5f, 0.5f);
beta = 0.7 * Math.Pow(hiddenNodes, 1.0 / inputNodes);
}
/// <summary>
/// Initializes a new instance of the <see cref="DeltaRuleLearning"/> class.
/// </summary>
///
/// <param name="network">Network to teach.</param>
///
/// <exception cref="ArgumentException">Invalid nuaral network. It should have one layer only.</exception>
///
public DeltaRuleLearning(ActivationNetwork network)
{
// check layers count
if (network.Layers.Length != 1)
{
throw new ArgumentException("Invalid nuaral network. It should have one layer only.");
}
this.network = network;
}
/// <summary>
/// Initializes a new instance of the <see cref="LevenbergMarquardtLearning"/> class.
/// </summary>
///
/// <param name="network">Network to teach.</param>
/// <param name="useRegularization">True to use Bayesian regularization, false otherwise.</param>
/// <param name="method">The method by which the Jacobian matrix will be calculated.</param>
///
public LevenbergMarquardtLearning(ActivationNetwork network, bool useRegularization, JacobianMethod method)
{
this.ParallelOptions = new ParallelOptions();
this.network = network;
this.numberOfParameters = getNumberOfParameters(network);
this.outputCount = network.Layers[network.Layers.Length - 1].Neurons.Length;
this.useBayesianRegularization = useRegularization;
this.method = method;
this.weights = new float[numberOfParameters];
this.hessian = new float[numberOfParameters][];
for (int i = 0; i < hessian.Length; i++)
{
hessian[i] = new float[numberOfParameters];
}
this.diagonal = new float[numberOfParameters];
this.gradient = new float[numberOfParameters];
this.jacobian = new float[numberOfParameters][];
// Will use Backpropagation method for Jacobian computation
if (method == JacobianMethod.ByBackpropagation)
{
// create weight derivatives arrays
this.weightDerivatives = new float[network.Layers.Length][][];
this.thresholdsDerivatives = new float[network.Layers.Length][];
// initialize arrays
for (int i = 0; i < network.Layers.Length; i++)
{
ActivationLayer layer = (ActivationLayer)network.Layers[i];
this.weightDerivatives[i] = new float[layer.Neurons.Length][];
this.thresholdsDerivatives[i] = new float[layer.Neurons.Length];
for (int j = 0; j < layer.Neurons.Length; j++)
{
this.weightDerivatives[i][j] = new float[layer.InputsCount];
}
}
}
else // Will use finite difference method for Jacobian computation
{
// create differential coefficient arrays
this.differentialCoefficients = createCoefficients(3);
this.derivativeStepSize = new double[numberOfParameters];
// initialize arrays
for (int i = 0; i < numberOfParameters; i++)
{
this.derivativeStepSize[i] = derivativeStep;
}
}
}
public NeuralNetwork(Accord.Neuro.IActivationFunction function, int numberOfInputs, params int[] numberOfNeurons)
{
inputSize = numberOfInputs;
outputSize = numberOfNeurons[numberOfNeurons.Length - 1];
activationFunction = function;
this.numberOfNeurons = numberOfNeurons;
network = new ActivationNetwork(function, numberOfInputs, numberOfNeurons);
//InitalizeInputs(numberOfInputs);
//InitalizeOutputs(numberOfNeurons[numberOfNeurons.Length - 1]);
}
public void Reset()
{
PersonOccurenceDatabase = new WordOccurenceDatabase();
OrganizationOccurenceDatabase = new WordOccurenceDatabase();
LocationsOccurenceDatabase = new WordOccurenceDatabase();
SportSpecificWordsOccurenceDatabase = new WordOccurenceDatabase();
int hiddenLayerCount = (10 * _categoriesCount) / 3;
_classifier = new ActivationNetwork(new SigmoidFunction(), 4 * _categoriesCount, hiddenLayerCount, _categoriesCount);
_teacher = new BackPropagationLearning(_classifier);
}
public BackPropagationLearning(ActivationNetwork network)
{
this.network = network;
neuronErrors = new double[network.LayersCount][];
for (var i = 0; i < network.LayersCount; i++)
{
neuronErrors[i] = new double[network[i].NeuronsCount];
}
}
private static PredictiveValues CalculatePredictiveValues(ActivationNetwork ann, Data data)
{
var annOutputs = new double[data.ValidateOutputs.Length][];
for (int i = 0; i < data.ValidateInputs.Length; i++)
{
var vi = data.ValidateInputs[i];
annOutputs[i] = new double[] { Math.Round(ann.Compute(vi)[0], 0) };
}
return CalculatePredictiveValues(data.ValidateOutputs, annOutputs);
}
public double Treinar()
{
//var activationNetwork = new ActivationNetwork(new ThresholdFunction(), tamanhoEntrada, _classes.Count);
//var supervisedLearning = new PerceptronLearning(activationNetwork);
activationNetwork = new ActivationNetwork(new SigmoidFunction(0.5), tamanhoEntrada, _classes.Count);
//var supervisedLearning = new PerceptronLearning(activationNetwork);
var supervisedLearning = new BackPropagationLearning(activationNetwork);
supervisedLearning.LearningRate = 0.5;
int indice = 0;
int treinamento = -1;
var entrada = new List <double[]>();
var saida = new List <double[]>();
foreach (var modelo in _classes)
{
modelo.Value.SaidaEsperada = ObterSaidaEsperada(indice++);
int contadorTreinamento = 0;
foreach (var caracteristicas in modelo.Value.ObterCaracteristicas(msSegmentos))
{
entrada.Add(caracteristicas);
saida.Add(modelo.Value.SaidaEsperada);
//var erroAbsoluto = supervisedLearning.Run(caracteristicas, modelo.Value.SaidaEsperada);
if (treinamento == contadorTreinamento++)
{
break;
}
}
}
var entradaArray = entrada.ToArray();
var saidaArray = saida.ToArray();
return(supervisedLearning.RunEpoch(entradaArray, saidaArray));
//var resultado = supervisedLearning.RunEpoch(entradaArray, saidaArray);
//Testar(activationNetwork, msSegmentos, @".\Audios\Giovanni", @".\Audios\Giovanni\audio_02.wav");
//Testar(activationNetwork, msSegmentos, @".\Audios\Giovanni", @".\Audios\Giovanni\audio_03.wav");
//Testar(activationNetwork, msSegmentos, @".\Audios\Sidney", @".\Audios\Sidney\audio_03.wav");
//Testar(activationNetwork, msSegmentos, @".\Audios\Sidney", @".\Audios\Sidney\audio_04.wav");
//Testar(activationNetwork, msSegmentos, @".\Audios\Wellington", @".\Audios\Wellington\audio_03.wav");
}
public NeuralNetworkChromosome(ActivationNetwork activationNetwork)
{
this._activationNetwork = activationNetwork;
this._genes = new double[CountNumberOfGenesInNeuralNetwork()];
for (int i = 0; i < _genes.Length; i++)
{
_genes[i] = GenerateGene();
}
SetNeurons();
}
public AnnAgent(bool learn, int boardSize, byte player = 1)
{
learning = learn;
playerNumber = player;
int boardFields = boardSize * boardSize;
if(File.Exists("ann" + boardSize + ".bin"))
network = (ActivationNetwork)Serialization.LoadNetwork("ann" + boardSize + ".bin");
else
network = new ActivationNetwork(new BipolarSigmoidFunction(), boardFields, 5, boardFields * 2);
backProp = new BackPropagationLearning(network);
teacher = new MinimaxAgent(2, player);
}
/// <summary>
/// Initializes a new instance of the <see cref="EvolutionaryFitness"/> class.
/// </summary>
///
/// <param name="network">Neural network for which fitness will be calculated.</param>
/// <param name="input">Input data samples for neural network.</param>
/// <param name="output">Output data sampels for neural network (desired output).</param>
///
/// <exception cref="ArgumentException">Length of inputs and outputs arrays must be equal and greater than 0.</exception>
/// <exception cref="ArgumentException">Length of each input vector must be equal to neural network's inputs count.</exception>
///
public EvolutionaryFitness( ActivationNetwork network, double[][] input, double[][] output )
{
if ( ( input.Length == 0 ) || ( input.Length != output.Length ) )
{
throw new ArgumentException( "Length of inputs and outputs arrays must be equal and greater than 0." );
}
if ( network.InputsCount != input[0].Length )
{
throw new ArgumentException( "Length of each input vector must be equal to neural network's inputs count." );
}
this.network = network;
this.input = input;
this.output = output;
}
public void RunEpochTest1()
{
Accord.Math.Tools.SetupGenerator(0);
double[][] input =
{
new double[] { -1, -1 },
new double[] { -1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
double[][] output =
{
new double[] { -1 },
new double[] { 1 },
new double[] { 1 },
new double[] { -1 }
};
Neuron.RandGenerator = new ThreadSafeRandom(0);
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(2), 2, 2, 1);
var teacher = new ParallelResilientBackpropagationLearning(network);
double error = 1.0;
while (error > 1e-5)
error = teacher.RunEpoch(input, output);
for (int i = 0; i < input.Length; i++)
{
double actual = network.Compute(input[i])[0];
double expected = output[i][0];
Assert.AreEqual(expected, actual, 0.01);
Assert.IsFalse(Double.IsNaN(actual));
}
}
public IForecastingModel TrainNewModel(double[][] iInput, double[][] iOutput)
{
int inputSize = iInput[0].Length, samplesNum = iOutput.Length;
if (samplesNum != iInput.Length)
throw new ArgumentException();
for (int i = 0; i < samplesNum;++i)
if (iInput[i].Length != inputSize || iOutput[i].Length != 1) //iInput isn't a square matrix or iOutput isn't a vector
throw new ArgumentException();
int[] neuronsCount = (int[]) ModelParametersDict[NeuronsInLayersKey];
string activationFunction = (string) ModelParametersDict[ActivationFunctionKey];
long maxIterNum = (long) ModelParametersDict[MaxIterationsNumberKey];
double stopError = (double)ModelParametersDict[StopErrorKey];
ActivationNetwork netToTrain = new ActivationNetwork(ActivationFunctionsDict[activationFunction], inputSize, neuronsCount);
DataNormalizer normalizer = new DataNormalizer(iInput.Concat(iOutput).ToArray());
IForecastingModel aModel = new ANNforecastingModel(netToTrain, normalizer);
ISupervisedLearning teacher = new ResilientBackpropagationLearning(netToTrain);
double[][] trainInputSet, trainOutputSet;
TrainingSubsetGenerator.GenerateRandomly(iInput, iOutput, out trainInputSet, out trainOutputSet, iMultiplier: TrainSubsetMultiplier);
trainInputSet = normalizer.Normalize(trainInputSet); trainOutputSet = normalizer.Normalize(trainOutputSet);
long epochsCount = 0;
double nextError = ErrorCalculator.CalculateMSE(aModel, iInput, iOutput), prevError;
do
{
prevError = nextError;
teacher.RunEpoch(trainInputSet, trainOutputSet);
nextError = ErrorCalculator.CalculateMSE(aModel, iInput, iOutput);
}
while (epochsCount++ <= maxIterNum && Math.Abs(prevError - nextError) >= stopError);
return aModel;
}
// Worker thread
void SearchSolution( )
{
// initialize input and output values
double[][] input = null;
double[][] output = null;
if ( sigmoidType == 0 )
{
// unipolar data
input = new double[4][] {
new double[] {0, 0},
new double[] {0, 1},
new double[] {1, 0},
new double[] {1, 1}
};
output = new double[4][] {
new double[] {0},
new double[] {1},
new double[] {1},
new double[] {0}
};
}
else
{
// bipolar data
input = new double[4][] {
new double[] {-1, -1},
new double[] {-1, 1},
new double[] { 1, -1},
new double[] { 1, 1}
};
output = new double[4][] {
new double[] {-1},
new double[] { 1},
new double[] { 1},
new double[] {-1}
};
}
// create neural network
ActivationNetwork network = new ActivationNetwork(
( sigmoidType == 0 ) ?
(IActivationFunction) new SigmoidFunction( sigmoidAlphaValue ) :
(IActivationFunction) new BipolarSigmoidFunction( sigmoidAlphaValue ),
2, 2, 1 );
// create teacher
var teacher = new ParallelResilientBackpropagationLearning(network);
// set learning rate and momentum
teacher.Reset(initialStep);
// iterations
int iteration = 0;
// statistic files
StreamWriter errorsFile = null;
try
{
// check if we need to save statistics to files
if ( saveStatisticsToFiles )
{
// open files
errorsFile = File.CreateText( "errors.csv" );
}
// erros list
ArrayList errorsList = new ArrayList( );
// loop
while ( !needToStop )
{
// run epoch of learning procedure
double error = teacher.RunEpoch( input, output );
errorsList.Add( error );
// save current error
if ( errorsFile != null )
{
errorsFile.WriteLine( error );
}
// show current iteration & error
SetText( currentIterationBox, iteration.ToString( ) );
SetText( currentErrorBox, error.ToString( ) );
iteration++;
// check if we need to stop
if ( error <= learningErrorLimit )
break;
}
// show error's dynamics
double[,] errors = new double[errorsList.Count, 2];
for ( int i = 0, n = errorsList.Count; i < n; i++ )
{
errors[i, 0] = i;
errors[i, 1] = (double) errorsList[i];
}
errorChart.RangeX = new Range( 0, errorsList.Count - 1 );
errorChart.UpdateDataSeries( "error", errors );
}
catch ( IOException )
{
MessageBox.Show( "Failed writing file", "Error", MessageBoxButtons.OK, MessageBoxIcon.Error );
}
finally
{
// close files
if ( errorsFile != null )
errorsFile.Close( );
}
// enable settings controls
EnableControls( true );
}
public void ConstructorTest()
{
// Four training samples of the xor function
// two inputs (x and y)
double[][] input =
{
new double[] { -1, -1 },
new double[] { -1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
// one output (z = x ^ y)
double[][] output =
{
new double[] { -1 },
new double[] { 1 },
new double[] { 1 },
new double[] { -1 }
};
// create multi-layer neural network
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(2), // use a bipolar sigmoid activation function
2, // two inputs
3, // three hidden neurons
1 // one output neuron
);
// create teacher
LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(
network, // the neural network
false, // whether or not to use Bayesian regularization
JacobianMethod.ByBackpropagation // Jacobian calculation method
);
// set learning rate and momentum
teacher.LearningRate = 0.1f;
// start the supervisioned learning
for (int i = 0; i < 1000; i++)
{
double error = teacher.RunEpoch(input, output);
}
// If we reached here, the constructor test has passed.
}
public void RunEpochTest4()
{
Accord.Math.Tools.SetupGenerator(0);
double[][] input =
{
new double[] { 0, 0 },
};
double[][] output =
{
new double[] { 0 },
};
Neuron.RandGenerator = new ThreadSafeRandom(0);
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(2), 2, 1);
var teacher = new LevenbergMarquardtLearning(network,
true, JacobianMethod.ByBackpropagation);
double error = 1.0;
for (int i = 0; i < 1000; i++)
error = teacher.RunEpoch(input, output);
for (int i = 0; i < input.Length; i++)
Assert.AreEqual(network.Compute(input[i])[0], output[i][0], 0.1);
}
public void JacobianByChainRuleTest()
{
// Network with one hidden layer: 2-2-1
Accord.Math.Tools.SetupGenerator(0);
double[][] input =
{
new double[] { -1, -1 },
new double[] { -1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
double[][] output =
{
new double[] { -1 },
new double[] { 1 },
new double[] { 1 },
new double[] { -1 }
};
Neuron.RandGenerator = new ThreadSafeRandom(0);
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(2), 2, 2, 1);
var teacher1 = new LevenbergMarquardtLearning(network,
false, JacobianMethod.ByFiniteDifferences);
var teacher2 = new LevenbergMarquardtLearning(network,
false, JacobianMethod.ByBackpropagation);
// Set lambda to lambda max so no iterations are performed
teacher1.LearningRate = 1e30f;
teacher2.LearningRate = 1e30f;
teacher1.RunEpoch(input, output);
teacher2.RunEpoch(input, output);
PrivateObject privateTeacher1 = new PrivateObject(teacher1);
PrivateObject privateTeacher2 = new PrivateObject(teacher2);
var jacobian1 = (float[][])privateTeacher1.GetField("jacobian");
var jacobian2 = (float[][])privateTeacher2.GetField("jacobian");
Assert.AreEqual(jacobian1[0][0], -0.47895513745387097, 1e-6);
Assert.AreEqual(jacobian1[0][1], -0.05863886707282373, 1e-6);
Assert.AreEqual(jacobian1[0][2], 0.057751100929897485, 1e-6);
Assert.AreEqual(jacobian1[0][3], 0.0015185010717608583, 1e-6);
Assert.AreEqual(jacobian1[7][0], -0.185400783651892, 1e-6);
Assert.AreEqual(jacobian1[7][1], 0.025575161626462877, 1e-6);
Assert.AreEqual(jacobian1[7][2], 0.070494677797224889, 1e-6);
Assert.AreEqual(jacobian1[7][3], 0.037740463822781616, 1e-6);
Assert.AreEqual(jacobian2[0][0], -0.4789595904719437, 1e-6);
Assert.AreEqual(jacobian2[0][1], -0.058636153936941729, 1e-6);
Assert.AreEqual(jacobian2[0][2], 0.057748435491340212, 1e-6);
Assert.AreEqual(jacobian2[0][3], 0.0015184453425611988, 1e-6);
Assert.AreEqual(jacobian2[7][0], -0.1854008206574258, 1e-6);
Assert.AreEqual(jacobian2[7][1], 0.025575150379247645, 1e-6);
Assert.AreEqual(jacobian2[7][2], 0.070494269423259301, 1e-6);
Assert.AreEqual(jacobian2[7][3], 0.037740117733922635, 1e-6);
for (int i = 0; i < jacobian1.Length; i++)
{
for (int j = 0; j < jacobian1[i].Length; j++)
{
double j1 = jacobian1[i][j];
double j2 = jacobian2[i][j];
Assert.AreEqual(j1, j2, 1e-4);
Assert.IsFalse(Double.IsNaN(j1));
Assert.IsFalse(Double.IsNaN(j2));
}
}
}
public void BlockHessianTest1()
{
// Network with no hidden layers: 3-1
Accord.Math.Tools.SetupGenerator(0);
double[][] input =
{
new double[] {-1, -1 },
new double[] {-1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
double[][] output =
{
new double[] {-1 },
new double[] { 1 },
new double[] { 1 },
new double[] {-1 }
};
Neuron.RandGenerator = new ThreadSafeRandom(0);
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(2), 2, 1);
var teacher1 = new LevenbergMarquardtLearning(network,
false, JacobianMethod.ByFiniteDifferences);
var teacher2 = new LevenbergMarquardtLearning(network,
false, JacobianMethod.ByBackpropagation);
teacher2.Blocks = 2;
// Set lambda to lambda max so no iterations are performed
teacher1.LearningRate = 1e30f;
teacher2.LearningRate = 1e30f;
teacher1.RunEpoch(input, output);
teacher2.RunEpoch(input, output);
var hessian1 = teacher1.Hessian;
var hessian2 = teacher1.Hessian;
for (int i = 0; i < hessian1.Length; i++)
{
for (int j = 0; j < hessian1[i].Length; j++)
{
double j1 = hessian1[i][j];
double j2 = hessian2[i][j];
Assert.AreEqual(j1, j2, 1e-4);
Assert.IsFalse(Double.IsNaN(j1));
Assert.IsFalse(Double.IsNaN(j2));
}
}
Assert.IsTrue(hessian1.IsUpperTriangular());
Assert.IsTrue(hessian2.IsUpperTriangular());
var gradient1 = teacher1.Gradient;
var gradient2 = teacher2.Gradient;
for (int i = 0; i < gradient1.Length; i++)
{
double j1 = gradient1[i];
double j2 = gradient2[i];
Assert.AreEqual(j1, j2, 1e-5);
Assert.IsFalse(Double.IsNaN(j1));
Assert.IsFalse(Double.IsNaN(j2));
}
}
/// <summary>
/// Initializes a new instance of the <see cref="EvolutionaryLearning"/> class.
/// </summary>
///
/// <param name="activationNetwork">Activation network to be trained.</param>
/// <param name="populationSize">Size of genetic population.</param>
/// <param name="chromosomeGenerator">Random numbers generator used for initialization of genetic
/// population representing neural network's weights and thresholds (see <see cref="DoubleArrayChromosome.chromosomeGenerator"/>).</param>
/// <param name="mutationMultiplierGenerator">Random numbers generator used to generate random
/// factors for multiplication of network's weights and thresholds during genetic mutation
/// (ses <see cref="DoubleArrayChromosome.mutationMultiplierGenerator"/>.)</param>
/// <param name="mutationAdditionGenerator">Random numbers generator used to generate random
/// values added to neural network's weights and thresholds during genetic mutation
/// (see <see cref="DoubleArrayChromosome.mutationAdditionGenerator"/>).</param>
/// <param name="selectionMethod">Method of selection best chromosomes in genetic population.</param>
/// <param name="crossOverRate">Crossover rate in genetic population (see
/// <see cref="Population.CrossoverRate"/>).</param>
/// <param name="mutationRate">Mutation rate in genetic population (see
/// <see cref="Population.MutationRate"/>).</param>
/// <param name="randomSelectionRate">Rate of injection of random chromosomes during selection
/// in genetic population (see <see cref="Population.RandomSelectionPortion"/>).</param>
///
public EvolutionaryLearning( ActivationNetwork activationNetwork, int populationSize,
IRandomNumberGenerator chromosomeGenerator,
IRandomNumberGenerator mutationMultiplierGenerator,
IRandomNumberGenerator mutationAdditionGenerator,
ISelectionMethod selectionMethod,
double crossOverRate, double mutationRate, double randomSelectionRate )
{
// Check of assumptions during debugging only
Debug.Assert( activationNetwork != null );
Debug.Assert( populationSize > 0 );
Debug.Assert( chromosomeGenerator != null );
Debug.Assert( mutationMultiplierGenerator != null );
Debug.Assert( mutationAdditionGenerator != null );
Debug.Assert( selectionMethod != null );
Debug.Assert( crossOverRate >= 0.0 && crossOverRate <= 1.0 );
Debug.Assert( mutationRate >= 0.0 && crossOverRate <= 1.0 );
Debug.Assert( randomSelectionRate >= 0.0 && randomSelectionRate <= 1.0 );
// networks's parameters
this.network = activationNetwork;
this.numberOfNetworksWeights = CalculateNetworkSize( activationNetwork );
// population parameters
this.populationSize = populationSize;
this.chromosomeGenerator = chromosomeGenerator;
this.mutationMultiplierGenerator = mutationMultiplierGenerator;
this.mutationAdditionGenerator = mutationAdditionGenerator;
this.selectionMethod = selectionMethod;
this.crossOverRate = crossOverRate;
this.mutationRate = mutationRate;
this.randomSelectionRate = randomSelectionRate;
}
// Worker thread
void SearchSolution()
{
// number of learning samples
int samples = sourceMatrix.GetLength(0);
// prepare learning data
double[][] inputs = sourceMatrix.Submatrix(null, 0, 1).ToArray();
double[][] outputs = sourceMatrix.GetColumn(2).Transpose().ToArray();
// create multi-layer neural network
ann = new ActivationNetwork(new GaussianFunction(sigmoidAlphaValue),
//new BipolarSigmoidFunction(sigmoidAlphaValue),
2, neuronsInFirstLayer, 1);
if (useNguyenWidrow)
{
if (useSameWeights)
Accord.Math.Tools.SetupGenerator(0);
NguyenWidrow initializer = new NguyenWidrow(ann);
initializer.Randomize();
}
// create teacher
LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(ann, useRegularization);
// set learning rate and momentum
teacher.LearningRate = learningRate;
// iterations
iteration = 1;
var ranges = Matrix.Range(sourceMatrix, 0);
double[][] map = Matrix.Mesh(ranges[0], ranges[1], 1, 1);
var sw = Stopwatch.StartNew();
// loop
while (!needToStop)
{
// run epoch of learning procedure
error = teacher.RunEpoch(inputs, outputs) / samples;
//var result = map.Apply(ann.Compute).GetColumn(0).Apply(Math.Sign);
var result = map.Apply(ann.Compute).GetColumn(0).Apply(d =>
{
if (d > 0.66) return 2;
else if (d < 0.33)
return 0;
return 1;
});
var graph = map.ToMatrix().InsertColumn(result.ToDouble());
CreateScatterplot(zedGraphControl2, graph);
// increase current iteration
iteration++;
elapsed = sw.Elapsed;
updateStatus();
// check if we need to stop
if ((iterations != 0) && (iteration > iterations))
break;
}
sw.Stop();
// enable settings controls
EnableControls(true);
}
private static double computeError(double[][] inputs, double[][] outputs, ActivationNetwork ann)
{
// Compute the machine outputs
int miss = 0;
for (int i = 0; i < inputs.Length; i++)
{
var y = System.Math.Sign(ann.Compute(inputs[i])[0]);
var o = outputs[i][0];
if (y != o) miss++;
}
return (double)miss / inputs.Length;
}
// Worker thread
void SearchSolution()
{
// number of learning samples
int samples = data.GetLength(0);
// data transformation factor
double yFactor = 1.7 / chart.RangeY.Length;
double yMin = chart.RangeY.Min;
double xFactor = 2.0 / chart.RangeX.Length;
double xMin = chart.RangeX.Min;
// prepare learning data
double[][] input = new double[samples][];
double[][] output = new double[samples][];
for (int i = 0; i < samples; i++)
{
input[i] = new double[1];
output[i] = new double[1];
// set input
input[i][0] = (data[i, 0] - xMin) * xFactor - 1.0;
// set output
output[i][0] = (data[i, 1] - yMin) * yFactor - 0.85;
}
// create multi-layer neural network
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(sigmoidAlphaValue),
1, neuronsInFirstLayer, 1);
if (useNguyenWidrow)
{
NguyenWidrow initializer = new NguyenWidrow(network);
initializer.Randomize();
}
// create teacher
var teacher = new ParallelResilientBackpropagationLearning(network);
// iterations
int iteration = 1;
// solution array
double[,] solution = new double[50, 2];
double[] networkInput = new double[1];
// calculate X values to be used with solution function
for (int j = 0; j < 50; j++)
{
solution[j, 0] = chart.RangeX.Min + (double)j * chart.RangeX.Length / 49;
}
// loop
while (!needToStop)
{
// run epoch of learning procedure
double error = teacher.RunEpoch(input, output) / samples;
// calculate solution
for (int j = 0; j < 50; j++)
{
networkInput[0] = (solution[j, 0] - xMin) * xFactor - 1.0;
solution[j, 1] = (network.Compute(networkInput)[0] + 0.85) / yFactor + yMin;
}
chart.UpdateDataSeries("solution", solution);
// calculate error
double learningError = 0.0;
for (int j = 0, k = data.GetLength(0); j < k; j++)
{
networkInput[0] = input[j][0];
learningError += Math.Abs(data[j, 1] - ((network.Compute(networkInput)[0] + 0.85) / yFactor + yMin));
}
// set current iteration's info
SetText(currentIterationBox, iteration.ToString());
SetText(currentErrorBox, learningError.ToString("F3"));
// increase current iteration
iteration++;
// check if we need to stop
if ((iterations != 0) && (iteration > iterations))
break;
}
// enable settings controls
EnableControls(true);
}
public BinarySplitForm()
{
InitializeComponent();
watch = new Stopwatch();
backgroundWorkerTrainer.Disposed += backgroundWorkerTrainer_Disposed;
backgroundWorkerTrainer.DoWork += backgroundWorkerTrainer_DoWork;
backgroundWorkerTrainer.ProgressChanged += backgroundWorkerTrainer_ProgressChanged;
backgroundWorkerTrainer.WorkerSupportsCancellation = true;
backgroundWorkerTrainer.WorkerReportsProgress = true;
saveFileDialog1.Filter = "feed forward network files (*.ffn)|*.ffn";
saveFileDialog1.Title = "Save neural networkfile";
saveFileDialog1.InitialDirectory = null;
saveFileDialog1.FileName = null;
openFileDialog1.Filter = "feed forward network files (*.ffn)|*.ffn";
openFileDialog1.Title = "Load neural network file";
openFileDialog1.InitialDirectory = null;
openFileDialog1.FileName = null;
backgroundWorkerSignal.Disposed += backgroundWorkerSignal_Disposed;
backgroundWorkerSignal.WorkerSupportsCancellation = true;
backgroundWorkerSignal.WorkerReportsProgress = true;
backgroundWorkerSignal.DoWork += backgroundWorkerSignal_DoWork;
backgroundWorkerSignal.RunWorkerCompleted += backgroundWorkerSignal_RunWorkerCompleted;
// initialize input and output values
input = new double[4][] {
new double[] {0, 0}, new double[] {0, 1},
new double[] {1, 0}, new double[] {1, 1}
};
output = new double[4][] {
new double[] {0}, new double[] {1},
new double[] {1}, new double[] {0}
};
network = new ActivationNetwork(new SigmoidFunction(2), 2, 2, 1);
teacher = new BackPropagationLearning(network);
//logg.Show();
// pane used to draw your chart
myPane = new GraphPane();
// poing pair lists
listPointsOne = new PointPairList();
}
// Worker thread
void SearchSolution()
{
// number of learning samples
int samples = data.GetLength(0);
// prepare learning data
DoubleRange unit = new DoubleRange(-1, 1);
double[][] input = Tools.Scale(from: xRange, to: unit, x: data.GetColumn(0)).ToArray();
double[][] output = Tools.Scale(from: yRange, to: unit, x: data.GetColumn(1)).ToArray();
// create multi-layer neural network
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(sigmoidAlphaValue),
1, neuronsInFirstLayer, 1);
if (useNguyenWidrow)
{
new NguyenWidrow(network).Randomize();
}
// create teacher
var teacher = new ParallelResilientBackpropagationLearning(network);
// iterations
int iteration = 1;
// solution array
double[,] solution = new double[samples, 2];
// loop
while (!needToStop)
{
// run epoch of learning procedure
double error = teacher.RunEpoch(input, output) / samples;
// calculate solution
for (int j = 0; j < samples; j++)
{
double x = input[j][0];
double y = network.Compute(new[] { x })[0];
solution[j, 0] = Tools.Scale(from: unit, to: xRange, x: x);
solution[j, 1] = Tools.Scale(from: unit, to: yRange, x: y);
}
chart.UpdateDataSeries("solution", solution);
// calculate error
double learningError = 0.0;
for (int j = 0; j < samples; j++)
{
double x = input[j][0];
double expected = data[j, 1];
double actual = network.Compute(new[] { x })[0];
learningError += Math.Abs(expected - actual);
}
// set current iteration's info
SetText(currentIterationBox, iteration.ToString());
SetText(currentErrorBox, learningError.ToString("F3"));
// increase current iteration
iteration++;
// check if we need to stop
if ((iterations != 0) && (iteration > iterations))
break;
}
// enable settings controls
EnableControls(true);
}
public void JacobianByChainRuleTest4()
{
// Network with no hidden layers: 3-1
double[][] input =
{
new double[] {-1, -1 },
new double[] {-1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
double[][] output =
{
new double[] {-1 },
new double[] { 1 },
new double[] { 1 },
new double[] {-1 }
};
Neuron.RandGenerator = new ThreadSafeRandom(0);
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(2), 2, 1);
var teacher1 = new LevenbergMarquardtLearning(network,
false, JacobianMethod.ByFiniteDifferences);
var teacher2 = new LevenbergMarquardtLearning(network,
false, JacobianMethod.ByBackpropagation);
// Set lambda to lambda max so no iterations are performed
teacher1.LearningRate = 1e30f;
teacher2.LearningRate = 1e30f;
teacher1.RunEpoch(input, output);
teacher2.RunEpoch(input, output);
PrivateObject privateTeacher1 = new PrivateObject(teacher1);
PrivateObject privateTeacher2 = new PrivateObject(teacher2);
var jacobian1 = (float[][])privateTeacher1.GetField("jacobian");
var jacobian2 = (float[][])privateTeacher2.GetField("jacobian");
for (int i = 0; i < jacobian1.Length; i++)
{
for (int j = 0; j < jacobian1[i].Length; j++)
{
double j1 = jacobian1[i][j];
double j2 = jacobian2[i][j];
Assert.AreEqual(j1, j2, 1e-5);
Assert.IsFalse(Double.IsNaN(j1));
Assert.IsFalse(Double.IsNaN(j2));
}
}
}
public void JacobianByChainRuleTest_MultipleOutput()
{
// Network with no hidden layers: 3-4
int numberOfInputs = 3;
int numberOfClasses = 4;
double[][] input =
{
new double[] { -1, -1, -1 }, // 0
new double[] { -1, 1, -1 }, // 1
new double[] { 1, -1, -1 }, // 1
new double[] { 1, 1, -1 }, // 0
new double[] { -1, -1, 1 }, // 2
new double[] { -1, 1, 1 }, // 3
new double[] { 1, -1, 1 }, // 3
new double[] { 1, 1, 1 } // 2
};
int[] labels =
{
0,
1,
1,
0,
2,
3,
3,
2,
};
double[][] output = Accord.Statistics.Tools
.Expand(labels, numberOfClasses, -1, 1);
Neuron.RandGenerator = new ThreadSafeRandom(0);
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(2), numberOfInputs, numberOfClasses);
var teacher1 = new LevenbergMarquardtLearning(network,
false, JacobianMethod.ByFiniteDifferences);
var teacher2 = new LevenbergMarquardtLearning(network,
false, JacobianMethod.ByBackpropagation);
// Set lambda to lambda max so no iterations are performed
teacher1.LearningRate = 1e30f;
teacher2.LearningRate = 1e30f;
teacher1.RunEpoch(input, output);
teacher2.RunEpoch(input, output);
PrivateObject privateTeacher1 = new PrivateObject(teacher1);
PrivateObject privateTeacher2 = new PrivateObject(teacher2);
var jacobian1 = (float[][])privateTeacher1.GetField("jacobian");
var jacobian2 = (float[][])privateTeacher2.GetField("jacobian");
for (int i = 0; i < jacobian1.Length; i++)
{
for (int j = 0; j < jacobian1[i].Length; j++)
{
double j1 = jacobian1[i][j];
double j2 = jacobian2[i][j];
Assert.AreEqual(j1, j2, 1e-3);
Assert.IsFalse(Double.IsNaN(j1));
Assert.IsFalse(Double.IsNaN(j2));
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="EvolutionaryLearning"/> class.
/// </summary>
///
/// <param name="activationNetwork">Activation network to be trained.</param>
/// <param name="populationSize">Size of genetic population.</param>
///
/// <remarks><para>This version of constructor is used to create genetic population
/// for searching optimal neural network's weight using default set of parameters, which are:
/// <list type="bullet">
/// <item>Selection method - elite;</item>
/// <item>Crossover rate - 0.75;</item>
/// <item>Mutation rate - 0.25;</item>
/// <item>Rate of injection of random chromosomes during selection - 0.20;</item>
/// <item>Random numbers generator for initializing new chromosome -
/// <c>UniformGenerator( new Range( -1, 1 ) )</c>;</item>
/// <item>Random numbers generator used during mutation for genes' multiplication -
/// <c>ExponentialGenerator( 1 )</c>;</item>
/// <item>Random numbers generator used during mutation for adding random value to genes -
/// <c>UniformGenerator( new Range( -0.5f, 0.5f ) )</c>.</item>
/// </list></para>
///
/// <para>In order to have full control over the above default parameters, it is possible to
/// used extended version of constructor, which allows to specify all of the parameters.</para>
/// </remarks>
///
public EvolutionaryLearning( ActivationNetwork activationNetwork, int populationSize )
{
// Check of assumptions during debugging only
Debug.Assert( activationNetwork != null );
Debug.Assert( populationSize > 0 );
// networks's parameters
this.network = activationNetwork;
this.numberOfNetworksWeights = CalculateNetworkSize( activationNetwork );
// population parameters
this.populationSize = populationSize;
this.chromosomeGenerator = new UniformGenerator( new Range( -1, 1 ) );
this.mutationMultiplierGenerator = new ExponentialGenerator( 1 );
this.mutationAdditionGenerator = new UniformGenerator( new Range( -0.5f, 0.5f ) );
this.selectionMethod = new EliteSelection( );
this.crossOverRate = 0.75;
this.mutationRate = 0.25;
this.randomSelectionRate = 0.2;
}
public void ZeroLambdaTest()
{
double[,] data = null;
// open selected file
using (TextReader stream = new StringReader(Properties.Resources.ZeroLambda))
using (CsvReader reader = new CsvReader(stream, false))
{
data = reader.ToTable().ToMatrix();
}
// number of learning samples
int samples = data.GetLength(0);
var ranges = data.Range(dimension: 0);
Assert.AreEqual(2, ranges.Length);
var rangeX = ranges[0];
var rangeY = ranges[1];
// data transformation factor
double yFactor = 1.7 / rangeY.Length;
double yMin = rangeY.Min;
double xFactor = 2.0 / rangeX.Length;
double xMin = rangeX.Min;
// prepare learning data
double[][] input = new double[samples][];
double[][] output = new double[samples][];
for (int i = 0; i < samples; i++)
{
input[i] = new double[1];
output[i] = new double[1];
input[i][0] = (data[i, 0] - xMin) * xFactor - 1.0; // set input
output[i][0] = (data[i, 1] - yMin) * yFactor - 0.85; // set output
}
// create multi-layer neural network
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(5),
1, 12, 1);
// create teacher
LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(network, true);
teacher.LearningRate = 1;
// iterations
int iteration = 1;
int iterations = 2000;
// solution array
double[,] solution = new double[samples, 2];
double[] networkInput = new double[1];
bool needToStop = false;
double learningError = 0;
// loop
while (!needToStop)
{
Assert.AreNotEqual(0, teacher.LearningRate);
// run epoch of learning procedure
double error = teacher.RunEpoch(input, output) / samples;
// calculate solution
for (int j = 0; j < samples; j++)
{
networkInput[0] = (solution[j, 0] - xMin) * xFactor - 1.0;
solution[j, 1] = (network.Compute(networkInput)[0] + 0.85) / yFactor + yMin;
}
// calculate error
learningError = 0.0;
for (int j = 0; j < samples; j++)
{
networkInput[0] = input[j][0];
learningError += Math.Abs(data[j, 1] - ((network.Compute(networkInput)[0] + 0.85) / yFactor + yMin));
}
// increase current iteration
iteration++;
// check if we need to stop
if ((iterations != 0) && (iteration > iterations))
break;
}
Assert.IsTrue(learningError < 0.13);
}
// Create and initialize genetic population
private int CalculateNetworkSize( ActivationNetwork activationNetwork )
{
// caclculate total amount of weight in neural network
int networkSize = 0;
for ( int i = 0; i < network.Layers.Length; i++ )
{
Layer layer = network.Layers[i];
for ( int j = 0; j < layer.Neurons.Length; j++ )
{
// sum all weights and threshold
networkSize += layer.Neurons[j].Weights.Length + 1;
}
}
return networkSize;
}
private void loadFileToolStripMenuItem_Click(object sender, EventArgs e)
{
var r = openFileDialog1.ShowDialog();
if (r != System.Windows.Forms.DialogResult.Cancel)
{
try
{
var n = Network.Load(openFileDialog1.FileName);
network = n as ActivationNetwork;
}
catch (Exception eg)
{
MessageBox.Show("Error occured!");
}
}
}
public void RunEpochTest3()
{
double[,] dataset = yinyang;
double[][] input = dataset.GetColumns(0, 1).ToArray();
double[][] output = dataset.GetColumn(2).ToArray();
Neuron.RandGenerator = new ThreadSafeRandom(0);
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(2), 2, 5, 1);
var teacher = new LevenbergMarquardtLearning(network,
true, JacobianMethod.ByBackpropagation);
Assert.IsTrue(teacher.UseRegularization);
double error = 1.0;
for (int i = 0; i < 500; i++)
error = teacher.RunEpoch(input, output);
double[][] actual = new double[output.Length][];
for (int i = 0; i < input.Length; i++)
actual[i] = network.Compute(input[i]);
for (int i = 0; i < input.Length; i++)
Assert.AreEqual(Math.Sign(output[i][0]), Math.Sign(actual[i][0]));
}
// Create and initialize genetic population
private int CalculateNetworkSize( ActivationNetwork activationNetwork )
{
// caclculate total amount of weight in neural network
int networkSize = 0;
for ( int i = 0, layersCount = network.LayersCount; i < layersCount; i++ )
{
ActivationLayer layer = network[i];
for ( int j = 0, neuronsCount = layer.NeuronsCount; j < neuronsCount; j++ )
{
// sum all weights and threshold
networkSize += layer[j].InputsCount + 1;
}
}
return networkSize;
}
/// <summary>
/// Initializes a new instance of the <see cref="ResilientBackpropagationLearning"/> class.
/// </summary>
///
/// <param name="network">Network to teach.</param>
///
public ResilientBackpropagationLearning( ActivationNetwork network )
{
this.network = network;
int layersCount = network.Layers.Length;
neuronErrors = new double[layersCount][];
weightsDerivatives = new double[layersCount][][];
thresholdsDerivatives = new double[layersCount][];
weightsPreviousDerivatives = new double[layersCount][][];
thresholdsPreviousDerivatives = new double[layersCount][];
weightsUpdates = new double[layersCount][][];
thresholdsUpdates = new double[layersCount][];
// initialize errors, derivatives and steps
for ( int i = 0; i < network.Layers.Length; i++ )
{
Layer layer = network.Layers[i];
int neuronsCount = layer.Neurons.Length;
neuronErrors[i] = new double[neuronsCount];
weightsDerivatives[i] = new double[neuronsCount][];
weightsPreviousDerivatives[i] = new double[neuronsCount][];
weightsUpdates[i] = new double[neuronsCount][];
thresholdsDerivatives[i] = new double[neuronsCount];
thresholdsPreviousDerivatives[i] = new double[neuronsCount];
thresholdsUpdates[i] = new double[neuronsCount];
// for each neuron
for ( int j = 0; j < layer.Neurons.Length; j++ )
{
weightsDerivatives[i][j] = new double[layer.InputsCount];
weightsPreviousDerivatives[i][j] = new double[layer.InputsCount];
weightsUpdates[i][j] = new double[layer.InputsCount];
}
}
// intialize steps
ResetUpdates( learningRate );
}
