public void ExampleTest1()
{
Accord.Math.Tools.SetupGenerator(0);
// We'll use a simple XOR function as input.
double[][] inputs =
{
new double[] { 0, 0 }, // 0 xor 0
new double[] { 0, 1 }, // 0 xor 1
new double[] { 1, 0 }, // 1 xor 0
new double[] { 1, 1 }, // 1 xor 1
};
// XOR output, corresponding to the input.
double[][] outputs =
{
new double[] { 0 }, // 0 xor 0 = 0
new double[] { 1 }, // 0 xor 1 = 1
new double[] { 1 }, // 1 xor 0 = 1
new double[] { 0 }, // 1 xor 1 = 0
};
// Setup the deep belief network (2 inputs, 3 hidden, 1 output)
DeepBeliefNetwork network = new DeepBeliefNetwork(2, 3, 1);
// Initialize the network with Gaussian weights
new GaussianWeights(network, 0.1).Randomize();
// Update the visible layer with the new weights
network.UpdateVisibleWeights();
// Setup the learning algorithm.
DeepBeliefNetworkLearning teacher = new DeepBeliefNetworkLearning(network)
{
Algorithm = (h, v, i) => new ContrastiveDivergenceLearning(h, v)
{
LearningRate = 0.1,
Momentum = 0.5,
Decay = 0.001,
}
};
// Unsupervised learning on each hidden layer, except for the output.
for (int i = 0; i < network.Layers.Length - 1; i++)
{
teacher.LayerIndex = i;
// Compute the learning data with should be used
var layerInput = teacher.GetLayerInput(inputs);
// Train the layer iteratively
for (int j = 0; j < 5000; j++)
teacher.RunEpoch(layerInput);
}
// Supervised learning on entire network, to provide output classification.
var backpropagation = new BackPropagationLearning(network)
{
LearningRate = 0.1,
Momentum = 0.5
};
// Run supervised learning.
for (int i = 0; i < 5000; i++)
backpropagation.RunEpoch(inputs, outputs);
// Test the resulting accuracy.
int correct = 0;
for (int i = 0; i < inputs.Length; i++)
{
double[] outputValues = network.Compute(inputs[i]);
double outputResult = outputValues.First() >= 0.5 ? 1 : 0;
if (outputResult == outputs[i].First())
{
correct++;
}
}
Assert.AreEqual(4, correct);
}
private void button1_Click(object sender, EventArgs e)
{
needToStop = false;
workThread = new Thread(() =>
{
// Learning
while (!needToStop)
{
var error = teacher.RunEpoch(sampleStorage.InputLearn, sampleStorage.OutPutLearn);
errorVal = error.ToString(CultureInfo.InvariantCulture);
if (error < 0.1)
{
needToStop = true;
}
}
});
workThread.Priority = ThreadPriority.Highest;
workThread.Start();
}
private void RunEpoch(BackPropagationLearning teacher, double[][] input, double[][] output, bool isParallel)
{
if (isParallel)
{
var data = input.Zip(output, (n, w) => new { singleInput = n, singleOutput = w });
//var q = from v in data.AsParallel()
// select teacher.Run(v.singleInput, v.singleOutput);
//q.ToArray();
Parallel.ForEach(data, v =>
{
teacher.Run(v.singleInput, v.singleOutput);
});
}
else
{
teacher.RunEpoch(input, output);
}
}
public void bpnnTraining()
{
double[][] input_data = new double[Data.instance.images.Count][];
double[][] output_data = new double[Data.instance.indexClasses.Count][];
int max_output = Data.instance.classes.Count - 1, min_output = 0;
for (int i = 0; i < Data.instance.images.Count; i++)
{
//Pilih gambar berwarna
Bitmap image = new Bitmap(Data.instance.images[i]);
//Preprocess jadi 10 x 10 hitam putih
image = Data.instance.preProcess(image);
// dari pixel 0-255 jadi 0-1
ImageToArray converter = new ImageToArray(0, 1);
converter.Convert(image, out input_data[i]);
output_data[i] = new double[1];
output_data[i][0] = Data.instance.indexClasses[i];
output_data[i][0] = 0 + (output_data[i][0] - min_output) * (1 - 0) / (max_output - min_output);
}
bpnnNetwork = new ActivationNetwork(new SigmoidFunction(), 100, 3, 1);
bpnnLearning = new BackPropagationLearning(bpnnNetwork);
//o Error Goals: 0.000001
//o Max Epochs: 1000000
int max_iter = 1000000;
double max_error = 0.000001;
for (int i = 0; i < max_iter; i++)
{
double error = bpnnLearning.RunEpoch(input_data, output_data);
if (error < max_error)
{
break;
}
}
bpnnNetwork.Save("an.bin");
}
public static ActivationNetwork CreateANN(double[][] input, double[][] output, double learningRate)
{
var watch = System.Diagnostics.Stopwatch.StartNew();
int nrOfOutputs = output[0].Length;
int nrOfInputs = input[0].Length;
int[] layers = { 4, 4, nrOfOutputs }; //Neurons per layer
ActivationNetwork network = new ActivationNetwork(
new SigmoidFunction(), //Activation function
2, //Input
layers); //Hidden layers + output
BackPropagationLearning teacher = new BackPropagationLearning(network);
teacher.LearningRate = learningRate;
bool needToStop = false;
int epochs = 0;
double prevError = 999999.9;
while (!needToStop)
{
double error = teacher.RunEpoch(input, output);
if (Math.Abs(prevError - error) < 0.0000001 || epochs == 10000)
{
needToStop = true;
watch.Stop();
var elapsedMs = watch.ElapsedMilliseconds;
Console.WriteLine("\n" + 2 + ", " + nrOfOutputs + " Stop at: " + epochs +
" Error: " + error + " Time: " + elapsedMs / 1000 + "s");
}
if (epochs % 1000 == 0 && epochs != 0)
{
Console.Write(".");
}
prevError = error;
epochs++;
}
return(network);
}
public void Learn(ref ActivationNetwork network, double[] crossInput, double[] circleInput, double[] blankInput)
{
NetworkInput networkInput = new NetworkInput(crossInput, circleInput, blankInput);
double[][] input = networkInput.Input;
NetworkOutput networkOutput = new NetworkOutput(0);
double[][] output = networkOutput.Output;
BackPropagationLearning teacher =
new BackPropagationLearning(network);
for (int i = 0; i < 100000; i++)
{
double error = teacher.RunEpoch(input, output);
}
network.Save("Net.bin");
MessageBox.Show("Learning finished");
}
public void Ucz() //algorytm uczacy siec neuronowa
{
int NumberOfImputs = 6;
int Classnes = 11;
int NumberOfHiddenNeurons = 6;
network = new ActivationNetwork(new BipolarSigmoidFunction(2), NumberOfImputs, NumberOfHiddenNeurons, Classnes);
var teacher = new BackPropagationLearning(network);
double error = 1;
double[][] input = odczytDanychIN();
int[] label = odczytDanychOUT();
double[][] output = Jagged.OneHot(label, Classnes);
for (int i = 0; i < 10000; i++)
{
error = teacher.RunEpoch(input, output);
}
label2.Text = error.ToString();
}
public NeuralNetwork Train(BackProbArgs X)
{
if (!Info.Moduls.Contains("NeuralNetworks/Learning/BackProb.cs"))
{
Info.Moduls.Add("NeuralNetworks/Learning/BackProb.cs");
}
BackPropagationLearning Trainer
= new BackPropagationLearning(X.NetworkToTrain.ANetwork);
Trainer.Momentum = (X.Momentem <= 0) ? (3) : (X.Momentem);
Trainer.LearningRate = (X.LearningRate <= 0) ? (0.01) : (X.LearningRate);
uint tick = Settings.LearningUpdateTick;
while (true)
{
double Error = Trainer.RunEpoch(X.Inputs, X.Outputs);
if (Error <= X.WantedERROR)
{
break;
}
if (tick == 0)
{
Settings.OnUpdateEvent(this, new LearningUpdateArgs()
{
ERROR = Error,
Inputs = X.Inputs,
Outputs = X.Outputs,
NetworkState = X.NetworkToTrain,
});
tick = Settings.LearningUpdateTick;
}
else
{
tick--;
}
}
return(X.NetworkToTrain);
}
private void SearchSolution(out BackPropagationLearning teacher, double[][] input, double[][] output)
{
int gen = 0;
double error = 0;
ActivationNetwork network = new ActivationNetwork(
new BipolarSigmoidFunction(2),
Input_layer, Hidden_layer, Output_layer);
teacher = new BackPropagationLearning(network);
int iteration = 1;
error = 1.0;
while (error > 0.001)
{
error = teacher.RunEpoch(input, output) / OBJECTNUM;
iteration++;
if ((iterations != 0) && (iteration > iterations))
{
break;
}
}
}
public void TeachNetworkByEpochs(List <List <double> > Input, List <int> Output, int Epochs)
{
var teacher = new BackPropagationLearning(network)
{
LearningRate = 0.1f,
Momentum = 0.9f
};
/*var teacher = new Accord.Neuro.Learning.DeepNeuralNetworkLearning(network)
* {
* Algorithm = (ann, i) => new ParallelResilientBackpropagationLearning(ann),
* LayerIndex = network.Layers.Length - 1,
* };*/
double[][] inputs, outputs;
//Main.Database.Training.GetInstances(out inputs, out outputs);
inputs = new double[Input.Count][];
outputs = new double[Output.Count][];
for (int i = 0; i < Input.Count; i++)
{
inputs[i] = Input[i].ToArray();
outputs[i] = new double[6] {
0, 0, 0, 0, 0, 0
};
outputs[i][Output[i]] = 1;
}
// Start running the learning procedure
for (int i = 0; i < Epochs; i++)
{
//teacher.Run(Input.ToArray(), Output.ToArray());
double error = teacher.RunEpoch(inputs, outputs);
}
//network.UpdateVisibleWeights();
}
public void test()
{
// initialize input and output values
var input = new double[4][] {
new double[] { 0, 0 }, new double[] { 0, 1 },
new double[] { 1, 0 }, new double[] { 1, 1 }
};
var output = new double[4][] {
new double[] { 0 }, new double[] { 1 },
new double[] { 1 }, new double[] { 1 }
};
// create neural network
var network = new ActivationNetwork(
new SigmoidFunction(2),
2, // two inputs in the network
//2, // two neurons in the first layer
1); // one neuron in the second layer
// create teacher
var teacher =
new BackPropagationLearning(network);
// loop
while (true)
{
// run epoch of learning procedure
var error = teacher.RunEpoch(input, output);
// check error value to see if we need to stop
// ...
if (error < 0.001)
{
break;
}
}
Console.WriteLine(network.Compute(new double[] { 0, 0 })[0] + ","
+ network.Compute(new double[] { 0, 1 })[0] + ","
+ network.Compute(new double[] { 1, 0 })[0] + ","
+ network.Compute(new double[] { 1, 1 })[0]);
}
public void TrainBPNet(double [][] trainInput, double [][] trainOutput)//建立BP神经网络，以获得适应度函数
{
Console.WriteLine("神经网络训练开始:\n");
ActivationNetwork network = new ActivationNetwork(new SigmoidFunction(1.5), InputDimension, HiddenDimension, OutputDimension);//定义网络
BackPropagationLearning teacher = new BackPropagationLearning(network);
teacher.LearningRate = 0.1;
teacher.Momentum = 0;
int iteration = 1;
double error = 1.0;
Console.WriteLine("神经网络训练中...\n");
while (iteration < BP_Iter_Max)
{
error = teacher.RunEpoch(trainInput, trainOutput);
iteration++;
}
Console.WriteLine("训练完毕，神经网络权值保存在\\bin\\" + SaveNetFile + "中\n");
SaveNet(network, SaveNetFile);
Console.WriteLine("训练结果已保存在神经网络训练结果.xls中！\n");
IO.WriteTitleToXls("神经网络训练结果.xls", IO.TTitle);
StreamWriter sw = new StreamWriter("神经网络训练结果.xls", true);
for (int i = 0; i < trainInput.Length; i++)
{
var result = network.Compute(trainInput[i]);
for (int j = 0; j < result.Length; j++)
{
sw.Write(Convert.ToString(result[j]) + "\t");
}
sw.WriteLine();
}
sw.WriteLine("\n误差=" + Convert.ToString(error));
sw.Close();
}
public void Learn(double[][] input, double[][] output)
{
BackPropagationLearning teacher = new BackPropagationLearning(network);
double error = 1;
int k = input.Length;
double prerr = 0;
while (k >= input.Length / 100)
{
// run epoch of learning procedure
error = teacher.RunEpoch(input, output);
prerr = error;
k = 0;
for (int i = 0; i < input.Length; i++)
{
var c = network.Compute(input[i]);
if (c[0] > c[1] & output[i][0] < output[i][1] || c[0] < c[1] & output[i][0] > output[i][1])
{
k++;
}
/*
* if (Math.Round(c[0]) != output[i][0])
* {
*
* }*/
}
Console.WriteLine(k + " " + error + " " + teacher.LearningRate);
//c = network.Compute(input[1000]);
// check error value to see if we need to stop
// ...
}
}
private void train_Click(object sender, EventArgs e)
{
double[][] input = new double[Data.getInstance().images.Count()][];
double[][] output = new double[Data.getInstance().images.Count()][];
int max = Data.getInstance().classes.Count() - 1;
int min = 0;
for (int i = 0; i < Data.getInstance().images.Count(); i++)
{
Bitmap img = Data.getInstance().preprocessing(Data.getInstance().images[i]);
ImageToArray converter = new ImageToArray(0, 1);
converter.Convert(img, out input[i]);
output[i] = new double[1];
output[i][0] = Data.getInstance().classindex[i];
output[i][0] = 0 + (output[i][0] - min) * (1 - 0) / (max - min);
}
bpnn_net = new ActivationNetwork(new SigmoidFunction(), 100, 4, 1);
bpnn_learn = new BackPropagationLearning(bpnn_net);
double max_iter = 1000000;
double max_err = 0.000001;
for (double i = 0; i < max_iter; i++)
{
double err = bpnn_learn.RunEpoch(input, output);
if (err < max_err)
{
break;
}
}
predict.Enabled = true;
MessageBox.Show("Computing BPNN Success");
}
static ActivationNetwork GetNetwork(double[][] input, double[][] output)
{
ActivationNetwork net = new ActivationNetwork(new SigmoidFunction(), 18, 12, 8, 4, 2);
BackPropagationLearning trainer = new BackPropagationLearning(net);
// Переменная, сохраняющая значение ошибки сети на предыдущем шаге
double early = 1E6;
// Ошибка сети
double error = 1E3;
// Сначала скорость обучения должна быть высока
trainer.LearningRate = 1;
// Обучаем сеть пока ошибка сети станет небольшой
while (error > 0.01)
{
// Получаем ошибку сети
error = trainer.RunEpoch(input, output);
// Если ошибка сети изменилась на небольшое значения, в сравнении ошибкой предыдущей эпохи
if (Math.Abs(error - early) < 1E-9)
{
// Уменьшаем коэффициент скорости обучения на 2
trainer.LearningRate /= 2;
if (trainer.LearningRate < 0.001)
{
trainer.LearningRate = 0.001;
}
}
Console.WriteLine(error + " / " + trainer.LearningRate);
early = error;
}
return(net);
}
public void Init()
{
var dataset = CreateDataSet();
int counter = 0;
needToStop = false;
// initialize input and output values
double[][] input = new double[dataSetArrayCount][];
double[][] output = new double[dataSetArrayCount][];
for (int i = 0; i < dataset.Count; i++)
{
input[i] = new double[6];
input[i][0] = dataset[i].Erytrocyt;
input[i][1] = dataset[i].Fibrinogen;
input[i][2] = dataset[i].Hemocyt;
input[i][3] = dataset[i].Leukocyt;
input[i][4] = dataset[i].Protrombin;
input[i][5] = dataset[i].Trombocyt;
output[i] = new double[2];
output[i][0] = dataset[i].Result1;
output[i][1] = dataset[i].Result2;
}
SigmoidFunction sigmoidFunction = new SigmoidFunction(5);
// create neural network
ActivationNetwork network = new ActivationNetwork(
sigmoidFunction,
6, // two inputs in the network
5, // two neurons in the first layer
2); // one neuron in the second layer
// create teacher
network.Randomize();
BackPropagationLearning teacher =
new BackPropagationLearning(network);
teacher.LearningRate = 5;
// loop
while (!needToStop)
{
// run epoch of learning procedure
double error = teacher.RunEpoch(input, output);
// check error value to see if we need to stop
// ...
counter++;
if (counter == 2000)
{
Console.WriteLine(error);
counter = 0;
}
//if (error < 200)
if (error < 0.05)
{
needToStop = true;
}
}
BloodDonatonNeuralNet bloodDonatonNeuralNet = new BloodDonatonNeuralNet();
bloodDonatonNeuralNet.NeuralNetwork = network;
AssetManager assets = this.Assets;
IFormatter formatter;
using (MemoryStream stream = new MemoryStream())
{
formatter = new BinaryFormatter();
formatter.Serialize(stream, bloodDonatonNeuralNet);
var FullFilePath = System.IO.Path.Combine(System.Environment.GetFolderPath(System.Environment.SpecialFolder.Personal), filename);
stream.Seek(0, SeekOrigin.Begin);
using (FileStream fs = new FileStream(FullFilePath, FileMode.OpenOrCreate))
{
stream.CopyTo(fs);
fs.Flush();
}
}
}
static void Main(string[] args)
{
int trainNum = 75;
int testNum = 75;
double [][] trainInput = new double[trainNum][];
double[][] trainOutput = new double[trainNum][];
double [] max = new double [4];
double [] min = new double [4];
for (int i = 0; i < 4; ++i)
{
max[i] = double.MinValue;
min[i] = double.MaxValue;
}
// 读取训练数据
StreamReader reader = new StreamReader("trainData.txt");
for (int i = 0; i < trainNum; ++i)
{
string value = reader.ReadLine();
string[] temp = value.Split('\t');
trainInput[i] = new double[4];
trainOutput[i] = new double[3];
for (int j = 0; j < 4; ++j)
{
trainInput[i][j] = double.Parse(temp[j]);
if (trainInput[i][j] > max[j])
{
max[j] = trainInput[i][j];
}
if (trainInput[i][j] < min[j])
{
min[j] = trainInput[i][j];
}
}
for (int j = 0; j < 3; ++j)
{
trainOutput[i][j] = 0;
}
trainOutput[i][int.Parse(temp[4]) - 1] = 1;
}
// 归一化
for (int i = 0; i < trainNum; ++i)
{
for (int j = 0; j < 4; ++j)
{
trainInput[i][j] = premnmx(trainInput[i][j], min[j], max[j]);
}
}
//训练网络
// create multi-layer neural network
ActivationNetwork network = new ActivationNetwork(new SigmoidFunction(3), 4, 5, 3);
// create teacher
BackPropagationLearning teacher = new BackPropagationLearning(network);
// set learning rate and momentum
teacher.LearningRate = 0.1;
teacher.Momentum = 0;
int iteration = 1;
while (iteration < 500)
{
teacher.RunEpoch(trainInput, trainOutput);
++iteration;
}
// 读取测试数据
double [][] testInput = new double [testNum][];
double [] testOutput = new double [testNum];
StreamReader reader1 = new StreamReader("testData.txt");
for (int i = 0; i < testNum; ++i)
{
string value = reader1.ReadLine();
string[] temp = value.Split('\t');
testInput[i] = new double[4];
for (int j = 0; j < 4; ++j)
{
testInput[i][j] = double.Parse(temp[j]);
}
testOutput[i] = int.Parse(temp[4]);
}
// 对测试数据进行分类， 并统计正确率
int hitNum = 0;
for (int i = 0; i < testNum; ++i)
{
double[] inp = new double[4] {
testInput[i][0], testInput[i][1], testInput[i][2], testInput[i][3]
};
for (int j = 0; j < 4; ++j)
{
inp[j] = premnmx(inp[j], min[j], max[j]);
}
double[] t = { inp[0], inp[1], inp[2], inp[3] };
// 使用网络对训练样本计算输出
double[] result = network.Compute(t);
int m = 0;
for (int j = 0; j < 3; ++j)
{
if (result[j] > result[m])
{
m = j;
}
}
if (m + 1 == testOutput[i])
{
++hitNum;
}
}
System.Console.WriteLine("分类正确率为 {0}% ", 100.0 * hitNum / testNum);
System.Console.Read();
}
void findSolution()
{
// Ilość danych zbioru uczącego
int quantitySamples = inputData.Length - amountOfOmittedEndsPoints - amountOfOmittedInitialPoints;
// Różnica pomiędzy wartością największą a najmniejszą ze zbioru uczącego
double diffrenceMinMaxValue = chart.RangeY.Length;
// Współczynnik transformacji danych - factor - wartość oszacowania odchylenia standardowego populacji dla całkowitej zmienności
// Jest on użyty po to, żeby przy normalizacji danych nie przekroczyć zakresu -1 do 1
double dtc = 1.7 / diffrenceMinMaxValue;
//Najmniejsza wartość zbioru uczącego
double yMin = chart.RangeY.Min;
// Przygotowanie danych uczących
double[][] inputTmpData = new double[quantitySamples][];
double[][] outputTmpData = new double[quantitySamples][];
// Normalizacja danych z zakresu -1 do 1
for (int i = 0; i < quantitySamples; i++)
{
// Ustawienie nowych danych, nie włączając w to danych pominiętych przez linie ograniczające,
// przypisanie danych do tablicy outputTmpData
inputTmpData[i] = new double[amountOfOmittedInitialPoints];
outputTmpData[i] = new double[1];
// Ustawienie danych wejściowych - normalizacja danych
for (int j = 0; j < amountOfOmittedInitialPoints; j++)
{
inputTmpData[i][j] = (inputData[i + j] - yMin) * dtc - 0.85;
}
// Ustawienie danych końcowych
outputTmpData[i][0] = (inputData[i + amountOfOmittedInitialPoints] - yMin) * dtc - 0.85;
}
createMultiLayerNeuralNetwork(out network, out teacher);
int iteration = 1;
// Tablice rozwiązań
int solutionSize = inputData.Length - amountOfOmittedInitialPoints;
double[,] solution = new double[solutionSize, 2];
double[] networkInput = new double[amountOfOmittedInitialPoints];
// Obliczanie wartosci, które będą użyte do funkcji rozwiązującej - do przypisania danych dla tablicy estymowanej
for (int j = 0; j < solutionSize; j++)
{
solution[j, 0] = j + amountOfOmittedInitialPoints;
}
// Pętla rozpoczynająca uczenie
while (!needToStop)
{
// Uruchamiamy epokę procedury uczenia
double error = teacher.RunEpoch(inputTmpData, outputTmpData) / quantitySamples;
// Obliczanie wyniku oraz błędu uczenia i błędu predykcji
double learningError = 0.0;
double predictionError = 0.0;
for (int i = 0, n = inputData.Length - amountOfOmittedInitialPoints; i < n; i++)
{
// Wstawianie wartości bieżących(dla wartości od lini progowej poczatkowej) podlegających normalizacji
for (int j = 0; j < amountOfOmittedInitialPoints; j++)
{
networkInput[j] = (inputData[i + j] - yMin) * dtc - 0.85;
}
// Ocenianie funkcji
solution[i, 1] = (network.Compute(networkInput)[0] + 0.85) / dtc + yMin;
//Obliczanie błędu predykcji
if (i >= n - amountOfOmittedEndsPoints)
{
predictionError += Math.Abs(solution[i, 1] - inputData[amountOfOmittedInitialPoints + i]);
}
else
{
learningError += Math.Abs(solution[i, 1] - inputData[amountOfOmittedInitialPoints + i]);
}
}
// Aktualizowanie wykresu
chart.UpdateDataSeries("solution", solution);
// Aktualizowanie danych - iteracji, błędu uczenia oraz blędu predykcji w czasie iteracji od 1 do 1000
updateText(textBox5, iteration.ToString());
updateText(textBox6, learningError.ToString("F3"));
updateText(textBox7, predictionError.ToString("F3"));
iteration++;
// Sprawdzenie czy trzeba się zatrzymać
if ((iterations != 0) && (iteration > iterations))
{
break;
}
}
// Dodanie wyników do estymowanej kolumny wynikowej
for (int j = amountOfOmittedInitialPoints, k = 0, n = inputData.Length; j < n; j++, k++)
{
addSolutionToEstimatedColumn(listView1, j, solution[k, 1].ToString());
}
}
public void ExampleTest1()
{
Accord.Math.Tools.SetupGenerator(0);
// We'll use a simple XOR function as input.
double[][] inputs =
{
new double[] { 0, 0 }, // 0 xor 0
new double[] { 0, 1 }, // 0 xor 1
new double[] { 1, 0 }, // 1 xor 0
new double[] { 1, 1 }, // 1 xor 1
};
// XOR output, corresponding to the input.
double[][] outputs =
{
new double[] { 0 }, // 0 xor 0 = 0
new double[] { 1 }, // 0 xor 1 = 1
new double[] { 1 }, // 1 xor 0 = 1
new double[] { 0 }, // 1 xor 1 = 0
};
// Setup the deep belief network (2 inputs, 3 hidden, 1 output)
DeepBeliefNetwork network = new DeepBeliefNetwork(2, 3, 1);
// Initialize the network with Gaussian weights
new GaussianWeights(network, 0.1).Randomize();
// Update the visible layer with the new weights
network.UpdateVisibleWeights();
// Setup the learning algorithm.
DeepBeliefNetworkLearning teacher = new DeepBeliefNetworkLearning(network)
{
Algorithm = (h, v, i) => new ContrastiveDivergenceLearning(h, v)
{
LearningRate = 0.1,
Momentum = 0.5,
Decay = 0.001,
}
};
// Unsupervised learning on each hidden layer, except for the output.
for (int i = 0; i < network.Layers.Length - 1; i++)
{
teacher.LayerIndex = i;
// Compute the learning data with should be used
var layerInput = teacher.GetLayerInput(inputs);
// Train the layer iteratively
for (int j = 0; j < 5000; j++)
{
teacher.RunEpoch(layerInput);
}
}
// Supervised learning on entire network, to provide output classification.
var backpropagation = new BackPropagationLearning(network)
{
LearningRate = 0.1,
Momentum = 0.5
};
// Run supervised learning.
for (int i = 0; i < 5000; i++)
{
backpropagation.RunEpoch(inputs, outputs);
}
// Test the resulting accuracy.
int correct = 0;
for (int i = 0; i < inputs.Length; i++)
{
double[] outputValues = network.Compute(inputs[i]);
double outputResult = outputValues.First() >= 0.5 ? 1 : 0;
if (outputResult == outputs[i].First())
{
correct++;
}
}
Assert.AreEqual(4, correct);
}
private void backgroundWorker_DoWork(object sender, DoWorkEventArgs e)
{
TrainingThreadParameters parameters = (TrainingThreadParameters)e.Argument;
TrainingOptions options = parameters.Options;
double[][] inputs = parameters.Inputs;
double[][] outputs = parameters.Outputs;
//Create Teacher
BackPropagationLearning networkTeacher = new BackPropagationLearning(activationNetwork);
networkTeacher.LearningRate = options.LearningRate1;
networkTeacher.Momentum = options.Momentum;
//Start Training
bool stop = false;
bool complete = false;
int lastStatusEpoch = 0;
int lastSaveEpoch = 0;
int lastValidateEpoch = 0;
backgroundWorker.ReportProgress(0, BackgroundWorkerUpdate.Starting);
#region Main Training Loop
while (!stop)
{
// Run training epoch
status.TrainingError = networkTeacher.RunEpoch(inputs, outputs);
if (options.Validate && status.Epoch >= lastValidateEpoch + options.ValidateEpochs)
{
status.ValidationError = networkTeacher.MeasureEpochError(inputs, outputs);
}
// Adjust Training Rate
if (options.ChangeLearningRate)
{
networkTeacher.LearningRate = options.LearningRate2;
}
// Register Savepoint
if (options.MarkSavepoints &&
status.Epoch >= lastSaveEpoch + options.MarkSavepointsEpochs)
{
backgroundWorker.ReportProgress(0, BackgroundWorkerUpdate.NetworkSave);
lastSaveEpoch = status.Epoch;
}
// Progress Indicator
if (options.ReportProgress &&
status.Epoch >= lastStatusEpoch + options.ReportProgressEpochs)
{
if (options.Limit == TrainingOptions.TrainingLimit.ByError)
{
if (status.TrainingError != 0) // Check to avoid division by zero
{
status.Progress = Math.Max(Math.Min((int)((options.LimitByError * 100) / status.TrainingError), 100), 0);
}
}
else if (options.Limit == TrainingOptions.TrainingLimit.ByEpoch)
{
if (status.Epoch != 0) // Check to avoid division by zero
{
status.Progress = Math.Max(Math.Min((int)((status.Epoch * 100) / options.LimitByEpochs), 100), 0);
}
}
backgroundWorker.ReportProgress(0, BackgroundWorkerUpdate.Progress);
lastStatusEpoch = status.Epoch;
}
// Increment epoch counter
++status.Epoch;
if (options.Delay)
{
// Sleep thread according to specified delay
System.Threading.Thread.Sleep(options.DelayMilliseconds);
}
// Check stop conditions
if ((options.Limit == TrainingOptions.TrainingLimit.ByError &&
status.TrainingError <= options.LimitByError) ||
(options.Limit == TrainingOptions.TrainingLimit.ByEpoch &&
status.Epoch >= options.LimitByEpochs))
{
complete = true;
stop = true;
}
// Check for cancellation requests
if (backgroundWorker.CancellationPending)
{
e.Cancel = true;
stop = true;
}
while (state == SessionState.Paused)
{
System.Threading.Thread.Sleep(1000);
}
} // End main training loop
#endregion
// Do a last update before exiting the thread
if (complete)
{
backgroundWorker.ReportProgress(0, BackgroundWorkerUpdate.Completed);
}
else
{
backgroundWorker.ReportProgress(0, BackgroundWorkerUpdate.Progress);
}
}
public void Train(double[][] trainInput, double[][] trainOutput)
{
teacher.RunEpoch(trainInput, trainOutput);
}
// Worker thread
void SearchSolution()
{
// number of learning samples
var samples = trainData.GetLength(0);
// data normalization factors
const int numerator = 2;
for (var i = 0; i <= numberOfInputs; i++)
{
var factor = numerator / (maxes[i] - mins[i]);
factors[i] = float.IsPositiveInfinity(factor) || float.IsNegativeInfinity(factor) ? 1 : numerator / (maxes[i] - mins[i]);
}
var input = new double[samples][];
var output = new double[samples][];
for (var sample = 0; sample < samples; sample++)
{
input[sample] = new double[numberOfInputs];
output[sample] = new double[numberOfOutputs];
for (var i = 0; i < numberOfInputs; i++)
{
input[sample][i] = normalizeInput(trainData[sample, i], i);
}
output[sample] = normalizeOutput(sample);
}
network = new ActivationNetwork(
new BipolarSigmoidFunction(sigmoidAlphaValue),
numberOfInputs,
neuronsInLayers.ToArray());
teacher = new BackPropagationLearning(network)
{
LearningRate = learningRate, Momentum = momentum
};
var iteration = 1;
trainingSetError = new List <double>();
//////////////////////////////////////////////////////////////////////////////////////////////////////////
var periods = PeriodicAdd(gamma);
while (!needToStop)
{
if (dynamicAdding && periods.Contains(iteration))
{
((ActivationLayer)network.Layers[0]).AddNeuron(network);
}
var error = teacher.RunEpoch(input, output) / samples;
trainingSetError.Add(error);
iteration++;
if (((iterations == 0) || (iteration <= iterations)) && !(error < eps))
{
continue;
}
elapsedIterations = iteration - 1;
break;
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////
totalConnections = network.Layers.Select(x => x.InputsCount * x.Neurons.Length).Sum();
if (dynamicPruning)
{
Pruning(input, output);
}
TextWriter twError = new StreamWriter(_dir + "\\classification_error.csv");
var nfi = new NumberFormatInfo {
NumberDecimalSeparator = "."
};
for (var i = 0; i < trainingSetError.Count; i++)
{
twError.WriteLine(i + "," + trainingSetError[i].ToString(nfi));
}
twError.Close();
}
/// <summary>
/// The main application entry point.
/// </summary>
/// <param name="args">Command line arguments.</param>
public static void Main(string[] args)
{
// read data
Console.WriteLine("Loading data....");
var path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..");
path = Path.Combine(path, "..");
path = Path.Combine(path, "handwritten_digits_medium.csv");
var digits = Frame.ReadCsv(path, separators: ",", hasHeaders: false);
Console.WriteLine($" {digits.RowCount} rows loaded");
// normalize pixel values to 0..1
for (var i = 2; i <= 785; i++)
{
digits[$"Column{i}"] /= 255.0;
}
// grab a random digit
var rnd = new Random();
var row = rnd.Next(1, digits.RowCount);
var label = digits.Rows[row]["Column1"].ToString();
// plot the digit
var x = Enumerable.Range(0, 784).Select(v => (double)(v % 28));
var y = Enumerable.Range(0, 784).Select(v => (double)(-v / 28));
var z = from i in Enumerable.Range(2, 784)
let v = (double)digits.Rows[row][$"Column{i}"]
select v > 0.5 ? 1 : 0;
Scatterplot plot = new Scatterplot($"Digit {label}", "x", "y");
plot.Compute(x.ToArray(), y.ToArray(), z.ToArray());
ScatterplotBox.Show(plot);
// create one-hot label columns
for (var i = 0; i < 10; i++)
{
digits.AddColumn($"Label{i}", from v in digits["Column1"].Values select(int) v == i ? 1.0 : 0.0);
}
// print label columns
digits.Columns[new string[] { "Column1", "Label0", "Label1", "Label2", "Label3", "Label4",
"Label5", "Label6", "Label7", "Label8", "Label9" }].Print();
// create training and validation partitions
var numRows = digits.RowKeys.Count();
var pivot = (int)(numRows * 0.8);
var training = digits.Rows[Enumerable.Range(0, pivot)];
var validation = digits.Rows[Enumerable.Range(pivot, numRows - pivot)];
// set up feature and label column names
var featureColumns = Enumerable.Range(2, 784).Select(v => $"Column{v}").ToArray();
var labelColumns = Enumerable.Range(0, 10).Select(v => $"Label{v}").ToArray();
// set up feature and label arrays
var features = training.Columns[featureColumns].ToArray2D <double>().ToJagged();
var labels = training.Columns[labelColumns].ToArray2D <double>().ToJagged();
// build a neural network
var network = new ActivationNetwork(
new SigmoidFunction(),
784,
100,
100,
10);
// randomize network weights
new GaussianWeights(network, 0.1).Randomize();
// set up a backpropagation learner
var learner = new BackPropagationLearning(network)
{
LearningRate = 0.05
};
// train the network and validate it in each epoch
Console.WriteLine("Training neural network....");
var errors = new List <double>();
var validationErrors = new List <double>();
for (var epoch = 0; epoch < 50; epoch++)
{
var error = learner.RunEpoch(features, labels) / labels.GetLength(0);
var validationError = Validate(validation, network);
errors.Add(error);
validationErrors.Add(validationError);
Console.WriteLine($"Epoch: {epoch}, Training error: {error}, Validation error: {validationError}");
}
// test the network on the validation data
Console.WriteLine($"Validating neural network on {validation.RowCount} records....");
int mistakes = 0;
foreach (var key in validation.RowKeys)
{
var record = validation.Rows[key];
var digit = (int)record.Values.First();
var input = record.Values.Skip(1).Take(784).Cast <double>();
var predictions = network.Compute(input.ToArray());
// Console.Write($" {digit}: {predictions.ToString("0.00")} ");
// calculate best prediction
var best = Enumerable.Range(0, 10)
.Select(v => new { Digit = v, Prediction = predictions[v] })
.OrderByDescending(v => v.Prediction)
.First();
//Console.Write($" -> {digit} = {best.Digit} ({100 * best.Prediction:0.00}%) ");
// count incorrect predictions
if (best.Digit != digit)
{
Console.Write($" {digit}: {predictions.ToString("0.00")} ");
Console.WriteLine($" -> {digit} = {best.Digit} ({100 * best.Prediction:0.00}%) WRONG");
//Console.Write("WRONG");
mistakes++;
}
//Console.WriteLine();
}
// report total mistakes
var accuracy = 100.0 * (validation.Rows.KeyCount - mistakes) / validation.Rows.KeyCount;
Console.WriteLine($"Total mistakes: {mistakes}, Accuracy: {accuracy:0.00}%");
// plot the training and validation curves
var tmp = Enumerable.Range(1, 50).Select(v => (double)v);
x = tmp.Concat(tmp);
y = errors.Concat(validationErrors);
z = Enumerable.Repeat(1, 50).Concat(Enumerable.Repeat(2, 50));
plot = new Scatterplot("Training & validation curves", "epochs", "training error");
plot.Compute(x.ToArray(), y.ToArray(), z.ToArray());
ScatterplotBox.Show(plot);
Console.ReadLine();
}
// Worker thread
void SearchSolution( )
{
// number of learning samples
var samples = this.data.Length - this.predictionSize - this.windowSize;
// data transformation factor
var factor = 1.7 / this.chart.RangeY.Length;
var yMin = this.chart.RangeY.Min;
// prepare learning data
var input = new double[samples][];
var output = new double[samples][];
for (var i = 0; i < samples; i++)
{
input[i] = new double[this.windowSize];
output[i] = new double[1];
// set input
for (var j = 0; j < this.windowSize; j++)
{
input[i][j] = (this.data[i + j] - yMin) * factor - 0.85;
}
// set output
output[i][0] = (this.data[i + this.windowSize] - yMin) * factor - 0.85;
}
// create multi-layer neural network
var network = new ActivationNetwork(
new BipolarSigmoidFunction(this.sigmoidAlphaValue), this.windowSize, this.windowSize * 2, 1);
// create teacher
var teacher = new BackPropagationLearning(network);
// set learning rate and momentum
teacher.LearningRate = this.learningRate;
teacher.Momentum = this.momentum;
// iterations
var iteration = 1;
// solution array
var solutionSize = this.data.Length - this.windowSize;
var solution = new double[solutionSize, 2];
var networkInput = new double[this.windowSize];
// calculate X values to be used with solution function
for (var j = 0; j < solutionSize; j++)
{
solution[j, 0] = j + this.windowSize;
}
// loop
while (!this.needToStop)
{
// run epoch of learning procedure
var error = teacher.RunEpoch(input, output) / samples;
// calculate solution and learning and prediction errors
var learningError = 0.0;
var predictionError = 0.0;
// go through all the data
for (int i = 0, n = this.data.Length - this.windowSize; i < n; i++)
{
// put values from current window as network's input
for (var j = 0; j < this.windowSize; j++)
{
networkInput[j] = (this.data[i + j] - yMin) * factor - 0.85;
}
// evalue the function
solution[i, 1] = (network.Compute(networkInput)[0] + 0.85) / factor + yMin;
// calculate prediction error
if (i >= n - this.predictionSize)
{
predictionError += Math.Abs(solution[i, 1] - this.data[this.windowSize + i]);
}
else
{
learningError += Math.Abs(solution[i, 1] - this.data[this.windowSize + i]);
}
}
// update solution on the chart
this.chart.UpdateDataSeries("solution", solution);
// set current iteration's info
this.currentIterationBox.Text = iteration.ToString( );
this.currentLearningErrorBox.Text = learningError.ToString("F3");
this.currentPredictionErrorBox.Text = predictionError.ToString("F3");
// increase current iteration
iteration++;
// check if we need to stop
if ((this.iterations != 0) && (iteration > this.iterations))
{
break;
}
}
// show new solution
for (int j = this.windowSize, k = 0, n = this.data.Length; j < n; j++, k++)
{
this.dataList.Items[j].SubItems.Add(solution[k, 1].ToString( ));
}
// enable settings controls
EnableControls(true);
}
static void Main(string[] args)
{
double learningRate = 0.1;
double sigmoidAlphaValue = 2;
// iterations
int iterations = 100;
bool useNguyenWidrow = false;
var X_train = LoadData("X_train.csv");
var y_train = LoadData("y_train.csv");
var X_val = LoadData("X_val.csv");
var y_val = LoadData("y_val.csv");
var X_holdout = LoadData("X_holdout.csv");
var y_holdout = LoadData("y_holdout.csv");
int nb_samples = X_train.Length;
int input_size = X_train[0].Length;
// create multi-layer neural network
ActivationNetwork ann = new ActivationNetwork(
new BipolarSigmoidFunction(sigmoidAlphaValue),
input_size,
input_size, 1);
if (useNguyenWidrow)
{
NguyenWidrow initializer = new NguyenWidrow(ann);
initializer.Randomize();
}
// create teacher
//LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(ann, useRegularization);
BackPropagationLearning teacher = new BackPropagationLearning(ann);
// set learning rate and momentum
teacher.LearningRate = learningRate;
teacher.Momentum = 0.8;
// iterations
int iteration = 1;
//var ranges = Matrix.Range(sourceMatrix, 0);
//double[][] map = Matrix.Mesh(ranges[0], ranges[1], 0.05, 0.05);
// var sw = Stopwatch.StartNew();
bool use_batches = false;
int batch_size = 5000;
double[][] X_batch = new double[batch_size][];
double[][] y_batch = new double[batch_size][];
var rng = new Random();
while (true)
{
double error = 0.0;
if (use_batches)
{
int[] sample_indeces = Enumerable.Range(0, nb_samples).OrderBy(z => rng.Next()).ToArray();
int nb_batch = nb_samples / batch_size;
int n_grad = 50;
Console.Write("|{0}| error=\r", new string('-', n_grad));
for (int ibatch = 0; ibatch < nb_batch; ++ibatch)
{
for (int i = 0; i < batch_size; ++i)
{
X_batch[i] = X_train[sample_indeces[i]];
y_batch[i] = y_train[sample_indeces[i]];
}
error += teacher.RunEpoch(X_batch, y_batch);
int ngrad1 = (int)Math.Ceiling((ibatch + 1) * n_grad / (double)(nb_batch));
int ngrad0 = n_grad - ngrad1;
double cur_err = error / (batch_size * (ibatch + 1));
Console.Write("|{0}{1}| error={2:0.####}\r", new string('#', ngrad1), new string('-', ngrad0), cur_err);
}
error /= (batch_size * nb_batch);
}
else
{
error = teacher.RunEpoch(X_train, y_train) / nb_samples;
}
// Получим оценку точности на валидационном наборе
int n_hit2 = 0;
for (int i = 0; i < X_val.Length; ++i)
{
double[] output = ann.Compute(X_val[i]);
double y_bool = output[0] > 0.5 ? 1.0 : 0.0;
n_hit2 += y_bool == y_val[i][0] ? 1 : 0;
}
double val_acc = n_hit2 / (double)X_val.Length;
// TODO: тут надо бы смотреть, увеличилось ли качество сетки на валидации по сравнению с предыдущей эпохой,
// сохранять веса сетки в случае улучшения (через deep copy ann), и делать остановку в случае, если качество
// не растет уже 5-10 эпох.
Console.WriteLine($"\niteration={iteration} train_mse={error} val_acc={val_acc}");
iteration++;
if ((iterations != 0) && (iteration > iterations))
{
break;
}
}
// Получим оценку точности на отложенном наборе
int n_hit = 0;
for (int i = 0; i < X_holdout.Length; ++i)
{
double[] output = ann.Compute(X_holdout[i]);
double y_bool = output[0] > 0.5 ? 1.0 : 0.0;
n_hit += y_bool == y_holdout[i][0] ? 1 : 0;
}
double holdout_acc = n_hit / (double)X_holdout.Length;
Console.WriteLine($"holdout_acc={holdout_acc}");
return;
}
static void Main(string[] args)
{
double[][] inputs;
double[][] outputs;
double[][] testInputs;
double[][] testOutputs;
// Load ascii digits dataset.
inputs = DataManager.Load(@"../../../data/data.txt", out outputs);
// The first 500 data rows will be for training. The rest will be for testing.
testInputs = inputs.Skip(500).ToArray();
testOutputs = outputs.Skip(500).ToArray();
inputs = inputs.Take(500).ToArray();
outputs = outputs.Take(500).ToArray();
// Setup the deep belief network and initialize with random weights.
DeepBeliefNetwork network = new DeepBeliefNetwork(inputs.First().Length, 10, 10);
new GaussianWeights(network, 0.1).Randomize();
network.UpdateVisibleWeights();
// Setup the learning algorithm.
DeepBeliefNetworkLearning teacher = new DeepBeliefNetworkLearning(network)
{
Algorithm = (h, v, i) => new ContrastiveDivergenceLearning(h, v)
{
LearningRate = 0.1,
Momentum = 0.5,
Decay = 0.001,
}
};
// Setup batches of input for learning.
int batchCount = Math.Max(1, inputs.Length / 100);
// Create mini-batches to speed learning.
int[] groups = Accord.Statistics.Tools.RandomGroups(inputs.Length, batchCount);
double[][][] batches = inputs.Subgroups(groups);
// Learning data for the specified layer.
double[][][] layerData;
// Unsupervised learning on each hidden layer, except for the output layer.
for (int layerIndex = 0; layerIndex < network.Machines.Count - 1; layerIndex++)
{
teacher.LayerIndex = layerIndex;
layerData = teacher.GetLayerInput(batches);
for (int i = 0; i < 200; i++)
{
double error = teacher.RunEpoch(layerData) / inputs.Length;
if (i % 10 == 0)
{
Console.WriteLine(i + ", Error = " + error);
}
}
}
// Supervised learning on entire network, to provide output classification.
var teacher2 = new BackPropagationLearning(network)
{
LearningRate = 0.1,
Momentum = 0.5
};
// Run supervised learning.
for (int i = 0; i < 500; i++)
{
double error = teacher2.RunEpoch(inputs, outputs) / inputs.Length;
if (i % 10 == 0)
{
Console.WriteLine(i + ", Error = " + error);
}
}
// Test the resulting accuracy.
int correct = 0;
for (int i = 0; i < inputs.Length; i++)
{
double[] outputValues = network.Compute(testInputs[i]);
if (DataManager.FormatOutputResult(outputValues) == DataManager.FormatOutputResult(testOutputs[i]))
{
correct++;
}
}
Console.WriteLine("Correct " + correct + "/" + inputs.Length + ", " + Math.Round(((double)correct / (double)inputs.Length * 100), 2) + "%");
Console.Write("Press any key to quit ..");
Console.ReadKey();
}
// Worker thread
void SearchSolution()
{
// initialize input and output values
double[][] input = null;
double[][] output = null;
if (sigmoidType == 0)
{
// unipolar data
input = new double[4][] {
new double[] { 0, 0 },
new double[] { 0, 1 },
new double[] { 1, 0 },
new double[] { 1, 1 }
};
output = new double[4][] {
new double[] { 0 },
new double[] { 1 },
new double[] { 1 },
new double[] { 0 }
};
}
else
{
// biipolar data
input = new double[4][] {
new double[] { -1, -1 },
new double[] { -1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
output = new double[4][] {
new double[] { -1 },
new double[] { 1 },
new double[] { 1 },
new double[] { -1 }
};
}
// create neural network
var network = new ActivationNetwork(
(sigmoidType == 0) ?
new SigmoidFunction(sigmoidAlphaValue) :
(IActivationFunction) new BipolarSigmoidFunction(sigmoidAlphaValue),
2, 2, 1);
// create teacher
var teacher = new BackPropagationLearning(network)
{
// set learning rate and momentum
LearningRate = learningRate,
Momentum = momentum
};
// iterations
var iteration = 1;
// statistic files
StreamWriter errorsFile = null;
try
{
// check if we need to save statistics to files
if (saveStatisticsToFiles)
{
// open files
errorsFile = File.CreateText("errors.csv");
}
// erros list
var errorsList = new ArrayList();
// loop
while (!needToStop)
{
// run epoch of learning procedure
var error = teacher.RunEpoch(input, output);
errorsList.Add(error);
// save current error
if (errorsFile != null)
{
errorsFile.WriteLine(error);
}
// show current iteration & error
SetText(currentIterationBox, iteration.ToString());
SetText(currentErrorBox, error.ToString());
iteration++;
// check if we need to stop
if (error <= learningErrorLimit)
{
break;
}
}
// show error's dynamics
var errors = new double[errorsList.Count, 2];
for (int i = 0, n = errorsList.Count; i < n; i++)
{
errors[i, 0] = i;
errors[i, 1] = (double)errorsList[i];
}
errorChart.RangeX = new Range(0, errorsList.Count - 1);
errorChart.UpdateDataSeries("error", errors);
}
catch (IOException)
{
MessageBox.Show("Failed writing file", "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
finally
{
// close files
if (errorsFile != null)
{
errorsFile.Close();
}
}
// enable settings controls
EnableControls(true);
}
// Worker thread
void SearchSolution()
{
// number of learning samples
int samples = data.GetLength(0);
// prepare learning data
//80% training, 20% for validate data(to do 70% lear., 20% validate, 10% test)
double[][] input = new double[samples * 4 / 5][];
double[][] output = new double[samples * 4 / 5][];
double[][] validateInput = new double[samples / 5][];
double[][] validateOutput = new double[samples / 5][];
// create multi-layer neural network
if (alpha > 0.0)
{
activationFunc = new SigmoidFunction(alpha);
}
else
{
activationFunc = new SigmoidFunction();
}
int k = 0;
int j = 0;
for (int i = 1; i < samples; i++)
{
//80% training, 20% for validate data(to do 70% lear., 20% validate, 10% test)
if ((i % 5) == 0) // validate input 20 %
{
validateInput[k] = new double[colCountData - 1];
validateOutput[k] = new double[1];
for (int c = 0; c < colCountData - 1; c++)
{
validateInput[k][c] = data[i, c];
}
validateOutput[k][0] = data[i, colCountData - 1];
k++;
}
else //forward input 80 %
{
// input data
input[j] = new double[colCountData - 1];
for (int c = 0; c < colCountData - 1; c++)
{
input[j][c] = data[i, c];
}
//output data
output[j] = new double[1];
output[j][0] = data[i, colCountData - 1];
j++;
}
}
network = new ActivationNetwork(activationFunc,
colCountData - 1, neuronsAndLayers);
// create teacher
teacherBack = new BackPropagationLearning(network);
// set learning rate and momentum
teacherBack.LearningRate = learningRate;
teacherBack.Momentum = momentum;
// iterations
int iteration = 1;
double error = 0.0;
double validateError = 0.0;
//int j = 0;
// erros list
ArrayList errorsList = new ArrayList();
ArrayList validateList = new ArrayList();
// loop
while (!needToStop)
{
// run epoch of learning procedure
error = teacherBack.RunEpoch(input, output);
if (errorsList.Count - 1 >= 2000)
{
errorsList.RemoveAt(0);
errorsList[0] = 5.0;
}
errorsList.Add(error);
validateError = 0.0;
for (int count = 0; count < validateInput.GetLength(0) - 1; count++)
{
validateError += network.Compute(validateInput[count])[0] - validateOutput[count][0];
}
if (validateList.Count - 1 >= 2000)
{
validateList.RemoveAt(0);
validateList[0] = 5.0;
}
validateList.Add(validateError);
// set current iteration's info
currentIterationBox.Invoke(new Action <string>((s) => currentIterationBox.Text = s), iteration.ToString());
errorPercent.Invoke(new Action <string>((s) => errorPercent.Text = s), error.ToString("F14"));
validErrorBox.Invoke(new Action <string>((s) => validErrorBox.Text = s), (validateError / 1000).ToString("F14"));
// show error's dynamics
double[,] errors = new double[errorsList.Count, 2];
double[,] valid = new double[validateList.Count, 2];
double[,] zeros = new double[errorsList.Count, 2];
for (int i = 0, n = errorsList.Count; i < n; i++)
{
errors[i, 0] = i;
zeros[i, 0] = i;
errors[i, 1] = (double)errorsList[i];
zeros[i, 1] = 0.0;
}
for (int i = 0, n = validateList.Count; i < n; i++)
{
valid[i, 0] = i;
valid[i, 1] = (double)validateList[i];
}
errorChart.RangeX = new Range(0, errorsList.Count - 1);
errorChart.UpdateDataSeries("error", errors);
errorChart.UpdateDataSeries("zero", zeros);
validChart.RangeX = new Range(0, validateList.Count - 1);
validChart.UpdateDataSeries("validate", valid);
validChart.UpdateDataSeries("zero", zeros);
// increase current iteration
iteration++;
}
// enable settings controls
EnableControls(true);
}
protected virtual void LearningIteration()
{
teacher.RunEpoch(Inputs, Answers);
}
public double[] Run(int[] numOfHiddenNeurals)
{
// validation and testing error
double[] errors = new double[2];
// number of learning samples
int samples = _data.Length - _windowSize;
int trainingSamples = samples - _predictionSize;
// prepare learning data
double[][] input = new double[samples][];
double[][] output = new double[samples][];
// sample indices
int[] indices = new int[samples];
int[] trainingIndices = new int[trainingSamples];
// normalization function
var normalizeFunc = new MinMaxNormalization(_xMax, _xMin, _data.Max(), _data.Min());
for (int i = 0; i < samples; i++)
{
input[i] = new double[_windowSize];
output[i] = new double[_outputNum];
// set input
for (int j = 0; j < _windowSize; j++)
{
input[i][j] = normalizeFunc.Compute(_data[i + j]);
}
// set output
for (int j = 0; j < _outputNum; j++)
{
output[i][j] = normalizeFunc.Compute(_data[i + _windowSize + j]);
}
indices[i] = i;
}
// randomize the sample indices
Utils.Shuffle<int>(indices);
output.Swap(indices);
input.Swap(indices);
// get training samples
double[][] trainingInput = new double[trainingSamples][];
double[][] trainingOutput = new double[trainingSamples][];
for (int i = 0; i < trainingSamples; i++)
{
trainingInput[i] = new double[_windowSize];
trainingOutput[i] = new double[_outputNum];
// set input
for (int j = 0; j < _windowSize; j++)
{
trainingInput[i][j] = input[i][j];
}
// set output
for (int j = 0; j < _outputNum; j++)
{
trainingOutput[i][j] = output[i][j];
}
trainingIndices[i] = i;
}
//// randomize the sample indices
//Utils.Shuffle<int>(trainingIndices);
//trainingOutput.Swap(trainingIndices);
//trainingInput.Swap(trainingIndices);
// create multi-layer neural network
int[] neuronsCount = numOfHiddenNeurals.Concat(new int[] { _outputNum }).ToArray();
ActivationNetwork network = new ActivationNetwork(
_function,
_windowSize, neuronsCount);
// create teacher
BackPropagationLearning teacher = new BackPropagationLearning(network);
// set learning rate and momentum
teacher.LearningRate = _learningRate;
teacher.Momentum = 0.0;
//var teacher = new ParallelResilientBackpropagationLearning(network);
//teacher.Reset(_learningRate);
// iterations
int iteration = 1;
// solution array
int solutionSize = _data.Length - _windowSize;
double[,] solution = new double[solutionSize, 1 + _outputNum];
// calculate X values to be used with solution function
for (int j = 0; j < solutionSize; j++)
{
solution[j, 0] = j + _windowSize;
}
// loop
var needToStop = false;
while (!needToStop)
{
// run epoch of learning procedure
double error = teacher.RunEpoch(trainingInput, trainingOutput) / trainingSamples;
// calculate solution and learning and prediction errors every 5
double learningError = 0.0;
double predictionError = 0.0;
if (iteration % 5 == 0)
{
// go through all the data
for (int i = 0; i < samples; i++)
{
double y = 0.0;
double o = 0.0;
double err = 0.0;
for (int j = 0; j < _outputNum; j++)
{
y = output[i][j];
o = network.Compute(input[i])[j];
err += (o - y) * (o - y) / _outputNum;
// evalue the function
solution[i, j + 1] = normalizeFunc.Inverse(o);
}
// calculate prediction error (MSE)
if (i >= trainingSamples)
{
predictionError += err;// Math.Pow((solution[i, 1] - normalizeFunc.Inverse(output[i][0])), 2.0);
}
else
{
learningError += err;
}
}
}
// Adaptive Learning - decrease the learning rate
// n(t) = n0 * a^(t/T)
// a = 1/10^x, x >= 1
teacher.LearningRate = _learningRate * Math.Pow(0.1, iteration / _iterations);
// increase iteration
iteration++;
// check if we need to stop
if ((_iterations != 0) && (iteration > _iterations))
{
errors[0] = learningError / trainingSamples;
errors[1] = predictionError / _predictionSize;
Console.WriteLine("Final Learning MSE Error: " + errors[0]);
Console.WriteLine("Final Prediction MSE Error: " + errors[1]);
Console.WriteLine("Final Learning Rate: " + teacher.LearningRate);
Console.WriteLine("Window Size: " + _windowSize + "\n"
+ "Number of Hidden Neurons: " + neuronsCount[0] + "\n"
+ "Output Size: " + _outputNum);
break;
}
}
////print result to file
//Console.WriteLine("Real Values\t\tRegression Values");
//for (int i = samples - 1; i >= trainingSamples; --i)
//{
// Console.WriteLine(normalizeFunc.Inverse(output[i][0]) + "\t\t" + solution[i, 1]);
//}
return errors;
}
static void Main(string[] args)
{
Console.OutputEncoding = Encoding.UTF8;
Console.WriteLine(@"Загрузка данных для обучения и проверки...");
Frame <int, string> training = DataService.GetTrainingData();
Frame <int, string> validation = DataService.GetValidationData(500);
Console.WriteLine(@"Проводится нормализация пикселей...");
NormalizePixels(training);
NormalizePixels(validation);
Console.WriteLine(@"Кодирование цифр в обучающей и проверочной выборках...");
EncodeDigit(training);
EncodeDigit(validation);
// Инициализация нейронной сети
ActivationNetwork network = new ActivationNetwork(
new SigmoidFunction(), // сигмоидная активационная функция
InputNeuronsCount, // количество входных нейронов (784)
FirstHiddenLayerNeuronsCount, // количество нейронов в первом скрытом слое (100)
SecondHiddenLayerNeuronsCount, // количество нейронов во втором скрытом слое (100)
OutputNeuronsCount); // количество нейронов в выходном слое (10)
// Данный класс отвечает за реализацию алгоритма обратного распространения ошибки
BackPropagationLearning learner = new BackPropagationLearning(network)
{
LearningRate = 0.1 // Начальная скорость обучения
};
// Производим инициализацию весов сети при помощи распределения Гаусса
new GaussianWeights(network).Randomize();
List <double> errors = new List <double>();
List <double> validationErrors = new List <double>();
double[][] features = training.Columns[FeatureColumns].ToArray2D <double>().ToJagged();
double[][] labels = training.Columns[LabelColumns].ToArray2D <double>().ToJagged();
Console.WriteLine($@"Запущен процесс обучения нейронной сети, количество эпох: {EpochCount}...");
for (int i = 0; i < EpochCount; i++)
{
double error = learner.RunEpoch(features, labels) / labels.GetLength(0);
errors.Add(error); // сохраняем ошибку для построения графика
// Обучаем сеть методом обратного распространения ошибки
// и, для каждой эпохи, получаем ошибку обучения
double validationError = NetworkValidationService
.Validate(network, // Нейронная сеть
validation, // данные для тестирования
labels); // колонки с данными
// сохраняем квадратичную ошибку для построения графика
validationErrors.Add(validationError);
Console.WriteLine($@"{DateTime.Now} Эпоха: {i},
Ошибка обучения: {error},
Ошибка тестирования: {validationError}");
}
PlotService.PlotValidationCurve(validationErrors, EpochCount);
PlotService.PlotTrainingCurve(errors, EpochCount);
PrintValidationReport(network, NetworkValidationService, training);
Console.Write("Сохраняем состояние обученной сети на диск... ");
DataService.SaveNetworkState(network);
Console.ReadKey();
}
private void learnNetworkSupervised()
{
if (!Main.CanClassify) return;
Dispatcher dispatcher = Dispatcher.CurrentDispatcher;
new Task(() =>
{
var teacher = new BackPropagationLearning(Main.Network)
{
LearningRate = LearningRate,
Momentum = Momentum
};
double[][] inputs, outputs;
Main.Database.Training.GetInstances(out inputs, out outputs);
// Start running the learning procedure
for (int i = 0; i < Epochs && !shouldStop; i++)
{
double error = teacher.RunEpoch(inputs, outputs);
dispatcher.BeginInvoke((Action<int, double>)updateError,
DispatcherPriority.ContextIdle, i + 1, error);
}
Main.Network.UpdateVisibleWeights();
IsLearning = false;
}).Start();
}
private void Pruning(double[][] input, double[][] output)
{
var samples = input.Length;
var keepPruning = true;
Network checkpoint = null;
var pruned = 0;
while (keepPruning)
{
using (var clonestream = new MemoryStream())
{
var trainError = trainingSetError.Last();
var formatter = new BinaryFormatter();
formatter.Serialize(clonestream, network);
clonestream.Position = 0;
checkpoint = (Network)formatter.Deserialize(clonestream);
var min = Double.PositiveInfinity;
var minpos = new[] { 0, 0, 0 };
var originalPruning = true;
var minRatio = network.Layers.SelectMany(x => x.neurons).Min(y => y.minRatio());
if (minRatio <= 0.3)
{
originalPruning = false;
}
else
{
keepPruning = false;
}
originalPruning = true;
for (var i = 0; i < network.Layers.Length; i++)
{
var layer = network.Layers[i];
for (var j = 0; j < layer.neurons.Length; j++)
{
var neuron = layer.neurons[j];
if (originalPruning)
{
for (var k = 0; k < neuron.weights.Length; k++)
{
var weight = neuron.weights[k];
if (neuron.freezes[k] == 0 || Math.Abs(min) < Math.Abs(weight))
{
continue;
}
min = weight;
minpos = new[] { i, j, k };
}
}
else
{
var ratio = neuron.minRatio();
if (ratio > min)
{
continue;
}
min = ratio;
minpos = new[] { i, j };
}
}
}
if (originalPruning)
{
var minneuron = network.Layers[minpos[0]].neurons[minpos[1]];
minneuron.Freeze(minpos[2]);
}
else
{
network.Layers[minpos[0]].neurons[minpos[1]].FreezeMinRatio();
}
var iteration = 1;
var trainingSetErrorPrune = new List <double>();
while (!needToStop)
{
var error = teacher.RunEpoch(input, output) / samples;
trainingSetErrorPrune.Add(error);
iteration++;
if ((iteration < iterations / 10) && (error > trainError))
{
continue;
}
if (iteration >= iterations / 10 && error > trainError)
{
keepPruning = false;
}
else
{
trainingSetError.AddRange(trainingSetErrorPrune);
pruned++;
}
break;
}
}
}
var percentage = (Decimal.Parse(pruned.ToString()) / Decimal.Parse(totalConnections.ToString())).ToString("P", CultureInfo.InvariantCulture);
///////////////////////////////////// MESSAGE BOX /////////////////////////////////////////////////////////////////////////////
/* MessageBox.Show("Pruned " + pruned + " out of " + totalConnections + " connections" +
* " ( " + percentage + " ) in a total of: " + trainingSetError.Count + " iterations.");*/
raports.Add(new Raport {
connections = totalConnections, error = trainingSetError.Last(), iterations = trainingSetError.Count, pruned = pruned
});
network = (ActivationNetwork)checkpoint;
}
// Worker thread
public void SearchSolution()
{
// prepare learning data
double[][] input = new double[samples][];
double[][] hidden = new double[samples][];
double[][] output = new double[samples][];
for (int i = 0; i < samples; i++)
{
input[i] = new double[variables];
hidden[i] = new double[hneuron];
output[i] = new double[1];
// copy input
for (int j = 0; j < variables; j++)
{
input[i][j] = data[i, j];
}
// copy output
output[i][0] = classes[i];
}
// create backpropagation
ActivationNetwork network = new ActivationNetwork(new SigmoidFunction(2), variables, hneuron, 1);
ActivationNeuron neuron = network[0][0];
ActivationLayer layer = network[0];
// create teacher
BackPropagationLearning teacher = new BackPropagationLearning(network);
// set learning rate
teacher.LearningRate = learningRate;
// iterations
int iteration = 1;
// statistic files
StreamWriter errorsFile = null;
StreamWriter weightsFile = null;
try
{
// check if we need to save statistics to files
if (saveStatisticsToFiles)
{
// open files
errorsFile = File.CreateText("C:\\aaaaaa\\program\\s\\error.csv");
weightsFile = File.CreateText("C:\\aaaaaa\\program\\s\\weights.csv");
}
// erros list
ArrayList errorsList = new ArrayList();
// loop
while (!needToStop)
{
// run epoch of learning procedure
double error = teacher.RunEpoch(input, output);
errorsList.Add(error);
// save current weights
if (weightsFile != null)
{
weightsFile.Write("\n" + "iterasi" + iteration);
for (int i = 0; i < 1; i++)
{
weightsFile.Write("\n" + "nilai bobot-HO = " + layer[i].Threshold);
for (int j = 0; j < hneuron; j++)
{
weightsFile.Write("\n" + "nilai bobot-IH neuron " + j + "=" + layer[j].Threshold);
weightsFile.Write("\n" + "IHneuron" + j + "," + "HOneuron" + j);
for (int k = 0; k < variables; k++)
{
weightsFile.Write("\n" + layer[j][k] + "," + layer[i][j]);
}
}
}
}
// save current error
if (errorsFile != null)
{
errorsFile.WriteLine(error);
}
// show current iteration
SetText(textBox3, iteration.ToString());
SetText(textBox11, error.ToString());
// stop if no error
if (error < maxLearningError || iteration >= maxepoch)
{
break;
}
iteration++;
}
// show Backpropagation's weights
ClearList(weightsList);
for (int i = 0; i < 1; i++)
{
string neuro = string.Format("neuron {0}", i + 1);
ListViewItem item = null;
for (int j = 0; j < hneuron; j++)
{
item = AddListItem(weightsList, neuro);
AddListSubitem(item, string.Format("HOWeight {0}", j + 1));
AddListSubitem(item, layer[i][j].ToString("F6"));
for (int k = 0; k < variables; k++)
{
item = AddListItem(weightsList, neuro);
AddListSubitem(item, string.Format("IHWeight {0}", k + 1));
AddListSubitem(item, layer[j][k].ToString("F6"));
}
item = AddListItem(weightsList, neuro);
AddListSubitem(item, string.Format("IHthreshold{0}", j + 1));
AddListSubitem(item, layer[j].Threshold.ToString("F6"));
}
item = AddListItem(weightsList, neuro);
AddListSubitem(item, string.Format("HOthreshold{0}", i + 1));
AddListSubitem(item, layer[i].Threshold.ToString("F6"));
}
network.Save("D:\\My Documents\\Ihsan's Document\\skripsi\\s\\JST\\net_latih.net");
}
catch (IOException)
{
MessageBox.Show("Failed writing file", "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
finally
{
// close files
if (errorsFile != null)
{
errorsFile.Close();
}
if (weightsFile != null)
{
weightsFile.Close();
}
}
// enable settings controls
EnableControls(true);
}