public void ExampleTest1() { Accord.Math.Tools.SetupGenerator(0); // We'll use a simple XOR function as input. double[][] inputs = { new double[] { 0, 0 }, // 0 xor 0 new double[] { 0, 1 }, // 0 xor 1 new double[] { 1, 0 }, // 1 xor 0 new double[] { 1, 1 }, // 1 xor 1 }; // XOR output, corresponding to the input. double[][] outputs = { new double[] { 0 }, // 0 xor 0 = 0 new double[] { 1 }, // 0 xor 1 = 1 new double[] { 1 }, // 1 xor 0 = 1 new double[] { 0 }, // 1 xor 1 = 0 }; // Setup the deep belief network (2 inputs, 3 hidden, 1 output) DeepBeliefNetwork network = new DeepBeliefNetwork(2, 3, 1); // Initialize the network with Gaussian weights new GaussianWeights(network, 0.1).Randomize(); // Update the visible layer with the new weights network.UpdateVisibleWeights(); // Setup the learning algorithm. DeepBeliefNetworkLearning teacher = new DeepBeliefNetworkLearning(network) { Algorithm = (h, v, i) => new ContrastiveDivergenceLearning(h, v) { LearningRate = 0.1, Momentum = 0.5, Decay = 0.001, } }; // Unsupervised learning on each hidden layer, except for the output. for (int i = 0; i < network.Layers.Length - 1; i++) { teacher.LayerIndex = i; // Compute the learning data with should be used var layerInput = teacher.GetLayerInput(inputs); // Train the layer iteratively for (int j = 0; j < 5000; j++) teacher.RunEpoch(layerInput); } // Supervised learning on entire network, to provide output classification. var backpropagation = new BackPropagationLearning(network) { LearningRate = 0.1, Momentum = 0.5 }; // Run supervised learning. for (int i = 0; i < 5000; i++) backpropagation.RunEpoch(inputs, outputs); // Test the resulting accuracy. int correct = 0; for (int i = 0; i < inputs.Length; i++) { double[] outputValues = network.Compute(inputs[i]); double outputResult = outputValues.First() >= 0.5 ? 1 : 0; if (outputResult == outputs[i].First()) { correct++; } } Assert.AreEqual(4, correct); }
private void button1_Click(object sender, EventArgs e) { needToStop = false; workThread = new Thread(() => { // Learning while (!needToStop) { var error = teacher.RunEpoch(sampleStorage.InputLearn, sampleStorage.OutPutLearn); errorVal = error.ToString(CultureInfo.InvariantCulture); if (error < 0.1) { needToStop = true; } } }); workThread.Priority = ThreadPriority.Highest; workThread.Start(); }
private void RunEpoch(BackPropagationLearning teacher, double[][] input, double[][] output, bool isParallel) { if (isParallel) { var data = input.Zip(output, (n, w) => new { singleInput = n, singleOutput = w }); //var q = from v in data.AsParallel() // select teacher.Run(v.singleInput, v.singleOutput); //q.ToArray(); Parallel.ForEach(data, v => { teacher.Run(v.singleInput, v.singleOutput); }); } else { teacher.RunEpoch(input, output); } }
public void bpnnTraining() { double[][] input_data = new double[Data.instance.images.Count][]; double[][] output_data = new double[Data.instance.indexClasses.Count][]; int max_output = Data.instance.classes.Count - 1, min_output = 0; for (int i = 0; i < Data.instance.images.Count; i++) { //Pilih gambar berwarna Bitmap image = new Bitmap(Data.instance.images[i]); //Preprocess jadi 10 x 10 hitam putih image = Data.instance.preProcess(image); // dari pixel 0-255 jadi 0-1 ImageToArray converter = new ImageToArray(0, 1); converter.Convert(image, out input_data[i]); output_data[i] = new double[1]; output_data[i][0] = Data.instance.indexClasses[i]; output_data[i][0] = 0 + (output_data[i][0] - min_output) * (1 - 0) / (max_output - min_output); } bpnnNetwork = new ActivationNetwork(new SigmoidFunction(), 100, 3, 1); bpnnLearning = new BackPropagationLearning(bpnnNetwork); //o Error Goals: 0.000001 //o Max Epochs: 1000000 int max_iter = 1000000; double max_error = 0.000001; for (int i = 0; i < max_iter; i++) { double error = bpnnLearning.RunEpoch(input_data, output_data); if (error < max_error) { break; } } bpnnNetwork.Save("an.bin"); }
public static ActivationNetwork CreateANN(double[][] input, double[][] output, double learningRate) { var watch = System.Diagnostics.Stopwatch.StartNew(); int nrOfOutputs = output[0].Length; int nrOfInputs = input[0].Length; int[] layers = { 4, 4, nrOfOutputs }; //Neurons per layer ActivationNetwork network = new ActivationNetwork( new SigmoidFunction(), //Activation function 2, //Input layers); //Hidden layers + output BackPropagationLearning teacher = new BackPropagationLearning(network); teacher.LearningRate = learningRate; bool needToStop = false; int epochs = 0; double prevError = 999999.9; while (!needToStop) { double error = teacher.RunEpoch(input, output); if (Math.Abs(prevError - error) < 0.0000001 || epochs == 10000) { needToStop = true; watch.Stop(); var elapsedMs = watch.ElapsedMilliseconds; Console.WriteLine("\n" + 2 + ", " + nrOfOutputs + " Stop at: " + epochs + " Error: " + error + " Time: " + elapsedMs / 1000 + "s"); } if (epochs % 1000 == 0 && epochs != 0) { Console.Write("."); } prevError = error; epochs++; } return(network); }
public void Learn(ref ActivationNetwork network, double[] crossInput, double[] circleInput, double[] blankInput) { NetworkInput networkInput = new NetworkInput(crossInput, circleInput, blankInput); double[][] input = networkInput.Input; NetworkOutput networkOutput = new NetworkOutput(0); double[][] output = networkOutput.Output; BackPropagationLearning teacher = new BackPropagationLearning(network); for (int i = 0; i < 100000; i++) { double error = teacher.RunEpoch(input, output); } network.Save("Net.bin"); MessageBox.Show("Learning finished"); }
public void Ucz() //algorytm uczacy siec neuronowa { int NumberOfImputs = 6; int Classnes = 11; int NumberOfHiddenNeurons = 6; network = new ActivationNetwork(new BipolarSigmoidFunction(2), NumberOfImputs, NumberOfHiddenNeurons, Classnes); var teacher = new BackPropagationLearning(network); double error = 1; double[][] input = odczytDanychIN(); int[] label = odczytDanychOUT(); double[][] output = Jagged.OneHot(label, Classnes); for (int i = 0; i < 10000; i++) { error = teacher.RunEpoch(input, output); } label2.Text = error.ToString(); }
public NeuralNetwork Train(BackProbArgs X) { if (!Info.Moduls.Contains("NeuralNetworks/Learning/BackProb.cs")) { Info.Moduls.Add("NeuralNetworks/Learning/BackProb.cs"); } BackPropagationLearning Trainer = new BackPropagationLearning(X.NetworkToTrain.ANetwork); Trainer.Momentum = (X.Momentem <= 0) ? (3) : (X.Momentem); Trainer.LearningRate = (X.LearningRate <= 0) ? (0.01) : (X.LearningRate); uint tick = Settings.LearningUpdateTick; while (true) { double Error = Trainer.RunEpoch(X.Inputs, X.Outputs); if (Error <= X.WantedERROR) { break; } if (tick == 0) { Settings.OnUpdateEvent(this, new LearningUpdateArgs() { ERROR = Error, Inputs = X.Inputs, Outputs = X.Outputs, NetworkState = X.NetworkToTrain, }); tick = Settings.LearningUpdateTick; } else { tick--; } } return(X.NetworkToTrain); }
private void SearchSolution(out BackPropagationLearning teacher, double[][] input, double[][] output) { int gen = 0; double error = 0; ActivationNetwork network = new ActivationNetwork( new BipolarSigmoidFunction(2), Input_layer, Hidden_layer, Output_layer); teacher = new BackPropagationLearning(network); int iteration = 1; error = 1.0; while (error > 0.001) { error = teacher.RunEpoch(input, output) / OBJECTNUM; iteration++; if ((iterations != 0) && (iteration > iterations)) { break; } } }
public void TeachNetworkByEpochs(List <List <double> > Input, List <int> Output, int Epochs) { var teacher = new BackPropagationLearning(network) { LearningRate = 0.1f, Momentum = 0.9f }; /*var teacher = new Accord.Neuro.Learning.DeepNeuralNetworkLearning(network) * { * Algorithm = (ann, i) => new ParallelResilientBackpropagationLearning(ann), * LayerIndex = network.Layers.Length - 1, * };*/ double[][] inputs, outputs; //Main.Database.Training.GetInstances(out inputs, out outputs); inputs = new double[Input.Count][]; outputs = new double[Output.Count][]; for (int i = 0; i < Input.Count; i++) { inputs[i] = Input[i].ToArray(); outputs[i] = new double[6] { 0, 0, 0, 0, 0, 0 }; outputs[i][Output[i]] = 1; } // Start running the learning procedure for (int i = 0; i < Epochs; i++) { //teacher.Run(Input.ToArray(), Output.ToArray()); double error = teacher.RunEpoch(inputs, outputs); } //network.UpdateVisibleWeights(); }
public void test() { // initialize input and output values var input = new double[4][] { new double[] { 0, 0 }, new double[] { 0, 1 }, new double[] { 1, 0 }, new double[] { 1, 1 } }; var output = new double[4][] { new double[] { 0 }, new double[] { 1 }, new double[] { 1 }, new double[] { 1 } }; // create neural network var network = new ActivationNetwork( new SigmoidFunction(2), 2, // two inputs in the network //2, // two neurons in the first layer 1); // one neuron in the second layer // create teacher var teacher = new BackPropagationLearning(network); // loop while (true) { // run epoch of learning procedure var error = teacher.RunEpoch(input, output); // check error value to see if we need to stop // ... if (error < 0.001) { break; } } Console.WriteLine(network.Compute(new double[] { 0, 0 })[0] + "," + network.Compute(new double[] { 0, 1 })[0] + "," + network.Compute(new double[] { 1, 0 })[0] + "," + network.Compute(new double[] { 1, 1 })[0]); }
public void TrainBPNet(double [][] trainInput, double [][] trainOutput)//建立BP神经网络,以获得适应度函数 { Console.WriteLine("神经网络训练开始:\n"); ActivationNetwork network = new ActivationNetwork(new SigmoidFunction(1.5), InputDimension, HiddenDimension, OutputDimension);//定义网络 BackPropagationLearning teacher = new BackPropagationLearning(network); teacher.LearningRate = 0.1; teacher.Momentum = 0; int iteration = 1; double error = 1.0; Console.WriteLine("神经网络训练中...\n"); while (iteration < BP_Iter_Max) { error = teacher.RunEpoch(trainInput, trainOutput); iteration++; } Console.WriteLine("训练完毕,神经网络权值保存在\\bin\\" + SaveNetFile + "中\n"); SaveNet(network, SaveNetFile); Console.WriteLine("训练结果已保存在神经网络训练结果.xls中!\n"); IO.WriteTitleToXls("神经网络训练结果.xls", IO.TTitle); StreamWriter sw = new StreamWriter("神经网络训练结果.xls", true); for (int i = 0; i < trainInput.Length; i++) { var result = network.Compute(trainInput[i]); for (int j = 0; j < result.Length; j++) { sw.Write(Convert.ToString(result[j]) + "\t"); } sw.WriteLine(); } sw.WriteLine("\n误差=" + Convert.ToString(error)); sw.Close(); }
public void Learn(double[][] input, double[][] output) { BackPropagationLearning teacher = new BackPropagationLearning(network); double error = 1; int k = input.Length; double prerr = 0; while (k >= input.Length / 100) { // run epoch of learning procedure error = teacher.RunEpoch(input, output); prerr = error; k = 0; for (int i = 0; i < input.Length; i++) { var c = network.Compute(input[i]); if (c[0] > c[1] & output[i][0] < output[i][1] || c[0] < c[1] & output[i][0] > output[i][1]) { k++; } /* * if (Math.Round(c[0]) != output[i][0]) * { * * }*/ } Console.WriteLine(k + " " + error + " " + teacher.LearningRate); //c = network.Compute(input[1000]); // check error value to see if we need to stop // ... } }
private void train_Click(object sender, EventArgs e) { double[][] input = new double[Data.getInstance().images.Count()][]; double[][] output = new double[Data.getInstance().images.Count()][]; int max = Data.getInstance().classes.Count() - 1; int min = 0; for (int i = 0; i < Data.getInstance().images.Count(); i++) { Bitmap img = Data.getInstance().preprocessing(Data.getInstance().images[i]); ImageToArray converter = new ImageToArray(0, 1); converter.Convert(img, out input[i]); output[i] = new double[1]; output[i][0] = Data.getInstance().classindex[i]; output[i][0] = 0 + (output[i][0] - min) * (1 - 0) / (max - min); } bpnn_net = new ActivationNetwork(new SigmoidFunction(), 100, 4, 1); bpnn_learn = new BackPropagationLearning(bpnn_net); double max_iter = 1000000; double max_err = 0.000001; for (double i = 0; i < max_iter; i++) { double err = bpnn_learn.RunEpoch(input, output); if (err < max_err) { break; } } predict.Enabled = true; MessageBox.Show("Computing BPNN Success"); }
static ActivationNetwork GetNetwork(double[][] input, double[][] output) { ActivationNetwork net = new ActivationNetwork(new SigmoidFunction(), 18, 12, 8, 4, 2); BackPropagationLearning trainer = new BackPropagationLearning(net); // Переменная, сохраняющая значение ошибки сети на предыдущем шаге double early = 1E6; // Ошибка сети double error = 1E3; // Сначала скорость обучения должна быть высока trainer.LearningRate = 1; // Обучаем сеть пока ошибка сети станет небольшой while (error > 0.01) { // Получаем ошибку сети error = trainer.RunEpoch(input, output); // Если ошибка сети изменилась на небольшое значения, в сравнении ошибкой предыдущей эпохи if (Math.Abs(error - early) < 1E-9) { // Уменьшаем коэффициент скорости обучения на 2 trainer.LearningRate /= 2; if (trainer.LearningRate < 0.001) { trainer.LearningRate = 0.001; } } Console.WriteLine(error + " / " + trainer.LearningRate); early = error; } return(net); }
public void Init() { var dataset = CreateDataSet(); int counter = 0; needToStop = false; // initialize input and output values double[][] input = new double[dataSetArrayCount][]; double[][] output = new double[dataSetArrayCount][]; for (int i = 0; i < dataset.Count; i++) { input[i] = new double[6]; input[i][0] = dataset[i].Erytrocyt; input[i][1] = dataset[i].Fibrinogen; input[i][2] = dataset[i].Hemocyt; input[i][3] = dataset[i].Leukocyt; input[i][4] = dataset[i].Protrombin; input[i][5] = dataset[i].Trombocyt; output[i] = new double[2]; output[i][0] = dataset[i].Result1; output[i][1] = dataset[i].Result2; } SigmoidFunction sigmoidFunction = new SigmoidFunction(5); // create neural network ActivationNetwork network = new ActivationNetwork( sigmoidFunction, 6, // two inputs in the network 5, // two neurons in the first layer 2); // one neuron in the second layer // create teacher network.Randomize(); BackPropagationLearning teacher = new BackPropagationLearning(network); teacher.LearningRate = 5; // loop while (!needToStop) { // run epoch of learning procedure double error = teacher.RunEpoch(input, output); // check error value to see if we need to stop // ... counter++; if (counter == 2000) { Console.WriteLine(error); counter = 0; } //if (error < 200) if (error < 0.05) { needToStop = true; } } BloodDonatonNeuralNet bloodDonatonNeuralNet = new BloodDonatonNeuralNet(); bloodDonatonNeuralNet.NeuralNetwork = network; AssetManager assets = this.Assets; IFormatter formatter; using (MemoryStream stream = new MemoryStream()) { formatter = new BinaryFormatter(); formatter.Serialize(stream, bloodDonatonNeuralNet); var FullFilePath = System.IO.Path.Combine(System.Environment.GetFolderPath(System.Environment.SpecialFolder.Personal), filename); stream.Seek(0, SeekOrigin.Begin); using (FileStream fs = new FileStream(FullFilePath, FileMode.OpenOrCreate)) { stream.CopyTo(fs); fs.Flush(); } } }
static void Main(string[] args) { int trainNum = 75; int testNum = 75; double [][] trainInput = new double[trainNum][]; double[][] trainOutput = new double[trainNum][]; double [] max = new double [4]; double [] min = new double [4]; for (int i = 0; i < 4; ++i) { max[i] = double.MinValue; min[i] = double.MaxValue; } // 读取训练数据 StreamReader reader = new StreamReader("trainData.txt"); for (int i = 0; i < trainNum; ++i) { string value = reader.ReadLine(); string[] temp = value.Split('\t'); trainInput[i] = new double[4]; trainOutput[i] = new double[3]; for (int j = 0; j < 4; ++j) { trainInput[i][j] = double.Parse(temp[j]); if (trainInput[i][j] > max[j]) { max[j] = trainInput[i][j]; } if (trainInput[i][j] < min[j]) { min[j] = trainInput[i][j]; } } for (int j = 0; j < 3; ++j) { trainOutput[i][j] = 0; } trainOutput[i][int.Parse(temp[4]) - 1] = 1; } // 归一化 for (int i = 0; i < trainNum; ++i) { for (int j = 0; j < 4; ++j) { trainInput[i][j] = premnmx(trainInput[i][j], min[j], max[j]); } } //训练网络 // create multi-layer neural network ActivationNetwork network = new ActivationNetwork(new SigmoidFunction(3), 4, 5, 3); // create teacher BackPropagationLearning teacher = new BackPropagationLearning(network); // set learning rate and momentum teacher.LearningRate = 0.1; teacher.Momentum = 0; int iteration = 1; while (iteration < 500) { teacher.RunEpoch(trainInput, trainOutput); ++iteration; } // 读取测试数据 double [][] testInput = new double [testNum][]; double [] testOutput = new double [testNum]; StreamReader reader1 = new StreamReader("testData.txt"); for (int i = 0; i < testNum; ++i) { string value = reader1.ReadLine(); string[] temp = value.Split('\t'); testInput[i] = new double[4]; for (int j = 0; j < 4; ++j) { testInput[i][j] = double.Parse(temp[j]); } testOutput[i] = int.Parse(temp[4]); } // 对测试数据进行分类, 并统计正确率 int hitNum = 0; for (int i = 0; i < testNum; ++i) { double[] inp = new double[4] { testInput[i][0], testInput[i][1], testInput[i][2], testInput[i][3] }; for (int j = 0; j < 4; ++j) { inp[j] = premnmx(inp[j], min[j], max[j]); } double[] t = { inp[0], inp[1], inp[2], inp[3] }; // 使用网络对训练样本计算输出 double[] result = network.Compute(t); int m = 0; for (int j = 0; j < 3; ++j) { if (result[j] > result[m]) { m = j; } } if (m + 1 == testOutput[i]) { ++hitNum; } } System.Console.WriteLine("分类正确率为 {0}% ", 100.0 * hitNum / testNum); System.Console.Read(); }
void findSolution() { // Ilość danych zbioru uczącego int quantitySamples = inputData.Length - amountOfOmittedEndsPoints - amountOfOmittedInitialPoints; // Różnica pomiędzy wartością największą a najmniejszą ze zbioru uczącego double diffrenceMinMaxValue = chart.RangeY.Length; // Współczynnik transformacji danych - factor - wartość oszacowania odchylenia standardowego populacji dla całkowitej zmienności // Jest on użyty po to, żeby przy normalizacji danych nie przekroczyć zakresu -1 do 1 double dtc = 1.7 / diffrenceMinMaxValue; //Najmniejsza wartość zbioru uczącego double yMin = chart.RangeY.Min; // Przygotowanie danych uczących double[][] inputTmpData = new double[quantitySamples][]; double[][] outputTmpData = new double[quantitySamples][]; // Normalizacja danych z zakresu -1 do 1 for (int i = 0; i < quantitySamples; i++) { // Ustawienie nowych danych, nie włączając w to danych pominiętych przez linie ograniczające, // przypisanie danych do tablicy outputTmpData inputTmpData[i] = new double[amountOfOmittedInitialPoints]; outputTmpData[i] = new double[1]; // Ustawienie danych wejściowych - normalizacja danych for (int j = 0; j < amountOfOmittedInitialPoints; j++) { inputTmpData[i][j] = (inputData[i + j] - yMin) * dtc - 0.85; } // Ustawienie danych końcowych outputTmpData[i][0] = (inputData[i + amountOfOmittedInitialPoints] - yMin) * dtc - 0.85; } createMultiLayerNeuralNetwork(out network, out teacher); int iteration = 1; // Tablice rozwiązań int solutionSize = inputData.Length - amountOfOmittedInitialPoints; double[,] solution = new double[solutionSize, 2]; double[] networkInput = new double[amountOfOmittedInitialPoints]; // Obliczanie wartosci, które będą użyte do funkcji rozwiązującej - do przypisania danych dla tablicy estymowanej for (int j = 0; j < solutionSize; j++) { solution[j, 0] = j + amountOfOmittedInitialPoints; } // Pętla rozpoczynająca uczenie while (!needToStop) { // Uruchamiamy epokę procedury uczenia double error = teacher.RunEpoch(inputTmpData, outputTmpData) / quantitySamples; // Obliczanie wyniku oraz błędu uczenia i błędu predykcji double learningError = 0.0; double predictionError = 0.0; for (int i = 0, n = inputData.Length - amountOfOmittedInitialPoints; i < n; i++) { // Wstawianie wartości bieżących(dla wartości od lini progowej poczatkowej) podlegających normalizacji for (int j = 0; j < amountOfOmittedInitialPoints; j++) { networkInput[j] = (inputData[i + j] - yMin) * dtc - 0.85; } // Ocenianie funkcji solution[i, 1] = (network.Compute(networkInput)[0] + 0.85) / dtc + yMin; //Obliczanie błędu predykcji if (i >= n - amountOfOmittedEndsPoints) { predictionError += Math.Abs(solution[i, 1] - inputData[amountOfOmittedInitialPoints + i]); } else { learningError += Math.Abs(solution[i, 1] - inputData[amountOfOmittedInitialPoints + i]); } } // Aktualizowanie wykresu chart.UpdateDataSeries("solution", solution); // Aktualizowanie danych - iteracji, błędu uczenia oraz blędu predykcji w czasie iteracji od 1 do 1000 updateText(textBox5, iteration.ToString()); updateText(textBox6, learningError.ToString("F3")); updateText(textBox7, predictionError.ToString("F3")); iteration++; // Sprawdzenie czy trzeba się zatrzymać if ((iterations != 0) && (iteration > iterations)) { break; } } // Dodanie wyników do estymowanej kolumny wynikowej for (int j = amountOfOmittedInitialPoints, k = 0, n = inputData.Length; j < n; j++, k++) { addSolutionToEstimatedColumn(listView1, j, solution[k, 1].ToString()); } }
public void ExampleTest1() { Accord.Math.Tools.SetupGenerator(0); // We'll use a simple XOR function as input. double[][] inputs = { new double[] { 0, 0 }, // 0 xor 0 new double[] { 0, 1 }, // 0 xor 1 new double[] { 1, 0 }, // 1 xor 0 new double[] { 1, 1 }, // 1 xor 1 }; // XOR output, corresponding to the input. double[][] outputs = { new double[] { 0 }, // 0 xor 0 = 0 new double[] { 1 }, // 0 xor 1 = 1 new double[] { 1 }, // 1 xor 0 = 1 new double[] { 0 }, // 1 xor 1 = 0 }; // Setup the deep belief network (2 inputs, 3 hidden, 1 output) DeepBeliefNetwork network = new DeepBeliefNetwork(2, 3, 1); // Initialize the network with Gaussian weights new GaussianWeights(network, 0.1).Randomize(); // Update the visible layer with the new weights network.UpdateVisibleWeights(); // Setup the learning algorithm. DeepBeliefNetworkLearning teacher = new DeepBeliefNetworkLearning(network) { Algorithm = (h, v, i) => new ContrastiveDivergenceLearning(h, v) { LearningRate = 0.1, Momentum = 0.5, Decay = 0.001, } }; // Unsupervised learning on each hidden layer, except for the output. for (int i = 0; i < network.Layers.Length - 1; i++) { teacher.LayerIndex = i; // Compute the learning data with should be used var layerInput = teacher.GetLayerInput(inputs); // Train the layer iteratively for (int j = 0; j < 5000; j++) { teacher.RunEpoch(layerInput); } } // Supervised learning on entire network, to provide output classification. var backpropagation = new BackPropagationLearning(network) { LearningRate = 0.1, Momentum = 0.5 }; // Run supervised learning. for (int i = 0; i < 5000; i++) { backpropagation.RunEpoch(inputs, outputs); } // Test the resulting accuracy. int correct = 0; for (int i = 0; i < inputs.Length; i++) { double[] outputValues = network.Compute(inputs[i]); double outputResult = outputValues.First() >= 0.5 ? 1 : 0; if (outputResult == outputs[i].First()) { correct++; } } Assert.AreEqual(4, correct); }
private void backgroundWorker_DoWork(object sender, DoWorkEventArgs e) { TrainingThreadParameters parameters = (TrainingThreadParameters)e.Argument; TrainingOptions options = parameters.Options; double[][] inputs = parameters.Inputs; double[][] outputs = parameters.Outputs; //Create Teacher BackPropagationLearning networkTeacher = new BackPropagationLearning(activationNetwork); networkTeacher.LearningRate = options.LearningRate1; networkTeacher.Momentum = options.Momentum; //Start Training bool stop = false; bool complete = false; int lastStatusEpoch = 0; int lastSaveEpoch = 0; int lastValidateEpoch = 0; backgroundWorker.ReportProgress(0, BackgroundWorkerUpdate.Starting); #region Main Training Loop while (!stop) { // Run training epoch status.TrainingError = networkTeacher.RunEpoch(inputs, outputs); if (options.Validate && status.Epoch >= lastValidateEpoch + options.ValidateEpochs) { status.ValidationError = networkTeacher.MeasureEpochError(inputs, outputs); } // Adjust Training Rate if (options.ChangeLearningRate) { networkTeacher.LearningRate = options.LearningRate2; } // Register Savepoint if (options.MarkSavepoints && status.Epoch >= lastSaveEpoch + options.MarkSavepointsEpochs) { backgroundWorker.ReportProgress(0, BackgroundWorkerUpdate.NetworkSave); lastSaveEpoch = status.Epoch; } // Progress Indicator if (options.ReportProgress && status.Epoch >= lastStatusEpoch + options.ReportProgressEpochs) { if (options.Limit == TrainingOptions.TrainingLimit.ByError) { if (status.TrainingError != 0) // Check to avoid division by zero { status.Progress = Math.Max(Math.Min((int)((options.LimitByError * 100) / status.TrainingError), 100), 0); } } else if (options.Limit == TrainingOptions.TrainingLimit.ByEpoch) { if (status.Epoch != 0) // Check to avoid division by zero { status.Progress = Math.Max(Math.Min((int)((status.Epoch * 100) / options.LimitByEpochs), 100), 0); } } backgroundWorker.ReportProgress(0, BackgroundWorkerUpdate.Progress); lastStatusEpoch = status.Epoch; } // Increment epoch counter ++status.Epoch; if (options.Delay) { // Sleep thread according to specified delay System.Threading.Thread.Sleep(options.DelayMilliseconds); } // Check stop conditions if ((options.Limit == TrainingOptions.TrainingLimit.ByError && status.TrainingError <= options.LimitByError) || (options.Limit == TrainingOptions.TrainingLimit.ByEpoch && status.Epoch >= options.LimitByEpochs)) { complete = true; stop = true; } // Check for cancellation requests if (backgroundWorker.CancellationPending) { e.Cancel = true; stop = true; } while (state == SessionState.Paused) { System.Threading.Thread.Sleep(1000); } } // End main training loop #endregion // Do a last update before exiting the thread if (complete) { backgroundWorker.ReportProgress(0, BackgroundWorkerUpdate.Completed); } else { backgroundWorker.ReportProgress(0, BackgroundWorkerUpdate.Progress); } }
public void Train(double[][] trainInput, double[][] trainOutput) { teacher.RunEpoch(trainInput, trainOutput); }
// Worker thread void SearchSolution() { // number of learning samples var samples = trainData.GetLength(0); // data normalization factors const int numerator = 2; for (var i = 0; i <= numberOfInputs; i++) { var factor = numerator / (maxes[i] - mins[i]); factors[i] = float.IsPositiveInfinity(factor) || float.IsNegativeInfinity(factor) ? 1 : numerator / (maxes[i] - mins[i]); } var input = new double[samples][]; var output = new double[samples][]; for (var sample = 0; sample < samples; sample++) { input[sample] = new double[numberOfInputs]; output[sample] = new double[numberOfOutputs]; for (var i = 0; i < numberOfInputs; i++) { input[sample][i] = normalizeInput(trainData[sample, i], i); } output[sample] = normalizeOutput(sample); } network = new ActivationNetwork( new BipolarSigmoidFunction(sigmoidAlphaValue), numberOfInputs, neuronsInLayers.ToArray()); teacher = new BackPropagationLearning(network) { LearningRate = learningRate, Momentum = momentum }; var iteration = 1; trainingSetError = new List <double>(); ////////////////////////////////////////////////////////////////////////////////////////////////////////// var periods = PeriodicAdd(gamma); while (!needToStop) { if (dynamicAdding && periods.Contains(iteration)) { ((ActivationLayer)network.Layers[0]).AddNeuron(network); } var error = teacher.RunEpoch(input, output) / samples; trainingSetError.Add(error); iteration++; if (((iterations == 0) || (iteration <= iterations)) && !(error < eps)) { continue; } elapsedIterations = iteration - 1; break; } ////////////////////////////////////////////////////////////////////////////////////////////////////////// totalConnections = network.Layers.Select(x => x.InputsCount * x.Neurons.Length).Sum(); if (dynamicPruning) { Pruning(input, output); } TextWriter twError = new StreamWriter(_dir + "\\classification_error.csv"); var nfi = new NumberFormatInfo { NumberDecimalSeparator = "." }; for (var i = 0; i < trainingSetError.Count; i++) { twError.WriteLine(i + "," + trainingSetError[i].ToString(nfi)); } twError.Close(); }
/// <summary> /// The main application entry point. /// </summary> /// <param name="args">Command line arguments.</param> public static void Main(string[] args) { // read data Console.WriteLine("Loading data...."); var path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, ".."); path = Path.Combine(path, ".."); path = Path.Combine(path, "handwritten_digits_medium.csv"); var digits = Frame.ReadCsv(path, separators: ",", hasHeaders: false); Console.WriteLine($" {digits.RowCount} rows loaded"); // normalize pixel values to 0..1 for (var i = 2; i <= 785; i++) { digits[$"Column{i}"] /= 255.0; } // grab a random digit var rnd = new Random(); var row = rnd.Next(1, digits.RowCount); var label = digits.Rows[row]["Column1"].ToString(); // plot the digit var x = Enumerable.Range(0, 784).Select(v => (double)(v % 28)); var y = Enumerable.Range(0, 784).Select(v => (double)(-v / 28)); var z = from i in Enumerable.Range(2, 784) let v = (double)digits.Rows[row][$"Column{i}"] select v > 0.5 ? 1 : 0; Scatterplot plot = new Scatterplot($"Digit {label}", "x", "y"); plot.Compute(x.ToArray(), y.ToArray(), z.ToArray()); ScatterplotBox.Show(plot); // create one-hot label columns for (var i = 0; i < 10; i++) { digits.AddColumn($"Label{i}", from v in digits["Column1"].Values select(int) v == i ? 1.0 : 0.0); } // print label columns digits.Columns[new string[] { "Column1", "Label0", "Label1", "Label2", "Label3", "Label4", "Label5", "Label6", "Label7", "Label8", "Label9" }].Print(); // create training and validation partitions var numRows = digits.RowKeys.Count(); var pivot = (int)(numRows * 0.8); var training = digits.Rows[Enumerable.Range(0, pivot)]; var validation = digits.Rows[Enumerable.Range(pivot, numRows - pivot)]; // set up feature and label column names var featureColumns = Enumerable.Range(2, 784).Select(v => $"Column{v}").ToArray(); var labelColumns = Enumerable.Range(0, 10).Select(v => $"Label{v}").ToArray(); // set up feature and label arrays var features = training.Columns[featureColumns].ToArray2D <double>().ToJagged(); var labels = training.Columns[labelColumns].ToArray2D <double>().ToJagged(); // build a neural network var network = new ActivationNetwork( new SigmoidFunction(), 784, 100, 100, 10); // randomize network weights new GaussianWeights(network, 0.1).Randomize(); // set up a backpropagation learner var learner = new BackPropagationLearning(network) { LearningRate = 0.05 }; // train the network and validate it in each epoch Console.WriteLine("Training neural network...."); var errors = new List <double>(); var validationErrors = new List <double>(); for (var epoch = 0; epoch < 50; epoch++) { var error = learner.RunEpoch(features, labels) / labels.GetLength(0); var validationError = Validate(validation, network); errors.Add(error); validationErrors.Add(validationError); Console.WriteLine($"Epoch: {epoch}, Training error: {error}, Validation error: {validationError}"); } // test the network on the validation data Console.WriteLine($"Validating neural network on {validation.RowCount} records...."); int mistakes = 0; foreach (var key in validation.RowKeys) { var record = validation.Rows[key]; var digit = (int)record.Values.First(); var input = record.Values.Skip(1).Take(784).Cast <double>(); var predictions = network.Compute(input.ToArray()); // Console.Write($" {digit}: {predictions.ToString("0.00")} "); // calculate best prediction var best = Enumerable.Range(0, 10) .Select(v => new { Digit = v, Prediction = predictions[v] }) .OrderByDescending(v => v.Prediction) .First(); //Console.Write($" -> {digit} = {best.Digit} ({100 * best.Prediction:0.00}%) "); // count incorrect predictions if (best.Digit != digit) { Console.Write($" {digit}: {predictions.ToString("0.00")} "); Console.WriteLine($" -> {digit} = {best.Digit} ({100 * best.Prediction:0.00}%) WRONG"); //Console.Write("WRONG"); mistakes++; } //Console.WriteLine(); } // report total mistakes var accuracy = 100.0 * (validation.Rows.KeyCount - mistakes) / validation.Rows.KeyCount; Console.WriteLine($"Total mistakes: {mistakes}, Accuracy: {accuracy:0.00}%"); // plot the training and validation curves var tmp = Enumerable.Range(1, 50).Select(v => (double)v); x = tmp.Concat(tmp); y = errors.Concat(validationErrors); z = Enumerable.Repeat(1, 50).Concat(Enumerable.Repeat(2, 50)); plot = new Scatterplot("Training & validation curves", "epochs", "training error"); plot.Compute(x.ToArray(), y.ToArray(), z.ToArray()); ScatterplotBox.Show(plot); Console.ReadLine(); }
// Worker thread void SearchSolution( ) { // number of learning samples var samples = this.data.Length - this.predictionSize - this.windowSize; // data transformation factor var factor = 1.7 / this.chart.RangeY.Length; var yMin = this.chart.RangeY.Min; // prepare learning data var input = new double[samples][]; var output = new double[samples][]; for (var i = 0; i < samples; i++) { input[i] = new double[this.windowSize]; output[i] = new double[1]; // set input for (var j = 0; j < this.windowSize; j++) { input[i][j] = (this.data[i + j] - yMin) * factor - 0.85; } // set output output[i][0] = (this.data[i + this.windowSize] - yMin) * factor - 0.85; } // create multi-layer neural network var network = new ActivationNetwork( new BipolarSigmoidFunction(this.sigmoidAlphaValue), this.windowSize, this.windowSize * 2, 1); // create teacher var teacher = new BackPropagationLearning(network); // set learning rate and momentum teacher.LearningRate = this.learningRate; teacher.Momentum = this.momentum; // iterations var iteration = 1; // solution array var solutionSize = this.data.Length - this.windowSize; var solution = new double[solutionSize, 2]; var networkInput = new double[this.windowSize]; // calculate X values to be used with solution function for (var j = 0; j < solutionSize; j++) { solution[j, 0] = j + this.windowSize; } // loop while (!this.needToStop) { // run epoch of learning procedure var error = teacher.RunEpoch(input, output) / samples; // calculate solution and learning and prediction errors var learningError = 0.0; var predictionError = 0.0; // go through all the data for (int i = 0, n = this.data.Length - this.windowSize; i < n; i++) { // put values from current window as network's input for (var j = 0; j < this.windowSize; j++) { networkInput[j] = (this.data[i + j] - yMin) * factor - 0.85; } // evalue the function solution[i, 1] = (network.Compute(networkInput)[0] + 0.85) / factor + yMin; // calculate prediction error if (i >= n - this.predictionSize) { predictionError += Math.Abs(solution[i, 1] - this.data[this.windowSize + i]); } else { learningError += Math.Abs(solution[i, 1] - this.data[this.windowSize + i]); } } // update solution on the chart this.chart.UpdateDataSeries("solution", solution); // set current iteration's info this.currentIterationBox.Text = iteration.ToString( ); this.currentLearningErrorBox.Text = learningError.ToString("F3"); this.currentPredictionErrorBox.Text = predictionError.ToString("F3"); // increase current iteration iteration++; // check if we need to stop if ((this.iterations != 0) && (iteration > this.iterations)) { break; } } // show new solution for (int j = this.windowSize, k = 0, n = this.data.Length; j < n; j++, k++) { this.dataList.Items[j].SubItems.Add(solution[k, 1].ToString( )); } // enable settings controls EnableControls(true); }
static void Main(string[] args) { double learningRate = 0.1; double sigmoidAlphaValue = 2; // iterations int iterations = 100; bool useNguyenWidrow = false; var X_train = LoadData("X_train.csv"); var y_train = LoadData("y_train.csv"); var X_val = LoadData("X_val.csv"); var y_val = LoadData("y_val.csv"); var X_holdout = LoadData("X_holdout.csv"); var y_holdout = LoadData("y_holdout.csv"); int nb_samples = X_train.Length; int input_size = X_train[0].Length; // create multi-layer neural network ActivationNetwork ann = new ActivationNetwork( new BipolarSigmoidFunction(sigmoidAlphaValue), input_size, input_size, 1); if (useNguyenWidrow) { NguyenWidrow initializer = new NguyenWidrow(ann); initializer.Randomize(); } // create teacher //LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(ann, useRegularization); BackPropagationLearning teacher = new BackPropagationLearning(ann); // set learning rate and momentum teacher.LearningRate = learningRate; teacher.Momentum = 0.8; // iterations int iteration = 1; //var ranges = Matrix.Range(sourceMatrix, 0); //double[][] map = Matrix.Mesh(ranges[0], ranges[1], 0.05, 0.05); // var sw = Stopwatch.StartNew(); bool use_batches = false; int batch_size = 5000; double[][] X_batch = new double[batch_size][]; double[][] y_batch = new double[batch_size][]; var rng = new Random(); while (true) { double error = 0.0; if (use_batches) { int[] sample_indeces = Enumerable.Range(0, nb_samples).OrderBy(z => rng.Next()).ToArray(); int nb_batch = nb_samples / batch_size; int n_grad = 50; Console.Write("|{0}| error=\r", new string('-', n_grad)); for (int ibatch = 0; ibatch < nb_batch; ++ibatch) { for (int i = 0; i < batch_size; ++i) { X_batch[i] = X_train[sample_indeces[i]]; y_batch[i] = y_train[sample_indeces[i]]; } error += teacher.RunEpoch(X_batch, y_batch); int ngrad1 = (int)Math.Ceiling((ibatch + 1) * n_grad / (double)(nb_batch)); int ngrad0 = n_grad - ngrad1; double cur_err = error / (batch_size * (ibatch + 1)); Console.Write("|{0}{1}| error={2:0.####}\r", new string('#', ngrad1), new string('-', ngrad0), cur_err); } error /= (batch_size * nb_batch); } else { error = teacher.RunEpoch(X_train, y_train) / nb_samples; } // Получим оценку точности на валидационном наборе int n_hit2 = 0; for (int i = 0; i < X_val.Length; ++i) { double[] output = ann.Compute(X_val[i]); double y_bool = output[0] > 0.5 ? 1.0 : 0.0; n_hit2 += y_bool == y_val[i][0] ? 1 : 0; } double val_acc = n_hit2 / (double)X_val.Length; // TODO: тут надо бы смотреть, увеличилось ли качество сетки на валидации по сравнению с предыдущей эпохой, // сохранять веса сетки в случае улучшения (через deep copy ann), и делать остановку в случае, если качество // не растет уже 5-10 эпох. Console.WriteLine($"\niteration={iteration} train_mse={error} val_acc={val_acc}"); iteration++; if ((iterations != 0) && (iteration > iterations)) { break; } } // Получим оценку точности на отложенном наборе int n_hit = 0; for (int i = 0; i < X_holdout.Length; ++i) { double[] output = ann.Compute(X_holdout[i]); double y_bool = output[0] > 0.5 ? 1.0 : 0.0; n_hit += y_bool == y_holdout[i][0] ? 1 : 0; } double holdout_acc = n_hit / (double)X_holdout.Length; Console.WriteLine($"holdout_acc={holdout_acc}"); return; }
static void Main(string[] args) { double[][] inputs; double[][] outputs; double[][] testInputs; double[][] testOutputs; // Load ascii digits dataset. inputs = DataManager.Load(@"../../../data/data.txt", out outputs); // The first 500 data rows will be for training. The rest will be for testing. testInputs = inputs.Skip(500).ToArray(); testOutputs = outputs.Skip(500).ToArray(); inputs = inputs.Take(500).ToArray(); outputs = outputs.Take(500).ToArray(); // Setup the deep belief network and initialize with random weights. DeepBeliefNetwork network = new DeepBeliefNetwork(inputs.First().Length, 10, 10); new GaussianWeights(network, 0.1).Randomize(); network.UpdateVisibleWeights(); // Setup the learning algorithm. DeepBeliefNetworkLearning teacher = new DeepBeliefNetworkLearning(network) { Algorithm = (h, v, i) => new ContrastiveDivergenceLearning(h, v) { LearningRate = 0.1, Momentum = 0.5, Decay = 0.001, } }; // Setup batches of input for learning. int batchCount = Math.Max(1, inputs.Length / 100); // Create mini-batches to speed learning. int[] groups = Accord.Statistics.Tools.RandomGroups(inputs.Length, batchCount); double[][][] batches = inputs.Subgroups(groups); // Learning data for the specified layer. double[][][] layerData; // Unsupervised learning on each hidden layer, except for the output layer. for (int layerIndex = 0; layerIndex < network.Machines.Count - 1; layerIndex++) { teacher.LayerIndex = layerIndex; layerData = teacher.GetLayerInput(batches); for (int i = 0; i < 200; i++) { double error = teacher.RunEpoch(layerData) / inputs.Length; if (i % 10 == 0) { Console.WriteLine(i + ", Error = " + error); } } } // Supervised learning on entire network, to provide output classification. var teacher2 = new BackPropagationLearning(network) { LearningRate = 0.1, Momentum = 0.5 }; // Run supervised learning. for (int i = 0; i < 500; i++) { double error = teacher2.RunEpoch(inputs, outputs) / inputs.Length; if (i % 10 == 0) { Console.WriteLine(i + ", Error = " + error); } } // Test the resulting accuracy. int correct = 0; for (int i = 0; i < inputs.Length; i++) { double[] outputValues = network.Compute(testInputs[i]); if (DataManager.FormatOutputResult(outputValues) == DataManager.FormatOutputResult(testOutputs[i])) { correct++; } } Console.WriteLine("Correct " + correct + "/" + inputs.Length + ", " + Math.Round(((double)correct / (double)inputs.Length * 100), 2) + "%"); Console.Write("Press any key to quit .."); Console.ReadKey(); }
// Worker thread void SearchSolution() { // initialize input and output values double[][] input = null; double[][] output = null; if (sigmoidType == 0) { // unipolar data input = new double[4][] { new double[] { 0, 0 }, new double[] { 0, 1 }, new double[] { 1, 0 }, new double[] { 1, 1 } }; output = new double[4][] { new double[] { 0 }, new double[] { 1 }, new double[] { 1 }, new double[] { 0 } }; } else { // biipolar data input = new double[4][] { new double[] { -1, -1 }, new double[] { -1, 1 }, new double[] { 1, -1 }, new double[] { 1, 1 } }; output = new double[4][] { new double[] { -1 }, new double[] { 1 }, new double[] { 1 }, new double[] { -1 } }; } // create neural network var network = new ActivationNetwork( (sigmoidType == 0) ? new SigmoidFunction(sigmoidAlphaValue) : (IActivationFunction) new BipolarSigmoidFunction(sigmoidAlphaValue), 2, 2, 1); // create teacher var teacher = new BackPropagationLearning(network) { // set learning rate and momentum LearningRate = learningRate, Momentum = momentum }; // iterations var iteration = 1; // statistic files StreamWriter errorsFile = null; try { // check if we need to save statistics to files if (saveStatisticsToFiles) { // open files errorsFile = File.CreateText("errors.csv"); } // erros list var errorsList = new ArrayList(); // loop while (!needToStop) { // run epoch of learning procedure var error = teacher.RunEpoch(input, output); errorsList.Add(error); // save current error if (errorsFile != null) { errorsFile.WriteLine(error); } // show current iteration & error SetText(currentIterationBox, iteration.ToString()); SetText(currentErrorBox, error.ToString()); iteration++; // check if we need to stop if (error <= learningErrorLimit) { break; } } // show error's dynamics var errors = new double[errorsList.Count, 2]; for (int i = 0, n = errorsList.Count; i < n; i++) { errors[i, 0] = i; errors[i, 1] = (double)errorsList[i]; } errorChart.RangeX = new Range(0, errorsList.Count - 1); errorChart.UpdateDataSeries("error", errors); } catch (IOException) { MessageBox.Show("Failed writing file", "Error", MessageBoxButtons.OK, MessageBoxIcon.Error); } finally { // close files if (errorsFile != null) { errorsFile.Close(); } } // enable settings controls EnableControls(true); }
// Worker thread void SearchSolution() { // number of learning samples int samples = data.GetLength(0); // prepare learning data //80% training, 20% for validate data(to do 70% lear., 20% validate, 10% test) double[][] input = new double[samples * 4 / 5][]; double[][] output = new double[samples * 4 / 5][]; double[][] validateInput = new double[samples / 5][]; double[][] validateOutput = new double[samples / 5][]; // create multi-layer neural network if (alpha > 0.0) { activationFunc = new SigmoidFunction(alpha); } else { activationFunc = new SigmoidFunction(); } int k = 0; int j = 0; for (int i = 1; i < samples; i++) { //80% training, 20% for validate data(to do 70% lear., 20% validate, 10% test) if ((i % 5) == 0) // validate input 20 % { validateInput[k] = new double[colCountData - 1]; validateOutput[k] = new double[1]; for (int c = 0; c < colCountData - 1; c++) { validateInput[k][c] = data[i, c]; } validateOutput[k][0] = data[i, colCountData - 1]; k++; } else //forward input 80 % { // input data input[j] = new double[colCountData - 1]; for (int c = 0; c < colCountData - 1; c++) { input[j][c] = data[i, c]; } //output data output[j] = new double[1]; output[j][0] = data[i, colCountData - 1]; j++; } } network = new ActivationNetwork(activationFunc, colCountData - 1, neuronsAndLayers); // create teacher teacherBack = new BackPropagationLearning(network); // set learning rate and momentum teacherBack.LearningRate = learningRate; teacherBack.Momentum = momentum; // iterations int iteration = 1; double error = 0.0; double validateError = 0.0; //int j = 0; // erros list ArrayList errorsList = new ArrayList(); ArrayList validateList = new ArrayList(); // loop while (!needToStop) { // run epoch of learning procedure error = teacherBack.RunEpoch(input, output); if (errorsList.Count - 1 >= 2000) { errorsList.RemoveAt(0); errorsList[0] = 5.0; } errorsList.Add(error); validateError = 0.0; for (int count = 0; count < validateInput.GetLength(0) - 1; count++) { validateError += network.Compute(validateInput[count])[0] - validateOutput[count][0]; } if (validateList.Count - 1 >= 2000) { validateList.RemoveAt(0); validateList[0] = 5.0; } validateList.Add(validateError); // set current iteration's info currentIterationBox.Invoke(new Action <string>((s) => currentIterationBox.Text = s), iteration.ToString()); errorPercent.Invoke(new Action <string>((s) => errorPercent.Text = s), error.ToString("F14")); validErrorBox.Invoke(new Action <string>((s) => validErrorBox.Text = s), (validateError / 1000).ToString("F14")); // show error's dynamics double[,] errors = new double[errorsList.Count, 2]; double[,] valid = new double[validateList.Count, 2]; double[,] zeros = new double[errorsList.Count, 2]; for (int i = 0, n = errorsList.Count; i < n; i++) { errors[i, 0] = i; zeros[i, 0] = i; errors[i, 1] = (double)errorsList[i]; zeros[i, 1] = 0.0; } for (int i = 0, n = validateList.Count; i < n; i++) { valid[i, 0] = i; valid[i, 1] = (double)validateList[i]; } errorChart.RangeX = new Range(0, errorsList.Count - 1); errorChart.UpdateDataSeries("error", errors); errorChart.UpdateDataSeries("zero", zeros); validChart.RangeX = new Range(0, validateList.Count - 1); validChart.UpdateDataSeries("validate", valid); validChart.UpdateDataSeries("zero", zeros); // increase current iteration iteration++; } // enable settings controls EnableControls(true); }
protected virtual void LearningIteration() { teacher.RunEpoch(Inputs, Answers); }
public double[] Run(int[] numOfHiddenNeurals) { // validation and testing error double[] errors = new double[2]; // number of learning samples int samples = _data.Length - _windowSize; int trainingSamples = samples - _predictionSize; // prepare learning data double[][] input = new double[samples][]; double[][] output = new double[samples][]; // sample indices int[] indices = new int[samples]; int[] trainingIndices = new int[trainingSamples]; // normalization function var normalizeFunc = new MinMaxNormalization(_xMax, _xMin, _data.Max(), _data.Min()); for (int i = 0; i < samples; i++) { input[i] = new double[_windowSize]; output[i] = new double[_outputNum]; // set input for (int j = 0; j < _windowSize; j++) { input[i][j] = normalizeFunc.Compute(_data[i + j]); } // set output for (int j = 0; j < _outputNum; j++) { output[i][j] = normalizeFunc.Compute(_data[i + _windowSize + j]); } indices[i] = i; } // randomize the sample indices Utils.Shuffle<int>(indices); output.Swap(indices); input.Swap(indices); // get training samples double[][] trainingInput = new double[trainingSamples][]; double[][] trainingOutput = new double[trainingSamples][]; for (int i = 0; i < trainingSamples; i++) { trainingInput[i] = new double[_windowSize]; trainingOutput[i] = new double[_outputNum]; // set input for (int j = 0; j < _windowSize; j++) { trainingInput[i][j] = input[i][j]; } // set output for (int j = 0; j < _outputNum; j++) { trainingOutput[i][j] = output[i][j]; } trainingIndices[i] = i; } //// randomize the sample indices //Utils.Shuffle<int>(trainingIndices); //trainingOutput.Swap(trainingIndices); //trainingInput.Swap(trainingIndices); // create multi-layer neural network int[] neuronsCount = numOfHiddenNeurals.Concat(new int[] { _outputNum }).ToArray(); ActivationNetwork network = new ActivationNetwork( _function, _windowSize, neuronsCount); // create teacher BackPropagationLearning teacher = new BackPropagationLearning(network); // set learning rate and momentum teacher.LearningRate = _learningRate; teacher.Momentum = 0.0; //var teacher = new ParallelResilientBackpropagationLearning(network); //teacher.Reset(_learningRate); // iterations int iteration = 1; // solution array int solutionSize = _data.Length - _windowSize; double[,] solution = new double[solutionSize, 1 + _outputNum]; // calculate X values to be used with solution function for (int j = 0; j < solutionSize; j++) { solution[j, 0] = j + _windowSize; } // loop var needToStop = false; while (!needToStop) { // run epoch of learning procedure double error = teacher.RunEpoch(trainingInput, trainingOutput) / trainingSamples; // calculate solution and learning and prediction errors every 5 double learningError = 0.0; double predictionError = 0.0; if (iteration % 5 == 0) { // go through all the data for (int i = 0; i < samples; i++) { double y = 0.0; double o = 0.0; double err = 0.0; for (int j = 0; j < _outputNum; j++) { y = output[i][j]; o = network.Compute(input[i])[j]; err += (o - y) * (o - y) / _outputNum; // evalue the function solution[i, j + 1] = normalizeFunc.Inverse(o); } // calculate prediction error (MSE) if (i >= trainingSamples) { predictionError += err;// Math.Pow((solution[i, 1] - normalizeFunc.Inverse(output[i][0])), 2.0); } else { learningError += err; } } } // Adaptive Learning - decrease the learning rate // n(t) = n0 * a^(t/T) // a = 1/10^x, x >= 1 teacher.LearningRate = _learningRate * Math.Pow(0.1, iteration / _iterations); // increase iteration iteration++; // check if we need to stop if ((_iterations != 0) && (iteration > _iterations)) { errors[0] = learningError / trainingSamples; errors[1] = predictionError / _predictionSize; Console.WriteLine("Final Learning MSE Error: " + errors[0]); Console.WriteLine("Final Prediction MSE Error: " + errors[1]); Console.WriteLine("Final Learning Rate: " + teacher.LearningRate); Console.WriteLine("Window Size: " + _windowSize + "\n" + "Number of Hidden Neurons: " + neuronsCount[0] + "\n" + "Output Size: " + _outputNum); break; } } ////print result to file //Console.WriteLine("Real Values\t\tRegression Values"); //for (int i = samples - 1; i >= trainingSamples; --i) //{ // Console.WriteLine(normalizeFunc.Inverse(output[i][0]) + "\t\t" + solution[i, 1]); //} return errors; }
static void Main(string[] args) { Console.OutputEncoding = Encoding.UTF8; Console.WriteLine(@"Загрузка данных для обучения и проверки..."); Frame <int, string> training = DataService.GetTrainingData(); Frame <int, string> validation = DataService.GetValidationData(500); Console.WriteLine(@"Проводится нормализация пикселей..."); NormalizePixels(training); NormalizePixels(validation); Console.WriteLine(@"Кодирование цифр в обучающей и проверочной выборках..."); EncodeDigit(training); EncodeDigit(validation); // Инициализация нейронной сети ActivationNetwork network = new ActivationNetwork( new SigmoidFunction(), // сигмоидная активационная функция InputNeuronsCount, // количество входных нейронов (784) FirstHiddenLayerNeuronsCount, // количество нейронов в первом скрытом слое (100) SecondHiddenLayerNeuronsCount, // количество нейронов во втором скрытом слое (100) OutputNeuronsCount); // количество нейронов в выходном слое (10) // Данный класс отвечает за реализацию алгоритма обратного распространения ошибки BackPropagationLearning learner = new BackPropagationLearning(network) { LearningRate = 0.1 // Начальная скорость обучения }; // Производим инициализацию весов сети при помощи распределения Гаусса new GaussianWeights(network).Randomize(); List <double> errors = new List <double>(); List <double> validationErrors = new List <double>(); double[][] features = training.Columns[FeatureColumns].ToArray2D <double>().ToJagged(); double[][] labels = training.Columns[LabelColumns].ToArray2D <double>().ToJagged(); Console.WriteLine($@"Запущен процесс обучения нейронной сети, количество эпох: {EpochCount}..."); for (int i = 0; i < EpochCount; i++) { double error = learner.RunEpoch(features, labels) / labels.GetLength(0); errors.Add(error); // сохраняем ошибку для построения графика // Обучаем сеть методом обратного распространения ошибки // и, для каждой эпохи, получаем ошибку обучения double validationError = NetworkValidationService .Validate(network, // Нейронная сеть validation, // данные для тестирования labels); // колонки с данными // сохраняем квадратичную ошибку для построения графика validationErrors.Add(validationError); Console.WriteLine($@"{DateTime.Now} Эпоха: {i}, Ошибка обучения: {error}, Ошибка тестирования: {validationError}"); } PlotService.PlotValidationCurve(validationErrors, EpochCount); PlotService.PlotTrainingCurve(errors, EpochCount); PrintValidationReport(network, NetworkValidationService, training); Console.Write("Сохраняем состояние обученной сети на диск... "); DataService.SaveNetworkState(network); Console.ReadKey(); }
private void learnNetworkSupervised() { if (!Main.CanClassify) return; Dispatcher dispatcher = Dispatcher.CurrentDispatcher; new Task(() => { var teacher = new BackPropagationLearning(Main.Network) { LearningRate = LearningRate, Momentum = Momentum }; double[][] inputs, outputs; Main.Database.Training.GetInstances(out inputs, out outputs); // Start running the learning procedure for (int i = 0; i < Epochs && !shouldStop; i++) { double error = teacher.RunEpoch(inputs, outputs); dispatcher.BeginInvoke((Action<int, double>)updateError, DispatcherPriority.ContextIdle, i + 1, error); } Main.Network.UpdateVisibleWeights(); IsLearning = false; }).Start(); }
private void Pruning(double[][] input, double[][] output) { var samples = input.Length; var keepPruning = true; Network checkpoint = null; var pruned = 0; while (keepPruning) { using (var clonestream = new MemoryStream()) { var trainError = trainingSetError.Last(); var formatter = new BinaryFormatter(); formatter.Serialize(clonestream, network); clonestream.Position = 0; checkpoint = (Network)formatter.Deserialize(clonestream); var min = Double.PositiveInfinity; var minpos = new[] { 0, 0, 0 }; var originalPruning = true; var minRatio = network.Layers.SelectMany(x => x.neurons).Min(y => y.minRatio()); if (minRatio <= 0.3) { originalPruning = false; } else { keepPruning = false; } originalPruning = true; for (var i = 0; i < network.Layers.Length; i++) { var layer = network.Layers[i]; for (var j = 0; j < layer.neurons.Length; j++) { var neuron = layer.neurons[j]; if (originalPruning) { for (var k = 0; k < neuron.weights.Length; k++) { var weight = neuron.weights[k]; if (neuron.freezes[k] == 0 || Math.Abs(min) < Math.Abs(weight)) { continue; } min = weight; minpos = new[] { i, j, k }; } } else { var ratio = neuron.minRatio(); if (ratio > min) { continue; } min = ratio; minpos = new[] { i, j }; } } } if (originalPruning) { var minneuron = network.Layers[minpos[0]].neurons[minpos[1]]; minneuron.Freeze(minpos[2]); } else { network.Layers[minpos[0]].neurons[minpos[1]].FreezeMinRatio(); } var iteration = 1; var trainingSetErrorPrune = new List <double>(); while (!needToStop) { var error = teacher.RunEpoch(input, output) / samples; trainingSetErrorPrune.Add(error); iteration++; if ((iteration < iterations / 10) && (error > trainError)) { continue; } if (iteration >= iterations / 10 && error > trainError) { keepPruning = false; } else { trainingSetError.AddRange(trainingSetErrorPrune); pruned++; } break; } } } var percentage = (Decimal.Parse(pruned.ToString()) / Decimal.Parse(totalConnections.ToString())).ToString("P", CultureInfo.InvariantCulture); ///////////////////////////////////// MESSAGE BOX ///////////////////////////////////////////////////////////////////////////// /* MessageBox.Show("Pruned " + pruned + " out of " + totalConnections + " connections" + * " ( " + percentage + " ) in a total of: " + trainingSetError.Count + " iterations.");*/ raports.Add(new Raport { connections = totalConnections, error = trainingSetError.Last(), iterations = trainingSetError.Count, pruned = pruned }); network = (ActivationNetwork)checkpoint; }
// Worker thread public void SearchSolution() { // prepare learning data double[][] input = new double[samples][]; double[][] hidden = new double[samples][]; double[][] output = new double[samples][]; for (int i = 0; i < samples; i++) { input[i] = new double[variables]; hidden[i] = new double[hneuron]; output[i] = new double[1]; // copy input for (int j = 0; j < variables; j++) { input[i][j] = data[i, j]; } // copy output output[i][0] = classes[i]; } // create backpropagation ActivationNetwork network = new ActivationNetwork(new SigmoidFunction(2), variables, hneuron, 1); ActivationNeuron neuron = network[0][0]; ActivationLayer layer = network[0]; // create teacher BackPropagationLearning teacher = new BackPropagationLearning(network); // set learning rate teacher.LearningRate = learningRate; // iterations int iteration = 1; // statistic files StreamWriter errorsFile = null; StreamWriter weightsFile = null; try { // check if we need to save statistics to files if (saveStatisticsToFiles) { // open files errorsFile = File.CreateText("C:\\aaaaaa\\program\\s\\error.csv"); weightsFile = File.CreateText("C:\\aaaaaa\\program\\s\\weights.csv"); } // erros list ArrayList errorsList = new ArrayList(); // loop while (!needToStop) { // run epoch of learning procedure double error = teacher.RunEpoch(input, output); errorsList.Add(error); // save current weights if (weightsFile != null) { weightsFile.Write("\n" + "iterasi" + iteration); for (int i = 0; i < 1; i++) { weightsFile.Write("\n" + "nilai bobot-HO = " + layer[i].Threshold); for (int j = 0; j < hneuron; j++) { weightsFile.Write("\n" + "nilai bobot-IH neuron " + j + "=" + layer[j].Threshold); weightsFile.Write("\n" + "IHneuron" + j + "," + "HOneuron" + j); for (int k = 0; k < variables; k++) { weightsFile.Write("\n" + layer[j][k] + "," + layer[i][j]); } } } } // save current error if (errorsFile != null) { errorsFile.WriteLine(error); } // show current iteration SetText(textBox3, iteration.ToString()); SetText(textBox11, error.ToString()); // stop if no error if (error < maxLearningError || iteration >= maxepoch) { break; } iteration++; } // show Backpropagation's weights ClearList(weightsList); for (int i = 0; i < 1; i++) { string neuro = string.Format("neuron {0}", i + 1); ListViewItem item = null; for (int j = 0; j < hneuron; j++) { item = AddListItem(weightsList, neuro); AddListSubitem(item, string.Format("HOWeight {0}", j + 1)); AddListSubitem(item, layer[i][j].ToString("F6")); for (int k = 0; k < variables; k++) { item = AddListItem(weightsList, neuro); AddListSubitem(item, string.Format("IHWeight {0}", k + 1)); AddListSubitem(item, layer[j][k].ToString("F6")); } item = AddListItem(weightsList, neuro); AddListSubitem(item, string.Format("IHthreshold{0}", j + 1)); AddListSubitem(item, layer[j].Threshold.ToString("F6")); } item = AddListItem(weightsList, neuro); AddListSubitem(item, string.Format("HOthreshold{0}", i + 1)); AddListSubitem(item, layer[i].Threshold.ToString("F6")); } network.Save("D:\\My Documents\\Ihsan's Document\\skripsi\\s\\JST\\net_latih.net"); } catch (IOException) { MessageBox.Show("Failed writing file", "Error", MessageBoxButtons.OK, MessageBoxIcon.Error); } finally { // close files if (errorsFile != null) { errorsFile.Close(); } if (weightsFile != null) { weightsFile.Close(); } } // enable settings controls EnableControls(true); }