public EvolutionaryAlgorithmBuilderContinuousMO( Population population, DecisionSpace decisionSpace, HyperParameterManager hyperParameters, IParentSelectionOperator parentSelector, IRecombinationOperator recombinationOperator, IMutationOperator mutationOperator, IReinsertionOperator reinsertionOperator) { this.decisionSpace = decisionSpace; HyperParameters.AddFromExistingHyperParameterSet(hyperParameters); this.population = population; this.parentSelector = parentSelector; this.recombinationOperator = recombinationOperator; this.mutationOperator = mutationOperator; this.reinsertionOperator = reinsertionOperator; }
/// <summary> /// Инструмент обучения. /// </summary> /// <param name="dataSet">Обучающая выборка.</param> /// <param name="trainType">Тип обучения.</param> /// <param name="hyperParameters">Гиперпараметры.</param> /// <param name="topology">Топология сети.</param> public TrainUtil(DataSet dataSet, TrainType trainType, HyperParameters hyperParameters, Topology topology) { if (!trainType.Equals(TrainType.Backpropagation)) { throw new NotImplementedException(); } if (!topology.IsClosed) { throw new Exception("Топология сети не замкнута! Обучение невозможно!"); } _trainType = trainType; _dataSet = dataSet; _hyperParameters = hyperParameters; _topology = topology; }
private static void dumpHyperParameters(HyperParameters hyperParameters) { Console.WriteLine(""); Console.WriteLine(" Hyper Parameters"); Console.WriteLine("======================================================"); Console.WriteLine(" Epochs: " + hyperParameters.Epochs); Console.WriteLine(" Mini Batch Size: " + hyperParameters.MiniBatchSize); Console.WriteLine(" Cost Function: " + hyperParameters.CostFunctionName); Console.WriteLine(" Learning Rate: " + hyperParameters.LearningRate); Console.WriteLine(" Use Dropouts: " + hyperParameters.UseDropouts); Console.WriteLine("Hidden Layer Dropout Probability: " + hyperParameters.HiddenLayerDropoutProbability); Console.WriteLine(" Input Layer Dropout Probability: " + hyperParameters.InputLayerDropoutProbability); Console.WriteLine(" Regulation Lambda: " + hyperParameters.RegulationLambda); Console.WriteLine(" Test Size: " + hyperParameters.TestSize); Console.WriteLine(" Auto-save: " + hyperParameters.AutoSave); Console.WriteLine(" Auto-save Threshold: " + hyperParameters.AutoSaveThreshold); Console.WriteLine("======================================================"); Console.WriteLine(""); }
static void Main(string[] args) { const string dataRoot = @"C:\HaishiRooster\Data\MINST"; //Load training data var trainingImages = MINSTDataLoader.LoadImages(Path.Combine(dataRoot, "train-images-idx3-ubyte.gz")); var trainingLabels = MINSTDataLoader.LoadLabels(Path.Combine(dataRoot, "train-labels-idx1-ubyte.gz")); var trainingSet = MINSTDataLoader.CombineImagesAndLabels(trainingImages, trainingLabels); //Load test data var testingImages = MINSTDataLoader.LoadImages(Path.Combine(dataRoot, "t10k-images-idx3-ubyte.gz")); var testingLabels = MINSTDataLoader.LoadLabels(Path.Combine(dataRoot, "t10k-labels-idx1-ubyte.gz")); var testingSet = MINSTDataLoader.CombineImagesAndLabels(testingImages, testingLabels); //Print training set information and a sample image Console.WriteLine("Training set size: " + trainingSet.Count); Console.WriteLine("Testing set size: " + testingSet.Count); int index = mRand.Next(trainingSet.Count); Console.WriteLine(string.Format("Here's a random picture from the training set: #{0} ({1})", index, convertToByte(trainingSet[index].Label))); MINSTDataVisualizer.PrintImage(trainingSet[index].Image, 28); Network network; if (args.Length > 0) { using (StreamReader reader = new StreamReader(File.OpenRead(args[0]))) { network = Network.Load(reader); } Console.WriteLine("\nNetwork is loaded!"); dumpHyperParameters(network.HyperParameters); } else { ////MSR //HyperParameters hyperParameters = new HyperParameters { CostFunctionName = "QuadraticCost", Epochs = 30, MiniBatchSize = 10, LearningRate = 3, TestSize = 10, AutoSave = true, AutoSaveThreshold = 0.951 }; //network = new Network(hyperParameters, 784, 30, 10); ////Cross-entropy //HyperParameters hyperParameters = new HyperParameters { CostFunctionName = "CrossEntropyCost", Epochs = 30, MiniBatchSize = 10, LearningRate = 0.5, TestSize = testingSet.Count, AutoSave = true, AutoSaveThreshold = 0.967 }; //network = new Network(hyperParameters, 784, 100, 10); ////Cross-entropy with regulation //HyperParameters hyperParameters = new HyperParameters { CostFunctionName = "CrossEntropyCost", Epochs = 60, MiniBatchSize = 10, LearningRate = 0.1, RegulationLambda = 5.0, TestSize = testingSet.Count, AutoSave = true, AutoSaveThreshold = 0.98 }; //network = new Network(hyperParameters, 784, 100, 10); ////Cross-entropy with regulation - 120 epochs //HyperParameters hyperParameters = new HyperParameters { CostFunctionName = "CrossEntropyCost", Epochs = 120, MiniBatchSize = 10, LearningRate = 0.1, RegulationLambda = 5.0, TestSize = testingSet.Count, AutoSave = true, AutoSaveThreshold = 0.98 }; //network = new Network(hyperParameters, 784, 100, 10); ////Cross-entropy with regulation - 120 epochs - with dropouts HyperParameters hyperParameters = new HyperParameters { CostFunctionName = "CrossEntropyCost", Epochs = 240, MiniBatchSize = 10, LearningRate = 1, RegulationLambda = 5.0, TestSize = testingSet.Count, AutoSave = true, AutoSaveThreshold = 0.98, UseDropouts = true }; network = new Network(hyperParameters, 784, 100, 10); hookupEvents(network); dumpHyperParameters(hyperParameters); //Train the network network.Train(trainingSet, (actual, expected) => { return(convertToByte(actual) == convertToByte(expected)); }, testingSet); Console.WriteLine("\nNetwork is trained!"); } //Now validate while (true) { Console.Write(string.Format("\nPlease enter a test image index [0 - {0}]. Enter '-1' to evaluate all test images. Enter '-2' to exit:", testingSet.Count - 1)); if (int.TryParse(Console.ReadLine(), out index)) { if (index == -1) { int count = 0; Console.Write(string.Format("Detecting {0} pictures", testingSet.Count)); for (int i = 0; i < testingSet.Count; i++) { var detection = network.Detect(testingSet[i].Image); if (convertToByte(detection) == convertToByte(testingSet[i].Label)) { Console.Write("."); count++; } else { Console.Write("X"); } } Console.WriteLine("\nDetected {0} out of {1} pictures, correct rate is: {2:0.0%}", count, testingSet.Count, count * 1.0 / testingSet.Count); } else if (index >= 0 && index < testingSet.Count) { Console.WriteLine(string.Format("Test image: #{0} ({1})", index, convertToByte(testingSet[index].Label))); MINSTDataVisualizer.PrintImage(testingSet[index].Image, 28); var detection = network.Detect(testingSet[index].Image); Console.WriteLine(string.Format("\nDetected number: {0} - {1}", detection, convertToByte(detection) == convertToByte(testingSet[index].Label) ? "SUCCESS!" : "FAIL!")); } else if (index == -2) { break; } else if (index == -3) { Console.Write("Please enter file name: "); string fileName = Console.ReadLine(); using (StreamWriter writer = new StreamWriter(File.Create(fileName))) { network.Save(writer); } } } } }
public abstract void Train(NeuralNetwork.NeuralNetwork neuralNetwork, Matrix <double> trainingSet, Matrix <double> crossValidationSet, Matrix <double> trainingSetOutput, Matrix <double> crossValidationSetOutput, HyperParameters hyperParameters = null);
private void _train(NeuralNetwork.NeuralNetwork neuralNetwork, Matrix <double> trainingSet, Matrix <double> crossValidationSet, Matrix <double> trainingSetOutput, Matrix <double> crossValidationSetOutput, HyperParameters hyperParamters = null) { TrainingAlgorithm.Train(neuralNetwork, trainingSet, crossValidationSet, trainingSetOutput, crossValidationSetOutput, hyperParamters); }
/// <summary> /// Обновить ячейку с помощтю карт входа/выхода. /// </summary> /// <param name="cell">Ячейка.</param> /// <param name="inputMap">Входная карта.</param> /// <param name="outputMap">Выходная карта.</param> /// <param name="filterMatrixSize">Размер матрицы фильтра.</param> /// <param name="hyperParameters">Гиперпараметры.</param> public static void UpdateCellByIOMaps(this ModifiedCell cell, FigureMap inputMap, FigureMap outputMap, int filterMatrixSize, HyperParameters hyperParameters) { var xEndPoint = cell.X + filterMatrixSize; var yEndPoint = cell.Y + filterMatrixSize; var gradient = 0d; var x = 0; for (var xStartPoint = cell.X; xStartPoint < xEndPoint; ++xStartPoint) { var y = 0; for (var yStartPoint = cell.Y; yStartPoint < yEndPoint; ++yStartPoint) { var cellFromInputMap = inputMap.Cells .Find(c => c.X.Equals(xStartPoint) && c.Y.Equals(yStartPoint)); var cellFromOutputMap = outputMap.Cells.Find(c => c.X.Equals(x) && c.Y.Equals(y)); gradient += cellFromInputMap.Value * cellFromOutputMap.Value; ++y; } ++x; } var deltaValue = hyperParameters.Epsilon * gradient + hyperParameters.Alpha * cell.LastValueDelta; cell.UpdateLastDeltValue(deltaValue); var newValue = cell.Value + deltaValue; cell.UpdatedValue(newValue); }
/// <summary> /// Выполнить обучение. /// </summary> /// <param name="pathToSettings">Путь до сохранённого файла настроек.</param> private static void DoTrain(out string pathToSettings) { Console.Clear(); ConsoleExtensions.WriteWithColors(ConsoleColor.Black, ConsoleColor.Green, "Вас приветствует обучение!\nУкажите директорию обучающей выборки " + "(enter для директории по-умолчанию):"); var input = Console.ReadLine(); if (input.Equals(string.Empty)) { input = PathHelper.GetResourcesPath(); } if (!Directory.Exists(input)) { throw new Exception($"Указанная директория не существует!\nДиректория: {input}"); } var directories = Directory.GetDirectories(input).ToList(); var matrixDictionary = new Dictionary <int, List <double[, ]> >(); var key = 0; foreach (var directory in directories) { var files = Directory.GetFiles(directory).ToList(); if (!files.Any()) { throw new Exception($"Файлы не найдены!\nДиректория: {directory}"); } var images = PathToImageConverter.LoadImages(files); var resizedImages = NormilizeUtil.ResizeImages(images, 6, 6); var normilizedMatrixies = NormilizeUtil.GetNormilizedMatrixesFromImages(resizedImages); matrixDictionary.Add(key, normilizedMatrixies); ++key; } var hyperParameters = new HyperParameters { EpochCount = 1, Epsilon = 0.75, Alpha = 0.001 }; var topology = new Topology(); topology.Add(2, 2, LayerType.Convolution); topology.Add(3, 2, LayerType.Subsampling); topology.Add(4, 2, LayerType.Hidden, true); if (!topology.IsClosed) { throw new Exception("Не удалось замкнуть топологию."); } var dataSet = new DataSet(DataSetType.ForNumberRecognizing); if (!matrixDictionary.Count.Equals(dataSet.MaxCountInDataSet())) { throw new Exception("Не соответсвие количества выборок для распознавания чисел!"); } foreach (var pair in matrixDictionary) { dataSet.Add(pair.Value); } var trainUtil = new TrainUtil(dataSet, TrainType.Backpropagation, hyperParameters, topology); trainUtil.Start(3, 2, out var error, out pathToSettings); var errorString = $"{Math.Round(error * 100, 2)}%"; ConsoleExtensions.WriteWithColors(ConsoleColor.Black, ConsoleColor.Yellow, $"\nОшибка: {errorString}"); if (pathToSettings.Equals(string.Empty)) { ConsoleExtensions.WriteWithColors(ConsoleColor.Black, ConsoleColor.Red, "\nДанные не были сохранены!"); } else { ConsoleExtensions.WriteWithColors(ConsoleColor.Black, ConsoleColor.Blue, $"\nДанные сохранены!\nДиректория: {pathToSettings}"); } }
public void Train(NeuralNetwork.NeuralNetwork neuralNetwork, Matrix <double> inputs, Matrix <double> outputs, HyperParameters hyperParamters = null) { if (TrainingAlgorithm is LevenbergAlgorithm) { _isLeven = true; } else { _isLeven = false; } TrainingAlgorithm.Subscribe(this); var temp = Divider.Divide(inputs, outputs); Console.WriteLine(temp.Item1); Console.WriteLine(temp.Item2); _train(neuralNetwork, temp.Item1, temp.Item3, temp.Item2, temp.Item4, hyperParamters); }
public override void Train(NeuralNetwork.NeuralNetwork neuralNetwork, Matrix <double> trainingSet, Matrix <double> crossValidationSet, Matrix <double> trainingSetOutput, Matrix <double> crossValidationSetOutput, HyperParameters hyperParameters = null) { double maxError = 0.01, error = 5, momentum = 0.9; int maxEpochs = 1000, epochs = 0; if (hyperParameters != null) { if (maxEpochs <= 0) { throw new ArgumentException("Max Epochs cannot be negative"); } if (maxError > 2 || maxError < 0) { throw new ArgumentException("Max error cannot be negative or very large"); } maxError = hyperParameters.MaxError; maxEpochs = hyperParameters.MaxEpochs; momentum = hyperParameters.Momentum; } TrainingErrorMessage message = new TrainingErrorMessage() { NeuralNetwork = neuralNetwork, TrainingSet = trainingSet, CrossValidationSet = crossValidationSet, TrainingSetOutput = trainingSetOutput, CrossValidationSetOutput = crossValidationSetOutput }; var layers = neuralNetwork.Layers; var weights = neuralNetwork.HiddenWeights; //for momentum List <Matrix <double> > prevDeltaW = new List <Matrix <double> >(); for (int i = 0; i < weights.Count; ++i) { prevDeltaW.Add(Matrix <double> .Build.Dense(layers[i + 1].NeuronsNumber, layers[i].NeuronsNumber + 1)); } //end //epochs++ => ++epochs while (error >= maxError && ++epochs <= maxEpochs) { prevDeltaW.ForEach(e => e.Clear()); for (int i = 0; i < trainingSet.RowCount; i++) { Vector <double> input = trainingSet.Row(i), output = trainingSetOutput.Column(i); var temp = neuralNetwork.ForwardInput(input); IList <Vector <double> > acs = temp.Item1, gs = temp.Item2; var D = (output - acs[acs.Count - 1]).PointwiseMultiply(gs[gs.Count - 1]).ToColumnMatrix(); // n(output) * 1 var deltaW = D * acs[acs.Count - 2].ToRowMatrix() * layers[layers.Count - 1].LearningRate; // (n(output) * 1) * ((n(output-1)+1) * 1)' = n(output) * (n(output-1)+1) //for momentum deltaW += computeAdditionalTerms(prevDeltaW[weights.Count - 1], momentum); prevDeltaW[weights.Count - 1] = deltaW; //end neuralNetwork.UpdateWeightsAt(deltaW, weights.Count - 1); for (int j = layers.Count - 2; j > 0; j--) { D = (weights[j].Transpose() * D).RemoveRow(0).PointwiseMultiply(gs[j - 1].ToColumnMatrix()); // (n(j+1) * (n(j)+1))' * (n(j+1) * 1) = (n(j)+1) * 1, then => (n(j) * 1) .* (n(j) * 1) deltaW = D * acs[j - 1].ToRowMatrix() * layers[j].LearningRate; // (n(j) * 1) * ((n(j-1)+1) * 1)' = n(j) * (n(j-1)+1) //for momentum deltaW += computeAdditionalTerms(prevDeltaW[j - 1], momentum); prevDeltaW[j - 1] = deltaW; //end neuralNetwork.UpdateWeightsAt(deltaW, j - 1); } } message.Epochs = epochs; base.Notify(message); error = message.Error; Console.WriteLine(error); } base.OnComplete(); }
public override void Train(NeuralNetwork.NeuralNetwork neuralNetwork, Matrix <double> trainingSet, Matrix <double> crossValidationSet, Matrix <double> trainingSetOutput, Matrix <double> crossValidationSetOutput, HyperParameters hyperParameters = null) { //Func<Math.IActivatorFunction, IActivationFunction> to = (fun) => //{ // if (fun is Math.SigmoidFunction) // return new SigmoidFunction(); // if (fun is Math.IdentityFunction) // return new IdentityFunction(); // return new SigmoidFunction(); //}; //IList<ActivationLayer> layers = new List<ActivationLayer>(); //layers.Add(new ActivationLayer(neuralNetwork.HiddenWeights[0].RowCount, trainingSet.ColumnCount, to(neuralNetwork.Layers[0].Applier.ActivatorFunction))); //for (int i = 1; i < neuralNetwork.HiddenWeights.Count; i++) //{ // layers.Add(new ActivationLayer(neuralNetwork.HiddenWeights[i - 1].RowCount, trainingSet.ColumnCount, to(neuralNetwork.Layers[i].Applier.ActivatorFunction))); //} double maxError = 0.01, error = 5, lr = 0.01; int maxEpochs = 1000, epochs = 0; if (hyperParameters != null) { if (maxEpochs <= 0) { throw new ArgumentException("Max Epochs cannot be negative"); } if (maxError > 2 || maxError < 0) { throw new ArgumentException("Max error cannot be negative or very large"); } maxError = hyperParameters.MaxError; maxEpochs = hyperParameters.MaxEpochs; lr = hyperParameters.Lr; } ActivationNetwork network = new ActivationNetwork(new SigmoidFunction(2), trainingSet.ColumnCount, neuralNetwork.Layers.Select(x => x.NeuronsNumber).ToArray()); LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(network, true) { LearningRate = lr }; TrainingErrorMessage message = new TrainingErrorMessage() { NeuralNetwork = neuralNetwork, TrainingSet = trainingSet, CrossValidationSet = crossValidationSet, TrainingSetOutput = trainingSetOutput, CrossValidationSetOutput = crossValidationSetOutput }; int iterations = 1; double[][] inputs = new double[trainingSet.RowCount][], crossInputs = new double[crossValidationSet.RowCount][], outputs = new double[trainingSetOutput.RowCount][], crossOutputs = new double[crossValidationSetOutput.RowCount][]; for (int i = 0; i < trainingSet.RowCount; i++) { inputs[i] = new double[trainingSet.ColumnCount]; for (int j = 0; j < trainingSet.ColumnCount; j++) { inputs[i][j] = trainingSet[i, j]; } } for (int i = 0; i < trainingSetOutput.RowCount; i++) { outputs[i] = new double[trainingSetOutput.ColumnCount]; for (int j = 0; j < trainingSet.ColumnCount; j++) { outputs[i][j] = trainingSetOutput[i, j]; } } for (int i = 0; i < crossValidationSet.RowCount; i++) { crossInputs[i] = new double[crossValidationSet.ColumnCount]; for (int j = 0; j < crossValidationSet.ColumnCount; j++) { crossInputs[i][j] = crossValidationSet[i, j]; } } for (int i = 0; i < crossValidationSetOutput.RowCount; i++) { crossOutputs[i] = new double[crossValidationSetOutput.ColumnCount]; for (int j = 0; j < crossValidationSetOutput.ColumnCount; j++) { crossOutputs[i][j] = crossValidationSetOutput[i, j]; } } while (error > maxError && iterations++ <= maxEpochs) { message.Epochs = iterations; error = teacher.RunEpoch(inputs, outputs); message.TrainError = error; message.CrossError = teacher.ComputeError(crossInputs, crossOutputs); base.Notify(message); } //double mue = 0.001, mue_adj = 10, max_mue = 1e10; //base.Notify(message); //double currentError = message.Error; //while (currentError >= maxError && epochs++ < maxEpochs) //{ // message.Epochs = epochs; // var temp = HissienAndGragient(neuralNetwork, trainingSet, trainingSetOutput); // var hessien = temp.Item1; // var gradient = temp.Item2; // Matrix<double> blendingMatrix = Matrix<double>.Build.DenseDiagonal(hessien.RowCount, hessien.ColumnCount); // var prevW = neuralNetwork.HiddenWeights.ToList(); // //Console.WriteLine("prev :"); // //prevW.ForEach(Console.WriteLine); // double nextError = 100000; // while (true) // { // var term = hessien + mue*blendingMatrix; // var det = term.Determinant(); // if (System.Math.Abs(det) > 0) // { // var deltaW = term*gradient; // neuralNetwork.UpdateWeightsFromVector(deltaW); // //Console.WriteLine("updated :"); // //neuralNetwork.HiddenWeights.ForEach(Console.WriteLine); // base.Notify(message); // nextError = message.Error; // } // if (!(System.Math.Abs(det) > 0) || nextError >= currentError) // { // neuralNetwork.SetWeights(prevW); // //Console.WriteLine("set to prev :"); // //neuralNetwork.HiddenWeights.ForEach(Console.WriteLine); // mue *= mue_adj; // if (mue > max_mue) // { // mue = max_mue; // break; // } // } // else // { // mue /= mue_adj; // currentError = nextError; // //Console.WriteLine("the shit is here"); // break; // } // } //} }
public NelderMeadBuilder(DecisionSpace decisionSpace, HyperParameterManager hyperParameters) { this.decisionSpace = decisionSpace; HyperParameters.AddFromExistingHyperParameterSet(hyperParameters); }
/// <summary> /// Получить дельту веса. /// </summary> /// <param name="hyperParameters">Гиперпараметры.</param> /// <param name="gradient">Градиент.</param> /// <param name="lastDeltaValue">Последние значение дельты веса.</param> /// <returns>Возвращает дельту веса.</returns> internal static double GetWeightsDelta(HyperParameters hyperParameters, double gradient, double lastDeltaValue) => hyperParameters.Epsilon * gradient + hyperParameters.Alpha * lastDeltaValue;
/// <summary> /// Получить дельту веса. /// </summary> /// <param name="hyperParameters">Гиперпараметры.</param> /// <param name="outputNeuron">Выходной связанные нейрон.</param> /// <param name="gradient">Градиент.</param> /// <param name="indexOfCurrentNeuron">Индекс текущего нейрона.</param> /// <returns>Возвращает дельту веса.</returns> internal static double GetWeightsDelta(HyperParameters hyperParameters, Neuron outputNeuron, double gradient, int indexOfCurrentNeuron) => hyperParameters.Epsilon * gradient + hyperParameters.Alpha * outputNeuron.LastWeightsDeltas[indexOfCurrentNeuron];
public GeneticSpecimen(string file) { FilePath = file; hyperParameters = new HyperParameters(FilePath); CreateChildren(); }
public HyperParameters GetHyperParameters() { return(HyperParameters.GetDefaults()); }