public EvolutionaryAlgorithmBuilderContinuousMO(
Population population,
DecisionSpace decisionSpace,
HyperParameterManager hyperParameters,
IParentSelectionOperator parentSelector,
IRecombinationOperator recombinationOperator,
IMutationOperator mutationOperator,
IReinsertionOperator reinsertionOperator)
{
this.decisionSpace = decisionSpace;
HyperParameters.AddFromExistingHyperParameterSet(hyperParameters);
this.population = population;
this.parentSelector = parentSelector;
this.recombinationOperator = recombinationOperator;
this.mutationOperator = mutationOperator;
this.reinsertionOperator = reinsertionOperator;
}
/// <summary>
/// Инструмент обучения.
/// </summary>
/// <param name="dataSet">Обучающая выборка.</param>
/// <param name="trainType">Тип обучения.</param>
/// <param name="hyperParameters">Гиперпараметры.</param>
/// <param name="topology">Топология сети.</param>
public TrainUtil(DataSet dataSet, TrainType trainType,
HyperParameters hyperParameters, Topology topology)
{
if (!trainType.Equals(TrainType.Backpropagation))
{
throw new NotImplementedException();
}
if (!topology.IsClosed)
{
throw new Exception("Топология сети не замкнута! Обучение невозможно!");
}
_trainType = trainType;
_dataSet = dataSet;
_hyperParameters = hyperParameters;
_topology = topology;
}
private static void dumpHyperParameters(HyperParameters hyperParameters)
{
Console.WriteLine("");
Console.WriteLine(" Hyper Parameters");
Console.WriteLine("======================================================");
Console.WriteLine(" Epochs: " + hyperParameters.Epochs);
Console.WriteLine(" Mini Batch Size: " + hyperParameters.MiniBatchSize);
Console.WriteLine(" Cost Function: " + hyperParameters.CostFunctionName);
Console.WriteLine(" Learning Rate: " + hyperParameters.LearningRate);
Console.WriteLine(" Use Dropouts: " + hyperParameters.UseDropouts);
Console.WriteLine("Hidden Layer Dropout Probability: " + hyperParameters.HiddenLayerDropoutProbability);
Console.WriteLine(" Input Layer Dropout Probability: " + hyperParameters.InputLayerDropoutProbability);
Console.WriteLine(" Regulation Lambda: " + hyperParameters.RegulationLambda);
Console.WriteLine(" Test Size: " + hyperParameters.TestSize);
Console.WriteLine(" Auto-save: " + hyperParameters.AutoSave);
Console.WriteLine(" Auto-save Threshold: " + hyperParameters.AutoSaveThreshold);
Console.WriteLine("======================================================");
Console.WriteLine("");
}
static void Main(string[] args)
{
const string dataRoot = @"C:\HaishiRooster\Data\MINST";
//Load training data
var trainingImages = MINSTDataLoader.LoadImages(Path.Combine(dataRoot, "train-images-idx3-ubyte.gz"));
var trainingLabels = MINSTDataLoader.LoadLabels(Path.Combine(dataRoot, "train-labels-idx1-ubyte.gz"));
var trainingSet = MINSTDataLoader.CombineImagesAndLabels(trainingImages, trainingLabels);
//Load test data
var testingImages = MINSTDataLoader.LoadImages(Path.Combine(dataRoot, "t10k-images-idx3-ubyte.gz"));
var testingLabels = MINSTDataLoader.LoadLabels(Path.Combine(dataRoot, "t10k-labels-idx1-ubyte.gz"));
var testingSet = MINSTDataLoader.CombineImagesAndLabels(testingImages, testingLabels);
//Print training set information and a sample image
Console.WriteLine("Training set size: " + trainingSet.Count);
Console.WriteLine("Testing set size: " + testingSet.Count);
int index = mRand.Next(trainingSet.Count);
Console.WriteLine(string.Format("Here's a random picture from the training set: #{0} ({1})", index, convertToByte(trainingSet[index].Label)));
MINSTDataVisualizer.PrintImage(trainingSet[index].Image, 28);
Network network;
if (args.Length > 0)
{
using (StreamReader reader = new StreamReader(File.OpenRead(args[0])))
{
network = Network.Load(reader);
}
Console.WriteLine("\nNetwork is loaded!");
dumpHyperParameters(network.HyperParameters);
}
else
{
////MSR
//HyperParameters hyperParameters = new HyperParameters { CostFunctionName = "QuadraticCost", Epochs = 30, MiniBatchSize = 10, LearningRate = 3, TestSize = 10, AutoSave = true, AutoSaveThreshold = 0.951 };
//network = new Network(hyperParameters, 784, 30, 10);
////Cross-entropy
//HyperParameters hyperParameters = new HyperParameters { CostFunctionName = "CrossEntropyCost", Epochs = 30, MiniBatchSize = 10, LearningRate = 0.5, TestSize = testingSet.Count, AutoSave = true, AutoSaveThreshold = 0.967 };
//network = new Network(hyperParameters, 784, 100, 10);
////Cross-entropy with regulation
//HyperParameters hyperParameters = new HyperParameters { CostFunctionName = "CrossEntropyCost", Epochs = 60, MiniBatchSize = 10, LearningRate = 0.1, RegulationLambda = 5.0, TestSize = testingSet.Count, AutoSave = true, AutoSaveThreshold = 0.98 };
//network = new Network(hyperParameters, 784, 100, 10);
////Cross-entropy with regulation - 120 epochs
//HyperParameters hyperParameters = new HyperParameters { CostFunctionName = "CrossEntropyCost", Epochs = 120, MiniBatchSize = 10, LearningRate = 0.1, RegulationLambda = 5.0, TestSize = testingSet.Count, AutoSave = true, AutoSaveThreshold = 0.98 };
//network = new Network(hyperParameters, 784, 100, 10);
////Cross-entropy with regulation - 120 epochs - with dropouts
HyperParameters hyperParameters = new HyperParameters {
CostFunctionName = "CrossEntropyCost", Epochs = 240, MiniBatchSize = 10, LearningRate = 1, RegulationLambda = 5.0, TestSize = testingSet.Count, AutoSave = true, AutoSaveThreshold = 0.98, UseDropouts = true
};
network = new Network(hyperParameters, 784, 100, 10);
hookupEvents(network);
dumpHyperParameters(hyperParameters);
//Train the network
network.Train(trainingSet, (actual, expected) =>
{
return(convertToByte(actual) == convertToByte(expected));
}, testingSet);
Console.WriteLine("\nNetwork is trained!");
}
//Now validate
while (true)
{
Console.Write(string.Format("\nPlease enter a test image index [0 - {0}]. Enter '-1' to evaluate all test images. Enter '-2' to exit:", testingSet.Count - 1));
if (int.TryParse(Console.ReadLine(), out index))
{
if (index == -1)
{
int count = 0;
Console.Write(string.Format("Detecting {0} pictures", testingSet.Count));
for (int i = 0; i < testingSet.Count; i++)
{
var detection = network.Detect(testingSet[i].Image);
if (convertToByte(detection) == convertToByte(testingSet[i].Label))
{
Console.Write(".");
count++;
}
else
{
Console.Write("X");
}
}
Console.WriteLine("\nDetected {0} out of {1} pictures, correct rate is: {2:0.0%}", count, testingSet.Count, count * 1.0 / testingSet.Count);
}
else if (index >= 0 && index < testingSet.Count)
{
Console.WriteLine(string.Format("Test image: #{0} ({1})", index, convertToByte(testingSet[index].Label)));
MINSTDataVisualizer.PrintImage(testingSet[index].Image, 28);
var detection = network.Detect(testingSet[index].Image);
Console.WriteLine(string.Format("\nDetected number: {0} - {1}", detection, convertToByte(detection) == convertToByte(testingSet[index].Label) ? "SUCCESS!" : "FAIL!"));
}
else if (index == -2)
{
break;
}
else if (index == -3)
{
Console.Write("Please enter file name: ");
string fileName = Console.ReadLine();
using (StreamWriter writer = new StreamWriter(File.Create(fileName)))
{
network.Save(writer);
}
}
}
}
}
public abstract void Train(NeuralNetwork.NeuralNetwork neuralNetwork, Matrix <double> trainingSet, Matrix <double> crossValidationSet, Matrix <double> trainingSetOutput, Matrix <double> crossValidationSetOutput, HyperParameters hyperParameters = null);
private void _train(NeuralNetwork.NeuralNetwork neuralNetwork, Matrix <double> trainingSet,
Matrix <double> crossValidationSet, Matrix <double> trainingSetOutput, Matrix <double> crossValidationSetOutput, HyperParameters hyperParamters = null)
{
TrainingAlgorithm.Train(neuralNetwork, trainingSet, crossValidationSet, trainingSetOutput,
crossValidationSetOutput, hyperParamters);
}
/// <summary>
/// Обновить ячейку с помощтю карт входа/выхода.
/// </summary>
/// <param name="cell">Ячейка.</param>
/// <param name="inputMap">Входная карта.</param>
/// <param name="outputMap">Выходная карта.</param>
/// <param name="filterMatrixSize">Размер матрицы фильтра.</param>
/// <param name="hyperParameters">Гиперпараметры.</param>
public static void UpdateCellByIOMaps(this ModifiedCell cell, FigureMap inputMap,
FigureMap outputMap, int filterMatrixSize, HyperParameters hyperParameters)
{
var xEndPoint = cell.X + filterMatrixSize;
var yEndPoint = cell.Y + filterMatrixSize;
var gradient = 0d;
var x = 0;
for (var xStartPoint = cell.X; xStartPoint < xEndPoint; ++xStartPoint)
{
var y = 0;
for (var yStartPoint = cell.Y; yStartPoint < yEndPoint; ++yStartPoint)
{
var cellFromInputMap = inputMap.Cells
.Find(c => c.X.Equals(xStartPoint) && c.Y.Equals(yStartPoint));
var cellFromOutputMap = outputMap.Cells.Find(c => c.X.Equals(x) && c.Y.Equals(y));
gradient += cellFromInputMap.Value * cellFromOutputMap.Value;
++y;
}
++x;
}
var deltaValue = hyperParameters.Epsilon * gradient + hyperParameters.Alpha * cell.LastValueDelta;
cell.UpdateLastDeltValue(deltaValue);
var newValue = cell.Value + deltaValue;
cell.UpdatedValue(newValue);
}
/// <summary>
/// Выполнить обучение.
/// </summary>
/// <param name="pathToSettings">Путь до сохранённого файла настроек.</param>
private static void DoTrain(out string pathToSettings)
{
Console.Clear();
ConsoleExtensions.WriteWithColors(ConsoleColor.Black, ConsoleColor.Green,
"Вас приветствует обучение!\nУкажите директорию обучающей выборки " +
"(enter для директории по-умолчанию):");
var input = Console.ReadLine();
if (input.Equals(string.Empty))
{
input = PathHelper.GetResourcesPath();
}
if (!Directory.Exists(input))
{
throw new Exception($"Указанная директория не существует!\nДиректория: {input}");
}
var directories = Directory.GetDirectories(input).ToList();
var matrixDictionary = new Dictionary <int, List <double[, ]> >();
var key = 0;
foreach (var directory in directories)
{
var files = Directory.GetFiles(directory).ToList();
if (!files.Any())
{
throw new Exception($"Файлы не найдены!\nДиректория: {directory}");
}
var images = PathToImageConverter.LoadImages(files);
var resizedImages = NormilizeUtil.ResizeImages(images, 6, 6);
var normilizedMatrixies = NormilizeUtil.GetNormilizedMatrixesFromImages(resizedImages);
matrixDictionary.Add(key, normilizedMatrixies);
++key;
}
var hyperParameters = new HyperParameters
{
EpochCount = 1,
Epsilon = 0.75,
Alpha = 0.001
};
var topology = new Topology();
topology.Add(2, 2, LayerType.Convolution);
topology.Add(3, 2, LayerType.Subsampling);
topology.Add(4, 2, LayerType.Hidden, true);
if (!topology.IsClosed)
{
throw new Exception("Не удалось замкнуть топологию.");
}
var dataSet = new DataSet(DataSetType.ForNumberRecognizing);
if (!matrixDictionary.Count.Equals(dataSet.MaxCountInDataSet()))
{
throw new Exception("Не соответсвие количества выборок для распознавания чисел!");
}
foreach (var pair in matrixDictionary)
{
dataSet.Add(pair.Value);
}
var trainUtil = new TrainUtil(dataSet, TrainType.Backpropagation, hyperParameters, topology);
trainUtil.Start(3, 2, out var error, out pathToSettings);
var errorString = $"{Math.Round(error * 100, 2)}%";
ConsoleExtensions.WriteWithColors(ConsoleColor.Black, ConsoleColor.Yellow,
$"\nОшибка: {errorString}");
if (pathToSettings.Equals(string.Empty))
{
ConsoleExtensions.WriteWithColors(ConsoleColor.Black, ConsoleColor.Red,
"\nДанные не были сохранены!");
}
else
{
ConsoleExtensions.WriteWithColors(ConsoleColor.Black, ConsoleColor.Blue,
$"\nДанные сохранены!\nДиректория: {pathToSettings}");
}
}
public void Train(NeuralNetwork.NeuralNetwork neuralNetwork, Matrix <double> inputs, Matrix <double> outputs, HyperParameters hyperParamters = null)
{
if (TrainingAlgorithm is LevenbergAlgorithm)
{
_isLeven = true;
}
else
{
_isLeven = false;
}
TrainingAlgorithm.Subscribe(this);
var temp = Divider.Divide(inputs, outputs);
Console.WriteLine(temp.Item1);
Console.WriteLine(temp.Item2);
_train(neuralNetwork, temp.Item1, temp.Item3, temp.Item2, temp.Item4, hyperParamters);
}
public override void Train(NeuralNetwork.NeuralNetwork neuralNetwork, Matrix <double> trainingSet, Matrix <double> crossValidationSet, Matrix <double> trainingSetOutput, Matrix <double> crossValidationSetOutput, HyperParameters hyperParameters = null)
{
double maxError = 0.01, error = 5, momentum = 0.9;
int maxEpochs = 1000, epochs = 0;
if (hyperParameters != null)
{
if (maxEpochs <= 0)
{
throw new ArgumentException("Max Epochs cannot be negative");
}
if (maxError > 2 || maxError < 0)
{
throw new ArgumentException("Max error cannot be negative or very large");
}
maxError = hyperParameters.MaxError;
maxEpochs = hyperParameters.MaxEpochs;
momentum = hyperParameters.Momentum;
}
TrainingErrorMessage message = new TrainingErrorMessage()
{
NeuralNetwork = neuralNetwork, TrainingSet = trainingSet, CrossValidationSet = crossValidationSet, TrainingSetOutput = trainingSetOutput, CrossValidationSetOutput = crossValidationSetOutput
};
var layers = neuralNetwork.Layers;
var weights = neuralNetwork.HiddenWeights;
//for momentum
List <Matrix <double> > prevDeltaW = new List <Matrix <double> >();
for (int i = 0; i < weights.Count; ++i)
{
prevDeltaW.Add(Matrix <double> .Build.Dense(layers[i + 1].NeuronsNumber, layers[i].NeuronsNumber + 1));
}
//end
//epochs++ => ++epochs
while (error >= maxError && ++epochs <= maxEpochs)
{
prevDeltaW.ForEach(e => e.Clear());
for (int i = 0; i < trainingSet.RowCount; i++)
{
Vector <double> input = trainingSet.Row(i),
output = trainingSetOutput.Column(i);
var temp = neuralNetwork.ForwardInput(input);
IList <Vector <double> > acs = temp.Item1, gs = temp.Item2;
var D = (output - acs[acs.Count - 1]).PointwiseMultiply(gs[gs.Count - 1]).ToColumnMatrix(); // n(output) * 1
var deltaW = D * acs[acs.Count - 2].ToRowMatrix() * layers[layers.Count - 1].LearningRate; // (n(output) * 1) * ((n(output-1)+1) * 1)' = n(output) * (n(output-1)+1)
//for momentum
deltaW += computeAdditionalTerms(prevDeltaW[weights.Count - 1], momentum);
prevDeltaW[weights.Count - 1] = deltaW;
//end
neuralNetwork.UpdateWeightsAt(deltaW, weights.Count - 1);
for (int j = layers.Count - 2; j > 0; j--)
{
D = (weights[j].Transpose() * D).RemoveRow(0).PointwiseMultiply(gs[j - 1].ToColumnMatrix()); // (n(j+1) * (n(j)+1))' * (n(j+1) * 1) = (n(j)+1) * 1, then => (n(j) * 1) .* (n(j) * 1)
deltaW = D * acs[j - 1].ToRowMatrix() * layers[j].LearningRate; // (n(j) * 1) * ((n(j-1)+1) * 1)' = n(j) * (n(j-1)+1)
//for momentum
deltaW += computeAdditionalTerms(prevDeltaW[j - 1], momentum);
prevDeltaW[j - 1] = deltaW;
//end
neuralNetwork.UpdateWeightsAt(deltaW, j - 1);
}
}
message.Epochs = epochs;
base.Notify(message);
error = message.Error;
Console.WriteLine(error);
}
base.OnComplete();
}
public override void Train(NeuralNetwork.NeuralNetwork neuralNetwork, Matrix <double> trainingSet,
Matrix <double> crossValidationSet, Matrix <double> trainingSetOutput, Matrix <double> crossValidationSetOutput,
HyperParameters hyperParameters = null)
{
//Func<Math.IActivatorFunction, IActivationFunction> to = (fun) =>
//{
// if (fun is Math.SigmoidFunction)
// return new SigmoidFunction();
// if (fun is Math.IdentityFunction)
// return new IdentityFunction();
// return new SigmoidFunction();
//};
//IList<ActivationLayer> layers = new List<ActivationLayer>();
//layers.Add(new ActivationLayer(neuralNetwork.HiddenWeights[0].RowCount, trainingSet.ColumnCount, to(neuralNetwork.Layers[0].Applier.ActivatorFunction)));
//for (int i = 1; i < neuralNetwork.HiddenWeights.Count; i++)
//{
// layers.Add(new ActivationLayer(neuralNetwork.HiddenWeights[i - 1].RowCount, trainingSet.ColumnCount, to(neuralNetwork.Layers[i].Applier.ActivatorFunction)));
//}
double maxError = 0.01, error = 5, lr = 0.01;
int maxEpochs = 1000, epochs = 0;
if (hyperParameters != null)
{
if (maxEpochs <= 0)
{
throw new ArgumentException("Max Epochs cannot be negative");
}
if (maxError > 2 || maxError < 0)
{
throw new ArgumentException("Max error cannot be negative or very large");
}
maxError = hyperParameters.MaxError;
maxEpochs = hyperParameters.MaxEpochs;
lr = hyperParameters.Lr;
}
ActivationNetwork network = new ActivationNetwork(new SigmoidFunction(2), trainingSet.ColumnCount, neuralNetwork.Layers.Select(x => x.NeuronsNumber).ToArray());
LevenbergMarquardtLearning teacher = new LevenbergMarquardtLearning(network, true)
{
LearningRate = lr
};
TrainingErrorMessage message = new TrainingErrorMessage()
{
NeuralNetwork = neuralNetwork,
TrainingSet = trainingSet,
CrossValidationSet = crossValidationSet,
TrainingSetOutput = trainingSetOutput,
CrossValidationSetOutput = crossValidationSetOutput
};
int iterations = 1;
double[][] inputs = new double[trainingSet.RowCount][],
crossInputs = new double[crossValidationSet.RowCount][],
outputs = new double[trainingSetOutput.RowCount][],
crossOutputs = new double[crossValidationSetOutput.RowCount][];
for (int i = 0; i < trainingSet.RowCount; i++)
{
inputs[i] = new double[trainingSet.ColumnCount];
for (int j = 0; j < trainingSet.ColumnCount; j++)
{
inputs[i][j] = trainingSet[i, j];
}
}
for (int i = 0; i < trainingSetOutput.RowCount; i++)
{
outputs[i] = new double[trainingSetOutput.ColumnCount];
for (int j = 0; j < trainingSet.ColumnCount; j++)
{
outputs[i][j] = trainingSetOutput[i, j];
}
}
for (int i = 0; i < crossValidationSet.RowCount; i++)
{
crossInputs[i] = new double[crossValidationSet.ColumnCount];
for (int j = 0; j < crossValidationSet.ColumnCount; j++)
{
crossInputs[i][j] = crossValidationSet[i, j];
}
}
for (int i = 0; i < crossValidationSetOutput.RowCount; i++)
{
crossOutputs[i] = new double[crossValidationSetOutput.ColumnCount];
for (int j = 0; j < crossValidationSetOutput.ColumnCount; j++)
{
crossOutputs[i][j] = crossValidationSetOutput[i, j];
}
}
while (error > maxError && iterations++ <= maxEpochs)
{
message.Epochs = iterations;
error = teacher.RunEpoch(inputs, outputs);
message.TrainError = error;
message.CrossError = teacher.ComputeError(crossInputs, crossOutputs);
base.Notify(message);
}
//double mue = 0.001, mue_adj = 10, max_mue = 1e10;
//base.Notify(message);
//double currentError = message.Error;
//while (currentError >= maxError && epochs++ < maxEpochs)
//{
// message.Epochs = epochs;
// var temp = HissienAndGragient(neuralNetwork, trainingSet, trainingSetOutput);
// var hessien = temp.Item1;
// var gradient = temp.Item2;
// Matrix<double> blendingMatrix = Matrix<double>.Build.DenseDiagonal(hessien.RowCount, hessien.ColumnCount);
// var prevW = neuralNetwork.HiddenWeights.ToList();
// //Console.WriteLine("prev :");
// //prevW.ForEach(Console.WriteLine);
// double nextError = 100000;
// while (true)
// {
// var term = hessien + mue*blendingMatrix;
// var det = term.Determinant();
// if (System.Math.Abs(det) > 0)
// {
// var deltaW = term*gradient;
// neuralNetwork.UpdateWeightsFromVector(deltaW);
// //Console.WriteLine("updated :");
// //neuralNetwork.HiddenWeights.ForEach(Console.WriteLine);
// base.Notify(message);
// nextError = message.Error;
// }
// if (!(System.Math.Abs(det) > 0) || nextError >= currentError)
// {
// neuralNetwork.SetWeights(prevW);
// //Console.WriteLine("set to prev :");
// //neuralNetwork.HiddenWeights.ForEach(Console.WriteLine);
// mue *= mue_adj;
// if (mue > max_mue)
// {
// mue = max_mue;
// break;
// }
// }
// else
// {
// mue /= mue_adj;
// currentError = nextError;
// //Console.WriteLine("the shit is here");
// break;
// }
// }
//}
}
public NelderMeadBuilder(DecisionSpace decisionSpace, HyperParameterManager hyperParameters)
{
this.decisionSpace = decisionSpace;
HyperParameters.AddFromExistingHyperParameterSet(hyperParameters);
}
/// <summary>
/// Получить дельту веса.
/// </summary>
/// <param name="hyperParameters">Гиперпараметры.</param>
/// <param name="gradient">Градиент.</param>
/// <param name="lastDeltaValue">Последние значение дельты веса.</param>
/// <returns>Возвращает дельту веса.</returns>
internal static double GetWeightsDelta(HyperParameters hyperParameters,
double gradient, double lastDeltaValue) =>
hyperParameters.Epsilon * gradient + hyperParameters.Alpha * lastDeltaValue;
/// <summary>
/// Получить дельту веса.
/// </summary>
/// <param name="hyperParameters">Гиперпараметры.</param>
/// <param name="outputNeuron">Выходной связанные нейрон.</param>
/// <param name="gradient">Градиент.</param>
/// <param name="indexOfCurrentNeuron">Индекс текущего нейрона.</param>
/// <returns>Возвращает дельту веса.</returns>
internal static double GetWeightsDelta(HyperParameters hyperParameters, Neuron outputNeuron,
double gradient, int indexOfCurrentNeuron) =>
hyperParameters.Epsilon * gradient +
hyperParameters.Alpha *
outputNeuron.LastWeightsDeltas[indexOfCurrentNeuron];
public GeneticSpecimen(string file)
{
FilePath = file;
hyperParameters = new HyperParameters(FilePath);
CreateChildren();
}
public HyperParameters GetHyperParameters()
{
return(HyperParameters.GetDefaults());
}
