public EvaluationModel Evaluate(InputModel[] testData) { var correctSolutions = 0; for (var i = 0; i < testData.Length; i++) { var model = testData[i]; var decision = Compute(model.Values); if (decision == model.Label) { correctSolutions++; } } var result = new EvaluationModel { Correct = correctSolutions, All = testData.Length }; return(result); }
public TrainingResult Train(TrainingModel trainingModel) { var trainingSet = trainingModel.TrainingSet; var testSet = trainingModel.TestSet; var validationSet = trainingModel.ValidationSet; var errorTreshold = trainingModel.ErrorThreshold; var maxEpochs = trainingModel.MaxEpochs; var batchSize = Math.Min(trainingModel.BatchSize, trainingModel.TrainingSet.Length); var learningRate = trainingModel.LearningRate; var lambda = trainingModel.Lambda; var momentum = trainingModel.Momentum; var isEncoder = trainingModel.IsEncoder; var takeBest = trainingModel.TakeBest; var isVerbose = trainingModel.IsVerbose; var evaluateOnEachEpoch = trainingModel.EvaluateOnEachEpoch; //Debugger.Launch(); IList <double> epochErrors = new List <double>(maxEpochs); var epochEvaluations = new List <EvaluationModel>(maxEpochs); var errorSum = double.PositiveInfinity; var epoch = 0; var layersCount = Sizes.Length - 1; #region create nablas arrays var nablaWeights = new Matrix <double> [layersCount]; var nablaBiases = new Vector <double> [layersCount]; for (var i = 0; i < layersCount; i++) { var nextLayer = Layers[i]; nablaBiases[i] = nextLayer.GetNewBiasesVector(true); nablaWeights[i] = nextLayer.GetNewWeightsMatrix(true); } #endregion if (isVerbose) { var activationFunctions = Layers.Select(l => l.CurrentActivationFunction.ToString()).ToArray(); var distributions = Layers.Select(l => l.InitialWeightsRange.ToString("#0.00")).ToArray(); Console.WriteLine("Starting with params:"); Console.WriteLine($"\tsizes- {JsonConvert.SerializeObject(Sizes)}"); Console.WriteLine($"\tlearning rate - {learningRate}"); Console.WriteLine($"\tmomentum- {momentum}"); Console.WriteLine($"\terror threshold - {errorTreshold}"); Console.WriteLine($"\tmax epochs - {maxEpochs}"); Console.WriteLine($"\tactivation functions - {JsonConvert.SerializeObject(activationFunctions, Formatting.None)}"); Console.WriteLine($"\tinitial weights ranges- {JsonConvert.SerializeObject(distributions, Formatting.None)}"); } InputModel sample = null; if (isEncoder) { Directory.CreateDirectory("encoder_logs"); sample = trainingSet[DateTime.Now.Ticks % trainingSet.Length]; var recreatedImage = MnistViewer.ToImage(sample.Values, 7); recreatedImage.Save($"encoder_logs/_original.png", ImageFormat.Png); } var prevWeightsChange = new Matrix <double> [layersCount]; var prevBiasChange = new Vector <double> [layersCount]; if (isVerbose && !isEncoder) { var initialPercentage = Evaluate(testSet).Percentage; Console.WriteLine($"Initial state, {initialPercentage.ToString("#0.00")}"); } #region log data //log data var path = "log.csv"; var log = new StringBuilder("sep=|"); log.AppendLine(); log.Append("epoch|evaluation_0|error_0"); log.AppendLine(); #endregion string bestNNJson = null; double bestError = double.PositiveInfinity; while (errorSum > errorTreshold && epoch < maxEpochs) { epoch++; errorSum = 0; var batch = HelperFunctions.RandomPermutation(trainingSet).Take(batchSize).ToList(); //Vector<double> activationsSum = new DenseVector(_hiddenLayers[0].NeuronsCount); //foreach (var item in batch) //{ // var activation = item.Values; // foreach (var layer in _hiddenLayers) // { // activation = layer.Feedforward(activation); // } // activationsSum += activation; //} //var avgActivations = activationsSum.Divide(batch.Count); foreach (var item in batch) { //const double beta = 0.5; //const double rho = 0.05; //var divergence = beta * (-rho / avgActivations + (1 - rho) / (1 - avgActivations)); //var bpResult = Backpropagate(item.Values, item.ExpectedSolution, divergence); var bpResult = Backpropagate(item.Values, item.ExpectedSolution); for (var i = 0; i < nablaWeights.Length; i++) { nablaBiases[i] = bpResult.Biases[i] + nablaBiases[i]; nablaWeights[i] = bpResult.Weights[i] + nablaWeights[i]; } var solution = bpResult.Solution; var expectedSolution = item.ExpectedSolution; errorSum += solution.Map2((y, o) => Math.Abs(y - o), expectedSolution).Sum(); } errorSum /= batchSize; #region update parameters for (var i = 0; i < layersCount; i++) { var weightsChange = learningRate / batchSize * nablaWeights[i]; if (prevWeightsChange[i] != null) { weightsChange += momentum * prevWeightsChange[i]; } //L2 var weights = Layers[i].Weights; if (lambda != 0) { weights = (1 - learningRate * lambda / batchSize) * weights; } Layers[i].Weights = weights - weightsChange; var biasesChange = learningRate / batchSize * nablaBiases[i]; if (prevBiasChange[i] != null) { biasesChange += momentum * prevBiasChange[i]; } //L2 var biases = Layers[i].Biases; if (lambda != 0) { biases = (1 - (learningRate * lambda) / batchSize) * biases; } Layers[i].Biases = biases - biasesChange; prevWeightsChange[i] = weightsChange; prevBiasChange[i] = biasesChange; } #endregion EvaluationModel epochEvaluation = null; if (evaluateOnEachEpoch) { epochEvaluation = Evaluate(testSet); epochEvaluations.Add(epochEvaluation); } if (isVerbose) { var percentage = isEncoder ? 0 : (epochEvaluation ?? Evaluate(testSet)).Percentage; Console.WriteLine($"Epoch - {epoch}," + $" error - {errorSum.ToString("#0.000")}," + $" test - {percentage.ToString("#0.00")}"); if (isEncoder) { var recreatedData = FeedForward(sample.Values); var recreatedImage = MnistViewer.ToImage(recreatedData, 7); recreatedImage.Save($"encoder_logs/{epoch}.png", ImageFormat.Png); } } #region dump data var eval = isEncoder ? 0 : (epochEvaluation ?? Evaluate(testSet)).Percentage; log.AppendLine(epoch + "|" + eval + "|" + errorSum); #endregion #region set nablas to zeroes for (var i = 0; i < layersCount; i++) { nablaBiases[i].Clear(); nablaWeights[i].Clear(); } #endregion epochErrors.Add(errorSum); if (takeBest && errorSum < bestError) { bestNNJson = ToJson(); } } #region log data File.WriteAllText(path, log.ToString()); #endregion var trainingResult = new TrainingResult { NeuralNetwork = takeBest ? FromJson(bestNNJson) : this, Epochs = epoch, EpochErrors = epochErrors.ToArray(), Evaluations = epochEvaluations.ToArray() }; return(trainingResult); }