public void RelativeConvergenceConstructorTest() { var criteria = new RelativeConvergence(iterations: 0, tolerance: 0.1); int progress = 1; do { // Do some processing... // Update current iteration information: criteria.NewValue = 12345.6 / progress++; } while (!criteria.HasConverged); // The method will converge after reaching the // maximum of 11 iterations with a final value // of 1234.56: int iterations = criteria.CurrentIteration; // 11 double value = criteria.OldValue; // 1234.56 Assert.AreEqual(11, criteria.CurrentIteration); Assert.AreEqual(1234.56, criteria.OldValue); }
/// <summary> /// Creates a new <see cref="ExpectationMaximization{TObservation}"/> algorithm. /// </summary> /// /// <param name="coefficients">The initial coefficient values.</param> /// <param name="distributions">The initial component distributions.</param> /// public ExpectationMaximization(double[] coefficients, IFittableDistribution <TObservation>[] distributions) { Coefficients = coefficients; Distributions = distributions; Convergence = new RelativeConvergence(0, 1e-3); Gamma = new double[coefficients.Length][]; }
private static TrainingSessionResult Optimize( SequentialNetwork network, BatchesCollection miniBatches, int epochs, float dropout, [NotNull] WeightsUpdater updater, [CanBeNull] IProgress <BatchProgress> batchProgress, [CanBeNull] IProgress <TrainingProgressEventArgs> trainingProgress, [CanBeNull] ValidationDataset validationDataset, [CanBeNull] TestDataset testDataset, CancellationToken token) { // Setup DateTime startTime = DateTime.Now; List <DatasetEvaluationResult> validationReports = new List <DatasetEvaluationResult>(), testReports = new List <DatasetEvaluationResult>(); TrainingSessionResult PrepareResult(TrainingStopReason reason, int loops) { return(new TrainingSessionResult(reason, loops, DateTime.Now.Subtract(startTime).RoundToSeconds(), validationReports, testReports)); } // Convergence manager for the validation dataset RelativeConvergence convergence = validationDataset == null ? null : new RelativeConvergence(validationDataset.Tolerance, validationDataset.EpochsInterval); // Optional batch monitor BatchProgressMonitor batchMonitor = batchProgress == null ? null : new BatchProgressMonitor(miniBatches.Count, batchProgress); // Create the training batches for (int i = 0; i < epochs; i++) { // Shuffle the training set miniBatches.CrossShuffle(); // Gradient descent over the current batches for (int j = 0; j < miniBatches.BatchesCount; j++) { if (token.IsCancellationRequested) { return(PrepareResult(TrainingStopReason.TrainingCanceled, i)); } network.Backpropagate(miniBatches.Batches[j], dropout, updater); batchMonitor?.NotifyCompletedBatch(miniBatches.Batches[j].X.GetLength(0)); } batchMonitor?.Reset(); // Check for overflows if (!Parallel.For(0, network._Layers.Length, (j, state) => { if (network._Layers[j] is WeightedLayerBase layer && !layer.ValidateWeights()) { state.Break(); } }).IsCompleted)
public void RelativeConvergenceConstructorTest4() { var criteria = new RelativeConvergence(iterations: 0, tolerance: 1e-5, startValue: 1); criteria.CurrentIteration = -2; criteria.NewValue /= 10.0; Assert.AreEqual(0.9, criteria.Delta, 1e-10); Assert.AreEqual(0.9, criteria.RelativeDelta, 1e-10); Assert.AreEqual(0.1, criteria.NewValue, 1e-10); Assert.AreEqual(1, criteria.OldValue, 1e-10); Assert.IsFalse(criteria.HasConverged); criteria.NewValue /= 10.0; Assert.AreEqual(0.09, criteria.Delta, 1e-10); Assert.AreEqual(0.9, criteria.RelativeDelta, 1e-10); Assert.AreEqual(0.01, criteria.NewValue, 1e-10); Assert.AreEqual(0.1, criteria.OldValue, 1e-10); Assert.IsFalse(criteria.HasConverged); criteria.NewValue /= 10.0; Assert.AreEqual(0.009, criteria.Delta, 1e-10); Assert.AreEqual(0.9, criteria.RelativeDelta, 1e-10); Assert.AreEqual(0.001, criteria.NewValue, 1e-10); Assert.AreEqual(0.01, criteria.OldValue, 1e-10); Assert.IsFalse(criteria.HasConverged); criteria.NewValue = criteria.NewValue * 1e-3; Assert.AreEqual(0.000999, criteria.Delta, 1e-10); Assert.AreEqual(0.999, criteria.RelativeDelta, 1e-10); Assert.AreEqual(1E-06, criteria.NewValue, 1e-10); Assert.AreEqual(0.001, criteria.OldValue, 1e-10); Assert.IsFalse(criteria.HasConverged); criteria.NewValue = criteria.NewValue - (criteria.NewValue * 1e-3); Assert.AreEqual(1e-9, criteria.Delta, 1e-10); Assert.AreEqual(1e-3, criteria.RelativeDelta, 1e-10); Assert.AreEqual(9.9899999999999988E-07, criteria.NewValue, 1e-10); Assert.AreEqual(1E-06, criteria.OldValue, 1e-10); Assert.IsFalse(criteria.HasConverged); criteria.NewValue = criteria.NewValue - (criteria.NewValue * 1e-5); Assert.AreEqual(9.999999999E-11, criteria.Delta, 1e-10); Assert.AreEqual(1e-5, criteria.RelativeDelta, 1e-10); Assert.AreEqual(9.9899000999999985E-07, criteria.NewValue, 1e-10); Assert.AreEqual(9.9899999999999988E-07, criteria.OldValue, 1e-10); Assert.IsFalse(criteria.HasConverged); criteria.NewValue = criteria.NewValue - (criteria.NewValue * 1e-6); Assert.AreEqual(9.999999999E-11, criteria.Delta, 1e-10); Assert.AreEqual(1.0000000000115289E-06, criteria.RelativeDelta, 1e-10); Assert.AreEqual(9.9898901100998984E-07, criteria.NewValue, 1e-10); Assert.AreEqual(9.9899000999999985E-07, criteria.OldValue, 1e-10); Assert.IsTrue(criteria.HasConverged); }
private void init(int numberOfVariables) { convergence = new RelativeConvergence(); gradient = new double[numberOfVariables]; previousGradient = new double[numberOfVariables]; weightsUpdates = new double[numberOfVariables]; // Initialize steps Reset(initialStep); }
/// <summary> /// Creates a new <see cref="ResilientBackpropagation"/> function optimizer. /// </summary> /// /// <param name="numberOfVariables">The number of parameters in the function to be optimized.</param> /// public ResilientBackpropagation(int numberOfVariables) : base(numberOfVariables) { convergence = new RelativeConvergence(); gradient = new double[numberOfVariables]; previousGradient = new double[numberOfVariables]; weightsUpdates = new double[numberOfVariables]; // Initialize steps Reset(initialStep); }
/// <summary> /// Creates a new <see cref="ResilientBackpropagation"/> function optimizer. /// </summary> /// /// <param name="numberOfVariables">The number of parameters in the function to be optimized.</param> /// public ResilientBackpropagation(int numberOfVariables) : base(numberOfVariables) { convergence = new RelativeConvergence(); gradient = new double[numberOfVariables]; previousGradient = new double[numberOfVariables]; weightsUpdates = new double[numberOfVariables]; // Initialize steps Reset(initialStep); }
/// <summary> /// Called when the <see cref="IOptimizationMethod{TInput, TOutput}.NumberOfVariables" /> property has changed. /// </summary> /// /// <param name="numberOfVariables">The number of variables.</param> /// protected override void OnNumberOfVariablesChanged(int numberOfVariables) { base.OnNumberOfVariablesChanged(numberOfVariables); convergence = new RelativeConvergence(); gradient = new double[numberOfVariables]; previousGradient = new double[numberOfVariables]; weightsUpdates = new double[numberOfVariables]; // Initialize steps Reset(initialStep); }
public void RelativeConvergenceConstructorTest3() { var criteria = new RelativeConvergence(iterations: 1, tolerance: 1e-5, startValue: 1); criteria.CurrentIteration = -2; do { criteria.NewValue /= 10.0; } while (!criteria.HasConverged); Assert.AreEqual(1, criteria.CurrentIteration); Assert.AreEqual(-2, Math.Log10(criteria.OldValue)); Assert.AreEqual(-3, Math.Log10(criteria.NewValue)); }
/// <summary> /// Initializes a new instance of the <see cref="AdaBoost<TModel>"/> class. /// </summary> /// /// <param name="model">The model to be learned.</param> /// <param name="creationFunction">The model fitting function.</param> /// public AdaBoost(Boost <TModel> model, ModelConstructor <TModel> creationFunction) { this.classifier = model; this.Creation = creationFunction; this.convergence = new RelativeConvergence(); }
/// <summary> /// Initializes a new instance of the <see cref="AdaBoost<TModel>"/> class. /// </summary> /// /// <param name="model">The model to be learned.</param> /// public AdaBoost(Boost <TModel> model) { this.classifier = model; this.convergence = new RelativeConvergence(); }
/// <summary> /// Creates a new instance of the Viterbi learning algorithm. /// </summary> /// protected BaseViterbiLearning() { this.convergence = new RelativeConvergence(); }
/// <summary> /// Initializes a new instance of the <see cref="BaseLeastSquaresMethod"/> class. /// </summary> /// public BaseLeastSquaresMethod() { this.convergence = new RelativeConvergence(0, 1e-5); }
private static TrainingSessionResult Optimize( NeuralNetworkBase network, BatchesCollection miniBatches, int epochs, float dropout, [NotNull] WeightsUpdater updater, [CanBeNull] IProgress <BatchProgress> batchProgress, [CanBeNull] IProgress <TrainingProgressEventArgs> trainingProgress, [CanBeNull] ValidationDataset validationDataset, [CanBeNull] TestDataset testDataset, CancellationToken token) { // Setup DateTime startTime = DateTime.Now; List <DatasetEvaluationResult> validationReports = new List <DatasetEvaluationResult>(), testReports = new List <DatasetEvaluationResult>(); TrainingSessionResult PrepareResult(TrainingStopReason reason, int loops) { return(new TrainingSessionResult(reason, loops, DateTime.Now.Subtract(startTime).RoundToSeconds(), validationReports, testReports)); } // Convergence manager for the validation dataset RelativeConvergence convergence = validationDataset == null ? null : new RelativeConvergence(validationDataset.Tolerance, validationDataset.EpochsInterval); // Optional batch monitor BatchProgressMonitor batchMonitor = batchProgress == null ? null : new BatchProgressMonitor(miniBatches.Count, batchProgress); // Create the training batches for (int i = 0; i < epochs; i++) { // Shuffle the training set miniBatches.CrossShuffle(); // Gradient descent over the current batches BackpropagationInProgress = true; for (int j = 0; j < miniBatches.BatchesCount; j++) { if (token.IsCancellationRequested) { BackpropagationInProgress = false; return(PrepareResult(TrainingStopReason.TrainingCanceled, i)); } network.Backpropagate(miniBatches.Batches[j], dropout, updater); batchMonitor?.NotifyCompletedBatch(miniBatches.Batches[j].X.GetLength(0)); } BackpropagationInProgress = false; batchMonitor?.Reset(); if (network.IsInNumericOverflow) { return(PrepareResult(TrainingStopReason.NumericOverflow, i)); } // Check the training progress if (trainingProgress != null) { (float cost, _, float accuracy) = network.Evaluate(miniBatches); trainingProgress.Report(new TrainingProgressEventArgs(i + 1, cost, accuracy)); } // Check the validation dataset if (convergence != null) { (float cost, _, float accuracy) = network.Evaluate(validationDataset.Dataset); validationReports.Add(new DatasetEvaluationResult(cost, accuracy)); convergence.Value = accuracy; if (convergence.HasConverged) { return(PrepareResult(TrainingStopReason.EarlyStopping, i)); } } // Report progress if necessary if (testDataset != null) { (float cost, _, float accuracy) = network.Evaluate(testDataset.Dataset); testReports.Add(new DatasetEvaluationResult(cost, accuracy)); testDataset.ThreadSafeProgressCallback?.Report(new TrainingProgressEventArgs(i + 1, cost, accuracy)); } } return(PrepareResult(TrainingStopReason.EpochsCompleted, epochs)); }
private void init(int numberOfVariables) { convergence = new RelativeConvergence(); gradient = new double[numberOfVariables]; previousGradient = new double[numberOfVariables]; weightsUpdates = new double[numberOfVariables]; // Initialize steps Reset(initialStep); }