/// <summary> /// Initializes a new instance of the <see cref="GradientCheckingMonitor"/> class. /// </summary> /// <param name="termCrit">The termination criterion</param> /// <param name="gradientCheckInterval">The gradient check interval.</param> public GradientCheckingMonitor(ITerminationCriterion termCrit, int gradientCheckInterval) { Contracts.CheckParam(gradientCheckInterval > 0, nameof(gradientCheckInterval), "gradientCheckInterval must be positive."); _termCrit = termCrit; _gradCheckInterval = gradientCheckInterval; }
/// <summary> /// Create an L1Optimizer with the supplied value of M and termination criterion /// </summary> /// <param name="env">The environment</param> /// <param name="biasCount">Number of biases</param> /// <param name="l1weight">Weight of L1 regularizer</param> /// <param name="m">The number of previous iterations to store</param> /// <param name="keepDense">Whether the optimizer will keep its internal state dense</param> /// <param name="term">Termination criterion</param> /// <param name="enforceNonNegativity">The flag enforcing the non-negativity constraint</param> public L1Optimizer(IHostEnvironment env, int biasCount, Float l1weight, int m = 20, bool keepDense = false, ITerminationCriterion term = null, bool enforceNonNegativity = false) : base(env, m, keepDense, term, enforceNonNegativity) { Env.Check(biasCount >= 0); Env.Check(l1weight >= 0); _biasCount = biasCount; _l1weight = l1weight; }
//Override default termination criterion MeanRelativeImprovementCriterion with protected override Optimizer InitializeOptimizer(IChannel ch, FloatLabelCursor.Factory cursorFactory, out VBuffer <float> init, out ITerminationCriterion terminationCriterion) { var opt = base.InitializeOptimizer(ch, cursorFactory, out init, out terminationCriterion); // MeanImprovementCriterion: // Terminates when the geometrically-weighted average improvement falls below the tolerance terminationCriterion = new MeanImprovementCriterion(OptTol, 0.25f, MaxIterations); return(opt); }
/// <summary> /// Create an optimizer with the supplied value of M and termination criterion /// </summary> /// <param name="env">The host environment</param> /// <param name="m">The number of previous iterations to store</param> /// <param name="keepDense">Whether the optimizer will keep its internal state dense</param> /// <param name="term">Termination criterion, defaults to MeanRelativeImprovement if null</param> /// <param name="enforceNonNegativity">The flag enforcing the non-negativity constraint</param> public Optimizer(IHostEnvironment env, int m = 20, bool keepDense = false, ITerminationCriterion term = null, bool enforceNonNegativity = false) { Contracts.CheckValue(env, nameof(env)); Env = env; M = m; KeepDense = keepDense; _staticTerm = term ?? new MeanRelativeImprovementCriterion(); TotalMemoryLimit = -1; EnforceNonNegativity = enforceNonNegativity; }
protected virtual Optimizer InitializeOptimizer(IChannel ch, FloatLabelCursor.Factory cursorFactory, out VBuffer <float> init, out ITerminationCriterion terminationCriterion) { // MeanRelativeImprovementCriterion: // Stops optimization when the average objective improvement over the last // n iterations, normalized by the function value, is small enough. terminationCriterion = new MeanRelativeImprovementCriterion(OptTol, 5, MaxIterations); Optimizer opt = (L1Weight > 0) ? new L1Optimizer(Host, BiasCount, L1Weight / NumGoodRows, MemorySize, DenseOptimizer, null, EnforceNonNegativity) : new Optimizer(Host, MemorySize, DenseOptimizer, null, EnforceNonNegativity); opt.Quiet = Quiet; if (_srcPredictor != null) { init = InitializeWeightsFromPredictor(_srcPredictor); } else if (InitWtsDiameter > 0) { float[] initWeights = new float[BiasCount + WeightCount]; for (int j = 0; j < initWeights.Length; j++) { initWeights[j] = InitWtsDiameter * (Host.Rand.NextSingle() - 0.5f); } init = new VBuffer <float>(initWeights.Length, initWeights); } else if (SgdInitializationTolerance > 0) { init = InitializeWeightsSgd(ch, cursorFactory); } else { init = VBufferUtils.CreateEmpty <float>(BiasCount + WeightCount); } return(opt); }
/// <summary> /// Minimize a function using the supplied termination criterion /// </summary> /// <param name="function">The function to minimize</param> /// <param name="initial">The initial point</param> /// <param name="term">termination criterion to use</param> /// <param name="result">The point at the optimum</param> /// <param name="optimum">The optimum function value</param> /// <exception cref="PrematureConvergenceException">Thrown if successive points are within numeric precision of each other, but termination condition is still unsatisfied.</exception> public void Minimize(DifferentiableFunction function, ref VBuffer <Float> initial, ITerminationCriterion term, ref VBuffer <Float> result, out Float optimum) { const string computationName = "LBFGS Optimizer"; using (var pch = Env.StartProgressChannel(computationName)) using (var ch = Env.Start(computationName)) { ch.Info("Beginning optimization"); ch.Info("num vars: {0}", initial.Length); ch.Info("improvement criterion: {0}", term.FriendlyName); OptimizerState state = MakeState(ch, pch, function, ref initial); term.Reset(); var header = new ProgressHeader(new[] { "Loss", "Improvement" }, new[] { "iterations", "gradients" }); pch.SetHeader(header, (Action <IProgressEntry>)(e => { e.SetProgress(0, (double)(state.Iter - 1)); e.SetProgress(1, state.GradientCalculations); })); bool finished = false; pch.Checkpoint(state.Value, null, 0); state.UpdateDir(); while (!finished) { bool success = state.LineSearch(ch, false); if (!success) { // problem may be numerical errors in previous gradients // try to save state of optimization by discarding them // and starting over with gradient descent. state.DiscardOldVectors(); state.UpdateDir(); state.LineSearch(ch, true); } string message; finished = term.Terminate(state, out message); double?improvement = null; double x; int end; if (message != null && DoubleParser.TryParse(out x, message, 0, message.Length, out end)) { improvement = x; } pch.Checkpoint(state.Value, improvement, state.Iter); if (!finished) { state.Shift(); state.UpdateDir(); } } state.X.CopyTo(ref result); optimum = state.Value; ch.Done(); } }