internal MultiExtremum(int count, MultiExtremumSettings settings, double[] point, double value, double precision, double[][] hessian) : base(count)
{
Debug.Assert(settings != null);
Debug.Assert(point != null);
this.point = point;
this.value = value;
this.precision = precision;
this.hessian = hessian;
this.settings = settings;
}
private static void SetDefaultOptimizationSettings(MultiExtremumSettings settings, int d)
{
if (settings.RelativePrecision < 0.0)
{
settings.RelativePrecision = Math.Pow(10.0, -(10.0 + 4.0 / d));
}
if (settings.AbsolutePrecision < 0.0)
{
settings.AbsolutePrecision = Math.Pow(10.0, -(10.0 + 4.0 / d));
}
if (settings.EvaluationBudget < 0)
{
settings.EvaluationBudget = 16 * (d + 1) * (d + 2) * (d + 3);
}
}
private static DifferentialEvolutionSettings GetDefaultSettings(MultiExtremumSettings settings, int d)
{
if (settings == null)
{
settings = new MultiExtremumSettings();
}
DifferentialEvolutionSettings deSettings = new DifferentialEvolutionSettings();
deSettings.Population = 8 * d + 4;
deSettings.CrossoverProbability = 1.0 - 1.0 / 8.0 - 1.0 / d;
deSettings.RelativePrecision = (settings.RelativePrecision < 0.0) ? Math.Pow(10.0, -(2.0 + 4.0 / d)) : settings.RelativePrecision;
deSettings.AbsolutePrecision = (settings.AbsolutePrecision < 0.0) ? Math.Pow(10.0, -(4.0 + 8.0 / d)) : settings.AbsolutePrecision;
deSettings.EvaluationBudget = (settings.EvaluationBudget < 0) ? 128 * d * d * d * d : settings.EvaluationBudget;
deSettings.Listener = settings.Listener;
return(deSettings);
}
private static MultiExtremum FindGlobalExtremum(Func <IReadOnlyList <double>, double> function, IReadOnlyList <Interval> volume, MultiExtremumSettings settings, bool negate)
{
if (function == null)
{
throw new ArgumentNullException(nameof(function));
}
if (volume == null)
{
throw new ArgumentNullException(nameof(volume));
}
MultiFunctor f = new MultiFunctor(function, negate);
DifferentialEvolutionSettings deSettings = GetDefaultSettings(settings, volume.Count);
MultiExtremum extremum = FindGlobalExtremum(f, volume, deSettings);
return(extremum);
}
/// <summary>
/// Finds the maximum of a function within the given volume, subject to the given evaluation constraints.
/// </summary>
/// <param name="function">The function.</param>
/// <param name="volume">The volume to search.</param>
/// <param name="settings">The evaluation constraints to apply.</param>
/// <returns>The global maximum.</returns>
public static MultiExtremum FindGlobalMaximum(Func <IReadOnlyList <double>, double> function, IReadOnlyList <Interval> volume, MultiExtremumSettings settings)
{
return(FindGlobalExtremum(function, volume, settings, true));
}
/// <summary>
/// Finds a local maximum of a multi-dimensional function in the vicinity of the given starting location,
/// subject to the given evaluation constraints.
/// </summary>
/// <param name="function">The multi-dimensional function to maximize.</param>
/// <param name="start">The starting location for the search.</param>
/// <param name="settings">The evaluation settings that govern the search for the maximum.</param>
/// <returns>The local maximum.</returns>
public static MultiExtremum FindLocalMaximum(Func <IReadOnlyList <double>, double> function, IReadOnlyList <double> start, MultiExtremumSettings settings)
{
if (function == null)
{
throw new ArgumentNullException(nameof(function));
}
if (start == null)
{
throw new ArgumentNullException(nameof(start));
}
if (settings == null)
{
throw new ArgumentNullException(nameof(settings));
}
return(FindLocalExtremum(function, start, settings, true));
}
// This method is due to Powell (http://en.wikipedia.org/wiki/Michael_J._D._Powell), but it is not what
// is usually called Powell's Method (http://en.wikipedia.org/wiki/Powell%27s_method); Powell
// developed that method in the 1960s, it was included in Numerical Recipes and is very popular.
// This is a model trust algorithm developed by Powell in the 2000s. It typically uses many
// fewer function evaluations, but does more intensive calculations between each evaluation.
// This is basically the UOBYQA variant of Powell's new methods. It maintains a quadratic model
// that interpolates between (d + 1) (d + 2) / 2 points. The model is trusted
// within a given radius. At each step, it moves to the minimum of the model (or the boundary of
// the trust region in that direction) and evaluates the function. The new value is incorporated
// into the model and the trust region expanded or contracted depending on how accurate its
// prediction of the function value was.
// Papers on these methods are collected at http://mat.uc.pt/~zhang/software.html#powell_software.
// The UOBYQA paper is here: http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.28.1756.
// The NEWUOA paper is here: http://www.damtp.cam.ac.uk/user/na/NA_papers/NA2004_08.pdf.
// The CONDOR system (http://www.applied-mathematics.net/optimization/CONDORdownload.html) is based on these same ideas.
// The thesis of CONDOR's author (http://www.applied-mathematics.net/mythesis/index.html) was also helpful.
// It should be very easy to extend this method to constrained optimization, either by incorporating the bounds into
// the step limits or by mapping hyper-space into a hyper-cube.
private static MultiExtremum FindMinimum_ModelTrust(MultiFunctor f, IReadOnlyList <double> x, double s, MultiExtremumSettings settings)
{
// Construct an initial model.
QuadraticInterpolationModel model = QuadraticInterpolationModel.Construct(f, x, s);
double trustRadius = s;
while (f.EvaluationCount < settings.EvaluationBudget)
{
// Find the minimum point of the model within the trust radius
double[] z = model.FindMinimum(trustRadius);
double expectedValue = model.Evaluate(z);
double deltaExpected = model.MinimumValue - expectedValue;
// Evaluate the function at the suggested minimum
double[] point = model.ConvertPoint(z);
double value = f.Evaluate(point);
double delta = model.MinimumValue - value;
double tol = settings.ComputePrecision(Math.Min(model.MinimumValue, value));
// Note value can be way off, so use better of old best and new value to compute tol.
// When we didn't do this before, we got value = infinity, so tol = infinity, and thus terminated!
if (delta > 0.0 && settings.Listener != null)
{
MultiExtremum report = new MultiExtremum(f.EvaluationCount, settings, point, value, Math.Max(Math.Abs(delta), 0.75 * tol), model.GetHessian());
settings.Listener(report);
}
// To terminate, we demand: a reduction, that the reduction be small, that the reduction be in line with
// its expected value, that we have not run up against the trust boundary, and that the gradient is small.
// I had wanted to demand delta > 0, but we run into some cases where delta keeps being very slightly
// negative, typically orders of magnitude less than tol, causing the trust radius to shrink in an
// endless cycle that causes our approximation to ultimately go sour, even though terminating on the original
// very slightly negative delta would have produced an accurate estimate. So we tolerate this case for now.
if ((delta <= tol) && (-0.25 * tol <= delta))
{
// We demand that the model be decent, i.e. that the expected delta was within tol of the measured delta.
if (Math.Abs(delta - deltaExpected) <= tol)
{
// We demand that the step not just be small because it ran up against the trust radius.
// If it ran up against the trust radius, there is probably more to be hand by continuing.
double zm = Blas1.dNrm2(z, 0, 1, z.Length);
if (zm < trustRadius)
{
// Finally, we demand that the gradient be small. You might think this was obvious since
// z was small, but if the Hessian is not positive definite
// the interplay of the Hessian and the gradient can produce a small z even if the model looks nothing like a quadratic minimum.
double gm = Blas1.dNrm2(model.GetGradient(), 0, 1, z.Length);
if (gm * zm <= tol)
{
if (f.IsNegated)
{
value = -value;
}
return(new MultiExtremum(f.EvaluationCount, settings, point, value, Math.Max(Math.Abs(delta), 0.75 * tol), model.GetHessian()));
}
}
}
}
// There are now three decisions to be made:
// 1. How to change the trust radius
// 2. Whether to accept the new point
// 3. Which existing point to replace
// If the actual change was very far from the expected change, reduce the trust radius.
// If the expected change did a good job of predicting the actual change, increase the trust radius.
if ((delta < 0.25 * deltaExpected) /*|| (8.0 * deltaExpected < delta)*/)
{
trustRadius = 0.5 * trustRadius;
}
else if ((0.75 * deltaExpected <= delta) /*&& (delta <= 2.0 * deltaExpected)*/)
{
trustRadius = 2.0 * trustRadius;
}
// It appears that the limits on delta being too large don't help, and even hurt if made too stringent.
// Replace an old point with the new point.
int iMax = 0; double fMax = model.values[0];
int iBad = 0; double fBad = model.ComputeBadness(0, z, point, value);
for (int i = 1; i < model.values.Length; i++)
{
if (model.values[i] > fMax)
{
iMax = i; fMax = model.values[i];
}
double bad = model.ComputeBadness(i, z, point, value);
if (bad > fBad)
{
iBad = i; fBad = bad;
}
}
// Use the new point as long as it is better than our worst existing point.
if (value < fMax)
{
Debug.Assert(!Double.IsPositiveInfinity(value) && !Double.IsNaN(value));
model.ReplacePoint(iBad, point, z, value);
}
// There is some question about how best to choose which point to replace.
// The largest value? The furthest away? The one closest to new min?
}
throw new NonconvergenceException();
}
private static MultiExtremum FindLocalExtremum(Func <IReadOnlyList <double>, double> function, IReadOnlyList <double> start, MultiExtremumSettings settings, bool negate)
{
MultiFunctor f = new MultiFunctor(function, negate);
// Pick an initial trust region radius; we need to do this better.
double s = 0.0;
foreach (double x in start)
{
s += (Math.Abs(x) + 1.0 / 4.0) / 4.0;
}
s = s / start.Count;
Debug.WriteLine("s={0}", s);
SetDefaultOptimizationSettings(settings, start.Count);
return(FindMinimum_ModelTrust(f, start, s, settings));
}