// Brent's algorithm: use 3-point parabolic interpolation,
// switching to golden section if interval does not shrink fast enough
// see Richard Brent, "Algorithms for Minimization Without Derivatives"
// The bracket is [a, b] and the three lowest points are (u,fu), (v,fv), (w, fw)
// Note that a or b may be u, v, or w.
private static Extremum FindMinimum(
Functor f,
double a, double b,
double u, double fu, double v, double fv, double w, double fw,
EvaluationSettings settings
)
{
double tol = 0.0; double fpp = Double.NaN;
while (f.EvaluationCount < settings.EvaluationBudget)
{
Debug.WriteLine(String.Format("n={0} tol={1}", f.EvaluationCount, tol));
Debug.WriteLine(String.Format("[{0} f({1})={2} f({3})={4} f({5})={6} {7}]", a, u, fu, v, fv, w, fw, b));
// While a < u < b is guaranteed, a < v, w < b is not guaranteed, since the bracket can sometimes be made tight enough to exclude v or w.
// For example, if u < v < w, then we can set b = v, placing w outside the bracket.
Debug.Assert(a < b);
Debug.Assert((a <= u) && (u <= b));
Debug.Assert((fu <= fv) && (fv <= fw));
// Expected final situation is a<tol><tol>u<tol><tol>b, leaving no point left to evaluate that is not within tol of an existing point.
if ((b - a) <= 4.0 * tol)
{
return(new Extremum(u, fu, fpp, f.EvaluationCount, settings));
}
double x; ParabolicFit(u, fu, v, fv, w, fw, out x, out fpp);
Debug.WriteLine(String.Format("parabolic x={0} f''={1}", x, fpp));
if (Double.IsNaN(fpp) || (fpp <= 0.0) || (x < a) || (x > b))
{
// the parabolic fit didn't work out, so do a golden section reduction instead
// to get the most reduction of the bracket, pick the larger of au and ub
// for self-similarity, pick a point inside it that divides it into two segments in the golden section ratio,
// i.e. 0.3820 = \frac{1}{\phi + 1} and 0.6180 = \frac{\phi}{\phi+1}
// put the smaller segment closer to u so that x is closer to u, the best minimum so far
double au = u - a;
double ub = b - u;
if (au > ub)
{
x = u - au / (AdvancedMath.GoldenRatio + 1.0);
}
else
{
x = u + ub / (AdvancedMath.GoldenRatio + 1.0);
}
Debug.WriteLine(String.Format("golden section x={0}", x));
}
// ensure we don't evaluate within tolerance of an existing point
if (Math.Abs(x - u) < tol)
{
Debug.WriteLine(String.Format("shift from u (x={0})", x)); x = (x > u) ? u + tol : u - tol;
}
if ((x - a) < tol)
{
Debug.WriteLine(String.Format("shift from a (x={0})", x)); x = a + tol;
}
if ((b - x) < tol)
{
Debug.WriteLine(String.Format("shift from b (x={0})", x)); x = b - tol;
}
// evaluate the function at the new point x
double fx = f.Evaluate(x);
Debug.WriteLine(String.Format("f({0}) = {1}", x, fx));
Debug.WriteLine(String.Format("delta={0}", fu - fx));
// update a, b and u, v, w based on new point x
if (fx < fu)
{
// the new point is lower than all the others; this is success
// u now becomes a bracket point
if (u < x)
{
a = u;
}
else
{
b = u;
}
// x -> u -> v -> w
w = v; fw = fv;
v = u; fv = fu;
u = x; fu = fx;
}
else
{
// x now becomes a bracket point
if (x < u)
{
a = x;
}
else
{
b = x;
}
if (fx < fv)
{
// the new point is higher than u, but still lower than v and w
// this isn't what we expected, but we have lower points that before
// x -> v -> w
w = v; fw = fv;
v = x; fv = fx;
}
else if (fx < fw)
{
// x -> w
w = x; fw = fx;
}
else
{
// the new point is higher than all our other points; this is the worst case
// we might still want to replace w with x because
// (i) otherwise a parabolic fit will reproduce the same x and
// (ii) w is quite likely far outside the new bracket and not telling us much about the behavior near u
// w = x; fw = fx;
// but tests with Rosenbrock function indicate this increases evaluation count
Debug.WriteLine("bad point");
}
}
// if the user has specified a tollerance, use it
if ((settings.RelativePrecision > 0.0 || settings.AbsolutePrecision > 0.0))
{
tol = Math.Max(Math.Abs(u) * settings.RelativePrecision, settings.AbsolutePrecision);
}
else
{
// otherwise, try to get the tollerance from the curvature
if (fpp > 0.0)
{
tol = Math.Sqrt(2.0 * Global.Accuracy * (Math.Abs(fu) + Global.Accuracy) / fpp);
}
else
{
// but if we don't have a useable curvature either, wing it
if (tol == 0.0)
{
tol = Math.Sqrt(Global.Accuracy);
}
}
}
}
throw new NonconvergenceException();
}
// Brent's algorithm: use 3-point parabolic interpolation,
// switching to golden section if interval does not shrink fast enough
// see Richard Brent, "Algorithms for Minimization Without Derivatives"
// The bracket is [a, b] and the three lowest points are (u,fu), (v,fv), (w, fw)
// Note that a or b may be u, v, or w.
private static Extremum FindMinimum(
Functor f,
double a, double b,
double u, double fu, double v, double fv, double w, double fw,
EvaluationSettings settings
)
{
double tol = 0.0; double fpp = Double.NaN;
while (f.EvaluationCount < settings.EvaluationBudget) {
Debug.WriteLine(String.Format("n={0} tol={1}", f.EvaluationCount, tol));
Debug.WriteLine(String.Format("[{0} f({1})={2} f({3})={4} f({5})={6} {7}]", a, u, fu, v, fv, w, fw, b));
// While a < u < b is guaranteed, a < v, w < b is not guaranteed, since the bracket can sometimes be made tight enough to exclude v or w.
// For example, if u < v < w, then we can set b = v, placing w outside the bracket.
Debug.Assert(a < b);
Debug.Assert((a <= u) && (u <= b));
Debug.Assert((fu <= fv) && (fv <= fw));
// Expected final situation is a<tol><tol>u<tol><tol>b, leaving no point left to evaluate that is not within tol of an existing point.
if ((b - a) <= 4.0 * tol) return (new Extremum(u, fu, fpp, f.EvaluationCount, settings));
double x; ParabolicFit(u, fu, v, fv, w, fw, out x, out fpp);
Debug.WriteLine(String.Format("parabolic x={0} f''={1}", x, fpp));
if (Double.IsNaN(fpp) || (fpp <= 0.0) || (x < a) || (x > b)) {
// the parabolic fit didn't work out, so do a golden section reduction instead
// to get the most reduction of the bracket, pick the larger of au and ub
// for self-similarity, pick a point inside it that divides it into two segments in the golden section ratio,
// i.e. 0.3820 = \frac{1}{\phi + 1} and 0.6180 = \frac{\phi}{\phi+1}
// put the smaller segment closer to u so that x is closer to u, the best minimum so far
double au = u - a;
double ub = b - u;
if (au > ub) {
x = u - au / (AdvancedMath.GoldenRatio + 1.0);
} else {
x = u + ub / (AdvancedMath.GoldenRatio + 1.0);
}
Debug.WriteLine(String.Format("golden section x={0}", x));
}
// ensure we don't evaluate within tolerance of an existing point
if (Math.Abs(x - u) < tol) { Debug.WriteLine(String.Format("shift from u (x={0})", x)); x = (x > u) ? u + tol : u - tol; }
if ((x - a) < tol) { Debug.WriteLine(String.Format("shift from a (x={0})", x)); x = a + tol; }
if ((b - x) < tol) { Debug.WriteLine(String.Format("shift from b (x={0})", x)); x = b - tol; }
// evaluate the function at the new point x
double fx = f.Evaluate(x);
Debug.WriteLine(String.Format("f({0}) = {1}", x, fx));
Debug.WriteLine(String.Format("delta={0}", fu - fx));
// update a, b and u, v, w based on new point x
if (fx < fu) {
// the new point is lower than all the others; this is success
// u now becomes a bracket point
if (u < x) {
a = u;
} else {
b = u;
}
// x -> u -> v -> w
w = v; fw = fv;
v = u; fv = fu;
u = x; fu = fx;
} else {
// x now becomes a bracket point
if (x < u) {
a = x;
} else {
b = x;
}
if (fx < fv) {
// the new point is higher than u, but still lower than v and w
// this isn't what we expected, but we have lower points that before
// x -> v -> w
w = v; fw = fv;
v = x; fv = fx;
} else if (fx < fw) {
// x -> w
w = x; fw = fx;
} else {
// the new point is higher than all our other points; this is the worst case
// we might still want to replace w with x because
// (i) otherwise a parabolic fit will reproduce the same x and
// (ii) w is quite likely far outside the new bracket and not telling us much about the behavior near u
// w = x; fw = fx;
// but tests with Rosenbrock function indicate this increases evaluation count
Debug.WriteLine("bad point");
}
}
// if the user has specified a tollerance, use it
if ((settings.RelativePrecision > 0.0 || settings.AbsolutePrecision > 0.0)) {
tol = Math.Max(Math.Abs(u) * settings.RelativePrecision, settings.AbsolutePrecision);
} else {
// otherwise, try to get the tollerance from the curvature
if (fpp > 0.0) {
tol = Math.Sqrt(2.0 * Global.Accuracy * (Math.Abs(fu) + Global.Accuracy) / fpp);
} else {
// but if we don't have a useable curvature either, wing it
if (tol == 0.0) tol = Math.Sqrt(Global.Accuracy);
}
}
}
throw new NonconvergenceException();
}
private static Extremum FindMinimum(
Functor f,
double a, double b,
EvaluationSettings settings
)
{
// evaluate three points within the bracket
double u = (3.0 * a + b) / 4.0;
double v = (a + b) / 2.0;
double w = (a + 3.0 * b) / 4.0;
double fu = f.Evaluate(u); double fv = f.Evaluate(v); double fw = f.Evaluate(w);
Debug.WriteLine(String.Format("f({0})={1} f({2})={3} f({4})={5}", u, fu, v, fv, w, fw));
// move in the bracket boundaries, if possible
if (fv < fu) { a = u; if (fw < fv) a = v; }
if (fv < fw) { b = w; if (fu < fv) b = v; }
Debug.WriteLine(String.Format("[{0} {1}]", a, b));
// sort u, v, w by fu, fv, fw values
// these three comparisons are the most efficient three-item sort
if (fv < fu) { Global.Swap(ref v, ref u); Global.Swap(ref fv, ref fu); }
if (fw < fu) { Global.Swap(ref w, ref u); Global.Swap(ref fw, ref fu); }
if (fw < fv) { Global.Swap(ref w, ref v); Global.Swap(ref fw, ref fv); }
// pass to Brent's algorithm to shrink bracket
return (FindMinimum(f, a, b, u, fu, v, fv, w, fw, settings));
}
private static Extremum FindMinimum(Functor f, double x, double fx, double d, EvaluationSettings settings)
{
// This function brackets a minimum by starting from x and taking increasing steps downhill until it moves uphill again.
// We write this function assuming f(x) has already been evaluated because when it is called
// to do a line fit for Powell's multi-dimensional minimization routine, that is the case and
// we don't want to do a superfluous evaluation.
Debug.Assert(f != null); Debug.Assert(d > 0.0); Debug.Assert(settings != null);
// evaluate at x + d
double y = x + d;
double fy = f.Evaluate(y);
// if we stepped uphill, reverse direction of steps and exchange x & y
if (fy > fx) {
Global.Swap(ref x, ref y); Global.Swap(ref fx, ref fy);
d = -d;
}
// we now know f(x) >= f(y) and we are stepping downhill
// continue stepping until we step uphill
double z, fz;
while (true) {
if (f.EvaluationCount >= settings.EvaluationBudget) throw new NonconvergenceException();
z = y + d;
fz = f.Evaluate(z);
Debug.WriteLine(String.Format("f({0})={1} f({2})={3} f({4})={5} d={6}", x, fx, y, fy, z, fz, d));
if (fz > fy) break;
// increase the step size each time
d = AdvancedMath.GoldenRatio * d;
// x <- y <- z
x = y; fx = fy; y = z; fy = fz;
}
// we x and z now bracket a local minimum, with y the lowest point evaluated so far
double a = Math.Min(x, z); double b = Math.Max(x, z);
if (fz < fx) { Global.Swap(ref x, ref z); Global.Swap(ref fx, ref fz);}
return (FindMinimum(f, a, b, y, fy, x, fx, z, fz, settings));
}
private static Extremum FindMinimum(Functor f, double x, EvaluationSettings settings)
{
Debug.Assert(f != null); Debug.Assert(settings != null);
// To call the bracketing function, we need an initial value and an initial step.
double fx = f.Evaluate(x);
// Pick a relatively small initial value to try to avoid running into any nearly singularities.
double d = (Math.Abs(x) + 1.0 / 32.0) / 32.0;
return(FindMinimum(f, x, fx, d, settings));
}
private static Extremum FindMinimum(Functor f, double x, double fx, double d, ExtremumSettings settings)
{
// This function brackets a minimum by starting from x and taking increasing steps downhill until it moves uphill again.
// We write this function assuming f(x) has already been evaluated because when it is called
// to do a line fit for Powell's multi-dimensional minimization routine, that is the case and
// we don't want to do a superfluous evaluation.
Debug.Assert(f != null);
Debug.Assert(d > 0.0);
Debug.Assert(settings != null);
// evaluate at x + d
double y = x + d;
double fy = f.Evaluate(y);
// if we stepped uphill, reverse direction of steps and exchange x & y
if (fy > fx)
{
Global.Swap(ref x, ref y); Global.Swap(ref fx, ref fy);
d = -d;
}
// we now know f(x) >= f(y) and we are stepping downhill
// continue stepping until we step uphill
double z, fz;
while (true)
{
if (f.EvaluationCount >= settings.EvaluationBudget)
{
throw new NonconvergenceException();
}
z = y + d;
fz = f.Evaluate(z);
Debug.WriteLine($"f({x})={fx} f({y})={fy} f({z})={fz} d={d}");
if (fz > fy)
{
break;
}
// increase the step size each time
d = AdvancedMath.GoldenRatio * d;
// x <- y <- z
x = y; fx = fy; y = z; fy = fz;
}
// we x and z now bracket a local minimum, with y the lowest point evaluated so far
double a = Math.Min(x, z); double b = Math.Max(x, z);
if (fz < fx)
{
Global.Swap(ref x, ref z); Global.Swap(ref fx, ref fz);
}
return(FindMinimum(f, a, b, y, fy, x, fx, z, fz, settings));
}