internal static QuadraticInterpolationModel Construct(MultiFunctor f, IReadOnlyList <double> x, double s) { QuadraticInterpolationModel model = new QuadraticInterpolationModel(); model.Initialize(f, x, s); return(model); }
public static QuadraticInterpolationModel Construct(double[][] points, double[] values) { int m = points.Length; int d = points[0].Length; // find the minimum point, use it as the origin int iMin = 0; double fMin = values[0]; for (int i = 1; i < values.Length; i++) { if (values[i] < fMin) { iMin = i; fMin = values[i]; } } SquareMatrix A = new SquareMatrix(m); int c = 0; for (int r = 0; r < m; r++) { A[r, 0] = 1.0; } for (int i = 0; i < d; i++) { c++; for (int r = 0; r < m; r++) { A[r, c] = points[r][i] - points[iMin][i]; } } for (int i = 0; i < d; i++) { for (int j = 0; j <= i; j++) { c++; for (int r = 0; r < m; r++) { A[r, c] = (points[r][i] - points[iMin][i]) * (points[r][j] - points[iMin][j]); } } } ColumnVector b = new ColumnVector(values); SquareQRDecomposition QR = A.QRDecomposition(); ColumnVector a = QR.Solve(b); QuadraticInterpolationModel model = new QuadraticInterpolationModel(); model.d = d; model.origin = points[iMin]; model.f0 = a[0]; model.g = new double[d]; c = 0; for (int i = 0; i < d; i++) { c++; model.g[i] = a[c]; } model.h = new double[d][]; for (int i = 0; i < d; i++) { model.h[i] = new double[d]; } for (int i = 0; i < d; i++) { for (int j = 0; j <= i; j++) { c++; if (i == j) { model.h[i][j] = 2.0 * a[c]; } else { model.h[i][j] = a[c]; model.h[j][i] = a[c]; } } } return(model); }
public static QuadraticInterpolationModel Construct(double[][] points, double[] values) { int m = points.Length; int d = points[0].Length; // find the minimum point, use it as the origin int iMin = 0; double fMin = values[0]; for (int i = 1; i < values.Length; i++) { if (values[i] < fMin) { iMin = i; fMin = values[i]; } } SquareMatrix A = new SquareMatrix(m); int c = 0; for (int r = 0; r < m; r++) { A[r, 0] = 1.0; } for (int i = 0; i < d; i++) { c++; for (int r = 0; r < m; r++) { A[r, c] = points[r][i] - points[iMin][i]; } } for (int i = 0; i < d; i++) { for (int j = 0; j <= i; j++) { c++; for (int r = 0; r < m; r++) { A[r, c] = (points[r][i] - points[iMin][i]) * (points[r][j] - points[iMin][j]); } } } ColumnVector b = new ColumnVector(values); SquareQRDecomposition QR = A.QRDecomposition(); ColumnVector a = QR.Solve(b); QuadraticInterpolationModel model = new QuadraticInterpolationModel(); model.d = d; model.origin = points[iMin]; model.f0 = a[0]; model.g = new double[d]; c = 0; for (int i = 0; i < d; i++) { c++; model.g[i] = a[c]; } model.h = new double[d][]; for (int i = 0; i < d; i++) { model.h[i] = new double[d]; } for (int i = 0; i < d; i++) { for (int j = 0; j <= i; j++) { c++; if (i == j) { model.h[i][j] = 2.0 * a[c]; } else { model.h[i][j] = a[c]; model.h[j][i] = a[c]; } } } return (model); }
// This method is due to Powell (http://en.wikipedia.org/wiki/Michael_J._D._Powell), but it is not what // is usually called Powell's Method (http://en.wikipedia.org/wiki/Powell%27s_method); Powell // developed that method in the 1960s, it was included in Numerical Recipes and is very popular. // This is a model trust algorithm developed by Powell in the 2000s. It typically uses many // fewer function evaluations, but does more intensive calculations between each evaluation. // This is basically the UOBYQA variant of Powell's new methods. It maintains a quadratic model // that interpolates between (d + 1) (d + 2) / 2 points. The model is trusted // within a given radius. At each step, it moves to the minimum of the model (or the boundary of // the trust region in that direction) and evaluates the function. The new value is incorporated // into the model and the trust region expanded or contracted depending on how accurate its // prediction of the function value was. // Papers on these methods are collected at http://mat.uc.pt/~zhang/software.html#powell_software. // The UOBYQA paper is here: http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.28.1756. // The NEWUOA paper is here: http://www.damtp.cam.ac.uk/user/na/NA_papers/NA2004_08.pdf. // The CONDOR system (http://www.applied-mathematics.net/optimization/CONDORdownload.html) is based on these same ideas. // The thesis of CONDOR's author (http://www.applied-mathematics.net/mythesis/index.html) was also helpful. // It should be very easy to extend this method to constrained optimization, either by incorporating the bounds into // the step limits or by mapping hyper-space into a hyper-cube. private static MultiExtremum FindMinimum_ModelTrust(MultiFunctor f, IReadOnlyList <double> x, double s, MultiExtremumSettings settings) { // Construct an initial model. QuadraticInterpolationModel model = QuadraticInterpolationModel.Construct(f, x, s); double trustRadius = s; while (f.EvaluationCount < settings.EvaluationBudget) { // Find the minimum point of the model within the trust radius double[] z = model.FindMinimum(trustRadius); double expectedValue = model.Evaluate(z); double deltaExpected = model.MinimumValue - expectedValue; // Evaluate the function at the suggested minimum double[] point = model.ConvertPoint(z); double value = f.Evaluate(point); double delta = model.MinimumValue - value; double tol = settings.ComputePrecision(Math.Min(model.MinimumValue, value)); // Note value can be way off, so use better of old best and new value to compute tol. // When we didn't do this before, we got value = infinity, so tol = infinity, and thus terminated! if (delta > 0.0 && settings.Listener != null) { MultiExtremum report = new MultiExtremum(f.EvaluationCount, settings, point, value, Math.Max(Math.Abs(delta), 0.75 * tol), model.GetHessian()); settings.Listener(report); } // To terminate, we demand: a reduction, that the reduction be small, that the reduction be in line with // its expected value, that we have not run up against the trust boundary, and that the gradient is small. // I had wanted to demand delta > 0, but we run into some cases where delta keeps being very slightly // negative, typically orders of magnitude less than tol, causing the trust radius to shrink in an // endless cycle that causes our approximation to ultimately go sour, even though terminating on the original // very slightly negative delta would have produced an accurate estimate. So we tolerate this case for now. if ((delta <= tol) && (-0.25 * tol <= delta)) { // We demand that the model be decent, i.e. that the expected delta was within tol of the measured delta. if (Math.Abs(delta - deltaExpected) <= tol) { // We demand that the step not just be small because it ran up against the trust radius. // If it ran up against the trust radius, there is probably more to be hand by continuing. double zm = Blas1.dNrm2(z, 0, 1, z.Length); if (zm < trustRadius) { // Finally, we demand that the gradient be small. You might think this was obvious since // z was small, but if the Hessian is not positive definite // the interplay of the Hessian and the gradient can produce a small z even if the model looks nothing like a quadratic minimum. double gm = Blas1.dNrm2(model.GetGradient(), 0, 1, z.Length); if (gm * zm <= tol) { if (f.IsNegated) { value = -value; } return(new MultiExtremum(f.EvaluationCount, settings, point, value, Math.Max(Math.Abs(delta), 0.75 * tol), model.GetHessian())); } } } } // There are now three decisions to be made: // 1. How to change the trust radius // 2. Whether to accept the new point // 3. Which existing point to replace // If the actual change was very far from the expected change, reduce the trust radius. // If the expected change did a good job of predicting the actual change, increase the trust radius. if ((delta < 0.25 * deltaExpected) /*|| (8.0 * deltaExpected < delta)*/) { trustRadius = 0.5 * trustRadius; } else if ((0.75 * deltaExpected <= delta) /*&& (delta <= 2.0 * deltaExpected)*/) { trustRadius = 2.0 * trustRadius; } // It appears that the limits on delta being too large don't help, and even hurt if made too stringent. // Replace an old point with the new point. int iMax = 0; double fMax = model.values[0]; int iBad = 0; double fBad = model.ComputeBadness(0, z, point, value); for (int i = 1; i < model.values.Length; i++) { if (model.values[i] > fMax) { iMax = i; fMax = model.values[i]; } double bad = model.ComputeBadness(i, z, point, value); if (bad > fBad) { iBad = i; fBad = bad; } } // Use the new point as long as it is better than our worst existing point. if (value < fMax) { Debug.Assert(!Double.IsPositiveInfinity(value) && !Double.IsNaN(value)); model.ReplacePoint(iBad, point, z, value); } // There is some question about how best to choose which point to replace. // The largest value? The furthest away? The one closest to new min? } throw new NonconvergenceException(); }
internal static QuadraticInterpolationModel Construct(MultiFunctor f, IList<double> x, double s) { QuadraticInterpolationModel model = new QuadraticInterpolationModel(); model.Initialize(f, x, s); return (model); }