public static void FindExtremum_Amobea(Func <IList <double>, double> function, IList <double> x0, EvaluationSettings settings)
{
MultiFunctor f = new MultiFunctor(function);
int d = x0.Count;
Vertex[] vertexes = new Vertex[d + 1];
for (int i = 0; i < d; i++)
{
double[] x = new double[d];
x0.CopyTo(x, 0);
x[i] = x[i] + 1.0 + Math.Abs(x[i]);
double y = f.Evaluate(x);
vertexes[i] = new Vertex()
{
X = x, Y = y
};
}
double[] x00 = new double[d];
x0.CopyTo(x00, 0);
double y00 = f.Evaluate(x00);
vertexes[d] = new Vertex()
{
X = x00, Y = y00
};
FindExtremum_Amobea(f, vertexes, settings);
}
internal static QuadraticInterpolationModel Construct(MultiFunctor f, IReadOnlyList <double> x, double s)
{
QuadraticInterpolationModel model = new QuadraticInterpolationModel();
model.Initialize(f, x, s);
return(model);
}
private static MultiExtremum FindLocalExtremum(Func <IList <double>, double> function, IList <double> start, EvaluationSettings settings, bool negate)
{
MultiFunctor f = new MultiFunctor(function, negate);
// Pick an initial radius; we need to do this better.
/*
* double s = Double.MaxValue;
* foreach (double x in start) s = Math.Min((Math.Abs(x) + 1.0 / 8.0) / 8.0, s);
*/
double s = 0.0;
foreach (double x in start)
{
s += (Math.Abs(x) + 1.0 / 4.0) / 4.0;
}
s = s / start.Count;
//double s = 0.2;
Debug.WriteLine("s={0}", s);
return(FindMinimum_ModelTrust(f, start, s, settings));
}
private static UncertainValue Integrate_Adaptive(MultiFunctor f, CoordinateTransform[] map, IntegrationRegion r, EvaluationSettings settings)
{
// Create an evaluation rule
GenzMalik75Rule rule = new GenzMalik75Rule(r.Dimension);
// Use it on the whole region and put the answer in a linked list
rule.Evaluate(f, map, r);
LinkedList<IntegrationRegion> regionList = new LinkedList<IntegrationRegion>();
regionList.AddFirst(r);
// Iterate until convergence
while (f.EvaluationCount < settings.EvaluationBudget) {
// Add up value and errors in all regions.
// While we are at it, take note of the region with the largest error.
double value = 0.0;
double error = 0.0;
LinkedListNode<IntegrationRegion> regionNode = regionList.First;
double maxError = 0.0;
LinkedListNode<IntegrationRegion> maxErrorNode = null;
while (regionNode != null) {
IntegrationRegion region = regionNode.Value;
value += region.Value;
error += region.Error;
//error += MoreMath.Sqr(region.Error);
if (region.Error > maxError) {
maxError = region.Error;
maxErrorNode = regionNode;
}
regionNode = regionNode.Next;
}
// Check for convergence.
if ((error <= settings.AbsolutePrecision) || (error <= settings.RelativePrecision * Math.Abs(value))) {
return (new UncertainValue(value, error));
}
// Split the region with the largest error, and evaluate each subregion.
IntegrationRegion maxErrorRegion = maxErrorNode.Value;
regionList.Remove(maxErrorNode);
IList<IntegrationRegion> subRegions = maxErrorRegion.Split(maxErrorRegion.SplitIndex);
/*
Countdown cnt = new Countdown(2);
ThreadPool.QueueUserWorkItem((object state) => { rule.Evaluate(f, subRegions[0]); cnt.Signal(); });
ThreadPool.QueueUserWorkItem((object state) => { rule.Evaluate(f, subRegions[1]); cnt.Signal(); });
cnt.Wait();
foreach (IntegrationRegion subRegion in subRegions) {
regionList.AddLast(subRegion);
}
*/
foreach (IntegrationRegion subRegion in subRegions) {
rule.Evaluate(f, map, subRegion);
regionList.AddLast(subRegion);
}
}
throw new NonconvergenceException();
}
private static MultiExtremum FindGlobalExtremum(Func<IList<double>, double> function, IList<Interval> volume, EvaluationSettings settings, bool negate)
{
if (function == null) throw new ArgumentNullException("function");
if (volume == null) throw new ArgumentNullException("volume");
MultiFunctor f = new MultiFunctor(function, negate);
DifferentialEvolutionSettings deSettings = GetDefaultSettings(settings, volume.Count);
MultiExtremum extremum = FindGlobalExtremum(f, volume, deSettings);
return (extremum);
}
// Sample a pre-determined number of points using the given generator and grid and return the average function value.
private static double Integrate_MonteCarlo_Cycle(MultiFunctor f, CoordinateTransform[] map, VectorGenerator g, LePageGrid grid, int n)
{
double sum = 0.0;
for (int i = 0; i < n; i++)
{
double[] x = g.NextVector();
//sum += f.Evaluate(x);
sum += grid.Evaluate(f, map, x);
}
return(sum / n);
}
public double Evaluate(MultiFunctor f, CoordinateTransform[] map, double[] x)
{
Debug.Assert(x.Length == dimension);
// Increase the evaluation count.
count++;
// We will need to record the bin number into which each coordinate falls
// in order to acrue the result to the proper bin.
int[] binIndexes = new int[dimension];
// Map incoming x into a grid cell and value based on grid.
double v = v0;
for (int i = 0; i < x.Length; i++)
{
Debug.Assert((0.0 <= x[i]) && (x[i] < 1.0));
double z = (grid[i].Length - 1) * x[i];
int j = (int)Math.Floor(z);
z = z - j;
double w = grid[i][j + 1] - grid[i][j];
x[i] = (1.0 - z) * grid[i][j] + z * grid[i][j + 1];
v *= w;
binIndexes[i] = j;
}
// Use the map to further transform that value.
if (map != null)
{
Debug.Assert(map.Length == dimension);
for (int i = 0; i < x.Length; i++)
{
map[i].TransformInPlace(ref x[i], ref v);
}
}
double y = f.Evaluate(x);
// Record the value in the appropriate bins.
for (int i = 0; i < binIndexes.Length; i++)
{
int j = binIndexes[i];
//binSum[i][j] += y * y * v / ((grid[i][j + 1] - grid[i][j]) * (grid[i].Length - 1));
//binSum[i][j] += Math.Abs(y) * (grid[i][j + 1] - grid[i][j]);
//binSum[i][j] += Math.Abs(v * y) * (grid[i][j + 1] - grid[i][j]);
binSum[i][j] += Math.Abs(v * y);
}
return(v * y);
}
private static MultiExtremum FindGlobalExtremum(Func <IReadOnlyList <double>, double> function, IReadOnlyList <Interval> volume, MultiExtremumSettings settings, bool negate)
{
if (function == null)
{
throw new ArgumentNullException(nameof(function));
}
if (volume == null)
{
throw new ArgumentNullException(nameof(volume));
}
MultiFunctor f = new MultiFunctor(function, negate);
DifferentialEvolutionSettings deSettings = GetDefaultSettings(settings, volume.Count);
MultiExtremum extremum = FindGlobalExtremum(f, volume, deSettings);
return(extremum);
}
private static MultiExtremum FindLocalExtremum(Func <IReadOnlyList <double>, double> function, IReadOnlyList <double> start, MultiExtremumSettings settings, bool negate)
{
MultiFunctor f = new MultiFunctor(function, negate);
// Pick an initial trust region radius; we need to do this better.
double s = 0.0;
foreach (double x in start)
{
s += (Math.Abs(x) + 1.0 / 4.0) / 4.0;
}
s = s / start.Count;
Debug.WriteLine("s={0}", s);
SetDefaultOptimizationSettings(settings, start.Count);
return(FindMinimum_ModelTrust(f, start, s, settings));
}
private static MultiExtremum FindLocalExtremum(Func<IList<double>, double> function, IList<double> start, EvaluationSettings settings, bool negate)
{
MultiFunctor f = new MultiFunctor(function, negate);
// Pick an initial radius; we need to do this better.
/*
double s = Double.MaxValue;
foreach (double x in start) s = Math.Min((Math.Abs(x) + 1.0 / 8.0) / 8.0, s);
*/
double s = 0.0;
foreach (double x in start) s += (Math.Abs(x) + 1.0 / 4.0) / 4.0;
s = s / start.Count;
//double s = 0.2;
Debug.WriteLine("s={0}", s);
return (FindMinimum_ModelTrust(f, start, s, settings));
}
internal static QuadraticInterpolationModel Construct(MultiFunctor f, IList<double> x, double s)
{
QuadraticInterpolationModel model = new QuadraticInterpolationModel();
model.Initialize(f, x, s);
return (model);
}
public static QuadraticInterpolationModel Construct(Func<IList<double>, double> f, double[] x, double s)
{
MultiFunctor mf = new MultiFunctor(f);
return (Construct(mf, x, s));
}
public static QuadraticInterpolationModel Construct(Func <IList <double>, double> f, double[] x, double s)
{
MultiFunctor mf = new MultiFunctor(f);
return(Construct(mf, x, s));
}
// Differential evoluation is a global optimization algorithm over continuous inputs that is adapted from genetic algorithms for finite inputs.
// The idea is maintain a population of input vectors ("agents") and to vary that population over cycles ("generations") according to rules that incorporate
// random mutation but on average tend to bring them closer to optima ("fitter").
private static MultiExtremum FindGlobalExtremum(MultiFunctor f, IList<Interval> volume, DifferentialEvolutionSettings settings)
{
int d = volume.Count;
// Choose a number of agents that increases with dimension and required precision.
int m = settings.Population;
Debug.WriteLine("d={0} m={1}", d, m);
Random rng = new Random(3);
//Random rng = new Random(1001110000);
// Start with random points in the allowed region.
double[][] points = new double[m][];
double[] values = new double[m];
for (int i = 0; i < m; i++) {
points[i] = new double[d];
for (int j = 0; j < d; j++) {
points[i][j] = volume[j].LeftEndpoint + rng.NextDouble() * volume[j].Width;
}
values[i] = f.Evaluate(points[i]);
}
while (f.EvaluationCount < settings.EvaluationBudget) {
//double mutationFactor = 0.5 + 0.5 * rng.NextDouble();
double[][] newPoints = new double[m][];
double[] newValues = new double[m];
for (int i = 0; i < m; i++) {
// Mutation
// construct donor vector
int a = i; while (a == i) a = rng.Next(m);
int b = i; while ((b == i) || (b == a)) b = rng.Next(m);
int c = i; while ((c == i) || (c == b) || (c == a)) c = rng.Next(m);
double[] donor = new double[d];
for (int j = 0; j < d; j++) {
donor[j] = points[a][j] + mutationFactor * (points[b][j] - points[c][j]);
if (donor[j] < volume[j].LeftEndpoint) donor[j] = volume[j].LeftEndpoint;
if (donor[j] > volume[j].RightEndpoint) donor[j] = volume[j].RightEndpoint;
}
// Recombination
double[] trial = new double[d];
int k = rng.Next(d);
for (int j = 0; j < d; j++) {
if ((j == k) || (rng.NextDouble() < settings.CrossoverProbability)) {
trial[j] = donor[j];
} else {
trial[j] = points[i][j];
}
}
// Selection
double value = f.Evaluate(trial);
if (value <= values[i]) {
newPoints[i] = trial;
newValues[i] = value;
} else {
newPoints[i] = points[i];
newValues[i] = values[i];
}
}
points = newPoints;
values = newValues;
// Check termination criteria
int minIndex = -1;
double minValue = Double.MaxValue;
double maxValue = Double.MinValue;
for (int i = 0; i < m; i++) {
if (values[i] < minValue) {
minValue = values[i];
minIndex = i;
}
if (values[i] > maxValue) maxValue = values[i];
}
double range = maxValue - minValue;
double tol = settings.ComputePrecision(minValue);
if (range <= tol) {
MultiExtremum result = new MultiExtremum(f.EvaluationCount, settings, points[minIndex], f.IsNegated ? -values[minIndex] : values[minIndex], Math.Max(range, 0.75 * tol), null);
return (result);
}
//settings.OnUpdate(new MultiExtremum(points[minIndex], values[minIndex], null, f.EvaluationCount));
}
throw new NonconvergenceException();
}
public void Evaluate(MultiFunctor f, CoordinateTransform[] map, IntegrationRegion r)
{
if (map == null)
{
Evaluate(f, r);
return;
}
// Evaluation at origin. Keep a vector of origin coordinates and jacobian values.
double[] x0 = new double[d];
double[] x = new double[d];
double[] v0 = new double[d];
double[] v = new double[d];
for (int i = 0; i < x.Length; i++)
{
x0[i] = map[i].Transform(r.MapCoordinateFromSymmetricUnitInterval(i, 0.0), out v0[i]);
x[i] = x0[i];
v[i] = v0[i];
}
double f0 = Jacobian(v) * f.Evaluate(x);
double I7 = w70 * f0;
double I5 = w50 * f0;
// near off-one-axis evaluations
double f1 = 0.0;
for (int i = 0; i < d; i++)
{
x[i] = map[i].Transform(r.MapCoordinateFromSymmetricUnitInterval(i, a1), out v[i]);
f1 += Jacobian(v) * f.Evaluate(x);
x[i] = map[i].Transform(r.MapCoordinateFromSymmetricUnitInterval(i, -a1), out v[i]);
f1 += Jacobian(v) * f.Evaluate(x);
x[i] = x0[i];
v[i] = v0[i];
}
I7 += w71 * f1;
I5 += w51 * f1;
// far off-one-axis evaluations
int sIndex = 0; double sMax = 0.0;
double f2 = 0.0;
for (int i = 0; i < d; i++)
{
x[i] = map[i].Transform(r.MapCoordinateFromSymmetricUnitInterval(i, a2), out v[i]);
double fp2 = Jacobian(v) * f.Evaluate(x);
f2 += fp2;
x[i] = map[i].Transform(r.MapCoordinateFromSymmetricUnitInterval(i, -a2), out v[i]);
double fm2 = Jacobian(v) * f.Evaluate(x);
f2 += fm2;
x[i] = x0[i];
v[i] = v0[i];
double s = Math.Abs(fm2 + fp2 - 2.0 * f0);
if (s > sMax)
{
sMax = s;
sIndex = i;
}
else if ((s == sMax) && (r.CoordinateWidth(i) > r.CoordinateWidth(sIndex)))
{
sIndex = i;
}
}
I7 += w72 * f2;
I5 += w52 * f2;
// far off-two-axis evaluations
double f3 = 0.0;
for (int i = 0; i < d; i++)
{
for (int j = 0; j < i; j++)
{
// ++
x[i] = map[i].Transform(r.MapCoordinateFromSymmetricUnitInterval(i, a3), out v[i]);
x[j] = map[j].Transform(r.MapCoordinateFromSymmetricUnitInterval(j, a3), out v[j]);
f3 += Jacobian(v) * f.Evaluate(x);
// +-
x[j] = map[j].Transform(r.MapCoordinateFromSymmetricUnitInterval(j, -a3), out v[j]);
f3 += Jacobian(v) * f.Evaluate(x);
// --
x[i] = map[i].Transform(r.MapCoordinateFromSymmetricUnitInterval(i, -a3), out v[i]);
f3 += Jacobian(v) * f.Evaluate(x);
// -+
x[j] = map[j].Transform(r.MapCoordinateFromSymmetricUnitInterval(j, a3), out v[j]);
f3 += Jacobian(v) * f.Evaluate(x);
x[i] = x0[i];
x[j] = x0[j];
v[i] = v0[i];
v[j] = v0[j];
}
}
I7 += w73 * f3;
I5 += w53 * f3;
// mid off-all-axis evaluations
// We need all 2^d permutations of + and - in each position.
// So that we only need to change one component each time, we proceed in Gray code order.
// We use a bit-vector to keep track of which component is + (0) and which is - (1).
int state = 0;
for (int j = 0; j < d; j++)
{
x[j] = map[j].Transform(r.MapCoordinateFromSymmetricUnitInterval(j, a4), out v[j]);
}
double f4 = Jacobian(v) * f.Evaluate(x);
for (int i = 0; i < (td - 1); i++)
{
int j = GrayFlipIndex(i);
int mask = 1 << j;
state = state ^ mask;
x[j] = map[j].Transform(r.MapCoordinateFromSymmetricUnitInterval(j, ((state & mask) > 0) ? -a4 : a4), out v[j]);
f4 += Jacobian(v) * f.Evaluate(x);
}
I7 += w74 * f4;
double V = r.Volume();
r.Value = V * I7;
r.Error = V * Math.Abs(I7 - I5);
r.SplitIndex = sIndex;
}
public void Evaluate(MultiFunctor f, IntegrationRegion r)
{
// evaluation at origin
// Keep a vector of origin coordinates, we will often re-set components to them.
double[] x0 = new double[d];
double[] x = new double[d];
for (int i = 0; i < x.Length; i++)
{
x0[i] = r.MapCoordinateFromSymmetricUnitInterval(i, 0.0);
x[i] = x0[i];
}
double f0 = f.Evaluate(x);
double I7 = w70 * f0;
double I5 = w50 * f0;
// near off-one-axis evaluations
double f1 = 0.0;
for (int i = 0; i < d; i++)
{
x[i] = r.MapCoordinateFromSymmetricUnitInterval(i, a1);
f1 += f.Evaluate(x);
x[i] = r.MapCoordinateFromSymmetricUnitInterval(i, -a1);
f1 += f.Evaluate(x);
x[i] = x0[i];
}
I7 += w71 * f1;
I5 += w51 * f1;
// far off-one-axis evaluations
// while doing this, determine along which direction we will split, if asked
int sIndex = 0; double sMax = 0.0;
double f2 = 0.0;
for (int i = 0; i < d; i++)
{
x[i] = r.MapCoordinateFromSymmetricUnitInterval(i, a2);
double fp2 = f.Evaluate(x);
f2 += fp2;
x[i] = r.MapCoordinateFromSymmetricUnitInterval(i, -a2);
double fm2 = f.Evaluate(x);
f2 += fm2;
x[i] = x0[i];
// One way to view fm2 + fp2 - 2 f0 is as numerical 2nd derivative
// Another way is as difference of f0 from value predicted by linear interpolation from fm2 and fp2
double s = Math.Abs(fm2 + fp2 - 2.0 * f0);
if (s > sMax)
{
// Use the direction of largest difference for future splits.
sMax = s;
sIndex = i;
}
else if ((s == sMax) && (r.CoordinateWidth(i) > r.CoordinateWidth(sIndex)))
{
// If two directions both have the same difference, pick the one with the larger width to split.
// This may seem like an unimportant corner case, but it turns out to be critical to convergence.
// Without this clause we get into cycles in which we split along the same direction forever.
// For example, given the 3D watson integral of [1 - \cos(x) \cos(y) \cos(z)]^{-1} over [0,\pi]^3,
// all directions have zero deficit from starting point because one of the cosines is always zero,
// and without this branch we split along the x-axis forever.
sIndex = i;
}
}
I7 += w72 * f2;
I5 += w52 * f2;
r.SplitIndex = sIndex;
// far off-two-axis evaluations
double f3 = 0.0;
for (int i = 0; i < d; i++)
{
for (int j = 0; j < i; j++)
{
// ++
x[i] = r.MapCoordinateFromSymmetricUnitInterval(i, a3);
x[j] = r.MapCoordinateFromSymmetricUnitInterval(j, a3);
f3 += f.Evaluate(x);
// +-
x[j] = r.MapCoordinateFromSymmetricUnitInterval(j, -a3);
f3 += f.Evaluate(x);
// --
x[i] = r.MapCoordinateFromSymmetricUnitInterval(i, -a3);
f3 += f.Evaluate(x);
// -+
x[j] = r.MapCoordinateFromSymmetricUnitInterval(j, a3);
f3 += f.Evaluate(x);
x[i] = x0[i];
x[j] = x0[j];
}
}
I7 += w73 * f3;
I5 += w53 * f3;
// mid off-all-axis evaluations
// We need all 2^d permutations of + and - in each position.
// So that we only need to change one component each time, we proceed in Gray code order.
// We use a bit-vector to keep track of which component is + (0) and which is - (1).
int state = 0;
for (int j = 0; j < d; j++)
{
x[j] = r.MapCoordinateFromSymmetricUnitInterval(j, a4);
}
double f4 = f.Evaluate(x);
for (int i = 0; i < (td - 1); i++)
{
int j = GrayFlipIndex(i);
int mask = 1 << j;
state = state ^ mask;
x[j] = r.MapCoordinateFromSymmetricUnitInterval(j, ((state & mask) > 0) ? -a4 : a4);
f4 += f.Evaluate(x);
}
I7 += w74 * f4;
double V = r.Volume();
r.Value = V * I7;
r.Error = V * Math.Abs(I7 - I5);
}
private static UncertainValue Integrate_Adaptive(MultiFunctor f, CoordinateTransform[] map, IntegrationRegion r, EvaluationSettings settings)
{
// Create an evaluation rule
GenzMalik75Rule rule = new GenzMalik75Rule(r.Dimension);
// Use it on the whole region and put the answer in a linked list
rule.Evaluate(f, map, r);
LinkedList <IntegrationRegion> regionList = new LinkedList <IntegrationRegion>();
regionList.AddFirst(r);
// Iterate until convergence
while (f.EvaluationCount < settings.EvaluationBudget)
{
// Add up value and errors in all regions.
// While we are at it, take note of the region with the largest error.
double value = 0.0;
double error = 0.0;
LinkedListNode <IntegrationRegion> regionNode = regionList.First;
double maxError = 0.0;
LinkedListNode <IntegrationRegion> maxErrorNode = null;
while (regionNode != null)
{
IntegrationRegion region = regionNode.Value;
value += region.Value;
error += region.Error;
//error += MoreMath.Sqr(region.Error);
if (region.Error > maxError)
{
maxError = region.Error;
maxErrorNode = regionNode;
}
regionNode = regionNode.Next;
}
// Check for convergence.
if ((error <= settings.AbsolutePrecision) || (error <= settings.RelativePrecision * Math.Abs(value)))
{
return(new UncertainValue(value, error));
}
// Split the region with the largest error, and evaluate each subregion.
IntegrationRegion maxErrorRegion = maxErrorNode.Value;
regionList.Remove(maxErrorNode);
IList <IntegrationRegion> subRegions = maxErrorRegion.Split(maxErrorRegion.SplitIndex);
/*
* Countdown cnt = new Countdown(2);
* ThreadPool.QueueUserWorkItem((object state) => { rule.Evaluate(f, subRegions[0]); cnt.Signal(); });
* ThreadPool.QueueUserWorkItem((object state) => { rule.Evaluate(f, subRegions[1]); cnt.Signal(); });
* cnt.Wait();
* foreach (IntegrationRegion subRegion in subRegions) {
* regionList.AddLast(subRegion);
* }
*/
foreach (IntegrationRegion subRegion in subRegions)
{
rule.Evaluate(f, map, subRegion);
regionList.AddLast(subRegion);
}
}
throw new NonconvergenceException();
}
// This method is due to Powell (http://en.wikipedia.org/wiki/Michael_J._D._Powell), but it is not what
// is usually called Powell's Method (http://en.wikipedia.org/wiki/Powell%27s_method); Powell
// developed that method in the 1960s, it was included in Numerical Recipes and is very popular.
// This is a model trust algorithm developed by Powell in the 2000s. It typically uses many
// fewer function evaluations, but does more intensive calculations between each evaluation.
// This is basically the UOBYQA variant of Powell's new methods. It maintains a quadratic model
// that interpolates between (d + 1) (d + 2) / 2 points. The model is trusted
// within a given radius. At each step, it moves to the minimum of the model (or the boundary of
// the trust region in that direction) and evaluates the function. The new value is incorporated
// into the model and the trust region expanded or contracted depending on how accurate its
// prediction of the function value was.
// Papers on these methods are collected at http://mat.uc.pt/~zhang/software.html#powell_software.
// The UOBYQA paper is here: http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.28.1756.
// The NEWUOA paper is here: http://www.damtp.cam.ac.uk/user/na/NA_papers/NA2004_08.pdf.
// The CONDOR system (http://www.applied-mathematics.net/optimization/CONDORdownload.html) is based on these same ideas.
// The thesis of CONDOR's author (http://www.applied-mathematics.net/mythesis/index.html) was also helpful.
// It should be very easy to extend this method to constrained optimization, either by incorporating the bounds into
// the step limits or by mapping hyper-space into a hyper-cube.
private static MultiExtremum FindMinimum_ModelTrust(MultiFunctor f, IReadOnlyList <double> x, double s, MultiExtremumSettings settings)
{
// Construct an initial model.
QuadraticInterpolationModel model = QuadraticInterpolationModel.Construct(f, x, s);
double trustRadius = s;
while (f.EvaluationCount < settings.EvaluationBudget)
{
// Find the minimum point of the model within the trust radius
double[] z = model.FindMinimum(trustRadius);
double expectedValue = model.Evaluate(z);
double deltaExpected = model.MinimumValue - expectedValue;
// Evaluate the function at the suggested minimum
double[] point = model.ConvertPoint(z);
double value = f.Evaluate(point);
double delta = model.MinimumValue - value;
double tol = settings.ComputePrecision(Math.Min(model.MinimumValue, value));
// Note value can be way off, so use better of old best and new value to compute tol.
// When we didn't do this before, we got value = infinity, so tol = infinity, and thus terminated!
if (delta > 0.0 && settings.Listener != null)
{
MultiExtremum report = new MultiExtremum(f.EvaluationCount, settings, point, value, Math.Max(Math.Abs(delta), 0.75 * tol), model.GetHessian());
settings.Listener(report);
}
// To terminate, we demand: a reduction, that the reduction be small, that the reduction be in line with
// its expected value, that we have not run up against the trust boundary, and that the gradient is small.
// I had wanted to demand delta > 0, but we run into some cases where delta keeps being very slightly
// negative, typically orders of magnitude less than tol, causing the trust radius to shrink in an
// endless cycle that causes our approximation to ultimately go sour, even though terminating on the original
// very slightly negative delta would have produced an accurate estimate. So we tolerate this case for now.
if ((delta <= tol) && (-0.25 * tol <= delta))
{
// We demand that the model be decent, i.e. that the expected delta was within tol of the measured delta.
if (Math.Abs(delta - deltaExpected) <= tol)
{
// We demand that the step not just be small because it ran up against the trust radius.
// If it ran up against the trust radius, there is probably more to be hand by continuing.
double zm = Blas1.dNrm2(z, 0, 1, z.Length);
if (zm < trustRadius)
{
// Finally, we demand that the gradient be small. You might think this was obvious since
// z was small, but if the Hessian is not positive definite
// the interplay of the Hessian and the gradient can produce a small z even if the model looks nothing like a quadratic minimum.
double gm = Blas1.dNrm2(model.GetGradient(), 0, 1, z.Length);
if (gm * zm <= tol)
{
if (f.IsNegated)
{
value = -value;
}
return(new MultiExtremum(f.EvaluationCount, settings, point, value, Math.Max(Math.Abs(delta), 0.75 * tol), model.GetHessian()));
}
}
}
}
// There are now three decisions to be made:
// 1. How to change the trust radius
// 2. Whether to accept the new point
// 3. Which existing point to replace
// If the actual change was very far from the expected change, reduce the trust radius.
// If the expected change did a good job of predicting the actual change, increase the trust radius.
if ((delta < 0.25 * deltaExpected) /*|| (8.0 * deltaExpected < delta)*/)
{
trustRadius = 0.5 * trustRadius;
}
else if ((0.75 * deltaExpected <= delta) /*&& (delta <= 2.0 * deltaExpected)*/)
{
trustRadius = 2.0 * trustRadius;
}
// It appears that the limits on delta being too large don't help, and even hurt if made too stringent.
// Replace an old point with the new point.
int iMax = 0; double fMax = model.values[0];
int iBad = 0; double fBad = model.ComputeBadness(0, z, point, value);
for (int i = 1; i < model.values.Length; i++)
{
if (model.values[i] > fMax)
{
iMax = i; fMax = model.values[i];
}
double bad = model.ComputeBadness(i, z, point, value);
if (bad > fBad)
{
iBad = i; fBad = bad;
}
}
// Use the new point as long as it is better than our worst existing point.
if (value < fMax)
{
Debug.Assert(!Double.IsPositiveInfinity(value) && !Double.IsNaN(value));
model.ReplacePoint(iBad, point, z, value);
}
// There is some question about how best to choose which point to replace.
// The largest value? The furthest away? The one closest to new min?
}
throw new NonconvergenceException();
}
private void Initialize(MultiFunctor f, IReadOnlyList <double> x, double s)
{
// Allocate storage
d = x.Count;
int m = (d + 1) * (d + 2) / 2;
origin = new double[d];
points = new double[m][];
for (int i = 0; i < m; i++)
{
points[i] = new double[d];
}
values = new double[m];
polynomials = new QuadraticModel[m];
for (int i = 0; i < m; i++)
{
polynomials[i] = new QuadraticModel(d);
}
// Start with x as the origin.
x.CopyTo(origin, 0);
// The first interpolation point is the origin.
x.CopyTo(points[0], 0);
values[0] = f.Evaluate(points[0]);
// Compute 2d more interpolation points one step along each axis.
int k = 0;
for (int i = 0; i < d; i++)
{
k++;
x.CopyTo(points[k], 0);
points[k][i] += s;
double plusValue = f.Evaluate(points[k]);
values[k] = plusValue;
k++;
x.CopyTo(points[k], 0);
points[k][i] -= s;
double minusValue = f.Evaluate(points[k]);
values[k] = minusValue;
}
// Compute d(d+1)/2 more interpolation points at the corners.
for (int i = 0; i < d; i++)
{
for (int j = 0; j < i; j++)
{
k++;
x.CopyTo(points[k], 0);
points[k][i] += s;
points[k][j] += s;
double cornerValue = f.Evaluate(points[k]);
values[k] = cornerValue;
}
}
double s1 = 1.0 / s;
double s2 = s1 * s1;
// Compute the Lagrange polynomial for each point
k = 0;
for (int i = 0; i < d; i++)
{
k++;
polynomials[2 * i + 1].g[i] = 0.5 * s1;
polynomials[2 * i + 1].h[i][i] = 0.5 * s2;
k++;
polynomials[2 * i + 2].g[i] = -0.5 * s1;
polynomials[2 * i + 2].h[i][i] = 0.5 * s2;
}
for (int i = 0; i < d; i++)
{
for (int j = 0; j < i; j++)
{
k++;
polynomials[k].h[i][j] = s2;
polynomials[2 * i + 1].h[i][j] -= s2;
polynomials[2 * j + 1].h[i][j] -= s2;
}
}
polynomials[0].f = 1.0;
for (int l = 1; l < m; l++)
{
for (int i = 0; i < d; i++)
{
polynomials[0].g[i] -= polynomials[l].g[i];
for (int j = 0; j <= i; j++)
{
polynomials[0].h[i][j] -= polynomials[l].h[i][j];
}
}
}
// Compute the total interpolating polynomial.
total = new QuadraticModel(d);
for (int l = 0; l < m; l++)
{
total.f += values[l] * polynomials[l].f;
for (int i = 0; i < d; i++)
{
total.g[i] += values[l] * polynomials[l].g[i];
for (int j = 0; j <= i; j++)
{
total.h[i][j] += values[l] * polynomials[l].h[i][j];
}
}
}
// Find the minimum point.
minValueIndex = 0;
for (int i = 1; i < m; i++)
{
if (values[i] < values[minValueIndex])
{
minValueIndex = i;
}
}
// move the origin to the minimum point
double[] z = new double[d];
for (int i = 0; i < z.Length; i++)
{
z[i] = points[minValueIndex][i] - origin[i];
}
ShiftOrigin(z);
// compute badnesses
//badnesses = new double[points.Length];
//for (int i = 0; i < points.Length; i++) {
// badnesses[i] = ComputeBadressAbsolute(points[i], values[i]);
//}
}
public void Evaluate(MultiFunctor f, CoordinateTransform[] map, IntegrationRegion r)
{
if (map == null) {
Evaluate(f, r);
return;
}
// Evaluation at origin. Keep a vector of origin coordinates and jacobian values.
double[] x0 = new double[d];
double[] x = new double[d];
double[] v0 = new double[d];
double[] v = new double[d];
for (int i = 0; i < x.Length; i++) {
x0[i] = map[i].Transform(r.MapCoordinateFromSymmetricUnitInterval(i, 0.0), out v0[i]);
x[i] = x0[i];
v[i] = v0[i];
}
double f0 = Jacobian(v) * f.Evaluate(x);
double I7 = w70 * f0;
double I5 = w50 * f0;
// near off-one-axis evaluations
double f1 = 0.0;
for (int i = 0; i < d; i++) {
x[i] = map[i].Transform(r.MapCoordinateFromSymmetricUnitInterval(i, a1), out v[i]);
f1 += Jacobian(v) * f.Evaluate(x);
x[i] = map[i].Transform(r.MapCoordinateFromSymmetricUnitInterval(i, -a1), out v[i]);
f1 += Jacobian(v) * f.Evaluate(x);
x[i] = x0[i];
v[i] = v0[i];
}
I7 += w71 * f1;
I5 += w51 * f1;
// far off-one-axis evaluations
int sIndex = 0; double sMax = 0.0;
double f2 = 0.0;
for (int i = 0; i < d; i++) {
x[i] = map[i].Transform(r.MapCoordinateFromSymmetricUnitInterval(i, a2), out v[i]);
double fp2 = Jacobian(v) * f.Evaluate(x);
f2 += fp2;
x[i] = map[i].Transform(r.MapCoordinateFromSymmetricUnitInterval(i, -a2), out v[i]);
double fm2 = Jacobian(v) * f.Evaluate(x);
f2 += fm2;
x[i] = x0[i];
v[i] = v0[i];
double s = Math.Abs(fm2 + fp2 - 2.0 * f0);
if (s > sMax) {
sMax = s;
sIndex = i;
} else if ((s == sMax) && (r.CoordinateWidth(i) > r.CoordinateWidth(sIndex))) {
sIndex = i;
}
}
I7 += w72 * f2;
I5 += w52 * f2;
// far off-two-axis evaluations
double f3 = 0.0;
for (int i = 0; i < d; i++) {
for (int j = 0; j < i; j++) {
// ++
x[i] = map[i].Transform(r.MapCoordinateFromSymmetricUnitInterval(i, a3), out v[i]);
x[j] = map[j].Transform(r.MapCoordinateFromSymmetricUnitInterval(j, a3), out v[j]);
f3 += Jacobian(v) * f.Evaluate(x);
// +-
x[j] = map[j].Transform(r.MapCoordinateFromSymmetricUnitInterval(j, -a3), out v[j]);
f3 += Jacobian(v) * f.Evaluate(x);
// --
x[i] = map[i].Transform(r.MapCoordinateFromSymmetricUnitInterval(i, -a3), out v[i]);
f3 += Jacobian(v) * f.Evaluate(x);
// -+
x[j] = map[j].Transform(r.MapCoordinateFromSymmetricUnitInterval(j, a3), out v[j]);
f3 += Jacobian(v) * f.Evaluate(x);
x[i] = x0[i];
x[j] = x0[j];
v[i] = v0[i];
v[j] = v0[j];
}
}
I7 += w73 * f3;
I5 += w53 * f3;
// mid off-all-axis evaluations
// We need all 2^d permutations of + and - in each position.
// So that we only need to change one component each time, we proceed in Gray code order.
// We use a bit-vector to keep track of which component is + (0) and which is - (1).
int state = 0;
for (int j = 0; j < d; j++) {
x[j] = map[j].Transform(r.MapCoordinateFromSymmetricUnitInterval(j, a4), out v[j]);
}
double f4 = Jacobian(v) * f.Evaluate(x);
for (int i = 0; i < (td - 1); i++) {
int j = GrayFlipIndex(i);
int mask = 1 << j;
state = state ^ mask;
x[j] = map[j].Transform(r.MapCoordinateFromSymmetricUnitInterval(j, ((state & mask) > 0) ? -a4 : a4), out v[j]);
f4 += Jacobian(v) * f.Evaluate(x);
}
I7 += w74 * f4;
double V = r.Volume();
r.Value = V * I7;
r.Error = V * Math.Abs(I7 - I5);
r.SplitIndex = sIndex;
}
private static void FindExtremum_Amobea(MultiFunctor f, Vertex[] vertexes, EvaluationSettings settings)
{
int d = vertexes.Length - 1;
while (f.EvaluationCount < settings.EvaluationBudget)
{
// Identify the best and worst vertexes.
int minVertex = 0; double minY = vertexes[0].Y;
int maxVertex = 0; double maxY = vertexes[0].Y;
int nextMaxVertex = 0; double nextMaxY = vertexes[0].Y;
for (int i = 1; i < vertexes.Length; i++)
{
double y = vertexes[i].Y;
if (y < minY)
{
minVertex = i; minY = y;
}
if (y > nextMaxY)
{
if (y > maxY)
{
nextMaxVertex = maxVertex; nextMaxY = maxY;
maxVertex = i; maxY = y;
}
else
{
nextMaxVertex = i; nextMaxY = y;
}
}
}
// Terminate based on spread between vertexes.
if ((maxY - minY) <= Math.Abs(maxY) * settings.RelativePrecision)
{
Debug.WriteLine(minY);
return;
}
// Produce a new candidate vertex by reflecting the worst vertex through the opposite face.
double[] centroid = new double[d];
for (int i = 0; i < vertexes.Length; i++)
{
if (i != maxVertex)
{
for (int j = 0; j < d; j++)
{
centroid[j] += vertexes[i].X[j] / d;
}
}
}
double[] newX = new double[d];
for (int j = 0; j < d; j++)
{
newX[j] = centroid[j] + alpha * (centroid[j] - vertexes[maxVertex].X[j]);
}
double newY = f.Evaluate(newX);
if (newY < nextMaxY)
{
// As long as the new point is not terrible, we are going to replace the worst point with it.
vertexes[maxVertex] = new Vertex()
{
X = newX, Y = newY
};
Debug.WriteLine("Reflect");
if (newY < minY)
{
// If the new point was very good, we will try to extend the simplex further in that direction.
double[] extendedX = new double[d];
for (int j = 0; j < d; j++)
{
extendedX[j] = centroid[j] + 2.0 * (centroid[j] - vertexes[maxVertex].X[j]);
}
double extendedY = f.Evaluate(extendedX);
if (extendedY < minY)
{
// If the extension is also very good, we replace the second worst point too.
vertexes[maxVertex] = new Vertex()
{
X = extendedX, Y = extendedY
};
Debug.WriteLine("No, Extend");
}
}
}
else
{
// The reflected point was pretty terrible, so we will try to produce a new candidate
// point by contracting the worst point toward the centroid instead.
for (int j = 0; j < d; j++)
{
newX[j] = centroid[j] + (vertexes[maxVertex].X[j] - centroid[j]) / 2.0;
}
newY = f.Evaluate(newX);
if (newY < nextMaxY)
{
// If that candidate is not terrible, accept it.
vertexes[maxVertex] = new Vertex()
{
X = newX, Y = newY
};
Debug.WriteLine("Contract");
}
else
{
// Otherwise, we give up and simply shrink our simplex down toward the minimum.
for (int i = 0; i < vertexes.Length; i++)
{
if (i != minVertex)
{
double[] shrunkX = new double[d];
for (int j = 0; j < d; j++)
{
shrunkX[j] = vertexes[minVertex].X[j] + (vertexes[i].X[j] - vertexes[minVertex].X[j]) / 2.0;
}
double shrunkY = f.Evaluate(shrunkX);
vertexes[i] = new Vertex()
{
X = shrunkX, Y = shrunkY
};
}
}
Debug.WriteLine("Shrink");
}
}
}
}
private void Initialize(MultiFunctor f, IList<double> x, double s)
{
// Allocate storage
d = x.Count;
int m = (d + 1) * (d + 2) / 2;
origin = new double[d];
points = new double[m][];
for (int i = 0; i < m; i++) points[i] = new double[d];
values = new double[m];
polynomials = new QuadraticModel[m];
for (int i = 0; i < m; i++) polynomials[i] = new QuadraticModel(d);
// Start with x as the origin.
x.CopyTo(origin, 0);
// The first interpolation point is the origin.
x.CopyTo(points[0], 0);
values[0] = f.Evaluate(points[0]);
// Compute 2d more interpolation points one step along each axis.
int k = 0;
for (int i = 0; i < d; i++) {
k++;
x.CopyTo(points[k], 0);
points[k][i] += s;
double plusValue = f.Evaluate(points[k]);
values[k] = plusValue;
k++;
x.CopyTo(points[k], 0);
points[k][i] -= s;
double minusValue = f.Evaluate(points[k]);
values[k] = minusValue;
}
// Compute d(d+1)/2 more interpolation points at the corners.
for (int i = 0; i < d; i++) {
for (int j = 0; j < i; j++) {
k++;
x.CopyTo(points[k], 0);
points[k][i] += s;
points[k][j] += s;
double cornerValue = f.Evaluate(points[k]);
values[k] = cornerValue;
}
}
double s1 = 1.0 / s;
double s2 = s1 * s1;
// Compute the Lagrange polynomial for each point
k = 0;
for (int i = 0; i < d; i++) {
k++;
polynomials[2 * i + 1].g[i] = 0.5 * s1;
polynomials[2 * i + 1].h[i][i] = 0.5 * s2;
k++;
polynomials[2 * i + 2].g[i] = -0.5 * s1;
polynomials[2 * i + 2].h[i][i] = 0.5 * s2;
}
for (int i = 0; i < d; i++) {
for (int j = 0; j < i; j++) {
k++;
polynomials[k].h[i][j] = s2;
polynomials[2 * i + 1].h[i][j] -= s2;
polynomials[2 * j + 1].h[i][j] -= s2;
}
}
polynomials[0].f = 1.0;
for (int l = 1; l < m; l++) {
for (int i = 0; i < d; i++) {
polynomials[0].g[i] -= polynomials[l].g[i];
for (int j = 0; j <= i; j++) {
polynomials[0].h[i][j] -= polynomials[l].h[i][j];
}
}
}
// Compute the total interpolating polynomial.
total = new QuadraticModel(d);
for (int l = 0; l < m; l++) {
total.f += values[l] * polynomials[l].f;
for (int i = 0; i < d; i++) {
total.g[i] += values[l] * polynomials[l].g[i];
for (int j = 0; j <= i; j++) {
total.h[i][j] += values[l] * polynomials[l].h[i][j];
}
}
}
// Find the minimum point.
minValueIndex = 0;
for (int i = 1; i < m; i++) {
if (values[i] < values[minValueIndex]) minValueIndex = i;
}
// move the origin to the minimum point
double[] z = new double[d];
for (int i = 0; i < z.Length; i++) z[i] = points[minValueIndex][i] - origin[i];
ShiftOrigin(z);
// compute badnesses
//badnesses = new double[points.Length];
//for (int i = 0; i < points.Length; i++) {
// badnesses[i] = ComputeBadressAbsolute(points[i], values[i]);
//}
}
public void Evaluate(MultiFunctor f, IntegrationRegion r)
{
// evaluation at origin
// Keep a vector of origin coordinates, we will often re-set components to them.
double[] x0 = new double[d];
double[] x = new double[d];
for (int i = 0; i < x.Length; i++) {
x0[i] = r.MapCoordinateFromSymmetricUnitInterval(i, 0.0);
x[i] = x0[i];
}
double f0 = f.Evaluate(x);
double I7 = w70 * f0;
double I5 = w50 * f0;
// near off-one-axis evaluations
double f1 = 0.0;
for (int i = 0; i < d; i++) {
x[i] = r.MapCoordinateFromSymmetricUnitInterval(i, a1);
f1 += f.Evaluate(x);
x[i] = r.MapCoordinateFromSymmetricUnitInterval(i, -a1);
f1 += f.Evaluate(x);
x[i] = x0[i];
}
I7 += w71 * f1;
I5 += w51 * f1;
// far off-one-axis evaluations
// while doing this, determine along which direction we will split, if asked
int sIndex = 0; double sMax = 0.0;
double f2 = 0.0;
for (int i = 0; i < d; i++) {
x[i] = r.MapCoordinateFromSymmetricUnitInterval(i, a2);
double fp2 = f.Evaluate(x);
f2 += fp2;
x[i] = r.MapCoordinateFromSymmetricUnitInterval(i, -a2);
double fm2 = f.Evaluate(x);
f2 += fm2;
x[i] = x0[i];
// One way to view fm2 + fp2 - 2 f0 is as numerical 2nd derivative
// Another way is as difference of f0 from value predicted by linear interpolation from fm2 and fp2
double s = Math.Abs(fm2 + fp2 - 2.0 * f0);
if (s > sMax) {
// Use the direction of largest difference for future splits.
sMax = s;
sIndex = i;
} else if ((s == sMax) && (r.CoordinateWidth(i) > r.CoordinateWidth(sIndex))) {
// If two directions both have the same difference, pick the one with the larger width to split.
// This may seem like an unimportant corner case, but it turns out to be critical to convergence.
// Without this clause we get into cycles in which we split along the same direction forever.
// For example, given the 3D watson integral of [1 - \cos(x) \cos(y) \cos(z)]^{-1} over [0,\pi]^3,
// all directions have zero deficit from starting point because one of the cosines is always zero,
// and without this branch we split along the x-axis forever.
sIndex = i;
}
}
I7 += w72 * f2;
I5 += w52 * f2;
r.SplitIndex = sIndex;
// far off-two-axis evaluations
double f3 = 0.0;
for (int i = 0; i < d; i++) {
for (int j = 0; j < i; j++) {
// ++
x[i] = r.MapCoordinateFromSymmetricUnitInterval(i, a3);
x[j] = r.MapCoordinateFromSymmetricUnitInterval(j, a3);
f3 += f.Evaluate(x);
// +-
x[j] = r.MapCoordinateFromSymmetricUnitInterval(j, -a3);
f3 += f.Evaluate(x);
// --
x[i] = r.MapCoordinateFromSymmetricUnitInterval(i, -a3);
f3 += f.Evaluate(x);
// -+
x[j] = r.MapCoordinateFromSymmetricUnitInterval(j, a3);
f3 += f.Evaluate(x);
x[i] = x0[i];
x[j] = x0[j];
}
}
I7 += w73 * f3;
I5 += w53 * f3;
// mid off-all-axis evaluations
// We need all 2^d permutations of + and - in each position.
// So that we only need to change one component each time, we proceed in Gray code order.
// We use a bit-vector to keep track of which component is + (0) and which is - (1).
int state = 0;
for (int j = 0; j < d; j++) {
x[j] = r.MapCoordinateFromSymmetricUnitInterval(j, a4);
}
double f4 = f.Evaluate(x);
for (int i = 0; i < (td - 1); i++) {
int j = GrayFlipIndex(i);
int mask = 1 << j;
state = state ^ mask;
x[j] = r.MapCoordinateFromSymmetricUnitInterval(j, ((state & mask) > 0) ? -a4 : a4);
f4 += f.Evaluate(x);
}
I7 += w74 * f4;
double V = r.Volume();
r.Value = V * I7;
r.Error = V * Math.Abs(I7 - I5);
}
private static UncertainValue Integrate_MonteCarlo(MultiFunctor f, CoordinateTransform[] map, IList<Interval> box, EvaluationSettings settings)
{
int d = box.Count;
// Use a Sobol quasi-random sequence. This give us 1/N accuracy instead of 1/\sqrt{N} accuracy.
//VectorGenerator g = new RandomVectorGenerator(d, new Random(314159265));
VectorGenerator g = new SobolVectorGenerator(d);
// Start with a trivial Lepage grid.
// We will increase the grid size every few cycles.
// My tests indicate that trying to increase every cycle or even every other cycle is too often.
// This makes sense, because we have no reason to believe our new grid will be better until we
// have substantially more evaluations per grid cell than we did for the previous grid.
LePageGrid grid = new LePageGrid(box, 1);
int refineCount = 0;
// Start with a reasonable number of evaluations per cycle that increases with the dimension.
int cycleCount = 8 * d;
//double lastValue = Integrate_MonteCarlo_Cycle(f, map, g, grid, cycleCount);
// Each cycle consists of three sets of evaluations.
// At first I did this with just two set and used the difference between the two sets as an error estimate.
// I found that it was pretty common for that difference to be low just by chance, causing error underestimatation.
double value1 = Integrate_MonteCarlo_Cycle(f, map, g, grid, cycleCount);
double value2 = Integrate_MonteCarlo_Cycle(f, map, g, grid, cycleCount);
double value3 = Integrate_MonteCarlo_Cycle(f, map, g, grid, cycleCount);
while (f.EvaluationCount < settings.EvaluationBudget) {
// Take the largest deviation as the error.
double value = (value1 + value2 + value3) / 3.0;
double error = Math.Max(Math.Abs(value1 - value3), Math.Max(Math.Abs(value1 - value2), Math.Abs(value2 - value3)));
Debug.WriteLine("{0} {1} {2}", f.EvaluationCount, value, error);
// Check for convergence.
if ((error <= settings.AbsolutePrecision) || (error <= Math.Abs(value) * settings.RelativePrecision)) {
return (new UncertainValue(value, error));
}
// Do more cycles. In order for new sets to be equal-sized, one of those must be at the current count and the next at twice that.
double smallValue = Integrate_MonteCarlo_Cycle(f, map, g, grid, cycleCount);
cycleCount *= 2;
double bigValue = Integrate_MonteCarlo_Cycle(f, map, g, grid, cycleCount);
// Combine all the cycles into new ones with twice the number of evaluations each.
value1 = (value1 + value2) / 2.0;
value2 = (value3 + smallValue) / 2.0;
value3 = bigValue;
//double currentValue = Integrate_MonteCarlo_Cycle(f, map, g, grid, cycleCount);
//double error = Math.Abs(currentValue - lastValue);
//double value = (currentValue + lastValue) / 2.0;
//lastValue = value;
// Increase the number of evaluations for the next cycle.
//cycleCount *= 2;
// Refine the grid for the next cycle.
refineCount++;
if (refineCount == 2) {
Debug.WriteLine("Replacing grid with {0} bins after {1} evaluations", grid.BinCount, grid.EvaluationCount);
grid = grid.ComputeNewGrid(grid.BinCount * 2);
refineCount = 0;
}
}
throw new NonconvergenceException();
}
// Differential evolution is a global optimization algorithm over continuous inputs that is adapted from genetic algorithms for finite inputs.
// The idea is to maintain a population of input vectors ("agents") and to vary that population over cycles ("generations") according to rules that incorporate
// random mutation but on average tend to bring them closer to optima ("fitter").
private static MultiExtremum FindGlobalExtremum(MultiFunctor f, IReadOnlyList <Interval> volume, DifferentialEvolutionSettings settings)
{
int d = volume.Count;
// Choose a number of agents that increases with dimension and required precision.
int m = settings.Population;
Debug.WriteLine("d={0} m={1}", d, m);
Random rng = new Random(3);
// Start with random points in the allowed region.
double[][] points = new double[m][];
double[] values = new double[m];
for (int i = 0; i < m; i++)
{
points[i] = new double[d];
for (int j = 0; j < d; j++)
{
points[i][j] = volume[j].LeftEndpoint + rng.NextDouble() * volume[j].Width;
}
values[i] = f.Evaluate(points[i]);
}
while (f.EvaluationCount < settings.EvaluationBudget)
{
double[][] newPoints = new double[m][];
double[] newValues = new double[m];
for (int i = 0; i < m; i++)
{
// Mutation
// construct donor vector
int a = i; while (a == i)
{
a = rng.Next(m);
}
int b = i; while ((b == i) || (b == a))
{
b = rng.Next(m);
}
int c = i; while ((c == i) || (c == b) || (c == a))
{
c = rng.Next(m);
}
double[] donor = new double[d];
for (int j = 0; j < d; j++)
{
donor[j] = points[a][j] + mutationFactor * (points[b][j] - points[c][j]);
if (donor[j] < volume[j].LeftEndpoint)
{
donor[j] = volume[j].LeftEndpoint;
}
if (donor[j] > volume[j].RightEndpoint)
{
donor[j] = volume[j].RightEndpoint;
}
}
// Recombination
double[] trial = new double[d];
int k = rng.Next(d);
for (int j = 0; j < d; j++)
{
if ((j == k) || (rng.NextDouble() < settings.CrossoverProbability))
{
trial[j] = donor[j];
}
else
{
trial[j] = points[i][j];
}
}
// Selection
double value = f.Evaluate(trial);
if (value <= values[i])
{
newPoints[i] = trial;
newValues[i] = value;
}
else
{
newPoints[i] = points[i];
newValues[i] = values[i];
}
}
points = newPoints;
values = newValues;
// Check termination criteria
int minIndex = -1;
double minValue = Double.MaxValue;
double maxValue = Double.MinValue;
for (int i = 0; i < m; i++)
{
if (values[i] < minValue)
{
minValue = values[i];
minIndex = i;
}
if (values[i] > maxValue)
{
maxValue = values[i];
}
}
double range = maxValue - minValue;
double tol = settings.ComputePrecision(minValue);
if (range <= tol)
{
MultiExtremum result = new MultiExtremum(f.EvaluationCount, settings, points[minIndex], f.IsNegated ? -values[minIndex] : values[minIndex], Math.Max(range, 0.75 * tol), null);
return(result);
}
else if (settings.Listener != null)
{
MultiExtremum report = new MultiExtremum(f.EvaluationCount, settings, points[minIndex], f.IsNegated ? -values[minIndex] : values[minIndex], Math.Max(range, 0.75 * tol), null);
settings.Listener(report);
}
}
throw new NonconvergenceException();
}
// Evaluate at x. x is assumed to be in [0,1]^d and is mapped to a bin.
// Then f(y) / p(y) is evaluated and the result recorded for future refinements
// before being returned. Note x is changed upon return.
public double Evaluate(MultiFunctor f, double[] x)
{
return (Evaluate(f, null, x));
}
// Evaluate at x. x is assumed to be in [0,1]^d and is mapped to a bin.
// Then f(y) / p(y) is evaluated and the result recorded for future refinements
// before being returned. Note x is changed upon return.
public double Evaluate(MultiFunctor f, double[] x)
{
return(Evaluate(f, null, x));
}
public double Evaluate(MultiFunctor f, CoordinateTransform[] map, double[] x)
{
Debug.Assert(x.Length == dimension);
// Increase the evaluation count.
count++;
// We will need to record the bin number into which each coordinate falls
// in order to acrue the result to the proper bin.
int[] binIndexes = new int[dimension];
// Map incoming x into a grid cell and value based on grid.
double v = v0;
for (int i = 0; i < x.Length; i++) {
Debug.Assert((0.0 <= x[i]) && (x[i] < 1.0));
double z = (grid[i].Length - 1) * x[i];
int j = (int) Math.Floor(z);
z = z - j;
double w = grid[i][j + 1] - grid[i][j];
x[i] = (1.0 - z) * grid[i][j] + z * grid[i][j + 1];
v *= w;
binIndexes[i] = j;
}
// Use the map to further transform that value.
if (map != null) {
Debug.Assert(map.Length == dimension);
for (int i = 0; i < x.Length; i++) {
map[i].TransformInPlace(ref x[i], ref v);
}
}
double y = f.Evaluate(x);
// Record the value in the appropriate bins.
for (int i = 0; i < binIndexes.Length; i++) {
int j = binIndexes[i];
//binSum[i][j] += y * y * v / ((grid[i][j + 1] - grid[i][j]) * (grid[i].Length - 1));
//binSum[i][j] += Math.Abs(y) * (grid[i][j + 1] - grid[i][j]);
//binSum[i][j] += Math.Abs(v * y) * (grid[i][j + 1] - grid[i][j]);
binSum[i][j] += Math.Abs(v * y);
}
return (v * y);
}
private static UncertainValue Integrate_MonteCarlo(MultiFunctor f, CoordinateTransform[] map, IList <Interval> box, EvaluationSettings settings)
{
int d = box.Count;
// Use a Sobol quasi-random sequence. This give us 1/N accuracy instead of 1/\sqrt{N} accuracy.
//VectorGenerator g = new RandomVectorGenerator(d, new Random(314159265));
VectorGenerator g = new SobolVectorGenerator(d);
// Start with a trivial Lepage grid.
// We will increase the grid size every few cycles.
// My tests indicate that trying to increase every cycle or even every other cycle is too often.
// This makes sense, because we have no reason to believe our new grid will be better until we
// have substantially more evaluations per grid cell than we did for the previous grid.
LePageGrid grid = new LePageGrid(box, 1);
int refineCount = 0;
// Start with a reasonable number of evaluations per cycle that increases with the dimension.
int cycleCount = 8 * d;
//double lastValue = Integrate_MonteCarlo_Cycle(f, map, g, grid, cycleCount);
// Each cycle consists of three sets of evaluations.
// At first I did this with just two set and used the difference between the two sets as an error estimate.
// I found that it was pretty common for that difference to be low just by chance, causing error underestimatation.
double value1 = Integrate_MonteCarlo_Cycle(f, map, g, grid, cycleCount);
double value2 = Integrate_MonteCarlo_Cycle(f, map, g, grid, cycleCount);
double value3 = Integrate_MonteCarlo_Cycle(f, map, g, grid, cycleCount);
while (f.EvaluationCount < settings.EvaluationBudget)
{
// Take the largest deviation as the error.
double value = (value1 + value2 + value3) / 3.0;
double error = Math.Max(Math.Abs(value1 - value3), Math.Max(Math.Abs(value1 - value2), Math.Abs(value2 - value3)));
Debug.WriteLine("{0} {1} {2}", f.EvaluationCount, value, error);
// Check for convergence.
if ((error <= settings.AbsolutePrecision) || (error <= Math.Abs(value) * settings.RelativePrecision))
{
return(new UncertainValue(value, error));
}
// Do more cycles. In order for new sets to be equal-sized, one of those must be at the current count and the next at twice that.
double smallValue = Integrate_MonteCarlo_Cycle(f, map, g, grid, cycleCount);
cycleCount *= 2;
double bigValue = Integrate_MonteCarlo_Cycle(f, map, g, grid, cycleCount);
// Combine all the cycles into new ones with twice the number of evaluations each.
value1 = (value1 + value2) / 2.0;
value2 = (value3 + smallValue) / 2.0;
value3 = bigValue;
//double currentValue = Integrate_MonteCarlo_Cycle(f, map, g, grid, cycleCount);
//double error = Math.Abs(currentValue - lastValue);
//double value = (currentValue + lastValue) / 2.0;
//lastValue = value;
// Increase the number of evaluations for the next cycle.
//cycleCount *= 2;
// Refine the grid for the next cycle.
refineCount++;
if (refineCount == 2)
{
Debug.WriteLine("Replacing grid with {0} bins after {1} evaluations", grid.BinCount, grid.EvaluationCount);
grid = grid.ComputeNewGrid(grid.BinCount * 2);
refineCount = 0;
}
}
throw new NonconvergenceException();
}
// Sample a pre-determined number of points using the given generator and grid and return the average function value.
private static double Integrate_MonteCarlo_Cycle(MultiFunctor f, CoordinateTransform[] map, VectorGenerator g, LePageGrid grid, int n)
{
double sum = 0.0;
for (int i = 0; i < n; i++) {
double[] x = g.NextVector();
//sum += f.Evaluate(x);
sum += grid.Evaluate(f, map, x);
}
return (sum / n);
}
/// <summary>
/// Estimates a multi-dimensional integral using the given evaluation settings.
/// </summary>
/// <param name="function">The function to be integrated.</param>
/// <param name="volume">The volume over which to integrate.</param>
/// <param name="settings">The integration settings.</param>
/// <returns>A numerical estimate of the multi-dimensional integral.</returns>
/// <remarks>
/// <para>Note that the integration function must not attempt to modify the argument passed to it.</para>
/// <para>Note that the integration volume must be a hyper-rectangle. You can integrate over regions with more complex boundaries by specifying the integration
/// volume as a bounding hyper-rectangle that encloses your desired integration region, and returing the value 0 for the integrand outside of the desired integration
/// region. For example, to find the volume of a unit d-sphere, you can integrate a function that is 1 inside the unit d-sphere and 0 outside it over the volume
/// [-1,1]<sup>d</sup>. You can integrate over infinite volumes by specifing volume endpoints of <see cref="Double.PositiveInfinity"/> and/or
/// <see cref="Double.NegativeInfinity"/>. Volumes with dimension greater than 15 are not currently supported.</para>
/// <para>Integrals with hard boundaries (like our hyper-sphere volume problem) typically require more evaluations than integrals of smooth functions to achieve the same accuracy.
/// Integrals with canceling positive and negative contributions also typically require more evaluations than integtrals of purely positive functions.</para>
/// <para>Numerical multi-dimensional integration is computationally expensive. To make problems more tractable, keep in mind some rules of thumb:</para>
/// <ul>
/// <li>Reduce the required accuracy to the minimum required. Alternatively, if you are willing to wait longer, increase the evaluation budget.</li>
/// <li>Exploit symmetries of the problem to reduce the integration volume. For example, to compute the volume of the unit d-sphere, it is better to
/// integrate over [0,1]<sup>d</sup> and multiply the result by 2<sup>d</sup> than to simply integrate over [-1,1]<sup>d</sup>.</li>
/// <li>Apply analytic techniques to reduce the dimension of the integral. For example, when computing the volume of the unit d-sphere, it is better
/// to do a (d-1)-dimesional integral over the function that is the height of the sphere in the dth dimension than to do a d-dimensional integral over the
/// indicator function that is 1 inside the sphere and 0 outside it.</li>
/// </ul>
/// </remarks>
/// <exception cref="ArgumentException"><paramref name="function"/>, <paramref name="volume"/>, or <paramref name="settings"/> are null, or
/// the dimension of <paramref name="volume"/> is larger than 15.</exception>
/// <exception cref="NonconvergenceException">The prescribed accuracy could not be achieved with the given evaluation budget.</exception>
public static IntegrationResult Integrate(Func <IList <double>, double> function, IList <Interval> volume, IntegrationSettings settings)
{
if (function == null)
{
throw new ArgumentNullException(nameof(function));
}
if (volume == null)
{
throw new ArgumentNullException(nameof(volume));
}
if (settings == null)
{
throw new ArgumentNullException(nameof(settings));
}
// Get the dimension from the box
int d = volume.Count;
settings = SetMultiIntegrationDefaults(settings, d);
// Translate the integration volume, which may be infinite, into a bounding box plus a coordinate transform.
Interval[] box = new Interval[d];
CoordinateTransform[] map = new CoordinateTransform[d];
for (int i = 0; i < d; i++)
{
Interval limits = volume[i];
if (Double.IsInfinity(limits.RightEndpoint))
{
if (Double.IsInfinity(limits.LeftEndpoint))
{
// -\infinity to +\infinity
box[i] = Interval.FromEndpoints(-1.0, +1.0);
map[i] = new TangentCoordinateTransform(0.0);
}
else
{
// a to +\infinity
box[i] = Interval.FromEndpoints(0.0, 1.0);
map[i] = new TangentCoordinateTransform(limits.LeftEndpoint);
}
}
else
{
if (Double.IsInfinity(limits.LeftEndpoint))
{
// -\infinity to b
box[i] = Interval.FromEndpoints(-1.0, 0.0);
map[i] = new TangentCoordinateTransform(limits.RightEndpoint);
}
else
{
// a to b
box[i] = limits;
map[i] = new IdentityCoordinateTransform();
}
}
}
// Use adaptive cubature for small dimensions, Monte-Carlo for large dimensions.
MultiFunctor f = new MultiFunctor(function);
f.IgnoreInfinity = true;
f.IgnoreNaN = true;
UncertainValue estimate;
if (d < 1)
{
throw new ArgumentException("The dimension of the integration volume must be at least 1.", "volume");
}
else if (d < 4)
{
IntegrationRegion r = new IntegrationRegion(box);
estimate = Integrate_Adaptive(f, map, r, settings);
}
else if (d < 16)
{
estimate = Integrate_MonteCarlo(f, map, box, settings);
}
else
{
throw new ArgumentException("The dimension of the integrtion volume must be less than 16.", "volume");
}
// Sometimes the estimated uncertainty drops precipitiously. We will not report an uncertainty less than 3/4 of that demanded.
double minUncertainty = Math.Max(0.75 * settings.AbsolutePrecision, Math.Abs(estimate.Value) * 0.75 * settings.RelativePrecision);
if (estimate.Uncertainty < minUncertainty)
{
estimate = new UncertainValue(estimate.Value, minUncertainty);
}
return(new IntegrationResult(estimate, f.EvaluationCount, settings));
}
// This method is due to Powell (http://en.wikipedia.org/wiki/Michael_J._D._Powell), but it is not what
// is usually called Powell's Method (http://en.wikipedia.org/wiki/Powell%27s_method); Powell
// developed that method in the 1960s, it was included in Numerical Recipies and is very popular.
// This is a model trust algorithm developed by Powell in the 2000s. It typically uses many
// fewer function evaluations, but does more intensive calcuations between each evaluation.
// This is basically the UOBYQA variant of Powell's new methods. It maintains a quadratic model
// that interpolates between (d + 1) (d + 2) / 2 points. The model is trusted
// within a given radius. At each step, it moves to the minimum of the model (or the boundary of
// the trust region in that direction) and evaluates the function. The new value is incorporated
// into the model and the trust region expanded or contracted depending on how accurate its
// prediction of the function value was.
// Papers on these methods are collected at http://mat.uc.pt/~zhang/software.html#powell_software.
// The UOBYQA paper is here: http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.28.1756.
// The NEWUOA paper is here: http://www.damtp.cam.ac.uk/user/na/NA_papers/NA2004_08.pdf.
// The CONDOR system (http://www.applied-mathematics.net/optimization/CONDORdownload.html) is based on these same ideas.
// The thesis of CONDOR's author (http://www.applied-mathematics.net/mythesis/index.html) was also helpful.
// It should be very easy to extend this method to constrained optimization, either by incorporating the bounds into
// the step limits or by mapping hyper-space into a hyper-cube.
private static MultiExtremum FindMinimum_ModelTrust(MultiFunctor f, IList<double> x, double s, EvaluationSettings settings)
{
// Construct an initial model.
QuadraticInterpolationModel model = QuadraticInterpolationModel.Construct(f, x, s);
double trustRadius = s;
while (f.EvaluationCount < settings.EvaluationBudget) {
// Find the minimum point of the model within the trust radius
double[] z = model.FindMinimum(trustRadius);
double expectedValue = model.Evaluate(z);
double deltaExpected = model.MinimumValue - expectedValue;
// Evaluate the function at the suggested minimum
double[] point = model.ConvertPoint(z);
double value = f.Evaluate(point);
double delta = model.MinimumValue - value;
double tol = settings.ComputePrecision(value);
// To terminate, we demand: a reduction, the reduction be small, the reduction be in line with its expected value, that we have run up against trust boundary,
// and that the gradient is small.
// I had wanted to demand delta > 0, but we run into some cases where delta keeps being very slightly negative, typically orders of magnitude less than tol,
// causing the trust radius to shrink in and endless cycle that causes our approximation to ultimately go sour, even though terminating on the original
// very slightly negative delta would have produced an accurate estimate. So we tolerate this case for now.
if ((-tol / 4.0 <= delta) && (delta <= tol)) {
// We demand that the model be decent, i.e. that the expected delta was within tol of the measured delta.
if (Math.Abs(delta - deltaExpected) <= tol) {
// We demand that the step not just be small because it ran up against the trust radius. If it ran up against the trust radius,
// there is probably more to be hand by continuing.
double zm = Blas1.dNrm2(z, 0, 1, z.Length);
if (zm < trustRadius) {
// Finally, we demand that the gradient be small. You might think this was obvious since z was small, but if the Hessian is not positive definite
// the interplay of the Hessian and the gradient can produce a small z even if the model looks nothing like a quadratic minimum.
double gm = Blas1.dNrm2(model.GetGradient(), 0, 1, z.Length);
if (gm * zm <= tol) {
if (f.IsNegated) value = -value;
return (new MultiExtremum(f.EvaluationCount, settings, point, value, Math.Max(Math.Abs(delta), 0.75 * tol), model.GetHessian()));
}
}
}
}
// There are now three decisions to be made:
// 1. How to change the trust radius
// 2. Whether to accept the new point
// 3. Which existing point to replace
// If the actual change was very far from the expected change, reduce the trust radius.
// If the expected change did a good job of predicting the actual change, increase the trust radius.
if ((delta < 0.25 * deltaExpected) /*|| (8.0 * deltaExpected < delta)*/) {
trustRadius = trustRadius / 2.0;
} else if ((0.75 * deltaExpected <= delta) /*&& (delta <= 2.0 * deltaExpected)*/) {
trustRadius = 2.0 * trustRadius;
}
// It appears that the limits on delta being too large don't help, and even hurt if made too stringent.
// Replace an old point with the new point.
int iMax = 0; double fMax = model.values[0];
int iBad = 0; double fBad = model.ComputeBadness(0, z, point, value);
for (int i = 1; i < model.values.Length; i++) {
if (model.values[i] > fMax) { iMax = i; fMax = model.values[i]; }
double bad = model.ComputeBadness(i, z, point, value);
if (bad > fBad) { iBad = i; fBad = bad; }
}
if (value < fMax) {
Debug.WriteLine("iMax={0}, iBad={1}", iMax, iBad);
model.ReplacePoint(iBad, point, z, value);
}
// There is some question about how best to choose which point to replace.
// The largest value? The furthest away? The one closest to new min?
}
throw new NonconvergenceException();
}
