/// <summary>
/// Returns the value x such that GetProbLessThan(x) == probability.
/// </summary>
/// <param name="probability">A real number in [0,1].</param>
/// <returns></returns>
public double GetQuantile(double probability)
{
if (probability < 0)
{
throw new ArgumentOutOfRangeException("probability < 0");
}
if (probability > 1)
{
throw new ArgumentOutOfRangeException("probability > 1");
}
if (this.IsPointMass)
{
return((probability == 1.0) ? MMath.NextDouble(this.Point) : this.Point);
}
else if (!IsProper())
{
throw new ImproperDistributionException(this);
}
else if (Shape == 1)
{
// cdf is 1 - exp(-x*rate)
return(-Math.Log(1 - probability) / Rate);
}
else
{
throw new NotImplementedException("Shape != 1 is not implemented");
}
}
public void QuantileEstimator_AllEqualTest()
{
double middle = 3.4;
double next = MMath.NextDouble(middle);
double[] x = { middle, middle, middle };
var outer = new OuterQuantiles(x);
Assert.Equal(0.0, outer.GetProbLessThan(middle));
Assert.Equal(middle, outer.GetQuantile(0.0));
Assert.Equal(middle, outer.GetQuantile(0.75));
Assert.Equal(1.0, outer.GetProbLessThan(next));
Assert.Equal(next, outer.GetQuantile(1.0));
foreach (int weight in new[] { 1, 2, 3 })
{
var est = new QuantileEstimator(0.01);
foreach (var item in x)
{
est.Add(item, weight);
}
Assert.Equal(0.0, est.GetProbLessThan(middle));
Assert.Equal(middle, est.GetQuantile(0.0));
Assert.Equal(1.0, est.GetProbLessThan(next));
Assert.Equal(next, est.GetQuantile(1.0));
}
}
public void QuantileEstimator_DoubleDuplicationTest()
{
double first = 1;
double second = 2;
double between = (first + second) / 2;
double next = MMath.NextDouble(second);
double[] x = { first, first, second, second };
// quantiles are 0, 1/3, 2/3, 1
var outer = new OuterQuantiles(x);
Assert.Equal(0.0, outer.GetProbLessThan(first));
Assert.Equal(first, outer.GetQuantile(0.0));
Assert.Equal(0.5, outer.GetProbLessThan(between));
Assert.Equal(between, outer.GetQuantile(0.5));
Assert.Equal(2.0 / 3, outer.GetProbLessThan(second));
Assert.Equal(second, outer.GetQuantile(2.0 / 3));
Assert.Equal(1.0, outer.GetProbLessThan(next));
Assert.Equal(next, outer.GetQuantile(1.0));
CheckGetQuantile(outer, outer);
var inner = InnerQuantiles.FromDistribution(5, outer);
CheckGetQuantile(inner, inner, (int)Math.Ceiling(100.0 / 6), (int)Math.Floor(100.0 * 5 / 6));
var est = new QuantileEstimator(0.01);
est.Add(first, 2);
est.Add(second, 2);
Assert.Equal(0.0, est.GetProbLessThan(first));
Assert.Equal(first, est.GetQuantile(0.0));
Assert.Equal(0.5, est.GetProbLessThan(between));
Assert.Equal(second, est.GetQuantile(2.0 / 3));
Assert.Equal(1.0, est.GetProbLessThan(next));
Assert.Equal(next, est.GetQuantile(1.0));
CheckGetQuantile(est, est);
}
/// <summary>
/// Returns the value x such that GetProbLessThan(x) == probability.
/// </summary>
/// <param name="probability">A real number in [0,1].</param>
/// <returns></returns>
public double GetQuantile(double probability)
{
if (probability < 0)
{
throw new ArgumentOutOfRangeException("probability < 0");
}
if (probability > 1)
{
throw new ArgumentOutOfRangeException("probability > 1");
}
if (this.IsPointMass)
{
return((probability == 1.0) ? MMath.NextDouble(this.Point) : this.Point);
}
else if (Precision <= 0.0)
{
throw new ImproperDistributionException(this);
}
else
{
double mean = MeanTimesPrecision / Precision;
double sqrtPrec = Math.Sqrt(Precision);
return(MMath.NormalCdfInv(probability) / sqrtPrec + mean);
}
}
/// <summary>
/// Returns the value x such that GetProbLessThan(x) == probability.
/// </summary>
/// <param name="probability">A real number in [0,1].</param>
/// <returns></returns>
public double GetQuantile(double probability)
{
if (probability < 0)
{
throw new ArgumentOutOfRangeException("probability < 0");
}
if (probability > 1)
{
throw new ArgumentOutOfRangeException("probability > 1");
}
if (this.IsPointMass)
{
return((probability == 1.0) ? MMath.NextDouble(this.Point) : this.Point);
}
else if (!IsProper())
{
throw new ImproperDistributionException(this);
}
else if (MMath.AreEqual(probability, 0))
{
return(0);
}
else if (MMath.AreEqual(probability, 1))
{
return(double.PositiveInfinity);
}
else if (Shape == 1)
{
// cdf is 1 - exp(-x*rate)
return(-Math.Log(1 - probability) / Rate);
}
else
{
// Binary search
double lowerBound = 0;
double upperBound = double.MaxValue;
while (lowerBound < upperBound)
{
double average = MMath.Average(lowerBound, upperBound);
double p = GetProbLessThan(average);
if (p == probability)
{
return(average);
}
else if (p < probability)
{
lowerBound = MMath.NextDouble(average);
}
else
{
upperBound = MMath.PreviousDouble(average);
}
}
return(lowerBound);
}
}
internal void CheckGetQuantile(CanGetQuantile canGetQuantile, CanGetProbLessThan canGetProbLessThan, int minPercent = 0, int maxPercent = 100)
{
for (int i = minPercent; i < maxPercent; i++)
{
// probability = 1.0 is excluded
double probability = i / 100.0;
double x = canGetQuantile.GetQuantile(probability);
double probLessThanX = canGetProbLessThan.GetProbLessThan(x);
Assert.True(probLessThanX <= probability);
double next = MMath.NextDouble(x);
double probLessThanNext = canGetProbLessThan.GetProbLessThan(next);
Assert.True(probLessThanNext > probability);
}
}
/// <summary>
/// Returns the largest value x such that GetProbLessThan(x) <= probability.
/// </summary>
/// <param name="probability">A real number in [0,1].</param>
/// <param name="quantiles">Numbers in increasing order corresponding to the quantiles for probabilities (0,...,n-1)/(n-1).</param>
/// <returns></returns>
public static double GetQuantile(double probability, double[] quantiles)
{
if (probability < 0)
{
throw new ArgumentOutOfRangeException(nameof(probability), "probability < 0");
}
if (probability > 1.0)
{
throw new ArgumentOutOfRangeException(nameof(probability), "probability > 1.0");
}
int n = quantiles.Length;
if (quantiles == null)
{
throw new ArgumentNullException(nameof(quantiles));
}
if (n == 0)
{
throw new ArgumentException("quantiles array is empty", nameof(quantiles));
}
if (probability == 1.0)
{
return(MMath.NextDouble(quantiles[n - 1]));
}
if (n == 1)
{
return(quantiles[0]);
}
double pos = MMath.LargestDoubleProduct(n - 1, probability);
int lower = (int)Math.Floor(pos);
int upper = (int)Math.Ceiling(pos);
if (upper == lower)
{
upper = lower + 1;
}
return(GetQuantile(probability, lower, quantiles[lower], quantiles[upper], n));
}
/// <summary>
/// Used to tune MMath.NormalCdfLn. The best tuning minimizes the number of messages printed.
/// </summary>
internal void NormalCdf2Test2()
{
// Call both routines now to speed up later calls.
MMath.NormalCdf(-2, -2, -0.5);
NormalCdf_Quadrature(-2, -2, -0.5);
Stopwatch watch = new Stopwatch();
double xmin = 0.1;
double xmax = 0.1;
double n = 20;
double xinc = (xmax - xmin) / (n - 1);
for (int xi = 0; xi < n; xi++)
{
if (xinc == 0 && xi > 0)
{
break;
}
double x = xmin + xi * xinc;
double ymin = -System.Math.Abs(x) * 10;
double ymax = -ymin;
double yinc = (ymax - ymin) / (n - 1);
for (int yi = 0; yi < n; yi++)
{
double y = ymin + yi * yinc;
double rmin = -0.999999;
double rmax = -0.000001;
rmin = MMath.NextDouble(-1);
rmax = -1 + 1e-6;
//rmax = -0.5;
//rmax = 0.1;
//rmax = -0.58;
//rmax = -0.9;
//rmin = -0.5;
rmax = 1;
double rinc = (rmax - rmin) / (n - 1);
for (int ri = 0; ri < n; ri++)
{
double r = rmin + ri * rinc;
string good = "good";
watch.Restart();
double result1 = double.NaN;
try
{
result1 = MMath.NormalCdfIntegral(x, y, r);
}
catch
{
good = "bad";
//throw;
}
watch.Stop();
long ticks = watch.ElapsedTicks;
watch.Restart();
double result2 = double.NaN;
try
{
result2 = NormalCdfLn_Quadrature(x, y, r);
}
catch
{
}
long ticks2 = watch.ElapsedTicks;
bool overtime = ticks > 10 * ticks2;
if (double.IsNaN(result1) /*|| overtime*/)
{
Trace.WriteLine($"({x:g17},{y:g17},{r:g17},{x - r * y}): {good} {ticks} {ticks2} {result1} {result2}");
}
}
}
}
}
/// <summary>
/// Returns the largest value x such that GetProbLessThan(x) <= probability.
/// </summary>
/// <param name="probability">A real number in [0,1].</param>
/// <returns></returns>
public double GetQuantile(double probability)
{
if (probability < 0)
{
throw new ArgumentOutOfRangeException(nameof(probability), "probability < 0");
}
if (probability > 1.0)
{
throw new ArgumentOutOfRangeException(nameof(probability), "probability > 1.0");
}
// compute the min and max of the retained items
double lowerBound = double.PositiveInfinity, upperBound = double.NegativeInfinity;
for (int bufferIndex = 0; bufferIndex < buffers.Length; bufferIndex++)
{
double[] buffer = buffers[bufferIndex];
int count = countInBuffer[bufferIndex];
for (int i = 0; i < count; i++)
{
double item = buffer[i];
if (item < lowerBound)
{
lowerBound = item;
}
if (item > upperBound)
{
upperBound = item;
}
}
}
if (probability == 1.0)
{
return(MMath.NextDouble(upperBound));
}
if (probability == 0.0)
{
return(lowerBound);
}
if (double.IsPositiveInfinity(lowerBound))
{
throw new Exception("QuantileEstimator has no data");
}
if (lowerBound == upperBound)
{
return(upperBound);
}
// use bisection
while (true)
{
double x = MMath.Average(lowerBound, upperBound);
double lowerItem;
double upperItem;
long lowerRank;
long lowerWeight, minLowerWeight, upperWeight, minUpperWeight;
long itemCount;
GetAdjacentItems(x, out lowerItem, out upperItem, out lowerRank, out lowerWeight, out upperWeight, out minLowerWeight, out minUpperWeight, out itemCount);
if (InterpolationType == 0)
{
double probabilityLessThanLowerItem = (double)(lowerRank - lowerWeight) / itemCount;
if (probability < probabilityLessThanLowerItem)
{
upperBound = MMath.PreviousDouble(lowerItem);
}
else
{
double probabilityLessThanUpperItem = (double)lowerRank / itemCount;
if (probability < probabilityLessThanUpperItem)
{
return(lowerItem);
}
double probabilityLessThanOrEqualUpperItem = (double)(lowerRank + upperWeight) / itemCount;
if (probability < probabilityLessThanOrEqualUpperItem)
{
return(upperItem);
}
lowerBound = MMath.NextDouble(upperItem);
}
}
else if (InterpolationType == 1)
{
// Find frac such that (lowerRank - 0.5 + frac) / itemCount == probability
double scaledProbability = MMath.LargestDoubleProduct(itemCount, probability);
if (scaledProbability < 0.5)
{
return(lowerBound);
}
if (scaledProbability >= itemCount - 0.5)
{
return(upperBound);
}
// probability of lowerItem ranges from (lowerRank-lowerWeight+0.5) / itemCount
// to (lowerRank - 0.5) / itemCount
//if (scaledProbability == lowerRank - lowerWeight + 0.5) return lowerItem;
if ((scaledProbability > lowerRank - lowerWeight + 0.5) && (scaledProbability < lowerRank - 0.5))
{
return(lowerItem);
}
// probability of upperItem ranges from (lowerRank + 0.5) / itemCount
// to (lowerRank + upperWeight - 0.5) / itemCount
if (scaledProbability == lowerRank + 0.5)
{
return(upperItem);
}
if ((scaledProbability > lowerRank + 0.5) && (scaledProbability < lowerRank + upperWeight - 0.5))
{
return(upperItem);
}
double frac = scaledProbability - (lowerRank - 0.5);
if (frac < 0)
{
upperBound = MMath.PreviousDouble(x);
}
else if (frac > 1)
{
lowerBound = MMath.NextDouble(x);
}
else
{
return(OuterQuantiles.GetQuantile(probability, lowerRank - 0.5, lowerItem, upperItem, itemCount + 1));
}
}
else
{
double scaledProbability = MMath.LargestDoubleProduct(itemCount - 1, probability);
// probability of lowerItem ranges from (lowerRank-lowerWeight) / (itemCount - 1)
// to (lowerRank - 1) / (itemCount - 1).
if (scaledProbability == lowerRank - lowerWeight)
{
return(lowerItem);
}
if ((scaledProbability > lowerRank - lowerWeight) && (scaledProbability < lowerRank - 1))
{
return(lowerItem);
}
// probability of upperItem ranges from lowerRank / (itemCount - 1)
// to (lowerRank + upperWeight-1) / (itemCount - 1)
if (scaledProbability == lowerRank)
{
return(upperItem);
}
if ((scaledProbability > lowerRank) && (scaledProbability < lowerRank + upperWeight - 1))
{
return(upperItem);
}
// solve for frac in (lowerRank-1 + frac) / (itemCount - 1) == probability
double frac = scaledProbability - (lowerRank - 1);
if (frac < 0)
{
upperBound = MMath.PreviousDouble(x);
}
else if (frac > 1)
{
lowerBound = MMath.NextDouble(x);
}
else
{
return(OuterQuantiles.GetQuantile(probability, lowerRank - 1, lowerItem, upperItem, itemCount));
}
}
}
}
/// <summary>
/// Get the mean and variance after truncation.
/// </summary>
/// <param name="mean"></param>
/// <param name="variance"></param>
public void GetMeanAndVariance(out double mean, out double variance)
{
if (this.Gamma.IsPointMass)
{
mean = this.Gamma.Point;
variance = 0.0;
}
else if (!IsProper())
{
throw new ImproperDistributionException(this);
}
else
{
// Apply the recurrence GammaUpper(s+1,x,false) = s*GammaUpper(s,x,false) + x^s*exp(-x)
// Z = GammaUpper(s,r*l,false) - GammaUpper(s,r*u,false)
// E[x] = (GammaUpper(s+1,r*l,false) - GammaUpper(s+1,r*u,false))/r/Z
// = (s + ((r*l)^s*exp(-r*l) - (r*u)^s*exp(-r*u))/Z)/r
double rl = this.Gamma.Rate * LowerBound;
double ru = this.Gamma.Rate * UpperBound;
double m = this.Gamma.Shape / this.Gamma.Rate;
double offset, offset2;
if (ru > double.MaxValue)
{
double logZ = GetLogNormalizer();
if (logZ < double.MinValue)
{
mean = GetMode();
variance = 0.0;
return;
}
offset = Math.Exp(MMath.GammaUpperLogScale(this.Gamma.Shape, rl) - logZ);
offset2 = (rl - this.Gamma.Shape) / this.Gamma.Rate * offset;
}
else
{
// This fails when GammaUpperScale underflows to 0
double Z = GetNormalizer();
if (Z == 0)
{
mean = GetMode();
variance = 0.0;
return;
}
double gammaUpperScaleLower = MMath.GammaUpperScale(this.Gamma.Shape, rl);
double gammaUpperScaleUpper = MMath.GammaUpperScale(this.Gamma.Shape, ru);
offset = (gammaUpperScaleLower - gammaUpperScaleUpper) / Z;
offset2 = ((rl - this.Gamma.Shape) / this.Gamma.Rate * gammaUpperScaleLower - (ru - this.Gamma.Shape) / this.Gamma.Rate * gammaUpperScaleUpper) / Z;
}
if (rl == this.Gamma.Shape)
{
mean = LowerBound + offset / this.Gamma.Rate;
}
else
{
mean = (this.Gamma.Shape + offset) / this.Gamma.Rate;
if (mean < LowerBound)
{
mean = MMath.NextDouble(mean);
}
if (mean < LowerBound)
{
mean = MMath.NextDouble(mean);
}
}
if (mean > double.MaxValue)
{
variance = mean;
}
else
{
variance = (m + offset2 + (1 - offset) * offset / this.Gamma.Rate) / this.Gamma.Rate;
}
}
}
/// <summary>
/// Returns the largest value x such that GetProbLessThan(x) <= probability.
/// </summary>
/// <param name="probability">A real number in [0,1].</param>
/// <returns></returns>
public double GetQuantile(double probability)
{
if (probability < 0)
{
throw new ArgumentOutOfRangeException(nameof(probability), "probability < 0");
}
if (probability > 1.0)
{
throw new ArgumentOutOfRangeException(nameof(probability), "probability > 1.0");
}
// compute the min and max of the retained items
double lowerBound = double.PositiveInfinity, upperBound = double.NegativeInfinity;
for (int bufferIndex = 0; bufferIndex < buffers.Length; bufferIndex++)
{
double[] buffer = buffers[bufferIndex];
int count = countInBuffer[bufferIndex];
for (int i = 0; i < count; i++)
{
double item = buffer[i];
if (item < lowerBound)
{
lowerBound = item;
}
if (item > upperBound)
{
upperBound = item;
}
}
}
if (probability == 1.0)
{
return(MMath.NextDouble(upperBound));
}
if (double.IsPositiveInfinity(lowerBound))
{
throw new Exception("QuantileEstimator has no data");
}
if (lowerBound == upperBound)
{
return(upperBound);
}
// use bisection
while (true)
{
double x = (lowerBound + upperBound) / 2;
double lowerItem;
double upperItem;
long lowerIndex;
long lowerWeight, minLowerWeight, upperWeight, minUpperWeight;
long itemCount;
GetAdjacentItems(x, out lowerItem, out upperItem, out lowerIndex, out lowerWeight, out upperWeight, out minLowerWeight, out minUpperWeight, out itemCount);
double scaledProbability = MMath.LargestDoubleProduct(itemCount - 1, probability);
// probability of lowerItem ranges from (lowerIndex - lowerWeight + 1) / (itemCount - 1)
// to lowerIndex / (itemCount - 1).
if ((scaledProbability >= lowerIndex - lowerWeight + 1) && (scaledProbability < lowerIndex))
{
return(lowerItem);
}
// probability of upperItem ranges from (lowerIndex + 1) / (itemCount - 1)
// to (lowerIndex + upperWeight) / (itemCount - 1)
if ((scaledProbability >= lowerIndex + 1) && (scaledProbability <= lowerIndex + upperWeight))
{
return(upperItem);
}
// solve for frac in (lowerIndex + frac) / (itemCount - 1) == probability
double frac = scaledProbability - lowerIndex;
if (frac < 0)
{
upperBound = x;
}
else if (frac > 1)
{
lowerBound = x;
}
else
{
return(OuterQuantiles.GetQuantile(probability, lowerIndex, lowerItem, upperItem, itemCount));
}
}
}
