public void QuantileEstimator_DoubleDuplicationTest2()
{
var data = new double[]
{
0.16659357378138889, // 0
0.70210023978217528, // 0.25
0.70210023978217528, // 0.5
0.70319732172768734, // 0.75
0.70319732172768734 // 1
};
var est = new QuantileEstimator(0.01);
est.AddRange(data);
Assert.Equal(data[4], est.GetQuantile(0.76));
Assert.Equal(data[2], est.GetQuantile(0.3));
CheckGetQuantile(est, est);
var outer = new OuterQuantiles(data);
Assert.Equal(data[4], outer.GetQuantile(0.76));
Assert.Equal(data[2], outer.GetQuantile(0.3));
CheckGetQuantile(outer, outer);
var inner = InnerQuantiles.FromDistribution(7, outer);
CheckGetQuantile(inner, inner, (int)Math.Ceiling(100.0 / 8), (int)Math.Floor(100.0 * 7 / 8));
}
public void QuantileEstimator_AllEqualTest()
{
double middle = 3.4;
double next = MMath.NextDouble(middle);
double[] x = { middle, middle, middle };
var outer = new OuterQuantiles(x);
Assert.Equal(0.0, outer.GetProbLessThan(middle));
Assert.Equal(middle, outer.GetQuantile(0.0));
Assert.Equal(middle, outer.GetQuantile(0.75));
Assert.Equal(1.0, outer.GetProbLessThan(next));
Assert.Equal(next, outer.GetQuantile(1.0));
foreach (int weight in new[] { 1, 2, 3 })
{
var est = new QuantileEstimator(0.01);
foreach (var item in x)
{
est.Add(item, weight);
}
Assert.Equal(0.0, est.GetProbLessThan(middle));
Assert.Equal(middle, est.GetQuantile(0.0));
Assert.Equal(1.0, est.GetProbLessThan(next));
Assert.Equal(next, est.GetQuantile(1.0));
}
}
public void QuantileEstimator_DuplicationTest()
{
double middle = 3.4;
double[] x = { 1.2, middle, middle, middle, 5.6 };
var outer = new OuterQuantiles(x);
Assert.Equal(0.25, outer.GetProbLessThan(middle));
Assert.Equal(outer.GetQuantile(0.3), middle);
Assert.Equal(outer.GetQuantile(0.5), middle);
Assert.Equal(outer.GetQuantile(0.7), middle);
CheckGetQuantile(outer, outer);
var inner = InnerQuantiles.FromDistribution(7, outer);
Assert.Equal(0.25, inner.GetProbLessThan(middle));
Assert.Equal(outer.GetQuantile(0.3), middle);
Assert.Equal(outer.GetQuantile(0.5), middle);
Assert.Equal(outer.GetQuantile(0.7), middle);
CheckGetQuantile(inner, inner, (int)Math.Ceiling(100.0 / 8), (int)Math.Floor(100.0 * 7 / 8));
var est = new QuantileEstimator(0.01);
est.AddRange(x);
Assert.Equal(est.GetQuantile(0.3), middle);
Assert.Equal(est.GetQuantile(0.5), middle);
// InterpolationType==1 returns NextDouble(middle)
Assert.Equal(est.GetQuantile(0.7), middle, 1e-15);
CheckGetQuantile(est, est);
}
public void QuantileEstimator_DoubleDuplicationTest()
{
double first = 1;
double second = 2;
double between = (first + second) / 2;
double next = MMath.NextDouble(second);
double[] x = { first, first, second, second };
// quantiles are 0, 1/3, 2/3, 1
var outer = new OuterQuantiles(x);
Assert.Equal(0.0, outer.GetProbLessThan(first));
Assert.Equal(first, outer.GetQuantile(0.0));
Assert.Equal(0.5, outer.GetProbLessThan(between));
Assert.Equal(between, outer.GetQuantile(0.5));
Assert.Equal(2.0 / 3, outer.GetProbLessThan(second));
Assert.Equal(second, outer.GetQuantile(2.0 / 3));
Assert.Equal(1.0, outer.GetProbLessThan(next));
Assert.Equal(next, outer.GetQuantile(1.0));
CheckGetQuantile(outer, outer);
var inner = InnerQuantiles.FromDistribution(5, outer);
CheckGetQuantile(inner, inner, (int)Math.Ceiling(100.0 / 6), (int)Math.Floor(100.0 * 5 / 6));
var est = new QuantileEstimator(0.01);
est.Add(first, 2);
est.Add(second, 2);
Assert.Equal(0.0, est.GetProbLessThan(first));
Assert.Equal(first, est.GetQuantile(0.0));
Assert.Equal(0.5, est.GetProbLessThan(between));
Assert.Equal(second, est.GetQuantile(2.0 / 3));
Assert.Equal(1.0, est.GetProbLessThan(next));
Assert.Equal(next, est.GetQuantile(1.0));
CheckGetQuantile(est, est);
}
public void QuantileEstimator_DuplicationTest()
{
double middle = 3.4;
double[] x = { 1.2, middle, middle, middle, 5.6 };
var outer = new OuterQuantiles(x);
Assert.Equal(0.25, outer.GetProbLessThan(middle));
Assert.Equal(outer.GetQuantile(0.3), middle);
Assert.Equal(outer.GetQuantile(0.5), middle);
Assert.Equal(outer.GetQuantile(0.7), middle);
CheckGetQuantile(outer, outer);
var inner = new InnerQuantiles(7, outer);
Assert.Equal(0.25, inner.GetProbLessThan(middle));
Assert.Equal(outer.GetQuantile(0.3), middle);
Assert.Equal(outer.GetQuantile(0.5), middle);
Assert.Equal(outer.GetQuantile(0.7), middle);
CheckGetQuantile(inner, inner, 100 / 8, 100 * 7 / 8);
var est = new QuantileEstimator(0.01);
est.AddRange(x);
Assert.Equal(0.25, est.GetProbLessThan(middle));
Assert.Equal(est.GetQuantile(0.3), middle);
Assert.Equal(est.GetQuantile(0.5), middle);
Assert.Equal(est.GetQuantile(0.7), middle);
CheckGetQuantile(est, est);
}
public void QuantileEstimator_InfinityTest()
{
var est = new QuantileEstimator(0.1);
est.Add(double.PositiveInfinity);
Assert.Equal(double.PositiveInfinity, est.GetQuantile(0.1));
}
/// <summary>
/// Test that QuantileEstimator can handle billions of items.
/// </summary>
//[TestMethod, TestCategory("Performance")]
internal void QuantileEstimatorBillionsTest()
{
long n = 100000;
double maximumError = 1e-2;
QuantileEstimator est = new QuantileEstimator(maximumError);
for (int i = 0; i < n; i++)
{
if (i % 100 == 0)
{
Console.WriteLine(i);
}
for (int j = 0; j < n; j++)
{
est.Add(i + j);
}
}
long expectedCount = n * n;
ulong actualCount = est.GetCount();
Console.WriteLine($"count = {actualCount} should be {expectedCount}");
Assert.Equal(expectedCount, actualCount, maximumError * expectedCount);
double expectedMedian = n - 2;
double actualMedian = est.GetQuantile(0.5);
Console.WriteLine($"median = {actualMedian} should be {expectedMedian}");
Assert.Equal(expectedMedian, actualMedian, maximumError * expectedMedian);
}
public override double GetValue(Sample a, Sample b, Probability probability)
{
Assertion.NotNull(nameof(a), a);
Assertion.NotNull(nameof(b), b);
try
{
double aMad = MedianAbsoluteDeviation.CalcMad(a);
double bMad = MedianAbsoluteDeviation.CalcMad(b);
if (aMad < Eps && bMad < Eps)
{
double aMedian = QuantileEstimator.GetMedian(a);
double bMedian = QuantileEstimator.GetMedian(b);
if (Math.Abs(aMedian - bMedian) < Eps)
{
return(0);
}
return(aMedian < bMedian
? double.PositiveInfinity
: double.NegativeInfinity);
}
double aQuantile = QuantileEstimator.GetQuantile(a, probability);
double bQuantile = QuantileEstimator.GetQuantile(b, probability);
double pooledMad = PooledMad(a.Count, b.Count, aMad, bMad);
return((bQuantile - aQuantile) / pooledMad);
}
catch (Exception)
{
return(double.NaN);
}
}
public void QuantileEstimator_SinglePointIsMedian()
{
QuantileEstimator est = new QuantileEstimator(0.1);
double point = 2;
est.Add(point);
Assert.Equal(point, est.GetQuantile(0.5));
OuterQuantiles outer = new OuterQuantiles(new[] { point });
Assert.Equal(point, outer.GetQuantile(0.5));
InnerQuantiles inner = new InnerQuantiles(new[] { point });
Assert.Equal(point, inner.GetQuantile(0.5));
}
public void QuantileEstimator_MedianTest()
{
double middle = 3.4;
double[] x = { 1.2, middle, 5.6 };
var outer = new OuterQuantiles(x);
Assert.Equal(outer.GetQuantile(0.5), middle);
var inner = new InnerQuantiles(3, outer);
Assert.Equal(inner.GetQuantile(0.5), middle);
var est = new QuantileEstimator(0.01);
est.AddRange(x);
Assert.Equal(est.GetQuantile(0.5), middle);
}
public void QuantileEstimator_MedianTest()
{
double left = 1.2;
double middle = 3.4;
double right = 5.6;
double[] x = { left, middle, right };
var outer = new OuterQuantiles(x);
Assert.Equal(middle, outer.GetQuantile(0.5));
var inner = InnerQuantiles.FromDistribution(3, outer);
Assert.Equal(middle, inner.GetQuantile(0.5));
inner = new InnerQuantiles(x);
CheckGetQuantile(inner, inner, 25, 75);
var est = new QuantileEstimator(0.01);
est.AddRange(x);
Assert.Equal(est.GetQuantile(0.5), middle);
}
