/// <summary>
/// Test that QuantileEstimator can handle billions of items.
/// </summary>
//[TestMethod, TestCategory("Performance")]
internal void QuantileEstimatorBillionsTest()
{
long n = 100000;
double maximumError = 1e-2;
QuantileEstimator est = new QuantileEstimator(maximumError);
for (int i = 0; i < n; i++)
{
if (i % 100 == 0)
{
Console.WriteLine(i);
}
for (int j = 0; j < n; j++)
{
est.Add(i + j);
}
}
long expectedCount = n * n;
ulong actualCount = est.GetCount();
Console.WriteLine($"count = {actualCount} should be {expectedCount}");
Assert.Equal(expectedCount, actualCount, maximumError * expectedCount);
double expectedMedian = n - 2;
double actualMedian = est.GetQuantile(0.5);
Console.WriteLine($"median = {actualMedian} should be {expectedMedian}");
Assert.Equal(expectedMedian, actualMedian, maximumError * expectedMedian);
}
public void QuantileEstimator_AllEqualTest()
{
double middle = 3.4;
double next = MMath.NextDouble(middle);
double[] x = { middle, middle, middle };
var outer = new OuterQuantiles(x);
Assert.Equal(0.0, outer.GetProbLessThan(middle));
Assert.Equal(middle, outer.GetQuantile(0.0));
Assert.Equal(middle, outer.GetQuantile(0.75));
Assert.Equal(1.0, outer.GetProbLessThan(next));
Assert.Equal(next, outer.GetQuantile(1.0));
foreach (int weight in new[] { 1, 2, 3 })
{
var est = new QuantileEstimator(0.01);
foreach (var item in x)
{
est.Add(item, weight);
}
Assert.Equal(0.0, est.GetProbLessThan(middle));
Assert.Equal(middle, est.GetQuantile(0.0));
Assert.Equal(1.0, est.GetProbLessThan(next));
Assert.Equal(next, est.GetQuantile(1.0));
}
}
public void QuantileEstimatorMergingTest()
{
double maximumError = 0.05;
// draw many samples from N(m,v)
Rand.Restart(0);
double m = 2;
double stddev = 3;
Gaussian prior = new Gaussian(m, stddev * stddev);
var est = new QuantileEstimator(maximumError);
List <double> x = new List <double>();
int batchCount = 10;
for (int batch = 0; batch < batchCount; batch++)
{
var est2 = new QuantileEstimator(maximumError);
int n = ((1 << 5) - 1) * 46 + 1;
for (int i = 0; i < n; i++)
{
double sample = prior.Sample();
x.Add(sample);
est2.Add(sample);
}
est.Add(est2);
}
CheckProbLessThan(est, x, maximumError);
}
public void InnerQuantiles_InfinityTest()
{
Assert.Throws <ArgumentOutOfRangeException>(() =>
{
var inner = new InnerQuantiles(new double[] { double.PositiveInfinity });
});
Assert.Throws <ArgumentOutOfRangeException>(() =>
{
var est = new QuantileEstimator(0.1);
est.Add(double.PositiveInfinity);
//est.Add(double.NegativeInfinity);
var inner = InnerQuantiles.FromDistribution(10, est);
});
}
public void QuantileEstimator_DoubleDuplicationTest()
{
double first = 1;
double second = 2;
double between = (first + second) / 2;
double next = MMath.NextDouble(second);
double[] x = { first, first, second, second };
// quantiles are 0, 1/3, 2/3, 1
var outer = new OuterQuantiles(x);
Assert.Equal(0.0, outer.GetProbLessThan(first));
Assert.Equal(first, outer.GetQuantile(0.0));
Assert.Equal(0.5, outer.GetProbLessThan(between));
Assert.Equal(between, outer.GetQuantile(0.5));
Assert.Equal(2.0 / 3, outer.GetProbLessThan(second));
Assert.Equal(second, outer.GetQuantile(2.0 / 3));
Assert.Equal(1.0, outer.GetProbLessThan(next));
Assert.Equal(next, outer.GetQuantile(1.0));
CheckGetQuantile(outer, outer);
var inner = new InnerQuantiles(5, outer);
CheckGetQuantile(inner, inner, (int)Math.Ceiling(100.0 / 6), (int)Math.Floor(100.0 * 5 / 6));
var est = new QuantileEstimator(0.01);
est.Add(first, 2);
est.Add(second, 2);
Assert.Equal(0.0, est.GetProbLessThan(first));
Assert.Equal(first, est.GetQuantile(0.0));
Assert.Equal(0.5, est.GetProbLessThan(between));
Assert.Equal(between, est.GetQuantile(0.5));
Assert.Equal(2.0 / 3, est.GetProbLessThan(second));
Assert.Equal(second, est.GetQuantile(2.0 / 3));
Assert.Equal(1.0, est.GetProbLessThan(next));
Assert.Equal(next, est.GetQuantile(1.0));
CheckGetQuantile(est, est);
}
public void QuantileEstimator_SinglePointIsMedian()
{
QuantileEstimator est = new QuantileEstimator(0.1);
double point = 2;
est.Add(point);
Assert.Equal(point, est.GetQuantile(0.5));
OuterQuantiles outer = new OuterQuantiles(new[] { point });
Assert.Equal(point, outer.GetQuantile(0.5));
InnerQuantiles inner = new InnerQuantiles(new[] { point });
Assert.Equal(point, inner.GetQuantile(0.5));
}
private void QuantileEstimatorTester(double maximumError, int n)
{
// draw many samples from N(m,v)
Rand.Restart(0);
double m = 2;
double stddev = 3;
Gaussian prior = new Gaussian(m, stddev * stddev);
var est = new QuantileEstimator(maximumError);
List <double> x = new List <double>();
for (int i = 0; i < n; i++)
{
double sample = prior.Sample();
x.Add(sample);
est.Add(sample);
}
CheckProbLessThan(est, x, maximumError);
}
public void QuantileSeedTest()
{
var seed = Rand.Int();
double[] GetRandomQuantiles()
{
var rand = new Random(seed);
var estimator = new QuantileEstimator(rand.NextDouble());
estimator.SetRandomSeed(rand.Next());
var numbers =
Enumerable.Range(0, rand.Next(100))
.Select(x => rand.Next())
.ToArray();
foreach (var item in numbers)
{
estimator.Add(item, 1 + rand.Next(20));
}
var quantiles =
numbers
.Select(x => estimator.GetProbLessThan(x))
.ToArray();
return(quantiles);
}
// Run the same estimation run twice with the same seed to test
// that the results are the same.
var firstRun = GetRandomQuantiles();
var secondRun = GetRandomQuantiles();
Assert.Equal(firstRun, secondRun);
}
