/// <summary> /// Test that QuantileEstimator can handle billions of items. /// </summary> //[TestMethod, TestCategory("Performance")] internal void QuantileEstimatorBillionsTest() { long n = 100000; double maximumError = 1e-2; QuantileEstimator est = new QuantileEstimator(maximumError); for (int i = 0; i < n; i++) { if (i % 100 == 0) { Console.WriteLine(i); } for (int j = 0; j < n; j++) { est.Add(i + j); } } long expectedCount = n * n; ulong actualCount = est.GetCount(); Console.WriteLine($"count = {actualCount} should be {expectedCount}"); Assert.Equal(expectedCount, actualCount, maximumError * expectedCount); double expectedMedian = n - 2; double actualMedian = est.GetQuantile(0.5); Console.WriteLine($"median = {actualMedian} should be {expectedMedian}"); Assert.Equal(expectedMedian, actualMedian, maximumError * expectedMedian); }
public void QuantileEstimator_AllEqualTest() { double middle = 3.4; double next = MMath.NextDouble(middle); double[] x = { middle, middle, middle }; var outer = new OuterQuantiles(x); Assert.Equal(0.0, outer.GetProbLessThan(middle)); Assert.Equal(middle, outer.GetQuantile(0.0)); Assert.Equal(middle, outer.GetQuantile(0.75)); Assert.Equal(1.0, outer.GetProbLessThan(next)); Assert.Equal(next, outer.GetQuantile(1.0)); foreach (int weight in new[] { 1, 2, 3 }) { var est = new QuantileEstimator(0.01); foreach (var item in x) { est.Add(item, weight); } Assert.Equal(0.0, est.GetProbLessThan(middle)); Assert.Equal(middle, est.GetQuantile(0.0)); Assert.Equal(1.0, est.GetProbLessThan(next)); Assert.Equal(next, est.GetQuantile(1.0)); } }
public void QuantileEstimatorMergingTest() { double maximumError = 0.05; // draw many samples from N(m,v) Rand.Restart(0); double m = 2; double stddev = 3; Gaussian prior = new Gaussian(m, stddev * stddev); var est = new QuantileEstimator(maximumError); List <double> x = new List <double>(); int batchCount = 10; for (int batch = 0; batch < batchCount; batch++) { var est2 = new QuantileEstimator(maximumError); int n = ((1 << 5) - 1) * 46 + 1; for (int i = 0; i < n; i++) { double sample = prior.Sample(); x.Add(sample); est2.Add(sample); } est.Add(est2); } CheckProbLessThan(est, x, maximumError); }
public void InnerQuantiles_InfinityTest() { Assert.Throws <ArgumentOutOfRangeException>(() => { var inner = new InnerQuantiles(new double[] { double.PositiveInfinity }); }); Assert.Throws <ArgumentOutOfRangeException>(() => { var est = new QuantileEstimator(0.1); est.Add(double.PositiveInfinity); //est.Add(double.NegativeInfinity); var inner = InnerQuantiles.FromDistribution(10, est); }); }
public void QuantileEstimator_DoubleDuplicationTest() { double first = 1; double second = 2; double between = (first + second) / 2; double next = MMath.NextDouble(second); double[] x = { first, first, second, second }; // quantiles are 0, 1/3, 2/3, 1 var outer = new OuterQuantiles(x); Assert.Equal(0.0, outer.GetProbLessThan(first)); Assert.Equal(first, outer.GetQuantile(0.0)); Assert.Equal(0.5, outer.GetProbLessThan(between)); Assert.Equal(between, outer.GetQuantile(0.5)); Assert.Equal(2.0 / 3, outer.GetProbLessThan(second)); Assert.Equal(second, outer.GetQuantile(2.0 / 3)); Assert.Equal(1.0, outer.GetProbLessThan(next)); Assert.Equal(next, outer.GetQuantile(1.0)); CheckGetQuantile(outer, outer); var inner = new InnerQuantiles(5, outer); CheckGetQuantile(inner, inner, (int)Math.Ceiling(100.0 / 6), (int)Math.Floor(100.0 * 5 / 6)); var est = new QuantileEstimator(0.01); est.Add(first, 2); est.Add(second, 2); Assert.Equal(0.0, est.GetProbLessThan(first)); Assert.Equal(first, est.GetQuantile(0.0)); Assert.Equal(0.5, est.GetProbLessThan(between)); Assert.Equal(between, est.GetQuantile(0.5)); Assert.Equal(2.0 / 3, est.GetProbLessThan(second)); Assert.Equal(second, est.GetQuantile(2.0 / 3)); Assert.Equal(1.0, est.GetProbLessThan(next)); Assert.Equal(next, est.GetQuantile(1.0)); CheckGetQuantile(est, est); }
public void QuantileEstimator_SinglePointIsMedian() { QuantileEstimator est = new QuantileEstimator(0.1); double point = 2; est.Add(point); Assert.Equal(point, est.GetQuantile(0.5)); OuterQuantiles outer = new OuterQuantiles(new[] { point }); Assert.Equal(point, outer.GetQuantile(0.5)); InnerQuantiles inner = new InnerQuantiles(new[] { point }); Assert.Equal(point, inner.GetQuantile(0.5)); }
private void QuantileEstimatorTester(double maximumError, int n) { // draw many samples from N(m,v) Rand.Restart(0); double m = 2; double stddev = 3; Gaussian prior = new Gaussian(m, stddev * stddev); var est = new QuantileEstimator(maximumError); List <double> x = new List <double>(); for (int i = 0; i < n; i++) { double sample = prior.Sample(); x.Add(sample); est.Add(sample); } CheckProbLessThan(est, x, maximumError); }
public void QuantileSeedTest() { var seed = Rand.Int(); double[] GetRandomQuantiles() { var rand = new Random(seed); var estimator = new QuantileEstimator(rand.NextDouble()); estimator.SetRandomSeed(rand.Next()); var numbers = Enumerable.Range(0, rand.Next(100)) .Select(x => rand.Next()) .ToArray(); foreach (var item in numbers) { estimator.Add(item, 1 + rand.Next(20)); } var quantiles = numbers .Select(x => estimator.GetProbLessThan(x)) .ToArray(); return(quantiles); } // Run the same estimation run twice with the same seed to test // that the results are the same. var firstRun = GetRandomQuantiles(); var secondRun = GetRandomQuantiles(); Assert.Equal(firstRun, secondRun); }