public void DirectCountingIsResetWhenMergingAlmostFullEstimators()
{
var addedEstimator = new CardinalityEstimator();
var mergedEstimator = new CardinalityEstimator();
for (int i = 0; i < 10_000; i++)
{
var guid = Guid.NewGuid().ToString();
addedEstimator.Add(guid);
// Simulate some intermediate estimators being merged together
var temporaryEstimator = new CardinalityEstimator();
temporaryEstimator.Add(guid);
mergedEstimator.Merge(temporaryEstimator);
}
var serializer = new CardinalityEstimatorSerializer();
var stream1 = new MemoryStream();
serializer.Serialize(stream1, addedEstimator, true);
var stream2 = new MemoryStream();
serializer.Serialize(stream2, mergedEstimator, true);
Assert.Equal(stream1.Length, stream2.Length);
}
public void StaticMergeHandlesNullElements()
{
const int expectedBitsPerIndex = 11;
var estimators = new List <CardinalityEstimator> {
null, new CardinalityEstimator(expectedBitsPerIndex, HashFunctionId.Fnv1A), null
};
CardinalityEstimator result = CardinalityEstimator.Merge(estimators);
Assert.NotNull(result);
Assert.Equal(expectedBitsPerIndex, result.GetState().BitsPerIndex);
}
/// <summary>
/// Generates <paramref name="expectedCount" /> random (or sequential) elements and adds them to CardinalityEstimators, then asserts that
/// the observed error rate is no more than <paramref name="maxAcceptedError" />
/// </summary>
/// <param name="stdError">Expected standard error of the estimators (upper bound)</param>
/// <param name="expectedCount">number of elements to generate in total</param>
/// <param name="maxAcceptedError">Maximum allowed error rate. Default is 4 times <paramref name="stdError" /></param>
/// <param name="numHllInstances">Number of estimators to create. Generated elements will be assigned to one of the estimators at random</param>
/// <param name="sequential">When false, elements will be generated at random. When true, elements will be 0,1,2...</param>
private void RunTest(double stdError, long expectedCount, double?maxAcceptedError = null, int numHllInstances = 1,
bool sequential = false)
{
maxAcceptedError = maxAcceptedError ?? 4 * stdError; // should fail once in A LOT of runs
int b = GetAccuracyInBits(stdError);
var runStopwatch = new Stopwatch();
long gcMemoryAtStart = GetGcMemory();
// init HLLs
var hlls = new CardinalityEstimator[numHllInstances];
for (var i = 0; i < numHllInstances; i++)
{
hlls[i] = new CardinalityEstimator(b);
}
var nextMember = new byte[ElementSizeInBytes];
runStopwatch.Start();
for (long i = 0; i < expectedCount; i++)
{
// pick random hll, add member
int chosenHll = Rand.Next(numHllInstances);
if (sequential)
{
hlls[chosenHll].Add(i);
}
else
{
Rand.NextBytes(nextMember);
hlls[chosenHll].Add(nextMember);
}
}
runStopwatch.Stop();
ReportMemoryCost(gcMemoryAtStart); // done here so references can't be GC'ed yet
// Merge
CardinalityEstimator mergedHll = CardinalityEstimator.Merge(hlls);
Console.WriteLine("Run time: {0}", runStopwatch.Elapsed);
Console.WriteLine("Expected {0}, got {1}", expectedCount, mergedHll.Count());
double obsError = Math.Abs(mergedHll.Count() / (double)(expectedCount) - 1.0);
Console.WriteLine("StdErr: {0}. Observed error: {1}", stdError, obsError);
Assert.True(obsError <= maxAcceptedError, string.Format("Observed error was over {0}", maxAcceptedError));
Console.WriteLine();
}
public void StaticMergeTest()
{
const int expectedBitsPerIndex = 11;
var estimators = new CardinalityEstimator[10];
for (var i = 0; i < estimators.Length; i++)
{
estimators[i] = new CardinalityEstimator(expectedBitsPerIndex);
estimators[i].Add(Rand.Next());
}
CardinalityEstimator merged = CardinalityEstimator.Merge(estimators);
Assert.Equal(10UL, merged.Count());
Assert.Equal(expectedBitsPerIndex, merged.GetState().BitsPerIndex);
}
public void TestSize()
{
var estimator = new CardinalityEstimator();
Assert.AreEqual(0UL, estimator.CountElementsAdded);
estimator.Add(0);
estimator.Add(0);
Assert.AreEqual(2UL, estimator.CountElementsAdded);
var estimator2 = new CardinalityEstimator();
estimator2.Add(0);
estimator.Merge(estimator2);
Assert.AreEqual(3UL, estimator.CountElementsAdded);
}
public void TestCountAdditions()
{
var estimator = new CardinalityEstimator();
Assert.Equal(0UL, estimator.CountAdditions);
estimator.Add(0);
estimator.Add(0);
Assert.Equal(2UL, estimator.CountAdditions);
var estimator2 = new CardinalityEstimator();
estimator2.Add(0);
estimator.Merge(estimator2);
Assert.Equal(3UL, estimator.CountAdditions);
}
public void StaticMergeHandlesNullParameter()
{
CardinalityEstimator result = CardinalityEstimator.Merge(null as IEnumerable <CardinalityEstimator>);
Assert.Null(result);
}
