public void HybridRemoveItemTest()
{
var addSize = 10000;
var testData = DataGenerator.Generate().Take(addSize).ToArray();
var errorRate = 0.001F;
var size = testData.Length;
var configuration = new HybridDefaultBloomFilterConfiguration();
var bloomFilter = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);
bloomFilter.Initialize(2 * size, errorRate);
foreach (var itm in testData)
{
bloomFilter.Add(itm);
}
var contained = testData.Count(item => bloomFilter.Contains(item));
foreach (var item in testData.Take(addSize / 2))
{
bloomFilter.Remove(item);
}
var containedAfterRemove = testData.Count(item => bloomFilter.Contains(item));
//tricky: assuming zero false positives.
Assert.AreEqual(contained, containedAfterRemove * 2, "Wrong item count after removal.");
}
public void HybridIntersectDifferentFiltersTest()
{
var addSize = 10000;
var testData = DataGenerator.Generate().Take(addSize).ToArray();
var errorRate = 0.001F;
var size = testData.Length;
var configuration = new HybridDefaultBloomFilterConfiguration();
var bloomFilter = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);
bloomFilter.Initialize(2 * size, errorRate);
foreach (var itm in testData)
{
bloomFilter.Add(itm);
}
var bloomFilter2 = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);
bloomFilter2.Initialize(2 * size, errorRate);
foreach (var itm in testData.Skip(1000))
{
bloomFilter2.Add(itm);
}
bloomFilter.Intersect(bloomFilter2);
Assert.AreEqual(9000, bloomFilter.ItemCount);
var count = testData.Skip(1000).Count(bloomFilter.Contains);
//Note: intersect introduces a horrible error rate when utilizing XOR, so don't actually use intersect.
//There are however definitions of operations possible where the intersect would not have this horrible effect.
Assert.IsTrue(count > 6000);
Assert.IsTrue(testData.Take(1000).All(d => !bloomFilter.Contains(d)));
}
public void HybridAddDifferentSizesTest()
{
var addSize = 10000;
var testData = DataGenerator.Generate().Take(addSize).ToArray();
var testData2 = DataGenerator.Generate().Skip(addSize).Take(addSize).ToArray();
var errorRate = 0.001F;
var size = testData.Length;
var configuration = new HybridDefaultBloomFilterConfiguration();
var bloomFilter = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);
bloomFilter.Initialize(4 * size, errorRate);
foreach (var itm in testData)
{
bloomFilter.Add(itm);
}
var bloomFilter2 = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);
//We should create a foldable counter part.
var data = bloomFilter.Extract();
var foldFactor = configuration.FoldingStrategy.GetAllFoldFactors(data.BlockSize).Where(f => f > 1).OrderBy(f => f).First();
bloomFilter2.Initialize(addSize, data.BlockSize / foldFactor, data.HashFunctionCount);
foreach (var itm in testData2)
{
bloomFilter2.Add(itm);
}
bloomFilter.Add(bloomFilter2);
var contained = testData.Union(testData2).Count(item => bloomFilter.Contains(item));
Assert.AreEqual(contained, 2 * addSize, "Not all items found in added Bloom filters");
}
public void HybridAddTest()
{
var addSize = 10000;
var testData = DataGenerator.Generate().Take(addSize).ToArray();
var testData2 = DataGenerator.Generate().Skip(addSize).Take(addSize).ToArray();
var errorRate = 0.001F;
var size = testData.Length;
var configuration = new HybridDefaultBloomFilterConfiguration();
var bloomFilter = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);
bloomFilter.Initialize(2 * size, errorRate);
foreach (var itm in testData)
{
bloomFilter.Add(itm);
}
var bloomFilter2 = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);
bloomFilter2.Initialize(2 * size, errorRate);
foreach (var itm in testData2)
{
bloomFilter2.Add(itm);
}
bloomFilter.Add(bloomFilter2);
var contained = testData.Union(testData2).Count(item => bloomFilter.Contains(item));
Assert.AreEqual(contained, 2 * addSize, "Not all items found in added Bloom filters");
}
public void HybridRemoveKeyTest()
{
var addSize = 10000;
var testData = DataGenerator.Generate().Take(addSize).ToArray();
var errorRate = 0.001F;
var size = testData.Length;
var configuration = new HybridDefaultBloomFilterConfiguration();
var bloomFilter = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);
bloomFilter.Initialize(2 * size, errorRate);
foreach (var itm in testData)
{
bloomFilter.Add(itm);
}
var contained = testData.Count(item => bloomFilter.Contains(item));
try
{
foreach (var item in testData.Take(addSize / 2))
{
bloomFilter.RemoveKey(item.Id);
}
Assert.Fail("RemoveKey should not be supported by a hybrid invertible Bloom filter");
}
catch (NotSupportedException)
{ };
}
public void HybridSimpleFold()
{
var addSize = 50;
var testData = DataGenerator.Generate().Take(addSize).ToArray();
var size = testData.Length;
var configuration = new HybridDefaultBloomFilterConfiguration();
var bloomFilter = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);
bloomFilter.Initialize(size, 1024, (uint)3);
foreach (var itm in testData)
{
bloomFilter.Add(itm);
}
var positiveCount = DataGenerator.Generate().Take(500).Count(itm => bloomFilter.Contains(itm));
var folded = bloomFilter.Fold(4);
var positiveCountAfterFold = DataGenerator.Generate().Take(500).Count(itm => bloomFilter.Contains(itm));
Assert.AreEqual(positiveCount, positiveCountAfterFold, "False positive count different after fold");
Assert.AreEqual(256, folded.BlockSize, "Folded block size is unexpected.");
Assert.IsTrue(testData.All(item => bloomFilter.Contains(item)), "False negative found");
}
public void HybridFalsePositiveTest()
{
var addSize = 10000;
var testData = DataGenerator.Generate().Take(addSize).ToArray();
var errorRate = 0.001F;
var size = testData.Length;
var configuration = new HybridDefaultBloomFilterConfiguration();
var bloomFilter = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);
bloomFilter.Initialize(size, errorRate);
foreach (var itm in testData)
{
bloomFilter.Add(itm);
}
var notFoundCount = testData.Count(itm => !bloomFilter.Contains(itm));
Assert.IsTrue(notFoundCount == 0, "False negative error rate violated");
notFoundCount = testData.Count(itm => !bloomFilter.ContainsKey(itm.Id));
Assert.IsTrue(notFoundCount == 0, "False negative error rate violated on ContainsKey");
notFoundCount = DataGenerator.Generate().Skip(addSize).Take(addSize).Count(itm => bloomFilter.Contains(itm));
Assert.IsTrue(notFoundCount <= errorRate * addSize, "False positive error rate violated");
notFoundCount = DataGenerator.Generate().Skip(addSize).Take(addSize).Count(itm => bloomFilter.ContainsKey(itm.Id));
Assert.IsTrue(notFoundCount <= errorRate * addSize, "False positive error rate violated on ContainsKey");
}
public void HybridCompressTest()
{
var addSize = 10000;
var errorRate = 0.001F;
var data = DataGenerator.Generate().Take(addSize).ToArray();
var hybridFilter = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(new HybridDefaultBloomFilterConfiguration());
hybridFilter.Initialize(50 * data.Length, errorRate);
Assert.AreEqual(hybridFilter.Capacity, 500000, "Unexpected size of hybrid Bloom filter.");
foreach (var item in data)
{
hybridFilter.Add(item);
}
//check error rate.
var notFoundCount = DataGenerator.Generate().Skip(addSize).Take(10000).Count(itm => hybridFilter.Contains(itm));
Assert.IsTrue(notFoundCount <= errorRate * addSize, "Uncompressed hybrid Bloom filter exceeded error rate.");
hybridFilter.Compress(true);
Assert.AreEqual(hybridFilter.Capacity, 12820, "Unexpected size of compressed hybrid Bloom filter.");
var compressNotFoundCount = DataGenerator.Generate().Skip(addSize).Take(10000).Count(itm => hybridFilter.Contains(itm));
Assert.IsTrue(compressNotFoundCount <= errorRate * addSize, "Compressed hybrid Bloom filter exceeded error rate.");
}
