public void ReverseRemoveItemTest() { var addSize = 10000; var testData = DataGenerator.Generate().Take(addSize).ToArray(); var errorRate = 0.001F; var size = testData.Length; var configuration = new KeyValueBloomFilterConfiguration(); var bloomFilter = new InvertibleReverseBloomFilter <TestEntity, long, sbyte>(configuration); bloomFilter.Initialize(2 * size, errorRate); foreach (var itm in testData) { bloomFilter.Add(itm); } var contained = testData.Count(item => bloomFilter.Contains(item)); foreach (var item in testData.Take(addSize / 2)) { bloomFilter.Remove(item); } var containedAfterRemove = testData.Count(item => bloomFilter.Contains(item)); //tricky: assuming zero false positives. Assert.AreEqual(contained, containedAfterRemove * 2, "Wrong item count after removal."); }
public void ReverseRemoveKeyTest() { var addSize = 10000; var testData = DataGenerator.Generate().Take(addSize).ToArray(); var errorRate = 0.001F; var size = testData.Length; var configuration = new KeyValueBloomFilterConfiguration(); var bloomFilter = new InvertibleReverseBloomFilter <TestEntity, long, sbyte>(configuration); bloomFilter.Initialize(2 * size, errorRate); foreach (var itm in testData) { bloomFilter.Add(itm); } var contained = testData.Count(item => bloomFilter.Contains(item)); try { foreach (var item in testData.Take(addSize / 2)) { bloomFilter.RemoveKey(item.Id); } Assert.Fail("RemoveKey should not be supported by a reverse invertible Bloom filter"); } catch (NotSupportedException) { }; }
public void HybridIntersectDifferentFiltersTest() { var addSize = 10000; var testData = DataGenerator.Generate().Take(addSize).ToArray(); var errorRate = 0.001F; var size = testData.Length; var configuration = new KeyValueBloomFilterConfiguration(); var bloomFilter = new InvertibleReverseBloomFilter <TestEntity, long, sbyte>(configuration); bloomFilter.Initialize(2 * size, errorRate); foreach (var itm in testData) { bloomFilter.Add(itm); } var bloomFilter2 = new InvertibleReverseBloomFilter <TestEntity, long, sbyte>(configuration); bloomFilter2.Initialize(2 * size, errorRate); foreach (var itm in testData.Skip(1000)) { bloomFilter2.Add(itm); } bloomFilter.Intersect(bloomFilter2); Assert.AreEqual(9000, bloomFilter.ItemCount); var count = testData.Skip(1000).Count(bloomFilter.Contains); //Note: intersect introduces a horrible error rate when utilizing XOR, so don't actually use intersect. //There are however definitions of operations possible where the intersect would not have this horrible effect. Assert.IsTrue(count > 6700); Assert.IsTrue(testData.Take(1000).All(d => !bloomFilter.Contains(d))); }
public void BasicFillAndEstimate() { var data = DataGenerator.Generate().Take(100000).ToList(); var data2 = DataGenerator.Generate().Take(100000).ToList(); var differences = 10000; data.Modify(differences); var configuration = new KeyValueBloomFilterConfiguration(); //create the first estimator var estimator = new BitMinwiseHashEstimator <TestEntity, long, sbyte>( configuration, 2, 20, data.LongCount()); foreach (var element in data) { estimator.Add(element); } //create the second estimator var estimator2 = new BitMinwiseHashEstimator <TestEntity, long, sbyte>(configuration, 2, 20, data.LongCount()); foreach (var element in data2) { //just making sure we do not depend upon the order of adding things. estimator2.Add(element); } var totalCount = data.LongCount() + data2.LongCount(); //calculate the similarity between the two estimators. var differenceCount = totalCount - estimator.Similarity(estimator2) * totalCount; //within 95% or higher of difference count. Assert.IsTrue(differenceCount >= 0.95 * differences); }
public void ReverseIntersectEqualFiltersTest() { var addSize = 10000; var testData = DataGenerator.Generate().Take(addSize).ToArray(); var errorRate = 0.001F; var size = testData.Length; var configuration = new KeyValueBloomFilterConfiguration(); var bloomFilter = new InvertibleReverseBloomFilter <TestEntity, long, sbyte>(configuration); bloomFilter.Initialize(2 * size, errorRate); foreach (var itm in testData) { bloomFilter.Add(itm); } var bloomFilter2 = new InvertibleReverseBloomFilter <TestEntity, long, sbyte>(configuration); bloomFilter2.Initialize(2 * size, errorRate); foreach (var itm in testData) { bloomFilter2.Add(itm); } bloomFilter.Intersect(bloomFilter2); Assert.AreEqual(addSize, bloomFilter.ItemCount); Assert.IsTrue(testData.All(bloomFilter.Contains)); }
public void ReverseAddDifferentSizesTest() { var addSize = 10000; var testData = DataGenerator.Generate().Take(addSize).ToArray(); var testData2 = DataGenerator.Generate().Skip(addSize).Take(addSize).ToArray(); var errorRate = 0.001F; var size = testData.Length; var configuration = new KeyValueBloomFilterConfiguration(); var bloomFilter = new InvertibleReverseBloomFilter <TestEntity, long, sbyte>(configuration); bloomFilter.Initialize(4 * size, errorRate); foreach (var itm in testData) { bloomFilter.Add(itm); } var bloomFilter2 = new InvertibleReverseBloomFilter <TestEntity, long, sbyte>(configuration); //We have to create a foldable version. var data = bloomFilter.Extract(); var foldFactor = configuration.FoldingStrategy.GetAllFoldFactors(data.BlockSize).Where(f => f > 1).OrderBy(f => f).First(); bloomFilter2.Initialize(addSize, data.BlockSize / foldFactor, data.HashFunctionCount); foreach (var itm in testData2) { bloomFilter2.Add(itm); } bloomFilter.Add(bloomFilter2); var contained = testData.Union(testData2).Count(item => bloomFilter.Contains(item)); Assert.AreEqual(contained, 2 * addSize, "Not all items found in added Bloom filters"); }
public void ReverseAddTest() { var addSize = 10000; var testData = DataGenerator.Generate().Take(addSize).ToArray(); var testData2 = DataGenerator.Generate().Skip(addSize).Take(addSize).ToArray(); var errorRate = 0.001F; var size = testData.Length; var configuration = new KeyValueBloomFilterConfiguration(); var bloomFilter = new InvertibleReverseBloomFilter <TestEntity, long, sbyte>(configuration); bloomFilter.Initialize(2 * size, errorRate); foreach (var itm in testData) { bloomFilter.Add(itm); } var bloomFilter2 = new InvertibleReverseBloomFilter <TestEntity, long, sbyte>(configuration); bloomFilter2.Initialize(2 * size, errorRate); foreach (var itm in testData2) { bloomFilter2.Add(itm); } bloomFilter.Add(bloomFilter2); var contained = testData.Union(testData2).Count(item => bloomFilter.Contains(item)); Assert.AreEqual(contained, 2 * addSize, "Not all items found in added Bloom filters"); }
public void ReverseFalsePositiveTest() { var addSize = 10000; var testData = DataGenerator.Generate().Take(addSize).ToArray(); var errorRate = 0.001F; var size = testData.Length; var configuration = new KeyValueBloomFilterConfiguration(); var bloomFilter = new InvertibleReverseBloomFilter <TestEntity, long, sbyte>(configuration); bloomFilter.Initialize(size, errorRate); foreach (var itm in testData) { bloomFilter.Add(itm); } var notFoundCount = testData.Count(itm => !bloomFilter.Contains(itm)); Assert.IsTrue(notFoundCount == 0, "False negative error rate violated"); try { notFoundCount = testData.Count(itm => !bloomFilter.ContainsKey(itm.Id)); Assert.Fail("Invertible reverse Bloom filter does not support ContainsKey."); } catch (NotSupportedException) { }; notFoundCount = DataGenerator.Generate().Skip(addSize).Take(addSize).Count(itm => bloomFilter.Contains(itm)); Assert.IsTrue(notFoundCount <= 20 * errorRate * addSize, "False positive error rate violated"); try { notFoundCount = DataGenerator.Generate().Skip(addSize).Take(addSize).Count(itm => bloomFilter.ContainsKey(itm.Id)); Assert.Fail("Invertible reverse Bloom filter does not support ContainsKey."); } catch (NotSupportedException) { }; }
public void BitMinwiseEstimatorCompressTest() { var data = DataGenerator.Generate().Take(100).ToArray(); var configuration = new KeyValueBloomFilterConfiguration(); var estimator = new BitMinwiseHashEstimator <TestEntity, long, sbyte>(configuration, 2, 20, 1000 * data.LongCount()); foreach (var element in data) { estimator.Add(element); } Assert.AreEqual(estimator.FullExtract().Capacity, 100170, "Unexpected capacity before compression."); var emptyEstimator = new BitMinwiseHashEstimator <TestEntity, long, sbyte>(configuration, 2, 20, 1000 * data.LongCount()); Assert.IsTrue(estimator.Similarity(emptyEstimator) > 0.999D, "Unexpected similartiy before compression."); estimator.Compress(true); Assert.AreEqual(estimator.FullExtract().Capacity, 105, "Unexpected capacity after compression."); Assert.IsTrue(estimator.Similarity(emptyEstimator) > 0.999D, "Unexpected similartiy after compression."); }
public void ReverseSimpleFold() { var addSize = 50; var testData = DataGenerator.Generate().Take(addSize).ToArray(); var size = testData.Length; var configuration = new KeyValueBloomFilterConfiguration(); var bloomFilter = new InvertibleReverseBloomFilter <TestEntity, long, sbyte>(configuration); bloomFilter.Initialize(size, 1024, (uint)3); foreach (var itm in testData) { bloomFilter.Add(itm); } var positiveCount = DataGenerator.Generate().Take(500).Count(itm => bloomFilter.Contains(itm)); var folded = bloomFilter.Fold(4); var positiveCountAfterFold = DataGenerator.Generate().Take(500).Count(itm => bloomFilter.Contains(itm)); Assert.AreEqual(positiveCount, positiveCountAfterFold, "False positive count different after fold"); Assert.AreEqual(256, folded.Extract().BlockSize); Assert.IsTrue(testData.All(item => bloomFilter.Contains(item)), "False negative found"); }
public void ReverseInvertibleBloomFilterEmptySetDiffTest() { var addSize = 1000; var modCount = addSize; var dataSet1 = DataGenerator.Generate().Take(0).ToList(); var dataSet2 = DataGenerator.Generate().Take(addSize).ToList(); dataSet2.Modify(modCount); var configuration = new KeyValueBloomFilterConfiguration(); var bloomFilter = new InvertibleReverseBloomFilter <TestEntity, long, sbyte>(configuration); bloomFilter.Initialize(10 * modCount, 0.0001F); foreach (var itm in dataSet1) { bloomFilter.Add(itm); } var secondBloomFilter = new InvertibleReverseBloomFilter <TestEntity, long, sbyte>(configuration); secondBloomFilter.Initialize(10 * modCount, 0.0001F); foreach (var itm in dataSet2) { secondBloomFilter.Add(itm); } var changed = new HashSet <long>(); var onlyInFirst = new HashSet <long>(); var onlyInSecond = new HashSet <long>(); var decoded = bloomFilter .SubtractAndDecode(secondBloomFilter, onlyInFirst, onlyInSecond, changed); var onlyInSet1 = dataSet1.Where(d => dataSet2.All(d2 => d2.Id != d.Id)).Select(d => d.Id).OrderBy(id => id).ToArray(); var onlyInSet2 = dataSet2.Where(d => dataSet1.All(d1 => d1.Id != d.Id)).Select(d => d.Id).OrderBy(id => id).ToArray(); var modified = dataSet1.Where(d => dataSet2.Any(d2 => d2.Id == d.Id && d2.Value != d.Value)).Select(d => d.Id).OrderBy(id => id).ToArray(); //fairly sensitive to decoding errors (due to the same reason as Contains is rather unreliable: the pure function does not check the id value and hash value) Assert.IsTrue(decoded.HasValue, "Decoding failed"); Assert.IsTrue(onlyInSet1.Length == onlyInFirst.Count, "Incorrect number of changes detected on 'only in set 1'"); Assert.IsTrue(onlyInSet2.Length == onlyInSecond.Count, "Incorrect number of changes detected on 'only in set 2'"); Assert.IsTrue(changed.Count == modified.Length, "Incorrect number of modified items detected"); }