예제 #1
0
        public void HybridIntersectEqualFiltersTest()
        {
            var addSize       = 10000;
            var testData      = DataGenerator.Generate().Take(addSize).ToArray();
            var errorRate     = 0.001F;
            var size          = testData.Length;
            var configuration = new HybridDefaultBloomFilterConfiguration();
            var bloomFilter   = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);

            bloomFilter.Initialize(2 * size, errorRate);
            foreach (var itm in testData)
            {
                bloomFilter.Add(itm);
            }
            var bloomFilter2 = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);

            bloomFilter2.Initialize(2 * size, errorRate);
            foreach (var itm in testData)
            {
                bloomFilter2.Add(itm);
            }
            bloomFilter.Intersect(bloomFilter2);
            Assert.AreEqual(addSize, bloomFilter.ItemCount);
            Assert.IsTrue(testData.All(bloomFilter.Contains));
        }
예제 #2
0
        public void HybridInvertibleBloomFilterEmptySetDiffTest()
        {
            var addSize       = 1000;
            var dataSet1      = DataGenerator.Generate().Take(0).ToList();
            var dataSet2      = DataGenerator.Generate().Take(addSize).ToList();
            var configuration = new HybridDefaultBloomFilterConfiguration();
            var bloomFilter   = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);

            bloomFilter.Initialize(10 * addSize, 0.0001F);
            foreach (var itm in dataSet1)
            {
                bloomFilter.Add(itm);
            }
            var secondBloomFilter = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);

            secondBloomFilter.Initialize(10 * addSize, 0.0001F);
            foreach (var itm in dataSet2)
            {
                secondBloomFilter.Add(itm);
            }
            var changed      = new HashSet <long>();
            var onlyInFirst  = new HashSet <long>();
            var onlyInSecond = new HashSet <long>();
            var decoded      = bloomFilter
                               .SubtractAndDecode(secondBloomFilter, onlyInFirst, onlyInSecond, changed);
            var onlyInSet1 = dataSet1.Where(d => dataSet2.All(d2 => d2.Id != d.Id)).Select(d => d.Id).OrderBy(id => id).ToArray();
            var onlyInSet2 = dataSet2.Where(d => dataSet1.All(d1 => d1.Id != d.Id)).Select(d => d.Id).OrderBy(id => id).ToArray();
            var modified   = dataSet1.Where(d => dataSet2.Any(d2 => d2.Id == d.Id && d2.Value != d.Value)).Select(d => d.Id).OrderBy(id => id).ToArray();

            Assert.IsTrue(decoded.HasValue, "Decoding failed");
            Assert.IsTrue(onlyInSet1.Length == onlyInFirst.Count, "Incorrect number of changes detected on 'only in set 1'");
            Assert.IsTrue(onlyInSet2.Length == onlyInSecond.Count, "Incorrect number of changes detected on 'only in set 2'");
            Assert.IsTrue(changed.Count == modified.Length, "Incorrect number of modified items detected");
        }
예제 #3
0
        public void HybridIntersectDifferentFiltersTest()
        {
            var addSize       = 10000;
            var testData      = DataGenerator.Generate().Take(addSize).ToArray();
            var errorRate     = 0.001F;
            var size          = testData.Length;
            var configuration = new HybridDefaultBloomFilterConfiguration();
            var bloomFilter   = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);

            bloomFilter.Initialize(2 * size, errorRate);
            foreach (var itm in testData)
            {
                bloomFilter.Add(itm);
            }
            var bloomFilter2 = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);

            bloomFilter2.Initialize(2 * size, errorRate);
            foreach (var itm in testData.Skip(1000))
            {
                bloomFilter2.Add(itm);
            }
            bloomFilter.Intersect(bloomFilter2);
            Assert.AreEqual(9000, bloomFilter.ItemCount);
            var count = testData.Skip(1000).Count(bloomFilter.Contains);

            //Note: intersect introduces a horrible error rate when utilizing XOR, so don't actually use intersect.
            //There are however definitions of operations possible where the intersect would not have this horrible effect.
            Assert.IsTrue(count > 6000);
            Assert.IsTrue(testData.Take(1000).All(d => !bloomFilter.Contains(d)));
        }
예제 #4
0
        public void HybridAddDifferentSizesTest()
        {
            var addSize       = 10000;
            var testData      = DataGenerator.Generate().Take(addSize).ToArray();
            var testData2     = DataGenerator.Generate().Skip(addSize).Take(addSize).ToArray();
            var errorRate     = 0.001F;
            var size          = testData.Length;
            var configuration = new HybridDefaultBloomFilterConfiguration();
            var bloomFilter   = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);

            bloomFilter.Initialize(4 * size, errorRate);
            foreach (var itm in testData)
            {
                bloomFilter.Add(itm);
            }
            var bloomFilter2 = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);
            //We should create a foldable counter part.
            var data       = bloomFilter.Extract();
            var foldFactor = configuration.FoldingStrategy.GetAllFoldFactors(data.BlockSize).Where(f => f > 1).OrderBy(f => f).First();

            bloomFilter2.Initialize(addSize, data.BlockSize / foldFactor, data.HashFunctionCount);
            foreach (var itm in testData2)
            {
                bloomFilter2.Add(itm);
            }
            bloomFilter.Add(bloomFilter2);
            var contained = testData.Union(testData2).Count(item => bloomFilter.Contains(item));

            Assert.AreEqual(contained, 2 * addSize, "Not all items found in added Bloom filters");
        }
예제 #5
0
        public void HybridAddTest()
        {
            var addSize       = 10000;
            var testData      = DataGenerator.Generate().Take(addSize).ToArray();
            var testData2     = DataGenerator.Generate().Skip(addSize).Take(addSize).ToArray();
            var errorRate     = 0.001F;
            var size          = testData.Length;
            var configuration = new HybridDefaultBloomFilterConfiguration();
            var bloomFilter   = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);

            bloomFilter.Initialize(2 * size, errorRate);
            foreach (var itm in testData)
            {
                bloomFilter.Add(itm);
            }
            var bloomFilter2 = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);

            bloomFilter2.Initialize(2 * size, errorRate);
            foreach (var itm in testData2)
            {
                bloomFilter2.Add(itm);
            }
            bloomFilter.Add(bloomFilter2);
            var contained = testData.Union(testData2).Count(item => bloomFilter.Contains(item));

            Assert.AreEqual(contained, 2 * addSize, "Not all items found in added Bloom filters");
        }
예제 #6
0
        public void HybridRemoveKeyTest()
        {
            var addSize       = 10000;
            var testData      = DataGenerator.Generate().Take(addSize).ToArray();
            var errorRate     = 0.001F;
            var size          = testData.Length;
            var configuration = new HybridDefaultBloomFilterConfiguration();
            var bloomFilter   = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);

            bloomFilter.Initialize(2 * size, errorRate);
            foreach (var itm in testData)
            {
                bloomFilter.Add(itm);
            }
            var contained = testData.Count(item => bloomFilter.Contains(item));

            try
            {
                foreach (var item in testData.Take(addSize / 2))
                {
                    bloomFilter.RemoveKey(item.Id);
                }
                Assert.Fail("RemoveKey should not be supported by a hybrid invertible Bloom filter");
            }
            catch (NotSupportedException)
            { };
        }
예제 #7
0
        public void HybridRemoveItemTest()
        {
            var addSize       = 10000;
            var testData      = DataGenerator.Generate().Take(addSize).ToArray();
            var errorRate     = 0.001F;
            var size          = testData.Length;
            var configuration = new HybridDefaultBloomFilterConfiguration();
            var bloomFilter   = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);

            bloomFilter.Initialize(2 * size, errorRate);
            foreach (var itm in testData)
            {
                bloomFilter.Add(itm);
            }
            var contained = testData.Count(item => bloomFilter.Contains(item));

            foreach (var item in testData.Take(addSize / 2))
            {
                bloomFilter.Remove(item);
            }
            var containedAfterRemove = testData.Count(item => bloomFilter.Contains(item));

            //tricky: assuming zero false positives.
            Assert.AreEqual(contained, containedAfterRemove * 2, "Wrong item count after removal.");
        }
예제 #8
0
        public void HybridSimpleFold()
        {
            var addSize       = 50;
            var testData      = DataGenerator.Generate().Take(addSize).ToArray();
            var size          = testData.Length;
            var configuration = new HybridDefaultBloomFilterConfiguration();
            var bloomFilter   = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);

            bloomFilter.Initialize(size, 1024, (uint)3);
            foreach (var itm in testData)
            {
                bloomFilter.Add(itm);
            }
            var positiveCount          = DataGenerator.Generate().Take(500).Count(itm => bloomFilter.Contains(itm));
            var folded                 = bloomFilter.Fold(4);
            var positiveCountAfterFold = DataGenerator.Generate().Take(500).Count(itm => bloomFilter.Contains(itm));

            Assert.AreEqual(positiveCount, positiveCountAfterFold, "False positive count different after fold");
            Assert.AreEqual(256, folded.BlockSize, "Folded block size is unexpected.");
            Assert.IsTrue(testData.All(item => bloomFilter.Contains(item)), "False negative found");
        }
예제 #9
0
        public void HybridCompressTest()
        {
            var addSize      = 10000;
            var errorRate    = 0.001F;
            var data         = DataGenerator.Generate().Take(addSize).ToArray();
            var hybridFilter = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(new HybridDefaultBloomFilterConfiguration());

            hybridFilter.Initialize(50 * data.Length, errorRate);
            Assert.AreEqual(hybridFilter.Capacity, 500000, "Unexpected size of hybrid Bloom filter.");
            foreach (var item in data)
            {
                hybridFilter.Add(item);
            }
            //check error rate.
            var notFoundCount = DataGenerator.Generate().Skip(addSize).Take(10000).Count(itm => hybridFilter.Contains(itm));

            Assert.IsTrue(notFoundCount <= errorRate * addSize, "Uncompressed hybrid Bloom filter exceeded error rate.");
            hybridFilter.Compress(true);
            Assert.AreEqual(hybridFilter.Capacity, 12820, "Unexpected size of compressed hybrid Bloom filter.");
            var compressNotFoundCount = DataGenerator.Generate().Skip(addSize).Take(10000).Count(itm => hybridFilter.Contains(itm));

            Assert.IsTrue(compressNotFoundCount <= errorRate * addSize, "Compressed hybrid Bloom filter exceeded error rate.");
        }
예제 #10
0
        public void HybridFalsePositiveTest()
        {
            var addSize       = 10000;
            var testData      = DataGenerator.Generate().Take(addSize).ToArray();
            var errorRate     = 0.001F;
            var size          = testData.Length;
            var configuration = new HybridDefaultBloomFilterConfiguration();
            var bloomFilter   = new InvertibleHybridBloomFilter <TestEntity, long, sbyte>(configuration);

            bloomFilter.Initialize(size, errorRate);
            foreach (var itm in testData)
            {
                bloomFilter.Add(itm);
            }
            var notFoundCount = testData.Count(itm => !bloomFilter.Contains(itm));

            Assert.IsTrue(notFoundCount == 0, "False negative error rate violated");
            notFoundCount = testData.Count(itm => !bloomFilter.ContainsKey(itm.Id));
            Assert.IsTrue(notFoundCount == 0, "False negative error rate violated on ContainsKey");
            notFoundCount = DataGenerator.Generate().Skip(addSize).Take(addSize).Count(itm => bloomFilter.Contains(itm));
            Assert.IsTrue(notFoundCount <= errorRate * addSize, "False positive error rate violated");
            notFoundCount = DataGenerator.Generate().Skip(addSize).Take(addSize).Count(itm => bloomFilter.ContainsKey(itm.Id));
            Assert.IsTrue(notFoundCount <= errorRate * addSize, "False positive error rate violated on ContainsKey");
        }
예제 #11
0
        public void HybridIbfDecodePerformance()
        {
            var configuration = new KeyValueLargeBloomFilterConfiguration();

            var size          = new[] { 1000, 10000, 100000 };
            var modPercentage = new[] { 0, 0.01D, 0.1D, 0.2D, 0.5D, 1.0D };

            foreach (var s in size)
            {
                using (
                    var writer =
                        new StreamWriter(File.Open($"hibfdecode-{s}.csv",
                                                   FileMode.Create)))
                {
                    writer.WriteLine("timeInMs,sizeInBytes,capacity,modCount,detectedModCount,countDiff,countDiffSd,decodeSuccessRate,listADiff,listBDiff,listCDiff");

                    foreach (var mod in modPercentage)
                    {
                        foreach (var capacityPercentage in new[] { 0.5, 1, 2, 5, 10, 100 })
                        {
                            var sizeInBytes             = new long[100];
                            var timeSpan                = new long[50];
                            var countAggregate          = new int[50];
                            var modCountResultAggregate = new int[50];
                            var listADiff               = new int[50];
                            var listBDiff               = new int[50];
                            var listCDiff               = new int[50];
                            var decodeResult            = new int[50];
                            for (var run = 0; run < 50; run++)
                            {
                                var dataSet1 = DataGenerator.Generate().Take(s).ToList();
                                var dataSet2 = DataGenerator.Generate().Take(s).ToList();
                                dataSet2.Modify((int)(s * mod));
                                var onlyInSet1    = dataSet1.Where(d => dataSet2.All(d2 => d2.Id != d.Id)).Select(d => d.Id).OrderBy(id => id).ToArray();
                                var onlyInSet2    = dataSet2.Where(d => dataSet1.All(d1 => d1.Id != d.Id)).Select(d => d.Id).OrderBy(id => id).ToArray();
                                var modified      = dataSet1.Where(d => dataSet2.Any(d2 => d2.Id == d.Id && d2.Value != d.Value)).Select(d => d.Id).OrderBy(id => id).ToArray();
                                var idealCapacity = Math.Max(15, onlyInSet1.Count() + onlyInSet2.Count() + modified.Count());
                                var stopWatch     = new Stopwatch();
                                stopWatch.Start();
                                var bloomFilter1 = new InvertibleHybridBloomFilter <TestEntity, long, int>(configuration);
                                bloomFilter1.Initialize((int)(idealCapacity * capacityPercentage), 0.01F);

                                foreach (var item in dataSet1)
                                {
                                    bloomFilter1.Add(item);
                                }
                                var bloomFilter2 = new InvertibleHybridBloomFilter <TestEntity, long, int>(configuration);
                                bloomFilter2.Initialize((int)(idealCapacity * capacityPercentage), 0.01F);
                                foreach (var item in dataSet2)
                                {
                                    bloomFilter2.Add(item);
                                }
                                var s1      = new HashSet <long>();
                                var s2      = new HashSet <long>();
                                var s3      = new HashSet <long>();
                                var success = bloomFilter1.SubtractAndDecode(bloomFilter2, s1, s2, s3);
                                stopWatch.Stop();
                                using (var stream = new MemoryStream())
                                {
                                    _protobufTypeModel.Serialize(stream, bloomFilter1.Extract());
                                    stream.Position  = 0;
                                    sizeInBytes[run] = stream.Length;
                                }
                                using (var stream = new MemoryStream())
                                {
                                    _protobufTypeModel.Serialize(stream, bloomFilter2.Extract());
                                    stream.Position       = 0;
                                    sizeInBytes[50 + run] = stream.Length;
                                }
                                timeSpan[run]                = stopWatch.ElapsedMilliseconds;
                                countAggregate[run]          = onlyInSet1.Count() + onlyInSet2.Count() + modified.Count();
                                modCountResultAggregate[run] = s1.Union(s2).Union(s3).Count(v => onlyInSet1.Contains(v) ||
                                                                                            onlyInSet2.Contains(v) || modified.Contains(v));
                                listADiff[run]    = s1.Count() - onlyInSet1.Count();
                                listBDiff[run]    = s2.Count() - onlyInSet2.Count();
                                listCDiff[run]    = s3.Count() - modified.Count();
                                decodeResult[run] = success == true? 1 : 0;
                            }
                            var countAvg         = (long)countAggregate.Average();
                            var modCountResult   = (long)modCountResultAggregate.Average();
                            var differenceResult =
                                modCountResultAggregate.Select((r, i) => r - countAggregate[i]).ToArray();
                            var differenceSd = Math.Sqrt(differenceResult.Variance());
                            writer
                            .WriteLine($"{timeSpan.Average()},{sizeInBytes.Average()},{Math.Max(15, capacityPercentage * (int)(s * mod))},{countAvg},{modCountResult},{(long)differenceResult.Average()},{differenceSd},{1.0D * decodeResult.Sum() / 50},{listADiff.Average()},{listBDiff.Average()},{listCDiff.Average()}");
                        }
                    }
                }
            }
        }