public void CanProperlyMatchComplex() { var bloomFilterPolicy = new BloomFilterPolicy(caseInsensitive: false); var builder = bloomFilterPolicy.CreateBuilder(); var filterBlockBuilder = new FilterBlockBuilder(new MemoryStream(), builder); filterBlockBuilder.StartBlock(0); filterBlockBuilder.Add(new InternalKey("test/1", 0, ItemType.Value).TheInternalKey); filterBlockBuilder.Add(new InternalKey("test/2", 0, ItemType.Value).TheInternalKey); filterBlockBuilder.Add(new InternalKey("test/3", 0, ItemType.Value).TheInternalKey); filterBlockBuilder.Add(new InternalKey("test/142", 0, ItemType.Value).TheInternalKey); filterBlockBuilder.Add(new InternalKey("test/3432", 0, ItemType.Value).TheInternalKey); var memoryStream = new MemoryStream(); filterBlockBuilder.Finish(memoryStream); var filter = bloomFilterPolicy.CreateFilter(ToMemoryMappedViewAccessor(memoryStream)); Assert.True(filter.KeyMayMatch(0, "test/1")); Assert.True(filter.KeyMayMatch(0, "test/142")); Assert.True(filter.KeyMayMatch(0, "test/3")); Assert.True(filter.KeyMayMatch(0, "test/3432")); Assert.False(filter.KeyMayMatch(0, "tester")); Assert.False(filter.KeyMayMatch(0, "test/5")); Assert.False(filter.KeyMayMatch(0, "test/133")); Assert.False(filter.KeyMayMatch(0, "test/133")); Assert.True(filter.KeyMayMatch(0, "test/144")); // even though this is wrong, false positives are okay with bloom filters }
public StreamOptions() { MaxInMemoryBuffer = 1024 * 1024 * 256; InitialInMemoryBuffer = 1024 * 1024 * 4; Comparator = new ByteWiseComparator(); FilterPolicy = new BloomFilterPolicy(caseInsensitive: false); Storage = new InMemoryLowLevelStorage(BufferPool); }
public BloomFilter(byte baseLg, int offset, IArrayAccessor accessor, BloomFilterPolicy bloomFilterPolicy) { _baseLg = baseLg; _accessor = accessor; _bloomFilterPolicy = bloomFilterPolicy; _offset = offset; _num = (accessor.Capacity - 5 - offset)/sizeof (int); }
public void CanProperlyMatchCaseInsensitive() { var bloomFilterPolicy = new BloomFilterPolicy(); var builder = bloomFilterPolicy.CreateBuilder(); var filterBlockBuilder = new FilterBlockBuilder(new MemoryStream(), builder); filterBlockBuilder.StartBlock(0); filterBlockBuilder.Add(new InternalKey("test", 0, ItemType.Value).TheInternalKey); var memoryStream = new MemoryStream(); filterBlockBuilder.Finish(memoryStream); var filter = bloomFilterPolicy.CreateFilter(ToMemoryMappedViewAccessor(memoryStream)); Assert.True(filter.KeyMayMatch(0, "Test")); Assert.True(filter.KeyMayMatch(0, "TEST")); }
public void CanConstructFilter() { var bloomFilterPolicy = new BloomFilterPolicy(); var builder = bloomFilterPolicy.CreateBuilder(); var filterBlockBuilder = new FilterBlockBuilder(new MemoryStream(), builder); filterBlockBuilder.StartBlock(0); filterBlockBuilder.Add(new InternalKey("test/1", 0, ItemType.Value).TheInternalKey); filterBlockBuilder.Add(new InternalKey("test/2", 0, ItemType.Value).TheInternalKey); filterBlockBuilder.Add(new InternalKey("test/231", 0, ItemType.Value).TheInternalKey); var memoryStream = new MemoryStream(); filterBlockBuilder.Finish(memoryStream); Assert.NotEqual(0, memoryStream.Length); }
public void CanReadFilter() { var bloomFilterPolicy = new BloomFilterPolicy(); var builder = bloomFilterPolicy.CreateBuilder(); var filterBlockBuilder = new FilterBlockBuilder(new MemoryStream(), builder); filterBlockBuilder.StartBlock(0); filterBlockBuilder.Add(new InternalKey("test/1", 0, ItemType.Value).TheInternalKey); filterBlockBuilder.Add(new InternalKey("test/2", 0, ItemType.Value).TheInternalKey); filterBlockBuilder.Add(new InternalKey("test/231", 0, ItemType.Value).TheInternalKey); var memoryStream = new MemoryStream(); filterBlockBuilder.Finish(memoryStream); var filter = bloomFilterPolicy.CreateFilter(ToMemoryMappedViewAccessor(memoryStream)); Assert.NotNull(filter); }
public BloomFilterBuilder(int bitsPerKey, int k, BloomFilterPolicy bloomFilterPolicy) { _bitsPerKey = bitsPerKey; _k = k; _bloomFilterPolicy = bloomFilterPolicy; }
public StorageOptions() { CreateIfMissing = true; BlockSize = 1024 * 4; MaxBlockCacheSizePerTableFile = 2048; MaxTablesCacheSize = 2048; BlockRestartInterval = 16; Comparator = new CaseInsensitiveComparator(); FilterPolicy = new BloomFilterPolicy(); MaximumExpectedKeySize = 2048; WriteBatchSize = 1024 * 1024 * 12; CacheSizeInMegabytes = 256; BufferPool = new BufferPool(); }