/// <summary>
/// Create a new table builder.
/// - storageOptions define the options for the table buildup.
/// - dataStream is where the data for the table will be written to.
/// REQUIRES: Being able to read dataStream.Position
/// - tempStream is where temporary data is written to avoid holding too much in memory
/// REQUIRES: Being able to read tempStream.Position AND change tempStream.Position
/// </summary>
public TableBuilder(StorageState storageState,
Stream dataStream,
TemporaryFiles temporaryFiles)
{
_temporaryFiles = temporaryFiles;
try
{
_storageState = storageState;
_dataStream = dataStream;
_indexStream = temporaryFiles.Create();
_originalIndexStreamPosition = _indexStream.Position;
_lastKeyBuffer = _storageState.Options.BufferPool.Take(storageState.Options.MaximumExpectedKeySize);
_scratchBuffer = _storageState.Options.BufferPool.Take(storageState.Options.MaximumExpectedKeySize);
if (storageState.Options.FilterPolicy != null)
{
var filterBuilder = storageState.Options.FilterPolicy.CreateBuilder();
_filterBlockStream = temporaryFiles.Create();
_filterBuilder = new FilterBlockBuilder(_filterBlockStream, filterBuilder);
_filterBuilder.StartBlock(0);
}
_indexBlock = new BlockBuilder(_indexStream, storageState, _storageState.InternalKeyComparator,
blockRestartInterval: 1);
_dataBlock = new BlockBuilder(_dataStream, storageState, _storageState.InternalKeyComparator, _storageState.Options.BlockRestartInterval);
}
catch (Exception)
{
Dispose();
throw;
}
}
public void CanProperlyMatchCaseInsensitive()
{
var bloomFilterPolicy = new BloomFilterPolicy();
var builder = bloomFilterPolicy.CreateBuilder();
var filterBlockBuilder = new FilterBlockBuilder(new MemoryStream(), builder);
filterBlockBuilder.StartBlock(0);
filterBlockBuilder.Add(new InternalKey("test", 0, ItemType.Value).TheInternalKey);
var memoryStream = new MemoryStream();
filterBlockBuilder.Finish(memoryStream);
var filter = bloomFilterPolicy.CreateFilter(ToMemoryMappedViewAccessor(memoryStream));
Assert.True(filter.KeyMayMatch(0, "Test"));
Assert.True(filter.KeyMayMatch(0, "TEST"));
}
public void CanConstructFilter()
{
var bloomFilterPolicy = new BloomFilterPolicy();
var builder = bloomFilterPolicy.CreateBuilder();
var filterBlockBuilder = new FilterBlockBuilder(new MemoryStream(), builder);
filterBlockBuilder.StartBlock(0);
filterBlockBuilder.Add(new InternalKey("test/1", 0, ItemType.Value).TheInternalKey);
filterBlockBuilder.Add(new InternalKey("test/2", 0, ItemType.Value).TheInternalKey);
filterBlockBuilder.Add(new InternalKey("test/231", 0, ItemType.Value).TheInternalKey);
var memoryStream = new MemoryStream();
filterBlockBuilder.Finish(memoryStream);
Assert.NotEqual(0, memoryStream.Length);
}
public void CanReadFilter()
{
var bloomFilterPolicy = new BloomFilterPolicy();
var builder = bloomFilterPolicy.CreateBuilder();
var filterBlockBuilder = new FilterBlockBuilder(new MemoryStream(), builder);
filterBlockBuilder.StartBlock(0);
filterBlockBuilder.Add(new InternalKey("test/1", 0, ItemType.Value).TheInternalKey);
filterBlockBuilder.Add(new InternalKey("test/2", 0, ItemType.Value).TheInternalKey);
filterBlockBuilder.Add(new InternalKey("test/231", 0, ItemType.Value).TheInternalKey);
var memoryStream = new MemoryStream();
filterBlockBuilder.Finish(memoryStream);
var filter = bloomFilterPolicy.CreateFilter(ToMemoryMappedViewAccessor(memoryStream));
Assert.NotNull(filter);
}
public void CanProperlyMatchComplex()
{
var bloomFilterPolicy = new BloomFilterPolicy(caseInsensitive: false);
var builder = bloomFilterPolicy.CreateBuilder();
var filterBlockBuilder = new FilterBlockBuilder(new MemoryStream(), builder);
filterBlockBuilder.StartBlock(0);
filterBlockBuilder.Add(new InternalKey("test/1", 0, ItemType.Value).TheInternalKey);
filterBlockBuilder.Add(new InternalKey("test/2", 0, ItemType.Value).TheInternalKey);
filterBlockBuilder.Add(new InternalKey("test/3", 0, ItemType.Value).TheInternalKey);
filterBlockBuilder.Add(new InternalKey("test/142", 0, ItemType.Value).TheInternalKey);
filterBlockBuilder.Add(new InternalKey("test/3432", 0, ItemType.Value).TheInternalKey);
var memoryStream = new MemoryStream();
filterBlockBuilder.Finish(memoryStream);
var filter = bloomFilterPolicy.CreateFilter(ToMemoryMappedViewAccessor(memoryStream));
Assert.True(filter.KeyMayMatch(0, "test/1"));
Assert.True(filter.KeyMayMatch(0, "test/142"));
Assert.True(filter.KeyMayMatch(0, "test/3"));
Assert.True(filter.KeyMayMatch(0, "test/3432"));
Assert.False(filter.KeyMayMatch(0, "tester"));
Assert.False(filter.KeyMayMatch(0, "test/5"));
Assert.False(filter.KeyMayMatch(0, "test/133"));
Assert.False(filter.KeyMayMatch(0, "test/133"));
Assert.True(filter.KeyMayMatch(0, "test/144")); // even though this is wrong, false positives are okay with bloom filters
}