public LogV2StreamExistenceFilterInitializerTests()
{
_log = new FakeInMemoryTfReader(recordOffset: _recordOffset);
_tableIndex = new TableIndex <string>(
directory: Fixture.Directory,
lowHasher: new XXHashUnsafe(),
highHasher: new Murmur3AUnsafe(),
emptyStreamId: string.Empty,
memTableFactory: () => new HashListMemTable(
version: PTableVersions.IndexV4,
maxSize: 1_000_000 * 2),
maxSizeForMemory: 100_000,
tfReaderFactory: () => new TFReaderLease(_log),
ptableVersion: PTableVersions.IndexV4,
maxAutoMergeIndexLevel: int.MaxValue,
pTableMaxReaderCount: 5);
_tableIndex.Initialize(0);
_sut = new LogV2StreamExistenceFilterInitializer(
tfReaderFactory: () => new TFReaderLease(_log),
tableIndex: _tableIndex);
var hasher = new CompositeHasher <string>(new XXHashUnsafe(), new Murmur3AUnsafe());
_filter = new MockExistenceFilter(hasher);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_tableIndex = new TableIndex(PathName,
() => new HashListMemTable(maxSize: 40),
() => { throw new InvalidOperationException(); },
maxSizeForMemory: 20);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(0, 0xDEAD, 0, 0xFF00);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF01);
_tableIndex.Add(0, 0xBEEF, 0, 0xFF00);
_tableIndex.Add(0, 0xBEEF, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 0, 0xFF00);
_tableIndex.Add(0, 0xABBA, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 2, 0xFF02);
_tableIndex.Add(0, 0xABBA, 3, 0xFF03);
_tableIndex.Add(0, 0xDEAD, 0, 0xFF10);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF11);
_tableIndex.Add(0, 0xADA, 0, 0xFF00);
}
protected override void when()
{
//ptable 1 with 32bit indexes
_tableIndex.Add(1, streamId, 0, 2);
_tableIndex.Add(1, streamId, 1, 4);
_tableIndex.Add(1, streamId, 2, 6);
System.Threading.Thread.Sleep(500);
_tableIndex.Close(false);
_tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, "",
() => new HashListMemTable(PTableVersions.IndexV2, maxSize: _maxMemTableSize),
() => _fakeReader,
PTableVersions.IndexV2,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: _maxMemTableSize,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_indexReader = new IndexReader <string>(
_indexBackend, _tableIndex,
_logFormat.StreamNamesProvider,
_logFormat.StreamIdValidator,
_logFormat.StreamExistenceFilterReader,
new EventStore.Core.Data.StreamMetadata(),
_hashCollisionReadLimit, skipIndexScanOnRead: false);
//memtable with 64bit indexes
_tableIndex.Add(1, streamId, 0, 8);
}
public void SetUp()
{
_indexDir = Path.Combine(Path.GetTempPath(), "idx-" + Guid.NewGuid().ToString());
_tableIndex = new TableIndex(_indexDir, () => new HashListMemTable(), new InMemoryCheckpoint(), maxSizeForMemory: 2, maxTablesPerLevel: 2);
_tableIndex.Initialize();
// ptable level 2
_tableIndex.Add(0xDEAD, 0, 0xFF00);
_tableIndex.Add(0xDEAD, 1, 0xFF01);
_tableIndex.Add(0xBEEF, 0, 0xFF00);
_tableIndex.Add(0xBEEF, 1, 0xFF01);
_tableIndex.Add(0xABBA, 0, 0xFF00);
_tableIndex.Add(0xABBA, 1, 0xFF01);
_tableIndex.Add(0xABBA, 0, 0xFF02);
_tableIndex.Add(0xABBA, 1, 0xFF03);
// ptable level 1
_tableIndex.Add(0xADA, 0, 0xFF00);
_tableIndex.Add(0xCEED, 10, 0xFFF1);
_tableIndex.Add(0xBABA, 0, 0xFF00);
_tableIndex.Add(0xDEAD, 0, 0xFF10);
// ptable level 0
_tableIndex.Add(0xBABA, 1, 0xFF01);
_tableIndex.Add(0xDEAD, 1, 0xFF11);
// memtable
_tableIndex.Add(0xADA, 0, 0xFF01);
Thread.Sleep(500);
}
public void Setup()
{
given();
_indexDir = PathName;
_fakeReader = new TFReaderLease(new FakeReader());
_indexBackend = new FakeIndexBackend <string>(_fakeReader);
_logFormat = LogFormatHelper <LogFormat.V2, string> .LogFormatFactory.Create(new() {
InMemory = true,
});
_lowHasher = _logFormat.LowHasher;
_highHasher = _logFormat.HighHasher;
_tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, _logFormat.EmptyStreamId,
() => new HashListMemTable(PTableVersions.IndexV1, maxSize: _maxMemTableSize),
() => _fakeReader,
PTableVersions.IndexV1,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: _maxMemTableSize,
maxTablesPerLevel: 2);
_logFormat.StreamNamesProvider.SetTableIndex(_tableIndex);
_tableIndex.Initialize(long.MaxValue);
_indexReader = new IndexReader <string>(_indexBackend, _tableIndex,
_logFormat.StreamNamesProvider,
_logFormat.StreamIdValidator,
_logFormat.StreamExistenceFilterReader,
new EventStore.Core.Data.StreamMetadata(),
_hashCollisionReadLimit, skipIndexScanOnRead: false);
when();
//wait for the mem table to be dumped
System.Threading.Thread.Sleep(500);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex <string>(PathName, _lowHasher, _highHasher, "",
() => new HashListMemTable(version: _ptableVersion, maxSize: 40),
() => { throw new InvalidOperationException(); },
_ptableVersion,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 20,
skipIndexVerify: _skipIndexVerify);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(0, "0xDEAD", 0, 0xFF00);
_tableIndex.Add(0, "0xDEAD", 1, 0xFF01);
_tableIndex.Add(0, "0xJEEP", 0, 0xFF00);
_tableIndex.Add(0, "0xJEEP", 1, 0xFF01);
_tableIndex.Add(0, "0xABBA", 0, 0xFF00);
_tableIndex.Add(0, "0xABBA", 1, 0xFF01);
_tableIndex.Add(0, "0xABBA", 2, 0xFF02);
_tableIndex.Add(0, "0xABBA", 3, 0xFF03);
_tableIndex.Add(0, "0xDEAD", 0, 0xFF10);
_tableIndex.Add(0, "0xDEAD", 1, 0xFF11);
_tableIndex.Add(0, "0xADA", 0, 0xFF00);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
_tableIndex = new TableIndex(_indexDir, () => new HashListMemTable(maxSize: 10), maxSizeForMemory: 5);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(0, 0xDEAD, 0, 0xFF00);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF01);
_tableIndex.Add(0, 0xBEEF, 0, 0xFF00);
_tableIndex.Add(0, 0xBEEF, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 0, 0xFF00); // 1st ptable0
_tableIndex.Add(0, 0xABBA, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 2, 0xFF02);
_tableIndex.Add(0, 0xABBA, 3, 0xFF03);
_tableIndex.Add(0, 0xADA, 0, 0xFF00); // simulates duplicate due to concurrency in TableIndex (see memtable below)
_tableIndex.Add(0, 0xDEAD, 0, 0xFF10); // 2nd ptable0
_tableIndex.Add(0, 0xDEAD, 1, 0xFF11); // in memtable
_tableIndex.Add(0, 0xADA, 0, 0xFF00); // in memtable
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeTfReader());
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 10),
() => fakeReader,
_ptableVersion,
maxSizeForMemory: 5);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(0, "0xDEAD", 0, 0xFF00);
_tableIndex.Add(0, "0xDEAD", 1, 0xFF01);
_tableIndex.Add(0, "0xBEEF", 0, 0xFF00);
_tableIndex.Add(0, "0xBEEF", 1, 0xFF01);
_tableIndex.Add(0, "0xABBA", 0, 0xFF00); // 1st ptable0
_tableIndex.Add(0, "0xABBA", 1, 0xFF01);
_tableIndex.Add(0, "0xABBA", 2, 0xFF02);
_tableIndex.Add(0, "0xABBA", 3, 0xFF03);
_tableIndex.Add(0, "0xADA", 0, 0xFF00); // simulates duplicate due to concurrency in TableIndex (see memtable below)
_tableIndex.Add(0, "0xDEAD", 0, 0xFF10); // 2nd ptable0
_tableIndex.Add(0, "0xDEAD", 1, 0xFF11); // in memtable
_tableIndex.Add(0, "0xADA", 0, 0xFF00); // in memtable
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(PathName, _lowHasher, _highHasher,
() => new HashListMemTable(version: _ptableVersion, maxSize: 40),
() => { throw new InvalidOperationException(); },
_ptableVersion,
maxSizeForMemory: 20,
skipIndexVerify: _skipIndexVerify);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(0, "0xDEAD", 0, 0xFF00);
_tableIndex.Add(0, "0xDEAD", 1, 0xFF01);
_tableIndex.Add(0, "0xJEEP", 0, 0xFF00);
_tableIndex.Add(0, "0xJEEP", 1, 0xFF01);
_tableIndex.Add(0, "0xABBA", 0, 0xFF00);
_tableIndex.Add(0, "0xABBA", 1, 0xFF01);
_tableIndex.Add(0, "0xABBA", 2, 0xFF02);
_tableIndex.Add(0, "0xABBA", 3, 0xFF03);
_tableIndex.Add(0, "0xDEAD", 0, 0xFF10);
_tableIndex.Add(0, "0xDEAD", 1, 0xFF11);
_tableIndex.Add(0, "0xADA", 0, 0xFF00);
}
public void ConstructTableIndexWithCorruptIndexEntries(byte version, bool skipIndexVerify, bool createForceVerifyFile = false)
{
base.TestFixtureSetUp();
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
var fakeReader = new TFReaderLease(new FakeIndexReader());
_tableIndex = new TableIndex(PathName, lowHasher, highHasher,
() => new HashListMemTable(version, maxSize: NumIndexEntries),
() => fakeReader,
version,
5,
maxSizeForMemory: NumIndexEntries,
skipIndexVerify: skipIndexVerify);
_tableIndex.Initialize(long.MaxValue);
//create index entries
for (int i = 1; i <= NumIndexEntries; i++)
{
_tableIndex.Add(i * 1337, StreamName, i, i * 1337);
}
_tableIndex.Close(false);
//load index map to obtain ptable filenames
_indexMap = IndexMap.FromFile(Path.Combine(PathName, TableIndex.IndexMapFilename));
List <string> ptableFiles = new List <string>();
foreach (string ptableFilename in _indexMap.GetAllFilenames())
{
ptableFiles.Add(ptableFilename);
}
_indexMap.Dispose(TimeSpan.FromSeconds(5));
//corrupt ptable files
foreach (string ptableFilename in ptableFiles)
{
CorruptPTableFile(ptableFilename, version, "zeroOutMiddleEntry");
}
//create force verify file if requested
if (createForceVerifyFile)
{
using (FileStream fs = new FileStream(Path.Combine(PathName, TableIndex.ForceIndexVerifyFilename), FileMode.OpenOrCreate)){
};
}
//load table index again
_tableIndex = new TableIndex(PathName, lowHasher, highHasher,
() => new HashListMemTable(version, maxSize: NumIndexEntries),
() => fakeReader,
version,
5,
maxSizeForMemory: NumIndexEntries,
skipIndexVerify: skipIndexVerify,
indexCacheDepth: 8);
_tableIndex.Initialize(long.MaxValue);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_tableIndex = new TableIndex(PathName,
() => new HashListMemTable(maxSize: 20),
() => { throw new InvalidOperationException(); },
maxSizeForMemory: 10);
_tableIndex.Initialize(long.MaxValue);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV1, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV1,
5,
maxSizeForMemory: 5 + _extraStreamHashesAtBeginning + _extraStreamHashesAtEnd,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
Assert.Greater(_lowHasher.Hash("abcd"), _lowHasher.Hash("LPN-FC002_LPK51001"));
for (int i = 0; i < _extraStreamHashesAtBeginning; i++)
{
_tableIndex.Add(1, "abcd", i, i + 1);
}
Assert.Less(_lowHasher.Hash("wxyz"), _lowHasher.Hash("LPN-FC002_LPK51001"));
for (int i = 0; i < _extraStreamHashesAtEnd; i++)
{
_tableIndex.Add(1, "wxyz", i, i + 1);
}
_tableIndex.Add(1, "LPN-FC002_LPK51001", 0, 1);
_tableIndex.Add(1, "account--696193173", 0, 2);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 1, 3);
_tableIndex.Add(1, "account--696193173", 1, 4);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 2, 5);
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 5),
() => fakeReader,
_ptableVersion,
5,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "account--696193173", 2, 6);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 3, 7);
_tableIndex.Add(1, "account--696193173", 3, 8);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 4, 9);
_tableIndex.Add(1, "account--696193173", 4, 10);
Thread.Sleep(500);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
int readerCount = 0;
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => {
readerCount++;
if (readerCount < 4) // One for each table add.
{
return(fakeReader);
}
throw new Exception("Expected exception");
},
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-1", 0, 0);
_tableIndex.Add(1, "testStream-1", 1, 100);
_tableIndex.Add(1, "testStream-1", 2, 200);
_tableIndex.Add(1, "testStream-1", 3, 300);
_tableIndex.Add(1, "testStream-1", 4, 400);
_tableIndex.Add(1, "testStream-1", 5, 500);
_log = new FakeTFScavengerLog();
Assert.That(() => _tableIndex.Scavenge(_log, CancellationToken.None),
Throws.Exception.With.Message.EqualTo("Expected exception"));
// Check it's loadable still.
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5);
_tableIndex.Initialize(long.MaxValue);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(PathName, lowHasher, highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 20),
() => { throw new InvalidOperationException(); },
_ptableVersion,
maxSizeForMemory: 10);
_tableIndex.Initialize(long.MaxValue);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var cancellationTokenSource = new CancellationTokenSource();
var fakeReader = new TFReaderLease(new FakeIndexReader(l => {
cancellationTokenSource.Cancel();
return(true);
}));
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-1", 0, 0);
_tableIndex.Add(1, "testStream-1", 1, 100);
_tableIndex.Add(1, "testStream-1", 2, 200);
_tableIndex.Add(1, "testStream-1", 3, 300);
_tableIndex.Add(1, "testStream-1", 4, 400);
_tableIndex.Add(1, "testStream-1", 5, 500);
_log = new FakeTFScavengerLog();
Assert.That(() => _tableIndex.Scavenge(_log, cancellationTokenSource.Token),
Throws.InstanceOf <OperationCanceledException>());
// Check it's loadable still.
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5);
_tableIndex.Initialize(long.MaxValue);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
ReadIndex.Close();
ReadIndex.Dispose();
Thread.Sleep(500);
TableIndex.ClearAll(removeFiles: false);
TableIndex = new TableIndex(Path.Combine(PathName, "index"), () => new HashListMemTable(), maxSizeForMemory: 5);
TableIndex.Initialize();
ReadIndex = new ReadIndex(new NoopPublisher(), 2, () => new TFChunkSequentialReader(Db, WriterCheckpoint, 0), () => new TFChunkReader(Db, WriterCheckpoint), TableIndex, new ByLengthHasher());
ReadIndex.Build();
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(PathName, lowHasher, highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 20),
() => { throw new InvalidOperationException(); },
_ptableVersion,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 10,
skipIndexVerify: _skipIndexVerify);
_tableIndex.Initialize(long.MaxValue);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, "",
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => throw new Exception("Expected exception") /* throw an exception when the first PTable scavenge starts and tries to acquire a reader */,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5,
useBloomFilter: _useBloomFilter);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-1", 0, 0);
_tableIndex.Add(1, "testStream-1", 1, 100);
_tableIndex.Add(1, "testStream-1", 2, 200);
_tableIndex.Add(1, "testStream-1", 3, 300);
_tableIndex.Add(1, "testStream-1", 4, 400);
_tableIndex.Add(1, "testStream-1", 5, 500);
_log = new FakeTFScavengerLog();
Assert.That(() => _tableIndex.Scavenge(_log, CancellationToken.None),
Throws.Exception.With.Message.EqualTo("Expected exception"));
// Check it's loadable still.
_tableIndex.Close(false);
_tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, "",
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5,
useBloomFilter: _useBloomFilter);
_tableIndex.Initialize(long.MaxValue);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader(l => !Deleted.Contains(l)));
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, "",
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5, skipIndexVerify: _skipIndexVerify,
useBloomFilter: _useBloomFilter);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-1", 0, 0);
_tableIndex.Add(1, "testStream-1", 1, 100);
_tableIndex.Add(1, "testStream-1", 2, 200);
_tableIndex.Add(1, "testStream-1", 3, 300);
_tableIndex.Add(1, "testStream-1", 4, 400);
_tableIndex.Add(1, "testStream-1", 5, 500);
_log = new FakeTFScavengerLog();
_tableIndex.Scavenge(_log, CancellationToken.None);
// Check it's loadable.
_tableIndex.Close(false);
_tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, "",
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5,
useBloomFilter: _useBloomFilter);
_tableIndex.Initialize(long.MaxValue);
}
protected override void when()
{
//ptable 1 with 32bit indexes
_tableIndex.Add(1, streamId, 0, 2);
_tableIndex.Add(1, streamId, 1, 4);
_tableIndex.Add(1, streamId, 2, 6);
System.Threading.Thread.Sleep(500);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.Index64Bit, maxSize: _maxMemTableSize),
() => _fakeReader,
PTableVersions.Index64Bit,
maxSizeForMemory: _maxMemTableSize,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_indexReader = new IndexReader(_indexBackend, _tableIndex, new EventStore.Core.Data.StreamMetadata(), _hashCollisionReadLimit);
//memtable with 64bit indexes
_tableIndex.Add(1, streamId, 0, 8);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new FakeIndexHasher();
_highHasher = new FakeIndexHasher();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 10),
() => fakeReader,
_ptableVersion,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 2,
skipIndexVerify: _skipIndexVerify);
_tableIndex.Initialize(long.MaxValue);
// ptable level 2
_tableIndex.Add(0, "1", 0, 0xFF00);
_tableIndex.Add(0, "1", 1, 0xFF01);
_tableIndex.Add(0, "2", 0, 0xFF00);
_tableIndex.Add(0, "2", 1, 0xFF01);
_tableIndex.Add(0, "3", 0, 0xFF00);
_tableIndex.Add(0, "3", 1, 0xFF01);
_tableIndex.Add(0, "3", 0, 0xFF02);
_tableIndex.Add(0, "3", 1, 0xFF03);
// ptable level 1
_tableIndex.Add(0, "4", 0, 0xFF00);
_tableIndex.Add(0, "5", 10, 0xFFF1);
_tableIndex.Add(0, "6", 0, 0xFF00);
_tableIndex.Add(0, "1", 0, 0xFF10);
// ptable level 0
_tableIndex.Add(0, "6", 1, 0xFF01);
_tableIndex.Add(0, "1", 1, 0xFF11);
// memtable
_tableIndex.Add(0, "4", 0, 0xFF01);
await Task.Delay(500);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new FakeIndexHasher();
_highHasher = new FakeIndexHasher();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 10),
() => fakeReader,
_ptableVersion,
5,
maxSizeForMemory: 2,
maxTablesPerLevel: 2,
skipIndexVerify: _skipIndexVerify);
_tableIndex.Initialize(long.MaxValue);
// ptable level 2
_tableIndex.Add(0, "1", 0, 0xFF00);
_tableIndex.Add(0, "1", 1, 0xFF01);
_tableIndex.Add(0, "2", 0, 0xFF00);
_tableIndex.Add(0, "2", 1, 0xFF01);
_tableIndex.Add(0, "3", 0, 0xFF00);
_tableIndex.Add(0, "3", 1, 0xFF01);
_tableIndex.Add(0, "3", 0, 0xFF02);
_tableIndex.Add(0, "3", 1, 0xFF03);
// ptable level 1
_tableIndex.Add(0, "4", 0, 0xFF00);
_tableIndex.Add(0, "5", 10, 0xFFF1);
_tableIndex.Add(0, "6", 0, 0xFF00);
_tableIndex.Add(0, "1", 0, 0xFF10);
// ptable level 0
_tableIndex.Add(0, "6", 1, 0xFF01);
_tableIndex.Add(0, "1", 1, 0xFF11);
// memtable
_tableIndex.Add(0, "4", 0, 0xFF01);
Thread.Sleep(500);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV2, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV2,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-1", 0, 1);
_tableIndex.Add(1, "testStream-2", 0, 2);
_tableIndex.Add(1, "testStream-1", 1, 3);
_tableIndex.Add(1, "testStream-2", 1, 4);
_tableIndex.Add(1, "testStream-1", 2, 5);
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 5),
() => fakeReader,
_ptableVersion,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-2", 2, 6);
_tableIndex.Add(1, "testStream-1", 3, 7);
_tableIndex.Add(1, "testStream-2", 3, 8);
_tableIndex.Add(1, "testStream-1", 4, 9);
_tableIndex.Add(1, "testStream-2", 4, 10);
await Task.Delay(500);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV1, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV1,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 0, 1);
_tableIndex.Add(1, "account--696193173", 0, 2);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 1, 3);
_tableIndex.Add(1, "account--696193173", 1, 4);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 2, 5);
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 5),
() => fakeReader,
_ptableVersion,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "account--696193173", 2, 6);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 3, 7);
_tableIndex.Add(1, "account--696193173", 3, 8);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 4, 9);
_tableIndex.Add(1, "account--696193173", 4, 10);
Thread.Sleep(500);
}
public void Setup()
{
given();
_indexDir = PathName;
_fakeReader = new TFReaderLease(new FakeReader());
_indexBackend = new FakeIndexBackend(_fakeReader);
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV1, maxSize: _maxMemTableSize),
() => _fakeReader,
PTableVersions.IndexV1,
maxSizeForMemory: _maxMemTableSize,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_indexReader = new IndexReader(_indexBackend, _tableIndex, new EventStore.Core.Data.StreamMetadata(), _hashCollisionReadLimit);
when();
//wait for the mem table to be dumped
System.Threading.Thread.Sleep(500);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader2());
_lowHasher = new ByLengthHasher();
_highHasher = new ByLengthHasher();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV1, maxSize: 3),
() => fakeReader,
PTableVersions.IndexV1,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 3,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, Stream1, 0, 1);
_tableIndex.Add(1, Stream2, 0, 2);
_tableIndex.Add(1, Stream3, 0, 3);
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 3),
() => fakeReader,
_ptableVersion,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 3,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, Stream3, 1, 4);
_tableIndex.Add(1, Stream2, 1, 5);
_tableIndex.Add(1, Stream1, 1, 6);
await Task.Delay(500);
}
public void SetUp()
{
_indexDir = Path.Combine(Path.GetTempPath(), "idx-" + Guid.NewGuid().ToString());
_tableIndex = new TableIndex(_indexDir, () => new HashListMemTable(), new InMemoryCheckpoint(), maxSizeForMemory: 2000);
_tableIndex.Initialize();
_tableIndex.Add(0xDEAD, 0, 0xFF00);
_tableIndex.Add(0xDEAD, 1, 0xFF01);
_tableIndex.Add(0xBEEF, 0, 0xFF00);
_tableIndex.Add(0xBEEF, 1, 0xFF01);
_tableIndex.Add(0xABBA, 0, 0xFF00);
_tableIndex.Add(0xABBA, 1, 0xFF01);
_tableIndex.Add(0xABBA, 2, 0xFF02);
_tableIndex.Add(0xABBA, 3, 0xFF03);
_tableIndex.Add(0xDEAD, 0, 0xFF10);
_tableIndex.Add(0xDEAD, 1, 0xFF11);
_tableIndex.Add(0xADA, 0, 0xFF00);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeTfReader());
_tableIndex = new TableIndex(_indexDir,
() => new HashListMemTable(maxSize: 10),
() => fakeReader,
maxSizeForMemory: 2,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
// ptable level 2
_tableIndex.Add(0, 0xDEAD, 0, 0xFF00);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF01);
_tableIndex.Add(0, 0xBEEF, 0, 0xFF00);
_tableIndex.Add(0, 0xBEEF, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 0, 0xFF00);
_tableIndex.Add(0, 0xABBA, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 0, 0xFF02);
_tableIndex.Add(0, 0xABBA, 1, 0xFF03);
// ptable level 1
_tableIndex.Add(0, 0xADA, 0, 0xFF00);
_tableIndex.Add(0, 0xCEED, 10, 0xFFF1);
_tableIndex.Add(0, 0xBABA, 0, 0xFF00);
_tableIndex.Add(0, 0xDEAD, 0, 0xFF10);
// ptable level 0
_tableIndex.Add(0, 0xBABA, 1, 0xFF01);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF11);
// memtable
_tableIndex.Add(0, 0xADA, 0, 0xFF01);
Thread.Sleep(500);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader2());
_lowHasher = new ByLengthHasher();
_highHasher = new ByLengthHasher();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV1, maxSize: 3),
() => fakeReader,
PTableVersions.IndexV1,
5,
maxSizeForMemory: 3,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, Stream1, 0, 1);
_tableIndex.Add(1, Stream2, 0, 2);
_tableIndex.Add(1, Stream3, 0, 3);
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 3),
() => fakeReader,
_ptableVersion,
5,
maxSizeForMemory: 3,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, Stream3, 1, 4);
_tableIndex.Add(1, Stream2, 1, 5);
_tableIndex.Add(1, Stream1, 1, 6);
Thread.Sleep(500);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeTfReader());
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, "",
() => new HashListMemTable(_ptableVersion, maxSize: 10),
() => fakeReader,
_ptableVersion,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 5,
skipIndexVerify: _skipIndexVerify);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(0, "0xDEAD", 0, 0xFF00);
_tableIndex.Add(0, "0xDEAD", 1, 0xFF01);
_tableIndex.Add(0, "0xBEEF", 0, 0xFF00);
_tableIndex.Add(0, "0xBEEF", 1, 0xFF01);
_tableIndex.Add(0, "0xABBA", 0, 0xFF00); // 1st ptable0
_tableIndex.Add(0, "0xABBA", 1, 0xFF01);
_tableIndex.Add(0, "0xABBA", 2, 0xFF02);
_tableIndex.Add(0, "0xABBA", 3, 0xFF03);
_tableIndex.Add(0, "0xADA", 0,
0xFF00); // simulates duplicate due to concurrency in TableIndex (see memtable below)
_tableIndex.Add(0, "0xDEAD", 0, 0xFF10); // 2nd ptable0
_tableIndex.Add(0, "0xDEAD", 1, 0xFF11); // in memtable
_tableIndex.Add(0, "0xADA", 0, 0xFF00); // in memtable
}
