public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
ReadIndex.Close();
ReadIndex.Dispose();
TableIndex.Close(removeFiles: false);
var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 2, () => new TFChunkReader(Db, WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.IndexV2, maxSize: MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
PTableVersions.IndexV2,
5,
maxSizeForMemory: MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: true,
metastreamMaxCount: 1,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
replicationCheckpoint: Db.Config.ReplicationCheckpoint);
ReadIndex.Init(ChaserCheckpoint.Read());
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
if (TruncateCheckpoint == long.MinValue)
{
throw new InvalidOperationException("AckCheckpoint must be set in WriteTestScenario.");
}
OnBeforeTruncating();
// need to close db before truncator can delete files
ReadIndex.Close();
ReadIndex.Dispose();
TableIndex.Close(removeFiles: false);
Db.Close();
Db.Dispose();
var truncator = new TFChunkDbTruncator(Db.Config);
truncator.TruncateDb(TruncateCheckpoint);
}
public void ConstructTableIndexWithCorruptIndexEntries(byte version, bool skipIndexVerify, bool createForceVerifyFile = false)
{
base.TestFixtureSetUp();
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
var fakeReader = new TFReaderLease(new FakeIndexReader());
_tableIndex = new TableIndex(PathName, lowHasher, highHasher,
() => new HashListMemTable(version, maxSize: NumIndexEntries),
() => fakeReader,
version,
5,
maxSizeForMemory: NumIndexEntries,
skipIndexVerify: skipIndexVerify);
_tableIndex.Initialize(long.MaxValue);
//create index entries
for (int i = 1; i <= NumIndexEntries; i++)
{
_tableIndex.Add(i * 1337, StreamName, i, i * 1337);
}
_tableIndex.Close(false);
//load index map to obtain ptable filenames
_indexMap = IndexMap.FromFile(Path.Combine(PathName, TableIndex.IndexMapFilename));
List <string> ptableFiles = new List <string>();
foreach (string ptableFilename in _indexMap.GetAllFilenames())
{
ptableFiles.Add(ptableFilename);
}
_indexMap.Dispose(TimeSpan.FromSeconds(5));
//corrupt ptable files
foreach (string ptableFilename in ptableFiles)
{
CorruptPTableFile(ptableFilename, version, "zeroOutMiddleEntry");
}
//create force verify file if requested
if (createForceVerifyFile)
{
using (FileStream fs = new FileStream(Path.Combine(PathName, TableIndex.ForceIndexVerifyFilename), FileMode.OpenOrCreate)){
};
}
//load table index again
_tableIndex = new TableIndex(PathName, lowHasher, highHasher,
() => new HashListMemTable(version, maxSize: NumIndexEntries),
() => fakeReader,
version,
5,
maxSizeForMemory: NumIndexEntries,
skipIndexVerify: skipIndexVerify,
indexCacheDepth: 8);
_tableIndex.Initialize(long.MaxValue);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV1, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV1,
5,
maxSizeForMemory: 5 + _extraStreamHashesAtBeginning + _extraStreamHashesAtEnd,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
Assert.Greater(_lowHasher.Hash("abcd"), _lowHasher.Hash("LPN-FC002_LPK51001"));
for (int i = 0; i < _extraStreamHashesAtBeginning; i++)
{
_tableIndex.Add(1, "abcd", i, i + 1);
}
Assert.Less(_lowHasher.Hash("wxyz"), _lowHasher.Hash("LPN-FC002_LPK51001"));
for (int i = 0; i < _extraStreamHashesAtEnd; i++)
{
_tableIndex.Add(1, "wxyz", i, i + 1);
}
_tableIndex.Add(1, "LPN-FC002_LPK51001", 0, 1);
_tableIndex.Add(1, "account--696193173", 0, 2);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 1, 3);
_tableIndex.Add(1, "account--696193173", 1, 4);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 2, 5);
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 5),
() => fakeReader,
_ptableVersion,
5,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "account--696193173", 2, 6);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 3, 7);
_tableIndex.Add(1, "account--696193173", 3, 8);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 4, 9);
_tableIndex.Add(1, "account--696193173", 4, 10);
Thread.Sleep(500);
}
public override Task TestFixtureTearDown()
{
ReadIndex.Close();
ReadIndex.Dispose();
_tableIndex.Close();
_db.Close();
_db.Dispose();
return(base.TestFixtureTearDown());
}
public override void TestFixtureTearDown()
{
ReadIndex.Close();
ReadIndex.Dispose();
TableIndex.Close();
Db.Close();
Db.Dispose();
base.TestFixtureTearDown();
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
int readerCount = 0;
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => {
readerCount++;
if (readerCount < 4) // One for each table add.
{
return(fakeReader);
}
throw new Exception("Expected exception");
},
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-1", 0, 0);
_tableIndex.Add(1, "testStream-1", 1, 100);
_tableIndex.Add(1, "testStream-1", 2, 200);
_tableIndex.Add(1, "testStream-1", 3, 300);
_tableIndex.Add(1, "testStream-1", 4, 400);
_tableIndex.Add(1, "testStream-1", 5, 500);
_log = new FakeTFScavengerLog();
Assert.That(() => _tableIndex.Scavenge(_log, CancellationToken.None),
Throws.Exception.With.Message.EqualTo("Expected exception"));
// Check it's loadable still.
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5);
_tableIndex.Initialize(long.MaxValue);
}
public override Task TestFixtureTearDown()
{
_logFormat?.Dispose();
ReadIndex?.Close();
ReadIndex?.Dispose();
TableIndex?.Close();
Db?.Close();
Db?.Dispose();
return(base.TestFixtureTearDown());
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var cancellationTokenSource = new CancellationTokenSource();
var fakeReader = new TFReaderLease(new FakeIndexReader(l => {
cancellationTokenSource.Cancel();
return(true);
}));
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-1", 0, 0);
_tableIndex.Add(1, "testStream-1", 1, 100);
_tableIndex.Add(1, "testStream-1", 2, 200);
_tableIndex.Add(1, "testStream-1", 3, 300);
_tableIndex.Add(1, "testStream-1", 4, 400);
_tableIndex.Add(1, "testStream-1", 5, 500);
_log = new FakeTFScavengerLog();
Assert.That(() => _tableIndex.Scavenge(_log, cancellationTokenSource.Token),
Throws.InstanceOf <OperationCanceledException>());
// Check it's loadable still.
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5);
_tableIndex.Initialize(long.MaxValue);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, "",
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => throw new Exception("Expected exception") /* throw an exception when the first PTable scavenge starts and tries to acquire a reader */,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5,
useBloomFilter: _useBloomFilter);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-1", 0, 0);
_tableIndex.Add(1, "testStream-1", 1, 100);
_tableIndex.Add(1, "testStream-1", 2, 200);
_tableIndex.Add(1, "testStream-1", 3, 300);
_tableIndex.Add(1, "testStream-1", 4, 400);
_tableIndex.Add(1, "testStream-1", 5, 500);
_log = new FakeTFScavengerLog();
Assert.That(() => _tableIndex.Scavenge(_log, CancellationToken.None),
Throws.Exception.With.Message.EqualTo("Expected exception"));
// Check it's loadable still.
_tableIndex.Close(false);
_tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, "",
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5,
useBloomFilter: _useBloomFilter);
_tableIndex.Initialize(long.MaxValue);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader(l => !Deleted.Contains(l)));
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, "",
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5, skipIndexVerify: _skipIndexVerify,
useBloomFilter: _useBloomFilter);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-1", 0, 0);
_tableIndex.Add(1, "testStream-1", 1, 100);
_tableIndex.Add(1, "testStream-1", 2, 200);
_tableIndex.Add(1, "testStream-1", 3, 300);
_tableIndex.Add(1, "testStream-1", 4, 400);
_tableIndex.Add(1, "testStream-1", 5, 500);
_log = new FakeTFScavengerLog();
_tableIndex.Scavenge(_log, CancellationToken.None);
// Check it's loadable.
_tableIndex.Close(false);
_tableIndex = new TableIndex <string>(_indexDir, _lowHasher, _highHasher, "",
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 2,
maxTablesPerLevel: 5,
useBloomFilter: _useBloomFilter);
_tableIndex.Initialize(long.MaxValue);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV2, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV2,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-1", 0, 1);
_tableIndex.Add(1, "testStream-2", 0, 2);
_tableIndex.Add(1, "testStream-1", 1, 3);
_tableIndex.Add(1, "testStream-2", 1, 4);
_tableIndex.Add(1, "testStream-1", 2, 5);
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 5),
() => fakeReader,
_ptableVersion,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-2", 2, 6);
_tableIndex.Add(1, "testStream-1", 3, 7);
_tableIndex.Add(1, "testStream-2", 3, 8);
_tableIndex.Add(1, "testStream-1", 4, 9);
_tableIndex.Add(1, "testStream-2", 4, 10);
await Task.Delay(500);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
ReadIndex.Close();
ReadIndex.Dispose();
TableIndex.Close(removeFiles: false);
var readers =
new ObjectPool <ITransactionFileReader>("Readers", 2, 2, () => new TFChunkReader(Db, WriterCheckpoint));
var lowHasher = _logFormat.LowHasher;
var highHasher = _logFormat.HighHasher;
var emptyStreamId = _logFormat.EmptyStreamId;
TableIndex = new TableIndex <TStreamId>(GetFilePathFor("index"), lowHasher, highHasher, emptyStreamId,
() => new HashListMemTable(PTableVersions.IndexV2, maxSize: MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
PTableVersions.IndexV2,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: MaxEntriesInMemTable);
var readIndex = new ReadIndex <TStreamId>(new NoopPublisher(),
readers,
TableIndex,
_logFormat.StreamNameIndexConfirmer,
_logFormat.StreamIds,
_logFormat.StreamNamesProvider,
_logFormat.EmptyStreamId,
_logFormat.StreamIdValidator,
_logFormat.StreamIdSizer,
_logFormat.StreamExistenceFilter,
_logFormat.StreamExistenceFilterReader,
_logFormat.EventTypeIndexConfirmer,
streamInfoCacheCapacity: 0,
additionalCommitChecks: true,
metastreamMaxCount: 1,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
replicationCheckpoint: Db.Config.ReplicationCheckpoint,
indexCheckpoint: Db.Config.IndexCheckpoint);
readIndex.IndexCommitter.Init(ChaserCheckpoint.Read());
ReadIndex = readIndex;
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV1, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV1,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 0, 1);
_tableIndex.Add(1, "account--696193173", 0, 2);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 1, 3);
_tableIndex.Add(1, "account--696193173", 1, 4);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 2, 5);
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 5),
() => fakeReader,
_ptableVersion,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "account--696193173", 2, 6);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 3, 7);
_tableIndex.Add(1, "account--696193173", 3, 8);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 4, 9);
_tableIndex.Add(1, "account--696193173", 4, 10);
Thread.Sleep(500);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader2());
_lowHasher = new ByLengthHasher();
_highHasher = new ByLengthHasher();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV1, maxSize: 3),
() => fakeReader,
PTableVersions.IndexV1,
5,
maxSizeForMemory: 3,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, Stream1, 0, 1);
_tableIndex.Add(1, Stream2, 0, 2);
_tableIndex.Add(1, Stream3, 0, 3);
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 3),
() => fakeReader,
_ptableVersion,
5,
maxSizeForMemory: 3,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, Stream3, 1, 4);
_tableIndex.Add(1, Stream2, 1, 5);
_tableIndex.Add(1, Stream1, 1, 6);
Thread.Sleep(500);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader2());
_lowHasher = new ByLengthHasher();
_highHasher = new ByLengthHasher();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV1, maxSize: 3),
() => fakeReader,
PTableVersions.IndexV1,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 3,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, Stream1, 0, 1);
_tableIndex.Add(1, Stream2, 0, 2);
_tableIndex.Add(1, Stream3, 0, 3);
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 3),
() => fakeReader,
_ptableVersion,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 3,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, Stream3, 1, 4);
_tableIndex.Add(1, Stream2, 1, 5);
_tableIndex.Add(1, Stream1, 1, 6);
await Task.Delay(500);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
ReadIndex.Close();
ReadIndex.Dispose();
TableIndex.Close(removeFiles: false);
TableIndex = new TableIndex(GetFilePathFor("index"),
() => new HashListMemTable(maxSize: MaxEntriesInMemTable * 2),
maxSizeForMemory: MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
2,
2,
() => new TFChunkReader(Db, WriterCheckpoint, 0),
TableIndex,
new ByLengthHasher(),
new NoLRUCache <string, StreamCacheInfo>(),
additionalCommitChecks: true,
metastreamMaxCount: 1);
ReadIndex.Init(WriterCheckpoint.Read(), ChaserCheckpoint.Read());
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
ReadIndex.Close();
ReadIndex.Dispose();
TableIndex.Close(removeFiles: false);
var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 2, () => new TFChunkReader(Db, WriterCheckpoint));
TableIndex = new TableIndex(GetFilePathFor("index"),
() => new HashListMemTable(maxSize: MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
maxSizeForMemory: MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
new ByLengthHasher(),
0,
additionalCommitChecks: true,
metastreamMaxCount: 1);
ReadIndex.Init(ChaserCheckpoint.Read());
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
var indexDirectory = GetFilePathFor("index");
_logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormatFactory.Create(new() {
IndexDirectory = indexDirectory,
});
var writerCheckpoint = new InMemoryCheckpoint(0);
var chaserCheckpoint = new InMemoryCheckpoint(0);
var bus = new InMemoryBus("bus");
new IODispatcher(bus, new PublishEnvelope(bus));
_db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, writerCheckpoint, chaserCheckpoint));
_db.Open();
// create db
Writer = new TFChunkWriter(_db);
Writer.Open();
SetupDB();
Writer.Close();
Writer = null;
writerCheckpoint.Flush();
chaserCheckpoint.Write(writerCheckpoint.Read());
chaserCheckpoint.Flush();
var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 5,
() => new TFChunkReader(_db, _db.Config.WriterCheckpoint));
var lowHasher = _logFormat.LowHasher;
var highHasher = _logFormat.HighHasher;
var emptyStreamId = _logFormat.EmptyStreamId;
_tableIndex = new TableIndex <TStreamId>(indexDirectory, lowHasher, highHasher, emptyStreamId,
() => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
IndexBitnessVersion,
int.MaxValue,
Constants.PTableMaxReaderCountDefault,
MaxEntriesInMemTable);
_logFormat.StreamNamesProvider.SetTableIndex(_tableIndex);
var readIndex = new ReadIndex <TStreamId>(new NoopPublisher(),
readers,
_tableIndex,
_logFormat.StreamNameIndexConfirmer,
_logFormat.StreamIds,
_logFormat.StreamNamesProvider,
_logFormat.EmptyStreamId,
_logFormat.StreamIdValidator,
_logFormat.StreamIdSizer,
_logFormat.StreamExistenceFilter,
_logFormat.StreamExistenceFilterReader,
_logFormat.EventTypeIndexConfirmer,
streamInfoCacheCapacity: 100_000,
additionalCommitChecks: PerformAdditionalCommitChecks,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
replicationCheckpoint: _db.Config.ReplicationCheckpoint,
indexCheckpoint: _db.Config.IndexCheckpoint);
readIndex.IndexCommitter.Init(chaserCheckpoint.Read());
ReadIndex = readIndex;
_tableIndex.Close(false);
Writer = new TFChunkWriter(_db);
Writer.Open();
Given();
Writer.Close();
Writer = null;
writerCheckpoint.Flush();
chaserCheckpoint.Write(writerCheckpoint.Read());
chaserCheckpoint.Flush();
_tableIndex = new TableIndex <TStreamId>(indexDirectory, lowHasher, highHasher, emptyStreamId,
() => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
IndexBitnessVersion,
int.MaxValue,
Constants.PTableMaxReaderCountDefault,
MaxEntriesInMemTable);
readIndex = new ReadIndex <TStreamId>(new NoopPublisher(),
readers,
_tableIndex,
_logFormat.StreamNameIndexConfirmer,
_logFormat.StreamIds,
_logFormat.StreamNamesProvider,
_logFormat.EmptyStreamId,
_logFormat.StreamIdValidator,
_logFormat.StreamIdSizer,
_logFormat.StreamExistenceFilter,
_logFormat.StreamExistenceFilterReader,
_logFormat.EventTypeIndexConfirmer,
streamInfoCacheCapacity: 100_000,
additionalCommitChecks: PerformAdditionalCommitChecks,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
replicationCheckpoint: _db.Config.ReplicationCheckpoint,
indexCheckpoint: _db.Config.IndexCheckpoint);
readIndex.IndexCommitter.Init(chaserCheckpoint.Read());
ReadIndex = readIndex;
}
public override void TestFixtureTearDown()
{
_tableIndex.Close();
base.TestFixtureTearDown();
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var scavengeBlocker = new ManualResetEventSlim(false);
var scavengeStarted = new ManualResetEventSlim(false);
var fakeReader = new TFReaderLease(new FakeIndexReader(l =>
{
scavengeStarted.Set();
if (!scavengeBlocker.Wait(5000))
{
throw new Exception("Failed to continue.");
}
return(false);
}));
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
maxSizeForMemory: 2,
maxTablesPerLevel: 5);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-1", 0, 0);
_tableIndex.Add(1, "testStream-1", 1, 100);
_tableIndex.WaitForBackgroundTasks();
_log = new FakeTFScavengerLog();
var task = Task.Run(() => _tableIndex.Scavenge(_log, CancellationToken.None));
Assert.That(scavengeStarted.Wait(5000));
// Add enough for 2 more tables
_tableIndex.Add(1, "testStream-1", 2, 200);
_tableIndex.Add(1, "testStream-1", 3, 300);
_tableIndex.Add(1, "testStream-1", 4, 400);
_tableIndex.Add(1, "testStream-1", 5, 500);
// Release the scavenge process
scavengeBlocker.Set();
task.Wait();
// Check it's loadable.
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
maxSizeForMemory: 2,
maxTablesPerLevel: 5);
_tableIndex.Initialize(long.MaxValue);
}
public override Task TestFixtureTearDown()
{
_logFormat.Dispose();
_tableIndex.Close();
return(base.TestFixtureTearDown());
}
public override Task TestFixtureTearDown()
{
_tableIndex.Close();
return(base.TestFixtureTearDown());
}
public void TearDown()
{
_tableIndex.Close();
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var writerCheckpoint = new InMemoryCheckpoint(0);
var chaserCheckpoint = new InMemoryCheckpoint(0);
var bus = new InMemoryBus("bus");
new IODispatcher(bus, new PublishEnvelope(bus));
_db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
writerCheckpoint,
chaserCheckpoint,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
_db.Open();
// create db
Writer = new TFChunkWriter(_db);
Writer.Open();
SetupDB();
Writer.Close();
Writer = null;
writerCheckpoint.Flush();
chaserCheckpoint.Write(writerCheckpoint.Read());
chaserCheckpoint.Flush();
var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(_db, _db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
IndexBitnessVersion,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
_tableIndex,
EventStore.Core.Settings.ESConsts.StreamInfoCacheCapacity,
additionalCommitChecks: PerformAdditionalCommitChecks,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault);
ReadIndex.Init(chaserCheckpoint.Read());
_tableIndex.Close(false);
Writer = new TFChunkWriter(_db);
Writer.Open();
Given();
Writer.Close();
Writer = null;
writerCheckpoint.Flush();
chaserCheckpoint.Write(writerCheckpoint.Read());
chaserCheckpoint.Flush();
_tableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
IndexBitnessVersion,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
_tableIndex,
EventStore.Core.Settings.ESConsts.StreamInfoCacheCapacity,
additionalCommitChecks: PerformAdditionalCommitChecks,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault);
ReadIndex.Init(chaserCheckpoint.Read());
}