public void when_checkpoint_is_exactly_on_the_boundary_of_chunk_the_last_chunk_could_be_present()
{
var config = TFChunkHelper.CreateDbConfig(PathName, 200, chunkSize: 100);
using (var db = new TFChunkDb(config))
{
DbUtil.CreateMultiChunk(config, 0, 1, GetFilePathFor("chunk-000000.000000"));
DbUtil.CreateOngoingChunk(config, 2, GetFilePathFor("chunk-000002.000001"));
Assert.DoesNotThrow(() => db.Open(verifyHash: false));
Assert.IsNotNull(db.Manager.GetChunk(2));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000")));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000002.000001")));
Assert.AreEqual(2, Directory.GetFiles(PathName, "*").Length);
}
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_config = TFChunkHelper.CreateDbConfig(PathName, 11111, 5500, 5500, 5757, 1000);
DbUtil.CreateMultiChunk(_config, 0, 2, GetFilePathFor("chunk-000000.000001"));
DbUtil.CreateMultiChunk(_config, 0, 2, GetFilePathFor("chunk-000000.000002"));
DbUtil.CreateMultiChunk(_config, 3, 10, GetFilePathFor("chunk-000003.000001"));
DbUtil.CreateMultiChunk(_config, 3, 10, GetFilePathFor("chunk-000003.000002"));
DbUtil.CreateMultiChunk(_config, 7, 8, GetFilePathFor("chunk-000007.000001"));
DbUtil.CreateOngoingChunk(_config, 11, GetFilePathFor("chunk-000011.000000"));
var truncator = new TFChunkDbTruncator(_config);
truncator.TruncateDb(_config.TruncateCheckpoint.ReadNonFlushed());
}
public void when_checkpoint_is_on_boundary_of_chunk_last_chunk_is_preserved()
{
var config = TFChunkHelper.CreateDbConfig(PathName, 200, chunkSize: 100);
using (var db = new TFChunkDb(config)) {
DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000"));
DbUtil.CreateSingleChunk(config, 1, GetFilePathFor("chunk-000001.000001"));
DbUtil.CreateOngoingChunk(config, 2, GetFilePathFor("chunk-000002.000005"));
Assert.DoesNotThrow(() => db.Open(verifyHash: false));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000")));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000001.000001")));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000002.000005")));
Assert.AreEqual(3, Directory.GetFiles(PathName, "*").Length);
}
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
var indexDirectory = GetFilePathFor("index");
_logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormatFactory.Create(new() {
IndexDirectory = indexDirectory,
});
_mainBus = new InMemoryBus(nameof(when_starting_having_TFLog_with_no_epochs <TLogFormat, TStreamId>));
_mainBus.Subscribe(new AdHocHandler <SystemMessage.EpochWritten>(m => _published.Add(m)));
_db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, 0));
_db.Open();
_reader = new TFChunkReader(_db, _db.Config.WriterCheckpoint);
_writer = new TFChunkWriter(_db);
}
public void allows_last_chunk_to_be_multichunk_when_checkpoint_point_at_the_start_of_next_chunk()
{
var config = TFChunkHelper.CreateDbConfig(PathName, 4000, chunkSize: 1000);
using (var db = new TFChunkDb(config))
{
DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000"));
DbUtil.CreateMultiChunk(config, 1, 3, GetFilePathFor("chunk-000001.000001"));
Assert.DoesNotThrow(() => db.Open(verifyHash: false));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000")));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000001.000001")));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000004.000000")));
Assert.AreEqual(3, Directory.GetFiles(PathName, "*").Length);
}
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
string dbPath = Path.Combine(PathName, string.Format("mini-node-db-{0}", Guid.NewGuid()));
Bus = new InMemoryBus("bus");
IODispatcher = new IODispatcher(Bus, new PublishEnvelope(Bus));
if (!Directory.Exists(dbPath))
{
Directory.CreateDirectory(dbPath);
}
var writerCheckFilename = Path.Combine(dbPath, Checkpoint.Writer + ".chk");
var chaserCheckFilename = Path.Combine(dbPath, Checkpoint.Chaser + ".chk");
WriterCheckpoint = new MemoryMappedFileCheckpoint(writerCheckFilename, Checkpoint.Writer, cached: true);
ChaserCheckpoint = new MemoryMappedFileCheckpoint(chaserCheckFilename, Checkpoint.Chaser, cached: true);
Db = new TFChunkDb(TFChunkHelper.CreateDbConfig(dbPath, WriterCheckpoint, ChaserCheckpoint, TFConsts.ChunkSize));
Db.Open();
// create DB
Writer = new TFChunkWriter(Db);
Writer.Open();
WriteTestScenario();
Writer.Close();
Writer = null;
WriterCheckpoint.Flush();
ChaserCheckpoint.Write(WriterCheckpoint.Read());
ChaserCheckpoint.Flush();
Db.Close();
// start node with our created DB
Node = new MiniNode(PathName, inMemDb: false, dbPath: dbPath);
await Node.Start();
try {
await Given().WithTimeout();
} catch (Exception ex) {
throw new Exception("Given Failed", ex);
}
}
public void when_checkpoint_is_on_boundary_of_new_chunk_and_last_chunk_is_truncated_no_exception_is_thrown()
{
var config = TFChunkHelper.CreateDbConfig(PathName, 300, chunkSize: 100);
using (var db = new TFChunkDb(config))
{
DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000"));
DbUtil.CreateMultiChunk(config, 1, 2, GetFilePathFor("chunk-000001.000001"), physicalSize: 50, logicalSize: 150);
Assert.DoesNotThrow(() => db.Open(verifyHash: false));
Assert.IsNotNull(db.Manager.GetChunk(2));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000")));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000001.000001")));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000003.000000")));
Assert.AreEqual(3, Directory.GetFiles(PathName, "*").Length);
}
}
public void CreateDb(params Rec[] records)
{
if (DbRes != null)
{
DbRes.Db.Close();
}
var dbConfig = TFChunkHelper.CreateDbConfig(PathName, 0, chunkSize: 1024 * 1024);
var dbHelper = new TFChunkDbCreationHelper(dbConfig);
DbRes = dbHelper.Chunk(records).CreateDb();
DbRes.Db.Config.WriterCheckpoint.Flush();
DbRes.Db.Config.ChaserCheckpoint.Write(DbRes.Db.Config.WriterCheckpoint.Read());
DbRes.Db.Config.ChaserCheckpoint.Flush();
var readers = new ObjectPool <ITransactionFileReader>(
"Readers", 2, 2, () => new TFChunkReader(DbRes.Db, DbRes.Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.IndexV3, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
PTableVersions.IndexV3,
int.MaxValue,
Constants.PTableMaxReaderCountDefault,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: true,
metastreamMaxCount: _metastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
replicationCheckpoint: DbRes.Db.Config.ReplicationCheckpoint,
indexCheckpoint: DbRes.Db.Config.IndexCheckpoint);
((ReadIndex)ReadIndex).IndexCommitter.Init(DbRes.Db.Config.ChaserCheckpoint.Read());
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_config = TFChunkHelper.CreateDbConfig(PathName, 1711, 5500, 5500, 1111, 1000);
var rnd = new Random();
_file1Contents = new byte[_config.ChunkSize];
_file2Contents = new byte[_config.ChunkSize];
rnd.NextBytes(_file1Contents);
rnd.NextBytes(_file2Contents);
DbUtil.CreateSingleChunk(_config, 0, GetFilePathFor("chunk-000000.000001"), contents: _file1Contents);
DbUtil.CreateSingleChunk(_config, 1, GetFilePathFor("chunk-000001.000002"), contents: _file2Contents);
var truncator = new TFChunkDbTruncator(_config);
truncator.TruncateDb(_config.TruncateCheckpoint.ReadNonFlushed());
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
_mainBus = new InMemoryBus(nameof(when_starting_having_TFLog_with_existing_epochs <TLogFormat, TStreamId>));
_mainBus.Subscribe(new AdHocHandler <SystemMessage.EpochWritten>(m => _published.Add(m)));
_db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, 0));
_db.Open();
_reader = new TFChunkReader(_db, _db.Config.WriterCheckpoint);
_writer = new TFChunkWriter(_db);
_epochs = new List <EpochRecord>();
var lastPos = 0L;
for (int i = 0; i < 30; i++)
{
var epoch = WriteEpoch(GetNextEpoch(), lastPos, _instanceId);
_epochs.Add(epoch);
lastPos = epoch.EpochPosition;
}
}
public void temporary_files_are_removed()
{
var config = TFChunkHelper.CreateDbConfig(PathName, 150, chunkSize: 100);
using (var db = new TFChunkDb(config)) {
DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000"));
DbUtil.CreateOngoingChunk(config, 1, GetFilePathFor("chunk-000001.000001"));
File.Create(GetFilePathFor("bla")).Close();
File.Create(GetFilePathFor("bla.scavenge.tmp")).Close();
File.Create(GetFilePathFor("bla.tmp")).Close();
Assert.DoesNotThrow(() => db.Open(verifyHash: false));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000")));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000001.000001")));
Assert.IsTrue(File.Exists(GetFilePathFor("bla")));
Assert.AreEqual(3, Directory.GetFiles(PathName, "*").Length);
}
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = TFChunkHelper.CreateDbConfig(PathName, 0, chunkSize: 1024 * 1024);
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = dbCreationHelper
.Chunk().CompleteLastChunk()
.Chunk().CompleteLastChunk()
.Chunk()
.CreateDb();
_dbResult.Db.Config.WriterCheckpoint.Flush();
_dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
_dbResult.Db.Config.ChaserCheckpoint.Flush();
Log = new FakeTFScavengerLog();
TfChunkScavenger = new TFChunkScavenger(_dbResult.Db, Log, new FakeTableIndex(), new FakeReadIndex(_ => false));
When();
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
var dbConfig = TFChunkHelper.CreateDbConfig(PathName, 0, chunkSize: 1024 * 1024);
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
DbRes = CreateDb(dbCreationHelper);
DbRes.Db.Config.WriterCheckpoint.Flush();
DbRes.Db.Config.ChaserCheckpoint.Write(DbRes.Db.Config.WriterCheckpoint.Read());
DbRes.Db.Config.ChaserCheckpoint.Flush();
var readers = new ObjectPool <ITransactionFileReader>(
"Readers", 2, 2, () => new TFChunkReader(DbRes.Db, DbRes.Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.IndexV2, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
PTableVersions.IndexV2,
int.MaxValue,
Constants.PTableMaxReaderCountDefault,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: true,
metastreamMaxCount: _metastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
replicationCheckpoint: DbRes.Db.Config.ReplicationCheckpoint);
ReadIndex.Init(DbRes.Db.Config.ChaserCheckpoint.Read());
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
var indexDirectory = GetFilePathFor("index");
_logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormatFactory.Create(new() {
IndexDirectory = indexDirectory,
});
_mainBus = new InMemoryBus(nameof(when_having_an_epoch_manager_and_empty_tf_log <TLogFormat, TStreamId>));
_mainBus.Subscribe(new AdHocHandler <SystemMessage.EpochWritten>(m => _published.Add(m)));
_db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, 0));
_db.Open();
_reader = new TFChunkReader(_db, _db.Config.WriterCheckpoint);
_writer = new TFChunkWriter(_db);
_epochManager = GetManager();
_epochManager.Init();
_cache = GetCache(_epochManager);
Assert.NotNull(_cache);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = TFChunkHelper.CreateDbConfig(PathName, 0, chunkSize: 1024 * 1024);
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = CreateDb(dbCreationHelper);
_keptRecords = KeptRecords(_dbResult);
_dbResult.Db.Config.WriterCheckpoint.Flush();
_dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
_dbResult.Db.Config.ChaserCheckpoint.Flush();
var indexPath = Path.Combine(PathName, "index");
var readerPool = new ObjectPool <ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
var tableIndex = new TableIndex(indexPath, lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.IndexV3, maxSize: 200),
() => new TFReaderLease(readerPool),
PTableVersions.IndexV3,
maxSizeForMemory: 100,
maxTablesPerLevel: 2);
ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, 100, true, _metastreamMaxCount, Opts.HashCollisionReadLimitDefault, Opts.SkipIndexScanOnReadsDefault, _dbResult.Db.Config.ReplicationCheckpoint);
ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read());
//var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount);
var bus = new InMemoryBus("Bus");
var ioDispatcher = new IODispatcher(bus, new PublishEnvelope(bus));
var scavenger = new TFChunkScavenger(_dbResult.Db, ioDispatcher, tableIndex, ReadIndex, Guid.NewGuid(), "fakeNodeIp",
unsafeIgnoreHardDeletes: UnsafeIgnoreHardDelete());
scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false);
}
private void ReOpenDb()
{
Db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, WriterCheckpoint, ChaserCheckpoint));
Db.Open();
var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 5,
() => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
var lowHasher = _logFormat.LowHasher;
var highHasher = _logFormat.HighHasher;
var emptyStreamId = _logFormat.EmptyStreamId;
TableIndex = new TableIndex <TStreamId>(Path.Combine(PathName, "index"), lowHasher, highHasher, emptyStreamId,
() => new HashListMemTable(PTableVersions.IndexV3, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
PTableVersions.IndexV3,
int.MaxValue,
Constants.PTableMaxReaderCountDefault,
MaxEntriesInMemTable);
var readIndex = new ReadIndex <TStreamId>(new NoopPublisher(),
readers,
TableIndex,
_logFormat.StreamIds,
_logFormat.StreamNamesProvider,
_logFormat.EmptyStreamId,
_logFormat.StreamIdValidator,
_logFormat.StreamIdSizer,
0,
additionalCommitChecks: true,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
replicationCheckpoint: Db.Config.ReplicationCheckpoint,
indexCheckpoint: Db.Config.IndexCheckpoint);
readIndex.IndexCommitter.Init(ChaserCheckpoint.Read());
ReadIndex = new TestReadIndex <TStreamId>(readIndex, _logFormat.StreamNameIndex);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
var dbConfig = TFChunkHelper.CreateDbConfig(PathName, 0, chunkSize: 1024 * 1024);
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = CreateDb(dbCreationHelper);
_keptRecords = KeptRecords(_dbResult);
_dbResult.Db.Config.WriterCheckpoint.Flush();
_dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
_dbResult.Db.Config.ChaserCheckpoint.Flush();
var indexPath = Path.Combine(PathName, "index");
var readerPool = new ObjectPool <ITransactionFileReader>(
"ReadIndex readers pool", Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault,
() => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
var tableIndex = new TableIndex(indexPath, lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.IndexV3, maxSize: 200),
() => new TFReaderLease(readerPool),
PTableVersions.IndexV3,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 100,
maxTablesPerLevel: 2);
ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, 100, true, _metastreamMaxCount,
Opts.HashCollisionReadLimitDefault, Opts.SkipIndexScanOnReadsDefault,
_dbResult.Db.Config.ReplicationCheckpoint, _dbResult.Db.Config.IndexCheckpoint);
((ReadIndex)ReadIndex).IndexCommitter.Init(_dbResult.Db.Config.WriterCheckpoint.Read());
var scavenger = new TFChunkScavenger(_dbResult.Db, new FakeTFScavengerLog(), tableIndex, ReadIndex,
unsafeIgnoreHardDeletes: UnsafeIgnoreHardDelete());
await scavenger.Scavenge(alwaysKeepScavenged : true, mergeChunks : false);
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
var indexDirectory = GetFilePathFor("index");
_logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormatFactory.Create(new() {
IndexDirectory = indexDirectory,
});
var writerCheckpoint = new InMemoryCheckpoint(0);
var chaserCheckpoint = new InMemoryCheckpoint(0);
var bus = new InMemoryBus("bus");
new IODispatcher(bus, new PublishEnvelope(bus));
_db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, writerCheckpoint, chaserCheckpoint));
_db.Open();
// create db
Writer = new TFChunkWriter(_db);
Writer.Open();
SetupDB();
Writer.Close();
Writer = null;
writerCheckpoint.Flush();
chaserCheckpoint.Write(writerCheckpoint.Read());
chaserCheckpoint.Flush();
var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 5,
() => new TFChunkReader(_db, _db.Config.WriterCheckpoint));
var lowHasher = _logFormat.LowHasher;
var highHasher = _logFormat.HighHasher;
var emptyStreamId = _logFormat.EmptyStreamId;
_tableIndex = new TableIndex <TStreamId>(indexDirectory, lowHasher, highHasher, emptyStreamId,
() => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
IndexBitnessVersion,
int.MaxValue,
Constants.PTableMaxReaderCountDefault,
MaxEntriesInMemTable);
_logFormat.StreamNamesProvider.SetTableIndex(_tableIndex);
var readIndex = new ReadIndex <TStreamId>(new NoopPublisher(),
readers,
_tableIndex,
_logFormat.StreamNameIndexConfirmer,
_logFormat.StreamIds,
_logFormat.StreamNamesProvider,
_logFormat.EmptyStreamId,
_logFormat.StreamIdValidator,
_logFormat.StreamIdSizer,
_logFormat.StreamExistenceFilter,
_logFormat.StreamExistenceFilterReader,
_logFormat.EventTypeIndexConfirmer,
streamInfoCacheCapacity: 100_000,
additionalCommitChecks: PerformAdditionalCommitChecks,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
replicationCheckpoint: _db.Config.ReplicationCheckpoint,
indexCheckpoint: _db.Config.IndexCheckpoint);
readIndex.IndexCommitter.Init(chaserCheckpoint.Read());
ReadIndex = readIndex;
_tableIndex.Close(false);
Writer = new TFChunkWriter(_db);
Writer.Open();
Given();
Writer.Close();
Writer = null;
writerCheckpoint.Flush();
chaserCheckpoint.Write(writerCheckpoint.Read());
chaserCheckpoint.Flush();
_tableIndex = new TableIndex <TStreamId>(indexDirectory, lowHasher, highHasher, emptyStreamId,
() => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
IndexBitnessVersion,
int.MaxValue,
Constants.PTableMaxReaderCountDefault,
MaxEntriesInMemTable);
readIndex = new ReadIndex <TStreamId>(new NoopPublisher(),
readers,
_tableIndex,
_logFormat.StreamNameIndexConfirmer,
_logFormat.StreamIds,
_logFormat.StreamNamesProvider,
_logFormat.EmptyStreamId,
_logFormat.StreamIdValidator,
_logFormat.StreamIdSizer,
_logFormat.StreamExistenceFilter,
_logFormat.StreamExistenceFilterReader,
_logFormat.EventTypeIndexConfirmer,
streamInfoCacheCapacity: 100_000,
additionalCommitChecks: PerformAdditionalCommitChecks,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
replicationCheckpoint: _db.Config.ReplicationCheckpoint,
indexCheckpoint: _db.Config.IndexCheckpoint);
readIndex.IndexCommitter.Init(chaserCheckpoint.Read());
ReadIndex = readIndex;
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
var writerCheckpoint = new InMemoryCheckpoint(0);
var chaserCheckpoint = new InMemoryCheckpoint(0);
var bus = new InMemoryBus("bus");
new IODispatcher(bus, new PublishEnvelope(bus));
_db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, writerCheckpoint, chaserCheckpoint));
_db.Open();
// create db
Writer = new TFChunkWriter(_db);
Writer.Open();
SetupDB();
Writer.Close();
Writer = null;
writerCheckpoint.Flush();
chaserCheckpoint.Write(writerCheckpoint.Read());
chaserCheckpoint.Flush();
var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 5,
() => new TFChunkReader(_db, _db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
IndexBitnessVersion,
int.MaxValue,
Constants.PTableMaxReaderCountDefault,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
_tableIndex,
EventStore.Core.Settings.ESConsts.StreamInfoCacheCapacity,
additionalCommitChecks: PerformAdditionalCommitChecks,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
replicationCheckpoint: _db.Config.ReplicationCheckpoint);
((ReadIndex)ReadIndex).IndexCommitter.Init(chaserCheckpoint.Read());
_tableIndex.Close(false);
Writer = new TFChunkWriter(_db);
Writer.Open();
Given();
Writer.Close();
Writer = null;
writerCheckpoint.Flush();
chaserCheckpoint.Write(writerCheckpoint.Read());
chaserCheckpoint.Flush();
_tableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
IndexBitnessVersion,
int.MaxValue,
Constants.PTableMaxReaderCountDefault,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
_tableIndex,
EventStore.Core.Settings.ESConsts.StreamInfoCacheCapacity,
additionalCommitChecks: PerformAdditionalCommitChecks,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
replicationCheckpoint: _db.Config.ReplicationCheckpoint);
((ReadIndex)ReadIndex).IndexCommitter.Init(chaserCheckpoint.Read());
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
var indexDirectory = GetFilePathFor("index");
_logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormatFactory.Create(new() {
IndexDirectory = indexDirectory,
});
_recordFactory = _logFormat.RecordFactory;
_streamNameIndex = _logFormat.StreamNameIndex;
_eventTypeIndex = _logFormat.EventTypeIndex;
WriterCheckpoint = new InMemoryCheckpoint(0);
ChaserCheckpoint = new InMemoryCheckpoint(0);
Db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, WriterCheckpoint, ChaserCheckpoint,
replicationCheckpoint: new InMemoryCheckpoint(-1), chunkSize: _chunkSize));
Db.Open();
// create db
Writer = new TFChunkWriter(Db);
Writer.Open();
WriteTestScenario();
Writer.Close();
Writer = null;
WriterCheckpoint.Flush();
ChaserCheckpoint.Write(WriterCheckpoint.Read());
ChaserCheckpoint.Flush();
var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 5,
() => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
var lowHasher = _logFormat.LowHasher;
var highHasher = _logFormat.HighHasher;
var emptyStreamId = _logFormat.EmptyStreamId;
TableIndex = new TableIndex <TStreamId>(indexDirectory, lowHasher, highHasher, emptyStreamId,
() => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
IndexBitnessVersion,
int.MaxValue,
Constants.PTableMaxReaderCountDefault,
MaxEntriesInMemTable);
_logFormat.StreamNamesProvider.SetTableIndex(TableIndex);
var readIndex = new ReadIndex <TStreamId>(new NoopPublisher(),
readers,
TableIndex,
_logFormat.StreamNameIndexConfirmer,
_logFormat.StreamIds,
_logFormat.StreamNamesProvider,
_logFormat.EmptyStreamId,
_logFormat.StreamIdValidator,
_logFormat.StreamIdSizer,
_logFormat.StreamExistenceFilter,
_logFormat.StreamExistenceFilterReader,
_logFormat.EventTypeIndexConfirmer,
streamInfoCacheCapacity: StreamInfoCacheCapacity,
additionalCommitChecks: PerformAdditionalCommitChecks,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
replicationCheckpoint: Db.Config.ReplicationCheckpoint,
indexCheckpoint: Db.Config.IndexCheckpoint);
readIndex.IndexCommitter.Init(ChaserCheckpoint.Read());
ReadIndex = readIndex;
// scavenge must run after readIndex is built
if (_scavenge)
{
if (_completeLastChunkOnScavenge)
{
Db.Manager.GetChunk(Db.Manager.ChunksCount - 1).Complete();
}
_scavenger = new TFChunkScavenger <TStreamId>(Db, new FakeTFScavengerLog(), TableIndex, ReadIndex, _logFormat.Metastreams);
await _scavenger.Scavenge(alwaysKeepScavenged : true, mergeChunks : _mergeChunks);
}
}
