protected override DbResult CreateDb(TFChunkDbCreationHelper dbCreator)
{
return dbCreator.Chunk(Rec.Prepare(0, "test"),
Rec.Commit(0, "test"),
Rec.Delete(2, "test"),
Rec.Commit(2, "test"))
.CreateDb();
}
protected override DbResult CreateDb(TFChunkDbCreationHelper dbCreator)
{
return dbCreator
.Chunk(Rec.Create(0, "bla"), Rec.Prepare(0, "bla"), Rec.Commit(0, "bla"))
.Chunk(Rec.Delete(1, "bla"), Rec.Commit(1, "bla"))
.CompleteLastChunk()
.CreateDb();
}
protected override DbResult CreateDb(TFChunkDbCreationHelper dbCreator)
{
return dbCreator.Chunk(Rec.Prepare(0, "$$test", "0", metadata: new StreamMetadata(10, null, null, null)),
Rec.Prepare(0, "$$test", "1", metadata: new StreamMetadata(9, null, null, null)),
Rec.Prepare(0, "$$test", "2", metadata: new StreamMetadata(8, null, null, null)),
Rec.Prepare(0, "$$test", "3", metadata: new StreamMetadata(7, null, null, null)),
Rec.Prepare(0, "$$test", "4", metadata: new StreamMetadata(6, null, null, null)),
Rec.Commit(0, "$$test"))
.CreateDb();
}
protected override DbResult CreateDb(TFChunkDbCreationHelper dbCreator)
{
return dbCreator.Chunk(Rec.Prepare(0, "$$test", "0", metadata: new StreamMetadata(maxCount: 10)),
Rec.Prepare(0, "$$test", "1", metadata: new StreamMetadata(maxCount: 9)),
Rec.Prepare(0, "$$test", "2", metadata: new StreamMetadata(maxCount: 8)),
Rec.Prepare(0, "$$test", "3", metadata: new StreamMetadata(maxCount: 7)),
Rec.Prepare(0, "$$test", "4", metadata: new StreamMetadata(maxCount: 6)),
Rec.Commit(0, "$$test"))
.CreateDb();
}
protected override DbResult CreateDb(TFChunkDbCreationHelper dbCreator)
{
return dbCreator.Chunk(Rec.Prepare(0, "test"),
Rec.Commit(0, "test"),
Rec.Prepare(1, "$$test", metadata: new StreamMetadata(2, null, null, null)),
Rec.Commit(1, "$$test"),
Rec.Delete(2, "test"),
Rec.Commit(2, "test"))
.CreateDb();
}
protected override DbResult CreateDb(TFChunkDbCreationHelper dbCreator)
{
return dbCreator
.Chunk(Rec.Prepare(0, "$$bla", metadata: new StreamMetadata(10, null, null, null)),
Rec.Prepare(0, "$$bla", metadata: new StreamMetadata(5, null, null, null)),
Rec.Prepare(0, "$$bla", metadata: new StreamMetadata(3, null, null, null)),
Rec.Prepare(0, "$$bla", metadata: new StreamMetadata(2, null, null, null)),
Rec.Commit(0, "$$bla"))
.CompleteLastChunk()
.CreateDb();
}
protected override DbResult CreateDb(TFChunkDbCreationHelper dbCreator)
{
return dbCreator.Chunk(Rec.Prepare(0, "$$test", metadata: new StreamMetadata(tempStream: true)),
Rec.Commit(0, "$$test"))
.Chunk(Rec.Prepare(1, "test"),
Rec.Commit(1, "test"),
Rec.Prepare(2, "test"),
Rec.Commit(2, "test"),
Rec.Prepare(3, "$$test", metadata: new StreamMetadata(truncateBefore: EventNumber.DeletedStream, tempStream: true)),
Rec.Commit(3, "$$test"))
.CompleteLastChunk()
.CreateDb();
}
protected override DbResult CreateDb(TFChunkDbCreationHelper dbCreator)
{
return dbCreator
.Chunk(Rec.Create(0, "bla", new StreamMetadata(null, TimeSpan.FromMinutes(1))),
Rec.Commit(0, "bla"),
Rec.Prepare(2, "bla", timestamp: DateTime.UtcNow - TimeSpan.FromMinutes(20)),
Rec.Prepare(2, "bla", timestamp: DateTime.UtcNow - TimeSpan.FromMinutes(10)),
Rec.Prepare(2, "bla", timestamp: DateTime.UtcNow - TimeSpan.FromMinutes(5)),
Rec.Prepare(2, "bla", timestamp: DateTime.UtcNow - TimeSpan.FromMinutes(3)),
Rec.Commit(2, "bla"))
.CompleteLastChunk()
.CreateDb();
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024*1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = CreateDb(dbCreationHelper);
_keptRecords = KeptRecords(_dbResult);
var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams);
var scavenger = new TFChunkScavenger(_dbResult.Db, scavengeReadIndex);
scavenger.Scavenge(alwaysKeepScavenged: true);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024*1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = CreateDb(dbCreationHelper);
_keptRecords = KeptRecords(_dbResult);
_dbResult.Db.Config.WriterCheckpoint.Flush();
_dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
_dbResult.Db.Config.ChaserCheckpoint.Flush();
var indexPath = Path.Combine(PathName, "index");
var readerPool = new ObjectPool<ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
var tableIndex = new TableIndex(indexPath, lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.Index64Bit, maxSize: 200),
() => new TFReaderLease(readerPool),
PTableVersions.Index64Bit,
maxSizeForMemory: 100,
maxTablesPerLevel: 2);
ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, 100, true, _metastreamMaxCount, Opts.HashCollisionReadLimitDefault);
ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read());
//var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount);
var bus = new InMemoryBus("Bus");
var ioDispatcher = new IODispatcher(bus, new PublishEnvelope(bus));
var scavenger = new TFChunkScavenger(_dbResult.Db, ioDispatcher, tableIndex, ReadIndex, Guid.NewGuid(), "fakeNodeIp",
unsafeIgnoreHardDeletes: UnsafeIgnoreHardDelete());
scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024*1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
DbRes = CreateDb(dbCreationHelper);
DbRes.Db.Config.WriterCheckpoint.Flush();
DbRes.Db.Config.ChaserCheckpoint.Write(DbRes.Db.Config.WriterCheckpoint.Read());
DbRes.Db.Config.ChaserCheckpoint.Flush();
var readers = new ObjectPool<ITransactionFileReader>(
"Readers", 2, 2, () => new TFChunkReader(DbRes.Db, DbRes.Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.Index64Bit, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
PTableVersions.Index64Bit,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: true,
metastreamMaxCount: _metastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault);
ReadIndex.Init(DbRes.Db.Config.ChaserCheckpoint.Read());
}
protected override DbResult CreateDb(TFChunkDbCreationHelper dbCreator)
{
return dbCreator
.Chunk(Rec.Prepare(3, "$$bla", metadata: new StreamMetadata(5, TimeSpan.FromMinutes(5), null, null)),
Rec.Commit(3, "$$bla"),
Rec.Prepare(0, "bla"),
Rec.Commit(0, "bla"),
Rec.Prepare(1, "bla", timestamp: DateTime.UtcNow - TimeSpan.FromMinutes(100)),
Rec.Prepare(1, "bla", timestamp: DateTime.UtcNow - TimeSpan.FromMinutes(90)),
Rec.Prepare(1, "bla", timestamp: DateTime.UtcNow - TimeSpan.FromMinutes(60)),
Rec.Prepare(1, "bla", timestamp: DateTime.UtcNow - TimeSpan.FromMinutes(40)),
Rec.Prepare(1, "bla", timestamp: DateTime.UtcNow - TimeSpan.FromMinutes(30)),
Rec.Commit(1, "bla"),
Rec.Prepare(2, "bla", timestamp: DateTime.UtcNow - TimeSpan.FromMinutes(20)),
Rec.Prepare(2, "bla", timestamp: DateTime.UtcNow - TimeSpan.FromMinutes(3)),
Rec.Prepare(2, "bla", timestamp: DateTime.UtcNow - TimeSpan.FromMinutes(2)),
Rec.Prepare(2, "bla", timestamp: DateTime.UtcNow - TimeSpan.FromMinutes(1)),
Rec.Commit(2, "bla"))
.CompleteLastChunk()
.CreateDb();
}
protected override DbResult CreateDb(TFChunkDbCreationHelper dbCreator)
{
return dbCreator
.Chunk(Rec.Prepare(0, "$$bla", metadata: new StreamMetadata(truncateBefore: 7)),
Rec.Commit(0, "$$bla"),
Rec.Prepare(1, "bla"), // event 0
Rec.Commit(1, "bla"),
Rec.Prepare(2, "bla"), // event 1
Rec.Prepare(2, "bla"), // event 2
Rec.Prepare(2, "bla"), // event 3
Rec.Prepare(2, "bla"), // event 4
Rec.Prepare(2, "bla"), // event 5
Rec.Commit(2, "bla"),
Rec.Prepare(3, "bla"), // event 6
Rec.Prepare(3, "bla"), // event 7
Rec.Prepare(3, "bla"), // event 8
Rec.Prepare(3, "bla"), // event 9
Rec.Commit(3, "bla"))
.CompleteLastChunk()
.CreateDb();
}
protected override DbResult CreateDb(TFChunkDbCreationHelper dbCreator)
{
return dbCreator
.Chunk(Rec.Prepare(0, "$$bla", metadata: new StreamMetadata(maxCount: 3)),
Rec.Commit(0, "$$bla"),
Rec.Prepare(1, "bla"),
Rec.Commit(1, "bla"),
Rec.Prepare(2, "bla"),
Rec.Prepare(2, "bla"),
Rec.Prepare(2, "bla"),
Rec.Prepare(2, "bla"),
Rec.Prepare(2, "bla"),
Rec.Commit(2, "bla"),
Rec.Prepare(3, "bla"),
Rec.Prepare(3, "bla"),
Rec.Prepare(3, "bla"),
Rec.Prepare(3, "bla"),
Rec.Commit(3, "bla"))
.CompleteLastChunk()
.CreateDb();
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024*1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = CreateDb(dbCreationHelper);
_keptRecords = KeptRecords(_dbResult);
_dbResult.Db.Config.WriterCheckpoint.Flush();
_dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
_dbResult.Db.Config.ChaserCheckpoint.Flush();
var indexPath = Path.Combine(PathName, "index");
var readerPool = new ObjectPool<ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint));
var tableIndex = new TableIndex(indexPath,
() => new HashListMemTable(maxSize: 200),
() => new TFReaderLease(readerPool),
maxSizeForMemory: 100,
maxTablesPerLevel: 2);
var hasher = new XXHashUnsafe();
ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, hasher, 100, true, _metastreamMaxCount);
ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read());
//var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount);
var scavenger = new TFChunkScavenger(_dbResult.Db, tableIndex, hasher, ReadIndex);
scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024*1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = CreateDb(dbCreationHelper);
_keptRecords = KeptRecords(_dbResult);
_dbResult.Db.Config.WriterCheckpoint.Flush();
_dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
_dbResult.Db.Config.ChaserCheckpoint.Flush();
var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount);
var scavenger = new TFChunkScavenger(_dbResult.Db, scavengeReadIndex);
scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024*1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
DbRes = CreateDb(dbCreationHelper);
DbRes.Db.Config.WriterCheckpoint.Flush();
DbRes.Db.Config.ChaserCheckpoint.Write(DbRes.Db.Config.WriterCheckpoint.Read());
DbRes.Db.Config.ChaserCheckpoint.Flush();
TableIndex = new TableIndex(GetFilePathFor("index"),
() => new HashListMemTable(MaxEntriesInMemTable * 2),
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
2,
2,
() => new TFChunkReader(DbRes.Db, DbRes.Db.Config.WriterCheckpoint),
TableIndex,
new ByLengthHasher(),
new NoLRUCache<string, StreamCacheInfo>(),
additionalCommitChecks: true,
metastreamMaxCount: _metastreamMaxCount);
ReadIndex.Init(DbRes.Db.Config.WriterCheckpoint.Read(), DbRes.Db.Config.ChaserCheckpoint.Read());
}
protected abstract DbResult CreateDb(TFChunkDbCreationHelper dbCreator);
protected abstract DbResult CreateDb(TFChunkDbCreationHelper <TLogFormat, TStreamId> dbCreator);
protected abstract DbResult CreateDb(TFChunkDbCreationHelper dbCreator);