protected override LogRecord[][] KeptRecords(DbResult dbResult)
{
return new[]
{
dbResult.Recs[0].Where((x, i) => new[] {1,6,7,8}.Contains(i)).ToArray(),
};
}
protected override LogRecord[][] KeptRecords(DbResult dbResult)
{
return new[]
{
dbResult.Recs[0].Where((x, i) => new[] {0, 1, 11, 12, 13, 14}.Contains(i)).ToArray()
};
}
protected override LogRecord[][] KeptRecords(DbResult dbResult)
{
return new[]
{
dbResult.Recs[0].Where((x, i) => i >= 3).ToArray()
};
}
protected override LogRecord[][] KeptRecords(DbResult dbResult)
{
return new []
{
new LogRecord[0],
new LogRecord[0]
};
}
protected override LogRecord[][] KeptRecords(DbResult dbResult)
{
return new[]
{
new[]{ dbResult.Recs[0][2] }, // first prepare in commit that is in different chunk
dbResult.Recs[1].Where((x, i) => i >= 3).ToArray(),
};
}
protected override LogRecord[][] KeptRecords(DbResult dbResult)
{
return new[]
{
dbResult.Recs[0].Where((x, i) => i <= 2).ToArray(),
dbResult.Recs[1].Where((x, i) => i == 6).ToArray(),
dbResult.Recs[2]
};
}
protected override LogRecord[][] KeptRecords(DbResult dbResult)
{
return new[]
{
new [] { dbResult.Recs[0][2] },
new [] { dbResult.Recs[1][6] }, // commit
dbResult.Recs[2]
};
}
protected override LogRecord[][] KeptRecords(DbResult dbResult)
{
return new[]
{
new LogRecord[0],
new[]
{
dbResult.Recs[1][2],
dbResult.Recs[1][3],
dbResult.Recs[1][4],
dbResult.Recs[1][5]
}
};
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
var dbConfig = TFChunkHelper.CreateSizedDbConfig(PathName, 0, chunkSize: 1024 * 1024);
var dbCreationHelper = new TFChunkDbCreationHelper <TLogFormat, TStreamId>(dbConfig);
_dbResult = CreateDb(dbCreationHelper);
_keptRecords = KeptRecords(_dbResult);
_dbResult.Db.Config.WriterCheckpoint.Flush();
_dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
_dbResult.Db.Config.ChaserCheckpoint.Flush();
var logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormat;
var indexPath = Path.Combine(PathName, "index");
var readerPool = new ObjectPool <ITransactionFileReader>(
"ReadIndex readers pool", Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault,
() => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint));
var lowHasher = logFormat.LowHasher;
var highHasher = logFormat.HighHasher;
var emptyStreamId = logFormat.EmptyStreamId;
var tableIndex = new TableIndex <TStreamId>(indexPath, lowHasher, highHasher, emptyStreamId,
() => new HashListMemTable(PTableVersions.IndexV3, maxSize: 200),
() => new TFReaderLease(readerPool),
PTableVersions.IndexV3,
5, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: 100,
maxTablesPerLevel: 2);
var readIndex = new ReadIndex <TStreamId>(new NoopPublisher(), readerPool, tableIndex,
logFormat.StreamIds,
logFormat.StreamNamesProvider,
logFormat.EmptyStreamId,
logFormat.StreamIdValidator,
logFormat.StreamIdSizer,
100, true, _metastreamMaxCount,
Opts.HashCollisionReadLimitDefault, Opts.SkipIndexScanOnReadsDefault,
_dbResult.Db.Config.ReplicationCheckpoint, _dbResult.Db.Config.IndexCheckpoint);
readIndex.IndexCommitter.Init(_dbResult.Db.Config.WriterCheckpoint.Read());
ReadIndex = new TestReadIndex <TStreamId>(readIndex, logFormat.StreamNameIndex);
var scavenger = new TFChunkScavenger <TStreamId>(_dbResult.Db, new FakeTFScavengerLog(), tableIndex, ReadIndex,
logFormat.SystemStreams,
unsafeIgnoreHardDeletes: UnsafeIgnoreHardDelete());
await scavenger.Scavenge(alwaysKeepScavenged : true, mergeChunks : false);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024 * 1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = CreateDb(dbCreationHelper);
_keptRecords = KeptRecords(_dbResult);
_dbResult.Db.Config.WriterCheckpoint.Flush();
_dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
_dbResult.Db.Config.ChaserCheckpoint.Flush();
var indexPath = Path.Combine(PathName, "index");
var readerPool = new ObjectPool <ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
var tableIndex = new TableIndex(indexPath, lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.IndexV3, maxSize: 200),
() => new TFReaderLease(readerPool),
PTableVersions.IndexV3,
maxSizeForMemory: 100,
maxTablesPerLevel: 2);
ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, 100, true, _metastreamMaxCount, Opts.HashCollisionReadLimitDefault);
ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read());
//var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount);
var bus = new InMemoryBus("Bus");
var ioDispatcher = new IODispatcher(bus, new PublishEnvelope(bus));
var scavenger = new TFChunkScavenger(_dbResult.Db, ioDispatcher, tableIndex, ReadIndex, Guid.NewGuid(), "fakeNodeIp",
unsafeIgnoreHardDeletes: UnsafeIgnoreHardDelete());
scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024*1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = CreateDb(dbCreationHelper);
_keptRecords = KeptRecords(_dbResult);
var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams);
var scavenger = new TFChunkScavenger(_dbResult.Db, scavengeReadIndex);
scavenger.Scavenge(alwaysKeepScavenged: true);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024*1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = CreateDb(dbCreationHelper);
_keptRecords = KeptRecords(_dbResult);
_dbResult.Db.Config.WriterCheckpoint.Flush();
_dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
_dbResult.Db.Config.ChaserCheckpoint.Flush();
var indexPath = Path.Combine(PathName, "index");
var readerPool = new ObjectPool<ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
var tableIndex = new TableIndex(indexPath, lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.Index64Bit, maxSize: 200),
() => new TFReaderLease(readerPool),
PTableVersions.Index64Bit,
maxSizeForMemory: 100,
maxTablesPerLevel: 2);
ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, 100, true, _metastreamMaxCount, Opts.HashCollisionReadLimitDefault);
ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read());
//var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount);
var bus = new InMemoryBus("Bus");
var ioDispatcher = new IODispatcher(bus, new PublishEnvelope(bus));
var scavenger = new TFChunkScavenger(_dbResult.Db, ioDispatcher, tableIndex, ReadIndex, Guid.NewGuid(), "fakeNodeIp",
unsafeIgnoreHardDeletes: UnsafeIgnoreHardDelete());
scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024*1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
DbRes = CreateDb(dbCreationHelper);
DbRes.Db.Config.WriterCheckpoint.Flush();
DbRes.Db.Config.ChaserCheckpoint.Write(DbRes.Db.Config.WriterCheckpoint.Read());
DbRes.Db.Config.ChaserCheckpoint.Flush();
var readers = new ObjectPool<ITransactionFileReader>(
"Readers", 2, 2, () => new TFChunkReader(DbRes.Db, DbRes.Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.Index64Bit, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
PTableVersions.Index64Bit,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: true,
metastreamMaxCount: _metastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault);
ReadIndex.Init(DbRes.Db.Config.ChaserCheckpoint.Read());
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024 * 1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = CreateDb(dbCreationHelper);
_keptRecords = KeptRecords(_dbResult);
_dbResult.Db.Config.WriterCheckpoint.Flush();
_dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
_dbResult.Db.Config.ChaserCheckpoint.Flush();
var indexPath = Path.Combine(PathName, "index");
var readerPool = new ObjectPool <ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint));
var tableIndex = new TableIndex(indexPath,
() => new HashListMemTable(maxSize: 200),
() => new TFReaderLease(readerPool),
maxSizeForMemory: 100,
maxTablesPerLevel: 2);
var hasher = new XXHashUnsafe();
ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, hasher, 100, true, _metastreamMaxCount);
ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read());
//var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount);
var scavenger = new TFChunkScavenger(_dbResult.Db, tableIndex, hasher, ReadIndex);
scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = TFChunkHelper.CreateDbConfig(PathName, 0, chunkSize: 1024 * 1024);
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = dbCreationHelper
.Chunk().CompleteLastChunk()
.Chunk().CompleteLastChunk()
.Chunk()
.CreateDb();
_dbResult.Db.Config.WriterCheckpoint.Flush();
_dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
_dbResult.Db.Config.ChaserCheckpoint.Flush();
Log = new FakeTFScavengerLog();
TfChunkScavenger = new TFChunkScavenger(_dbResult.Db, Log, new FakeTableIndex(), new FakeReadIndex(_ => false));
When();
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = TFChunkHelper.CreateDbConfig(PathName, 0, chunkSize: 1024 * 1024);
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = CreateDb(dbCreationHelper);
_keptRecords = KeptRecords(_dbResult);
_dbResult.Db.Config.WriterCheckpoint.Flush();
_dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
_dbResult.Db.Config.ChaserCheckpoint.Flush();
var indexPath = Path.Combine(PathName, "index");
var readerPool = new ObjectPool <ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
var tableIndex = new TableIndex(indexPath, lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.IndexV3, maxSize: 200),
() => new TFReaderLease(readerPool),
PTableVersions.IndexV3,
5,
maxSizeForMemory: 100,
maxTablesPerLevel: 2);
ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, 100, true, _metastreamMaxCount,
Opts.HashCollisionReadLimitDefault, Opts.SkipIndexScanOnReadsDefault,
_dbResult.Db.Config.ReplicationCheckpoint);
ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read());
var scavenger = new TFChunkScavenger(_dbResult.Db, new FakeTFScavengerLog(), tableIndex, ReadIndex,
unsafeIgnoreHardDeletes: UnsafeIgnoreHardDelete());
scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false).Wait();
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024*1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = CreateDb(dbCreationHelper);
_keptRecords = KeptRecords(_dbResult);
_dbResult.Db.Config.WriterCheckpoint.Flush();
_dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
_dbResult.Db.Config.ChaserCheckpoint.Flush();
var indexPath = Path.Combine(PathName, "index");
var readerPool = new ObjectPool<ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint));
var tableIndex = new TableIndex(indexPath,
() => new HashListMemTable(maxSize: 200),
() => new TFReaderLease(readerPool),
maxSizeForMemory: 100,
maxTablesPerLevel: 2);
var hasher = new XXHashUnsafe();
ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, hasher, 100, true, _metastreamMaxCount);
ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read());
//var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount);
var scavenger = new TFChunkScavenger(_dbResult.Db, tableIndex, hasher, ReadIndex);
scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024*1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = CreateDb(dbCreationHelper);
_keptRecords = KeptRecords(_dbResult);
_dbResult.Db.Config.WriterCheckpoint.Flush();
_dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
_dbResult.Db.Config.ChaserCheckpoint.Flush();
var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount);
var scavenger = new TFChunkScavenger(_dbResult.Db, scavengeReadIndex);
scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024*1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
DbRes = CreateDb(dbCreationHelper);
DbRes.Db.Config.WriterCheckpoint.Flush();
DbRes.Db.Config.ChaserCheckpoint.Write(DbRes.Db.Config.WriterCheckpoint.Read());
DbRes.Db.Config.ChaserCheckpoint.Flush();
TableIndex = new TableIndex(GetFilePathFor("index"),
() => new HashListMemTable(MaxEntriesInMemTable * 2),
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
2,
2,
() => new TFChunkReader(DbRes.Db, DbRes.Db.Config.WriterCheckpoint),
TableIndex,
new ByLengthHasher(),
new NoLRUCache<string, StreamCacheInfo>(),
additionalCommitChecks: true,
metastreamMaxCount: _metastreamMaxCount);
ReadIndex.Init(DbRes.Db.Config.WriterCheckpoint.Read(), DbRes.Db.Config.ChaserCheckpoint.Read());
}
protected abstract LogRecord[][] KeptRecords(DbResult dbResult);
protected override LogRecord[][] KeptRecords(DbResult dbResult)
{
return dbResult.Recs;
}
protected abstract LogRecord[][] KeptRecords(DbResult dbResult);