public TFChunkDb(TFChunkDbConfig config)
{
Ensure.NotNull(config, "config");
Config = config;
Manager = new TFChunkManager(Config);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_config = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1000,
0,
new InMemoryCheckpoint(1711),
new InMemoryCheckpoint(5500),
new InMemoryCheckpoint(5500),
new InMemoryCheckpoint(1111));
var rnd = new Random();
_file1Contents = new byte[_config.ChunkSize];
_file2Contents = new byte[_config.ChunkSize];
rnd.NextBytes(_file1Contents);
rnd.NextBytes(_file2Contents);
DbUtil.CreateSingleChunk(_config, 0, GetFilePathFor("chunk-000000.000001"), contents:_file1Contents);
DbUtil.CreateSingleChunk(_config, 1, GetFilePathFor("chunk-000001.000002"), contents: _file2Contents);
var truncator = new TFChunkDbTruncator(_config);
truncator.TruncateDb(_config.TruncateCheckpoint.ReadNonFlushed());
}
public TFChunkDb(TFChunkDbConfig config)
{
Ensure.NotNull(config, "config");
Config = config;
Manager = new TFChunkManager(Config);
}
public TFChunkDb(TFChunkDbConfig config, ILogger log = null)
{
Ensure.NotNull(config, "config");
Config = config;
Manager = new TFChunkManager(Config);
_log = log ?? Serilog.Log.ForContext <TFChunkDb>();
}
public virtual void TestFixtureSetUp()
{
_builder = TestVNodeBuilder.AsSingleNode()
.RunInMemory();
Given();
_node = _builder.Build();
_settings = ((TestVNodeBuilder)_builder).GetSettings();
_dbConfig = ((TestVNodeBuilder)_builder).GetDbConfig();
_node.Start();
}
public void a_null_checkpoint_throws_argument_null_exception()
{
var config = new TFChunkDbConfig(Path.GetTempPath(),
new PrefixFileNamingStrategy(Path.GetTempPath(), "prefix.tf"),
10000,
0,
new InMemoryCheckpoint(0),
new ICheckpoint[0]);
var db = new TFChunkDb(config);
Assert.Throws<ArgumentNullException>(() => new TFChunkReader(db, null));
}
public virtual void TestFixtureSetUp()
{
_builder = TestVNodeBuilder.AsClusterMember(_clusterSize)
.RunInMemory();
_quorumSize = _clusterSize / 2 + 1;
Given();
_node = _builder.Build();
_settings = ((TestVNodeBuilder)_builder).GetSettings();
_dbConfig = ((TestVNodeBuilder)_builder).GetDbConfig();
_node.Start();
}
public TFChunkDbCreationHelper(TFChunkDbConfig dbConfig)
{
Ensure.NotNull(dbConfig, "dbConfig");
_dbConfig = dbConfig;
_db = new TFChunkDb(_dbConfig);
_db.OpenVerifyAndClean();
if (_db.Config.WriterCheckpoint.ReadNonFlushed() > 0)
throw new Exception("The DB already contains some data.");
}
private void ValidateReaderChecksumsMustBeLess(TFChunkDbConfig config)
{
var current = config.WriterCheckpoint.Read();
foreach (var checkpoint in new[] { config.ChaserCheckpoint, config.EpochCheckpoint })
{
if (checkpoint.Read() > current)
{
throw new CorruptDatabaseException(new ReaderCheckpointHigherThanWriterException(checkpoint.Name));
}
}
}
public void allows_first_correct_file_when_checkpoint_is_zero()
{
var config = new TFChunkDbConfig(PathName,
new PrefixFileNamingStrategy(PathName, "prefix.tf"),
10000,
0,
new InMemoryCheckpoint(),
new ICheckpoint[0]);
var db = new TFChunkDb(config);
CreateChunk(Path.Combine(PathName, config.FileNamingStrategy.GetFilenameFor(0)), config.ChunkSize, config.ChunkSize);
Assert.DoesNotThrow(() => db.OpenVerifyAndClean(verifyHash: false));
db.Dispose();
}
public void a_null_checkpoint_throws_argument_null_exception()
{
var config = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var db = new TFChunkDb(config);
Assert.Throws<ArgumentNullException>(() => new TFChunkReader(db, null));
}
public void with_file_of_wrong_size_database_corruption_is_detected()
{
var config = new TFChunkDbConfig(PathName,
new PrefixFileNamingStrategy(PathName, "prefix.tf"),
10000,
0,
new InMemoryCheckpoint(500),
new ICheckpoint[0]);
var db = new TFChunkDb(config);
File.WriteAllText(Path.Combine(PathName, config.FileNamingStrategy.GetFilenameFor(0)), "this is just some test blahbydy blah");
var ex = Assert.Throws<CorruptDatabaseException>(db.OpenVerifyAndClean);
Assert.IsInstanceOf<BadChunkInDatabaseException>(ex.InnerException);
db.Dispose();
}
public void allows_next_new_chunk_when_checksum_is_exactly_in_between_two_chunks()
{
var config = new TFChunkDbConfig(PathName,
new PrefixFileNamingStrategy(PathName, "prefix.tf"),
10000,
0,
new InMemoryCheckpoint(10000),
new ICheckpoint[0]);
var db = new TFChunkDb(config);
CreateChunk(Path.Combine(PathName, config.FileNamingStrategy.GetFilenameFor(0)), config.ChunkSize, config.ChunkSize);
CreateChunk(Path.Combine(PathName, config.FileNamingStrategy.GetFilenameFor(1)), config.ChunkSize, config.ChunkSize);
Assert.DoesNotThrow(() => db.OpenVerifyAndClean(verifyHash: false));
db.Dispose();
}
public void with_wrong_actual_chunk_size_in_chunk_footer()
{
var config = new TFChunkDbConfig(PathName,
new PrefixFileNamingStrategy(PathName, "prefix.tf"),
10000,
0,
new InMemoryCheckpoint(10000),
new ICheckpoint[0]);
var db = new TFChunkDb(config);
CreateChunk(Path.Combine(PathName, config.FileNamingStrategy.GetFilenameFor(0)), 10000, 12000);
var ex = Assert.Throws<CorruptDatabaseException>(db.OpenVerifyAndClean);
Assert.IsInstanceOf<BadChunkInDatabaseException>(ex.InnerException);
db.Dispose();
}
public void with_not_enough_files_to_reach_checksum_throws()
{
var config = new TFChunkDbConfig(PathName,
new PrefixFileNamingStrategy(PathName, "prefix.tf"),
10000,
0,
new InMemoryCheckpoint(15000),
new ICheckpoint[0]);
var db = new TFChunkDb(config);
CreateChunk(Path.Combine(PathName, config.FileNamingStrategy.GetFilenameFor(0)), config.ChunkSize, config.ChunkSize);
var exc = Assert.Throws<CorruptDatabaseException>(db.OpenVerifyAndClean);
Assert.IsInstanceOf<ChunkNotFoundException>(exc.InnerException);
db.Dispose();
}
public static void CreateMultiChunk(TFChunkDbConfig config, int chunkStartNum, int chunkEndNum, string filename,
int? physicalSize = null, long? logicalSize = null)
{
if (chunkStartNum > chunkEndNum) throw new ArgumentException("chunkStartNum");
var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, config.ChunkSize, chunkStartNum, chunkEndNum, true, Guid.NewGuid());
var chunkBytes = chunkHeader.AsByteArray();
var physicalDataSize = physicalSize ?? config.ChunkSize;
var logicalDataSize = logicalSize ?? (chunkEndNum - chunkStartNum + 1) * config.ChunkSize;
var buf = new byte[ChunkHeader.Size + physicalDataSize + ChunkFooter.Size];
Buffer.BlockCopy(chunkBytes, 0, buf, 0, chunkBytes.Length);
var chunkFooter = new ChunkFooter(true, true, physicalDataSize, logicalDataSize, 0, new byte[ChunkFooter.ChecksumSize]);
chunkBytes = chunkFooter.AsByteArray();
Buffer.BlockCopy(chunkBytes, 0, buf, buf.Length - ChunkFooter.Size, chunkBytes.Length);
File.WriteAllBytes(filename, buf);
}
public void allows_with_exactly_enough_file_to_reach_checksum()
{
var config = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
new InMemoryCheckpoint(10000),
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
using (var db = new TFChunkDb(config))
{
DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000"));
Assert.DoesNotThrow(() => db.Open(verifyHash: false));
}
}
public void with_a_writer_checksum_of_zero_the_first_chunk_is_created_with_correct_name()
{
var config = new TFChunkDbConfig(PathName,
new PrefixFileNamingStrategy(PathName, "prefix.tf"),
10000,
0,
new InMemoryCheckpoint(0),
new ICheckpoint[0]);
var db = new TFChunkDb(config);
db.OpenVerifyAndClean();
db.Dispose();
Assert.AreEqual(1, Directory.GetFiles(PathName).Length);
Assert.IsTrue(File.Exists(Path.Combine(PathName, "prefix.tf0")));
var fileInfo = new FileInfo(Path.Combine(PathName, "prefix.tf0"));
Assert.AreEqual(10000 + ChunkHeader.Size + ChunkFooter.Size, fileInfo.Length);
}
public static void CreateOngoingChunk(TFChunkDbConfig config, int chunkNum, string filename, int? actualSize = null, byte[] contents = null)
{
var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, config.ChunkSize, chunkNum, chunkNum, false, Guid.NewGuid());
var chunkBytes = chunkHeader.AsByteArray();
var dataSize = actualSize ?? config.ChunkSize;
var buf = new byte[ChunkHeader.Size + dataSize + ChunkFooter.Size];
Buffer.BlockCopy(chunkBytes, 0, buf, 0, chunkBytes.Length);
if (contents != null)
{
if (contents.Length != dataSize)
throw new Exception("Wrong contents size.");
Buffer.BlockCopy(contents, 0, buf, ChunkHeader.Size, contents.Length);
}
File.WriteAllBytes(filename, buf);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024*1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = CreateDb(dbCreationHelper);
_keptRecords = KeptRecords(_dbResult);
var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams);
var scavenger = new TFChunkScavenger(_dbResult.Db, scavengeReadIndex);
scavenger.Scavenge(alwaysKeepScavenged: true);
}
public void with_checksum_inside_multi_chunk_throws()
{
var config = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
new InMemoryCheckpoint(25000),
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
using (var db = new TFChunkDb(config))
{
DbUtil.CreateMultiChunk(config, 0, 2, GetFilePathFor("chunk-000000.000000"));
Assert.That(() => db.Open(verifyHash: false),
Throws.Exception.InstanceOf<CorruptDatabaseException>()
.With.InnerException.InstanceOf<ChunkNotFoundException>());
}
}
public void with_file_of_wrong_size_database_corruption_is_detected()
{
var config = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
new InMemoryCheckpoint(500),
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
using (var db = new TFChunkDb(config))
{
File.WriteAllText(GetFilePathFor("chunk-000000.000000"), "this is just some test blahbydy blah");
Assert.That(() => db.Open(verifyHash: false),
Throws.Exception.InstanceOf<CorruptDatabaseException>()
.With.InnerException.InstanceOf<BadChunkInDatabaseException>());
}
}
public void with_a_writer_checksum_of_zero_the_first_chunk_is_created_with_correct_name()
{
var config = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var db = new TFChunkDb(config);
db.Open();
db.Dispose();
Assert.AreEqual(1, Directory.GetFiles(PathName).Length);
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000")));
var fileInfo = new FileInfo(GetFilePathFor("chunk-000000.000000"));
Assert.AreEqual(10000 + ChunkHeader.Size + ChunkFooter.Size, fileInfo.Length);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024*1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = CreateDb(dbCreationHelper);
_keptRecords = KeptRecords(_dbResult);
_dbResult.Db.Config.WriterCheckpoint.Flush();
_dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
_dbResult.Db.Config.ChaserCheckpoint.Flush();
var indexPath = Path.Combine(PathName, "index");
var readerPool = new ObjectPool<ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
var tableIndex = new TableIndex(indexPath, lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.Index64Bit, maxSize: 200),
() => new TFReaderLease(readerPool),
PTableVersions.Index64Bit,
maxSizeForMemory: 100,
maxTablesPerLevel: 2);
ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, 100, true, _metastreamMaxCount, Opts.HashCollisionReadLimitDefault);
ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read());
//var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount);
var bus = new InMemoryBus("Bus");
var ioDispatcher = new IODispatcher(bus, new PublishEnvelope(bus));
var scavenger = new TFChunkScavenger(_dbResult.Db, ioDispatcher, tableIndex, ReadIndex, Guid.NewGuid(), "fakeNodeIp",
unsafeIgnoreHardDeletes: UnsafeIgnoreHardDelete());
scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024*1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
DbRes = CreateDb(dbCreationHelper);
DbRes.Db.Config.WriterCheckpoint.Flush();
DbRes.Db.Config.ChaserCheckpoint.Write(DbRes.Db.Config.WriterCheckpoint.Read());
DbRes.Db.Config.ChaserCheckpoint.Flush();
var readers = new ObjectPool<ITransactionFileReader>(
"Readers", 2, 2, () => new TFChunkReader(DbRes.Db, DbRes.Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.Index64Bit, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
PTableVersions.Index64Bit,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: true,
metastreamMaxCount: _metastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault);
ReadIndex.Init(DbRes.Db.Config.ChaserCheckpoint.Read());
}
protected virtual void SetUp()
{
var dbPath = Path.Combine(Path.GetTempPath(), "EventStoreTests", Guid.NewGuid().ToString());
Directory.CreateDirectory(dbPath);
var chunkSize = 256*1024*1024;
var chunksToCache = 2;
if (Runtime.IsMono)
{
_writerChk = new FileCheckpoint(Path.Combine(dbPath, Checkpoint.Writer + ".chk"), Checkpoint.Writer, cached: true);
_chaserChk = new FileCheckpoint(Path.Combine(dbPath, Checkpoint.Chaser + ".chk"), Checkpoint.Chaser, cached: true);
}
else
{
_writerChk = new MemoryMappedFileCheckpoint(Path.Combine(dbPath, Checkpoint.Writer + ".chk"), Checkpoint.Writer, cached: true);
_chaserChk = new MemoryMappedFileCheckpoint(Path.Combine(dbPath, Checkpoint.Chaser + ".chk"), Checkpoint.Chaser, cached: true);
}
var nodeConfig = new TFChunkDbConfig(dbPath,
new VersionedPatternFileNamingStrategy(dbPath, "chunk-"),
chunkSize,
chunksToCache,
_writerChk,
new[] {_chaserChk});
var settings = new SingleVNodeSettings(new IPEndPoint(IPAddress.Loopback, 1111),
new IPEndPoint(IPAddress.Loopback, 2111),
new[] {new IPEndPoint(IPAddress.Loopback, 2111).ToHttpUrl()});
var appsets = new SingleVNodeAppSettings(TimeSpan.FromDays(1));
_db = new TFChunkDb(nodeConfig);
_vNode = new SingleVNode(_db, settings, appsets);
var startCallback = new EnvelopeCallback<SystemMessage.SystemStart>();
_vNode.Bus.Subscribe<SystemMessage.SystemStart>(startCallback);
_vNode.Start();
startCallback.Wait();
}
public void does_not_allow_not_completed_not_last_chunks()
{
var config = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1000,
0,
new InMemoryCheckpoint(4000),
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
using (var db = new TFChunkDb(config))
{
DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000"));
DbUtil.CreateOngoingChunk(config, 1, GetFilePathFor("chunk-000001.000000"));
DbUtil.CreateOngoingChunk(config, 2, GetFilePathFor("chunk-000002.000000"));
DbUtil.CreateOngoingChunk(config, 3, GetFilePathFor("chunk-000003.000000"));
Assert.That(() => db.Open(verifyHash: false),
Throws.Exception.InstanceOf<CorruptDatabaseException>()
.With.InnerException.InstanceOf<BadChunkInDatabaseException>());
}
}
public void allows_with_exactly_enough_file_to_reach_checksum_while_last_is_multi_chunk()
{
var config = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
new InMemoryCheckpoint(30000),
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
using (var db = new TFChunkDb(config))
{
DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000"));
DbUtil.CreateMultiChunk(config, 1, 2, GetFilePathFor("chunk-000001.000000"));
Assert.DoesNotThrow(() => db.Open(verifyHash: false));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000")));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000001.000000")));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000003.000000")));
Assert.AreEqual(3, Directory.GetFiles(PathName, "*").Length);
}
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_config = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1000,
0,
new InMemoryCheckpoint(11111),
new InMemoryCheckpoint(5500),
new InMemoryCheckpoint(5500),
new InMemoryCheckpoint(0));
DbUtil.CreateMultiChunk(_config, 0, 2, GetFilePathFor("chunk-000000.000001"));
DbUtil.CreateMultiChunk(_config, 0, 2, GetFilePathFor("chunk-000000.000002"));
DbUtil.CreateMultiChunk(_config, 3, 10, GetFilePathFor("chunk-000003.000001"));
DbUtil.CreateMultiChunk(_config, 3, 10, GetFilePathFor("chunk-000003.000002"));
DbUtil.CreateMultiChunk(_config, 7, 8, GetFilePathFor("chunk-000007.000001"));
DbUtil.CreateOngoingChunk(_config, 11, GetFilePathFor("chunk-000011.000000"));
var truncator = new TFChunkDbTruncator(_config);
truncator.TruncateDb(_config.TruncateCheckpoint.ReadNonFlushed());
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024*1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = CreateDb(dbCreationHelper);
_keptRecords = KeptRecords(_dbResult);
_dbResult.Db.Config.WriterCheckpoint.Flush();
_dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
_dbResult.Db.Config.ChaserCheckpoint.Flush();
var indexPath = Path.Combine(PathName, "index");
var readerPool = new ObjectPool<ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint));
var tableIndex = new TableIndex(indexPath,
() => new HashListMemTable(maxSize: 200),
() => new TFReaderLease(readerPool),
maxSizeForMemory: 100,
maxTablesPerLevel: 2);
var hasher = new XXHashUnsafe();
ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, hasher, 100, true, _metastreamMaxCount);
ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read());
//var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount);
var scavenger = new TFChunkScavenger(_dbResult.Db, tableIndex, hasher, ReadIndex);
scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024*1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
DbRes = CreateDb(dbCreationHelper);
DbRes.Db.Config.WriterCheckpoint.Flush();
DbRes.Db.Config.ChaserCheckpoint.Write(DbRes.Db.Config.WriterCheckpoint.Read());
DbRes.Db.Config.ChaserCheckpoint.Flush();
TableIndex = new TableIndex(GetFilePathFor("index"),
() => new HashListMemTable(MaxEntriesInMemTable * 2),
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
2,
2,
() => new TFChunkReader(DbRes.Db, DbRes.Db.Config.WriterCheckpoint),
TableIndex,
new ByLengthHasher(),
new NoLRUCache<string, StreamCacheInfo>(),
additionalCommitChecks: true,
metastreamMaxCount: _metastreamMaxCount);
ReadIndex.Init(DbRes.Db.Config.WriterCheckpoint.Read(), DbRes.Db.Config.ChaserCheckpoint.Read());
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024*1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = CreateDb(dbCreationHelper);
_keptRecords = KeptRecords(_dbResult);
_dbResult.Db.Config.WriterCheckpoint.Flush();
_dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
_dbResult.Db.Config.ChaserCheckpoint.Flush();
var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount);
var scavenger = new TFChunkScavenger(_dbResult.Db, scavengeReadIndex);
scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false);
}
public TFChunkManager(TFChunkDbConfig config)
{
Ensure.NotNull(config, "config");
_config = config;
}
private static TFChunkDbConfig CreateDbConfig(int chunkSize, string dbPath, long chunksCacheSize, bool inMemDb)
{
ICheckpoint writerChk;
ICheckpoint chaserChk;
ICheckpoint epochChk;
ICheckpoint truncateChk;
if (inMemDb)
{
writerChk = new InMemoryCheckpoint(Checkpoint.Writer);
chaserChk = new InMemoryCheckpoint(Checkpoint.Chaser);
epochChk = new InMemoryCheckpoint(Checkpoint.Epoch, initValue: -1);
truncateChk = new InMemoryCheckpoint(Checkpoint.Truncate, initValue: -1);
}
else
{
var writerCheckFilename = Path.Combine(dbPath, Checkpoint.Writer + ".chk");
var chaserCheckFilename = Path.Combine(dbPath, Checkpoint.Chaser + ".chk");
var epochCheckFilename = Path.Combine(dbPath, Checkpoint.Epoch + ".chk");
var truncateCheckFilename = Path.Combine(dbPath, Checkpoint.Truncate + ".chk");
if (Runtime.IsMono)
{
writerChk = new FileCheckpoint(writerCheckFilename, Checkpoint.Writer, cached: true);
chaserChk = new FileCheckpoint(chaserCheckFilename, Checkpoint.Chaser, cached: true);
epochChk = new FileCheckpoint(epochCheckFilename, Checkpoint.Epoch, cached: true, initValue: -1);
truncateChk = new FileCheckpoint(truncateCheckFilename, Checkpoint.Truncate, cached: true, initValue: -1);
}
else
{
writerChk = new MemoryMappedFileCheckpoint(writerCheckFilename, Checkpoint.Writer, cached: true);
chaserChk = new MemoryMappedFileCheckpoint(chaserCheckFilename, Checkpoint.Chaser, cached: true);
epochChk = new MemoryMappedFileCheckpoint(epochCheckFilename, Checkpoint.Epoch, cached: true, initValue: -1);
truncateChk = new MemoryMappedFileCheckpoint(truncateCheckFilename, Checkpoint.Truncate, cached: true, initValue: -1);
}
}
var nodeConfig = new TFChunkDbConfig(dbPath,
new VersionedPatternFileNamingStrategy(dbPath, "chunk-"),
chunkSize,
chunksCacheSize,
writerChk,
chaserChk,
epochChk,
truncateChk,
inMemDb);
return nodeConfig;
}
public TFChunkDbTruncator(TFChunkDbConfig config)
{
Ensure.NotNull(config, "config");
_config = config;
}
