public TFChunkDb(TFChunkDbConfig config) { Ensure.NotNull(config, "config"); Config = config; Manager = new TFChunkManager(Config); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1000, 0, new InMemoryCheckpoint(1711), new InMemoryCheckpoint(5500), new InMemoryCheckpoint(5500), new InMemoryCheckpoint(1111)); var rnd = new Random(); _file1Contents = new byte[_config.ChunkSize]; _file2Contents = new byte[_config.ChunkSize]; rnd.NextBytes(_file1Contents); rnd.NextBytes(_file2Contents); DbUtil.CreateSingleChunk(_config, 0, GetFilePathFor("chunk-000000.000001"), contents:_file1Contents); DbUtil.CreateSingleChunk(_config, 1, GetFilePathFor("chunk-000001.000002"), contents: _file2Contents); var truncator = new TFChunkDbTruncator(_config); truncator.TruncateDb(_config.TruncateCheckpoint.ReadNonFlushed()); }
public TFChunkDb(TFChunkDbConfig config, ILogger log = null) { Ensure.NotNull(config, "config"); Config = config; Manager = new TFChunkManager(Config); _log = log ?? Serilog.Log.ForContext <TFChunkDb>(); }
public virtual void TestFixtureSetUp() { _builder = TestVNodeBuilder.AsSingleNode() .RunInMemory(); Given(); _node = _builder.Build(); _settings = ((TestVNodeBuilder)_builder).GetSettings(); _dbConfig = ((TestVNodeBuilder)_builder).GetDbConfig(); _node.Start(); }
public void a_null_checkpoint_throws_argument_null_exception() { var config = new TFChunkDbConfig(Path.GetTempPath(), new PrefixFileNamingStrategy(Path.GetTempPath(), "prefix.tf"), 10000, 0, new InMemoryCheckpoint(0), new ICheckpoint[0]); var db = new TFChunkDb(config); Assert.Throws<ArgumentNullException>(() => new TFChunkReader(db, null)); }
public virtual void TestFixtureSetUp() { _builder = TestVNodeBuilder.AsClusterMember(_clusterSize) .RunInMemory(); _quorumSize = _clusterSize / 2 + 1; Given(); _node = _builder.Build(); _settings = ((TestVNodeBuilder)_builder).GetSettings(); _dbConfig = ((TestVNodeBuilder)_builder).GetDbConfig(); _node.Start(); }
public TFChunkDbCreationHelper(TFChunkDbConfig dbConfig) { Ensure.NotNull(dbConfig, "dbConfig"); _dbConfig = dbConfig; _db = new TFChunkDb(_dbConfig); _db.OpenVerifyAndClean(); if (_db.Config.WriterCheckpoint.ReadNonFlushed() > 0) throw new Exception("The DB already contains some data."); }
private void ValidateReaderChecksumsMustBeLess(TFChunkDbConfig config) { var current = config.WriterCheckpoint.Read(); foreach (var checkpoint in new[] { config.ChaserCheckpoint, config.EpochCheckpoint }) { if (checkpoint.Read() > current) { throw new CorruptDatabaseException(new ReaderCheckpointHigherThanWriterException(checkpoint.Name)); } } }
public void allows_first_correct_file_when_checkpoint_is_zero() { var config = new TFChunkDbConfig(PathName, new PrefixFileNamingStrategy(PathName, "prefix.tf"), 10000, 0, new InMemoryCheckpoint(), new ICheckpoint[0]); var db = new TFChunkDb(config); CreateChunk(Path.Combine(PathName, config.FileNamingStrategy.GetFilenameFor(0)), config.ChunkSize, config.ChunkSize); Assert.DoesNotThrow(() => db.OpenVerifyAndClean(verifyHash: false)); db.Dispose(); }
public void a_null_checkpoint_throws_argument_null_exception() { var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, new InMemoryCheckpoint(0), new InMemoryCheckpoint(0), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); var db = new TFChunkDb(config); Assert.Throws<ArgumentNullException>(() => new TFChunkReader(db, null)); }
public void with_file_of_wrong_size_database_corruption_is_detected() { var config = new TFChunkDbConfig(PathName, new PrefixFileNamingStrategy(PathName, "prefix.tf"), 10000, 0, new InMemoryCheckpoint(500), new ICheckpoint[0]); var db = new TFChunkDb(config); File.WriteAllText(Path.Combine(PathName, config.FileNamingStrategy.GetFilenameFor(0)), "this is just some test blahbydy blah"); var ex = Assert.Throws<CorruptDatabaseException>(db.OpenVerifyAndClean); Assert.IsInstanceOf<BadChunkInDatabaseException>(ex.InnerException); db.Dispose(); }
public void allows_next_new_chunk_when_checksum_is_exactly_in_between_two_chunks() { var config = new TFChunkDbConfig(PathName, new PrefixFileNamingStrategy(PathName, "prefix.tf"), 10000, 0, new InMemoryCheckpoint(10000), new ICheckpoint[0]); var db = new TFChunkDb(config); CreateChunk(Path.Combine(PathName, config.FileNamingStrategy.GetFilenameFor(0)), config.ChunkSize, config.ChunkSize); CreateChunk(Path.Combine(PathName, config.FileNamingStrategy.GetFilenameFor(1)), config.ChunkSize, config.ChunkSize); Assert.DoesNotThrow(() => db.OpenVerifyAndClean(verifyHash: false)); db.Dispose(); }
public void with_wrong_actual_chunk_size_in_chunk_footer() { var config = new TFChunkDbConfig(PathName, new PrefixFileNamingStrategy(PathName, "prefix.tf"), 10000, 0, new InMemoryCheckpoint(10000), new ICheckpoint[0]); var db = new TFChunkDb(config); CreateChunk(Path.Combine(PathName, config.FileNamingStrategy.GetFilenameFor(0)), 10000, 12000); var ex = Assert.Throws<CorruptDatabaseException>(db.OpenVerifyAndClean); Assert.IsInstanceOf<BadChunkInDatabaseException>(ex.InnerException); db.Dispose(); }
public void with_not_enough_files_to_reach_checksum_throws() { var config = new TFChunkDbConfig(PathName, new PrefixFileNamingStrategy(PathName, "prefix.tf"), 10000, 0, new InMemoryCheckpoint(15000), new ICheckpoint[0]); var db = new TFChunkDb(config); CreateChunk(Path.Combine(PathName, config.FileNamingStrategy.GetFilenameFor(0)), config.ChunkSize, config.ChunkSize); var exc = Assert.Throws<CorruptDatabaseException>(db.OpenVerifyAndClean); Assert.IsInstanceOf<ChunkNotFoundException>(exc.InnerException); db.Dispose(); }
public static void CreateMultiChunk(TFChunkDbConfig config, int chunkStartNum, int chunkEndNum, string filename, int? physicalSize = null, long? logicalSize = null) { if (chunkStartNum > chunkEndNum) throw new ArgumentException("chunkStartNum"); var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, config.ChunkSize, chunkStartNum, chunkEndNum, true, Guid.NewGuid()); var chunkBytes = chunkHeader.AsByteArray(); var physicalDataSize = physicalSize ?? config.ChunkSize; var logicalDataSize = logicalSize ?? (chunkEndNum - chunkStartNum + 1) * config.ChunkSize; var buf = new byte[ChunkHeader.Size + physicalDataSize + ChunkFooter.Size]; Buffer.BlockCopy(chunkBytes, 0, buf, 0, chunkBytes.Length); var chunkFooter = new ChunkFooter(true, true, physicalDataSize, logicalDataSize, 0, new byte[ChunkFooter.ChecksumSize]); chunkBytes = chunkFooter.AsByteArray(); Buffer.BlockCopy(chunkBytes, 0, buf, buf.Length - ChunkFooter.Size, chunkBytes.Length); File.WriteAllBytes(filename, buf); }
public void allows_with_exactly_enough_file_to_reach_checksum() { var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, new InMemoryCheckpoint(10000), new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); using (var db = new TFChunkDb(config)) { DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000")); Assert.DoesNotThrow(() => db.Open(verifyHash: false)); } }
public void with_a_writer_checksum_of_zero_the_first_chunk_is_created_with_correct_name() { var config = new TFChunkDbConfig(PathName, new PrefixFileNamingStrategy(PathName, "prefix.tf"), 10000, 0, new InMemoryCheckpoint(0), new ICheckpoint[0]); var db = new TFChunkDb(config); db.OpenVerifyAndClean(); db.Dispose(); Assert.AreEqual(1, Directory.GetFiles(PathName).Length); Assert.IsTrue(File.Exists(Path.Combine(PathName, "prefix.tf0"))); var fileInfo = new FileInfo(Path.Combine(PathName, "prefix.tf0")); Assert.AreEqual(10000 + ChunkHeader.Size + ChunkFooter.Size, fileInfo.Length); }
public static void CreateOngoingChunk(TFChunkDbConfig config, int chunkNum, string filename, int? actualSize = null, byte[] contents = null) { var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, config.ChunkSize, chunkNum, chunkNum, false, Guid.NewGuid()); var chunkBytes = chunkHeader.AsByteArray(); var dataSize = actualSize ?? config.ChunkSize; var buf = new byte[ChunkHeader.Size + dataSize + ChunkFooter.Size]; Buffer.BlockCopy(chunkBytes, 0, buf, 0, chunkBytes.Length); if (contents != null) { if (contents.Length != dataSize) throw new Exception("Wrong contents size."); Buffer.BlockCopy(contents, 0, buf, ChunkHeader.Size, contents.Length); } File.WriteAllBytes(filename, buf); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); var dbConfig = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1024*1024, 0, new InMemoryCheckpoint(0), new InMemoryCheckpoint(0)); var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig); _dbResult = CreateDb(dbCreationHelper); _keptRecords = KeptRecords(_dbResult); var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams); var scavenger = new TFChunkScavenger(_dbResult.Db, scavengeReadIndex); scavenger.Scavenge(alwaysKeepScavenged: true); }
public void with_checksum_inside_multi_chunk_throws() { var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, new InMemoryCheckpoint(25000), new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); using (var db = new TFChunkDb(config)) { DbUtil.CreateMultiChunk(config, 0, 2, GetFilePathFor("chunk-000000.000000")); Assert.That(() => db.Open(verifyHash: false), Throws.Exception.InstanceOf<CorruptDatabaseException>() .With.InnerException.InstanceOf<ChunkNotFoundException>()); } }
public void with_file_of_wrong_size_database_corruption_is_detected() { var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, new InMemoryCheckpoint(500), new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); using (var db = new TFChunkDb(config)) { File.WriteAllText(GetFilePathFor("chunk-000000.000000"), "this is just some test blahbydy blah"); Assert.That(() => db.Open(verifyHash: false), Throws.Exception.InstanceOf<CorruptDatabaseException>() .With.InnerException.InstanceOf<BadChunkInDatabaseException>()); } }
public void with_a_writer_checksum_of_zero_the_first_chunk_is_created_with_correct_name() { var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, new InMemoryCheckpoint(0), new InMemoryCheckpoint(0), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); var db = new TFChunkDb(config); db.Open(); db.Dispose(); Assert.AreEqual(1, Directory.GetFiles(PathName).Length); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000"))); var fileInfo = new FileInfo(GetFilePathFor("chunk-000000.000000")); Assert.AreEqual(10000 + ChunkHeader.Size + ChunkFooter.Size, fileInfo.Length); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); var dbConfig = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1024*1024, 0, new InMemoryCheckpoint(0), new InMemoryCheckpoint(0), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig); _dbResult = CreateDb(dbCreationHelper); _keptRecords = KeptRecords(_dbResult); _dbResult.Db.Config.WriterCheckpoint.Flush(); _dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read()); _dbResult.Db.Config.ChaserCheckpoint.Flush(); var indexPath = Path.Combine(PathName, "index"); var readerPool = new ObjectPool<ITransactionFileReader>( "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount, () => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint)); var lowHasher = new XXHashUnsafe(); var highHasher = new Murmur3AUnsafe(); var tableIndex = new TableIndex(indexPath, lowHasher, highHasher, () => new HashListMemTable(PTableVersions.Index64Bit, maxSize: 200), () => new TFReaderLease(readerPool), PTableVersions.Index64Bit, maxSizeForMemory: 100, maxTablesPerLevel: 2); ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, 100, true, _metastreamMaxCount, Opts.HashCollisionReadLimitDefault); ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read()); //var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount); var bus = new InMemoryBus("Bus"); var ioDispatcher = new IODispatcher(bus, new PublishEnvelope(bus)); var scavenger = new TFChunkScavenger(_dbResult.Db, ioDispatcher, tableIndex, ReadIndex, Guid.NewGuid(), "fakeNodeIp", unsafeIgnoreHardDeletes: UnsafeIgnoreHardDelete()); scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); var dbConfig = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1024*1024, 0, new InMemoryCheckpoint(0), new InMemoryCheckpoint(0), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig); DbRes = CreateDb(dbCreationHelper); DbRes.Db.Config.WriterCheckpoint.Flush(); DbRes.Db.Config.ChaserCheckpoint.Write(DbRes.Db.Config.WriterCheckpoint.Read()); DbRes.Db.Config.ChaserCheckpoint.Flush(); var readers = new ObjectPool<ITransactionFileReader>( "Readers", 2, 2, () => new TFChunkReader(DbRes.Db, DbRes.Db.Config.WriterCheckpoint)); var lowHasher = new XXHashUnsafe(); var highHasher = new Murmur3AUnsafe(); TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher, () => new HashListMemTable(PTableVersions.Index64Bit, MaxEntriesInMemTable * 2), () => new TFReaderLease(readers), PTableVersions.Index64Bit, MaxEntriesInMemTable); ReadIndex = new ReadIndex(new NoopPublisher(), readers, TableIndex, 0, additionalCommitChecks: true, metastreamMaxCount: _metastreamMaxCount, hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault); ReadIndex.Init(DbRes.Db.Config.ChaserCheckpoint.Read()); }
protected virtual void SetUp() { var dbPath = Path.Combine(Path.GetTempPath(), "EventStoreTests", Guid.NewGuid().ToString()); Directory.CreateDirectory(dbPath); var chunkSize = 256*1024*1024; var chunksToCache = 2; if (Runtime.IsMono) { _writerChk = new FileCheckpoint(Path.Combine(dbPath, Checkpoint.Writer + ".chk"), Checkpoint.Writer, cached: true); _chaserChk = new FileCheckpoint(Path.Combine(dbPath, Checkpoint.Chaser + ".chk"), Checkpoint.Chaser, cached: true); } else { _writerChk = new MemoryMappedFileCheckpoint(Path.Combine(dbPath, Checkpoint.Writer + ".chk"), Checkpoint.Writer, cached: true); _chaserChk = new MemoryMappedFileCheckpoint(Path.Combine(dbPath, Checkpoint.Chaser + ".chk"), Checkpoint.Chaser, cached: true); } var nodeConfig = new TFChunkDbConfig(dbPath, new VersionedPatternFileNamingStrategy(dbPath, "chunk-"), chunkSize, chunksToCache, _writerChk, new[] {_chaserChk}); var settings = new SingleVNodeSettings(new IPEndPoint(IPAddress.Loopback, 1111), new IPEndPoint(IPAddress.Loopback, 2111), new[] {new IPEndPoint(IPAddress.Loopback, 2111).ToHttpUrl()}); var appsets = new SingleVNodeAppSettings(TimeSpan.FromDays(1)); _db = new TFChunkDb(nodeConfig); _vNode = new SingleVNode(_db, settings, appsets); var startCallback = new EnvelopeCallback<SystemMessage.SystemStart>(); _vNode.Bus.Subscribe<SystemMessage.SystemStart>(startCallback); _vNode.Start(); startCallback.Wait(); }
public void does_not_allow_not_completed_not_last_chunks() { var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1000, 0, new InMemoryCheckpoint(4000), new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); using (var db = new TFChunkDb(config)) { DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000")); DbUtil.CreateOngoingChunk(config, 1, GetFilePathFor("chunk-000001.000000")); DbUtil.CreateOngoingChunk(config, 2, GetFilePathFor("chunk-000002.000000")); DbUtil.CreateOngoingChunk(config, 3, GetFilePathFor("chunk-000003.000000")); Assert.That(() => db.Open(verifyHash: false), Throws.Exception.InstanceOf<CorruptDatabaseException>() .With.InnerException.InstanceOf<BadChunkInDatabaseException>()); } }
public void allows_with_exactly_enough_file_to_reach_checksum_while_last_is_multi_chunk() { var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, new InMemoryCheckpoint(30000), new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); using (var db = new TFChunkDb(config)) { DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000")); DbUtil.CreateMultiChunk(config, 1, 2, GetFilePathFor("chunk-000001.000000")); Assert.DoesNotThrow(() => db.Open(verifyHash: false)); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000"))); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000001.000000"))); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000003.000000"))); Assert.AreEqual(3, Directory.GetFiles(PathName, "*").Length); } }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1000, 0, new InMemoryCheckpoint(11111), new InMemoryCheckpoint(5500), new InMemoryCheckpoint(5500), new InMemoryCheckpoint(0)); DbUtil.CreateMultiChunk(_config, 0, 2, GetFilePathFor("chunk-000000.000001")); DbUtil.CreateMultiChunk(_config, 0, 2, GetFilePathFor("chunk-000000.000002")); DbUtil.CreateMultiChunk(_config, 3, 10, GetFilePathFor("chunk-000003.000001")); DbUtil.CreateMultiChunk(_config, 3, 10, GetFilePathFor("chunk-000003.000002")); DbUtil.CreateMultiChunk(_config, 7, 8, GetFilePathFor("chunk-000007.000001")); DbUtil.CreateOngoingChunk(_config, 11, GetFilePathFor("chunk-000011.000000")); var truncator = new TFChunkDbTruncator(_config); truncator.TruncateDb(_config.TruncateCheckpoint.ReadNonFlushed()); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); var dbConfig = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1024*1024, 0, new InMemoryCheckpoint(0), new InMemoryCheckpoint(0), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig); _dbResult = CreateDb(dbCreationHelper); _keptRecords = KeptRecords(_dbResult); _dbResult.Db.Config.WriterCheckpoint.Flush(); _dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read()); _dbResult.Db.Config.ChaserCheckpoint.Flush(); var indexPath = Path.Combine(PathName, "index"); var readerPool = new ObjectPool<ITransactionFileReader>( "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount, () => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint)); var tableIndex = new TableIndex(indexPath, () => new HashListMemTable(maxSize: 200), () => new TFReaderLease(readerPool), maxSizeForMemory: 100, maxTablesPerLevel: 2); var hasher = new XXHashUnsafe(); ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, hasher, 100, true, _metastreamMaxCount); ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read()); //var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount); var scavenger = new TFChunkScavenger(_dbResult.Db, tableIndex, hasher, ReadIndex); scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); var dbConfig = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1024*1024, 0, new InMemoryCheckpoint(0), new InMemoryCheckpoint(0), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig); DbRes = CreateDb(dbCreationHelper); DbRes.Db.Config.WriterCheckpoint.Flush(); DbRes.Db.Config.ChaserCheckpoint.Write(DbRes.Db.Config.WriterCheckpoint.Read()); DbRes.Db.Config.ChaserCheckpoint.Flush(); TableIndex = new TableIndex(GetFilePathFor("index"), () => new HashListMemTable(MaxEntriesInMemTable * 2), MaxEntriesInMemTable); ReadIndex = new ReadIndex(new NoopPublisher(), 2, 2, () => new TFChunkReader(DbRes.Db, DbRes.Db.Config.WriterCheckpoint), TableIndex, new ByLengthHasher(), new NoLRUCache<string, StreamCacheInfo>(), additionalCommitChecks: true, metastreamMaxCount: _metastreamMaxCount); ReadIndex.Init(DbRes.Db.Config.WriterCheckpoint.Read(), DbRes.Db.Config.ChaserCheckpoint.Read()); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); var dbConfig = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1024*1024, 0, new InMemoryCheckpoint(0), new InMemoryCheckpoint(0), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig); _dbResult = CreateDb(dbCreationHelper); _keptRecords = KeptRecords(_dbResult); _dbResult.Db.Config.WriterCheckpoint.Flush(); _dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read()); _dbResult.Db.Config.ChaserCheckpoint.Flush(); var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount); var scavenger = new TFChunkScavenger(_dbResult.Db, scavengeReadIndex); scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false); }
public TFChunkManager(TFChunkDbConfig config) { Ensure.NotNull(config, "config"); _config = config; }
private static TFChunkDbConfig CreateDbConfig(int chunkSize, string dbPath, long chunksCacheSize, bool inMemDb) { ICheckpoint writerChk; ICheckpoint chaserChk; ICheckpoint epochChk; ICheckpoint truncateChk; if (inMemDb) { writerChk = new InMemoryCheckpoint(Checkpoint.Writer); chaserChk = new InMemoryCheckpoint(Checkpoint.Chaser); epochChk = new InMemoryCheckpoint(Checkpoint.Epoch, initValue: -1); truncateChk = new InMemoryCheckpoint(Checkpoint.Truncate, initValue: -1); } else { var writerCheckFilename = Path.Combine(dbPath, Checkpoint.Writer + ".chk"); var chaserCheckFilename = Path.Combine(dbPath, Checkpoint.Chaser + ".chk"); var epochCheckFilename = Path.Combine(dbPath, Checkpoint.Epoch + ".chk"); var truncateCheckFilename = Path.Combine(dbPath, Checkpoint.Truncate + ".chk"); if (Runtime.IsMono) { writerChk = new FileCheckpoint(writerCheckFilename, Checkpoint.Writer, cached: true); chaserChk = new FileCheckpoint(chaserCheckFilename, Checkpoint.Chaser, cached: true); epochChk = new FileCheckpoint(epochCheckFilename, Checkpoint.Epoch, cached: true, initValue: -1); truncateChk = new FileCheckpoint(truncateCheckFilename, Checkpoint.Truncate, cached: true, initValue: -1); } else { writerChk = new MemoryMappedFileCheckpoint(writerCheckFilename, Checkpoint.Writer, cached: true); chaserChk = new MemoryMappedFileCheckpoint(chaserCheckFilename, Checkpoint.Chaser, cached: true); epochChk = new MemoryMappedFileCheckpoint(epochCheckFilename, Checkpoint.Epoch, cached: true, initValue: -1); truncateChk = new MemoryMappedFileCheckpoint(truncateCheckFilename, Checkpoint.Truncate, cached: true, initValue: -1); } } var nodeConfig = new TFChunkDbConfig(dbPath, new VersionedPatternFileNamingStrategy(dbPath, "chunk-"), chunkSize, chunksCacheSize, writerChk, chaserChk, epochChk, truncateChk, inMemDb); return nodeConfig; }
public TFChunkDbTruncator(TFChunkDbConfig config) { Ensure.NotNull(config, "config"); _config = config; }