public void when_checkpoint_is_on_boundary_of_new_chunk_excessive_last_chunks_are_removed_if_present_and_new_empty_one_is_created() { var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 100, 0, new InMemoryCheckpoint(200), new InMemoryCheckpoint(), new ICheckpoint[0]); var db = new TFChunkDb(config); CreateChunk(Path.Combine(PathName, "chunk-000000.000000"), config.ChunkSize, config.ChunkSize); CreateChunk(Path.Combine(PathName, "chunk-000001.000001"), config.ChunkSize, config.ChunkSize); CreateChunk(Path.Combine(PathName, "chunk-000002.000000"), config.ChunkSize, config.ChunkSize); CreateChunk(Path.Combine(PathName, "chunk-000002.000001"), config.ChunkSize, config.ChunkSize); Assert.DoesNotThrow(() => db.OpenVerifyAndClean(verifyHash: false)); Assert.IsTrue(File.Exists(Path.Combine(PathName, "chunk-000000.000000"))); Assert.IsTrue(File.Exists(Path.Combine(PathName, "chunk-000001.000001"))); Assert.IsTrue(File.Exists(Path.Combine(PathName, "chunk-000002.000000"))); Assert.AreEqual(3, Directory.GetFiles(PathName, "*").Length); db.Dispose(); }
public void when_checkpoint_is_on_boundary_of_new_chunk_and_last_chunk_is_truncated_no_exception_is_thrown() { var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 100, 0, new InMemoryCheckpoint(300), new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); using (var db = new TFChunkDb(config)) { DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000")); DbUtil.CreateMultiChunk(config, 1, 2, GetFilePathFor("chunk-000001.000001"), physicalSize: 50, logicalSize: 150); Assert.DoesNotThrow(() => db.Open(verifyHash: false)); Assert.IsNotNull(db.Manager.GetChunk(2)); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000"))); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000001.000001"))); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000003.000000"))); Assert.AreEqual(3, Directory.GetFiles(PathName, "*").Length); } }
public void old_version_of_chunks_are_removed() { File.Create(GetFilePathFor("foo")).Close(); File.Create(GetFilePathFor("bla")).Close(); var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 100, 0, new InMemoryCheckpoint(350), new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); using (var db = new TFChunkDb(config)) { DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000")); DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000002")); DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000005")); DbUtil.CreateSingleChunk(config, 1, GetFilePathFor("chunk-000001.000000")); DbUtil.CreateMultiChunk(config, 1, 2, GetFilePathFor("chunk-000001.000001")); DbUtil.CreateSingleChunk(config, 2, GetFilePathFor("chunk-000002.000000")); DbUtil.CreateSingleChunk(config, 3, GetFilePathFor("chunk-000003.000007")); DbUtil.CreateOngoingChunk(config, 3, GetFilePathFor("chunk-000003.000008")); Assert.DoesNotThrow(() => db.Open(verifyHash: false)); Assert.IsTrue(File.Exists(GetFilePathFor("foo"))); Assert.IsTrue(File.Exists(GetFilePathFor("bla"))); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000005"))); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000001.000001"))); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000003.000008"))); Assert.AreEqual(5, Directory.GetFiles(PathName, "*").Length); } }
private static TFChunkDbConfig CreateTfDbConfig(int chunkSize, string dbPath, int chunksToCache) { if (!Directory.Exists(dbPath)) // mono crashes without this check { Directory.CreateDirectory(dbPath); } ICheckpoint writerChk; //TODO MM: WHO SHOULD DISPOSE THESE TWO?? ICheckpoint chaserChk; if (Runtime.IsMono) { writerChk = new FileCheckpoint(Path.Combine(dbPath, Checkpoint.Writer + ".chk"), Checkpoint.Writer, cached: true); chaserChk = new FileCheckpoint(Path.Combine(dbPath, Checkpoint.Chaser + ".chk"), Checkpoint.Chaser, cached: true); } else { writerChk = new MemoryMappedFileCheckpoint(Path.Combine(dbPath, Checkpoint.Writer + ".chk"), Checkpoint.Writer, cached: true); chaserChk = new MemoryMappedFileCheckpoint(Path.Combine(dbPath, Checkpoint.Chaser + ".chk"), Checkpoint.Chaser, cached: true); } var nodeConfig = new TFChunkDbConfig(dbPath, new VersionedPatternFileNamingStrategy(dbPath, "chunk-"), chunkSize, chunksToCache, writerChk, new[] { chaserChk }); return(nodeConfig); }
public void when_checkpoint_is_exactly_on_the_boundary_of_chunk_the_last_chunk_could_be_present() { var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 100, 0, new InMemoryCheckpoint(200), new InMemoryCheckpoint(), new ICheckpoint[0]); var db = new TFChunkDb(config); CreateChunk(Path.Combine(PathName, "chunk-000000.000000"), config.ChunkSize, config.ChunkSize); CreateChunk(Path.Combine(PathName, "chunk-000001.000001"), config.ChunkSize, config.ChunkSize); CreateOngoingChunk(Path.Combine(PathName, "chunk-000002.000000"), config.ChunkSize, config.ChunkSize); Assert.DoesNotThrow(() => db.OpenVerifyAndClean(verifyHash: false)); Assert.IsNotNull(db.Manager.GetChunk(2)); Assert.IsTrue(File.Exists(Path.Combine(PathName, "chunk-000000.000000"))); Assert.IsTrue(File.Exists(Path.Combine(PathName, "chunk-000001.000001"))); Assert.IsTrue(File.Exists(Path.Combine(PathName, "chunk-000002.000000"))); Assert.AreEqual(3, Directory.GetFiles(PathName, "*").Length); db.Dispose(); }
private TFChunkDbConfig CreateDbConfig(int chunkSize, string dbPath, long chunksCacheSize, bool inMemDb) { ICheckpoint writerChk; ICheckpoint chaserChk; ICheckpoint epochChk; ICheckpoint truncateChk; ICheckpoint replicationCheckpoint = new InMemoryCheckpoint(-1); if (inMemDb) { writerChk = new InMemoryCheckpoint(Checkpoint.Writer); chaserChk = new InMemoryCheckpoint(Checkpoint.Chaser); epochChk = new InMemoryCheckpoint(Checkpoint.Epoch, initValue: -1); truncateChk = new InMemoryCheckpoint(Checkpoint.Truncate, initValue: -1); } else { var writerCheckFilename = Path.Combine(dbPath, Checkpoint.Writer + ".chk"); var chaserCheckFilename = Path.Combine(dbPath, Checkpoint.Chaser + ".chk"); var epochCheckFilename = Path.Combine(dbPath, Checkpoint.Epoch + ".chk"); var truncateCheckFilename = Path.Combine(dbPath, Checkpoint.Truncate + ".chk"); writerChk = new MemoryMappedFileCheckpoint(writerCheckFilename, Checkpoint.Writer, cached: true); chaserChk = new MemoryMappedFileCheckpoint(chaserCheckFilename, Checkpoint.Chaser, cached: true); epochChk = new MemoryMappedFileCheckpoint( epochCheckFilename, Checkpoint.Epoch, cached: true, initValue: -1); truncateChk = new MemoryMappedFileCheckpoint( truncateCheckFilename, Checkpoint.Truncate, cached: true, initValue: -1); } var nodeConfig = new TFChunkDbConfig( dbPath, new VersionedPatternFileNamingStrategy(dbPath, "chunk-"), chunkSize, chunksCacheSize, writerChk, chaserChk, epochChk, truncateChk, replicationCheckpoint, Constants.TFChunkInitialReaderCountDefault, Constants.TFChunkMaxReaderCountDefault, inMemDb); return(nodeConfig); }
public void when_checkpoint_is_exactly_on_the_boundary_of_chunk_the_last_chunk_could_be_not_present_but_should_be_created() { var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 100, 0, new InMemoryCheckpoint(200), new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); using (var db = new TFChunkDb(config)) { DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000")); DbUtil.CreateSingleChunk(config, 1, GetFilePathFor("chunk-000001.000001")); Assert.DoesNotThrow(() => db.Open(verifyHash: false)); Assert.IsNotNull(db.Manager.GetChunk(2)); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000"))); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000001.000001"))); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000002.000000"))); Assert.AreEqual(3, Directory.GetFiles(PathName, "*").Length); } }
public void old_version_of_chunks_are_removed() { File.Create(Path.Combine(PathName, "foo")).Close(); File.Create(Path.Combine(PathName, "bla")).Close(); var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 100, 0, new InMemoryCheckpoint(400), new ICheckpoint[0]); var db = new TFChunkDb(config); CreateChunk(Path.Combine(PathName, "chunk-000000.000000"), config.ChunkSize, config.ChunkSize); CreateChunk(Path.Combine(PathName, "chunk-000000.000002"), config.ChunkSize, config.ChunkSize); CreateChunk(Path.Combine(PathName, "chunk-000000.000005"), config.ChunkSize, config.ChunkSize); CreateChunk(Path.Combine(PathName, "chunk-000001.000000"), config.ChunkSize, config.ChunkSize); CreateChunk(Path.Combine(PathName, "chunk-000001.000001"), config.ChunkSize, config.ChunkSize); CreateChunk(Path.Combine(PathName, "chunk-000002.000000"), config.ChunkSize, config.ChunkSize); CreateChunk(Path.Combine(PathName, "chunk-000003.000007"), config.ChunkSize, config.ChunkSize); CreateChunk(Path.Combine(PathName, "chunk-000003.000008"), config.ChunkSize, config.ChunkSize); Assert.DoesNotThrow(db.OpenVerifyAndClean); Assert.IsTrue(File.Exists(Path.Combine(PathName, "foo"))); Assert.IsTrue(File.Exists(Path.Combine(PathName, "bla"))); Assert.IsTrue(File.Exists(Path.Combine(PathName, "chunk-000000.000005"))); Assert.IsTrue(File.Exists(Path.Combine(PathName, "chunk-000001.000001"))); Assert.IsTrue(File.Exists(Path.Combine(PathName, "chunk-000002.000000"))); Assert.IsTrue(File.Exists(Path.Combine(PathName, "chunk-000003.000008"))); Assert.AreEqual(6, Directory.GetFiles(PathName, "*").Length); db.Dispose(); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1000, 0, new InMemoryCheckpoint(1711), new InMemoryCheckpoint(5500), new InMemoryCheckpoint(5500), new InMemoryCheckpoint(1111)); var rnd = new Random(); _file1Contents = new byte[_config.ChunkSize]; _file2Contents = new byte[_config.ChunkSize]; rnd.NextBytes(_file1Contents); rnd.NextBytes(_file2Contents); DbUtil.CreateSingleChunk(_config, 0, GetFilePathFor("chunk-000000.000001"), contents: _file1Contents); DbUtil.CreateOngoingChunk(_config, 1, GetFilePathFor("chunk-000001.000002"), contents: _file2Contents); var truncator = new TFChunkDbTruncator(_config); truncator.TruncateDb(_config.TruncateCheckpoint.ReadNonFlushed()); }
public void when_checkpoint_is_on_boundary_of_new_chunk_last_chunk_is_preserved_and_excessive_versions_are_removed_if_present() { var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 100, 0, new InMemoryCheckpoint(200), new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); using (var db = new TFChunkDb(config)) { DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000")); DbUtil.CreateSingleChunk(config, 1, GetFilePathFor("chunk-000001.000001")); DbUtil.CreateSingleChunk(config, 2, GetFilePathFor("chunk-000002.000000")); DbUtil.CreateOngoingChunk(config, 2, GetFilePathFor("chunk-000002.000001")); Assert.DoesNotThrow(() => db.Open(verifyHash: false)); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000"))); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000001.000001"))); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000002.000001"))); Assert.AreEqual(3, Directory.GetFiles(PathName, "*").Length); } }
public static void CreateSingleChunk(TFChunkDbConfig config, int chunkNum, string filename, int?actualDataSize = null, bool isScavenged = false, byte[] contents = null) { var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, config.ChunkSize, chunkNum, chunkNum, isScavenged, Guid.NewGuid()); var chunkBytes = chunkHeader.AsByteArray(); var dataSize = actualDataSize ?? config.ChunkSize; var buf = new byte[ChunkHeader.Size + dataSize + ChunkFooter.Size]; Buffer.BlockCopy(chunkBytes, 0, buf, 0, chunkBytes.Length); var chunkFooter = new ChunkFooter(true, true, dataSize, dataSize, 0, new byte[ChunkFooter.ChecksumSize]); chunkBytes = chunkFooter.AsByteArray(); Buffer.BlockCopy(chunkBytes, 0, buf, buf.Length - ChunkFooter.Size, chunkBytes.Length); if (contents != null) { if (contents.Length != dataSize) { throw new Exception("Wrong contents size."); } Buffer.BlockCopy(contents, 0, buf, ChunkHeader.Size, contents.Length); } File.WriteAllBytes(filename, buf); }
private TFChunkDbConfig CreateOneTimeDbConfig(int chunkSize, string dbPath, int chunksToCache) { if (Runtime.IsMono) { _writerChk = new FileCheckpoint(Path.Combine(dbPath, Checkpoint.Writer + ".chk"), Checkpoint.Writer, cached: true); _chaserChk = new FileCheckpoint(Path.Combine(dbPath, Checkpoint.Chaser + ".chk"), Checkpoint.Chaser, cached: true); } else { _writerChk = new MemoryMappedFileCheckpoint(Path.Combine(dbPath, Checkpoint.Writer + ".chk"), Checkpoint.Writer, cached: true); _chaserChk = new MemoryMappedFileCheckpoint(Path.Combine(dbPath, Checkpoint.Chaser + ".chk"), Checkpoint.Chaser, cached: true); } var nodeConfig = new TFChunkDbConfig(dbPath, new VersionedPatternFileNamingStrategy(dbPath, "chunk-"), chunkSize, chunksToCache, _writerChk, _chaserChk, new[] { _writerChk, _chaserChk }); return(nodeConfig); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1000, 0, new InMemoryCheckpoint(13500), new InMemoryCheckpoint(5500), new InMemoryCheckpoint(5500), new InMemoryCheckpoint(11000)); DbUtil.CreateMultiChunk(_config, 0, 2, GetFilePathFor("chunk-000000.000001")); DbUtil.CreateMultiChunk(_config, 0, 2, GetFilePathFor("chunk-000000.000002")); DbUtil.CreateMultiChunk(_config, 3, 10, GetFilePathFor("chunk-000003.000001")); DbUtil.CreateMultiChunk(_config, 3, 10, GetFilePathFor("chunk-000003.000002")); DbUtil.CreateMultiChunk(_config, 7, 8, GetFilePathFor("chunk-000007.000001")); DbUtil.CreateMultiChunk(_config, 11, 12, GetFilePathFor("chunk-000011.000001")); DbUtil.CreateOngoingChunk(_config, 13, GetFilePathFor("chunk-000013.000000")); var truncator = new TFChunkDbTruncator(_config); truncator.TruncateDb(_config.TruncateCheckpoint.ReadNonFlushed()); }
public void temporary_files_are_removed() { var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 100, 0, new InMemoryCheckpoint(150), new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); using (var db = new TFChunkDb(config)) { DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000")); DbUtil.CreateOngoingChunk(config, 1, GetFilePathFor("chunk-000001.000001")); File.Create(GetFilePathFor("bla")).Close(); File.Create(GetFilePathFor("bla.scavenge.tmp")).Close(); File.Create(GetFilePathFor("bla.tmp")).Close(); Assert.DoesNotThrow(() => db.Open(verifyHash: false)); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000"))); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000001.000001"))); Assert.IsTrue(File.Exists(GetFilePathFor("bla"))); Assert.AreEqual(3, Directory.GetFiles(PathName, "*").Length); } }
public override async Task TestFixtureSetUp() { await base.TestFixtureSetUp(); Publisher.Subscribe(new AdHocHandler <ReplicationTrackingMessage.ReplicaWriteAck>(msg => ReplicaWriteAcks.Enqueue(msg))); Publisher.Subscribe(new AdHocHandler <SystemMessage.VNodeConnectionLost>(msg => ReplicaLostMessages.Enqueue(msg))); TcpSendPublisher.Subscribe(new AdHocHandler <TcpMessage.TcpSend>(msg => TcpSends.Enqueue(msg))); DbConfig = CreateDbConfig(); var db = new TFChunkDb(DbConfig); db.Open(); Service = new LeaderReplicationService( publisher: Publisher, instanceId: LeaderId, db: db, tcpSendPublisher: TcpSendPublisher, epochManager: new FakeEpochManager(), clusterSize: ClusterSize, unsafeAllowSurplusNodes: false, queueStatsManager: new QueueStatsManager()); Service.Handle(new SystemMessage.SystemStart()); Service.Handle(new SystemMessage.BecomeLeader(Guid.NewGuid(), 0)); ReplicaSubscriptionId = AddSubscription(ReplicaId, true, out ReplicaManager1); ReplicaSubscriptionId2 = AddSubscription(ReplicaId2, true, out ReplicaManager2); ReadOnlyReplicaSubscriptionId = AddSubscription(ReadOnlyReplicaId, false, out ReadOnlyReplicaManager); When(); }
private TFChunkDbConfig CreateDbConfig() { ICheckpoint writerChk = new InMemoryCheckpoint(Checkpoint.Writer); ICheckpoint chaserChk = new InMemoryCheckpoint(Checkpoint.Chaser); ICheckpoint epochChk = new InMemoryCheckpoint(Checkpoint.Epoch, initValue: -1); ICheckpoint proposalChk = new InMemoryCheckpoint(Checkpoint.Proposal, initValue: -1); ICheckpoint truncateChk = new InMemoryCheckpoint(Checkpoint.Truncate, initValue: -1); ICheckpoint replicationCheckpoint = new InMemoryCheckpoint(-1); ICheckpoint indexCheckpoint = new InMemoryCheckpoint(-1); ICheckpoint streamExistenceFilterCheckpoint = new InMemoryCheckpoint(-1); var nodeConfig = new TFChunkDbConfig( PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1000, 10000, writerChk, chaserChk, epochChk, proposalChk, truncateChk, replicationCheckpoint, indexCheckpoint, streamExistenceFilterCheckpoint, Constants.TFChunkInitialReaderCountDefault, Constants.TFChunkMaxReaderCountDefault, true); return(nodeConfig); }
protected static TFChunkDbConfig CreateDbConfig(string dbPath, int chunksToCache) { if (!Directory.Exists(dbPath)) // mono crashes without this check { Directory.CreateDirectory(dbPath); } ICheckpoint writerChk; ICheckpoint chaserChk; var writerCheckFilename = Path.Combine(dbPath, Checkpoint.Writer + ".chk"); var chaserCheckFilename = Path.Combine(dbPath, Checkpoint.Chaser + ".chk"); if (Runtime.IsMono) { writerChk = new FileCheckpoint(writerCheckFilename, Checkpoint.Writer, cached: true); chaserChk = new FileCheckpoint(chaserCheckFilename, Checkpoint.Chaser, cached: true); } else { writerChk = new MemoryMappedFileCheckpoint(writerCheckFilename, Checkpoint.Writer, cached: true); chaserChk = new MemoryMappedFileCheckpoint(chaserCheckFilename, Checkpoint.Chaser, cached: true); } var nodeConfig = new TFChunkDbConfig(dbPath, new VersionedPatternFileNamingStrategy(dbPath, "chunk-"), TFConsts.ChunkSize, chunksToCache, writerChk, chaserChk, new[] { writerChk, chaserChk }); return(nodeConfig); }
public void temporary_files_are_removed() { var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 100, 0, new InMemoryCheckpoint(150), new ICheckpoint[0]); var db = new TFChunkDb(config); CreateChunk(Path.Combine(PathName, "chunk-000000.000000"), config.ChunkSize, config.ChunkSize); CreateChunk(Path.Combine(PathName, "chunk-000001.000001"), config.ChunkSize, config.ChunkSize); File.Create(Path.Combine(PathName, "bla")).Close(); File.Create(Path.Combine(PathName, "bla.scavenge.tmp")).Close(); File.Create(Path.Combine(PathName, "bla.tmp")).Close(); Assert.DoesNotThrow(db.OpenVerifyAndClean); Assert.IsTrue(File.Exists(Path.Combine(PathName, "chunk-000000.000000"))); Assert.IsTrue(File.Exists(Path.Combine(PathName, "chunk-000001.000001"))); Assert.IsTrue(File.Exists(Path.Combine(PathName, "bla"))); Assert.AreEqual(3, Directory.GetFiles(PathName, "*").Length); db.Dispose(); }
private TFChunkDbConfig CreateDbConfig() { var nodeConfig = new TFChunkDbConfig( PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1000, 10000, _writerChk, _chaserChk, _epochChk, _truncateChk, _replicationCheckpoint, _indexCheckpoint, Constants.TFChunkInitialReaderCountDefault, Constants.TFChunkMaxReaderCountDefault, true); return(nodeConfig); }
public virtual void TestFixtureSetUp() { _builder = TestVNodeBuilder.AsSingleNode() .RunInMemory(); Given(); _node = _builder.Build(); _settings = ((TestVNodeBuilder)_builder).GetSettings(); _dbConfig = ((TestVNodeBuilder)_builder).GetDbConfig(); _node.Start(); }
protected static TFChunkDbConfig CreateDbConfig(string dbPath, int cachedChunks, long chunksCacheSize, bool inMemDb) { ICheckpoint writerChk; ICheckpoint chaserChk; ICheckpoint epochChk; ICheckpoint truncateChk; if (inMemDb) { writerChk = new InMemoryCheckpoint(Checkpoint.Writer); chaserChk = new InMemoryCheckpoint(Checkpoint.Chaser); epochChk = new InMemoryCheckpoint(Checkpoint.Epoch, initValue: -1); truncateChk = new InMemoryCheckpoint(Checkpoint.Truncate, initValue: -1); } else { if (!Directory.Exists(dbPath)) // mono crashes without this check { Directory.CreateDirectory(dbPath); } var writerCheckFilename = Path.Combine(dbPath, Checkpoint.Writer + ".chk"); var chaserCheckFilename = Path.Combine(dbPath, Checkpoint.Chaser + ".chk"); var epochCheckFilename = Path.Combine(dbPath, Checkpoint.Epoch + ".chk"); var truncateCheckFilename = Path.Combine(dbPath, Checkpoint.Truncate + ".chk"); if (Runtime.IsMono) { writerChk = new FileCheckpoint(writerCheckFilename, Checkpoint.Writer, cached: true); chaserChk = new FileCheckpoint(chaserCheckFilename, Checkpoint.Chaser, cached: true); epochChk = new FileCheckpoint(epochCheckFilename, Checkpoint.Epoch, cached: true, initValue: -1); truncateChk = new FileCheckpoint(truncateCheckFilename, Checkpoint.Truncate, cached: true, initValue: -1); } else { writerChk = new MemoryMappedFileCheckpoint(writerCheckFilename, Checkpoint.Writer, cached: true); chaserChk = new MemoryMappedFileCheckpoint(chaserCheckFilename, Checkpoint.Chaser, cached: true); epochChk = new MemoryMappedFileCheckpoint(epochCheckFilename, Checkpoint.Epoch, cached: true, initValue: -1); truncateChk = new MemoryMappedFileCheckpoint(truncateCheckFilename, Checkpoint.Truncate, cached: true, initValue: -1); } } var cache = cachedChunks >= 0 ? cachedChunks * (long)(TFConsts.ChunkSize + ChunkHeader.Size + ChunkFooter.Size) : chunksCacheSize; var nodeConfig = new TFChunkDbConfig(dbPath, new VersionedPatternFileNamingStrategy(dbPath, "chunk-"), TFConsts.ChunkSize, cache, writerChk, chaserChk, epochChk, truncateChk, inMemDb); return(nodeConfig); }
public virtual void TestFixtureSetUp() { _builder = TestVNodeBuilder.AsSingleNode() .WithServerCertificate(ssl_connections.GetServerCertificate()) .RunInMemory(); Given(); _node = _builder.Build(); _settings = ((TestVNodeBuilder)_builder).GetSettings(); _dbConfig = ((TestVNodeBuilder)_builder).GetDbConfig(); _node.Start(); }
public virtual void TestFixtureSetUp() { _builder = TestVNodeBuilder.AsClusterMember(_clusterSize) .RunInMemory(); _quorumSize = _clusterSize / 2 + 1; Given(); _node = _builder.Build(); _settings = ((TestVNodeBuilder)_builder).GetSettings(); _dbConfig = ((TestVNodeBuilder)_builder).GetDbConfig(); _node.Start(); }
public void a_null_checkpoint_throws_argument_null_exception() { var config = new TFChunkDbConfig(Path.GetTempPath(), new PrefixFileNamingStrategy(Path.GetTempPath(), "prefix.tf"), 10000, 0, new InMemoryCheckpoint(0), new ICheckpoint[0]); var db = new TFChunkDb(config); Assert.Throws <ArgumentNullException>(() => new TFChunkReader(db, null)); }
public void allows_no_files_when_checkpoint_is_zero() { var config = new TFChunkDbConfig(PathName, new PrefixFileNamingStrategy(PathName, "prefix.tf"), 10000, 0, new InMemoryCheckpoint(), new ICheckpoint[0]); var db = new TFChunkDb(config); Assert.DoesNotThrow(db.OpenVerifyAndClean); db.Dispose(); }
public TFChunkDbCreationHelper(TFChunkDbConfig dbConfig) { Ensure.NotNull(dbConfig, "dbConfig"); _dbConfig = dbConfig; _db = new TFChunkDb(_dbConfig); _db.Open(); if (_db.Config.WriterCheckpoint.ReadNonFlushed() > 0) { throw new Exception("The DB already contains some data."); } }
public override async Task TestFixtureSetUp() { await base.TestFixtureSetUp(); // writer checkpoint = 5500, truncate to 0, max truncation = 1000 _config = TFChunkHelper.CreateDbConfig(PathName, 5500, 5500, 5500, 0, 1000, maxTruncation: 1000); DbUtil.CreateMultiChunk(_config, 0, 2, GetFilePathFor("chunk-000000.000001")); DbUtil.CreateMultiChunk(_config, 0, 2, GetFilePathFor("chunk-000000.000002")); DbUtil.CreateMultiChunk(_config, 3, 10, GetFilePathFor("chunk-000003.000001")); DbUtil.CreateMultiChunk(_config, 3, 10, GetFilePathFor("chunk-000003.000002")); DbUtil.CreateMultiChunk(_config, 7, 8, GetFilePathFor("chunk-000007.000001")); DbUtil.CreateOngoingChunk(_config, 11, GetFilePathFor("chunk-000011.000000")); }
public void a_null_checkpoint_throws_argument_null_exception() { var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, new InMemoryCheckpoint(0), new InMemoryCheckpoint(0), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); var db = new TFChunkDb(config); Assert.Throws <ArgumentNullException>(() => new TFChunkReader(db, null)); }
public void allows_first_correct_file_when_checkpoint_is_zero() { var config = new TFChunkDbConfig(PathName, new PrefixFileNamingStrategy(PathName, "prefix.tf"), 10000, 0, new InMemoryCheckpoint(), new ICheckpoint[0]); var db = new TFChunkDb(config); CreateChunk(Path.Combine(PathName, config.FileNamingStrategy.GetFilenameFor(0)), config.ChunkSize, config.ChunkSize); Assert.DoesNotThrow(db.OpenVerifyAndClean); db.Dispose(); }
public void allows_with_exactly_enough_file_to_reach_checksum() { var config = new TFChunkDbConfig(PathName, new PrefixFileNamingStrategy(PathName, "prefix.tf"), 10000, 0, new InMemoryCheckpoint(10000), new ICheckpoint[0]); var db = new TFChunkDb(config); CreateChunk(Path.Combine(PathName, config.FileNamingStrategy.GetFilenameFor(0)), config.ChunkSize, config.ChunkSize); Assert.DoesNotThrow(db.OpenVerifyAndClean); db.Dispose(); }