private void ReOpenDb() { Db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, WriterCheckpoint, ChaserCheckpoint, new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); Db.Open(); var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint)); var lowHasher = new XXHashUnsafe(); var highHasher = new Murmur3AUnsafe(); TableIndex = new TableIndex(Path.Combine(PathName, "index"), lowHasher, highHasher, () => new HashListMemTable(PTableVersions.Index64Bit, MaxEntriesInMemTable * 2), () => new TFReaderLease(readers), PTableVersions.Index64Bit, MaxEntriesInMemTable); ReadIndex = new ReadIndex(new NoopPublisher(), readers, TableIndex, 0, additionalCommitChecks: true, metastreamMaxCount: MetastreamMaxCount, hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault); ReadIndex.Init(ChaserCheckpoint.Read()); }
public void SetUp() { _writerCheckpoint = new InMemoryCheckpoint(); _db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1024, 0, _writerCheckpoint, new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); _db.Open(); _writer = new TFChunkWriter(_db); _writer.Open(); _record = new PrepareLogRecord(logPosition: 0, eventId: _eventId, correlationId: _correlationId, transactionPosition: 0xDEAD, transactionOffset: 0xBEEF, eventStreamId: "WorldEnding", expectedVersion: 1234, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.SingleWrite, eventType: "type", data: new byte[] { 1, 2, 3, 4, 5 }, metadata: new byte[] {7, 17}); long newPos; _writer.Write(_record, out newPos); _writer.Flush(); }
public void try_read_returns_false_when_writer_checksum_is_equal_to_reader_checksum() { var writerchk = new InMemoryCheckpoint(); var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0); var db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, writerchk, chaserchk, new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); db.Open(); writerchk.Write(12); writerchk.Flush(); chaserchk.Write(12); chaserchk.Flush(); var chaser = new TFChunkChaser(db, writerchk, chaserchk); chaser.Open(); LogRecord record; Assert.IsFalse(chaser.TryReadNext(out record)); Assert.AreEqual(12, chaserchk.Read()); chaser.Close(); db.Dispose(); }
private void ReOpenDb() { Db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, WriterCheckpoint, ChaserCheckpoint, new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); Db.Open(); var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint)); TableIndex = new TableIndex(Path.Combine(PathName, "index"), () => new HashListMemTable(MaxEntriesInMemTable * 2), () => new TFReaderLease(readers), MaxEntriesInMemTable); ReadIndex = new ReadIndex(new NoopPublisher(), readers, TableIndex, new ByLengthHasher(), 0, additionalCommitChecks: true, metastreamMaxCount: MetastreamMaxCount); ReadIndex.Init(ChaserCheckpoint.Read()); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 4096, 0, new InMemoryCheckpoint(), new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); _db.Open(); var chunk = _db.Manager.GetChunk(0); _records = new LogRecord[RecordsCount]; _results = new RecordWriteResult[RecordsCount]; for (int i = 0; i < _records.Length; ++i) { _records[i] = LogRecord.SingleWrite(i == 0 ? 0 : _results[i - 1].NewPosition, Guid.NewGuid(), Guid.NewGuid(), "es1", ExpectedVersion.Any, "et1", new byte[] { 0, 1, 2 }, new byte[] { 5, 7 }); _results[i] = chunk.TryAppend(_records[i]); } chunk.Flush(); _db.Config.WriterCheckpoint.Write(_results[RecordsCount - 1].NewPosition); _db.Config.WriterCheckpoint.Flush(); }
public void try_read_does_not_cache_anything_and_returns_record_once_it_is_written_later() { var writerchk = new InMemoryCheckpoint(0); var db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, writerchk, new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); db.Open(); var writer = new TFChunkWriter(db); writer.Open(); var reader = new TFChunkReader(db, writerchk, 0); Assert.IsFalse(reader.TryReadNext().Success); var rec = LogRecord.SingleWrite(0, Guid.NewGuid(), Guid.NewGuid(), "ES", -1, "ET", new byte[] { 7 }, null); long tmp; Assert.IsTrue(writer.Write(rec, out tmp)); writer.Flush(); writer.Close(); var res = reader.TryReadNext(); Assert.IsTrue(res.Success); Assert.AreEqual(rec, res.LogRecord); db.Close(); }
private void ReOpenDb() { Db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, WriterCheckpoint, ChaserCheckpoint, new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); Db.Open(); TableIndex = new TableIndex(Path.Combine(PathName, "index"), () => new HashListMemTable(MaxEntriesInMemTable * 2), MaxEntriesInMemTable); ReadIndex = new ReadIndex(new NoopPublisher(), 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint), TableIndex, new ByLengthHasher(), new NoLRUCache<string, StreamCacheInfo>(), additionalCommitChecks: true, metastreamMaxCount: MetastreamMaxCount); ReadIndex.Init(WriterCheckpoint.Read(), ChaserCheckpoint.Read()); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); WriterCheckpoint = new InMemoryCheckpoint(0); ChaserCheckpoint = new InMemoryCheckpoint(0); Bus = new InMemoryBus("bus"); IODispatcher = new IODispatcher(Bus, new PublishEnvelope(Bus)); Db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, WriterCheckpoint, ChaserCheckpoint, new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); Db.Open(); // create db Writer = new TFChunkWriter(Db); Writer.Open(); WriteTestScenario(); Writer.Close(); Writer = null; WriterCheckpoint.Flush(); ChaserCheckpoint.Write(WriterCheckpoint.Read()); ChaserCheckpoint.Flush(); var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint)); var lowHasher = new XXHashUnsafe(); var highHasher = new Murmur3AUnsafe(); TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher, () => new HashListMemTable(PTableVersions.Index64Bit, MaxEntriesInMemTable * 2), () => new TFReaderLease(readers), PTableVersions.Index64Bit, MaxEntriesInMemTable); ReadIndex = new ReadIndex(new NoopPublisher(), readers, TableIndex, 0, additionalCommitChecks: true, metastreamMaxCount: MetastreamMaxCount, hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault); ReadIndex.Init(ChaserCheckpoint.Read()); // scavenge must run after readIndex is built if (_scavenge) { if (_completeLastChunkOnScavenge) Db.Manager.GetChunk(Db.Manager.ChunksCount - 1).Complete(); _scavenger = new TFChunkScavenger(Db, IODispatcher, TableIndex, ReadIndex, Guid.NewGuid(), "fakeNodeIp"); _scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: _mergeChunks); } }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 16 * 1024, 0, new InMemoryCheckpoint(), new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); _db.Open(); var chunk = _db.Manager.GetChunkFor(0); _p1 = LogRecord.SingleWrite(0, Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1", new byte[] { 0, 1, 2 }, new byte[] { 5, 7 }); _res1 = chunk.TryAppend(_p1); _c1 = LogRecord.Commit(_res1.NewPosition, Guid.NewGuid(), _p1.LogPosition, 0); _cres1 = chunk.TryAppend(_c1); _p2 = LogRecord.SingleWrite(_cres1.NewPosition, Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1", new byte[] { 0, 1, 2 }, new byte[] { 5, 7 }); _res2 = chunk.TryAppend(_p2); _c2 = LogRecord.Commit(_res2.NewPosition, Guid.NewGuid(), _p2.LogPosition, 1); _cres2 = chunk.TryAppend(_c2); _p3 = LogRecord.SingleWrite(_cres2.NewPosition, Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1", new byte[] { 0, 1, 2 }, new byte[] { 5, 7 }); _res3 = chunk.TryAppend(_p3); _c3 = LogRecord.Commit(_res3.NewPosition, Guid.NewGuid(), _p3.LogPosition, 2); _cres3 = chunk.TryAppend(_c3); chunk.Complete(); _db.Config.WriterCheckpoint.Write(chunk.ChunkHeader.ChunkEndPosition); _db.Config.WriterCheckpoint.Flush(); _db.Config.ChaserCheckpoint.Write(chunk.ChunkHeader.ChunkEndPosition); _db.Config.ChaserCheckpoint.Flush(); var bus = new InMemoryBus("Bus"); var ioDispatcher = new IODispatcher(bus, new PublishEnvelope(bus)); var scavenger = new TFChunkScavenger(_db, ioDispatcher, new FakeTableIndex(), new FakeReadIndex(x => x == "es-to-scavenge"), Guid.NewGuid(), "fakeNodeIp"); scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false); _scavengedChunk = _db.Manager.GetChunk(0); }
public override void TestFixtureTearDown() { using (var db = new TFChunkDb(_config)) { Assert.DoesNotThrow(() => db.Open(verifyHash: false)); } Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000"))); Assert.AreEqual(1, Directory.GetFiles(PathName, "*").Length); base.TestFixtureTearDown(); }
public TFChunkDbCreationHelper(TFChunkDbConfig dbConfig) { Ensure.NotNull(dbConfig, "dbConfig"); _dbConfig = dbConfig; _db = new TFChunkDb(_dbConfig); _db.Open(); if (_db.Config.WriterCheckpoint.ReadNonFlushed() > 0) throw new Exception("The DB already contains some data."); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); WriterCheckpoint = new InMemoryCheckpoint(0); ChaserCheckpoint = new InMemoryCheckpoint(0); Db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, WriterCheckpoint, ChaserCheckpoint, new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); Db.Open(); // create db Writer = new TFChunkWriter(Db); Writer.Open(); WriteTestScenario(); Writer.Close(); Writer = null; WriterCheckpoint.Flush(); ChaserCheckpoint.Write(WriterCheckpoint.Read()); ChaserCheckpoint.Flush(); var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 2, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint)); TableIndex = new TableIndex(GetFilePathFor("index"), () => new HashListMemTable(MaxEntriesInMemTable * 2), () => new TFReaderLease(readers), MaxEntriesInMemTable); var hasher = new ByLengthHasher(); ReadIndex = new ReadIndex(new NoopPublisher(), readers, TableIndex, hasher, 0, additionalCommitChecks: true, metastreamMaxCount: MetastreamMaxCount); ReadIndex.Init(ChaserCheckpoint.Read()); // scavenge must run after readIndex is built if (_scavenge) { if (_completeLastChunkOnScavenge) Db.Manager.GetChunk(Db.Manager.ChunksCount - 1).Complete(); _scavenger = new TFChunkScavenger(Db, TableIndex, hasher, ReadIndex); _scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: _mergeChunks); } }
public void with_a_writer_checksum_of_zero_and_no_files_is_valid() { var db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, new InMemoryCheckpoint(0), new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); Assert.DoesNotThrow(() => db.Open()); db.Dispose(); }
public void a_record_can_be_written() { var filename = GetFilePathFor("chunk-000000.000000"); var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, 10000, 0, 0, false, Guid.NewGuid()); var chunkBytes = chunkHeader.AsByteArray(); var buf = new byte[ChunkHeader.Size + ChunkFooter.Size + chunkHeader.ChunkSize]; Buffer.BlockCopy(chunkBytes, 0, buf, 0, chunkBytes.Length); File.WriteAllBytes(filename, buf); _checkpoint = new InMemoryCheckpoint(137); var db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), chunkHeader.ChunkSize, 0, _checkpoint, new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); db.Open(); var bytes = new byte[3994]; // this gives exactly 4097 size of record, with 3993 (rec size 4096) everything works fine! new Random().NextBytes(bytes); var writer = new TFChunkWriter(db); var record = new PrepareLogRecord(logPosition: 137, correlationId: _correlationId, eventId: _eventId, transactionPosition: 789, transactionOffset: 543, eventStreamId: "WorldEnding", expectedVersion: 1234, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.SingleWrite, eventType: "type", data: bytes, metadata: new byte[] { 0x07, 0x17 }); long pos; Assert.IsTrue(writer.Write(record, out pos)); writer.Close(); db.Dispose(); Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix() + 137, _checkpoint.Read()); using (var filestream = File.Open(filename, FileMode.Open, FileAccess.Read)) { filestream.Seek(ChunkHeader.Size + 137 + sizeof(int), SeekOrigin.Begin); var reader = new BinaryReader(filestream); var read = LogRecord.ReadFrom(reader); Assert.AreEqual(record, read); } }
public void with_a_writer_checksum_of_nonzero_and_no_files_a_corrupted_database_exception_is_thrown() { var db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, new InMemoryCheckpoint(500), new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); var exc = Assert.Throws<CorruptDatabaseException>(() => db.Open()); Assert.IsInstanceOf<ChunkNotFoundException>(exc.InnerException); db.Dispose(); }
public void allows_with_exactly_enough_file_to_reach_checksum() { var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, new InMemoryCheckpoint(10000), new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); using (var db = new TFChunkDb(config)) { DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000")); Assert.DoesNotThrow(() => db.Open(verifyHash: false)); } }
public void a_record_can_be_written() { var filename = GetFilePathFor("chunk-000000.000000"); var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, 10000, 0, 0, false, chunkId: Guid.NewGuid()); var chunkBytes = chunkHeader.AsByteArray(); var bytes = new byte[ChunkHeader.Size + 10000 + ChunkFooter.Size]; Buffer.BlockCopy(chunkBytes, 0, bytes, 0, chunkBytes.Length); File.WriteAllBytes(filename, bytes); _checkpoint = new InMemoryCheckpoint(137); var db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, _checkpoint, new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); db.Open(); var tf = new TFChunkWriter(db); var record = new PrepareLogRecord(logPosition: _checkpoint.Read(), correlationId: _correlationId, eventId: _eventId, expectedVersion: 1234, transactionPosition: 0, transactionOffset: 0, eventStreamId: "WorldEnding", timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.None, eventType: "type", data: new byte[] { 1, 2, 3, 4, 5 }, metadata: new byte[] { 7, 17 }); long tmp; tf.Write(record, out tmp); tf.Close(); db.Dispose(); Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix() + 137, _checkpoint.Read()); //137 is fluff assigned to beginning of checkpoint using (var filestream = File.Open(filename, FileMode.Open, FileAccess.Read)) { filestream.Seek(ChunkHeader.Size + 137 + sizeof(int), SeekOrigin.Begin); var reader = new BinaryReader(filestream); var read = LogRecord.ReadFrom(reader); Assert.AreEqual(record, read); } }
public void try_read_returns_false_when_writer_checksum_is_zero() { var writerchk = new InMemoryCheckpoint(0); var db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, writerchk, new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); db.Open(); var reader = new TFChunkReader(db, writerchk, 0); Assert.IsFalse(reader.TryReadNext().Success); db.Close(); }
public void with_file_of_wrong_size_database_corruption_is_detected() { var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, new InMemoryCheckpoint(500), new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); using (var db = new TFChunkDb(config)) { File.WriteAllText(GetFilePathFor("chunk-000000.000000"), "this is just some test blahbydy blah"); Assert.That(() => db.Open(verifyHash: false), Throws.Exception.InstanceOf<CorruptDatabaseException>() .With.InnerException.InstanceOf<BadChunkInDatabaseException>()); } }
public void with_checksum_inside_multi_chunk_throws() { var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, new InMemoryCheckpoint(25000), new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); using (var db = new TFChunkDb(config)) { DbUtil.CreateMultiChunk(config, 0, 2, GetFilePathFor("chunk-000000.000000")); Assert.That(() => db.Open(verifyHash: false), Throws.Exception.InstanceOf<CorruptDatabaseException>() .With.InnerException.InstanceOf<ChunkNotFoundException>()); } }
public void with_a_writer_checksum_of_zero_the_first_chunk_is_created_with_correct_name() { var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, new InMemoryCheckpoint(0), new InMemoryCheckpoint(0), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); var db = new TFChunkDb(config); db.Open(); db.Dispose(); Assert.AreEqual(1, Directory.GetFiles(PathName).Length); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000"))); var fileInfo = new FileInfo(GetFilePathFor("chunk-000000.000000")); Assert.AreEqual(10000 + ChunkHeader.Size + ChunkFooter.Size, fileInfo.Length); }
public void a_record_can_be_written() { _checkpoint = new InMemoryCheckpoint(0); var db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1000, 0, _checkpoint, new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); db.Open(); var tf = new TFChunkWriter(db); tf.Open(); var record = new PrepareLogRecord(logPosition: 0, correlationId: _correlationId, eventId: _eventId, transactionPosition: 0, transactionOffset: 0, eventStreamId: "WorldEnding", expectedVersion: 1234, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.None, eventType: "type", data: new byte[] { 1, 2, 3, 4, 5 }, metadata: new byte[] { 7, 17 }); long tmp; tf.Write(record, out tmp); tf.Close(); db.Dispose(); Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix(), _checkpoint.Read()); using (var filestream = File.Open(GetFilePathFor("chunk-000000.000000"), FileMode.Open, FileAccess.Read)) { filestream.Position = ChunkHeader.Size; var reader = new BinaryReader(filestream); reader.ReadInt32(); var read = LogRecord.ReadFrom(reader); Assert.AreEqual(record, read); } }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 4096, 0, new InMemoryCheckpoint(), new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); _db.Open(); var chunk = _db.Manager.GetChunk(0); _records = new LogRecord[RecordsCount]; _results = new RecordWriteResult[RecordsCount]; var pos = 0; for (int i = 0; i < RecordsCount; ++i) { if (i > 0 && i % 3 == 0) { pos = i/3 * _db.Config.ChunkSize; chunk.Complete(); chunk = _db.Manager.AddNewChunk(); } _records[i] = LogRecord.SingleWrite(pos, Guid.NewGuid(), Guid.NewGuid(), "es1", ExpectedVersion.Any, "et1", new byte[1200], new byte[] { 5, 7 }); _results[i] = chunk.TryAppend(_records[i]); pos += _records[i].GetSizeWithLengthPrefixAndSuffix(); } chunk.Flush(); _db.Config.WriterCheckpoint.Write((RecordsCount / 3) * _db.Config.ChunkSize + _results[RecordsCount - 1].NewPosition); _db.Config.WriterCheckpoint.Flush(); }
public void allows_with_exactly_enough_file_to_reach_checksum_while_last_is_multi_chunk() { var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, new InMemoryCheckpoint(30000), new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); using (var db = new TFChunkDb(config)) { DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000")); DbUtil.CreateMultiChunk(config, 1, 2, GetFilePathFor("chunk-000001.000000")); Assert.DoesNotThrow(() => db.Open(verifyHash: false)); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000"))); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000001.000000"))); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000003.000000"))); Assert.AreEqual(3, Directory.GetFiles(PathName, "*").Length); } }
public void try_read_returns_false_when_writer_checkpoint_is_zero() { var writerchk = new InMemoryCheckpoint(0); var db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, writerchk, new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); db.Open(); var chaser = new TFChunkChaser(db, writerchk, new InMemoryCheckpoint()); chaser.Open(); LogRecord record; Assert.IsFalse(chaser.TryReadNext(out record)); chaser.Close(); db.Dispose(); }
public void SetUp() { _writerCheckpoint = new InMemoryCheckpoint(); _db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1024, 0, _writerCheckpoint, new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); _db.Open(); _writer = new TFChunkWriter(_db); _writer.Open(); _record = new CommitLogRecord(logPosition: 0, correlationId: _eventId, transactionPosition: 4321, timeStamp: new DateTime(2012, 12, 21), firstEventNumber: 10); long newPos; _writer.Write(_record, out newPos); _writer.Flush(); }
public SingleVNode(TFChunkDb db, SingleVNodeSettings vNodeSettings, bool dbVerifyHashes, int memTableEntryCount, params ISubsystem[] subsystems) { Ensure.NotNull(db, "db"); Ensure.NotNull(vNodeSettings, "vNodeSettings"); _settings = vNodeSettings; _mainBus = new InMemoryBus("MainBus"); _controller = new SingleVNodeController(_mainBus, _settings.ExternalHttpEndPoint, db, this); _mainQueue = new QueuedHandler(_controller, "MainQueue"); _controller.SetMainQueue(_mainQueue); _subsystems = subsystems; // MONITORING var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false); var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false); var monitoringQueue = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100)); var monitoring = new MonitoringService(monitoringQueue, monitoringRequestBus, _mainQueue, db.Config.WriterCheckpoint, db.Config.Path, vNodeSettings.StatsPeriod, _settings.ExternalHttpEndPoint, vNodeSettings.StatsStorage); _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.SystemInit, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.StateChangeMessage, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShutdown, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom<ClientMessage.WriteEventsCompleted, Message>()); monitoringInnerBus.Subscribe<SystemMessage.SystemInit>(monitoring); monitoringInnerBus.Subscribe<SystemMessage.StateChangeMessage>(monitoring); monitoringInnerBus.Subscribe<SystemMessage.BecomeShuttingDown>(monitoring); monitoringInnerBus.Subscribe<SystemMessage.BecomeShutdown>(monitoring); monitoringInnerBus.Subscribe<ClientMessage.WriteEventsCompleted>(monitoring); monitoringInnerBus.Subscribe<MonitoringMessage.GetFreshStats>(monitoring); var truncPos = db.Config.TruncateCheckpoint.Read(); if (truncPos != -1) { Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})", truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read()); var truncator = new TFChunkDbTruncator(db.Config); truncator.TruncateDb(truncPos); } db.Open(dbVerifyHashes); // STORAGE SUBSYSTEM var indexPath = Path.Combine(db.Config.Path, "index"); var readerPool = new ObjectPool<ITransactionFileReader>( "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount, () => new TFChunkReader(db, db.Config.WriterCheckpoint)); var tableIndex = new TableIndex(indexPath, () => new HashListMemTable(maxSize: memTableEntryCount * 2), () => new TFReaderLease(readerPool), maxSizeForMemory: memTableEntryCount, maxTablesPerLevel: 2, inMem: db.Config.InMemDb); var hasher = new XXHashUnsafe(); var readIndex = new ReadIndex(_mainQueue, readerPool, tableIndex, hasher, ESConsts.StreamInfoCacheCapacity, Application.IsDefined(Application.AdditionalCommitChecks), Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1); var writer = new TFChunkWriter(db); var epochManager = new EpochManager(ESConsts.CachedEpochCount, db.Config.EpochCheckpoint, writer, initialReaderCount: 1, maxReaderCount: 5, readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint)); epochManager.Init(); var storageWriter = new StorageWriterService(_mainQueue, _mainBus, _settings.MinFlushDelay, db, writer, readIndex.IndexWriter, epochManager); // subscribes internally monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageWriter); var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, ESConsts.StorageReaderThreadCount, db.Config.WriterCheckpoint); _mainBus.Subscribe<SystemMessage.SystemInit>(storageReader); _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageReader); _mainBus.Subscribe<SystemMessage.BecomeShutdown>(storageReader); monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageReader); var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint); var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, readIndex.IndexCommitter, epochManager); _mainBus.Subscribe<SystemMessage.SystemInit>(storageChaser); _mainBus.Subscribe<SystemMessage.SystemStart>(storageChaser); _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageChaser); var storageScavenger = new StorageScavenger(db, tableIndex, hasher, readIndex, Application.IsDefined(Application.AlwaysKeepScavenged), mergeChunks: false /*!Application.IsDefined(Application.DisableMergeChunks)*/); // ReSharper disable RedundantTypeArgumentsOfMethod _mainBus.Subscribe<ClientMessage.ScavengeDatabase>(storageScavenger); // ReSharper restore RedundantTypeArgumentsOfMethod // MISC WORKERS _workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum => new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1), watchSlowMsg: true, slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray(); _workersHandler = new MultiQueuedHandler( vNodeSettings.WorkerThreads, queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum], string.Format("Worker #{0}", queueNum + 1), groupName: "Workers", watchSlowMsg: true, slowMsgThreshold: TimeSpan.FromMilliseconds(200))); // AUTHENTICATION INFRASTRUCTURE var passwordHashAlgorithm = new Rfc2898PasswordHashAlgorithm(); var dispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_workersHandler, crossThread: true)); var internalAuthenticationProvider = new InternalAuthenticationProvider(dispatcher, passwordHashAlgorithm, ESConsts.CachedPrincipalCount); var passwordChangeNotificationReader = new PasswordChangeNotificationReader(_mainQueue, dispatcher); _mainBus.Subscribe<SystemMessage.SystemStart>(passwordChangeNotificationReader); _mainBus.Subscribe<SystemMessage.BecomeShutdown>(passwordChangeNotificationReader); _mainBus.Subscribe(internalAuthenticationProvider); SubscribeWorkers(bus => { bus.Subscribe(dispatcher.ForwardReader); bus.Subscribe(dispatcher.BackwardReader); bus.Subscribe(dispatcher.Writer); bus.Subscribe(dispatcher.StreamDeleter); bus.Subscribe(dispatcher); }); // TCP { var tcpService = new TcpService(_mainQueue, _settings.ExternalTcpEndPoint, _workersHandler, TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(), ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout, internalAuthenticationProvider, null); _mainBus.Subscribe<SystemMessage.SystemInit>(tcpService); _mainBus.Subscribe<SystemMessage.SystemStart>(tcpService); _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(tcpService); } // SECURE TCP if (vNodeSettings.ExternalSecureTcpEndPoint != null) { var secureTcpService = new TcpService(_mainQueue, _settings.ExternalSecureTcpEndPoint, _workersHandler, TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(), ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout, internalAuthenticationProvider, _settings.Certificate); _mainBus.Subscribe<SystemMessage.SystemInit>(secureTcpService); _mainBus.Subscribe<SystemMessage.SystemStart>(secureTcpService); _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(secureTcpService); } SubscribeWorkers(bus => { var tcpSendService = new TcpSendService(); // ReSharper disable RedundantTypeArgumentsOfMethod bus.Subscribe<TcpMessage.TcpSend>(tcpSendService); // ReSharper restore RedundantTypeArgumentsOfMethod }); // HTTP var httpAuthenticationProviders = new List<HttpAuthenticationProvider> { new BasicHttpAuthenticationProvider(internalAuthenticationProvider), }; if (_settings.EnableTrustedAuth) httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider()); httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider()); var httpPipe = new HttpMessagePipe(); var httpSendService = new HttpSendService(httpPipe, forwardRequests: false); _mainBus.Subscribe<SystemMessage.StateChangeMessage>(httpSendService); _mainBus.Subscribe(new WideningHandler<HttpMessage.SendOverHttp, Message>(_workersHandler)); SubscribeWorkers(bus => { bus.Subscribe<HttpMessage.HttpSend>(httpSendService); bus.Subscribe<HttpMessage.HttpSendPart>(httpSendService); bus.Subscribe<HttpMessage.HttpBeginSend>(httpSendService); bus.Subscribe<HttpMessage.HttpEndSend>(httpSendService); bus.Subscribe<HttpMessage.SendOverHttp>(httpSendService); }); _httpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(), _workersHandler, vNodeSettings.HttpPrefixes); _httpService.SetupController(new AdminController(_mainQueue)); _httpService.SetupController(new PingController()); _httpService.SetupController(new StatController(monitoringQueue, _workersHandler)); _httpService.SetupController(new AtomController(httpSendService, _mainQueue, _workersHandler)); _httpService.SetupController(new GuidController(_mainQueue)); _httpService.SetupController(new UsersController(httpSendService, _mainQueue, _workersHandler)); _mainBus.Subscribe<SystemMessage.SystemInit>(_httpService); _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(_httpService); _mainBus.Subscribe<HttpMessage.PurgeTimedOutRequests>(_httpService); SubscribeWorkers(bus => { HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray()); }); // REQUEST MANAGEMENT var requestManagement = new RequestManagementService(_mainQueue, 1, 1, vNodeSettings.PrepareTimeout, vNodeSettings.CommitTimeout); _mainBus.Subscribe<SystemMessage.SystemInit>(requestManagement); _mainBus.Subscribe<ClientMessage.WriteEvents>(requestManagement); _mainBus.Subscribe<ClientMessage.TransactionStart>(requestManagement); _mainBus.Subscribe<ClientMessage.TransactionWrite>(requestManagement); _mainBus.Subscribe<ClientMessage.TransactionCommit>(requestManagement); _mainBus.Subscribe<ClientMessage.DeleteStream>(requestManagement); _mainBus.Subscribe<StorageMessage.RequestCompleted>(requestManagement); _mainBus.Subscribe<StorageMessage.CheckStreamAccessCompleted>(requestManagement); _mainBus.Subscribe<StorageMessage.AlreadyCommitted>(requestManagement); _mainBus.Subscribe<StorageMessage.CommitAck>(requestManagement); _mainBus.Subscribe<StorageMessage.PrepareAck>(requestManagement); _mainBus.Subscribe<StorageMessage.WrongExpectedVersion>(requestManagement); _mainBus.Subscribe<StorageMessage.InvalidTransaction>(requestManagement); _mainBus.Subscribe<StorageMessage.StreamDeleted>(requestManagement); _mainBus.Subscribe<StorageMessage.RequestManagerTimerTick>(requestManagement); // SUBSCRIPTIONS var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50)); var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false); _mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.SystemStart, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom<TcpMessage.ConnectionClosed, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.SubscribeToStream, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.UnsubscribeFromStream, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.PollStream, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.CheckPollTimeout, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom<StorageMessage.EventCommited, Message>()); var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex); subscrBus.Subscribe<SystemMessage.SystemStart>(subscription); subscrBus.Subscribe<SystemMessage.BecomeShuttingDown>(subscription); subscrBus.Subscribe<TcpMessage.ConnectionClosed>(subscription); subscrBus.Subscribe<ClientMessage.SubscribeToStream>(subscription); subscrBus.Subscribe<ClientMessage.UnsubscribeFromStream>(subscription); subscrBus.Subscribe<SubscriptionMessage.PollStream>(subscription); subscrBus.Subscribe<SubscriptionMessage.CheckPollTimeout>(subscription); subscrBus.Subscribe<StorageMessage.EventCommited>(subscription); // USER MANAGEMENT var ioDispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_mainQueue)); _mainBus.Subscribe(ioDispatcher.BackwardReader); _mainBus.Subscribe(ioDispatcher.ForwardReader); _mainBus.Subscribe(ioDispatcher.Writer); _mainBus.Subscribe(ioDispatcher.StreamDeleter); _mainBus.Subscribe(ioDispatcher); var userManagement = new UserManagementService( _mainQueue, ioDispatcher, passwordHashAlgorithm, vNodeSettings.SkipInitializeStandardUsersCheck); _mainBus.Subscribe<UserManagementMessage.Create>(userManagement); _mainBus.Subscribe<UserManagementMessage.Update>(userManagement); _mainBus.Subscribe<UserManagementMessage.Enable>(userManagement); _mainBus.Subscribe<UserManagementMessage.Disable>(userManagement); _mainBus.Subscribe<UserManagementMessage.Delete>(userManagement); _mainBus.Subscribe<UserManagementMessage.ResetPassword>(userManagement); _mainBus.Subscribe<UserManagementMessage.ChangePassword>(userManagement); _mainBus.Subscribe<UserManagementMessage.Get>(userManagement); _mainBus.Subscribe<UserManagementMessage.GetAll>(userManagement); _mainBus.Subscribe<SystemMessage.BecomeMaster>(userManagement); // TIMER _timeProvider = new RealTimeProvider(); _timerService = new TimerService(new ThreadBasedScheduler(_timeProvider)); _mainBus.Subscribe<SystemMessage.BecomeShutdown>(_timerService); _mainBus.Subscribe<TimerMessage.Schedule>(_timerService); _workersHandler.Start(); _mainQueue.Start(); monitoringQueue.Start(); subscrQueue.Start(); if (subsystems != null) foreach (var subsystem in subsystems) subsystem.Register(db, _mainQueue, _mainBus, _timerService, _timeProvider, httpSendService, new[]{_httpService}, _workersHandler); }
public void allows_last_chunk_to_be_multichunk_when_checkpoint_point_at_the_start_of_next_chunk() { var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1000, 0, new InMemoryCheckpoint(4000), new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); using (var db = new TFChunkDb(config)) { DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000")); DbUtil.CreateMultiChunk(config, 1, 3, GetFilePathFor("chunk-000001.000001")); Assert.DoesNotThrow(() => db.Open(verifyHash: false)); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000"))); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000001.000001"))); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000004.000000"))); Assert.AreEqual(3, Directory.GetFiles(PathName, "*").Length); } }
public void does_not_allow_checkpoint_to_point_into_the_middle_of_multichunk_chunk() { var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1000, 0, new InMemoryCheckpoint(1500), new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); using (var db = new TFChunkDb(config)) { DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000")); DbUtil.CreateMultiChunk(config, 1, 10, GetFilePathFor("chunk-000001.000001")); Assert.That(() => db.Open(verifyHash: false), Throws.Exception.InstanceOf<CorruptDatabaseException>() .With.InnerException.InstanceOf<BadChunkInDatabaseException>()); } }
public void when_checkpoint_is_on_boundary_of_new_chunk_and_last_chunk_is_truncated_no_exception_is_thrown() { var config = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 100, 0, new InMemoryCheckpoint(300), new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); using (var db = new TFChunkDb(config)) { DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000")); DbUtil.CreateMultiChunk(config, 1, 2, GetFilePathFor("chunk-000001.000001"), physicalSize: 50, logicalSize: 150); Assert.DoesNotThrow(() => db.Open(verifyHash: false)); Assert.IsNotNull(db.Manager.GetChunk(2)); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000"))); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000001.000001"))); Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000003.000000"))); Assert.AreEqual(3, Directory.GetFiles(PathName, "*").Length); } }