public void all_records_could_be_read_doing_forward_backward_pass() { var seqReader = new TFChunkReader(_db, _db.Config.WriterCheckpoint, 0); SeqReadResult res; int count1 = 0; while ((res = seqReader.TryReadNext()).Success) { var rec = _records[count1]; Assert.AreEqual(rec, res.LogRecord); Assert.AreEqual(rec.LogPosition, res.RecordPrePosition); Assert.AreEqual(rec.LogPosition + rec.GetSizeWithLengthPrefixAndSuffix(), res.RecordPostPosition); ++count1; } Assert.AreEqual(RecordsCount, count1); int count2 = 0; while ((res = seqReader.TryReadPrev()).Success) { var rec = _records[RecordsCount - count2 - 1]; Assert.AreEqual(rec, res.LogRecord); Assert.AreEqual(rec.LogPosition, res.RecordPrePosition); Assert.AreEqual(rec.LogPosition + rec.GetSizeWithLengthPrefixAndSuffix(), res.RecordPostPosition); ++count2; } Assert.AreEqual(RecordsCount, count2); }
public void try_read_does_not_cache_anything_and_returns_record_once_it_is_written_later() { var writerchk = new InMemoryCheckpoint(0); var chaserchk = new InMemoryCheckpoint(0); var db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, writerchk, chaserchk)); db.Open(); var writer = new TFChunkWriter(db); writer.Open(); var reader = new TFChunkReader(db, writerchk, 0); Assert.IsFalse(reader.TryReadNext().Success); var rec = LogRecord.SingleWrite(0, Guid.NewGuid(), Guid.NewGuid(), "ES", -1, "ET", new byte[] { 7 }, null); long tmp; Assert.IsTrue(writer.Write(rec, out tmp)); writer.Flush(); writer.Close(); var res = reader.TryReadNext(); Assert.IsTrue(res.Success); Assert.AreEqual(rec, res.LogRecord); db.Close(); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); WriterCheckpoint = new InMemoryCheckpoint(0); var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0); Db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, WriterCheckpoint, new[] { chaserchk })); Db.OpenVerifyAndClean(); // create db Writer = new TFChunkWriter(Db); Writer.Open(); WriteTestScenario(); Writer.Close(); Writer = null; WriterCheckpoint.Flush(); chaserchk.Write(WriterCheckpoint.Read()); chaserchk.Flush(); TableIndex = new TableIndex(Path.Combine(PathName, "index"), () => new HashListMemTable(), _maxEntriesInMemTable); TableIndex.Initialize(); var reader = new TFChunkReader(Db, Db.Config.WriterCheckpoint); ReadIndex = new ReadIndex(new NoopPublisher(), 2, () => new TFChunkSequentialReader(Db, Db.Config.WriterCheckpoint, 0), () => reader, TableIndex, new ByLengthHasher()); ReadIndex.Build(); }
public override async Task TestFixtureSetUp() { await base.TestFixtureSetUp(); var indexDirectory = GetFilePathFor("index"); _logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormatFactory.Create(new() { IndexDirectory = indexDirectory, }); _mainBus = new InMemoryBus(nameof(when_starting_having_TFLog_with_existing_epochs <TLogFormat, TStreamId>)); _mainBus.Subscribe(new AdHocHandler <SystemMessage.EpochWritten>(m => _published.Add(m))); _db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, 0)); _db.Open(); _reader = new TFChunkReader(_db, _db.Config.WriterCheckpoint); _writer = new TFChunkWriter(_db); _epochs = new List <EpochRecord>(); var lastPos = 0L; for (int i = 0; i < 30; i++) { var epoch = WriteEpoch(GetNextEpoch(), lastPos, _instanceId); _epochs.Add(epoch); lastPos = epoch.EpochPosition; } }
public override async Task TestFixtureSetUp() { await base.TestFixtureSetUp(); _mainBus = new InMemoryBus(nameof(when_having_an_epoch_manager_and_empty_tf_log <TLogFormat, TStreamId>)); _mainBus.Subscribe(new AdHocHandler <SystemMessage.EpochWritten>(m => _published.Add(m))); _db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, 0)); _db.Open(); _reader = new TFChunkReader(_db, _db.Config.WriterCheckpoint); _writer = new TFChunkWriter(_db); _epochManager = GetManager(); _epochManager.Init(); _cache = GetCache(_epochManager); Assert.NotNull(_cache); Assert.That(_cache.Count == 0); _epochs = new List <EpochRecord>(); var lastPos = 0L; for (int i = 0; i < 30; i++) { var epoch = WriteEpoch(GetNextEpoch(), lastPos, _instanceId); _epochs.Add(epoch); lastPos = epoch.EpochPosition; } }
public void trying_to_read_past_writer_checksum_returns_false() { using (var reader = new TFChunkReader(_db, _writerCheckpoint)) { reader.Open(); Assert.IsFalse(reader.TryReadAt(_writerCheckpoint.Read()).Success); } }
public override async Task TestFixtureSetUp() { await base.TestFixtureSetUp(); _mainBus = new InMemoryBus(nameof(when_starting_having_TFLog_with_no_epochs <TLogFormat, TStreamId>)); _mainBus.Subscribe(new AdHocHandler <SystemMessage.EpochWritten>(m => _published.Add(m))); _db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, 0)); _db.Open(); _reader = new TFChunkReader(_db, _db.Config.WriterCheckpoint); _writer = new TFChunkWriter(_db); }
public override async Task TestFixtureSetUp() { await base.TestFixtureSetUp(); _mainBus = new InMemoryBus(nameof(when_having_an_epoch_manager_and_empty_tf_log)); _mainBus.Subscribe(new AdHocHandler <SystemMessage.EpochWritten>(m => _published.Add(m))); _db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, 0)); _db.Open(); _reader = new TFChunkReader(_db, _db.Config.WriterCheckpoint); _writer = new TFChunkWriter(_db); }
public void only_the_last_record_is_marked_eof() { var seqReader = new TFChunkReader(_db, _db.Config.WriterCheckpoint, 0); SeqReadResult res; int count = 0; while ((res = seqReader.TryReadNext()).Success) { ++count; Assert.AreEqual(count == RecordsCount, res.Eof); } Assert.AreEqual(RecordsCount, count); }
public void try_read_returns_false_when_writer_checksum_is_zero() { var writerchk = new InMemoryCheckpoint(0); var chaserchk = new InMemoryCheckpoint(0); var db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, writerchk, chaserchk)); db.Open(); var reader = new TFChunkReader(db, writerchk, 0); Assert.IsFalse(reader.TryReadNext().Success); db.Close(); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); WriterChecksum = new InMemoryCheckpoint(0); ChaserChecksum = new InMemoryCheckpoint(0); Db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, WriterChecksum, ChaserChecksum, new[] { WriterChecksum, ChaserChecksum })); Db.OpenVerifyAndClean(); // create db Writer = new TFChunkWriter(Db); Writer.Open(); WriteTestScenario(); Writer.Close(); Writer = null; WriterChecksum.Flush(); ChaserChecksum.Write(WriterChecksum.Read()); ChaserChecksum.Flush(); TableIndex = new TableIndex(Path.Combine(PathName, "index"), () => new HashListMemTable(MaxEntriesInMemTable * 2), MaxEntriesInMemTable); var reader = new TFChunkReader(Db, Db.Config.WriterCheckpoint); ReadIndex = new ReadIndex(new NoopPublisher(), 2, () => new TFChunkSequentialReader(Db, Db.Config.WriterCheckpoint, 0), () => reader, TableIndex, new ByLengthHasher(), new NoLRUCache <string, StreamCacheInfo>()); ReadIndex.Build(); // scavenge must run after readIndex is built if (_scavenge) { _scavenger = new TFChunkScavenger(Db, ReadIndex); _scavenger.Scavenge(alwaysKeepScavenged: true); } }
public override async Task TestFixtureSetUp() { await base.TestFixtureSetUp(); _mainBus = new InMemoryBus(nameof(when_having_an_epoch_manager_and_empty_tf_log <TLogFormat, TStreamId>)); _mainBus.Subscribe(new AdHocHandler <SystemMessage.EpochWritten>(m => _published.Add(m))); _db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, 0)); _db.Open(); _reader = new TFChunkReader(_db, _db.Config.WriterCheckpoint); _writer = new TFChunkWriter(_db); _epochManager = GetManager(); _epochManager.Init(); _cache = GetCache(_epochManager); Assert.NotNull(_cache); }
public void records_can_be_read_backward_starting_from_any_position() { for (int i = 0; i < RecordsCount; ++i) { var seqReader = new TFChunkReader(_db, _db.Config.WriterCheckpoint, _records[i].LogPosition); SeqReadResult res; int count = 0; while ((res = seqReader.TryReadPrev()).Success) { var rec = _records[i - count - 1]; Assert.AreEqual(rec, res.LogRecord); Assert.AreEqual(rec.LogPosition, res.RecordPrePosition); Assert.AreEqual(rec.LogPosition + rec.GetSizeWithLengthPrefixAndSuffix(), res.RecordPostPosition); ++count; } Assert.AreEqual(i, count); } }
public void try_read_returns_false_when_writer_checksum_is_zero() { var writerchk = new InMemoryCheckpoint(0); var db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, writerchk, new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); db.Open(); var reader = new TFChunkReader(db, writerchk, 0); Assert.IsFalse(reader.TryReadNext().Success); db.Close(); }