public void should_not_read_buffered_data() { var writerchk = new InMemoryCheckpoint(0); var readerchk = new InMemoryCheckpoint("reader", 0); var config = new TransactionFileDatabaseConfig(PathName, "prefix.tf", 10000, writerchk, new List <ICheckpoint> { readerchk }); var fileName = Path.Combine(PathName, "prefix.tf0"); File.Create(fileName).Close(); var reader = new MultifileTransactionFileBulkRetriever(config); reader.Open(0); Assert.IsTrue(reader.ReadNextBulk().Length == 0); var bytes = new byte[100]; new Random().NextBytes(bytes); using (var f = new FileStream(fileName, FileMode.Append, FileAccess.Write, FileShare.ReadWrite)) { f.Write(bytes, 0, bytes.Length); f.Flush(flushToDisk: true); } writerchk.Write(bytes.Length); var readBytes = reader.ReadNextBulk(); Assert.AreEqual(bytes, readBytes); bytes = new byte[100]; new Random().NextBytes(bytes); using (var f = new FileStream(fileName, FileMode.Append, FileAccess.Write, FileShare.ReadWrite)) { f.Write(bytes, 0, bytes.Length); f.Flush(flushToDisk: true); } writerchk.Write(writerchk.Read() + bytes.Length); readBytes = reader.ReadNextBulk(); Assert.AreEqual(bytes, readBytes); reader.Close(); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); WriterCheckpoint = new InMemoryCheckpoint(0); var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0); Db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, WriterCheckpoint, new[] { chaserchk })); Db.OpenVerifyAndClean(); // create db Writer = new TFChunkWriter(Db); Writer.Open(); WriteTestScenario(); Writer.Close(); Writer = null; WriterCheckpoint.Flush(); chaserchk.Write(WriterCheckpoint.Read()); chaserchk.Flush(); TableIndex = new TableIndex(Path.Combine(PathName, "index"), () => new HashListMemTable(), _maxEntriesInMemTable); TableIndex.Initialize(); var reader = new TFChunkReader(Db, Db.Config.WriterCheckpoint); ReadIndex = new ReadIndex(new NoopPublisher(), 2, () => new TFChunkSequentialReader(Db, Db.Config.WriterCheckpoint, 0), () => reader, TableIndex, new ByLengthHasher()); ReadIndex.Build(); }
public void try_read_returns_false_when_writer_checksum_is_equal_to_reader_checksum() { var writerchk = new InMemoryCheckpoint(); var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0); var db = new TFChunkDb(new TFChunkDbConfig(PathName, new PrefixFileNamingStrategy(PathName, "prefix.tf"), 10000, 0, writerchk, chaserchk, new[] { writerchk, chaserchk })); db.OpenVerifyAndClean(); writerchk.Write(12); writerchk.Flush(); chaserchk.Write(12); chaserchk.Flush(); var chaser = new TFChunkChaser(db, writerchk, chaserchk); chaser.Open(); LogRecord record; Assert.IsFalse(chaser.TryReadNext(out record)); Assert.AreEqual(12, chaserchk.Read()); chaser.Close(); db.Dispose(); }
public void try_read_returns_record_when_record_bigger_than_internal_buffer() { var writerchk = new InMemoryCheckpoint(0); var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0); var db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, writerchk, chaserchk, new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); db.Open(); var recordToWrite = new PrepareLogRecord(logPosition: 0, correlationId: _correlationId, eventId: _eventId, transactionPosition: 0, transactionOffset: 0, eventStreamId: "WorldEnding", expectedVersion: 1234, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.None, eventType: "type", data: new byte[9000], metadata: new byte[] { 7, 17 }); var writer = new TFChunkWriter(db); writer.Open(); long pos; Assert.IsTrue(writer.Write(recordToWrite, out pos)); writer.Close(); writerchk.Write(recordToWrite.GetSizeWithLengthPrefixAndSuffix()); var reader = new TFChunkChaser(db, writerchk, chaserchk); reader.Open(); LogRecord record; var readRecord = reader.TryReadNext(out record); reader.Close(); Assert.IsTrue(readRecord); Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix(), chaserchk.Read()); Assert.AreEqual(recordToWrite, record); db.Close(); }
public void try_read_returns_record_when_record_bigger_than_internal_buffer() { var writerchk = new InMemoryCheckpoint(0); var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0); var db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, writerchk, chaserchk)); db.Open(); var recordFactory = LogFormatHelper <TLogFormat, TStreamId> .RecordFactory; var streamId = LogFormatHelper <TLogFormat, TStreamId> .StreamId; var eventTypeId = LogFormatHelper <TLogFormat, TStreamId> .EventTypeId; var recordToWrite = LogRecord.Prepare( factory: recordFactory, logPosition: 0, correlationId: _correlationId, eventId: _eventId, transactionPos: 0, transactionOffset: 0, eventStreamId: streamId, expectedVersion: 1234, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.None, eventType: eventTypeId, data: new byte[9000], metadata: new byte[] { 7, 17 }); var writer = new TFChunkWriter(db); writer.Open(); long pos; Assert.IsTrue(writer.Write(recordToWrite, out pos)); writer.Close(); writerchk.Write(recordToWrite.GetSizeWithLengthPrefixAndSuffix()); var reader = new TFChunkChaser(db, writerchk, chaserchk, false); reader.Open(); ILogRecord record; var readRecord = reader.TryReadNext(out record); reader.Close(); Assert.IsTrue(readRecord); Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix(), chaserchk.Read()); Assert.AreEqual(recordToWrite, record); db.Close(); }
public void try_read_returns_record_when_writerchecksum_equal() { var writerchk = new InMemoryCheckpoint(0); var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0); var db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, writerchk, chaserchk)); db.Open(); _logFormat.StreamNameIndex.GetOrAddId("WorldEnding", out var streamId, out _, out _); var recordToWrite = LogRecord.Prepare( factory: _logFormat.RecordFactory, logPosition: 0, correlationId: _correlationId, eventId: _eventId, transactionPos: 0, transactionOffset: 0, eventStreamId: streamId, expectedVersion: 1234, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.None, eventType: "type", data: new byte[] { 1, 2, 3, 4, 5 }, metadata: new byte[] { 7, 17 }); var writer = new TFChunkWriter(db); writer.Open(); long pos; Assert.IsTrue(writer.Write(recordToWrite, out pos)); writer.Close(); writerchk.Write(recordToWrite.GetSizeWithLengthPrefixAndSuffix()); var chaser = new TFChunkChaser(db, writerchk, chaserchk, false); chaser.Open(); ILogRecord record; var readRecord = chaser.TryReadNext(out record); chaser.Close(); Assert.IsTrue(readRecord); Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix(), chaserchk.Read()); Assert.AreEqual(recordToWrite, record); db.Close(); }
public void try_read_returns_record_when_writerchecksum_equal() { var writerchk = new InMemoryCheckpoint(0); var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0); var db = new TFChunkDb(new TFChunkDbConfig(PathName, new PrefixFileNamingStrategy(PathName, "prefix.tf"), 10000, 0, writerchk, new[] { chaserchk })); db.OpenVerifyAndClean(); var recordToWrite = new PrepareLogRecord(logPosition: 0, correlationId: _correlationId, eventId: _eventId, transactionPosition: 0, eventStreamId: "WorldEnding", expectedVersion: 1234, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.None, eventType: "type", data: new byte[] { 1, 2, 3, 4, 5 }, metadata: new byte[] { 7, 17 }); var writer = new TFChunkWriter(db); writer.Open(); long pos; Assert.IsTrue(writer.Write(recordToWrite, out pos)); writer.Close(); writerchk.Write(recordToWrite.GetSizeWithLengthPrefix()); var chaser = new TFChunkChaser(db, writerchk, chaserchk); chaser.Open(); LogRecord record; var readRecord = chaser.TryReadNext(out record); chaser.Close(); Assert.IsTrue(readRecord); Assert.AreEqual(record.GetSizeWithLengthPrefix(), chaserchk.Read()); Assert.AreEqual(recordToWrite, record); db.Close(); }
public void try_read_returns_record_when_writerchecksum_equal() { var writerchk = new InMemoryCheckpoint(0); var readerchk = new InMemoryCheckpoint("reader", 0); var config = new TransactionFileDatabaseConfig(PathName, "prefix.tf", 10000, writerchk, new List <ICheckpoint> { readerchk }); var recordToWrite = new PrepareLogRecord(logPosition: 0, correlationId: _correlationId, eventId: _eventId, transactionPosition: 0, transactionOffset: 0, eventStreamId: "WorldEnding", expectedVersion: 1234, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.None, eventType: "type", data: new byte[] { 1, 2, 3, 4, 5 }, metadata: new byte[] { 7, 17 }); using (var fs = new FileStream(Path.Combine(PathName, "prefix.tf0"), FileMode.CreateNew, FileAccess.Write)) { var writer = new BinaryWriter(fs); recordToWrite.WriteWithLengthPrefixAndSuffixTo(writer); fs.Close(); } writerchk.Write(recordToWrite.GetSizeWithLengthPrefixAndSuffix()); var reader = new MultifileTransactionFileChaser(config, readerchk); reader.Open(); LogRecord record; var readRecord = reader.TryReadNext(out record); reader.Close(); Assert.IsTrue(readRecord); Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix(), readerchk.Read()); Assert.AreEqual(recordToWrite, record); }
public void try_read_returns_false_when_writer_checksum_is_equal_to_reader_checksum() { var writerchk = new InMemoryCheckpoint(); var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0); var db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, writerchk, chaserchk)); db.Open(); writerchk.Write(12); writerchk.Flush(); chaserchk.Write(12); chaserchk.Flush(); var chaser = new TFChunkChaser(db, writerchk, chaserchk, false); chaser.Open(); ILogRecord record; Assert.IsFalse(chaser.TryReadNext(out record)); Assert.AreEqual(12, chaserchk.Read()); chaser.Close(); db.Dispose(); }
public override async Task TestFixtureSetUp() { await base.TestFixtureSetUp(); var indexDirectory = GetFilePathFor("index"); _logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormatFactory.Create(new() { IndexDirectory = indexDirectory, }); var writerCheckpoint = new InMemoryCheckpoint(0); var chaserCheckpoint = new InMemoryCheckpoint(0); var bus = new InMemoryBus("bus"); new IODispatcher(bus, new PublishEnvelope(bus)); _db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, writerCheckpoint, chaserCheckpoint)); _db.Open(); // create db Writer = new TFChunkWriter(_db); Writer.Open(); SetupDB(); Writer.Close(); Writer = null; writerCheckpoint.Flush(); chaserCheckpoint.Write(writerCheckpoint.Read()); chaserCheckpoint.Flush(); var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(_db, _db.Config.WriterCheckpoint)); var lowHasher = _logFormat.LowHasher; var highHasher = _logFormat.HighHasher; var emptyStreamId = _logFormat.EmptyStreamId; _tableIndex = new TableIndex <TStreamId>(indexDirectory, lowHasher, highHasher, emptyStreamId, () => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2), () => new TFReaderLease(readers), IndexBitnessVersion, int.MaxValue, Constants.PTableMaxReaderCountDefault, MaxEntriesInMemTable); _logFormat.StreamNamesProvider.SetTableIndex(_tableIndex); var readIndex = new ReadIndex <TStreamId>(new NoopPublisher(), readers, _tableIndex, _logFormat.StreamNameIndexConfirmer, _logFormat.StreamIds, _logFormat.StreamNamesProvider, _logFormat.EmptyStreamId, _logFormat.StreamIdValidator, _logFormat.StreamIdSizer, _logFormat.StreamExistenceFilter, _logFormat.StreamExistenceFilterReader, _logFormat.EventTypeIndexConfirmer, streamInfoCacheCapacity: 100_000, additionalCommitChecks: PerformAdditionalCommitChecks, metastreamMaxCount: MetastreamMaxCount, hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault, skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault, replicationCheckpoint: _db.Config.ReplicationCheckpoint, indexCheckpoint: _db.Config.IndexCheckpoint); readIndex.IndexCommitter.Init(chaserCheckpoint.Read()); ReadIndex = readIndex; _tableIndex.Close(false); Writer = new TFChunkWriter(_db); Writer.Open(); Given(); Writer.Close(); Writer = null; writerCheckpoint.Flush(); chaserCheckpoint.Write(writerCheckpoint.Read()); chaserCheckpoint.Flush(); _tableIndex = new TableIndex <TStreamId>(indexDirectory, lowHasher, highHasher, emptyStreamId, () => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2), () => new TFReaderLease(readers), IndexBitnessVersion, int.MaxValue, Constants.PTableMaxReaderCountDefault, MaxEntriesInMemTable); readIndex = new ReadIndex <TStreamId>(new NoopPublisher(), readers, _tableIndex, _logFormat.StreamNameIndexConfirmer, _logFormat.StreamIds, _logFormat.StreamNamesProvider, _logFormat.EmptyStreamId, _logFormat.StreamIdValidator, _logFormat.StreamIdSizer, _logFormat.StreamExistenceFilter, _logFormat.StreamExistenceFilterReader, _logFormat.EventTypeIndexConfirmer, streamInfoCacheCapacity: 100_000, additionalCommitChecks: PerformAdditionalCommitChecks, metastreamMaxCount: MetastreamMaxCount, hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault, skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault, replicationCheckpoint: _db.Config.ReplicationCheckpoint, indexCheckpoint: _db.Config.IndexCheckpoint); readIndex.IndexCommitter.Init(chaserCheckpoint.Read()); ReadIndex = readIndex; }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); var writerCheckpoint = new InMemoryCheckpoint(0); var chaserCheckpoint = new InMemoryCheckpoint(0); var bus = new InMemoryBus("bus"); new IODispatcher(bus, new PublishEnvelope(bus)); _db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, writerCheckpoint, chaserCheckpoint, new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); _db.Open(); // create db Writer = new TFChunkWriter(_db); Writer.Open(); SetupDB(); Writer.Close(); Writer = null; writerCheckpoint.Flush(); chaserCheckpoint.Write(writerCheckpoint.Read()); chaserCheckpoint.Flush(); var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(_db, _db.Config.WriterCheckpoint)); var lowHasher = new XXHashUnsafe(); var highHasher = new Murmur3AUnsafe(); _tableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher, () => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2), () => new TFReaderLease(readers), IndexBitnessVersion, MaxEntriesInMemTable); ReadIndex = new ReadIndex(new NoopPublisher(), readers, _tableIndex, EventStore.Core.Settings.ESConsts.StreamInfoCacheCapacity, additionalCommitChecks: PerformAdditionalCommitChecks, metastreamMaxCount: MetastreamMaxCount, hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault, skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault); ReadIndex.Init(chaserCheckpoint.Read()); _tableIndex.Close(false); Writer = new TFChunkWriter(_db); Writer.Open(); Given(); Writer.Close(); Writer = null; writerCheckpoint.Flush(); chaserCheckpoint.Write(writerCheckpoint.Read()); chaserCheckpoint.Flush(); _tableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher, () => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2), () => new TFReaderLease(readers), IndexBitnessVersion, MaxEntriesInMemTable); ReadIndex = new ReadIndex(new NoopPublisher(), readers, _tableIndex, EventStore.Core.Settings.ESConsts.StreamInfoCacheCapacity, additionalCommitChecks: PerformAdditionalCommitChecks, metastreamMaxCount: MetastreamMaxCount, hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault, skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault); ReadIndex.Init(chaserCheckpoint.Read()); }
public void try_read_returns_properly_when_writer_is_written_to_while_chasing() { var writerchk = new InMemoryCheckpoint(0); var readerchk = new InMemoryCheckpoint("reader", 0); var config = new TransactionFileDatabaseConfig(PathName, "prefix.tf", 10000, writerchk, new[] { readerchk }); var fileName = Path.Combine(PathName, "prefix.tf0"); File.Create(fileName).Close(); var reader = new MultifileTransactionFileChaser(config, "reader"); reader.Open(); LogRecord record; Assert.IsFalse(reader.TryReadNext(out record)); var recordToWrite = new PrepareLogRecord(logPosition: 0, correlationId: _correlationId, eventId: _eventId, transactionPosition: 0, eventStreamId: "WorldEnding", expectedVersion: 1234, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.None, eventType: "type", data: new byte[] { 1, 2, 3, 4, 5 }, metadata: new byte[] { 7, 17 }); var memstream = new MemoryStream(); var writer = new BinaryWriter(memstream); recordToWrite.WriteWithLengthPrefixTo(writer); using (var fs = new FileStream(fileName, FileMode.Append, FileAccess.Write, FileShare.ReadWrite)) { fs.Write(memstream.ToArray(), 0, (int)memstream.Length); fs.Flush(flushToDisk: true); } writerchk.Write(memstream.Length); Assert.IsTrue(reader.TryReadNext(out record)); Assert.AreEqual(record, recordToWrite); var recordToWrite2 = new PrepareLogRecord(logPosition: 0, correlationId: _correlationId, eventId: _eventId, transactionPosition: 0, eventStreamId: "WorldEnding", expectedVersion: 4321, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.None, eventType: "type", data: new byte[] { 3, 2, 1 }, metadata: new byte[] { 9 }); memstream.SetLength(0); recordToWrite2.WriteWithLengthPrefixTo(writer); using (var fs = new FileStream(fileName, FileMode.Append, FileAccess.Write, FileShare.ReadWrite)) { fs.Write(memstream.ToArray(), 0, (int)memstream.Length); fs.Flush(flushToDisk: true); } writerchk.Write(writerchk.Read() + memstream.Length); Assert.IsTrue(reader.TryReadNext(out record)); Assert.AreEqual(record, recordToWrite2); reader.Close(); }