public void SetUp() { _writerCheckpoint = new InMemoryCheckpoint(); _db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, _writerCheckpoint, new InMemoryCheckpoint(), 1024)); _db.Open(); _writer = new TFChunkWriter(_db); _writer.Open(); _record = new PrepareLogRecord(logPosition: 0, eventId: _eventId, correlationId: _correlationId, transactionPosition: 0xDEAD, transactionOffset: 0xBEEF, eventStreamId: "WorldEnding", expectedVersion: 1234, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.SingleWrite, eventType: "type", data: new byte[] { 1, 2, 3, 4, 5 }, metadata: new byte[] { 7, 17 }); long newPos; _writer.Write(_record, out newPos); _writer.Flush(); }
public void try_read_does_not_cache_anything_and_returns_record_once_it_is_written_later() { var writerchk = new InMemoryCheckpoint(0); var config = new TransactionFileDatabaseConfig(PathName, "prefix.tf", 10000, writerchk, new ICheckpoint[0]); // create db var writer = new MultifileTransactionFileWriter(config); writer.Open(); var reader = new MultifileTransactionFileChaser(config); reader.Open(); LogRecord record; Assert.IsFalse(reader.TryReadNext(out record)); var rec = LogRecord.SingleWrite(0, Guid.NewGuid(), Guid.NewGuid(), "ES", -1, "ET", new byte[] { 7 }, null); long tmp; Assert.IsTrue(writer.Write(rec, out tmp)); writer.Flush(); writer.Close(); Assert.IsTrue(reader.TryReadNext(out record)); Assert.AreEqual(rec, record); reader.Close(); }
public void a_record_can_be_written() { var filename = Path.Combine(PathName, "prefix.tf0"); File.WriteAllBytes(filename, new byte[10000]); _checkpoint = new InMemoryCheckpoint(137); var tf = new MultifileTransactionFileWriter( new TransactionFileDatabaseConfig(PathName, "prefix.tf", 100000, _checkpoint, new List <ICheckpoint>())); tf.Open(); var record = new PrepareLogRecord(logPosition: 0, correlationId: _correlationId, eventId: _eventId, transactionPosition: 0, transactionOffset: 0, eventStreamId: "WorldEnding", expectedVersion: 1234, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.None, eventType: "type", data: new byte[8000], metadata: new byte[] { 7, 17 }); long tmp; tf.Write(record, out tmp); tf.Close(); Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix() + 137, _checkpoint.Read()); using (var filestream = File.Open(filename, FileMode.Open, FileAccess.Read)) { filestream.Seek(137 + sizeof(int), SeekOrigin.Begin); var reader = new BinaryReader(filestream); var read = LogRecord.ReadFrom(reader); Assert.AreEqual(record, read); } }
public void Setup() { _consumer = new TestHandler <Message>(); _bus = new InMemoryBus("temp"); _bus.Subscribe(_consumer); ICheckpoint writerCheckpoint = new InMemoryCheckpoint(1000); var ioDispatcher = new IODispatcher(_bus, new PublishEnvelope(_bus)); _readerService = new EventReaderCoreService(_bus, ioDispatcher, 10, writerCheckpoint, runHeadingReader: true); _subscriptionDispatcher = new ReaderSubscriptionDispatcher(_bus); _spoolProcessingResponseDispatcher = new SpooledStreamReadingDispatcher(_bus); _service = new ProjectionCoreService( _bus, _bus, _subscriptionDispatcher, new RealTimeProvider(), ioDispatcher, _spoolProcessingResponseDispatcher); _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.CheckpointSuggested>()); _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.CommittedEventReceived>()); _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.EofReached>()); _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.PartitionEofReached>()); _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.PartitionMeasured>()); _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.ProgressChanged>()); _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.SubscriptionStarted>()); _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.NotAuthorized>()); _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.ReaderAssignedReader>()); _bus.Subscribe(_spoolProcessingResponseDispatcher.CreateSubscriber <PartitionProcessingResult>()); _readerService.Handle(new Messages.ReaderCoreServiceMessage.StartReader()); _service.Handle(new ProjectionCoreServiceMessage.StartCore()); }
private TFChunkDbConfig CreateDbConfig() { ICheckpoint writerChk = new InMemoryCheckpoint(Checkpoint.Writer); ICheckpoint chaserChk = new InMemoryCheckpoint(Checkpoint.Chaser); ICheckpoint epochChk = new InMemoryCheckpoint(Checkpoint.Epoch, initValue: -1); ICheckpoint proposalChk = new InMemoryCheckpoint(Checkpoint.Proposal, initValue: -1); ICheckpoint truncateChk = new InMemoryCheckpoint(Checkpoint.Truncate, initValue: -1); ICheckpoint replicationCheckpoint = new InMemoryCheckpoint(-1); ICheckpoint indexCheckpoint = new InMemoryCheckpoint(-1); var nodeConfig = new TFChunkDbConfig( PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1000, 10000, writerChk, chaserChk, epochChk, proposalChk, truncateChk, replicationCheckpoint, indexCheckpoint, Constants.TFChunkInitialReaderCountDefault, Constants.TFChunkMaxReaderCountDefault, true); return(nodeConfig); }
public void try_read_returns_false_when_writer_checkpoint_is_zero() { var writerchk = new InMemoryCheckpoint(0); var db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, writerchk, new InMemoryCheckpoint(), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); db.Open(); var chaser = new TFChunkChaser(db, writerchk, new InMemoryCheckpoint()); chaser.Open(); LogRecord record; Assert.IsFalse(chaser.TryReadNext(out record)); chaser.Close(); db.Dispose(); }
public void try_read_does_not_cache_anything_and_returns_record_once_it_is_written_later() { var writerchk = new InMemoryCheckpoint(0); var chaserchk = new InMemoryCheckpoint(0); var db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, writerchk, chaserchk)); db.Open(); var writer = new TFChunkWriter(db); writer.Open(); var reader = new TFChunkReader(db, writerchk, 0); Assert.IsFalse(reader.TryReadNext().Success); var rec = LogRecord.SingleWrite(0, Guid.NewGuid(), Guid.NewGuid(), "ES", -1, "ET", new byte[] { 7 }, null); long tmp; Assert.IsTrue(writer.Write(rec, out tmp)); writer.Flush(); writer.Close(); var res = reader.TryReadNext(); Assert.IsTrue(res.Success); Assert.AreEqual(rec, res.LogRecord); db.Close(); }
public void SetUp() { _writerCheckpoint = new InMemoryCheckpoint(); _db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, _writerCheckpoint, new InMemoryCheckpoint(), 1024)); _db.Open(); _writer = new TFChunkWriter(_db); _writer.Open(); var recordFactory = LogFormatHelper <TLogFormat, TStreamId> .RecordFactory; var streamId = LogFormatHelper <TLogFormat, TStreamId> .StreamId; var eventTypeId = LogFormatHelper <TLogFormat, TStreamId> .EventTypeId; _record = LogRecord.Prepare( factory: recordFactory, logPosition: 0, eventId: _eventId, correlationId: _correlationId, transactionPos: 0xDEAD, transactionOffset: 0xBEEF, eventStreamId: streamId, expectedVersion: 1234, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.SingleWrite, eventType: eventTypeId, data: new byte[] { 1, 2, 3, 4, 5 }, metadata: new byte[] { 7, 17 }); long newPos; _writer.Write(_record, out newPos); _writer.Flush(); }
public void try_read_returns_false_when_writer_checksum_is_equal_to_reader_checksum() { var writerchk = new InMemoryCheckpoint(); var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0); var db = new TFChunkDb(new TFChunkDbConfig(PathName, new PrefixFileNamingStrategy(PathName, "prefix.tf"), 10000, 0, writerchk, chaserchk, new[] { writerchk, chaserchk })); db.OpenVerifyAndClean(); writerchk.Write(12); writerchk.Flush(); chaserchk.Write(12); chaserchk.Flush(); var chaser = new TFChunkChaser(db, writerchk, chaserchk); chaser.Open(); LogRecord record; Assert.IsFalse(chaser.TryReadNext(out record)); Assert.AreEqual(12, chaserchk.Read()); chaser.Close(); db.Dispose(); }
public void try_read_returns_false_when_writer_checksum_is_ahead_but_not_enough_to_read_record() { var writerchk = new InMemoryCheckpoint(50); var readerchk = new InMemoryCheckpoint("reader", 0); var config = new TransactionFileDatabaseConfig(PathName, "prefix.tf", 10000, writerchk, new[] { readerchk }); var recordToWrite = new PrepareLogRecord(logPosition: 0, correlationId: _correlationId, eventId: _eventId, transactionPosition: 0, eventStreamId: "WorldEnding", expectedVersion: 1234, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.None, eventType: "type", data: new byte[] { 1, 2, 3, 4, 5 }, metadata: new byte[] { 7, 17 }); using (var fs = new FileStream(Path.Combine(PathName, "prefix.tf0"), FileMode.CreateNew, FileAccess.Write)) { var writer = new BinaryWriter(fs); recordToWrite.WriteWithLengthPrefixTo(writer); fs.Close(); } var reader = new MultifileTransactionFileChaser(config, "reader"); reader.Open(); LogRecord record = null; var readRecord = reader.TryReadNext(out record); reader.Close(); Assert.IsFalse(readRecord); Assert.AreEqual(0, readerchk.Read()); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); WriterCheckpoint = new InMemoryCheckpoint(0); var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0); Db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, WriterCheckpoint, new[] { chaserchk })); Db.OpenVerifyAndClean(); // create db Writer = new TFChunkWriter(Db); Writer.Open(); WriteTestScenario(); Writer.Close(); Writer = null; WriterCheckpoint.Flush(); chaserchk.Write(WriterCheckpoint.Read()); chaserchk.Flush(); TableIndex = new TableIndex(Path.Combine(PathName, "index"), () => new HashListMemTable(), _maxEntriesInMemTable); TableIndex.Initialize(); var reader = new TFChunkReader(Db, Db.Config.WriterCheckpoint); ReadIndex = new ReadIndex(new NoopPublisher(), 2, () => new TFChunkSequentialReader(Db, Db.Config.WriterCheckpoint, 0), () => reader, TableIndex, new ByLengthHasher()); ReadIndex.Build(); }
public void try_read_does_not_cache_anything_and_returns_record_once_it_is_written_later() { var writerchk = new InMemoryCheckpoint(0); var db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, writerchk, new InMemoryCheckpoint())); db.OpenVerifyAndClean(); var writer = new TFChunkWriter(db); writer.Open(); var reader = new TFChunkSequentialReader(db, writerchk, 0); LogRecord record; Assert.IsFalse(reader.TryReadNext(out record)); var rec = LogRecord.SingleWrite(0, Guid.NewGuid(), Guid.NewGuid(), "ES", -1, "ET", new byte[] { 7 }, null); long tmp; Assert.IsTrue(writer.Write(rec, out tmp)); writer.Flush(); writer.Close(); Assert.IsTrue(reader.TryReadNext(out record)); Assert.AreEqual(rec, record); reader.Close(); db.Close(); }
public static TFChunkDbConfig CreateDbConfig( string pathName, ICheckpoint writerCheckpoint, ICheckpoint chaserCheckpoint, int chunkSize = 10000, ICheckpoint replicationCheckpoint = null) { if (replicationCheckpoint == null) { replicationCheckpoint = new InMemoryCheckpoint(-1); } return(new TFChunkDbConfig( pathName, new VersionedPatternFileNamingStrategy(pathName, "chunk-"), chunkSize, 0, writerCheckpoint, chaserCheckpoint, new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1), replicationCheckpoint, new InMemoryCheckpoint(-1), Constants.TFChunkInitialReaderCountDefault, Constants.TFChunkMaxReaderCountDefault)); }
public void a_record_can_be_written() { _checkpoint = new InMemoryCheckpoint(0); var tf = new MultifileTransactionFileWriter( new TransactionFileDatabaseConfig(PathName, "prefix.tf", 10000, _checkpoint, new List <ICheckpoint>())); tf.Open(); var record = new PrepareLogRecord(logPosition: 0, correlationId: _correlationId, eventId: _eventId, transactionPosition: 0, eventStreamId: "WorldEnding", expectedVersion: 1234, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.None, eventType: "type", data: new byte[] { 1, 2, 3, 4, 5 }, metadata: new byte[] { 7, 17 }); long tmp; tf.Write(record, out tmp); tf.Close(); Assert.AreEqual(record.GetSizeWithLengthPrefix(), _checkpoint.Read()); using (var filestream = File.Open(Path.Combine(PathName, "prefix.tf0"), FileMode.Open, FileAccess.Read)) { var reader = new BinaryReader(filestream); reader.ReadInt32(); var read = LogRecord.ReadFrom(reader); Assert.AreEqual(record, read); } }
private TFChunkDbConfig CreateDbConfig(int chunkSize, string dbPath, long chunksCacheSize, bool inMemDb) { ICheckpoint writerChk; ICheckpoint chaserChk; ICheckpoint epochChk; ICheckpoint truncateChk; ICheckpoint replicationCheckpoint = new InMemoryCheckpoint(-1); if (inMemDb) { writerChk = new InMemoryCheckpoint(Checkpoint.Writer); chaserChk = new InMemoryCheckpoint(Checkpoint.Chaser); epochChk = new InMemoryCheckpoint(Checkpoint.Epoch, initValue: -1); truncateChk = new InMemoryCheckpoint(Checkpoint.Truncate, initValue: -1); } else { var writerCheckFilename = Path.Combine(dbPath, Checkpoint.Writer + ".chk"); var chaserCheckFilename = Path.Combine(dbPath, Checkpoint.Chaser + ".chk"); var epochCheckFilename = Path.Combine(dbPath, Checkpoint.Epoch + ".chk"); var truncateCheckFilename = Path.Combine(dbPath, Checkpoint.Truncate + ".chk"); writerChk = new MemoryMappedFileCheckpoint(writerCheckFilename, Checkpoint.Writer, cached: true); chaserChk = new MemoryMappedFileCheckpoint(chaserCheckFilename, Checkpoint.Chaser, cached: true); epochChk = new MemoryMappedFileCheckpoint( epochCheckFilename, Checkpoint.Epoch, cached: true, initValue: -1); truncateChk = new MemoryMappedFileCheckpoint( truncateCheckFilename, Checkpoint.Truncate, cached: true, initValue: -1); } var nodeConfig = new TFChunkDbConfig( dbPath, new VersionedPatternFileNamingStrategy(dbPath, "chunk-"), chunkSize, chunksCacheSize, writerChk, chaserChk, epochChk, truncateChk, replicationCheckpoint, Constants.TFChunkInitialReaderCountDefault, Constants.TFChunkMaxReaderCountDefault, inMemDb); return(nodeConfig); }
public void a_record_can_be_written() { var filename = Path.Combine(PathName, "prefix.tf0"); var secondfilename = Path.Combine(PathName, "prefix.tf1"); var thirdfilename = Path.Combine(PathName, "prefix.tf2"); File.WriteAllBytes(filename, new byte[50]); _checkpoint = new InMemoryCheckpoint(40); var tf = new MultifileTransactionFileWriter( new TransactionFileDatabaseConfig(PathName, "prefix.tf", 50, _checkpoint, new ICheckpoint[0])); tf.Open(); var record = new PrepareLogRecord(logPosition: 0, correlationId: _correlationId, eventId: _eventId, transactionPosition: 0, transactionOffset: 0, eventStreamId: "WorldEnding", expectedVersion: 1234, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.None, eventType: "type", data: new byte[] { 1, 2, 3, 4, 5 }, metadata: new byte[] { 7, 17 }); long tmp; tf.Write(record, out tmp); tf.Close(); Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix() + 40, _checkpoint.Read()); //+orginal checkpoint position Assert.IsTrue(File.Exists(secondfilename)); var stream = new MemoryStream(); var buffer = new byte[256]; using (var filestream = File.Open(filename, FileMode.Open, FileAccess.Read)) { filestream.Seek(40, SeekOrigin.Begin); filestream.Read(buffer, 0, 10); stream.Write(buffer, 0, 10); } using (var filestream = File.Open(secondfilename, FileMode.Open, FileAccess.Read)) { filestream.Seek(0, SeekOrigin.Begin); filestream.Read(buffer, 0, 50); stream.Write(buffer, 0, 50); } using (var filestream = File.Open(thirdfilename, FileMode.Open, FileAccess.Read)) { filestream.Seek(0, SeekOrigin.Begin); filestream.Read(buffer, 0, 50); stream.Write(buffer, 0, 50); } stream.Seek(0 + sizeof(int), SeekOrigin.Begin); var reader = new BinaryReader(stream); var read = LogRecord.ReadFrom(reader); Assert.AreEqual(record, read); }
public void a_record_can_be_written() { var filename = Path.Combine(PathName, "prefix.tf0"); var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, 10000, 0, 0, 0); var chunkBytes = chunkHeader.AsByteArray(); var buf = new byte[ChunkHeader.Size + ChunkFooter.Size + chunkHeader.ChunkSize]; Buffer.BlockCopy(chunkBytes, 0, buf, 0, chunkBytes.Length); File.WriteAllBytes(filename, buf); _checkpoint = new InMemoryCheckpoint(137); var db = new TFChunkDb(new TFChunkDbConfig(PathName, new PrefixFileNamingStrategy(PathName, "prefix.tf"), chunkHeader.ChunkSize, 0, _checkpoint, new ICheckpoint[0])); db.OpenVerifyAndClean(); var writer = new TFChunkWriter(db); var record = new PrepareLogRecord(logPosition: 0, correlationId: _correlationId, eventId: _eventId, transactionPosition: 0, transactionOffset: 0, eventStreamId: "WorldEnding", expectedVersion: 1234, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.None, eventType: "type", data: new byte[8000], metadata: new byte[] { 7, 17 }); Console.WriteLine(record.GetSizeWithLengthPrefixAndSuffix()); Console.WriteLine(record.GetSizeWithLengthPrefixAndSuffix() + 137); long pos; Assert.IsTrue(writer.Write(record, out pos)); writer.Close(); db.Dispose(); Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix() + 137, _checkpoint.Read()); using (var filestream = File.Open(filename, FileMode.Open, FileAccess.Read)) { filestream.Seek(ChunkHeader.Size + 137 + sizeof(int), SeekOrigin.Begin); var reader = new BinaryReader(filestream); var read = LogRecord.ReadFrom(reader); Console.WriteLine(string.Join("\n", Directory.EnumerateFiles(PathName))); Assert.AreEqual(record, read); } }
protected static TFChunkDbConfig CreateDbConfig(string dbPath, int cachedChunks, long chunksCacheSize, bool inMemDb) { ICheckpoint writerChk; ICheckpoint chaserChk; ICheckpoint epochChk; ICheckpoint truncateChk; if (inMemDb) { writerChk = new InMemoryCheckpoint(Checkpoint.Writer); chaserChk = new InMemoryCheckpoint(Checkpoint.Chaser); epochChk = new InMemoryCheckpoint(Checkpoint.Epoch, initValue: -1); truncateChk = new InMemoryCheckpoint(Checkpoint.Truncate, initValue: -1); } else { if (!Directory.Exists(dbPath)) // mono crashes without this check { Directory.CreateDirectory(dbPath); } var writerCheckFilename = Path.Combine(dbPath, Checkpoint.Writer + ".chk"); var chaserCheckFilename = Path.Combine(dbPath, Checkpoint.Chaser + ".chk"); var epochCheckFilename = Path.Combine(dbPath, Checkpoint.Epoch + ".chk"); var truncateCheckFilename = Path.Combine(dbPath, Checkpoint.Truncate + ".chk"); if (Runtime.IsMono) { writerChk = new FileCheckpoint(writerCheckFilename, Checkpoint.Writer, cached: true); chaserChk = new FileCheckpoint(chaserCheckFilename, Checkpoint.Chaser, cached: true); epochChk = new FileCheckpoint(epochCheckFilename, Checkpoint.Epoch, cached: true, initValue: -1); truncateChk = new FileCheckpoint(truncateCheckFilename, Checkpoint.Truncate, cached: true, initValue: -1); } else { writerChk = new MemoryMappedFileCheckpoint(writerCheckFilename, Checkpoint.Writer, cached: true); chaserChk = new MemoryMappedFileCheckpoint(chaserCheckFilename, Checkpoint.Chaser, cached: true); epochChk = new MemoryMappedFileCheckpoint(epochCheckFilename, Checkpoint.Epoch, cached: true, initValue: -1); truncateChk = new MemoryMappedFileCheckpoint(truncateCheckFilename, Checkpoint.Truncate, cached: true, initValue: -1); } } var cache = cachedChunks >= 0 ? cachedChunks * (long)(TFConsts.ChunkSize + ChunkHeader.Size + ChunkFooter.Size) : chunksCacheSize; var nodeConfig = new TFChunkDbConfig(dbPath, new VersionedPatternFileNamingStrategy(dbPath, "chunk-"), TFConsts.ChunkSize, cache, writerChk, chaserChk, epochChk, truncateChk, inMemDb); return(nodeConfig); }
public void Setup() { _bus.Subscribe(_consumer); ICheckpoint writerCheckpoint = new InMemoryCheckpoint(1000); _readerService = new EventReaderCoreService( GetInputQueue(), _ioDispatcher, 10, writerCheckpoint, runHeadingReader: GivenHeadingReaderRunning(), faultOutOfOrderProjections: true); _subscriptionDispatcher = new ReaderSubscriptionDispatcher(GetInputQueue()); _bus.Subscribe( _subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.CheckpointSuggested>()); _bus.Subscribe( _subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.CommittedEventReceived>()); _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.EofReached>()); _bus.Subscribe( _subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.PartitionEofReached>()); _bus.Subscribe(_subscriptionDispatcher .CreateSubscriber <EventReaderSubscriptionMessage.PartitionMeasured>()); _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.ProgressChanged>()); _bus.Subscribe( _subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.SubscriptionStarted>()); _bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.NotAuthorized>()); _bus.Subscribe( _subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.ReaderAssignedReader>()); _bus.Subscribe <ReaderCoreServiceMessage.StartReader>(_readerService); _bus.Subscribe <ReaderCoreServiceMessage.StopReader>(_readerService); _bus.Subscribe <ReaderCoreServiceMessage.ReaderTick>(_readerService); _bus.Subscribe <ReaderSubscriptionMessage.CommittedEventDistributed>(_readerService); _bus.Subscribe <ReaderSubscriptionMessage.EventReaderEof>(_readerService); _bus.Subscribe <ReaderSubscriptionMessage.EventReaderPartitionEof>(_readerService); _bus.Subscribe <ReaderSubscriptionMessage.EventReaderPartitionDeleted>(_readerService); _bus.Subscribe <ReaderSubscriptionMessage.EventReaderNotAuthorized>(_readerService); _bus.Subscribe <ReaderSubscriptionMessage.EventReaderIdle>(_readerService); _bus.Subscribe <ReaderSubscriptionMessage.EventReaderStarting>(_readerService); _bus.Subscribe <ReaderSubscriptionManagement.Pause>(_readerService); _bus.Subscribe <ReaderSubscriptionManagement.Resume>(_readerService); _bus.Subscribe <ReaderSubscriptionManagement.Subscribe>(_readerService); _bus.Subscribe <ReaderSubscriptionManagement.Unsubscribe>(_readerService); _bus.Subscribe <ReaderSubscriptionManagement.SpoolStreamReadingCore>(_readerService); _bus.Subscribe <ReaderSubscriptionManagement.CompleteSpooledStreamReading>(_readerService); GivenAdditionalServices(); _bus.Publish(new ReaderCoreServiceMessage.StartReader(Guid.NewGuid())); WhenLoop(); }
public void a_record_can_be_written() { var filename = GetFilePathFor("chunk-000000.000000"); var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, 10000, 0, 0, false, Guid.NewGuid()); var chunkBytes = chunkHeader.AsByteArray(); var buf = new byte[ChunkHeader.Size + ChunkFooter.Size + chunkHeader.ChunkSize]; Buffer.BlockCopy(chunkBytes, 0, buf, 0, chunkBytes.Length); File.WriteAllBytes(filename, buf); _checkpoint = new InMemoryCheckpoint(137); var db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, _checkpoint, new InMemoryCheckpoint(), chunkSize: chunkHeader.ChunkSize)); db.Open(); var bytes = new byte[3994]; // this gives exactly 4097 size of record, with 3993 (rec size 4096) everything works fine! new Random().NextBytes(bytes); var writer = new TFChunkWriter(db); var logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormat; logFormat.StreamNameIndex.GetOrAddId("WorldEnding", out var streamId, out _, out _); var record = LogRecord.Prepare( factory: logFormat.RecordFactory, logPosition: 137, correlationId: _correlationId, eventId: _eventId, transactionPos: 789, transactionOffset: 543, eventStreamId: streamId, expectedVersion: 1234, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.SingleWrite, eventType: "type", data: bytes, metadata: new byte[] { 0x07, 0x17 }); long pos; Assert.IsTrue(writer.Write(record, out pos)); writer.Close(); db.Dispose(); Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix() + 137, _checkpoint.Read()); using (var filestream = File.Open(filename, FileMode.Open, FileAccess.Read)) { filestream.Seek(ChunkHeader.Size + 137 + sizeof(int), SeekOrigin.Begin); var reader = new BinaryReader(filestream); var read = LogRecord.ReadFrom(reader, (int)reader.BaseStream.Length); Assert.AreEqual(record, read); } }
public void try_read_returns_record_when_record_bigger_than_internal_buffer() { var writerchk = new InMemoryCheckpoint(0); var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0); var db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, writerchk, chaserchk, new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); db.Open(); var recordToWrite = new PrepareLogRecord(logPosition: 0, correlationId: _correlationId, eventId: _eventId, transactionPosition: 0, transactionOffset: 0, eventStreamId: "WorldEnding", expectedVersion: 1234, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.None, eventType: "type", data: new byte[9000], metadata: new byte[] { 7, 17 }); var writer = new TFChunkWriter(db); writer.Open(); long pos; Assert.IsTrue(writer.Write(recordToWrite, out pos)); writer.Close(); writerchk.Write(recordToWrite.GetSizeWithLengthPrefixAndSuffix()); var reader = new TFChunkChaser(db, writerchk, chaserchk); reader.Open(); LogRecord record; var readRecord = reader.TryReadNext(out record); reader.Close(); Assert.IsTrue(readRecord); Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix(), chaserchk.Read()); Assert.AreEqual(recordToWrite, record); db.Close(); }
public void try_read_returns_record_when_writerchecksum_ahead() { var recordToWrite = new PrepareLogRecord(logPosition: 0, correlationId: _correlationId, eventId: _eventId, transactionPosition: 0, transactionOffset: 0, eventStreamId: "WorldEnding", expectedVersion: 1234, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.None, eventType: "type", data: new byte[] { 1, 2, 3, 4, 5 }, metadata: new byte[] { 7, 17 }); using (var fs = new FileStream(GetFilePathFor("chunk-000000.000000"), FileMode.CreateNew, FileAccess.Write)) { fs.SetLength(ChunkHeader.Size + ChunkFooter.Size + 10000); var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, 10000, 0, 0, false, Guid.NewGuid()).AsByteArray(); var writer = new BinaryWriter(fs); writer.Write(chunkHeader); recordToWrite.WriteWithLengthPrefixAndSuffixTo(writer); fs.Close(); } var writerchk = new InMemoryCheckpoint(128); var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0); var db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, writerchk, chaserchk, new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); db.Open(); var chaser = new TFChunkChaser(db, writerchk, chaserchk); chaser.Open(); LogRecord record; var recordRead = chaser.TryReadNext(out record); chaser.Close(); Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix(), chaserchk.Read()); Assert.IsTrue(recordRead); Assert.AreEqual(recordToWrite, record); db.Close(); }
public void a_record_can_be_written() { var filename = Path.Combine(PathName, "prefix.tf0"); var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, 10000, 0, 0, 0); var chunkBytes = chunkHeader.AsByteArray(); var buf = new byte[ChunkHeader.Size + ChunkFooter.Size + chunkHeader.ChunkSize]; Buffer.BlockCopy(chunkBytes, 0, buf, 0, chunkBytes.Length); File.WriteAllBytes(filename, buf); _checkpoint = new InMemoryCheckpoint(137); var db = new TFChunkDb(new TFChunkDbConfig(PathName, new PrefixFileNamingStrategy(PathName, "prefix.tf"), chunkHeader.ChunkSize, 0, _checkpoint, new InMemoryCheckpoint(), new ICheckpoint[0])); db.OpenVerifyAndClean(); var bytes = new byte[3994]; // this gives exactly 4097 size of record, with 3993 (rec size 4096) everything works fine! new Random().NextBytes(bytes); var writer = new TFChunkWriter(db); var record = new PrepareLogRecord(logPosition: 123, correlationId: _correlationId, eventId: _eventId, transactionPosition: 789, transactionOffset: 543, eventStreamId: "WorldEnding", expectedVersion: 1234, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.SingleWrite, eventType: "type", data: bytes, metadata: new byte[] { 0x07, 0x17 }); long pos; Assert.IsTrue(writer.Write(record, out pos)); writer.Close(); db.Dispose(); Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix() + 137, _checkpoint.Read()); using (var filestream = File.Open(filename, FileMode.Open, FileAccess.Read)) { filestream.Seek(ChunkHeader.Size + 137 + sizeof(int), SeekOrigin.Begin); var reader = new BinaryReader(filestream); var read = LogRecord.ReadFrom(reader); Assert.AreEqual(record, read); } }
public void try_read_returns_record_when_record_bigger_than_internal_buffer() { var writerchk = new InMemoryCheckpoint(0); var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0); var db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, writerchk, chaserchk)); db.Open(); var recordFactory = LogFormatHelper <TLogFormat, TStreamId> .RecordFactory; var streamId = LogFormatHelper <TLogFormat, TStreamId> .StreamId; var eventTypeId = LogFormatHelper <TLogFormat, TStreamId> .EventTypeId; var recordToWrite = LogRecord.Prepare( factory: recordFactory, logPosition: 0, correlationId: _correlationId, eventId: _eventId, transactionPos: 0, transactionOffset: 0, eventStreamId: streamId, expectedVersion: 1234, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.None, eventType: eventTypeId, data: new byte[9000], metadata: new byte[] { 7, 17 }); var writer = new TFChunkWriter(db); writer.Open(); long pos; Assert.IsTrue(writer.Write(recordToWrite, out pos)); writer.Close(); writerchk.Write(recordToWrite.GetSizeWithLengthPrefixAndSuffix()); var reader = new TFChunkChaser(db, writerchk, chaserchk, false); reader.Open(); ILogRecord record; var readRecord = reader.TryReadNext(out record); reader.Close(); Assert.IsTrue(readRecord); Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix(), chaserchk.Read()); Assert.AreEqual(recordToWrite, record); db.Close(); }
public void try_read_returns_record_when_writerchecksum_ahead() { var recordFactory = LogFormatHelper <TLogFormat, TStreamId> .RecordFactory; var streamId = LogFormatHelper <TLogFormat, TStreamId> .StreamId; var eventTypeId = LogFormatHelper <TLogFormat, TStreamId> .EventTypeId; var recordToWrite = LogRecord.Prepare( factory: recordFactory, logPosition: 0, correlationId: _correlationId, eventId: _eventId, transactionPos: 0, transactionOffset: 0, eventStreamId: streamId, expectedVersion: 1234, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.None, eventType: eventTypeId, data: new byte[] { 1, 2, 3, 4, 5 }, metadata: new byte[] { 7, 17 }); using (var fs = new FileStream(GetFilePathFor("chunk-000000.000000"), FileMode.CreateNew, FileAccess.Write)) { fs.SetLength(ChunkHeader.Size + ChunkFooter.Size + 10000); var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, 10000, 0, 0, false, Guid.NewGuid()) .AsByteArray(); var writer = new BinaryWriter(fs); writer.Write(chunkHeader); recordToWrite.WriteWithLengthPrefixAndSuffixTo(writer); fs.Close(); } var writerchk = new InMemoryCheckpoint(recordToWrite.GetSizeWithLengthPrefixAndSuffix() + 16); var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0); var db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, writerchk, chaserchk)); db.Open(); var chaser = new TFChunkChaser(db, writerchk, chaserchk, false); chaser.Open(); ILogRecord record; var recordRead = chaser.TryReadNext(out record); chaser.Close(); Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix(), chaserchk.Read()); Assert.IsTrue(recordRead); Assert.AreEqual(recordToWrite, record); db.Close(); }
public void a_record_can_be_written() { var filename = Path.Combine(PathName, "prefix.tf0"); var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, 10000, 0, 0, 0); var chunkBytes = chunkHeader.AsByteArray(); var bytes = new byte[ChunkHeader.Size + 10000 + ChunkFooter.Size]; Buffer.BlockCopy(chunkBytes, 0, bytes, 0, chunkBytes.Length); File.WriteAllBytes(filename, bytes); _checkpoint = new InMemoryCheckpoint(137); var db = new TFChunkDb(new TFChunkDbConfig(PathName, new PrefixFileNamingStrategy(PathName, "prefix.tf"), 10000, 0, _checkpoint, new InMemoryCheckpoint(), new ICheckpoint[0])); db.OpenVerifyAndClean(); var tf = new TFChunkWriter(db); var record = new PrepareLogRecord(logPosition: 0, correlationId: _correlationId, eventId: _eventId, expectedVersion: 1234, transactionPosition: 0, transactionOffset: 0, eventStreamId: "WorldEnding", timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.None, eventType: "type", data: new byte[] { 1, 2, 3, 4, 5 }, metadata: new byte[] { 7, 17 }); long tmp; tf.Write(record, out tmp); //tf.Flush(); tf.Close(); db.Dispose(); Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix() + 137, _checkpoint.Read()); //137 is fluff assigned to beginning of checkpoint //TODO actually read the event using (var filestream = File.Open(filename, FileMode.Open, FileAccess.Read)) { filestream.Seek(ChunkHeader.Size + 137 + sizeof(int), SeekOrigin.Begin); var reader = new BinaryReader(filestream); var read = LogRecord.ReadFrom(reader); Assert.AreEqual(record, read); } }
public void try_read_returns_false_when_writer_checksum_is_zero() { var writerchk = new InMemoryCheckpoint(0); var config = new TransactionFileDatabaseConfig(PathName, "prefix.tf", 10000, writerchk, new List <ICheckpoint>()); File.WriteAllBytes(Path.Combine(PathName, "prefix.tf0"), new byte[10000]); var reader = new MultifileTransactionFileChaser(config); reader.Open(); LogRecord record; Assert.IsFalse(reader.TryReadNext(out record)); reader.Close(); }
public void try_read_returns_false_when_writer_checksum_is_zero() { var writerchk = new InMemoryCheckpoint(0); var chaserchk = new InMemoryCheckpoint(0); var db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, writerchk, chaserchk)); db.Open(); var reader = new TFChunkReader(db, writerchk, 0); Assert.IsFalse(reader.TryReadNext().Success); db.Close(); }
public void try_read_returns_record_when_writerchecksum_equal() { var writerchk = new InMemoryCheckpoint(0); var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0); var db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, writerchk, chaserchk)); db.Open(); _logFormat.StreamNameIndex.GetOrAddId("WorldEnding", out var streamId, out _, out _); var recordToWrite = LogRecord.Prepare( factory: _logFormat.RecordFactory, logPosition: 0, correlationId: _correlationId, eventId: _eventId, transactionPos: 0, transactionOffset: 0, eventStreamId: streamId, expectedVersion: 1234, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.None, eventType: "type", data: new byte[] { 1, 2, 3, 4, 5 }, metadata: new byte[] { 7, 17 }); var writer = new TFChunkWriter(db); writer.Open(); long pos; Assert.IsTrue(writer.Write(recordToWrite, out pos)); writer.Close(); writerchk.Write(recordToWrite.GetSizeWithLengthPrefixAndSuffix()); var chaser = new TFChunkChaser(db, writerchk, chaserchk, false); chaser.Open(); ILogRecord record; var readRecord = chaser.TryReadNext(out record); chaser.Close(); Assert.IsTrue(readRecord); Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix(), chaserchk.Read()); Assert.AreEqual(recordToWrite, record); db.Close(); }
public void a_record_can_be_written() { var filename = GetFilePathFor("chunk-000000.000000"); var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, 10000, 0, 0, false, chunkId: Guid.NewGuid()); var chunkBytes = chunkHeader.AsByteArray(); var bytes = new byte[ChunkHeader.Size + 10000 + ChunkFooter.Size]; Buffer.BlockCopy(chunkBytes, 0, bytes, 0, chunkBytes.Length); File.WriteAllBytes(filename, bytes); _checkpoint = new InMemoryCheckpoint(137); var db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, _checkpoint, new InMemoryCheckpoint())); db.Open(); var recordFactory = LogFormatHelper <TLogFormat, TStreamId> .RecordFactory; var streamId = LogFormatHelper <TLogFormat, TStreamId> .StreamId; var eventTypeId = LogFormatHelper <TLogFormat, TStreamId> .EventTypeId; var tf = new TFChunkWriter(db); var record = LogRecord.Prepare( factory: recordFactory, logPosition: _checkpoint.Read(), correlationId: _correlationId, eventId: _eventId, expectedVersion: 1234, transactionPos: 0, transactionOffset: 0, eventStreamId: streamId, timeStamp: new DateTime(2012, 12, 21), flags: PrepareFlags.None, eventType: eventTypeId, data: new byte[] { 1, 2, 3, 4, 5 }, metadata: new byte[] { 7, 17 }); long tmp; tf.Write(record, out tmp); tf.Close(); db.Dispose(); Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix() + 137, _checkpoint.Read()); //137 is fluff assigned to beginning of checkpoint using (var filestream = File.Open(filename, FileMode.Open, FileAccess.Read)) { filestream.Seek(ChunkHeader.Size + 137 + sizeof(int), SeekOrigin.Begin); var reader = new BinaryReader(filestream); var read = LogRecord.ReadFrom(reader, (int)reader.BaseStream.Length); Assert.AreEqual(record, read); } }