public void SetUp()
{
_writerCheckpoint = new InMemoryCheckpoint();
_db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, _writerCheckpoint, new InMemoryCheckpoint(),
1024));
_db.Open();
_writer = new TFChunkWriter(_db);
_writer.Open();
_record = new PrepareLogRecord(logPosition: 0,
eventId: _eventId,
correlationId: _correlationId,
transactionPosition: 0xDEAD,
transactionOffset: 0xBEEF,
eventStreamId: "WorldEnding",
expectedVersion: 1234,
timeStamp: new DateTime(2012, 12, 21),
flags: PrepareFlags.SingleWrite,
eventType: "type",
data: new byte[] { 1, 2, 3, 4, 5 },
metadata: new byte[] { 7, 17 });
long newPos;
_writer.Write(_record, out newPos);
_writer.Flush();
}
public void try_read_does_not_cache_anything_and_returns_record_once_it_is_written_later()
{
var writerchk = new InMemoryCheckpoint(0);
var config = new TransactionFileDatabaseConfig(PathName, "prefix.tf", 10000, writerchk, new ICheckpoint[0]);
// create db
var writer = new MultifileTransactionFileWriter(config);
writer.Open();
var reader = new MultifileTransactionFileChaser(config);
reader.Open();
LogRecord record;
Assert.IsFalse(reader.TryReadNext(out record));
var rec = LogRecord.SingleWrite(0, Guid.NewGuid(), Guid.NewGuid(), "ES", -1, "ET", new byte[] { 7 }, null);
long tmp;
Assert.IsTrue(writer.Write(rec, out tmp));
writer.Flush();
writer.Close();
Assert.IsTrue(reader.TryReadNext(out record));
Assert.AreEqual(rec, record);
reader.Close();
}
public void a_record_can_be_written()
{
var filename = Path.Combine(PathName, "prefix.tf0");
File.WriteAllBytes(filename, new byte[10000]);
_checkpoint = new InMemoryCheckpoint(137);
var tf = new MultifileTransactionFileWriter(
new TransactionFileDatabaseConfig(PathName, "prefix.tf", 100000, _checkpoint, new List <ICheckpoint>()));
tf.Open();
var record = new PrepareLogRecord(logPosition: 0,
correlationId: _correlationId,
eventId: _eventId,
transactionPosition: 0,
transactionOffset: 0,
eventStreamId: "WorldEnding",
expectedVersion: 1234,
timeStamp: new DateTime(2012, 12, 21),
flags: PrepareFlags.None,
eventType: "type",
data: new byte[8000],
metadata: new byte[] { 7, 17 });
long tmp;
tf.Write(record, out tmp);
tf.Close();
Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix() + 137, _checkpoint.Read());
using (var filestream = File.Open(filename, FileMode.Open, FileAccess.Read))
{
filestream.Seek(137 + sizeof(int), SeekOrigin.Begin);
var reader = new BinaryReader(filestream);
var read = LogRecord.ReadFrom(reader);
Assert.AreEqual(record, read);
}
}
public void Setup()
{
_consumer = new TestHandler <Message>();
_bus = new InMemoryBus("temp");
_bus.Subscribe(_consumer);
ICheckpoint writerCheckpoint = new InMemoryCheckpoint(1000);
var ioDispatcher = new IODispatcher(_bus, new PublishEnvelope(_bus));
_readerService = new EventReaderCoreService(_bus, ioDispatcher, 10, writerCheckpoint, runHeadingReader: true);
_subscriptionDispatcher =
new ReaderSubscriptionDispatcher(_bus);
_spoolProcessingResponseDispatcher = new SpooledStreamReadingDispatcher(_bus);
_service = new ProjectionCoreService(
_bus, _bus, _subscriptionDispatcher, new RealTimeProvider(), ioDispatcher,
_spoolProcessingResponseDispatcher);
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.CheckpointSuggested>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.CommittedEventReceived>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.EofReached>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.PartitionEofReached>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.PartitionMeasured>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.ProgressChanged>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.SubscriptionStarted>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.NotAuthorized>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.ReaderAssignedReader>());
_bus.Subscribe(_spoolProcessingResponseDispatcher.CreateSubscriber <PartitionProcessingResult>());
_readerService.Handle(new Messages.ReaderCoreServiceMessage.StartReader());
_service.Handle(new ProjectionCoreServiceMessage.StartCore());
}
private TFChunkDbConfig CreateDbConfig()
{
ICheckpoint writerChk = new InMemoryCheckpoint(Checkpoint.Writer);
ICheckpoint chaserChk = new InMemoryCheckpoint(Checkpoint.Chaser);
ICheckpoint epochChk = new InMemoryCheckpoint(Checkpoint.Epoch, initValue: -1);
ICheckpoint proposalChk = new InMemoryCheckpoint(Checkpoint.Proposal, initValue: -1);
ICheckpoint truncateChk = new InMemoryCheckpoint(Checkpoint.Truncate, initValue: -1);
ICheckpoint replicationCheckpoint = new InMemoryCheckpoint(-1);
ICheckpoint indexCheckpoint = new InMemoryCheckpoint(-1);
var nodeConfig = new TFChunkDbConfig(
PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1000,
10000,
writerChk,
chaserChk,
epochChk,
proposalChk,
truncateChk,
replicationCheckpoint,
indexCheckpoint,
Constants.TFChunkInitialReaderCountDefault,
Constants.TFChunkMaxReaderCountDefault,
true);
return(nodeConfig);
}
public void try_read_returns_false_when_writer_checkpoint_is_zero()
{
var writerchk = new InMemoryCheckpoint(0);
var db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
writerchk,
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
db.Open();
var chaser = new TFChunkChaser(db, writerchk, new InMemoryCheckpoint());
chaser.Open();
LogRecord record;
Assert.IsFalse(chaser.TryReadNext(out record));
chaser.Close();
db.Dispose();
}
public void try_read_does_not_cache_anything_and_returns_record_once_it_is_written_later()
{
var writerchk = new InMemoryCheckpoint(0);
var chaserchk = new InMemoryCheckpoint(0);
var db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, writerchk, chaserchk));
db.Open();
var writer = new TFChunkWriter(db);
writer.Open();
var reader = new TFChunkReader(db, writerchk, 0);
Assert.IsFalse(reader.TryReadNext().Success);
var rec = LogRecord.SingleWrite(0, Guid.NewGuid(), Guid.NewGuid(), "ES", -1, "ET", new byte[] { 7 }, null);
long tmp;
Assert.IsTrue(writer.Write(rec, out tmp));
writer.Flush();
writer.Close();
var res = reader.TryReadNext();
Assert.IsTrue(res.Success);
Assert.AreEqual(rec, res.LogRecord);
db.Close();
}
public void SetUp()
{
_writerCheckpoint = new InMemoryCheckpoint();
_db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, _writerCheckpoint, new InMemoryCheckpoint(),
1024));
_db.Open();
_writer = new TFChunkWriter(_db);
_writer.Open();
var recordFactory = LogFormatHelper <TLogFormat, TStreamId> .RecordFactory;
var streamId = LogFormatHelper <TLogFormat, TStreamId> .StreamId;
var eventTypeId = LogFormatHelper <TLogFormat, TStreamId> .EventTypeId;
_record = LogRecord.Prepare(
factory: recordFactory,
logPosition: 0,
eventId: _eventId,
correlationId: _correlationId,
transactionPos: 0xDEAD,
transactionOffset: 0xBEEF,
eventStreamId: streamId,
expectedVersion: 1234,
timeStamp: new DateTime(2012, 12, 21),
flags: PrepareFlags.SingleWrite,
eventType: eventTypeId,
data: new byte[] { 1, 2, 3, 4, 5 },
metadata: new byte[] { 7, 17 });
long newPos;
_writer.Write(_record, out newPos);
_writer.Flush();
}
public void try_read_returns_false_when_writer_checksum_is_equal_to_reader_checksum()
{
var writerchk = new InMemoryCheckpoint();
var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0);
var db = new TFChunkDb(new TFChunkDbConfig(PathName,
new PrefixFileNamingStrategy(PathName, "prefix.tf"),
10000,
0,
writerchk,
chaserchk,
new[] { writerchk, chaserchk }));
db.OpenVerifyAndClean();
writerchk.Write(12);
writerchk.Flush();
chaserchk.Write(12);
chaserchk.Flush();
var chaser = new TFChunkChaser(db, writerchk, chaserchk);
chaser.Open();
LogRecord record;
Assert.IsFalse(chaser.TryReadNext(out record));
Assert.AreEqual(12, chaserchk.Read());
chaser.Close();
db.Dispose();
}
public void try_read_returns_false_when_writer_checksum_is_ahead_but_not_enough_to_read_record()
{
var writerchk = new InMemoryCheckpoint(50);
var readerchk = new InMemoryCheckpoint("reader", 0);
var config = new TransactionFileDatabaseConfig(PathName, "prefix.tf", 10000, writerchk, new[] { readerchk });
var recordToWrite = new PrepareLogRecord(logPosition: 0,
correlationId: _correlationId,
eventId: _eventId,
transactionPosition: 0,
eventStreamId: "WorldEnding",
expectedVersion: 1234,
timeStamp: new DateTime(2012, 12, 21),
flags: PrepareFlags.None,
eventType: "type",
data: new byte[] { 1, 2, 3, 4, 5 },
metadata: new byte[] { 7, 17 });
using (var fs = new FileStream(Path.Combine(PathName, "prefix.tf0"), FileMode.CreateNew, FileAccess.Write))
{
var writer = new BinaryWriter(fs);
recordToWrite.WriteWithLengthPrefixTo(writer);
fs.Close();
}
var reader = new MultifileTransactionFileChaser(config, "reader");
reader.Open();
LogRecord record = null;
var readRecord = reader.TryReadNext(out record);
reader.Close();
Assert.IsFalse(readRecord);
Assert.AreEqual(0, readerchk.Read());
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
WriterCheckpoint = new InMemoryCheckpoint(0);
var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0);
Db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
WriterCheckpoint,
new[] { chaserchk }));
Db.OpenVerifyAndClean();
// create db
Writer = new TFChunkWriter(Db);
Writer.Open();
WriteTestScenario();
Writer.Close();
Writer = null;
WriterCheckpoint.Flush();
chaserchk.Write(WriterCheckpoint.Read());
chaserchk.Flush();
TableIndex = new TableIndex(Path.Combine(PathName, "index"), () => new HashListMemTable(), _maxEntriesInMemTable);
TableIndex.Initialize();
var reader = new TFChunkReader(Db, Db.Config.WriterCheckpoint);
ReadIndex = new ReadIndex(new NoopPublisher(), 2, () => new TFChunkSequentialReader(Db, Db.Config.WriterCheckpoint, 0), () => reader, TableIndex, new ByLengthHasher());
ReadIndex.Build();
}
public void try_read_does_not_cache_anything_and_returns_record_once_it_is_written_later()
{
var writerchk = new InMemoryCheckpoint(0);
var db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
writerchk,
new InMemoryCheckpoint()));
db.OpenVerifyAndClean();
var writer = new TFChunkWriter(db);
writer.Open();
var reader = new TFChunkSequentialReader(db, writerchk, 0);
LogRecord record;
Assert.IsFalse(reader.TryReadNext(out record));
var rec = LogRecord.SingleWrite(0, Guid.NewGuid(), Guid.NewGuid(), "ES", -1, "ET", new byte[] { 7 }, null);
long tmp;
Assert.IsTrue(writer.Write(rec, out tmp));
writer.Flush();
writer.Close();
Assert.IsTrue(reader.TryReadNext(out record));
Assert.AreEqual(rec, record);
reader.Close();
db.Close();
}
public static TFChunkDbConfig CreateDbConfig(
string pathName,
ICheckpoint writerCheckpoint,
ICheckpoint chaserCheckpoint,
int chunkSize = 10000,
ICheckpoint replicationCheckpoint = null)
{
if (replicationCheckpoint == null)
{
replicationCheckpoint = new InMemoryCheckpoint(-1);
}
return(new TFChunkDbConfig(
pathName,
new VersionedPatternFileNamingStrategy(pathName, "chunk-"),
chunkSize,
0,
writerCheckpoint,
chaserCheckpoint,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1),
replicationCheckpoint,
new InMemoryCheckpoint(-1),
Constants.TFChunkInitialReaderCountDefault,
Constants.TFChunkMaxReaderCountDefault));
}
public void a_record_can_be_written()
{
_checkpoint = new InMemoryCheckpoint(0);
var tf = new MultifileTransactionFileWriter(
new TransactionFileDatabaseConfig(PathName, "prefix.tf", 10000, _checkpoint, new List <ICheckpoint>()));
tf.Open();
var record = new PrepareLogRecord(logPosition: 0,
correlationId: _correlationId,
eventId: _eventId,
transactionPosition: 0,
eventStreamId: "WorldEnding",
expectedVersion: 1234,
timeStamp: new DateTime(2012, 12, 21),
flags: PrepareFlags.None,
eventType: "type",
data: new byte[] { 1, 2, 3, 4, 5 },
metadata: new byte[] { 7, 17 });
long tmp;
tf.Write(record, out tmp);
tf.Close();
Assert.AreEqual(record.GetSizeWithLengthPrefix(), _checkpoint.Read());
using (var filestream = File.Open(Path.Combine(PathName, "prefix.tf0"), FileMode.Open, FileAccess.Read))
{
var reader = new BinaryReader(filestream);
reader.ReadInt32();
var read = LogRecord.ReadFrom(reader);
Assert.AreEqual(record, read);
}
}
private TFChunkDbConfig CreateDbConfig(int chunkSize, string dbPath, long chunksCacheSize, bool inMemDb)
{
ICheckpoint writerChk;
ICheckpoint chaserChk;
ICheckpoint epochChk;
ICheckpoint truncateChk;
ICheckpoint replicationCheckpoint = new InMemoryCheckpoint(-1);
if (inMemDb)
{
writerChk = new InMemoryCheckpoint(Checkpoint.Writer);
chaserChk = new InMemoryCheckpoint(Checkpoint.Chaser);
epochChk = new InMemoryCheckpoint(Checkpoint.Epoch, initValue: -1);
truncateChk = new InMemoryCheckpoint(Checkpoint.Truncate, initValue: -1);
}
else
{
var writerCheckFilename = Path.Combine(dbPath, Checkpoint.Writer + ".chk");
var chaserCheckFilename = Path.Combine(dbPath, Checkpoint.Chaser + ".chk");
var epochCheckFilename = Path.Combine(dbPath, Checkpoint.Epoch + ".chk");
var truncateCheckFilename = Path.Combine(dbPath, Checkpoint.Truncate + ".chk");
writerChk = new MemoryMappedFileCheckpoint(writerCheckFilename, Checkpoint.Writer, cached: true);
chaserChk = new MemoryMappedFileCheckpoint(chaserCheckFilename, Checkpoint.Chaser, cached: true);
epochChk = new MemoryMappedFileCheckpoint(
epochCheckFilename, Checkpoint.Epoch, cached: true, initValue: -1);
truncateChk = new MemoryMappedFileCheckpoint(
truncateCheckFilename, Checkpoint.Truncate, cached: true, initValue: -1);
}
var nodeConfig = new TFChunkDbConfig(
dbPath, new VersionedPatternFileNamingStrategy(dbPath, "chunk-"), chunkSize, chunksCacheSize, writerChk,
chaserChk, epochChk, truncateChk, replicationCheckpoint, Constants.TFChunkInitialReaderCountDefault, Constants.TFChunkMaxReaderCountDefault, inMemDb);
return(nodeConfig);
}
public void a_record_can_be_written()
{
var filename = Path.Combine(PathName, "prefix.tf0");
var secondfilename = Path.Combine(PathName, "prefix.tf1");
var thirdfilename = Path.Combine(PathName, "prefix.tf2");
File.WriteAllBytes(filename, new byte[50]);
_checkpoint = new InMemoryCheckpoint(40);
var tf = new MultifileTransactionFileWriter(
new TransactionFileDatabaseConfig(PathName, "prefix.tf", 50, _checkpoint, new ICheckpoint[0]));
tf.Open();
var record = new PrepareLogRecord(logPosition: 0,
correlationId: _correlationId,
eventId: _eventId,
transactionPosition: 0,
transactionOffset: 0,
eventStreamId: "WorldEnding",
expectedVersion: 1234,
timeStamp: new DateTime(2012, 12, 21),
flags: PrepareFlags.None,
eventType: "type",
data: new byte[] { 1, 2, 3, 4, 5 },
metadata: new byte[] { 7, 17 });
long tmp;
tf.Write(record, out tmp);
tf.Close();
Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix() + 40, _checkpoint.Read()); //+orginal checkpoint position
Assert.IsTrue(File.Exists(secondfilename));
var stream = new MemoryStream();
var buffer = new byte[256];
using (var filestream = File.Open(filename, FileMode.Open, FileAccess.Read))
{
filestream.Seek(40, SeekOrigin.Begin);
filestream.Read(buffer, 0, 10);
stream.Write(buffer, 0, 10);
}
using (var filestream = File.Open(secondfilename, FileMode.Open, FileAccess.Read))
{
filestream.Seek(0, SeekOrigin.Begin);
filestream.Read(buffer, 0, 50);
stream.Write(buffer, 0, 50);
}
using (var filestream = File.Open(thirdfilename, FileMode.Open, FileAccess.Read))
{
filestream.Seek(0, SeekOrigin.Begin);
filestream.Read(buffer, 0, 50);
stream.Write(buffer, 0, 50);
}
stream.Seek(0 + sizeof(int), SeekOrigin.Begin);
var reader = new BinaryReader(stream);
var read = LogRecord.ReadFrom(reader);
Assert.AreEqual(record, read);
}
public void a_record_can_be_written()
{
var filename = Path.Combine(PathName, "prefix.tf0");
var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, 10000, 0, 0, 0);
var chunkBytes = chunkHeader.AsByteArray();
var buf = new byte[ChunkHeader.Size + ChunkFooter.Size + chunkHeader.ChunkSize];
Buffer.BlockCopy(chunkBytes, 0, buf, 0, chunkBytes.Length);
File.WriteAllBytes(filename, buf);
_checkpoint = new InMemoryCheckpoint(137);
var db = new TFChunkDb(new TFChunkDbConfig(PathName,
new PrefixFileNamingStrategy(PathName, "prefix.tf"),
chunkHeader.ChunkSize,
0,
_checkpoint,
new ICheckpoint[0]));
db.OpenVerifyAndClean();
var writer = new TFChunkWriter(db);
var record = new PrepareLogRecord(logPosition: 0,
correlationId: _correlationId,
eventId: _eventId,
transactionPosition: 0,
transactionOffset: 0,
eventStreamId: "WorldEnding",
expectedVersion: 1234,
timeStamp: new DateTime(2012, 12, 21),
flags: PrepareFlags.None,
eventType: "type",
data: new byte[8000],
metadata: new byte[] { 7, 17 });
Console.WriteLine(record.GetSizeWithLengthPrefixAndSuffix());
Console.WriteLine(record.GetSizeWithLengthPrefixAndSuffix() + 137);
long pos;
Assert.IsTrue(writer.Write(record, out pos));
writer.Close();
db.Dispose();
Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix() + 137, _checkpoint.Read());
using (var filestream = File.Open(filename, FileMode.Open, FileAccess.Read))
{
filestream.Seek(ChunkHeader.Size + 137 + sizeof(int), SeekOrigin.Begin);
var reader = new BinaryReader(filestream);
var read = LogRecord.ReadFrom(reader);
Console.WriteLine(string.Join("\n", Directory.EnumerateFiles(PathName)));
Assert.AreEqual(record, read);
}
}
protected static TFChunkDbConfig CreateDbConfig(string dbPath, int cachedChunks, long chunksCacheSize, bool inMemDb)
{
ICheckpoint writerChk;
ICheckpoint chaserChk;
ICheckpoint epochChk;
ICheckpoint truncateChk;
if (inMemDb)
{
writerChk = new InMemoryCheckpoint(Checkpoint.Writer);
chaserChk = new InMemoryCheckpoint(Checkpoint.Chaser);
epochChk = new InMemoryCheckpoint(Checkpoint.Epoch, initValue: -1);
truncateChk = new InMemoryCheckpoint(Checkpoint.Truncate, initValue: -1);
}
else
{
if (!Directory.Exists(dbPath)) // mono crashes without this check
{
Directory.CreateDirectory(dbPath);
}
var writerCheckFilename = Path.Combine(dbPath, Checkpoint.Writer + ".chk");
var chaserCheckFilename = Path.Combine(dbPath, Checkpoint.Chaser + ".chk");
var epochCheckFilename = Path.Combine(dbPath, Checkpoint.Epoch + ".chk");
var truncateCheckFilename = Path.Combine(dbPath, Checkpoint.Truncate + ".chk");
if (Runtime.IsMono)
{
writerChk = new FileCheckpoint(writerCheckFilename, Checkpoint.Writer, cached: true);
chaserChk = new FileCheckpoint(chaserCheckFilename, Checkpoint.Chaser, cached: true);
epochChk = new FileCheckpoint(epochCheckFilename, Checkpoint.Epoch, cached: true, initValue: -1);
truncateChk = new FileCheckpoint(truncateCheckFilename, Checkpoint.Truncate, cached: true, initValue: -1);
}
else
{
writerChk = new MemoryMappedFileCheckpoint(writerCheckFilename, Checkpoint.Writer, cached: true);
chaserChk = new MemoryMappedFileCheckpoint(chaserCheckFilename, Checkpoint.Chaser, cached: true);
epochChk = new MemoryMappedFileCheckpoint(epochCheckFilename, Checkpoint.Epoch, cached: true, initValue: -1);
truncateChk = new MemoryMappedFileCheckpoint(truncateCheckFilename, Checkpoint.Truncate, cached: true, initValue: -1);
}
}
var cache = cachedChunks >= 0
? cachedChunks * (long)(TFConsts.ChunkSize + ChunkHeader.Size + ChunkFooter.Size)
: chunksCacheSize;
var nodeConfig = new TFChunkDbConfig(dbPath,
new VersionedPatternFileNamingStrategy(dbPath, "chunk-"),
TFConsts.ChunkSize,
cache,
writerChk,
chaserChk,
epochChk,
truncateChk,
inMemDb);
return(nodeConfig);
}
public void Setup()
{
_bus.Subscribe(_consumer);
ICheckpoint writerCheckpoint = new InMemoryCheckpoint(1000);
_readerService = new EventReaderCoreService(
GetInputQueue(), _ioDispatcher, 10, writerCheckpoint, runHeadingReader: GivenHeadingReaderRunning(),
faultOutOfOrderProjections: true);
_subscriptionDispatcher =
new ReaderSubscriptionDispatcher(GetInputQueue());
_bus.Subscribe(
_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.CheckpointSuggested>());
_bus.Subscribe(
_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.CommittedEventReceived>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.EofReached>());
_bus.Subscribe(
_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.PartitionEofReached>());
_bus.Subscribe(_subscriptionDispatcher
.CreateSubscriber <EventReaderSubscriptionMessage.PartitionMeasured>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.ProgressChanged>());
_bus.Subscribe(
_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.SubscriptionStarted>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.NotAuthorized>());
_bus.Subscribe(
_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.ReaderAssignedReader>());
_bus.Subscribe <ReaderCoreServiceMessage.StartReader>(_readerService);
_bus.Subscribe <ReaderCoreServiceMessage.StopReader>(_readerService);
_bus.Subscribe <ReaderCoreServiceMessage.ReaderTick>(_readerService);
_bus.Subscribe <ReaderSubscriptionMessage.CommittedEventDistributed>(_readerService);
_bus.Subscribe <ReaderSubscriptionMessage.EventReaderEof>(_readerService);
_bus.Subscribe <ReaderSubscriptionMessage.EventReaderPartitionEof>(_readerService);
_bus.Subscribe <ReaderSubscriptionMessage.EventReaderPartitionDeleted>(_readerService);
_bus.Subscribe <ReaderSubscriptionMessage.EventReaderNotAuthorized>(_readerService);
_bus.Subscribe <ReaderSubscriptionMessage.EventReaderIdle>(_readerService);
_bus.Subscribe <ReaderSubscriptionMessage.EventReaderStarting>(_readerService);
_bus.Subscribe <ReaderSubscriptionManagement.Pause>(_readerService);
_bus.Subscribe <ReaderSubscriptionManagement.Resume>(_readerService);
_bus.Subscribe <ReaderSubscriptionManagement.Subscribe>(_readerService);
_bus.Subscribe <ReaderSubscriptionManagement.Unsubscribe>(_readerService);
_bus.Subscribe <ReaderSubscriptionManagement.SpoolStreamReadingCore>(_readerService);
_bus.Subscribe <ReaderSubscriptionManagement.CompleteSpooledStreamReading>(_readerService);
GivenAdditionalServices();
_bus.Publish(new ReaderCoreServiceMessage.StartReader(Guid.NewGuid()));
WhenLoop();
}
public void a_record_can_be_written()
{
var filename = GetFilePathFor("chunk-000000.000000");
var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, 10000, 0, 0, false, Guid.NewGuid());
var chunkBytes = chunkHeader.AsByteArray();
var buf = new byte[ChunkHeader.Size + ChunkFooter.Size + chunkHeader.ChunkSize];
Buffer.BlockCopy(chunkBytes, 0, buf, 0, chunkBytes.Length);
File.WriteAllBytes(filename, buf);
_checkpoint = new InMemoryCheckpoint(137);
var db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, _checkpoint, new InMemoryCheckpoint(),
chunkSize: chunkHeader.ChunkSize));
db.Open();
var
bytes = new byte[3994]; // this gives exactly 4097 size of record, with 3993 (rec size 4096) everything works fine!
new Random().NextBytes(bytes);
var writer = new TFChunkWriter(db);
var logFormat = LogFormatHelper <TLogFormat, TStreamId> .LogFormat;
logFormat.StreamNameIndex.GetOrAddId("WorldEnding", out var streamId, out _, out _);
var record = LogRecord.Prepare(
factory: logFormat.RecordFactory,
logPosition: 137,
correlationId: _correlationId,
eventId: _eventId,
transactionPos: 789,
transactionOffset: 543,
eventStreamId: streamId,
expectedVersion: 1234,
timeStamp: new DateTime(2012, 12, 21),
flags: PrepareFlags.SingleWrite,
eventType: "type",
data: bytes,
metadata: new byte[] { 0x07, 0x17 });
long pos;
Assert.IsTrue(writer.Write(record, out pos));
writer.Close();
db.Dispose();
Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix() + 137, _checkpoint.Read());
using (var filestream = File.Open(filename, FileMode.Open, FileAccess.Read)) {
filestream.Seek(ChunkHeader.Size + 137 + sizeof(int), SeekOrigin.Begin);
var reader = new BinaryReader(filestream);
var read = LogRecord.ReadFrom(reader, (int)reader.BaseStream.Length);
Assert.AreEqual(record, read);
}
}
public void try_read_returns_record_when_record_bigger_than_internal_buffer()
{
var writerchk = new InMemoryCheckpoint(0);
var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0);
var db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
writerchk,
chaserchk,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
db.Open();
var recordToWrite = new PrepareLogRecord(logPosition: 0,
correlationId: _correlationId,
eventId: _eventId,
transactionPosition: 0,
transactionOffset: 0,
eventStreamId: "WorldEnding",
expectedVersion: 1234,
timeStamp: new DateTime(2012, 12, 21),
flags: PrepareFlags.None,
eventType: "type",
data: new byte[9000],
metadata: new byte[] { 7, 17 });
var writer = new TFChunkWriter(db);
writer.Open();
long pos;
Assert.IsTrue(writer.Write(recordToWrite, out pos));
writer.Close();
writerchk.Write(recordToWrite.GetSizeWithLengthPrefixAndSuffix());
var reader = new TFChunkChaser(db, writerchk, chaserchk);
reader.Open();
LogRecord record;
var readRecord = reader.TryReadNext(out record);
reader.Close();
Assert.IsTrue(readRecord);
Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix(), chaserchk.Read());
Assert.AreEqual(recordToWrite, record);
db.Close();
}
public void try_read_returns_record_when_writerchecksum_ahead()
{
var recordToWrite = new PrepareLogRecord(logPosition: 0,
correlationId: _correlationId,
eventId: _eventId,
transactionPosition: 0,
transactionOffset: 0,
eventStreamId: "WorldEnding",
expectedVersion: 1234,
timeStamp: new DateTime(2012, 12, 21),
flags: PrepareFlags.None,
eventType: "type",
data: new byte[] { 1, 2, 3, 4, 5 },
metadata: new byte[] { 7, 17 });
using (var fs = new FileStream(GetFilePathFor("chunk-000000.000000"), FileMode.CreateNew, FileAccess.Write))
{
fs.SetLength(ChunkHeader.Size + ChunkFooter.Size + 10000);
var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, 10000, 0, 0, false, Guid.NewGuid()).AsByteArray();
var writer = new BinaryWriter(fs);
writer.Write(chunkHeader);
recordToWrite.WriteWithLengthPrefixAndSuffixTo(writer);
fs.Close();
}
var writerchk = new InMemoryCheckpoint(128);
var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0);
var db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
writerchk,
chaserchk,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
db.Open();
var chaser = new TFChunkChaser(db, writerchk, chaserchk);
chaser.Open();
LogRecord record;
var recordRead = chaser.TryReadNext(out record);
chaser.Close();
Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix(), chaserchk.Read());
Assert.IsTrue(recordRead);
Assert.AreEqual(recordToWrite, record);
db.Close();
}
public void a_record_can_be_written()
{
var filename = Path.Combine(PathName, "prefix.tf0");
var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, 10000, 0, 0, 0);
var chunkBytes = chunkHeader.AsByteArray();
var buf = new byte[ChunkHeader.Size + ChunkFooter.Size + chunkHeader.ChunkSize];
Buffer.BlockCopy(chunkBytes, 0, buf, 0, chunkBytes.Length);
File.WriteAllBytes(filename, buf);
_checkpoint = new InMemoryCheckpoint(137);
var db = new TFChunkDb(new TFChunkDbConfig(PathName,
new PrefixFileNamingStrategy(PathName, "prefix.tf"),
chunkHeader.ChunkSize,
0,
_checkpoint,
new InMemoryCheckpoint(),
new ICheckpoint[0]));
db.OpenVerifyAndClean();
var bytes = new byte[3994]; // this gives exactly 4097 size of record, with 3993 (rec size 4096) everything works fine!
new Random().NextBytes(bytes);
var writer = new TFChunkWriter(db);
var record = new PrepareLogRecord(logPosition: 123,
correlationId: _correlationId,
eventId: _eventId,
transactionPosition: 789,
transactionOffset: 543,
eventStreamId: "WorldEnding",
expectedVersion: 1234,
timeStamp: new DateTime(2012, 12, 21),
flags: PrepareFlags.SingleWrite,
eventType: "type",
data: bytes,
metadata: new byte[] { 0x07, 0x17 });
long pos;
Assert.IsTrue(writer.Write(record, out pos));
writer.Close();
db.Dispose();
Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix() + 137, _checkpoint.Read());
using (var filestream = File.Open(filename, FileMode.Open, FileAccess.Read))
{
filestream.Seek(ChunkHeader.Size + 137 + sizeof(int), SeekOrigin.Begin);
var reader = new BinaryReader(filestream);
var read = LogRecord.ReadFrom(reader);
Assert.AreEqual(record, read);
}
}
public void try_read_returns_record_when_record_bigger_than_internal_buffer()
{
var writerchk = new InMemoryCheckpoint(0);
var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0);
var db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, writerchk, chaserchk));
db.Open();
var recordFactory = LogFormatHelper <TLogFormat, TStreamId> .RecordFactory;
var streamId = LogFormatHelper <TLogFormat, TStreamId> .StreamId;
var eventTypeId = LogFormatHelper <TLogFormat, TStreamId> .EventTypeId;
var recordToWrite = LogRecord.Prepare(
factory: recordFactory,
logPosition: 0,
correlationId: _correlationId,
eventId: _eventId,
transactionPos: 0,
transactionOffset: 0,
eventStreamId: streamId,
expectedVersion: 1234,
timeStamp: new DateTime(2012, 12, 21),
flags: PrepareFlags.None,
eventType: eventTypeId,
data: new byte[9000],
metadata: new byte[] { 7, 17 });
var writer = new TFChunkWriter(db);
writer.Open();
long pos;
Assert.IsTrue(writer.Write(recordToWrite, out pos));
writer.Close();
writerchk.Write(recordToWrite.GetSizeWithLengthPrefixAndSuffix());
var reader = new TFChunkChaser(db, writerchk, chaserchk, false);
reader.Open();
ILogRecord record;
var readRecord = reader.TryReadNext(out record);
reader.Close();
Assert.IsTrue(readRecord);
Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix(), chaserchk.Read());
Assert.AreEqual(recordToWrite, record);
db.Close();
}
public void try_read_returns_record_when_writerchecksum_ahead()
{
var recordFactory = LogFormatHelper <TLogFormat, TStreamId> .RecordFactory;
var streamId = LogFormatHelper <TLogFormat, TStreamId> .StreamId;
var eventTypeId = LogFormatHelper <TLogFormat, TStreamId> .EventTypeId;
var recordToWrite = LogRecord.Prepare(
factory: recordFactory,
logPosition: 0,
correlationId: _correlationId,
eventId: _eventId,
transactionPos: 0,
transactionOffset: 0,
eventStreamId: streamId,
expectedVersion: 1234,
timeStamp: new DateTime(2012, 12, 21),
flags: PrepareFlags.None,
eventType: eventTypeId,
data: new byte[] { 1, 2, 3, 4, 5 },
metadata: new byte[] { 7, 17 });
using (var fs = new FileStream(GetFilePathFor("chunk-000000.000000"), FileMode.CreateNew,
FileAccess.Write)) {
fs.SetLength(ChunkHeader.Size + ChunkFooter.Size + 10000);
var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, 10000, 0, 0, false, Guid.NewGuid())
.AsByteArray();
var writer = new BinaryWriter(fs);
writer.Write(chunkHeader);
recordToWrite.WriteWithLengthPrefixAndSuffixTo(writer);
fs.Close();
}
var writerchk = new InMemoryCheckpoint(recordToWrite.GetSizeWithLengthPrefixAndSuffix() + 16);
var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0);
var db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, writerchk, chaserchk));
db.Open();
var chaser = new TFChunkChaser(db, writerchk, chaserchk, false);
chaser.Open();
ILogRecord record;
var recordRead = chaser.TryReadNext(out record);
chaser.Close();
Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix(), chaserchk.Read());
Assert.IsTrue(recordRead);
Assert.AreEqual(recordToWrite, record);
db.Close();
}
public void a_record_can_be_written()
{
var filename = Path.Combine(PathName, "prefix.tf0");
var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, 10000, 0, 0, 0);
var chunkBytes = chunkHeader.AsByteArray();
var bytes = new byte[ChunkHeader.Size + 10000 + ChunkFooter.Size];
Buffer.BlockCopy(chunkBytes, 0, bytes, 0, chunkBytes.Length);
File.WriteAllBytes(filename, bytes);
_checkpoint = new InMemoryCheckpoint(137);
var db = new TFChunkDb(new TFChunkDbConfig(PathName,
new PrefixFileNamingStrategy(PathName, "prefix.tf"),
10000,
0,
_checkpoint,
new InMemoryCheckpoint(),
new ICheckpoint[0]));
db.OpenVerifyAndClean();
var tf = new TFChunkWriter(db);
var record = new PrepareLogRecord(logPosition: 0,
correlationId: _correlationId,
eventId: _eventId,
expectedVersion: 1234,
transactionPosition: 0,
transactionOffset: 0,
eventStreamId: "WorldEnding",
timeStamp: new DateTime(2012, 12, 21),
flags: PrepareFlags.None,
eventType: "type",
data: new byte[] { 1, 2, 3, 4, 5 },
metadata: new byte[] { 7, 17 });
long tmp;
tf.Write(record, out tmp);
//tf.Flush();
tf.Close();
db.Dispose();
Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix() + 137, _checkpoint.Read()); //137 is fluff assigned to beginning of checkpoint
//TODO actually read the event
using (var filestream = File.Open(filename, FileMode.Open, FileAccess.Read))
{
filestream.Seek(ChunkHeader.Size + 137 + sizeof(int), SeekOrigin.Begin);
var reader = new BinaryReader(filestream);
var read = LogRecord.ReadFrom(reader);
Assert.AreEqual(record, read);
}
}
public void try_read_returns_false_when_writer_checksum_is_zero()
{
var writerchk = new InMemoryCheckpoint(0);
var config = new TransactionFileDatabaseConfig(PathName, "prefix.tf", 10000, writerchk, new List <ICheckpoint>());
File.WriteAllBytes(Path.Combine(PathName, "prefix.tf0"), new byte[10000]);
var reader = new MultifileTransactionFileChaser(config);
reader.Open();
LogRecord record;
Assert.IsFalse(reader.TryReadNext(out record));
reader.Close();
}
public void try_read_returns_false_when_writer_checksum_is_zero()
{
var writerchk = new InMemoryCheckpoint(0);
var chaserchk = new InMemoryCheckpoint(0);
var db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, writerchk, chaserchk));
db.Open();
var reader = new TFChunkReader(db, writerchk, 0);
Assert.IsFalse(reader.TryReadNext().Success);
db.Close();
}
public void try_read_returns_record_when_writerchecksum_equal()
{
var writerchk = new InMemoryCheckpoint(0);
var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0);
var db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, writerchk, chaserchk));
db.Open();
_logFormat.StreamNameIndex.GetOrAddId("WorldEnding", out var streamId, out _, out _);
var recordToWrite = LogRecord.Prepare(
factory: _logFormat.RecordFactory,
logPosition: 0,
correlationId: _correlationId,
eventId: _eventId,
transactionPos: 0,
transactionOffset: 0,
eventStreamId: streamId,
expectedVersion: 1234,
timeStamp: new DateTime(2012, 12, 21),
flags: PrepareFlags.None,
eventType: "type",
data: new byte[] { 1, 2, 3, 4, 5 },
metadata: new byte[] { 7, 17 });
var writer = new TFChunkWriter(db);
writer.Open();
long pos;
Assert.IsTrue(writer.Write(recordToWrite, out pos));
writer.Close();
writerchk.Write(recordToWrite.GetSizeWithLengthPrefixAndSuffix());
var chaser = new TFChunkChaser(db, writerchk, chaserchk, false);
chaser.Open();
ILogRecord record;
var readRecord = chaser.TryReadNext(out record);
chaser.Close();
Assert.IsTrue(readRecord);
Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix(), chaserchk.Read());
Assert.AreEqual(recordToWrite, record);
db.Close();
}
public void a_record_can_be_written()
{
var filename = GetFilePathFor("chunk-000000.000000");
var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, 10000, 0, 0, false, chunkId: Guid.NewGuid());
var chunkBytes = chunkHeader.AsByteArray();
var bytes = new byte[ChunkHeader.Size + 10000 + ChunkFooter.Size];
Buffer.BlockCopy(chunkBytes, 0, bytes, 0, chunkBytes.Length);
File.WriteAllBytes(filename, bytes);
_checkpoint = new InMemoryCheckpoint(137);
var db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, _checkpoint, new InMemoryCheckpoint()));
db.Open();
var recordFactory = LogFormatHelper <TLogFormat, TStreamId> .RecordFactory;
var streamId = LogFormatHelper <TLogFormat, TStreamId> .StreamId;
var eventTypeId = LogFormatHelper <TLogFormat, TStreamId> .EventTypeId;
var tf = new TFChunkWriter(db);
var record = LogRecord.Prepare(
factory: recordFactory,
logPosition: _checkpoint.Read(),
correlationId: _correlationId,
eventId: _eventId,
expectedVersion: 1234,
transactionPos: 0,
transactionOffset: 0,
eventStreamId: streamId,
timeStamp: new DateTime(2012, 12, 21),
flags: PrepareFlags.None,
eventType: eventTypeId,
data: new byte[] { 1, 2, 3, 4, 5 },
metadata: new byte[] { 7, 17 });
long tmp;
tf.Write(record, out tmp);
tf.Close();
db.Dispose();
Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix() + 137,
_checkpoint.Read()); //137 is fluff assigned to beginning of checkpoint
using (var filestream = File.Open(filename, FileMode.Open, FileAccess.Read)) {
filestream.Seek(ChunkHeader.Size + 137 + sizeof(int), SeekOrigin.Begin);
var reader = new BinaryReader(filestream);
var read = LogRecord.ReadFrom(reader, (int)reader.BaseStream.Length);
Assert.AreEqual(record, read);
}
}
