private void ReOpenDb()
{
Db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
WriterCheckpoint,
ChaserCheckpoint,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
Db.Open();
var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(Path.Combine(PathName, "index"), lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.Index64Bit, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
PTableVersions.Index64Bit,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: true,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault);
ReadIndex.Init(ChaserCheckpoint.Read());
}
public void SetUp()
{
_writerCheckpoint = new InMemoryCheckpoint();
_db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024,
0,
_writerCheckpoint,
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
_db.Open();
_writer = new TFChunkWriter(_db);
_writer.Open();
_record = new PrepareLogRecord(logPosition: 0,
eventId: _eventId,
correlationId: _correlationId,
transactionPosition: 0xDEAD,
transactionOffset: 0xBEEF,
eventStreamId: "WorldEnding",
expectedVersion: 1234,
timeStamp: new DateTime(2012, 12, 21),
flags: PrepareFlags.SingleWrite,
eventType: "type",
data: new byte[] { 1, 2, 3, 4, 5 },
metadata: new byte[] {7, 17});
long newPos;
_writer.Write(_record, out newPos);
_writer.Flush();
}
public void try_read_returns_false_when_writer_checksum_is_equal_to_reader_checksum()
{
var writerchk = new InMemoryCheckpoint();
var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0);
var db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
writerchk,
chaserchk,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
db.Open();
writerchk.Write(12);
writerchk.Flush();
chaserchk.Write(12);
chaserchk.Flush();
var chaser = new TFChunkChaser(db, writerchk, chaserchk);
chaser.Open();
LogRecord record;
Assert.IsFalse(chaser.TryReadNext(out record));
Assert.AreEqual(12, chaserchk.Read());
chaser.Close();
db.Dispose();
}
private void ReOpenDb()
{
Db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
WriterCheckpoint,
ChaserCheckpoint,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
Db.Open();
var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
TableIndex = new TableIndex(Path.Combine(PathName, "index"),
() => new HashListMemTable(MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
new ByLengthHasher(),
0,
additionalCommitChecks: true,
metastreamMaxCount: MetastreamMaxCount);
ReadIndex.Init(ChaserCheckpoint.Read());
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
4096,
0,
new InMemoryCheckpoint(),
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
_db.Open();
var chunk = _db.Manager.GetChunk(0);
_records = new LogRecord[RecordsCount];
_results = new RecordWriteResult[RecordsCount];
for (int i = 0; i < _records.Length; ++i)
{
_records[i] = LogRecord.SingleWrite(i == 0 ? 0 : _results[i - 1].NewPosition,
Guid.NewGuid(), Guid.NewGuid(), "es1", ExpectedVersion.Any, "et1",
new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
_results[i] = chunk.TryAppend(_records[i]);
}
chunk.Flush();
_db.Config.WriterCheckpoint.Write(_results[RecordsCount - 1].NewPosition);
_db.Config.WriterCheckpoint.Flush();
}
public void try_read_does_not_cache_anything_and_returns_record_once_it_is_written_later()
{
var writerchk = new InMemoryCheckpoint(0);
var db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
writerchk,
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
db.Open();
var writer = new TFChunkWriter(db);
writer.Open();
var reader = new TFChunkReader(db, writerchk, 0);
Assert.IsFalse(reader.TryReadNext().Success);
var rec = LogRecord.SingleWrite(0, Guid.NewGuid(), Guid.NewGuid(), "ES", -1, "ET", new byte[] { 7 }, null);
long tmp;
Assert.IsTrue(writer.Write(rec, out tmp));
writer.Flush();
writer.Close();
var res = reader.TryReadNext();
Assert.IsTrue(res.Success);
Assert.AreEqual(rec, res.LogRecord);
db.Close();
}
private void ReOpenDb()
{
Db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
WriterCheckpoint,
ChaserCheckpoint,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
Db.Open();
TableIndex = new TableIndex(Path.Combine(PathName, "index"),
() => new HashListMemTable(MaxEntriesInMemTable * 2),
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
2,
5,
() => new TFChunkReader(Db, Db.Config.WriterCheckpoint),
TableIndex,
new ByLengthHasher(),
new NoLRUCache<string, StreamCacheInfo>(),
additionalCommitChecks: true,
metastreamMaxCount: MetastreamMaxCount);
ReadIndex.Init(WriterCheckpoint.Read(), ChaserCheckpoint.Read());
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
WriterCheckpoint = new InMemoryCheckpoint(0);
ChaserCheckpoint = new InMemoryCheckpoint(0);
Bus = new InMemoryBus("bus");
IODispatcher = new IODispatcher(Bus, new PublishEnvelope(Bus));
Db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
WriterCheckpoint,
ChaserCheckpoint,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
Db.Open();
// create db
Writer = new TFChunkWriter(Db);
Writer.Open();
WriteTestScenario();
Writer.Close();
Writer = null;
WriterCheckpoint.Flush();
ChaserCheckpoint.Write(WriterCheckpoint.Read());
ChaserCheckpoint.Flush();
var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.Index64Bit, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
PTableVersions.Index64Bit,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: true,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault);
ReadIndex.Init(ChaserCheckpoint.Read());
// scavenge must run after readIndex is built
if (_scavenge)
{
if (_completeLastChunkOnScavenge)
Db.Manager.GetChunk(Db.Manager.ChunksCount - 1).Complete();
_scavenger = new TFChunkScavenger(Db, IODispatcher, TableIndex, ReadIndex, Guid.NewGuid(), "fakeNodeIp");
_scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: _mergeChunks);
}
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
16 * 1024,
0,
new InMemoryCheckpoint(),
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
_db.Open();
var chunk = _db.Manager.GetChunkFor(0);
_p1 = LogRecord.SingleWrite(0, Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1",
new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
_res1 = chunk.TryAppend(_p1);
_c1 = LogRecord.Commit(_res1.NewPosition, Guid.NewGuid(), _p1.LogPosition, 0);
_cres1 = chunk.TryAppend(_c1);
_p2 = LogRecord.SingleWrite(_cres1.NewPosition,
Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1",
new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
_res2 = chunk.TryAppend(_p2);
_c2 = LogRecord.Commit(_res2.NewPosition, Guid.NewGuid(), _p2.LogPosition, 1);
_cres2 = chunk.TryAppend(_c2);
_p3 = LogRecord.SingleWrite(_cres2.NewPosition,
Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1",
new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
_res3 = chunk.TryAppend(_p3);
_c3 = LogRecord.Commit(_res3.NewPosition, Guid.NewGuid(), _p3.LogPosition, 2);
_cres3 = chunk.TryAppend(_c3);
chunk.Complete();
_db.Config.WriterCheckpoint.Write(chunk.ChunkHeader.ChunkEndPosition);
_db.Config.WriterCheckpoint.Flush();
_db.Config.ChaserCheckpoint.Write(chunk.ChunkHeader.ChunkEndPosition);
_db.Config.ChaserCheckpoint.Flush();
var bus = new InMemoryBus("Bus");
var ioDispatcher = new IODispatcher(bus, new PublishEnvelope(bus));
var scavenger = new TFChunkScavenger(_db, ioDispatcher, new FakeTableIndex(),
new FakeReadIndex(x => x == "es-to-scavenge"), Guid.NewGuid(), "fakeNodeIp");
scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false);
_scavengedChunk = _db.Manager.GetChunk(0);
}
public override void TestFixtureTearDown()
{
using (var db = new TFChunkDb(_config))
{
Assert.DoesNotThrow(() => db.Open(verifyHash: false));
}
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000")));
Assert.AreEqual(1, Directory.GetFiles(PathName, "*").Length);
base.TestFixtureTearDown();
}
public TFChunkDbCreationHelper(TFChunkDbConfig dbConfig)
{
Ensure.NotNull(dbConfig, "dbConfig");
_dbConfig = dbConfig;
_db = new TFChunkDb(_dbConfig);
_db.Open();
if (_db.Config.WriterCheckpoint.ReadNonFlushed() > 0)
throw new Exception("The DB already contains some data.");
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
WriterCheckpoint = new InMemoryCheckpoint(0);
ChaserCheckpoint = new InMemoryCheckpoint(0);
Db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
WriterCheckpoint,
ChaserCheckpoint,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
Db.Open();
// create db
Writer = new TFChunkWriter(Db);
Writer.Open();
WriteTestScenario();
Writer.Close();
Writer = null;
WriterCheckpoint.Flush();
ChaserCheckpoint.Write(WriterCheckpoint.Read());
ChaserCheckpoint.Flush();
var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 2, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
TableIndex = new TableIndex(GetFilePathFor("index"),
() => new HashListMemTable(MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
MaxEntriesInMemTable);
var hasher = new ByLengthHasher();
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
hasher,
0,
additionalCommitChecks: true,
metastreamMaxCount: MetastreamMaxCount);
ReadIndex.Init(ChaserCheckpoint.Read());
// scavenge must run after readIndex is built
if (_scavenge)
{
if (_completeLastChunkOnScavenge)
Db.Manager.GetChunk(Db.Manager.ChunksCount - 1).Complete();
_scavenger = new TFChunkScavenger(Db, TableIndex, hasher, ReadIndex);
_scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: _mergeChunks);
}
}
public void with_a_writer_checksum_of_zero_and_no_files_is_valid()
{
var db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
Assert.DoesNotThrow(() => db.Open());
db.Dispose();
}
public void a_record_can_be_written()
{
var filename = GetFilePathFor("chunk-000000.000000");
var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, 10000, 0, 0, false, Guid.NewGuid());
var chunkBytes = chunkHeader.AsByteArray();
var buf = new byte[ChunkHeader.Size + ChunkFooter.Size + chunkHeader.ChunkSize];
Buffer.BlockCopy(chunkBytes, 0, buf, 0, chunkBytes.Length);
File.WriteAllBytes(filename, buf);
_checkpoint = new InMemoryCheckpoint(137);
var db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
chunkHeader.ChunkSize,
0,
_checkpoint,
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
db.Open();
var bytes = new byte[3994]; // this gives exactly 4097 size of record, with 3993 (rec size 4096) everything works fine!
new Random().NextBytes(bytes);
var writer = new TFChunkWriter(db);
var record = new PrepareLogRecord(logPosition: 137,
correlationId: _correlationId,
eventId: _eventId,
transactionPosition: 789,
transactionOffset: 543,
eventStreamId: "WorldEnding",
expectedVersion: 1234,
timeStamp: new DateTime(2012, 12, 21),
flags: PrepareFlags.SingleWrite,
eventType: "type",
data: bytes,
metadata: new byte[] { 0x07, 0x17 });
long pos;
Assert.IsTrue(writer.Write(record, out pos));
writer.Close();
db.Dispose();
Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix() + 137, _checkpoint.Read());
using (var filestream = File.Open(filename, FileMode.Open, FileAccess.Read))
{
filestream.Seek(ChunkHeader.Size + 137 + sizeof(int), SeekOrigin.Begin);
var reader = new BinaryReader(filestream);
var read = LogRecord.ReadFrom(reader);
Assert.AreEqual(record, read);
}
}
public void with_a_writer_checksum_of_nonzero_and_no_files_a_corrupted_database_exception_is_thrown()
{
var db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
new InMemoryCheckpoint(500),
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
var exc = Assert.Throws<CorruptDatabaseException>(() => db.Open());
Assert.IsInstanceOf<ChunkNotFoundException>(exc.InnerException);
db.Dispose();
}
public void allows_with_exactly_enough_file_to_reach_checksum()
{
var config = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
new InMemoryCheckpoint(10000),
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
using (var db = new TFChunkDb(config))
{
DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000"));
Assert.DoesNotThrow(() => db.Open(verifyHash: false));
}
}
public void a_record_can_be_written()
{
var filename = GetFilePathFor("chunk-000000.000000");
var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, 10000, 0, 0, false, chunkId: Guid.NewGuid());
var chunkBytes = chunkHeader.AsByteArray();
var bytes = new byte[ChunkHeader.Size + 10000 + ChunkFooter.Size];
Buffer.BlockCopy(chunkBytes, 0, bytes, 0, chunkBytes.Length);
File.WriteAllBytes(filename, bytes);
_checkpoint = new InMemoryCheckpoint(137);
var db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
_checkpoint,
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
db.Open();
var tf = new TFChunkWriter(db);
var record = new PrepareLogRecord(logPosition: _checkpoint.Read(),
correlationId: _correlationId,
eventId: _eventId,
expectedVersion: 1234,
transactionPosition: 0,
transactionOffset: 0,
eventStreamId: "WorldEnding",
timeStamp: new DateTime(2012, 12, 21),
flags: PrepareFlags.None,
eventType: "type",
data: new byte[] { 1, 2, 3, 4, 5 },
metadata: new byte[] { 7, 17 });
long tmp;
tf.Write(record, out tmp);
tf.Close();
db.Dispose();
Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix() + 137, _checkpoint.Read()); //137 is fluff assigned to beginning of checkpoint
using (var filestream = File.Open(filename, FileMode.Open, FileAccess.Read))
{
filestream.Seek(ChunkHeader.Size + 137 + sizeof(int), SeekOrigin.Begin);
var reader = new BinaryReader(filestream);
var read = LogRecord.ReadFrom(reader);
Assert.AreEqual(record, read);
}
}
public void try_read_returns_false_when_writer_checksum_is_zero()
{
var writerchk = new InMemoryCheckpoint(0);
var db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
writerchk,
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
db.Open();
var reader = new TFChunkReader(db, writerchk, 0);
Assert.IsFalse(reader.TryReadNext().Success);
db.Close();
}
public void with_file_of_wrong_size_database_corruption_is_detected()
{
var config = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
new InMemoryCheckpoint(500),
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
using (var db = new TFChunkDb(config))
{
File.WriteAllText(GetFilePathFor("chunk-000000.000000"), "this is just some test blahbydy blah");
Assert.That(() => db.Open(verifyHash: false),
Throws.Exception.InstanceOf<CorruptDatabaseException>()
.With.InnerException.InstanceOf<BadChunkInDatabaseException>());
}
}
public void with_checksum_inside_multi_chunk_throws()
{
var config = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
new InMemoryCheckpoint(25000),
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
using (var db = new TFChunkDb(config))
{
DbUtil.CreateMultiChunk(config, 0, 2, GetFilePathFor("chunk-000000.000000"));
Assert.That(() => db.Open(verifyHash: false),
Throws.Exception.InstanceOf<CorruptDatabaseException>()
.With.InnerException.InstanceOf<ChunkNotFoundException>());
}
}
public void with_a_writer_checksum_of_zero_the_first_chunk_is_created_with_correct_name()
{
var config = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var db = new TFChunkDb(config);
db.Open();
db.Dispose();
Assert.AreEqual(1, Directory.GetFiles(PathName).Length);
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000")));
var fileInfo = new FileInfo(GetFilePathFor("chunk-000000.000000"));
Assert.AreEqual(10000 + ChunkHeader.Size + ChunkFooter.Size, fileInfo.Length);
}
public void a_record_can_be_written()
{
_checkpoint = new InMemoryCheckpoint(0);
var db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1000,
0,
_checkpoint,
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
db.Open();
var tf = new TFChunkWriter(db);
tf.Open();
var record = new PrepareLogRecord(logPosition: 0,
correlationId: _correlationId,
eventId: _eventId,
transactionPosition: 0,
transactionOffset: 0,
eventStreamId: "WorldEnding",
expectedVersion: 1234,
timeStamp: new DateTime(2012, 12, 21),
flags: PrepareFlags.None,
eventType: "type",
data: new byte[] { 1, 2, 3, 4, 5 },
metadata: new byte[] { 7, 17 });
long tmp;
tf.Write(record, out tmp);
tf.Close();
db.Dispose();
Assert.AreEqual(record.GetSizeWithLengthPrefixAndSuffix(), _checkpoint.Read());
using (var filestream = File.Open(GetFilePathFor("chunk-000000.000000"), FileMode.Open, FileAccess.Read))
{
filestream.Position = ChunkHeader.Size;
var reader = new BinaryReader(filestream);
reader.ReadInt32();
var read = LogRecord.ReadFrom(reader);
Assert.AreEqual(record, read);
}
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
4096,
0,
new InMemoryCheckpoint(),
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
_db.Open();
var chunk = _db.Manager.GetChunk(0);
_records = new LogRecord[RecordsCount];
_results = new RecordWriteResult[RecordsCount];
var pos = 0;
for (int i = 0; i < RecordsCount; ++i)
{
if (i > 0 && i % 3 == 0)
{
pos = i/3 * _db.Config.ChunkSize;
chunk.Complete();
chunk = _db.Manager.AddNewChunk();
}
_records[i] = LogRecord.SingleWrite(pos,
Guid.NewGuid(), Guid.NewGuid(), "es1", ExpectedVersion.Any, "et1",
new byte[1200], new byte[] { 5, 7 });
_results[i] = chunk.TryAppend(_records[i]);
pos += _records[i].GetSizeWithLengthPrefixAndSuffix();
}
chunk.Flush();
_db.Config.WriterCheckpoint.Write((RecordsCount / 3) * _db.Config.ChunkSize + _results[RecordsCount - 1].NewPosition);
_db.Config.WriterCheckpoint.Flush();
}
public void allows_with_exactly_enough_file_to_reach_checksum_while_last_is_multi_chunk()
{
var config = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
new InMemoryCheckpoint(30000),
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
using (var db = new TFChunkDb(config))
{
DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000"));
DbUtil.CreateMultiChunk(config, 1, 2, GetFilePathFor("chunk-000001.000000"));
Assert.DoesNotThrow(() => db.Open(verifyHash: false));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000")));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000001.000000")));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000003.000000")));
Assert.AreEqual(3, Directory.GetFiles(PathName, "*").Length);
}
}
public void try_read_returns_false_when_writer_checkpoint_is_zero()
{
var writerchk = new InMemoryCheckpoint(0);
var db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
writerchk,
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
db.Open();
var chaser = new TFChunkChaser(db, writerchk, new InMemoryCheckpoint());
chaser.Open();
LogRecord record;
Assert.IsFalse(chaser.TryReadNext(out record));
chaser.Close();
db.Dispose();
}
public void SetUp()
{
_writerCheckpoint = new InMemoryCheckpoint();
_db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024,
0,
_writerCheckpoint,
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
_db.Open();
_writer = new TFChunkWriter(_db);
_writer.Open();
_record = new CommitLogRecord(logPosition: 0,
correlationId: _eventId,
transactionPosition: 4321,
timeStamp: new DateTime(2012, 12, 21),
firstEventNumber: 10);
long newPos;
_writer.Write(_record, out newPos);
_writer.Flush();
}
public SingleVNode(TFChunkDb db,
SingleVNodeSettings vNodeSettings,
bool dbVerifyHashes,
int memTableEntryCount,
params ISubsystem[] subsystems)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(vNodeSettings, "vNodeSettings");
_settings = vNodeSettings;
_mainBus = new InMemoryBus("MainBus");
_controller = new SingleVNodeController(_mainBus, _settings.ExternalHttpEndPoint, db, this);
_mainQueue = new QueuedHandler(_controller, "MainQueue");
_controller.SetMainQueue(_mainQueue);
_subsystems = subsystems;
// MONITORING
var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
var monitoringQueue = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
var monitoring = new MonitoringService(monitoringQueue,
monitoringRequestBus,
_mainQueue,
db.Config.WriterCheckpoint,
db.Config.Path,
vNodeSettings.StatsPeriod,
_settings.ExternalHttpEndPoint,
vNodeSettings.StatsStorage);
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.SystemInit, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.StateChangeMessage, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShutdown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<ClientMessage.WriteEventsCompleted, Message>());
monitoringInnerBus.Subscribe<SystemMessage.SystemInit>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.StateChangeMessage>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.BecomeShuttingDown>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.BecomeShutdown>(monitoring);
monitoringInnerBus.Subscribe<ClientMessage.WriteEventsCompleted>(monitoring);
monitoringInnerBus.Subscribe<MonitoringMessage.GetFreshStats>(monitoring);
var truncPos = db.Config.TruncateCheckpoint.Read();
if (truncPos != -1)
{
Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})",
truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read());
var truncator = new TFChunkDbTruncator(db.Config);
truncator.TruncateDb(truncPos);
}
db.Open(dbVerifyHashes);
// STORAGE SUBSYSTEM
var indexPath = Path.Combine(db.Config.Path, "index");
var readerPool = new ObjectPool<ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(db, db.Config.WriterCheckpoint));
var tableIndex = new TableIndex(indexPath,
() => new HashListMemTable(maxSize: memTableEntryCount * 2),
() => new TFReaderLease(readerPool),
maxSizeForMemory: memTableEntryCount,
maxTablesPerLevel: 2,
inMem: db.Config.InMemDb);
var hasher = new XXHashUnsafe();
var readIndex = new ReadIndex(_mainQueue,
readerPool,
tableIndex,
hasher,
ESConsts.StreamInfoCacheCapacity,
Application.IsDefined(Application.AdditionalCommitChecks),
Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1);
var writer = new TFChunkWriter(db);
var epochManager = new EpochManager(ESConsts.CachedEpochCount,
db.Config.EpochCheckpoint,
writer,
initialReaderCount: 1,
maxReaderCount: 5,
readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint));
epochManager.Init();
var storageWriter = new StorageWriterService(_mainQueue, _mainBus, _settings.MinFlushDelay,
db, writer, readIndex.IndexWriter, epochManager); // subscribes internally
monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageWriter);
var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, ESConsts.StorageReaderThreadCount, db.Config.WriterCheckpoint);
_mainBus.Subscribe<SystemMessage.SystemInit>(storageReader);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageReader);
_mainBus.Subscribe<SystemMessage.BecomeShutdown>(storageReader);
monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageReader);
var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, readIndex.IndexCommitter, epochManager);
_mainBus.Subscribe<SystemMessage.SystemInit>(storageChaser);
_mainBus.Subscribe<SystemMessage.SystemStart>(storageChaser);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageChaser);
var storageScavenger = new StorageScavenger(db,
tableIndex,
hasher,
readIndex,
Application.IsDefined(Application.AlwaysKeepScavenged),
mergeChunks: false /*!Application.IsDefined(Application.DisableMergeChunks)*/);
// ReSharper disable RedundantTypeArgumentsOfMethod
_mainBus.Subscribe<ClientMessage.ScavengeDatabase>(storageScavenger);
// ReSharper restore RedundantTypeArgumentsOfMethod
// MISC WORKERS
_workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray();
_workersHandler = new MultiQueuedHandler(
vNodeSettings.WorkerThreads,
queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
string.Format("Worker #{0}", queueNum + 1),
groupName: "Workers",
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200)));
// AUTHENTICATION INFRASTRUCTURE
var passwordHashAlgorithm = new Rfc2898PasswordHashAlgorithm();
var dispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_workersHandler, crossThread: true));
var internalAuthenticationProvider = new InternalAuthenticationProvider(dispatcher, passwordHashAlgorithm, ESConsts.CachedPrincipalCount);
var passwordChangeNotificationReader = new PasswordChangeNotificationReader(_mainQueue, dispatcher);
_mainBus.Subscribe<SystemMessage.SystemStart>(passwordChangeNotificationReader);
_mainBus.Subscribe<SystemMessage.BecomeShutdown>(passwordChangeNotificationReader);
_mainBus.Subscribe(internalAuthenticationProvider);
SubscribeWorkers(bus =>
{
bus.Subscribe(dispatcher.ForwardReader);
bus.Subscribe(dispatcher.BackwardReader);
bus.Subscribe(dispatcher.Writer);
bus.Subscribe(dispatcher.StreamDeleter);
bus.Subscribe(dispatcher);
});
// TCP
{
var tcpService = new TcpService(_mainQueue, _settings.ExternalTcpEndPoint, _workersHandler,
TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
internalAuthenticationProvider, null);
_mainBus.Subscribe<SystemMessage.SystemInit>(tcpService);
_mainBus.Subscribe<SystemMessage.SystemStart>(tcpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(tcpService);
}
// SECURE TCP
if (vNodeSettings.ExternalSecureTcpEndPoint != null)
{
var secureTcpService = new TcpService(_mainQueue, _settings.ExternalSecureTcpEndPoint, _workersHandler,
TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(),
ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
internalAuthenticationProvider, _settings.Certificate);
_mainBus.Subscribe<SystemMessage.SystemInit>(secureTcpService);
_mainBus.Subscribe<SystemMessage.SystemStart>(secureTcpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(secureTcpService);
}
SubscribeWorkers(bus =>
{
var tcpSendService = new TcpSendService();
// ReSharper disable RedundantTypeArgumentsOfMethod
bus.Subscribe<TcpMessage.TcpSend>(tcpSendService);
// ReSharper restore RedundantTypeArgumentsOfMethod
});
// HTTP
var httpAuthenticationProviders = new List<HttpAuthenticationProvider>
{
new BasicHttpAuthenticationProvider(internalAuthenticationProvider),
};
if (_settings.EnableTrustedAuth)
httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider());
httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider());
var httpPipe = new HttpMessagePipe();
var httpSendService = new HttpSendService(httpPipe, forwardRequests: false);
_mainBus.Subscribe<SystemMessage.StateChangeMessage>(httpSendService);
_mainBus.Subscribe(new WideningHandler<HttpMessage.SendOverHttp, Message>(_workersHandler));
SubscribeWorkers(bus =>
{
bus.Subscribe<HttpMessage.HttpSend>(httpSendService);
bus.Subscribe<HttpMessage.HttpSendPart>(httpSendService);
bus.Subscribe<HttpMessage.HttpBeginSend>(httpSendService);
bus.Subscribe<HttpMessage.HttpEndSend>(httpSendService);
bus.Subscribe<HttpMessage.SendOverHttp>(httpSendService);
});
_httpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(),
_workersHandler, vNodeSettings.HttpPrefixes);
_httpService.SetupController(new AdminController(_mainQueue));
_httpService.SetupController(new PingController());
_httpService.SetupController(new StatController(monitoringQueue, _workersHandler));
_httpService.SetupController(new AtomController(httpSendService, _mainQueue, _workersHandler));
_httpService.SetupController(new GuidController(_mainQueue));
_httpService.SetupController(new UsersController(httpSendService, _mainQueue, _workersHandler));
_mainBus.Subscribe<SystemMessage.SystemInit>(_httpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(_httpService);
_mainBus.Subscribe<HttpMessage.PurgeTimedOutRequests>(_httpService);
SubscribeWorkers(bus =>
{
HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray());
});
// REQUEST MANAGEMENT
var requestManagement = new RequestManagementService(_mainQueue, 1, 1, vNodeSettings.PrepareTimeout, vNodeSettings.CommitTimeout);
_mainBus.Subscribe<SystemMessage.SystemInit>(requestManagement);
_mainBus.Subscribe<ClientMessage.WriteEvents>(requestManagement);
_mainBus.Subscribe<ClientMessage.TransactionStart>(requestManagement);
_mainBus.Subscribe<ClientMessage.TransactionWrite>(requestManagement);
_mainBus.Subscribe<ClientMessage.TransactionCommit>(requestManagement);
_mainBus.Subscribe<ClientMessage.DeleteStream>(requestManagement);
_mainBus.Subscribe<StorageMessage.RequestCompleted>(requestManagement);
_mainBus.Subscribe<StorageMessage.CheckStreamAccessCompleted>(requestManagement);
_mainBus.Subscribe<StorageMessage.AlreadyCommitted>(requestManagement);
_mainBus.Subscribe<StorageMessage.CommitAck>(requestManagement);
_mainBus.Subscribe<StorageMessage.PrepareAck>(requestManagement);
_mainBus.Subscribe<StorageMessage.WrongExpectedVersion>(requestManagement);
_mainBus.Subscribe<StorageMessage.InvalidTransaction>(requestManagement);
_mainBus.Subscribe<StorageMessage.StreamDeleted>(requestManagement);
_mainBus.Subscribe<StorageMessage.RequestManagerTimerTick>(requestManagement);
// SUBSCRIPTIONS
var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false);
_mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.SystemStart, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<TcpMessage.ConnectionClosed, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.SubscribeToStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.UnsubscribeFromStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.PollStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.CheckPollTimeout, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<StorageMessage.EventCommited, Message>());
var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex);
subscrBus.Subscribe<SystemMessage.SystemStart>(subscription);
subscrBus.Subscribe<SystemMessage.BecomeShuttingDown>(subscription);
subscrBus.Subscribe<TcpMessage.ConnectionClosed>(subscription);
subscrBus.Subscribe<ClientMessage.SubscribeToStream>(subscription);
subscrBus.Subscribe<ClientMessage.UnsubscribeFromStream>(subscription);
subscrBus.Subscribe<SubscriptionMessage.PollStream>(subscription);
subscrBus.Subscribe<SubscriptionMessage.CheckPollTimeout>(subscription);
subscrBus.Subscribe<StorageMessage.EventCommited>(subscription);
// USER MANAGEMENT
var ioDispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_mainQueue));
_mainBus.Subscribe(ioDispatcher.BackwardReader);
_mainBus.Subscribe(ioDispatcher.ForwardReader);
_mainBus.Subscribe(ioDispatcher.Writer);
_mainBus.Subscribe(ioDispatcher.StreamDeleter);
_mainBus.Subscribe(ioDispatcher);
var userManagement = new UserManagementService(
_mainQueue, ioDispatcher, passwordHashAlgorithm, vNodeSettings.SkipInitializeStandardUsersCheck);
_mainBus.Subscribe<UserManagementMessage.Create>(userManagement);
_mainBus.Subscribe<UserManagementMessage.Update>(userManagement);
_mainBus.Subscribe<UserManagementMessage.Enable>(userManagement);
_mainBus.Subscribe<UserManagementMessage.Disable>(userManagement);
_mainBus.Subscribe<UserManagementMessage.Delete>(userManagement);
_mainBus.Subscribe<UserManagementMessage.ResetPassword>(userManagement);
_mainBus.Subscribe<UserManagementMessage.ChangePassword>(userManagement);
_mainBus.Subscribe<UserManagementMessage.Get>(userManagement);
_mainBus.Subscribe<UserManagementMessage.GetAll>(userManagement);
_mainBus.Subscribe<SystemMessage.BecomeMaster>(userManagement);
// TIMER
_timeProvider = new RealTimeProvider();
_timerService = new TimerService(new ThreadBasedScheduler(_timeProvider));
_mainBus.Subscribe<SystemMessage.BecomeShutdown>(_timerService);
_mainBus.Subscribe<TimerMessage.Schedule>(_timerService);
_workersHandler.Start();
_mainQueue.Start();
monitoringQueue.Start();
subscrQueue.Start();
if (subsystems != null)
foreach (var subsystem in subsystems)
subsystem.Register(db, _mainQueue, _mainBus, _timerService, _timeProvider, httpSendService, new[]{_httpService}, _workersHandler);
}
public void allows_last_chunk_to_be_multichunk_when_checkpoint_point_at_the_start_of_next_chunk()
{
var config = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1000,
0,
new InMemoryCheckpoint(4000),
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
using (var db = new TFChunkDb(config))
{
DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000"));
DbUtil.CreateMultiChunk(config, 1, 3, GetFilePathFor("chunk-000001.000001"));
Assert.DoesNotThrow(() => db.Open(verifyHash: false));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000")));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000001.000001")));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000004.000000")));
Assert.AreEqual(3, Directory.GetFiles(PathName, "*").Length);
}
}
public void does_not_allow_checkpoint_to_point_into_the_middle_of_multichunk_chunk()
{
var config = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1000,
0,
new InMemoryCheckpoint(1500),
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
using (var db = new TFChunkDb(config))
{
DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000"));
DbUtil.CreateMultiChunk(config, 1, 10, GetFilePathFor("chunk-000001.000001"));
Assert.That(() => db.Open(verifyHash: false),
Throws.Exception.InstanceOf<CorruptDatabaseException>()
.With.InnerException.InstanceOf<BadChunkInDatabaseException>());
}
}
public void when_checkpoint_is_on_boundary_of_new_chunk_and_last_chunk_is_truncated_no_exception_is_thrown()
{
var config = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
100,
0,
new InMemoryCheckpoint(300),
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
using (var db = new TFChunkDb(config))
{
DbUtil.CreateSingleChunk(config, 0, GetFilePathFor("chunk-000000.000000"));
DbUtil.CreateMultiChunk(config, 1, 2, GetFilePathFor("chunk-000001.000001"), physicalSize: 50, logicalSize: 150);
Assert.DoesNotThrow(() => db.Open(verifyHash: false));
Assert.IsNotNull(db.Manager.GetChunk(2));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000000.000000")));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000001.000001")));
Assert.IsTrue(File.Exists(GetFilePathFor("chunk-000003.000000")));
Assert.AreEqual(3, Directory.GetFiles(PathName, "*").Length);
}
}