public void SetUp()
{
_indexDir = Path.Combine(Path.GetTempPath(), "idx-" + Guid.NewGuid().ToString());
_tableIndex = new TableIndex(_indexDir,
() => new HashListMemTable(),
maxSizeForMemory: 2,
maxTablesPerLevel: 2);
_tableIndex.Initialize();
// ptable level 2
_tableIndex.Add(0, 0xDEAD, 0, 0xFF00);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF01);
_tableIndex.Add(0, 0xBEEF, 0, 0xFF00);
_tableIndex.Add(0, 0xBEEF, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 0, 0xFF00);
_tableIndex.Add(0, 0xABBA, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 0, 0xFF02);
_tableIndex.Add(0, 0xABBA, 1, 0xFF03);
// ptable level 1
_tableIndex.Add(0, 0xADA, 0, 0xFF00);
_tableIndex.Add(0, 0xCEED, 10, 0xFFF1);
_tableIndex.Add(0, 0xBABA, 0, 0xFF00);
_tableIndex.Add(0, 0xDEAD, 0, 0xFF10);
// ptable level 0
_tableIndex.Add(0, 0xBABA, 1, 0xFF01);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF11);
// memtable
_tableIndex.Add(0, 0xADA, 0, 0xFF01);
Thread.Sleep(500);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeTfReader());
_tableIndex = new TableIndex(_indexDir,
() => new HashListMemTable(maxSize: 10),
() => fakeReader,
maxSizeForMemory: 5);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(0, 0xDEAD, 0, 0xFF00);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF01);
_tableIndex.Add(0, 0xBEEF, 0, 0xFF00);
_tableIndex.Add(0, 0xBEEF, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 0, 0xFF00); // 1st ptable0
_tableIndex.Add(0, 0xABBA, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 2, 0xFF02);
_tableIndex.Add(0, 0xABBA, 3, 0xFF03);
_tableIndex.Add(0, 0xADA, 0, 0xFF00); // simulates duplicate due to concurrency in TableIndex (see memtable below)
_tableIndex.Add(0, 0xDEAD, 0, 0xFF10); // 2nd ptable0
_tableIndex.Add(0, 0xDEAD, 1, 0xFF11); // in memtable
_tableIndex.Add(0, 0xADA, 0, 0xFF00); // in memtable
}
private void ReOpenDb()
{
Db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
WriterCheckpoint,
ChaserCheckpoint,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
Db.Open();
TableIndex = new TableIndex(Path.Combine(PathName, "index"),
() => new HashListMemTable(MaxEntriesInMemTable * 2),
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
2,
5,
() => new TFChunkReader(Db, Db.Config.WriterCheckpoint),
TableIndex,
new ByLengthHasher(),
new NoLRUCache<string, StreamCacheInfo>(),
additionalCommitChecks: true,
metastreamMaxCount: MetastreamMaxCount);
ReadIndex.Init(WriterCheckpoint.Read(), ChaserCheckpoint.Read());
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
ReadIndex.Close();
ReadIndex.Dispose();
TableIndex.Close(removeFiles: false);
var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 2, () => new TFChunkReader(Db, WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.Index64Bit, maxSize: MaxEntriesInMemTable*2),
() => new TFReaderLease(readers),
PTableVersions.Index64Bit,
maxSizeForMemory: MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: true,
metastreamMaxCount: 1,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault);
ReadIndex.Init(ChaserCheckpoint.Read());
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(PathName, _lowHasher, _highHasher,
() => new HashListMemTable(version: _ptableVersion, maxSize: 40),
() => { throw new InvalidOperationException(); },
_ptableVersion,
maxSizeForMemory: 20);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(0, "0xDEAD", 0, 0xFF00);
_tableIndex.Add(0, "0xDEAD", 1, 0xFF01);
_tableIndex.Add(0, "0xJEEP", 0, 0xFF00);
_tableIndex.Add(0, "0xJEEP", 1, 0xFF01);
_tableIndex.Add(0, "0xABBA", 0, 0xFF00);
_tableIndex.Add(0, "0xABBA", 1, 0xFF01);
_tableIndex.Add(0, "0xABBA", 2, 0xFF02);
_tableIndex.Add(0, "0xABBA", 3, 0xFF03);
_tableIndex.Add(0, "0xDEAD", 0, 0xFF10);
_tableIndex.Add(0, "0xDEAD", 1, 0xFF11);
_tableIndex.Add(0, "0xADA", 0, 0xFF00);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_tableIndex = new TableIndex(PathName,
() => new HashListMemTable(maxSize: 40),
() => { throw new InvalidOperationException(); },
maxSizeForMemory: 20);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(0, 0xDEAD, 0, 0xFF00);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF01);
_tableIndex.Add(0, 0xBEEF, 0, 0xFF00);
_tableIndex.Add(0, 0xBEEF, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 0, 0xFF00);
_tableIndex.Add(0, 0xABBA, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 2, 0xFF02);
_tableIndex.Add(0, 0xABBA, 3, 0xFF03);
_tableIndex.Add(0, 0xDEAD, 0, 0xFF10);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF11);
_tableIndex.Add(0, 0xADA, 0, 0xFF00);
}
private void ReOpenDb()
{
Db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
WriterCheckpoint,
ChaserCheckpoint,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
Db.Open();
var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
TableIndex = new TableIndex(Path.Combine(PathName, "index"),
() => new HashListMemTable(MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
new ByLengthHasher(),
0,
additionalCommitChecks: true,
metastreamMaxCount: MetastreamMaxCount);
ReadIndex.Init(ChaserCheckpoint.Read());
}
private void ReOpenDb()
{
Db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
WriterCheckpoint,
ChaserCheckpoint,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
Db.Open();
var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(Path.Combine(PathName, "index"), lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.Index64Bit, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
PTableVersions.Index64Bit,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: true,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault);
ReadIndex.Init(ChaserCheckpoint.Read());
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_tableIndex = new TableIndex(PathName,
() => new HashListMemTable(maxSize: 20),
() => { throw new InvalidOperationException(); },
maxSizeForMemory: 10);
_tableIndex.Initialize(long.MaxValue);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
WriterCheckpoint = new InMemoryCheckpoint(0);
ChaserCheckpoint = new InMemoryCheckpoint(0);
Bus = new InMemoryBus("bus");
IODispatcher = new IODispatcher(Bus, new PublishEnvelope(Bus));
Db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
WriterCheckpoint,
ChaserCheckpoint,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
Db.Open();
// create db
Writer = new TFChunkWriter(Db);
Writer.Open();
WriteTestScenario();
Writer.Close();
Writer = null;
WriterCheckpoint.Flush();
ChaserCheckpoint.Write(WriterCheckpoint.Read());
ChaserCheckpoint.Flush();
var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
TableIndex = new TableIndex(GetFilePathFor("index"),
() => new HashListMemTable(MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
MaxEntriesInMemTable);
var hasher = new ByLengthHasher();
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
hasher,
0,
additionalCommitChecks: true,
metastreamMaxCount: MetastreamMaxCount);
ReadIndex.Init(ChaserCheckpoint.Read());
// scavenge must run after readIndex is built
if (_scavenge)
{
if (_completeLastChunkOnScavenge)
Db.Manager.GetChunk(Db.Manager.ChunksCount - 1).Complete();
_scavenger = new TFChunkScavenger(Db, IODispatcher, TableIndex, hasher, ReadIndex, Guid.NewGuid(), "fakeNodeIp");
_scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: _mergeChunks);
}
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(PathName, lowHasher, highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 20),
() => { throw new InvalidOperationException(); },
_ptableVersion,
maxSizeForMemory: 10);
_tableIndex.Initialize(long.MaxValue);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
ReadIndex.Close();
ReadIndex.Dispose();
Thread.Sleep(500);
TableIndex.ClearAll(removeFiles: false);
TableIndex = new TableIndex(Path.Combine(PathName, "index"), () => new HashListMemTable(), maxSizeForMemory: 5);
TableIndex.Initialize();
ReadIndex = new ReadIndex(new NoopPublisher(), 2, () => new TFChunkSequentialReader(Db, WriterCheckpoint, 0), () => new TFChunkReader(Db, WriterCheckpoint), TableIndex, new ByLengthHasher());
ReadIndex.Build();
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024*1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = CreateDb(dbCreationHelper);
_keptRecords = KeptRecords(_dbResult);
_dbResult.Db.Config.WriterCheckpoint.Flush();
_dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
_dbResult.Db.Config.ChaserCheckpoint.Flush();
var indexPath = Path.Combine(PathName, "index");
var readerPool = new ObjectPool<ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
var tableIndex = new TableIndex(indexPath, lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.Index64Bit, maxSize: 200),
() => new TFReaderLease(readerPool),
PTableVersions.Index64Bit,
maxSizeForMemory: 100,
maxTablesPerLevel: 2);
ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, 100, true, _metastreamMaxCount, Opts.HashCollisionReadLimitDefault);
ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read());
//var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount);
var bus = new InMemoryBus("Bus");
var ioDispatcher = new IODispatcher(bus, new PublishEnvelope(bus));
var scavenger = new TFChunkScavenger(_dbResult.Db, ioDispatcher, tableIndex, ReadIndex, Guid.NewGuid(), "fakeNodeIp",
unsafeIgnoreHardDeletes: UnsafeIgnoreHardDelete());
scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false);
}
public void Setup(){
given();
_indexDir = PathName;
_fakeReader = new TFReaderLease(new FakeReader());
_indexBackend = new FakeIndexBackend(_fakeReader);
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.Index32Bit, maxSize: _maxMemTableSize),
() => _fakeReader,
PTableVersions.Index32Bit,
maxSizeForMemory: _maxMemTableSize,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_indexReader = new IndexReader(_indexBackend, _tableIndex, new EventStore.Core.Data.StreamMetadata(), _hashCollisionReadLimit);
when();
//wait for the mem table to be dumped
System.Threading.Thread.Sleep(500);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new FakeIndexHasher();
_highHasher = new FakeIndexHasher();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(_ptableVersion, maxSize: 10),
() => fakeReader,
_ptableVersion,
maxSizeForMemory: 2,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
// ptable level 2
_tableIndex.Add(0, "1", 0, 0xFF00);
_tableIndex.Add(0, "1", 1, 0xFF01);
_tableIndex.Add(0, "2", 0, 0xFF00);
_tableIndex.Add(0, "2", 1, 0xFF01);
_tableIndex.Add(0, "3", 0, 0xFF00);
_tableIndex.Add(0, "3", 1, 0xFF01);
_tableIndex.Add(0, "3", 0, 0xFF02);
_tableIndex.Add(0, "3", 1, 0xFF03);
// ptable level 1
_tableIndex.Add(0, "4", 0, 0xFF00);
_tableIndex.Add(0, "5", 10, 0xFFF1);
_tableIndex.Add(0, "6", 0, 0xFF00);
_tableIndex.Add(0, "1", 0, 0xFF10);
// ptable level 0
_tableIndex.Add(0, "6", 1, 0xFF01);
_tableIndex.Add(0, "1", 1, 0xFF11);
// memtable
_tableIndex.Add(0, "4", 0, 0xFF01);
Thread.Sleep(500);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024*1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
DbRes = CreateDb(dbCreationHelper);
DbRes.Db.Config.WriterCheckpoint.Flush();
DbRes.Db.Config.ChaserCheckpoint.Write(DbRes.Db.Config.WriterCheckpoint.Read());
DbRes.Db.Config.ChaserCheckpoint.Flush();
var readers = new ObjectPool<ITransactionFileReader>(
"Readers", 2, 2, () => new TFChunkReader(DbRes.Db, DbRes.Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.Index64Bit, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
PTableVersions.Index64Bit,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: true,
metastreamMaxCount: _metastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault);
ReadIndex.Init(DbRes.Db.Config.ChaserCheckpoint.Read());
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
ReadIndex.Close();
ReadIndex.Dispose();
TableIndex.Close(removeFiles: false);
TableIndex = new TableIndex(GetFilePathFor("index"),
() => new HashListMemTable(maxSize: MaxEntriesInMemTable*2),
maxSizeForMemory: MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
2,
() => new TFChunkSequentialReader(Db, WriterChecksum, 0),
() => new TFChunkReader(Db, WriterChecksum),
TableIndex,
new ByLengthHasher(),
new NoLRUCache<string, StreamCacheInfo>());
ReadIndex.Build();
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeIndexReader());
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.Index32Bit, maxSize: 5),
() => fakeReader,
PTableVersions.Index32Bit,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 0, 1);
_tableIndex.Add(1, "account--696193173", 0, 2);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 1, 3);
_tableIndex.Add(1, "account--696193173", 1, 4);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 2, 5);
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.Index64Bit, maxSize: 5),
() => fakeReader,
PTableVersions.Index64Bit,
maxSizeForMemory: 5,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "account--696193173", 2, 6);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 3, 7);
_tableIndex.Add(1, "account--696193173", 3, 8);
_tableIndex.Add(1, "LPN-FC002_LPK51001", 4, 9);
_tableIndex.Add(1, "account--696193173", 4, 10);
Thread.Sleep(500);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
ReadIndex.Close();
ReadIndex.Dispose();
TableIndex.Close(removeFiles: false);
var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 2, () => new TFChunkReader(Db, WriterCheckpoint));
TableIndex = new TableIndex(GetFilePathFor("index"),
() => new HashListMemTable(maxSize: MaxEntriesInMemTable*2),
() => new TFReaderLease(readers),
maxSizeForMemory: MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
new ByLengthHasher(),
0,
additionalCommitChecks: true,
metastreamMaxCount: 1);
ReadIndex.Init(ChaserCheckpoint.Read());
}
public void SetUp()
{
_indexDir = Path.Combine(Path.GetTempPath(), "idx-" + Guid.NewGuid().ToString());
_tableIndex = new TableIndex(_indexDir, () => new HashListMemTable(), new InMemoryCheckpoint(), maxSizeForMemory: 2000);
_tableIndex.Initialize();
_tableIndex.Add(0xDEAD, 0, 0xFF00);
_tableIndex.Add(0xDEAD, 1, 0xFF01);
_tableIndex.Add(0xBEEF, 0, 0xFF00);
_tableIndex.Add(0xBEEF, 1, 0xFF01);
_tableIndex.Add(0xABBA, 0, 0xFF00);
_tableIndex.Add(0xABBA, 1, 0xFF01);
_tableIndex.Add(0xABBA, 2, 0xFF02);
_tableIndex.Add(0xABBA, 3, 0xFF03);
_tableIndex.Add(0xDEAD, 0, 0xFF10);
_tableIndex.Add(0xDEAD, 1, 0xFF11);
_tableIndex.Add(0xADA, 0, 0xFF00);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var fakeReader = new TFReaderLease(new FakeTfReader());
_tableIndex = new TableIndex(_indexDir,
() => new HashListMemTable(maxSize: 10),
() => fakeReader,
maxSizeForMemory: 2,
maxTablesPerLevel: 2);
_tableIndex.Initialize(long.MaxValue);
// ptable level 2
_tableIndex.Add(0, 0xDEAD, 0, 0xFF00);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF01);
_tableIndex.Add(0, 0xBEEF, 0, 0xFF00);
_tableIndex.Add(0, 0xBEEF, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 0, 0xFF00);
_tableIndex.Add(0, 0xABBA, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 0, 0xFF02);
_tableIndex.Add(0, 0xABBA, 1, 0xFF03);
// ptable level 1
_tableIndex.Add(0, 0xADA, 0, 0xFF00);
_tableIndex.Add(0, 0xCEED, 10, 0xFFF1);
_tableIndex.Add(0, 0xBABA, 0, 0xFF00);
_tableIndex.Add(0, 0xDEAD, 0, 0xFF10);
// ptable level 0
_tableIndex.Add(0, 0xBABA, 1, 0xFF01);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF11);
// memtable
_tableIndex.Add(0, 0xADA, 0, 0xFF01);
Thread.Sleep(500);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
ReadIndex.Close();
ReadIndex.Dispose();
TableIndex.Close(removeFiles: false);
TableIndex = new TableIndex(GetFilePathFor("index"),
() => new HashListMemTable(maxSize: MaxEntriesInMemTable*2),
maxSizeForMemory: MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
2,
2,
() => new TFChunkReader(Db, WriterCheckpoint, 0),
TableIndex,
new ByLengthHasher(),
new NoLRUCache<string, StreamCacheInfo>(),
additionalCommitChecks: true,
metastreamMaxCount: 1);
ReadIndex.Init(WriterCheckpoint.Read(), ChaserCheckpoint.Read());
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_tableIndex = new TableIndex(PathName, () => new HashListMemTable(maxSize: 40), maxSizeForMemory: 20);
_tableIndex.Initialize();
_tableIndex.Add(0, 0xDEAD, 0, 0xFF00);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF01);
_tableIndex.Add(0, 0xBEEF, 0, 0xFF00);
_tableIndex.Add(0, 0xBEEF, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 0, 0xFF00);
_tableIndex.Add(0, 0xABBA, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 2, 0xFF02);
_tableIndex.Add(0, 0xABBA, 3, 0xFF03);
_tableIndex.Add(0, 0xDEAD, 0, 0xFF10);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF11);
_tableIndex.Add(0, 0xADA, 0, 0xFF00);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024*1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
_dbResult = CreateDb(dbCreationHelper);
_keptRecords = KeptRecords(_dbResult);
_dbResult.Db.Config.WriterCheckpoint.Flush();
_dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
_dbResult.Db.Config.ChaserCheckpoint.Flush();
var indexPath = Path.Combine(PathName, "index");
var readerPool = new ObjectPool<ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint));
var tableIndex = new TableIndex(indexPath,
() => new HashListMemTable(maxSize: 200),
() => new TFReaderLease(readerPool),
maxSizeForMemory: 100,
maxTablesPerLevel: 2);
var hasher = new XXHashUnsafe();
ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, hasher, 100, true, _metastreamMaxCount);
ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read());
//var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount);
var scavenger = new TFChunkScavenger(_dbResult.Db, tableIndex, hasher, ReadIndex);
scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false);
}
public override void SetUp()
{
base.SetUp();
_indexDir = base.PathName;
_tableIndex = new TableIndex(_indexDir, () => new HashListMemTable(), maxSizeForMemory: 2000);
_tableIndex.Initialize();
_tableIndex.Add(0, 0xDEAD, 0, 0xFF00);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF01);
_tableIndex.Add(0, 0xBEEF, 0, 0xFF00);
_tableIndex.Add(0, 0xBEEF, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 0, 0xFF00);
_tableIndex.Add(0, 0xABBA, 1, 0xFF01);
_tableIndex.Add(0, 0xABBA, 2, 0xFF02);
_tableIndex.Add(0, 0xABBA, 3, 0xFF03);
_tableIndex.Add(0, 0xDEAD, 0, 0xFF10);
_tableIndex.Add(0, 0xDEAD, 1, 0xFF11);
_tableIndex.Add(0, 0xADA, 0, 0xFF00);
}
public void SetUp()
{
_indexDir = Path.Combine(Path.GetTempPath(), "idx-" + Guid.NewGuid().ToString());
_tableIndex = new TableIndex(_indexDir, () => new HashListMemTable(), new InMemoryCheckpoint(), maxSizeForMemory: 5);
_tableIndex.Initialize();
_tableIndex.Add(0xDEAD, 0, 0xFF00);
_tableIndex.Add(0xDEAD, 1, 0xFF01);
_tableIndex.Add(0xBEEF, 0, 0xFF00);
_tableIndex.Add(0xBEEF, 1, 0xFF01);
_tableIndex.Add(0xABBA, 0, 0xFF00); // 1st ptable0
_tableIndex.Add(0xABBA, 1, 0xFF01);
_tableIndex.Add(0xABBA, 2, 0xFF02);
_tableIndex.Add(0xABBA, 3, 0xFF03);
_tableIndex.Add(0xADA, 0, 0xFF00); // simulates duplicate due to concurrency in TableIndex (see memtable below)
_tableIndex.Add(0xDEAD, 0, 0xFF10); // 2nd ptable0
_tableIndex.Add(0xDEAD, 1, 0xFF11); // in memtable
_tableIndex.Add(0xADA, 0, 0xFF00); // in memtable
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
var dbConfig = new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
1024*1024,
0,
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(0),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1));
var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
DbRes = CreateDb(dbCreationHelper);
DbRes.Db.Config.WriterCheckpoint.Flush();
DbRes.Db.Config.ChaserCheckpoint.Write(DbRes.Db.Config.WriterCheckpoint.Read());
DbRes.Db.Config.ChaserCheckpoint.Flush();
TableIndex = new TableIndex(GetFilePathFor("index"),
() => new HashListMemTable(MaxEntriesInMemTable * 2),
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
2,
2,
() => new TFChunkReader(DbRes.Db, DbRes.Db.Config.WriterCheckpoint),
TableIndex,
new ByLengthHasher(),
new NoLRUCache<string, StreamCacheInfo>(),
additionalCommitChecks: true,
metastreamMaxCount: _metastreamMaxCount);
ReadIndex.Init(DbRes.Db.Config.WriterCheckpoint.Read(), DbRes.Db.Config.ChaserCheckpoint.Read());
}
public SingleVNode(TFChunkDb db, SingleVNodeSettings vNodeSettings, SingleVNodeAppSettings appSettings)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(vNodeSettings, "vNodeSettings");
db.OpenVerifyAndClean();
_tcpEndPoint = vNodeSettings.ExternalTcpEndPoint;
_httpEndPoint = vNodeSettings.HttpEndPoint;
_outputBus = new InMemoryBus("OutputBus");
_controller = new SingleVNodeController(Bus, _httpEndPoint);
_mainQueue = new QueuedHandler(_controller, "MainQueue");
_controller.SetMainQueue(MainQueue);
//MONITORING
var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
var monitoringQueue = new QueuedHandler(monitoringInnerBus, "MonitoringQueue", watchSlowMsg: true, slowMsgThresholdMs: 100);
var monitoring = new MonitoringService(monitoringQueue, monitoringRequestBus, db.Config.WriterCheckpoint, appSettings.StatsPeriod);
Bus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.SystemInit, Message>());
Bus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
monitoringInnerBus.Subscribe<SystemMessage.SystemInit>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.BecomeShuttingDown>(monitoring);
monitoringInnerBus.Subscribe<MonitoringMessage.GetFreshStats>(monitoring);
//STORAGE SUBSYSTEM
var indexPath = Path.Combine(db.Config.Path, "index");
var tableIndex = new TableIndex(indexPath,
() => new HashListMemTable(),
new InMemoryCheckpoint(),
maxSizeForMemory: 1000000,
maxTablesPerLevel: 2);
var readIndex = new ReadIndex(_mainQueue,
db,
() => new TFChunkReader(db, db.Config.WriterCheckpoint),
TFConsts.ReadIndexReaderCount,
db.Config.WriterCheckpoint,
tableIndex,
new XXHashUnsafe());
var writer = new TFChunkWriter(db);
var storageWriter = new StorageWriter(_mainQueue, _outputBus, writer, readIndex);
var storageReader = new StorageReader(_mainQueue, _outputBus, readIndex, TFConsts.StorageReaderHandlerCount);
monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageReader);
var chaser = new TFChunkChaser(db,
db.Config.WriterCheckpoint,
db.Config.GetNamedCheckpoint(Checkpoint.Chaser));
var storageChaser = new StorageChaser(_mainQueue, chaser);
_outputBus.Subscribe<SystemMessage.SystemInit>(storageChaser);
_outputBus.Subscribe<SystemMessage.SystemStart>(storageChaser);
_outputBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageChaser);
var storageScavenger = new StorageScavenger(db, readIndex);
_outputBus.Subscribe<SystemMessage.ScavengeDatabase>(storageScavenger);
//TCP
var tcpService = new TcpService(MainQueue, _tcpEndPoint);
Bus.Subscribe<SystemMessage.SystemInit>(tcpService);
Bus.Subscribe<SystemMessage.SystemStart>(tcpService);
Bus.Subscribe<SystemMessage.BecomeShuttingDown>(tcpService);
//HTTP
HttpService = new HttpService(MainQueue, _httpEndPoint.ToHttpUrl());
Bus.Subscribe<SystemMessage.SystemInit>(HttpService);
Bus.Subscribe<SystemMessage.BecomeShuttingDown>(HttpService);
Bus.Subscribe<HttpMessage.SendOverHttp>(HttpService);
Bus.Subscribe<HttpMessage.UpdatePendingRequests>(HttpService);
HttpService.SetupController(new AdminController(MainQueue));
HttpService.SetupController(new PingController());
HttpService.SetupController(new StatController(monitoringQueue));
HttpService.SetupController(new ReadEventDataController(MainQueue));
HttpService.SetupController(new AtomController(MainQueue));
HttpService.SetupController(new WebSiteController(MainQueue));
//REQUEST MANAGEMENT
var requestManagement = new RequestManagementService(MainQueue, 1, 1);
Bus.Subscribe<ReplicationMessage.EventCommited>(requestManagement);
Bus.Subscribe<ReplicationMessage.CreateStreamRequestCreated>(requestManagement);
Bus.Subscribe<ReplicationMessage.WriteRequestCreated>(requestManagement);
Bus.Subscribe<ReplicationMessage.TransactionStartRequestCreated>(requestManagement);
Bus.Subscribe<ReplicationMessage.TransactionWriteRequestCreated>(requestManagement);
Bus.Subscribe<ReplicationMessage.TransactionCommitRequestCreated>(requestManagement);
Bus.Subscribe<ReplicationMessage.DeleteStreamRequestCreated>(requestManagement);
Bus.Subscribe<ReplicationMessage.RequestCompleted>(requestManagement);
Bus.Subscribe<ReplicationMessage.CommitAck>(requestManagement);
Bus.Subscribe<ReplicationMessage.PrepareAck>(requestManagement);
Bus.Subscribe<ReplicationMessage.WrongExpectedVersion>(requestManagement);
Bus.Subscribe<ReplicationMessage.InvalidTransaction>(requestManagement);
Bus.Subscribe<ReplicationMessage.StreamDeleted>(requestManagement);
Bus.Subscribe<ReplicationMessage.PreparePhaseTimeout>(requestManagement);
Bus.Subscribe<ReplicationMessage.CommitPhaseTimeout>(requestManagement);
var clientService = new ClientService();
Bus.Subscribe<TcpMessage.ConnectionClosed>(clientService);
Bus.Subscribe<ClientMessage.SubscribeToStream>(clientService);
Bus.Subscribe<ClientMessage.UnsubscribeFromStream>(clientService);
Bus.Subscribe<ClientMessage.SubscribeToAllStreams>(clientService);
Bus.Subscribe<ClientMessage.UnsubscribeFromAllStreams>(clientService);
Bus.Subscribe<ReplicationMessage.EventCommited>(clientService);
//TIMER
//var timer = new TimerService(new TimerBasedScheduler(new RealTimer(), new RealTimeProvider()));
TimerService = new TimerService(new ThreadBasedScheduler(new RealTimeProvider()));
Bus.Subscribe<TimerMessage.Schedule>(TimerService);
MainQueue.Start();
monitoringQueue.Start();
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
WriterCheckpoint = new InMemoryCheckpoint(0);
ChaserCheckpoint = new InMemoryCheckpoint(0);
Db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
WriterCheckpoint,
ChaserCheckpoint,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
Db.Open();
// create db
Writer = new TFChunkWriter(Db);
Writer.Open();
WriteTestScenario();
Writer.Close();
Writer = null;
WriterCheckpoint.Flush();
ChaserCheckpoint.Write(WriterCheckpoint.Read());
ChaserCheckpoint.Flush();
TableIndex = new TableIndex(GetFilePathFor("index"),
() => new HashListMemTable(MaxEntriesInMemTable * 2),
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
2,
2,
() => new TFChunkReader(Db, Db.Config.WriterCheckpoint),
TableIndex,
new ByLengthHasher(),
new NoLRUCache<string, StreamCacheInfo>(),
additionalCommitChecks: true,
metastreamMaxCount: MetastreamMaxCount);
ReadIndex.Init(WriterCheckpoint.Read(), ChaserCheckpoint.Read());
// scavenge must run after readIndex is built
if (_scavenge)
{
if (_completeLastChunkOnScavenge)
Db.Manager.GetChunk(Db.Manager.ChunksCount - 1).Complete();
_scavenger = new TFChunkScavenger(Db, ReadIndex);
_scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: _mergeChunks);
}
}
public SingleVNode(TFChunkDb db,
SingleVNodeSettings vNodeSettings,
bool dbVerifyHashes,
int memTableEntryCount,
params ISubsystem[] subsystems)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(vNodeSettings, "vNodeSettings");
_settings = vNodeSettings;
_mainBus = new InMemoryBus("MainBus");
_controller = new SingleVNodeController(_mainBus, _settings.ExternalHttpEndPoint, db, this);
_mainQueue = new QueuedHandler(_controller, "MainQueue");
_controller.SetMainQueue(_mainQueue);
_subsystems = subsystems;
// MONITORING
var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
var monitoringQueue = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
var monitoring = new MonitoringService(monitoringQueue,
monitoringRequestBus,
_mainQueue,
db.Config.WriterCheckpoint,
db.Config.Path,
vNodeSettings.StatsPeriod,
_settings.ExternalHttpEndPoint,
vNodeSettings.StatsStorage);
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.SystemInit, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.StateChangeMessage, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShutdown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<ClientMessage.WriteEventsCompleted, Message>());
monitoringInnerBus.Subscribe<SystemMessage.SystemInit>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.StateChangeMessage>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.BecomeShuttingDown>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.BecomeShutdown>(monitoring);
monitoringInnerBus.Subscribe<ClientMessage.WriteEventsCompleted>(monitoring);
monitoringInnerBus.Subscribe<MonitoringMessage.GetFreshStats>(monitoring);
var truncPos = db.Config.TruncateCheckpoint.Read();
if (truncPos != -1)
{
Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})",
truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read());
var truncator = new TFChunkDbTruncator(db.Config);
truncator.TruncateDb(truncPos);
}
db.Open(dbVerifyHashes);
// STORAGE SUBSYSTEM
var indexPath = Path.Combine(db.Config.Path, "index");
var readerPool = new ObjectPool<ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(db, db.Config.WriterCheckpoint));
var tableIndex = new TableIndex(indexPath,
() => new HashListMemTable(maxSize: memTableEntryCount * 2),
() => new TFReaderLease(readerPool),
maxSizeForMemory: memTableEntryCount,
maxTablesPerLevel: 2,
inMem: db.Config.InMemDb);
var hasher = new XXHashUnsafe();
var readIndex = new ReadIndex(_mainQueue,
readerPool,
tableIndex,
hasher,
ESConsts.StreamInfoCacheCapacity,
Application.IsDefined(Application.AdditionalCommitChecks),
Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1);
var writer = new TFChunkWriter(db);
var epochManager = new EpochManager(ESConsts.CachedEpochCount,
db.Config.EpochCheckpoint,
writer,
initialReaderCount: 1,
maxReaderCount: 5,
readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint));
epochManager.Init();
var storageWriter = new StorageWriterService(_mainQueue, _mainBus, _settings.MinFlushDelay,
db, writer, readIndex.IndexWriter, epochManager); // subscribes internally
monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageWriter);
var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, ESConsts.StorageReaderThreadCount, db.Config.WriterCheckpoint);
_mainBus.Subscribe<SystemMessage.SystemInit>(storageReader);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageReader);
_mainBus.Subscribe<SystemMessage.BecomeShutdown>(storageReader);
monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageReader);
var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, readIndex.IndexCommitter, epochManager);
_mainBus.Subscribe<SystemMessage.SystemInit>(storageChaser);
_mainBus.Subscribe<SystemMessage.SystemStart>(storageChaser);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageChaser);
var storageScavenger = new StorageScavenger(db,
tableIndex,
hasher,
readIndex,
Application.IsDefined(Application.AlwaysKeepScavenged),
mergeChunks: false /*!Application.IsDefined(Application.DisableMergeChunks)*/);
// ReSharper disable RedundantTypeArgumentsOfMethod
_mainBus.Subscribe<ClientMessage.ScavengeDatabase>(storageScavenger);
// ReSharper restore RedundantTypeArgumentsOfMethod
// MISC WORKERS
_workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray();
_workersHandler = new MultiQueuedHandler(
vNodeSettings.WorkerThreads,
queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
string.Format("Worker #{0}", queueNum + 1),
groupName: "Workers",
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200)));
// AUTHENTICATION INFRASTRUCTURE
var passwordHashAlgorithm = new Rfc2898PasswordHashAlgorithm();
var dispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_workersHandler, crossThread: true));
var internalAuthenticationProvider = new InternalAuthenticationProvider(dispatcher, passwordHashAlgorithm, ESConsts.CachedPrincipalCount);
var passwordChangeNotificationReader = new PasswordChangeNotificationReader(_mainQueue, dispatcher);
_mainBus.Subscribe<SystemMessage.SystemStart>(passwordChangeNotificationReader);
_mainBus.Subscribe<SystemMessage.BecomeShutdown>(passwordChangeNotificationReader);
_mainBus.Subscribe(internalAuthenticationProvider);
SubscribeWorkers(bus =>
{
bus.Subscribe(dispatcher.ForwardReader);
bus.Subscribe(dispatcher.BackwardReader);
bus.Subscribe(dispatcher.Writer);
bus.Subscribe(dispatcher.StreamDeleter);
bus.Subscribe(dispatcher);
});
// TCP
{
var tcpService = new TcpService(_mainQueue, _settings.ExternalTcpEndPoint, _workersHandler,
TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
internalAuthenticationProvider, null);
_mainBus.Subscribe<SystemMessage.SystemInit>(tcpService);
_mainBus.Subscribe<SystemMessage.SystemStart>(tcpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(tcpService);
}
// SECURE TCP
if (vNodeSettings.ExternalSecureTcpEndPoint != null)
{
var secureTcpService = new TcpService(_mainQueue, _settings.ExternalSecureTcpEndPoint, _workersHandler,
TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(),
ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
internalAuthenticationProvider, _settings.Certificate);
_mainBus.Subscribe<SystemMessage.SystemInit>(secureTcpService);
_mainBus.Subscribe<SystemMessage.SystemStart>(secureTcpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(secureTcpService);
}
SubscribeWorkers(bus =>
{
var tcpSendService = new TcpSendService();
// ReSharper disable RedundantTypeArgumentsOfMethod
bus.Subscribe<TcpMessage.TcpSend>(tcpSendService);
// ReSharper restore RedundantTypeArgumentsOfMethod
});
// HTTP
var httpAuthenticationProviders = new List<HttpAuthenticationProvider>
{
new BasicHttpAuthenticationProvider(internalAuthenticationProvider),
};
if (_settings.EnableTrustedAuth)
httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider());
httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider());
var httpPipe = new HttpMessagePipe();
var httpSendService = new HttpSendService(httpPipe, forwardRequests: false);
_mainBus.Subscribe<SystemMessage.StateChangeMessage>(httpSendService);
_mainBus.Subscribe(new WideningHandler<HttpMessage.SendOverHttp, Message>(_workersHandler));
SubscribeWorkers(bus =>
{
bus.Subscribe<HttpMessage.HttpSend>(httpSendService);
bus.Subscribe<HttpMessage.HttpSendPart>(httpSendService);
bus.Subscribe<HttpMessage.HttpBeginSend>(httpSendService);
bus.Subscribe<HttpMessage.HttpEndSend>(httpSendService);
bus.Subscribe<HttpMessage.SendOverHttp>(httpSendService);
});
_httpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(),
_workersHandler, vNodeSettings.HttpPrefixes);
_httpService.SetupController(new AdminController(_mainQueue));
_httpService.SetupController(new PingController());
_httpService.SetupController(new StatController(monitoringQueue, _workersHandler));
_httpService.SetupController(new AtomController(httpSendService, _mainQueue, _workersHandler));
_httpService.SetupController(new GuidController(_mainQueue));
_httpService.SetupController(new UsersController(httpSendService, _mainQueue, _workersHandler));
_mainBus.Subscribe<SystemMessage.SystemInit>(_httpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(_httpService);
_mainBus.Subscribe<HttpMessage.PurgeTimedOutRequests>(_httpService);
SubscribeWorkers(bus =>
{
HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray());
});
// REQUEST MANAGEMENT
var requestManagement = new RequestManagementService(_mainQueue, 1, 1, vNodeSettings.PrepareTimeout, vNodeSettings.CommitTimeout);
_mainBus.Subscribe<SystemMessage.SystemInit>(requestManagement);
_mainBus.Subscribe<ClientMessage.WriteEvents>(requestManagement);
_mainBus.Subscribe<ClientMessage.TransactionStart>(requestManagement);
_mainBus.Subscribe<ClientMessage.TransactionWrite>(requestManagement);
_mainBus.Subscribe<ClientMessage.TransactionCommit>(requestManagement);
_mainBus.Subscribe<ClientMessage.DeleteStream>(requestManagement);
_mainBus.Subscribe<StorageMessage.RequestCompleted>(requestManagement);
_mainBus.Subscribe<StorageMessage.CheckStreamAccessCompleted>(requestManagement);
_mainBus.Subscribe<StorageMessage.AlreadyCommitted>(requestManagement);
_mainBus.Subscribe<StorageMessage.CommitAck>(requestManagement);
_mainBus.Subscribe<StorageMessage.PrepareAck>(requestManagement);
_mainBus.Subscribe<StorageMessage.WrongExpectedVersion>(requestManagement);
_mainBus.Subscribe<StorageMessage.InvalidTransaction>(requestManagement);
_mainBus.Subscribe<StorageMessage.StreamDeleted>(requestManagement);
_mainBus.Subscribe<StorageMessage.RequestManagerTimerTick>(requestManagement);
// SUBSCRIPTIONS
var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false);
_mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.SystemStart, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<TcpMessage.ConnectionClosed, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.SubscribeToStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.UnsubscribeFromStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.PollStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.CheckPollTimeout, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<StorageMessage.EventCommited, Message>());
var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex);
subscrBus.Subscribe<SystemMessage.SystemStart>(subscription);
subscrBus.Subscribe<SystemMessage.BecomeShuttingDown>(subscription);
subscrBus.Subscribe<TcpMessage.ConnectionClosed>(subscription);
subscrBus.Subscribe<ClientMessage.SubscribeToStream>(subscription);
subscrBus.Subscribe<ClientMessage.UnsubscribeFromStream>(subscription);
subscrBus.Subscribe<SubscriptionMessage.PollStream>(subscription);
subscrBus.Subscribe<SubscriptionMessage.CheckPollTimeout>(subscription);
subscrBus.Subscribe<StorageMessage.EventCommited>(subscription);
// USER MANAGEMENT
var ioDispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_mainQueue));
_mainBus.Subscribe(ioDispatcher.BackwardReader);
_mainBus.Subscribe(ioDispatcher.ForwardReader);
_mainBus.Subscribe(ioDispatcher.Writer);
_mainBus.Subscribe(ioDispatcher.StreamDeleter);
_mainBus.Subscribe(ioDispatcher);
var userManagement = new UserManagementService(
_mainQueue, ioDispatcher, passwordHashAlgorithm, vNodeSettings.SkipInitializeStandardUsersCheck);
_mainBus.Subscribe<UserManagementMessage.Create>(userManagement);
_mainBus.Subscribe<UserManagementMessage.Update>(userManagement);
_mainBus.Subscribe<UserManagementMessage.Enable>(userManagement);
_mainBus.Subscribe<UserManagementMessage.Disable>(userManagement);
_mainBus.Subscribe<UserManagementMessage.Delete>(userManagement);
_mainBus.Subscribe<UserManagementMessage.ResetPassword>(userManagement);
_mainBus.Subscribe<UserManagementMessage.ChangePassword>(userManagement);
_mainBus.Subscribe<UserManagementMessage.Get>(userManagement);
_mainBus.Subscribe<UserManagementMessage.GetAll>(userManagement);
_mainBus.Subscribe<SystemMessage.BecomeMaster>(userManagement);
// TIMER
_timeProvider = new RealTimeProvider();
_timerService = new TimerService(new ThreadBasedScheduler(_timeProvider));
_mainBus.Subscribe<SystemMessage.BecomeShutdown>(_timerService);
_mainBus.Subscribe<TimerMessage.Schedule>(_timerService);
_workersHandler.Start();
_mainQueue.Start();
monitoringQueue.Start();
subscrQueue.Start();
if (subsystems != null)
foreach (var subsystem in subsystems)
subsystem.Register(db, _mainQueue, _mainBus, _timerService, _timeProvider, httpSendService, new[]{_httpService}, _workersHandler);
}