public void SetUp() { _indexDir = Path.Combine(Path.GetTempPath(), "idx-" + Guid.NewGuid().ToString()); _tableIndex = new TableIndex(_indexDir, () => new HashListMemTable(), maxSizeForMemory: 2, maxTablesPerLevel: 2); _tableIndex.Initialize(); // ptable level 2 _tableIndex.Add(0, 0xDEAD, 0, 0xFF00); _tableIndex.Add(0, 0xDEAD, 1, 0xFF01); _tableIndex.Add(0, 0xBEEF, 0, 0xFF00); _tableIndex.Add(0, 0xBEEF, 1, 0xFF01); _tableIndex.Add(0, 0xABBA, 0, 0xFF00); _tableIndex.Add(0, 0xABBA, 1, 0xFF01); _tableIndex.Add(0, 0xABBA, 0, 0xFF02); _tableIndex.Add(0, 0xABBA, 1, 0xFF03); // ptable level 1 _tableIndex.Add(0, 0xADA, 0, 0xFF00); _tableIndex.Add(0, 0xCEED, 10, 0xFFF1); _tableIndex.Add(0, 0xBABA, 0, 0xFF00); _tableIndex.Add(0, 0xDEAD, 0, 0xFF10); // ptable level 0 _tableIndex.Add(0, 0xBABA, 1, 0xFF01); _tableIndex.Add(0, 0xDEAD, 1, 0xFF11); // memtable _tableIndex.Add(0, 0xADA, 0, 0xFF01); Thread.Sleep(500); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _indexDir = PathName; var fakeReader = new TFReaderLease(new FakeTfReader()); _tableIndex = new TableIndex(_indexDir, () => new HashListMemTable(maxSize: 10), () => fakeReader, maxSizeForMemory: 5); _tableIndex.Initialize(long.MaxValue); _tableIndex.Add(0, 0xDEAD, 0, 0xFF00); _tableIndex.Add(0, 0xDEAD, 1, 0xFF01); _tableIndex.Add(0, 0xBEEF, 0, 0xFF00); _tableIndex.Add(0, 0xBEEF, 1, 0xFF01); _tableIndex.Add(0, 0xABBA, 0, 0xFF00); // 1st ptable0 _tableIndex.Add(0, 0xABBA, 1, 0xFF01); _tableIndex.Add(0, 0xABBA, 2, 0xFF02); _tableIndex.Add(0, 0xABBA, 3, 0xFF03); _tableIndex.Add(0, 0xADA, 0, 0xFF00); // simulates duplicate due to concurrency in TableIndex (see memtable below) _tableIndex.Add(0, 0xDEAD, 0, 0xFF10); // 2nd ptable0 _tableIndex.Add(0, 0xDEAD, 1, 0xFF11); // in memtable _tableIndex.Add(0, 0xADA, 0, 0xFF00); // in memtable }
private void ReOpenDb() { Db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, WriterCheckpoint, ChaserCheckpoint, new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); Db.Open(); TableIndex = new TableIndex(Path.Combine(PathName, "index"), () => new HashListMemTable(MaxEntriesInMemTable * 2), MaxEntriesInMemTable); ReadIndex = new ReadIndex(new NoopPublisher(), 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint), TableIndex, new ByLengthHasher(), new NoLRUCache<string, StreamCacheInfo>(), additionalCommitChecks: true, metastreamMaxCount: MetastreamMaxCount); ReadIndex.Init(WriterCheckpoint.Read(), ChaserCheckpoint.Read()); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); ReadIndex.Close(); ReadIndex.Dispose(); TableIndex.Close(removeFiles: false); var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 2, () => new TFChunkReader(Db, WriterCheckpoint)); var lowHasher = new XXHashUnsafe(); var highHasher = new Murmur3AUnsafe(); TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher, () => new HashListMemTable(PTableVersions.Index64Bit, maxSize: MaxEntriesInMemTable*2), () => new TFReaderLease(readers), PTableVersions.Index64Bit, maxSizeForMemory: MaxEntriesInMemTable); ReadIndex = new ReadIndex(new NoopPublisher(), readers, TableIndex, 0, additionalCommitChecks: true, metastreamMaxCount: 1, hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault); ReadIndex.Init(ChaserCheckpoint.Read()); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _lowHasher = new XXHashUnsafe(); _highHasher = new Murmur3AUnsafe(); _tableIndex = new TableIndex(PathName, _lowHasher, _highHasher, () => new HashListMemTable(version: _ptableVersion, maxSize: 40), () => { throw new InvalidOperationException(); }, _ptableVersion, maxSizeForMemory: 20); _tableIndex.Initialize(long.MaxValue); _tableIndex.Add(0, "0xDEAD", 0, 0xFF00); _tableIndex.Add(0, "0xDEAD", 1, 0xFF01); _tableIndex.Add(0, "0xJEEP", 0, 0xFF00); _tableIndex.Add(0, "0xJEEP", 1, 0xFF01); _tableIndex.Add(0, "0xABBA", 0, 0xFF00); _tableIndex.Add(0, "0xABBA", 1, 0xFF01); _tableIndex.Add(0, "0xABBA", 2, 0xFF02); _tableIndex.Add(0, "0xABBA", 3, 0xFF03); _tableIndex.Add(0, "0xDEAD", 0, 0xFF10); _tableIndex.Add(0, "0xDEAD", 1, 0xFF11); _tableIndex.Add(0, "0xADA", 0, 0xFF00); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _tableIndex = new TableIndex(PathName, () => new HashListMemTable(maxSize: 40), () => { throw new InvalidOperationException(); }, maxSizeForMemory: 20); _tableIndex.Initialize(long.MaxValue); _tableIndex.Add(0, 0xDEAD, 0, 0xFF00); _tableIndex.Add(0, 0xDEAD, 1, 0xFF01); _tableIndex.Add(0, 0xBEEF, 0, 0xFF00); _tableIndex.Add(0, 0xBEEF, 1, 0xFF01); _tableIndex.Add(0, 0xABBA, 0, 0xFF00); _tableIndex.Add(0, 0xABBA, 1, 0xFF01); _tableIndex.Add(0, 0xABBA, 2, 0xFF02); _tableIndex.Add(0, 0xABBA, 3, 0xFF03); _tableIndex.Add(0, 0xDEAD, 0, 0xFF10); _tableIndex.Add(0, 0xDEAD, 1, 0xFF11); _tableIndex.Add(0, 0xADA, 0, 0xFF00); }
private void ReOpenDb() { Db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, WriterCheckpoint, ChaserCheckpoint, new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); Db.Open(); var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint)); TableIndex = new TableIndex(Path.Combine(PathName, "index"), () => new HashListMemTable(MaxEntriesInMemTable * 2), () => new TFReaderLease(readers), MaxEntriesInMemTable); ReadIndex = new ReadIndex(new NoopPublisher(), readers, TableIndex, new ByLengthHasher(), 0, additionalCommitChecks: true, metastreamMaxCount: MetastreamMaxCount); ReadIndex.Init(ChaserCheckpoint.Read()); }
private void ReOpenDb() { Db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, WriterCheckpoint, ChaserCheckpoint, new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); Db.Open(); var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint)); var lowHasher = new XXHashUnsafe(); var highHasher = new Murmur3AUnsafe(); TableIndex = new TableIndex(Path.Combine(PathName, "index"), lowHasher, highHasher, () => new HashListMemTable(PTableVersions.Index64Bit, MaxEntriesInMemTable * 2), () => new TFReaderLease(readers), PTableVersions.Index64Bit, MaxEntriesInMemTable); ReadIndex = new ReadIndex(new NoopPublisher(), readers, TableIndex, 0, additionalCommitChecks: true, metastreamMaxCount: MetastreamMaxCount, hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault); ReadIndex.Init(ChaserCheckpoint.Read()); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _tableIndex = new TableIndex(PathName, () => new HashListMemTable(maxSize: 20), () => { throw new InvalidOperationException(); }, maxSizeForMemory: 10); _tableIndex.Initialize(long.MaxValue); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); WriterCheckpoint = new InMemoryCheckpoint(0); ChaserCheckpoint = new InMemoryCheckpoint(0); Bus = new InMemoryBus("bus"); IODispatcher = new IODispatcher(Bus, new PublishEnvelope(Bus)); Db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, WriterCheckpoint, ChaserCheckpoint, new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); Db.Open(); // create db Writer = new TFChunkWriter(Db); Writer.Open(); WriteTestScenario(); Writer.Close(); Writer = null; WriterCheckpoint.Flush(); ChaserCheckpoint.Write(WriterCheckpoint.Read()); ChaserCheckpoint.Flush(); var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint)); TableIndex = new TableIndex(GetFilePathFor("index"), () => new HashListMemTable(MaxEntriesInMemTable * 2), () => new TFReaderLease(readers), MaxEntriesInMemTable); var hasher = new ByLengthHasher(); ReadIndex = new ReadIndex(new NoopPublisher(), readers, TableIndex, hasher, 0, additionalCommitChecks: true, metastreamMaxCount: MetastreamMaxCount); ReadIndex.Init(ChaserCheckpoint.Read()); // scavenge must run after readIndex is built if (_scavenge) { if (_completeLastChunkOnScavenge) Db.Manager.GetChunk(Db.Manager.ChunksCount - 1).Complete(); _scavenger = new TFChunkScavenger(Db, IODispatcher, TableIndex, hasher, ReadIndex, Guid.NewGuid(), "fakeNodeIp"); _scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: _mergeChunks); } }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); var lowHasher = new XXHashUnsafe(); var highHasher = new Murmur3AUnsafe(); _tableIndex = new TableIndex(PathName, lowHasher, highHasher, () => new HashListMemTable(_ptableVersion, maxSize: 20), () => { throw new InvalidOperationException(); }, _ptableVersion, maxSizeForMemory: 10); _tableIndex.Initialize(long.MaxValue); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); ReadIndex.Close(); ReadIndex.Dispose(); Thread.Sleep(500); TableIndex.ClearAll(removeFiles: false); TableIndex = new TableIndex(Path.Combine(PathName, "index"), () => new HashListMemTable(), maxSizeForMemory: 5); TableIndex.Initialize(); ReadIndex = new ReadIndex(new NoopPublisher(), 2, () => new TFChunkSequentialReader(Db, WriterCheckpoint, 0), () => new TFChunkReader(Db, WriterCheckpoint), TableIndex, new ByLengthHasher()); ReadIndex.Build(); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); var dbConfig = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1024*1024, 0, new InMemoryCheckpoint(0), new InMemoryCheckpoint(0), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig); _dbResult = CreateDb(dbCreationHelper); _keptRecords = KeptRecords(_dbResult); _dbResult.Db.Config.WriterCheckpoint.Flush(); _dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read()); _dbResult.Db.Config.ChaserCheckpoint.Flush(); var indexPath = Path.Combine(PathName, "index"); var readerPool = new ObjectPool<ITransactionFileReader>( "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount, () => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint)); var lowHasher = new XXHashUnsafe(); var highHasher = new Murmur3AUnsafe(); var tableIndex = new TableIndex(indexPath, lowHasher, highHasher, () => new HashListMemTable(PTableVersions.Index64Bit, maxSize: 200), () => new TFReaderLease(readerPool), PTableVersions.Index64Bit, maxSizeForMemory: 100, maxTablesPerLevel: 2); ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, 100, true, _metastreamMaxCount, Opts.HashCollisionReadLimitDefault); ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read()); //var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount); var bus = new InMemoryBus("Bus"); var ioDispatcher = new IODispatcher(bus, new PublishEnvelope(bus)); var scavenger = new TFChunkScavenger(_dbResult.Db, ioDispatcher, tableIndex, ReadIndex, Guid.NewGuid(), "fakeNodeIp", unsafeIgnoreHardDeletes: UnsafeIgnoreHardDelete()); scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false); }
public void Setup(){ given(); _indexDir = PathName; _fakeReader = new TFReaderLease(new FakeReader()); _indexBackend = new FakeIndexBackend(_fakeReader); _lowHasher = new XXHashUnsafe(); _highHasher = new Murmur3AUnsafe(); _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher, () => new HashListMemTable(PTableVersions.Index32Bit, maxSize: _maxMemTableSize), () => _fakeReader, PTableVersions.Index32Bit, maxSizeForMemory: _maxMemTableSize, maxTablesPerLevel: 2); _tableIndex.Initialize(long.MaxValue); _indexReader = new IndexReader(_indexBackend, _tableIndex, new EventStore.Core.Data.StreamMetadata(), _hashCollisionReadLimit); when(); //wait for the mem table to be dumped System.Threading.Thread.Sleep(500); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _indexDir = PathName; var fakeReader = new TFReaderLease(new FakeIndexReader()); _lowHasher = new FakeIndexHasher(); _highHasher = new FakeIndexHasher(); _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher, () => new HashListMemTable(_ptableVersion, maxSize: 10), () => fakeReader, _ptableVersion, maxSizeForMemory: 2, maxTablesPerLevel: 2); _tableIndex.Initialize(long.MaxValue); // ptable level 2 _tableIndex.Add(0, "1", 0, 0xFF00); _tableIndex.Add(0, "1", 1, 0xFF01); _tableIndex.Add(0, "2", 0, 0xFF00); _tableIndex.Add(0, "2", 1, 0xFF01); _tableIndex.Add(0, "3", 0, 0xFF00); _tableIndex.Add(0, "3", 1, 0xFF01); _tableIndex.Add(0, "3", 0, 0xFF02); _tableIndex.Add(0, "3", 1, 0xFF03); // ptable level 1 _tableIndex.Add(0, "4", 0, 0xFF00); _tableIndex.Add(0, "5", 10, 0xFFF1); _tableIndex.Add(0, "6", 0, 0xFF00); _tableIndex.Add(0, "1", 0, 0xFF10); // ptable level 0 _tableIndex.Add(0, "6", 1, 0xFF01); _tableIndex.Add(0, "1", 1, 0xFF11); // memtable _tableIndex.Add(0, "4", 0, 0xFF01); Thread.Sleep(500); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); var dbConfig = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1024*1024, 0, new InMemoryCheckpoint(0), new InMemoryCheckpoint(0), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig); DbRes = CreateDb(dbCreationHelper); DbRes.Db.Config.WriterCheckpoint.Flush(); DbRes.Db.Config.ChaserCheckpoint.Write(DbRes.Db.Config.WriterCheckpoint.Read()); DbRes.Db.Config.ChaserCheckpoint.Flush(); var readers = new ObjectPool<ITransactionFileReader>( "Readers", 2, 2, () => new TFChunkReader(DbRes.Db, DbRes.Db.Config.WriterCheckpoint)); var lowHasher = new XXHashUnsafe(); var highHasher = new Murmur3AUnsafe(); TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher, () => new HashListMemTable(PTableVersions.Index64Bit, MaxEntriesInMemTable * 2), () => new TFReaderLease(readers), PTableVersions.Index64Bit, MaxEntriesInMemTable); ReadIndex = new ReadIndex(new NoopPublisher(), readers, TableIndex, 0, additionalCommitChecks: true, metastreamMaxCount: _metastreamMaxCount, hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault); ReadIndex.Init(DbRes.Db.Config.ChaserCheckpoint.Read()); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); ReadIndex.Close(); ReadIndex.Dispose(); TableIndex.Close(removeFiles: false); TableIndex = new TableIndex(GetFilePathFor("index"), () => new HashListMemTable(maxSize: MaxEntriesInMemTable*2), maxSizeForMemory: MaxEntriesInMemTable); ReadIndex = new ReadIndex(new NoopPublisher(), 2, () => new TFChunkSequentialReader(Db, WriterChecksum, 0), () => new TFChunkReader(Db, WriterChecksum), TableIndex, new ByLengthHasher(), new NoLRUCache<string, StreamCacheInfo>()); ReadIndex.Build(); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _indexDir = PathName; var fakeReader = new TFReaderLease(new FakeIndexReader()); _lowHasher = new XXHashUnsafe(); _highHasher = new Murmur3AUnsafe(); _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher, () => new HashListMemTable(PTableVersions.Index32Bit, maxSize: 5), () => fakeReader, PTableVersions.Index32Bit, maxSizeForMemory: 5, maxTablesPerLevel: 2); _tableIndex.Initialize(long.MaxValue); _tableIndex.Add(1, "LPN-FC002_LPK51001", 0, 1); _tableIndex.Add(1, "account--696193173", 0, 2); _tableIndex.Add(1, "LPN-FC002_LPK51001", 1, 3); _tableIndex.Add(1, "account--696193173", 1, 4); _tableIndex.Add(1, "LPN-FC002_LPK51001", 2, 5); _tableIndex.Close(false); _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher, () => new HashListMemTable(PTableVersions.Index64Bit, maxSize: 5), () => fakeReader, PTableVersions.Index64Bit, maxSizeForMemory: 5, maxTablesPerLevel: 2); _tableIndex.Initialize(long.MaxValue); _tableIndex.Add(1, "account--696193173", 2, 6); _tableIndex.Add(1, "LPN-FC002_LPK51001", 3, 7); _tableIndex.Add(1, "account--696193173", 3, 8); _tableIndex.Add(1, "LPN-FC002_LPK51001", 4, 9); _tableIndex.Add(1, "account--696193173", 4, 10); Thread.Sleep(500); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); ReadIndex.Close(); ReadIndex.Dispose(); TableIndex.Close(removeFiles: false); var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 2, () => new TFChunkReader(Db, WriterCheckpoint)); TableIndex = new TableIndex(GetFilePathFor("index"), () => new HashListMemTable(maxSize: MaxEntriesInMemTable*2), () => new TFReaderLease(readers), maxSizeForMemory: MaxEntriesInMemTable); ReadIndex = new ReadIndex(new NoopPublisher(), readers, TableIndex, new ByLengthHasher(), 0, additionalCommitChecks: true, metastreamMaxCount: 1); ReadIndex.Init(ChaserCheckpoint.Read()); }
public void SetUp() { _indexDir = Path.Combine(Path.GetTempPath(), "idx-" + Guid.NewGuid().ToString()); _tableIndex = new TableIndex(_indexDir, () => new HashListMemTable(), new InMemoryCheckpoint(), maxSizeForMemory: 2000); _tableIndex.Initialize(); _tableIndex.Add(0xDEAD, 0, 0xFF00); _tableIndex.Add(0xDEAD, 1, 0xFF01); _tableIndex.Add(0xBEEF, 0, 0xFF00); _tableIndex.Add(0xBEEF, 1, 0xFF01); _tableIndex.Add(0xABBA, 0, 0xFF00); _tableIndex.Add(0xABBA, 1, 0xFF01); _tableIndex.Add(0xABBA, 2, 0xFF02); _tableIndex.Add(0xABBA, 3, 0xFF03); _tableIndex.Add(0xDEAD, 0, 0xFF10); _tableIndex.Add(0xDEAD, 1, 0xFF11); _tableIndex.Add(0xADA, 0, 0xFF00); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _indexDir = PathName; var fakeReader = new TFReaderLease(new FakeTfReader()); _tableIndex = new TableIndex(_indexDir, () => new HashListMemTable(maxSize: 10), () => fakeReader, maxSizeForMemory: 2, maxTablesPerLevel: 2); _tableIndex.Initialize(long.MaxValue); // ptable level 2 _tableIndex.Add(0, 0xDEAD, 0, 0xFF00); _tableIndex.Add(0, 0xDEAD, 1, 0xFF01); _tableIndex.Add(0, 0xBEEF, 0, 0xFF00); _tableIndex.Add(0, 0xBEEF, 1, 0xFF01); _tableIndex.Add(0, 0xABBA, 0, 0xFF00); _tableIndex.Add(0, 0xABBA, 1, 0xFF01); _tableIndex.Add(0, 0xABBA, 0, 0xFF02); _tableIndex.Add(0, 0xABBA, 1, 0xFF03); // ptable level 1 _tableIndex.Add(0, 0xADA, 0, 0xFF00); _tableIndex.Add(0, 0xCEED, 10, 0xFFF1); _tableIndex.Add(0, 0xBABA, 0, 0xFF00); _tableIndex.Add(0, 0xDEAD, 0, 0xFF10); // ptable level 0 _tableIndex.Add(0, 0xBABA, 1, 0xFF01); _tableIndex.Add(0, 0xDEAD, 1, 0xFF11); // memtable _tableIndex.Add(0, 0xADA, 0, 0xFF01); Thread.Sleep(500); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); ReadIndex.Close(); ReadIndex.Dispose(); TableIndex.Close(removeFiles: false); TableIndex = new TableIndex(GetFilePathFor("index"), () => new HashListMemTable(maxSize: MaxEntriesInMemTable*2), maxSizeForMemory: MaxEntriesInMemTable); ReadIndex = new ReadIndex(new NoopPublisher(), 2, 2, () => new TFChunkReader(Db, WriterCheckpoint, 0), TableIndex, new ByLengthHasher(), new NoLRUCache<string, StreamCacheInfo>(), additionalCommitChecks: true, metastreamMaxCount: 1); ReadIndex.Init(WriterCheckpoint.Read(), ChaserCheckpoint.Read()); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); _tableIndex = new TableIndex(PathName, () => new HashListMemTable(maxSize: 40), maxSizeForMemory: 20); _tableIndex.Initialize(); _tableIndex.Add(0, 0xDEAD, 0, 0xFF00); _tableIndex.Add(0, 0xDEAD, 1, 0xFF01); _tableIndex.Add(0, 0xBEEF, 0, 0xFF00); _tableIndex.Add(0, 0xBEEF, 1, 0xFF01); _tableIndex.Add(0, 0xABBA, 0, 0xFF00); _tableIndex.Add(0, 0xABBA, 1, 0xFF01); _tableIndex.Add(0, 0xABBA, 2, 0xFF02); _tableIndex.Add(0, 0xABBA, 3, 0xFF03); _tableIndex.Add(0, 0xDEAD, 0, 0xFF10); _tableIndex.Add(0, 0xDEAD, 1, 0xFF11); _tableIndex.Add(0, 0xADA, 0, 0xFF00); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); var dbConfig = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1024*1024, 0, new InMemoryCheckpoint(0), new InMemoryCheckpoint(0), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig); _dbResult = CreateDb(dbCreationHelper); _keptRecords = KeptRecords(_dbResult); _dbResult.Db.Config.WriterCheckpoint.Flush(); _dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read()); _dbResult.Db.Config.ChaserCheckpoint.Flush(); var indexPath = Path.Combine(PathName, "index"); var readerPool = new ObjectPool<ITransactionFileReader>( "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount, () => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint)); var tableIndex = new TableIndex(indexPath, () => new HashListMemTable(maxSize: 200), () => new TFReaderLease(readerPool), maxSizeForMemory: 100, maxTablesPerLevel: 2); var hasher = new XXHashUnsafe(); ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, hasher, 100, true, _metastreamMaxCount); ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read()); //var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount); var scavenger = new TFChunkScavenger(_dbResult.Db, tableIndex, hasher, ReadIndex); scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false); }
public override void SetUp() { base.SetUp(); _indexDir = base.PathName; _tableIndex = new TableIndex(_indexDir, () => new HashListMemTable(), maxSizeForMemory: 2000); _tableIndex.Initialize(); _tableIndex.Add(0, 0xDEAD, 0, 0xFF00); _tableIndex.Add(0, 0xDEAD, 1, 0xFF01); _tableIndex.Add(0, 0xBEEF, 0, 0xFF00); _tableIndex.Add(0, 0xBEEF, 1, 0xFF01); _tableIndex.Add(0, 0xABBA, 0, 0xFF00); _tableIndex.Add(0, 0xABBA, 1, 0xFF01); _tableIndex.Add(0, 0xABBA, 2, 0xFF02); _tableIndex.Add(0, 0xABBA, 3, 0xFF03); _tableIndex.Add(0, 0xDEAD, 0, 0xFF10); _tableIndex.Add(0, 0xDEAD, 1, 0xFF11); _tableIndex.Add(0, 0xADA, 0, 0xFF00); }
public void SetUp() { _indexDir = Path.Combine(Path.GetTempPath(), "idx-" + Guid.NewGuid().ToString()); _tableIndex = new TableIndex(_indexDir, () => new HashListMemTable(), new InMemoryCheckpoint(), maxSizeForMemory: 5); _tableIndex.Initialize(); _tableIndex.Add(0xDEAD, 0, 0xFF00); _tableIndex.Add(0xDEAD, 1, 0xFF01); _tableIndex.Add(0xBEEF, 0, 0xFF00); _tableIndex.Add(0xBEEF, 1, 0xFF01); _tableIndex.Add(0xABBA, 0, 0xFF00); // 1st ptable0 _tableIndex.Add(0xABBA, 1, 0xFF01); _tableIndex.Add(0xABBA, 2, 0xFF02); _tableIndex.Add(0xABBA, 3, 0xFF03); _tableIndex.Add(0xADA, 0, 0xFF00); // simulates duplicate due to concurrency in TableIndex (see memtable below) _tableIndex.Add(0xDEAD, 0, 0xFF10); // 2nd ptable0 _tableIndex.Add(0xDEAD, 1, 0xFF11); // in memtable _tableIndex.Add(0xADA, 0, 0xFF00); // in memtable }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); var dbConfig = new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 1024*1024, 0, new InMemoryCheckpoint(0), new InMemoryCheckpoint(0), new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1)); var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig); DbRes = CreateDb(dbCreationHelper); DbRes.Db.Config.WriterCheckpoint.Flush(); DbRes.Db.Config.ChaserCheckpoint.Write(DbRes.Db.Config.WriterCheckpoint.Read()); DbRes.Db.Config.ChaserCheckpoint.Flush(); TableIndex = new TableIndex(GetFilePathFor("index"), () => new HashListMemTable(MaxEntriesInMemTable * 2), MaxEntriesInMemTable); ReadIndex = new ReadIndex(new NoopPublisher(), 2, 2, () => new TFChunkReader(DbRes.Db, DbRes.Db.Config.WriterCheckpoint), TableIndex, new ByLengthHasher(), new NoLRUCache<string, StreamCacheInfo>(), additionalCommitChecks: true, metastreamMaxCount: _metastreamMaxCount); ReadIndex.Init(DbRes.Db.Config.WriterCheckpoint.Read(), DbRes.Db.Config.ChaserCheckpoint.Read()); }
public SingleVNode(TFChunkDb db, SingleVNodeSettings vNodeSettings, SingleVNodeAppSettings appSettings) { Ensure.NotNull(db, "db"); Ensure.NotNull(vNodeSettings, "vNodeSettings"); db.OpenVerifyAndClean(); _tcpEndPoint = vNodeSettings.ExternalTcpEndPoint; _httpEndPoint = vNodeSettings.HttpEndPoint; _outputBus = new InMemoryBus("OutputBus"); _controller = new SingleVNodeController(Bus, _httpEndPoint); _mainQueue = new QueuedHandler(_controller, "MainQueue"); _controller.SetMainQueue(MainQueue); //MONITORING var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false); var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false); var monitoringQueue = new QueuedHandler(monitoringInnerBus, "MonitoringQueue", watchSlowMsg: true, slowMsgThresholdMs: 100); var monitoring = new MonitoringService(monitoringQueue, monitoringRequestBus, db.Config.WriterCheckpoint, appSettings.StatsPeriod); Bus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.SystemInit, Message>()); Bus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>()); monitoringInnerBus.Subscribe<SystemMessage.SystemInit>(monitoring); monitoringInnerBus.Subscribe<SystemMessage.BecomeShuttingDown>(monitoring); monitoringInnerBus.Subscribe<MonitoringMessage.GetFreshStats>(monitoring); //STORAGE SUBSYSTEM var indexPath = Path.Combine(db.Config.Path, "index"); var tableIndex = new TableIndex(indexPath, () => new HashListMemTable(), new InMemoryCheckpoint(), maxSizeForMemory: 1000000, maxTablesPerLevel: 2); var readIndex = new ReadIndex(_mainQueue, db, () => new TFChunkReader(db, db.Config.WriterCheckpoint), TFConsts.ReadIndexReaderCount, db.Config.WriterCheckpoint, tableIndex, new XXHashUnsafe()); var writer = new TFChunkWriter(db); var storageWriter = new StorageWriter(_mainQueue, _outputBus, writer, readIndex); var storageReader = new StorageReader(_mainQueue, _outputBus, readIndex, TFConsts.StorageReaderHandlerCount); monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageReader); var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.GetNamedCheckpoint(Checkpoint.Chaser)); var storageChaser = new StorageChaser(_mainQueue, chaser); _outputBus.Subscribe<SystemMessage.SystemInit>(storageChaser); _outputBus.Subscribe<SystemMessage.SystemStart>(storageChaser); _outputBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageChaser); var storageScavenger = new StorageScavenger(db, readIndex); _outputBus.Subscribe<SystemMessage.ScavengeDatabase>(storageScavenger); //TCP var tcpService = new TcpService(MainQueue, _tcpEndPoint); Bus.Subscribe<SystemMessage.SystemInit>(tcpService); Bus.Subscribe<SystemMessage.SystemStart>(tcpService); Bus.Subscribe<SystemMessage.BecomeShuttingDown>(tcpService); //HTTP HttpService = new HttpService(MainQueue, _httpEndPoint.ToHttpUrl()); Bus.Subscribe<SystemMessage.SystemInit>(HttpService); Bus.Subscribe<SystemMessage.BecomeShuttingDown>(HttpService); Bus.Subscribe<HttpMessage.SendOverHttp>(HttpService); Bus.Subscribe<HttpMessage.UpdatePendingRequests>(HttpService); HttpService.SetupController(new AdminController(MainQueue)); HttpService.SetupController(new PingController()); HttpService.SetupController(new StatController(monitoringQueue)); HttpService.SetupController(new ReadEventDataController(MainQueue)); HttpService.SetupController(new AtomController(MainQueue)); HttpService.SetupController(new WebSiteController(MainQueue)); //REQUEST MANAGEMENT var requestManagement = new RequestManagementService(MainQueue, 1, 1); Bus.Subscribe<ReplicationMessage.EventCommited>(requestManagement); Bus.Subscribe<ReplicationMessage.CreateStreamRequestCreated>(requestManagement); Bus.Subscribe<ReplicationMessage.WriteRequestCreated>(requestManagement); Bus.Subscribe<ReplicationMessage.TransactionStartRequestCreated>(requestManagement); Bus.Subscribe<ReplicationMessage.TransactionWriteRequestCreated>(requestManagement); Bus.Subscribe<ReplicationMessage.TransactionCommitRequestCreated>(requestManagement); Bus.Subscribe<ReplicationMessage.DeleteStreamRequestCreated>(requestManagement); Bus.Subscribe<ReplicationMessage.RequestCompleted>(requestManagement); Bus.Subscribe<ReplicationMessage.CommitAck>(requestManagement); Bus.Subscribe<ReplicationMessage.PrepareAck>(requestManagement); Bus.Subscribe<ReplicationMessage.WrongExpectedVersion>(requestManagement); Bus.Subscribe<ReplicationMessage.InvalidTransaction>(requestManagement); Bus.Subscribe<ReplicationMessage.StreamDeleted>(requestManagement); Bus.Subscribe<ReplicationMessage.PreparePhaseTimeout>(requestManagement); Bus.Subscribe<ReplicationMessage.CommitPhaseTimeout>(requestManagement); var clientService = new ClientService(); Bus.Subscribe<TcpMessage.ConnectionClosed>(clientService); Bus.Subscribe<ClientMessage.SubscribeToStream>(clientService); Bus.Subscribe<ClientMessage.UnsubscribeFromStream>(clientService); Bus.Subscribe<ClientMessage.SubscribeToAllStreams>(clientService); Bus.Subscribe<ClientMessage.UnsubscribeFromAllStreams>(clientService); Bus.Subscribe<ReplicationMessage.EventCommited>(clientService); //TIMER //var timer = new TimerService(new TimerBasedScheduler(new RealTimer(), new RealTimeProvider())); TimerService = new TimerService(new ThreadBasedScheduler(new RealTimeProvider())); Bus.Subscribe<TimerMessage.Schedule>(TimerService); MainQueue.Start(); monitoringQueue.Start(); }
public override void TestFixtureSetUp() { base.TestFixtureSetUp(); WriterCheckpoint = new InMemoryCheckpoint(0); ChaserCheckpoint = new InMemoryCheckpoint(0); Db = new TFChunkDb(new TFChunkDbConfig(PathName, new VersionedPatternFileNamingStrategy(PathName, "chunk-"), 10000, 0, WriterCheckpoint, ChaserCheckpoint, new InMemoryCheckpoint(-1), new InMemoryCheckpoint(-1))); Db.Open(); // create db Writer = new TFChunkWriter(Db); Writer.Open(); WriteTestScenario(); Writer.Close(); Writer = null; WriterCheckpoint.Flush(); ChaserCheckpoint.Write(WriterCheckpoint.Read()); ChaserCheckpoint.Flush(); TableIndex = new TableIndex(GetFilePathFor("index"), () => new HashListMemTable(MaxEntriesInMemTable * 2), MaxEntriesInMemTable); ReadIndex = new ReadIndex(new NoopPublisher(), 2, 2, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint), TableIndex, new ByLengthHasher(), new NoLRUCache<string, StreamCacheInfo>(), additionalCommitChecks: true, metastreamMaxCount: MetastreamMaxCount); ReadIndex.Init(WriterCheckpoint.Read(), ChaserCheckpoint.Read()); // scavenge must run after readIndex is built if (_scavenge) { if (_completeLastChunkOnScavenge) Db.Manager.GetChunk(Db.Manager.ChunksCount - 1).Complete(); _scavenger = new TFChunkScavenger(Db, ReadIndex); _scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: _mergeChunks); } }
public SingleVNode(TFChunkDb db, SingleVNodeSettings vNodeSettings, bool dbVerifyHashes, int memTableEntryCount, params ISubsystem[] subsystems) { Ensure.NotNull(db, "db"); Ensure.NotNull(vNodeSettings, "vNodeSettings"); _settings = vNodeSettings; _mainBus = new InMemoryBus("MainBus"); _controller = new SingleVNodeController(_mainBus, _settings.ExternalHttpEndPoint, db, this); _mainQueue = new QueuedHandler(_controller, "MainQueue"); _controller.SetMainQueue(_mainQueue); _subsystems = subsystems; // MONITORING var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false); var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false); var monitoringQueue = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100)); var monitoring = new MonitoringService(monitoringQueue, monitoringRequestBus, _mainQueue, db.Config.WriterCheckpoint, db.Config.Path, vNodeSettings.StatsPeriod, _settings.ExternalHttpEndPoint, vNodeSettings.StatsStorage); _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.SystemInit, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.StateChangeMessage, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShutdown, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom<ClientMessage.WriteEventsCompleted, Message>()); monitoringInnerBus.Subscribe<SystemMessage.SystemInit>(monitoring); monitoringInnerBus.Subscribe<SystemMessage.StateChangeMessage>(monitoring); monitoringInnerBus.Subscribe<SystemMessage.BecomeShuttingDown>(monitoring); monitoringInnerBus.Subscribe<SystemMessage.BecomeShutdown>(monitoring); monitoringInnerBus.Subscribe<ClientMessage.WriteEventsCompleted>(monitoring); monitoringInnerBus.Subscribe<MonitoringMessage.GetFreshStats>(monitoring); var truncPos = db.Config.TruncateCheckpoint.Read(); if (truncPos != -1) { Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})", truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read()); var truncator = new TFChunkDbTruncator(db.Config); truncator.TruncateDb(truncPos); } db.Open(dbVerifyHashes); // STORAGE SUBSYSTEM var indexPath = Path.Combine(db.Config.Path, "index"); var readerPool = new ObjectPool<ITransactionFileReader>( "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount, () => new TFChunkReader(db, db.Config.WriterCheckpoint)); var tableIndex = new TableIndex(indexPath, () => new HashListMemTable(maxSize: memTableEntryCount * 2), () => new TFReaderLease(readerPool), maxSizeForMemory: memTableEntryCount, maxTablesPerLevel: 2, inMem: db.Config.InMemDb); var hasher = new XXHashUnsafe(); var readIndex = new ReadIndex(_mainQueue, readerPool, tableIndex, hasher, ESConsts.StreamInfoCacheCapacity, Application.IsDefined(Application.AdditionalCommitChecks), Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1); var writer = new TFChunkWriter(db); var epochManager = new EpochManager(ESConsts.CachedEpochCount, db.Config.EpochCheckpoint, writer, initialReaderCount: 1, maxReaderCount: 5, readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint)); epochManager.Init(); var storageWriter = new StorageWriterService(_mainQueue, _mainBus, _settings.MinFlushDelay, db, writer, readIndex.IndexWriter, epochManager); // subscribes internally monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageWriter); var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, ESConsts.StorageReaderThreadCount, db.Config.WriterCheckpoint); _mainBus.Subscribe<SystemMessage.SystemInit>(storageReader); _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageReader); _mainBus.Subscribe<SystemMessage.BecomeShutdown>(storageReader); monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageReader); var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint); var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, readIndex.IndexCommitter, epochManager); _mainBus.Subscribe<SystemMessage.SystemInit>(storageChaser); _mainBus.Subscribe<SystemMessage.SystemStart>(storageChaser); _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageChaser); var storageScavenger = new StorageScavenger(db, tableIndex, hasher, readIndex, Application.IsDefined(Application.AlwaysKeepScavenged), mergeChunks: false /*!Application.IsDefined(Application.DisableMergeChunks)*/); // ReSharper disable RedundantTypeArgumentsOfMethod _mainBus.Subscribe<ClientMessage.ScavengeDatabase>(storageScavenger); // ReSharper restore RedundantTypeArgumentsOfMethod // MISC WORKERS _workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum => new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1), watchSlowMsg: true, slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray(); _workersHandler = new MultiQueuedHandler( vNodeSettings.WorkerThreads, queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum], string.Format("Worker #{0}", queueNum + 1), groupName: "Workers", watchSlowMsg: true, slowMsgThreshold: TimeSpan.FromMilliseconds(200))); // AUTHENTICATION INFRASTRUCTURE var passwordHashAlgorithm = new Rfc2898PasswordHashAlgorithm(); var dispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_workersHandler, crossThread: true)); var internalAuthenticationProvider = new InternalAuthenticationProvider(dispatcher, passwordHashAlgorithm, ESConsts.CachedPrincipalCount); var passwordChangeNotificationReader = new PasswordChangeNotificationReader(_mainQueue, dispatcher); _mainBus.Subscribe<SystemMessage.SystemStart>(passwordChangeNotificationReader); _mainBus.Subscribe<SystemMessage.BecomeShutdown>(passwordChangeNotificationReader); _mainBus.Subscribe(internalAuthenticationProvider); SubscribeWorkers(bus => { bus.Subscribe(dispatcher.ForwardReader); bus.Subscribe(dispatcher.BackwardReader); bus.Subscribe(dispatcher.Writer); bus.Subscribe(dispatcher.StreamDeleter); bus.Subscribe(dispatcher); }); // TCP { var tcpService = new TcpService(_mainQueue, _settings.ExternalTcpEndPoint, _workersHandler, TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(), ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout, internalAuthenticationProvider, null); _mainBus.Subscribe<SystemMessage.SystemInit>(tcpService); _mainBus.Subscribe<SystemMessage.SystemStart>(tcpService); _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(tcpService); } // SECURE TCP if (vNodeSettings.ExternalSecureTcpEndPoint != null) { var secureTcpService = new TcpService(_mainQueue, _settings.ExternalSecureTcpEndPoint, _workersHandler, TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(), ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout, internalAuthenticationProvider, _settings.Certificate); _mainBus.Subscribe<SystemMessage.SystemInit>(secureTcpService); _mainBus.Subscribe<SystemMessage.SystemStart>(secureTcpService); _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(secureTcpService); } SubscribeWorkers(bus => { var tcpSendService = new TcpSendService(); // ReSharper disable RedundantTypeArgumentsOfMethod bus.Subscribe<TcpMessage.TcpSend>(tcpSendService); // ReSharper restore RedundantTypeArgumentsOfMethod }); // HTTP var httpAuthenticationProviders = new List<HttpAuthenticationProvider> { new BasicHttpAuthenticationProvider(internalAuthenticationProvider), }; if (_settings.EnableTrustedAuth) httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider()); httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider()); var httpPipe = new HttpMessagePipe(); var httpSendService = new HttpSendService(httpPipe, forwardRequests: false); _mainBus.Subscribe<SystemMessage.StateChangeMessage>(httpSendService); _mainBus.Subscribe(new WideningHandler<HttpMessage.SendOverHttp, Message>(_workersHandler)); SubscribeWorkers(bus => { bus.Subscribe<HttpMessage.HttpSend>(httpSendService); bus.Subscribe<HttpMessage.HttpSendPart>(httpSendService); bus.Subscribe<HttpMessage.HttpBeginSend>(httpSendService); bus.Subscribe<HttpMessage.HttpEndSend>(httpSendService); bus.Subscribe<HttpMessage.SendOverHttp>(httpSendService); }); _httpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(), _workersHandler, vNodeSettings.HttpPrefixes); _httpService.SetupController(new AdminController(_mainQueue)); _httpService.SetupController(new PingController()); _httpService.SetupController(new StatController(monitoringQueue, _workersHandler)); _httpService.SetupController(new AtomController(httpSendService, _mainQueue, _workersHandler)); _httpService.SetupController(new GuidController(_mainQueue)); _httpService.SetupController(new UsersController(httpSendService, _mainQueue, _workersHandler)); _mainBus.Subscribe<SystemMessage.SystemInit>(_httpService); _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(_httpService); _mainBus.Subscribe<HttpMessage.PurgeTimedOutRequests>(_httpService); SubscribeWorkers(bus => { HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray()); }); // REQUEST MANAGEMENT var requestManagement = new RequestManagementService(_mainQueue, 1, 1, vNodeSettings.PrepareTimeout, vNodeSettings.CommitTimeout); _mainBus.Subscribe<SystemMessage.SystemInit>(requestManagement); _mainBus.Subscribe<ClientMessage.WriteEvents>(requestManagement); _mainBus.Subscribe<ClientMessage.TransactionStart>(requestManagement); _mainBus.Subscribe<ClientMessage.TransactionWrite>(requestManagement); _mainBus.Subscribe<ClientMessage.TransactionCommit>(requestManagement); _mainBus.Subscribe<ClientMessage.DeleteStream>(requestManagement); _mainBus.Subscribe<StorageMessage.RequestCompleted>(requestManagement); _mainBus.Subscribe<StorageMessage.CheckStreamAccessCompleted>(requestManagement); _mainBus.Subscribe<StorageMessage.AlreadyCommitted>(requestManagement); _mainBus.Subscribe<StorageMessage.CommitAck>(requestManagement); _mainBus.Subscribe<StorageMessage.PrepareAck>(requestManagement); _mainBus.Subscribe<StorageMessage.WrongExpectedVersion>(requestManagement); _mainBus.Subscribe<StorageMessage.InvalidTransaction>(requestManagement); _mainBus.Subscribe<StorageMessage.StreamDeleted>(requestManagement); _mainBus.Subscribe<StorageMessage.RequestManagerTimerTick>(requestManagement); // SUBSCRIPTIONS var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50)); var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false); _mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.SystemStart, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom<TcpMessage.ConnectionClosed, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.SubscribeToStream, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.UnsubscribeFromStream, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.PollStream, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.CheckPollTimeout, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom<StorageMessage.EventCommited, Message>()); var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex); subscrBus.Subscribe<SystemMessage.SystemStart>(subscription); subscrBus.Subscribe<SystemMessage.BecomeShuttingDown>(subscription); subscrBus.Subscribe<TcpMessage.ConnectionClosed>(subscription); subscrBus.Subscribe<ClientMessage.SubscribeToStream>(subscription); subscrBus.Subscribe<ClientMessage.UnsubscribeFromStream>(subscription); subscrBus.Subscribe<SubscriptionMessage.PollStream>(subscription); subscrBus.Subscribe<SubscriptionMessage.CheckPollTimeout>(subscription); subscrBus.Subscribe<StorageMessage.EventCommited>(subscription); // USER MANAGEMENT var ioDispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_mainQueue)); _mainBus.Subscribe(ioDispatcher.BackwardReader); _mainBus.Subscribe(ioDispatcher.ForwardReader); _mainBus.Subscribe(ioDispatcher.Writer); _mainBus.Subscribe(ioDispatcher.StreamDeleter); _mainBus.Subscribe(ioDispatcher); var userManagement = new UserManagementService( _mainQueue, ioDispatcher, passwordHashAlgorithm, vNodeSettings.SkipInitializeStandardUsersCheck); _mainBus.Subscribe<UserManagementMessage.Create>(userManagement); _mainBus.Subscribe<UserManagementMessage.Update>(userManagement); _mainBus.Subscribe<UserManagementMessage.Enable>(userManagement); _mainBus.Subscribe<UserManagementMessage.Disable>(userManagement); _mainBus.Subscribe<UserManagementMessage.Delete>(userManagement); _mainBus.Subscribe<UserManagementMessage.ResetPassword>(userManagement); _mainBus.Subscribe<UserManagementMessage.ChangePassword>(userManagement); _mainBus.Subscribe<UserManagementMessage.Get>(userManagement); _mainBus.Subscribe<UserManagementMessage.GetAll>(userManagement); _mainBus.Subscribe<SystemMessage.BecomeMaster>(userManagement); // TIMER _timeProvider = new RealTimeProvider(); _timerService = new TimerService(new ThreadBasedScheduler(_timeProvider)); _mainBus.Subscribe<SystemMessage.BecomeShutdown>(_timerService); _mainBus.Subscribe<TimerMessage.Schedule>(_timerService); _workersHandler.Start(); _mainQueue.Start(); monitoringQueue.Start(); subscrQueue.Start(); if (subsystems != null) foreach (var subsystem in subsystems) subsystem.Register(db, _mainQueue, _mainBus, _timerService, _timeProvider, httpSendService, new[]{_httpService}, _workersHandler); }