Inheritance: ITableIndex
        public void SetUp()
        {
            _indexDir = Path.Combine(Path.GetTempPath(), "idx-" + Guid.NewGuid().ToString());
            _tableIndex = new TableIndex(_indexDir,
                                         () => new HashListMemTable(),
                                         maxSizeForMemory: 2,
                                         maxTablesPerLevel: 2);
            _tableIndex.Initialize();

            // ptable level 2
            _tableIndex.Add(0, 0xDEAD, 0, 0xFF00);
            _tableIndex.Add(0, 0xDEAD, 1, 0xFF01);
            _tableIndex.Add(0, 0xBEEF, 0, 0xFF00);
            _tableIndex.Add(0, 0xBEEF, 1, 0xFF01);
            _tableIndex.Add(0, 0xABBA, 0, 0xFF00);
            _tableIndex.Add(0, 0xABBA, 1, 0xFF01);
            _tableIndex.Add(0, 0xABBA, 0, 0xFF02);
            _tableIndex.Add(0, 0xABBA, 1, 0xFF03);

            // ptable level 1
            _tableIndex.Add(0, 0xADA, 0, 0xFF00);
            _tableIndex.Add(0, 0xCEED, 10, 0xFFF1);
            _tableIndex.Add(0, 0xBABA, 0, 0xFF00);
            _tableIndex.Add(0, 0xDEAD, 0, 0xFF10);

            // ptable level 0
            _tableIndex.Add(0, 0xBABA, 1, 0xFF01);
            _tableIndex.Add(0, 0xDEAD, 1, 0xFF11);

            // memtable
            _tableIndex.Add(0, 0xADA, 0, 0xFF01);

            Thread.Sleep(500);
        }
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            _indexDir = PathName;
            var fakeReader = new TFReaderLease(new FakeTfReader());
            _tableIndex = new TableIndex(_indexDir,
                                         () => new HashListMemTable(maxSize: 10),
                                         () => fakeReader,
                                         maxSizeForMemory: 5);
            _tableIndex.Initialize(long.MaxValue);

            _tableIndex.Add(0, 0xDEAD, 0, 0xFF00);
            _tableIndex.Add(0, 0xDEAD, 1, 0xFF01); 
                             
            _tableIndex.Add(0, 0xBEEF, 0, 0xFF00);
            _tableIndex.Add(0, 0xBEEF, 1, 0xFF01); 
                             
            _tableIndex.Add(0, 0xABBA, 0, 0xFF00); // 1st ptable0
                             
            _tableIndex.Add(0, 0xABBA, 1, 0xFF01); 
            _tableIndex.Add(0, 0xABBA, 2, 0xFF02);
            _tableIndex.Add(0, 0xABBA, 3, 0xFF03); 
                             
            _tableIndex.Add(0, 0xADA, 0, 0xFF00); // simulates duplicate due to concurrency in TableIndex (see memtable below)
            _tableIndex.Add(0, 0xDEAD, 0, 0xFF10); // 2nd ptable0
                            
            _tableIndex.Add(0, 0xDEAD, 1, 0xFF11); // in memtable
            _tableIndex.Add(0, 0xADA, 0, 0xFF00); // in memtable
        }
        private void ReOpenDb()
        {
            Db = new TFChunkDb(new TFChunkDbConfig(PathName,
                                                   new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                                   10000,
                                                   0,
                                                   WriterCheckpoint,
                                                   ChaserCheckpoint,
                                                   new InMemoryCheckpoint(-1),
                                                   new InMemoryCheckpoint(-1)));

            Db.Open();

            TableIndex = new TableIndex(Path.Combine(PathName, "index"),
                                        () => new HashListMemTable(MaxEntriesInMemTable * 2),
                                        MaxEntriesInMemTable);
            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      2,
                                      5,
                                      () => new TFChunkReader(Db, Db.Config.WriterCheckpoint),
                                      TableIndex,
                                      new ByLengthHasher(),
                                      new NoLRUCache<string, StreamCacheInfo>(),
                                      additionalCommitChecks: true,
                                      metastreamMaxCount: MetastreamMaxCount);
            ReadIndex.Init(WriterCheckpoint.Read(), ChaserCheckpoint.Read());
        }
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            ReadIndex.Close();
            ReadIndex.Dispose();
            TableIndex.Close(removeFiles: false);

            var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 2, () => new TFChunkReader(Db, WriterCheckpoint));
            var lowHasher = new XXHashUnsafe();
            var highHasher = new Murmur3AUnsafe();
            TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
                                        () => new HashListMemTable(PTableVersions.Index64Bit, maxSize: MaxEntriesInMemTable*2),
                                        () => new TFReaderLease(readers),
                                        PTableVersions.Index64Bit,
                                        maxSizeForMemory: MaxEntriesInMemTable);
            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      readers,
                                      TableIndex,
                                      0,
                                      additionalCommitChecks: true, 
                                      metastreamMaxCount: 1,
                                      hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault);
            ReadIndex.Init(ChaserCheckpoint.Read());
        }
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            _lowHasher = new XXHashUnsafe();
            _highHasher = new Murmur3AUnsafe();
            _tableIndex = new TableIndex(PathName, _lowHasher, _highHasher,
                                         () => new HashListMemTable(version: _ptableVersion, maxSize: 40),
                                         () => { throw new InvalidOperationException(); },
                                         _ptableVersion,
                                         maxSizeForMemory: 20);
            _tableIndex.Initialize(long.MaxValue);

            _tableIndex.Add(0, "0xDEAD", 0, 0xFF00);
            _tableIndex.Add(0, "0xDEAD", 1, 0xFF01); 
                            
            _tableIndex.Add(0, "0xJEEP", 0, 0xFF00);
            _tableIndex.Add(0, "0xJEEP", 1, 0xFF01);
                             
            _tableIndex.Add(0, "0xABBA", 0, 0xFF00);
            _tableIndex.Add(0, "0xABBA", 1, 0xFF01); 
            _tableIndex.Add(0, "0xABBA", 2, 0xFF02);
            _tableIndex.Add(0, "0xABBA", 3, 0xFF03); 
                             
            _tableIndex.Add(0, "0xDEAD", 0, 0xFF10);
            _tableIndex.Add(0, "0xDEAD", 1, 0xFF11); 
                             
            _tableIndex.Add(0, "0xADA", 0, 0xFF00);
        }
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            _tableIndex = new TableIndex(PathName,
                                         () => new HashListMemTable(maxSize: 40),
                                         () => { throw new InvalidOperationException(); },
                                         maxSizeForMemory: 20);
            _tableIndex.Initialize(long.MaxValue);

            _tableIndex.Add(0, 0xDEAD, 0, 0xFF00);
            _tableIndex.Add(0, 0xDEAD, 1, 0xFF01); 
                            
            _tableIndex.Add(0, 0xBEEF, 0, 0xFF00);
            _tableIndex.Add(0, 0xBEEF, 1, 0xFF01); 
                             
            _tableIndex.Add(0, 0xABBA, 0, 0xFF00);
            _tableIndex.Add(0, 0xABBA, 1, 0xFF01); 
            _tableIndex.Add(0, 0xABBA, 2, 0xFF02);
            _tableIndex.Add(0, 0xABBA, 3, 0xFF03); 
                             
            _tableIndex.Add(0, 0xDEAD, 0, 0xFF10);
            _tableIndex.Add(0, 0xDEAD, 1, 0xFF11); 
                             
            _tableIndex.Add(0, 0xADA, 0, 0xFF00);
        }
        private void ReOpenDb()
        {
            Db = new TFChunkDb(new TFChunkDbConfig(PathName,
                                                   new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                                   10000,
                                                   0,
                                                   WriterCheckpoint,
                                                   ChaserCheckpoint,
                                                   new InMemoryCheckpoint(-1),
                                                   new InMemoryCheckpoint(-1)));

            Db.Open();

            var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
            TableIndex = new TableIndex(Path.Combine(PathName, "index"),
                                        () => new HashListMemTable(MaxEntriesInMemTable * 2),
                                        () => new TFReaderLease(readers),
                                        MaxEntriesInMemTable);
            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      readers,
                                      TableIndex,
                                      new ByLengthHasher(),
                                      0,
                                      additionalCommitChecks: true,
                                      metastreamMaxCount: MetastreamMaxCount);
            ReadIndex.Init(ChaserCheckpoint.Read());
        }
        private void ReOpenDb()
        {
            Db = new TFChunkDb(new TFChunkDbConfig(PathName,
                                                   new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                                   10000,
                                                   0,
                                                   WriterCheckpoint,
                                                   ChaserCheckpoint,
                                                   new InMemoryCheckpoint(-1),
                                                   new InMemoryCheckpoint(-1)));

            Db.Open();

            var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
            var lowHasher = new XXHashUnsafe();
            var highHasher = new Murmur3AUnsafe();
            TableIndex = new TableIndex(Path.Combine(PathName, "index"), lowHasher, highHasher,
                                        () => new HashListMemTable(PTableVersions.Index64Bit, MaxEntriesInMemTable * 2),
                                        () => new TFReaderLease(readers),
                                        PTableVersions.Index64Bit,
                                        MaxEntriesInMemTable);
            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      readers,
                                      TableIndex,
                                      0,
                                      additionalCommitChecks: true,
                                      metastreamMaxCount: MetastreamMaxCount,
                                      hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault);
            ReadIndex.Init(ChaserCheckpoint.Read());
        }
 public override void TestFixtureSetUp()
 {
     base.TestFixtureSetUp();
     _tableIndex = new TableIndex(PathName,
                                  () => new HashListMemTable(maxSize: 20),
                                  () => { throw new InvalidOperationException(); },
                                  maxSizeForMemory: 10);
     _tableIndex.Initialize(long.MaxValue);
 }
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            WriterCheckpoint = new InMemoryCheckpoint(0);
            ChaserCheckpoint = new InMemoryCheckpoint(0);

            Bus = new InMemoryBus("bus");
            IODispatcher = new IODispatcher(Bus, new PublishEnvelope(Bus));

            Db = new TFChunkDb(new TFChunkDbConfig(PathName,
                                                   new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                                   10000,
                                                   0,
                                                   WriterCheckpoint,
                                                   ChaserCheckpoint,
                                                   new InMemoryCheckpoint(-1),
                                                   new InMemoryCheckpoint(-1)));

            Db.Open();
            // create db
            Writer = new TFChunkWriter(Db);
            Writer.Open();
            WriteTestScenario();
            Writer.Close();
            Writer = null;

            WriterCheckpoint.Flush();
            ChaserCheckpoint.Write(WriterCheckpoint.Read());
            ChaserCheckpoint.Flush();

            var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
            TableIndex = new TableIndex(GetFilePathFor("index"),
                                        () => new HashListMemTable(MaxEntriesInMemTable * 2),
                                        () => new TFReaderLease(readers),
                                        MaxEntriesInMemTable);

            var hasher = new ByLengthHasher();
            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      readers,
                                      TableIndex,
                                      hasher,
                                      0,
                                      additionalCommitChecks: true,
                                      metastreamMaxCount: MetastreamMaxCount);

            ReadIndex.Init(ChaserCheckpoint.Read());

            // scavenge must run after readIndex is built
            if (_scavenge)
            {
                if (_completeLastChunkOnScavenge)
                    Db.Manager.GetChunk(Db.Manager.ChunksCount - 1).Complete();
                _scavenger = new TFChunkScavenger(Db, IODispatcher, TableIndex, hasher, ReadIndex, Guid.NewGuid(), "fakeNodeIp");
                _scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: _mergeChunks);
            }
        }
 public override void TestFixtureSetUp()
 {
     base.TestFixtureSetUp();
     var lowHasher = new XXHashUnsafe();
     var highHasher = new Murmur3AUnsafe();
     _tableIndex = new TableIndex(PathName, lowHasher, highHasher,
                                  () => new HashListMemTable(_ptableVersion, maxSize: 20),
                                  () => { throw new InvalidOperationException(); },
                                  _ptableVersion,
                                  maxSizeForMemory: 10);
     _tableIndex.Initialize(long.MaxValue);
 }
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            ReadIndex.Close();
            ReadIndex.Dispose();

            Thread.Sleep(500);
            TableIndex.ClearAll(removeFiles: false);

            TableIndex = new TableIndex(Path.Combine(PathName, "index"), () => new HashListMemTable(), maxSizeForMemory: 5);
            TableIndex.Initialize();

            ReadIndex = new ReadIndex(new NoopPublisher(), 2, () => new TFChunkSequentialReader(Db, WriterCheckpoint, 0), () => new TFChunkReader(Db, WriterCheckpoint), TableIndex, new ByLengthHasher());
            ReadIndex.Build();
        }
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            var dbConfig = new TFChunkDbConfig(PathName,
                                               new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                               1024*1024,
                                               0,
                                               new InMemoryCheckpoint(0),
                                               new InMemoryCheckpoint(0),
                                               new InMemoryCheckpoint(-1),
                                               new InMemoryCheckpoint(-1));
            var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
            _dbResult = CreateDb(dbCreationHelper);
            _keptRecords = KeptRecords(_dbResult);

            _dbResult.Db.Config.WriterCheckpoint.Flush();
            _dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
            _dbResult.Db.Config.ChaserCheckpoint.Flush();

            var indexPath = Path.Combine(PathName, "index");
            var readerPool = new ObjectPool<ITransactionFileReader>(
                "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
                () => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint));
            var lowHasher = new XXHashUnsafe();
            var highHasher = new Murmur3AUnsafe();
            var tableIndex = new TableIndex(indexPath, lowHasher, highHasher,
                                            () => new HashListMemTable(PTableVersions.Index64Bit, maxSize: 200),
                                            () => new TFReaderLease(readerPool),
                                            PTableVersions.Index64Bit,
                                            maxSizeForMemory: 100,
                                            maxTablesPerLevel: 2);
            ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, 100, true, _metastreamMaxCount, Opts.HashCollisionReadLimitDefault);
            ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read());

            //var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount);
            var bus = new InMemoryBus("Bus");
            var ioDispatcher = new IODispatcher(bus, new PublishEnvelope(bus));
            var scavenger = new TFChunkScavenger(_dbResult.Db, ioDispatcher, tableIndex, ReadIndex, Guid.NewGuid(), "fakeNodeIp",
                                            unsafeIgnoreHardDeletes: UnsafeIgnoreHardDelete());
            scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false);
        }
        public void Setup(){
            given();
            _indexDir = PathName;
            _fakeReader = new TFReaderLease(new FakeReader());
            _indexBackend = new FakeIndexBackend(_fakeReader);
            _lowHasher = new XXHashUnsafe();
            _highHasher = new Murmur3AUnsafe();
            _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
                                         () => new HashListMemTable(PTableVersions.Index32Bit, maxSize: _maxMemTableSize),
                                         () => _fakeReader,
                                         PTableVersions.Index32Bit,
                                         maxSizeForMemory: _maxMemTableSize,
                                         maxTablesPerLevel: 2);
            _tableIndex.Initialize(long.MaxValue);
            _indexReader = new IndexReader(_indexBackend, _tableIndex, new EventStore.Core.Data.StreamMetadata(), _hashCollisionReadLimit);

            when();
            //wait for the mem table to be dumped
            System.Threading.Thread.Sleep(500);
        }
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            _indexDir = PathName;
            var fakeReader = new TFReaderLease(new FakeIndexReader());
            _lowHasher = new FakeIndexHasher();
            _highHasher = new FakeIndexHasher();
            _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
                                         () => new HashListMemTable(_ptableVersion, maxSize: 10),
                                         () => fakeReader,
                                         _ptableVersion,
                                         maxSizeForMemory: 2,
                                         maxTablesPerLevel: 2);
            _tableIndex.Initialize(long.MaxValue);

            // ptable level 2
            _tableIndex.Add(0, "1", 0, 0xFF00); 
            _tableIndex.Add(0, "1", 1, 0xFF01);
            _tableIndex.Add(0, "2", 0, 0xFF00);
            _tableIndex.Add(0, "2", 1, 0xFF01);
            _tableIndex.Add(0, "3", 0, 0xFF00);
            _tableIndex.Add(0, "3", 1, 0xFF01);
            _tableIndex.Add(0, "3", 0, 0xFF02);
            _tableIndex.Add(0, "3", 1, 0xFF03);

            // ptable level 1
            _tableIndex.Add(0, "4", 0, 0xFF00);
            _tableIndex.Add(0, "5", 10, 0xFFF1);
            _tableIndex.Add(0, "6", 0, 0xFF00);
            _tableIndex.Add(0, "1", 0, 0xFF10);

            // ptable level 0
            _tableIndex.Add(0, "6", 1, 0xFF01);
            _tableIndex.Add(0, "1", 1, 0xFF11);

            // memtable
            _tableIndex.Add(0, "4", 0, 0xFF01);

            Thread.Sleep(500);
        }
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            var dbConfig = new TFChunkDbConfig(PathName,
                                               new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                               1024*1024,
                                               0,
                                               new InMemoryCheckpoint(0),
                                               new InMemoryCheckpoint(0),
                                               new InMemoryCheckpoint(-1),
                                               new InMemoryCheckpoint(-1));
            var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
            
            DbRes = CreateDb(dbCreationHelper);

            DbRes.Db.Config.WriterCheckpoint.Flush();
            DbRes.Db.Config.ChaserCheckpoint.Write(DbRes.Db.Config.WriterCheckpoint.Read());
            DbRes.Db.Config.ChaserCheckpoint.Flush();

            var readers = new ObjectPool<ITransactionFileReader>(
                "Readers", 2, 2, () => new TFChunkReader(DbRes.Db, DbRes.Db.Config.WriterCheckpoint));

            var lowHasher = new XXHashUnsafe();
            var highHasher = new Murmur3AUnsafe();
            TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
                                        () => new HashListMemTable(PTableVersions.Index64Bit, MaxEntriesInMemTable * 2),
                                        () => new TFReaderLease(readers),
                                        PTableVersions.Index64Bit,
                                        MaxEntriesInMemTable);

            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      readers,
                                      TableIndex,
                                      0,
                                      additionalCommitChecks: true,
                                      metastreamMaxCount: _metastreamMaxCount,
                                      hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault);

            ReadIndex.Init(DbRes.Db.Config.ChaserCheckpoint.Read());
        }
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            ReadIndex.Close();
            ReadIndex.Dispose();
            TableIndex.Close(removeFiles: false);

            TableIndex = new TableIndex(GetFilePathFor("index"),
                                        () => new HashListMemTable(maxSize: MaxEntriesInMemTable*2),
                                        maxSizeForMemory: MaxEntriesInMemTable);

            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      2,
                                      () => new TFChunkSequentialReader(Db, WriterChecksum, 0),
                                      () => new TFChunkReader(Db, WriterChecksum),
                                      TableIndex,
                                      new ByLengthHasher(),
                                      new NoLRUCache<string, StreamCacheInfo>());
            ReadIndex.Build();
        }
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            _indexDir = PathName;
            var fakeReader = new TFReaderLease(new FakeIndexReader());
            _lowHasher = new XXHashUnsafe();
            _highHasher = new Murmur3AUnsafe();
            _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
                                         () => new HashListMemTable(PTableVersions.Index32Bit, maxSize: 5),
                                         () => fakeReader,
                                         PTableVersions.Index32Bit,
                                         maxSizeForMemory: 5,
                                         maxTablesPerLevel: 2);
            _tableIndex.Initialize(long.MaxValue);

            _tableIndex.Add(1, "LPN-FC002_LPK51001", 0, 1);
            _tableIndex.Add(1, "account--696193173", 0, 2);
            _tableIndex.Add(1, "LPN-FC002_LPK51001", 1, 3);
            _tableIndex.Add(1, "account--696193173", 1, 4);
            _tableIndex.Add(1, "LPN-FC002_LPK51001", 2, 5);

            _tableIndex.Close(false);

            _tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
                                         () => new HashListMemTable(PTableVersions.Index64Bit, maxSize: 5),
                                         () => fakeReader,
                                         PTableVersions.Index64Bit,
                                         maxSizeForMemory: 5,
                                         maxTablesPerLevel: 2);
            _tableIndex.Initialize(long.MaxValue);

            _tableIndex.Add(1, "account--696193173", 2, 6);
            _tableIndex.Add(1, "LPN-FC002_LPK51001", 3, 7);
            _tableIndex.Add(1, "account--696193173", 3, 8);
            _tableIndex.Add(1, "LPN-FC002_LPK51001", 4, 9);
            _tableIndex.Add(1, "account--696193173", 4, 10);

            Thread.Sleep(500);
        }
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            ReadIndex.Close();
            ReadIndex.Dispose();
            TableIndex.Close(removeFiles: false);

            var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 2, () => new TFChunkReader(Db, WriterCheckpoint));
            TableIndex = new TableIndex(GetFilePathFor("index"),
                                        () => new HashListMemTable(maxSize: MaxEntriesInMemTable*2),
                                        () => new TFReaderLease(readers),
                                        maxSizeForMemory: MaxEntriesInMemTable);
            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      readers,
                                      TableIndex,
                                      new ByLengthHasher(),
                                      0,
                                      additionalCommitChecks: true, 
                                      metastreamMaxCount: 1);
            ReadIndex.Init(ChaserCheckpoint.Read());
        }
        public void SetUp()
        {
            _indexDir = Path.Combine(Path.GetTempPath(), "idx-" + Guid.NewGuid().ToString());
            _tableIndex = new TableIndex(_indexDir, () => new HashListMemTable(), new InMemoryCheckpoint(), maxSizeForMemory: 2000);
            _tableIndex.Initialize();

            _tableIndex.Add(0xDEAD, 0, 0xFF00);
            _tableIndex.Add(0xDEAD, 1, 0xFF01); 

            _tableIndex.Add(0xBEEF, 0, 0xFF00);
            _tableIndex.Add(0xBEEF, 1, 0xFF01); 

            _tableIndex.Add(0xABBA, 0, 0xFF00);
            _tableIndex.Add(0xABBA, 1, 0xFF01); 
            _tableIndex.Add(0xABBA, 2, 0xFF02);
            _tableIndex.Add(0xABBA, 3, 0xFF03); 

            _tableIndex.Add(0xDEAD, 0, 0xFF10);
            _tableIndex.Add(0xDEAD, 1, 0xFF11); 

            _tableIndex.Add(0xADA, 0, 0xFF00);
        }
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            _indexDir = PathName;
            var fakeReader = new TFReaderLease(new FakeTfReader());
            _tableIndex = new TableIndex(_indexDir,
                                         () => new HashListMemTable(maxSize: 10),
                                         () => fakeReader,
                                         maxSizeForMemory: 2,
                                         maxTablesPerLevel: 2);
            _tableIndex.Initialize(long.MaxValue);

            // ptable level 2
            _tableIndex.Add(0, 0xDEAD, 0, 0xFF00); 
            _tableIndex.Add(0, 0xDEAD, 1, 0xFF01);
            _tableIndex.Add(0, 0xBEEF, 0, 0xFF00);
            _tableIndex.Add(0, 0xBEEF, 1, 0xFF01);
            _tableIndex.Add(0, 0xABBA, 0, 0xFF00);
            _tableIndex.Add(0, 0xABBA, 1, 0xFF01);
            _tableIndex.Add(0, 0xABBA, 0, 0xFF02);
            _tableIndex.Add(0, 0xABBA, 1, 0xFF03);

            // ptable level 1
            _tableIndex.Add(0, 0xADA, 0, 0xFF00);
            _tableIndex.Add(0, 0xCEED, 10, 0xFFF1);
            _tableIndex.Add(0, 0xBABA, 0, 0xFF00);
            _tableIndex.Add(0, 0xDEAD, 0, 0xFF10);

            // ptable level 0
            _tableIndex.Add(0, 0xBABA, 1, 0xFF01);
            _tableIndex.Add(0, 0xDEAD, 1, 0xFF11);

            // memtable
            _tableIndex.Add(0, 0xADA, 0, 0xFF01);

            Thread.Sleep(500);
        }
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            ReadIndex.Close();
            ReadIndex.Dispose();
            TableIndex.Close(removeFiles: false);

            TableIndex = new TableIndex(GetFilePathFor("index"),
                                        () => new HashListMemTable(maxSize: MaxEntriesInMemTable*2),
                                        maxSizeForMemory: MaxEntriesInMemTable);

            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      2,
                                      2,
                                      () => new TFChunkReader(Db, WriterCheckpoint, 0),
                                      TableIndex,
                                      new ByLengthHasher(),
                                      new NoLRUCache<string, StreamCacheInfo>(),
                                      additionalCommitChecks: true, 
                                      metastreamMaxCount: 1);
            ReadIndex.Init(WriterCheckpoint.Read(), ChaserCheckpoint.Read());
        }
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            _tableIndex = new TableIndex(PathName, () => new HashListMemTable(maxSize: 40), maxSizeForMemory: 20);
            _tableIndex.Initialize();

            _tableIndex.Add(0, 0xDEAD, 0, 0xFF00);
            _tableIndex.Add(0, 0xDEAD, 1, 0xFF01); 
                            
            _tableIndex.Add(0, 0xBEEF, 0, 0xFF00);
            _tableIndex.Add(0, 0xBEEF, 1, 0xFF01); 
                             
            _tableIndex.Add(0, 0xABBA, 0, 0xFF00);
            _tableIndex.Add(0, 0xABBA, 1, 0xFF01); 
            _tableIndex.Add(0, 0xABBA, 2, 0xFF02);
            _tableIndex.Add(0, 0xABBA, 3, 0xFF03); 
                             
            _tableIndex.Add(0, 0xDEAD, 0, 0xFF10);
            _tableIndex.Add(0, 0xDEAD, 1, 0xFF11); 
                             
            _tableIndex.Add(0, 0xADA, 0, 0xFF00);
        }
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            var dbConfig = new TFChunkDbConfig(PathName,
                                               new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                               1024*1024,
                                               0,
                                               new InMemoryCheckpoint(0),
                                               new InMemoryCheckpoint(0),
                                               new InMemoryCheckpoint(-1),
                                               new InMemoryCheckpoint(-1));
            var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
            _dbResult = CreateDb(dbCreationHelper);
            _keptRecords = KeptRecords(_dbResult);

            _dbResult.Db.Config.WriterCheckpoint.Flush();
            _dbResult.Db.Config.ChaserCheckpoint.Write(_dbResult.Db.Config.WriterCheckpoint.Read());
            _dbResult.Db.Config.ChaserCheckpoint.Flush();

            var indexPath = Path.Combine(PathName, "index");
            var readerPool = new ObjectPool<ITransactionFileReader>(
                "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
                () => new TFChunkReader(_dbResult.Db, _dbResult.Db.Config.WriterCheckpoint));
            var tableIndex = new TableIndex(indexPath,
                                            () => new HashListMemTable(maxSize: 200),
                                            () => new TFReaderLease(readerPool),
                                            maxSizeForMemory: 100,
                                            maxTablesPerLevel: 2);
            var hasher = new XXHashUnsafe();
            ReadIndex = new ReadIndex(new NoopPublisher(), readerPool, tableIndex, hasher, 100, true, _metastreamMaxCount);
            ReadIndex.Init(_dbResult.Db.Config.WriterCheckpoint.Read());

            //var scavengeReadIndex = new ScavengeReadIndex(_dbResult.Streams, _metastreamMaxCount);
            var scavenger = new TFChunkScavenger(_dbResult.Db, tableIndex, hasher, ReadIndex);
            scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false);
        }
        public override void SetUp()
        {
            base.SetUp();

            _indexDir = base.PathName;
            _tableIndex = new TableIndex(_indexDir, () => new HashListMemTable(), maxSizeForMemory: 2000);
            _tableIndex.Initialize();

            _tableIndex.Add(0, 0xDEAD, 0, 0xFF00);
            _tableIndex.Add(0, 0xDEAD, 1, 0xFF01);

            _tableIndex.Add(0, 0xBEEF, 0, 0xFF00);
            _tableIndex.Add(0, 0xBEEF, 1, 0xFF01);

            _tableIndex.Add(0, 0xABBA, 0, 0xFF00);
            _tableIndex.Add(0, 0xABBA, 1, 0xFF01);
            _tableIndex.Add(0, 0xABBA, 2, 0xFF02);
            _tableIndex.Add(0, 0xABBA, 3, 0xFF03);

            _tableIndex.Add(0, 0xDEAD, 0, 0xFF10);
            _tableIndex.Add(0, 0xDEAD, 1, 0xFF11);

            _tableIndex.Add(0, 0xADA, 0, 0xFF00);
        }
        public void SetUp()
        {
            _indexDir = Path.Combine(Path.GetTempPath(), "idx-" + Guid.NewGuid().ToString());
            _tableIndex = new TableIndex(_indexDir, () => new HashListMemTable(), new InMemoryCheckpoint(), maxSizeForMemory: 5);
            _tableIndex.Initialize();

            _tableIndex.Add(0xDEAD, 0, 0xFF00);
            _tableIndex.Add(0xDEAD, 1, 0xFF01); 

            _tableIndex.Add(0xBEEF, 0, 0xFF00);
            _tableIndex.Add(0xBEEF, 1, 0xFF01); 

            _tableIndex.Add(0xABBA, 0, 0xFF00); // 1st ptable0

            _tableIndex.Add(0xABBA, 1, 0xFF01); 
            _tableIndex.Add(0xABBA, 2, 0xFF02);
            _tableIndex.Add(0xABBA, 3, 0xFF03); 

            _tableIndex.Add(0xADA, 0, 0xFF00); // simulates duplicate due to concurrency in TableIndex (see memtable below)
            _tableIndex.Add(0xDEAD, 0, 0xFF10); // 2nd ptable0
            
            _tableIndex.Add(0xDEAD, 1, 0xFF11); // in memtable
            _tableIndex.Add(0xADA, 0, 0xFF00); // in memtable
        }
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            var dbConfig = new TFChunkDbConfig(PathName,
                                               new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                               1024*1024,
                                               0,
                                               new InMemoryCheckpoint(0),
                                               new InMemoryCheckpoint(0),
                                               new InMemoryCheckpoint(-1),
                                               new InMemoryCheckpoint(-1));
            var dbCreationHelper = new TFChunkDbCreationHelper(dbConfig);
            
            DbRes = CreateDb(dbCreationHelper);

            DbRes.Db.Config.WriterCheckpoint.Flush();
            DbRes.Db.Config.ChaserCheckpoint.Write(DbRes.Db.Config.WriterCheckpoint.Read());
            DbRes.Db.Config.ChaserCheckpoint.Flush();

            TableIndex = new TableIndex(GetFilePathFor("index"),
                                        () => new HashListMemTable(MaxEntriesInMemTable * 2),
                                        MaxEntriesInMemTable);

            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      2,
                                      2,
                                      () => new TFChunkReader(DbRes.Db, DbRes.Db.Config.WriterCheckpoint),
                                      TableIndex,
                                      new ByLengthHasher(),
                                      new NoLRUCache<string, StreamCacheInfo>(),
                                      additionalCommitChecks: true,
                                      metastreamMaxCount: _metastreamMaxCount);

            ReadIndex.Init(DbRes.Db.Config.WriterCheckpoint.Read(), DbRes.Db.Config.ChaserCheckpoint.Read());
        }
Beispiel #28
0
        public SingleVNode(TFChunkDb db, SingleVNodeSettings vNodeSettings, SingleVNodeAppSettings appSettings)
        {
            Ensure.NotNull(db, "db");
            Ensure.NotNull(vNodeSettings, "vNodeSettings");

            db.OpenVerifyAndClean();

            _tcpEndPoint = vNodeSettings.ExternalTcpEndPoint;
            _httpEndPoint = vNodeSettings.HttpEndPoint;

            _outputBus = new InMemoryBus("OutputBus");
            _controller = new SingleVNodeController(Bus, _httpEndPoint);
            _mainQueue = new QueuedHandler(_controller, "MainQueue");
            _controller.SetMainQueue(MainQueue);

            //MONITORING
            var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
            var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
            var monitoringQueue = new QueuedHandler(monitoringInnerBus, "MonitoringQueue", watchSlowMsg: true, slowMsgThresholdMs: 100);
            var monitoring = new MonitoringService(monitoringQueue, monitoringRequestBus, db.Config.WriterCheckpoint, appSettings.StatsPeriod);
            Bus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.SystemInit, Message>());
            Bus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
            monitoringInnerBus.Subscribe<SystemMessage.SystemInit>(monitoring);
            monitoringInnerBus.Subscribe<SystemMessage.BecomeShuttingDown>(monitoring);
            monitoringInnerBus.Subscribe<MonitoringMessage.GetFreshStats>(monitoring);

            //STORAGE SUBSYSTEM
            var indexPath = Path.Combine(db.Config.Path, "index");
            var tableIndex = new TableIndex(indexPath,
                                            () => new HashListMemTable(),
                                            new InMemoryCheckpoint(),
                                            maxSizeForMemory: 1000000,
                                            maxTablesPerLevel: 2);

            var readIndex = new ReadIndex(_mainQueue,
                                          db,
                                          () => new TFChunkReader(db, db.Config.WriterCheckpoint),
                                          TFConsts.ReadIndexReaderCount,
                                          db.Config.WriterCheckpoint,
                                          tableIndex,
                                          new XXHashUnsafe());
            var writer = new TFChunkWriter(db);
            var storageWriter = new StorageWriter(_mainQueue, _outputBus, writer, readIndex);
            var storageReader = new StorageReader(_mainQueue, _outputBus, readIndex, TFConsts.StorageReaderHandlerCount);
            monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageReader);

            var chaser = new TFChunkChaser(db,
                                           db.Config.WriterCheckpoint,
                                           db.Config.GetNamedCheckpoint(Checkpoint.Chaser));
            var storageChaser = new StorageChaser(_mainQueue, chaser);
            _outputBus.Subscribe<SystemMessage.SystemInit>(storageChaser);
            _outputBus.Subscribe<SystemMessage.SystemStart>(storageChaser);
            _outputBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageChaser);

            var storageScavenger = new StorageScavenger(db, readIndex);
            _outputBus.Subscribe<SystemMessage.ScavengeDatabase>(storageScavenger);

            //TCP
            var tcpService = new TcpService(MainQueue, _tcpEndPoint);
            Bus.Subscribe<SystemMessage.SystemInit>(tcpService);
            Bus.Subscribe<SystemMessage.SystemStart>(tcpService);
            Bus.Subscribe<SystemMessage.BecomeShuttingDown>(tcpService);

            //HTTP
            HttpService = new HttpService(MainQueue, _httpEndPoint.ToHttpUrl());
            Bus.Subscribe<SystemMessage.SystemInit>(HttpService);
            Bus.Subscribe<SystemMessage.BecomeShuttingDown>(HttpService);
            Bus.Subscribe<HttpMessage.SendOverHttp>(HttpService);
            Bus.Subscribe<HttpMessage.UpdatePendingRequests>(HttpService);
            HttpService.SetupController(new AdminController(MainQueue));
            HttpService.SetupController(new PingController());
            HttpService.SetupController(new StatController(monitoringQueue));
            HttpService.SetupController(new ReadEventDataController(MainQueue));
            HttpService.SetupController(new AtomController(MainQueue));
            HttpService.SetupController(new WebSiteController(MainQueue));

            //REQUEST MANAGEMENT
            var requestManagement = new RequestManagementService(MainQueue, 1, 1);
            Bus.Subscribe<ReplicationMessage.EventCommited>(requestManagement);
            Bus.Subscribe<ReplicationMessage.CreateStreamRequestCreated>(requestManagement);
            Bus.Subscribe<ReplicationMessage.WriteRequestCreated>(requestManagement);
            Bus.Subscribe<ReplicationMessage.TransactionStartRequestCreated>(requestManagement);
            Bus.Subscribe<ReplicationMessage.TransactionWriteRequestCreated>(requestManagement);
            Bus.Subscribe<ReplicationMessage.TransactionCommitRequestCreated>(requestManagement);
            Bus.Subscribe<ReplicationMessage.DeleteStreamRequestCreated>(requestManagement);
            Bus.Subscribe<ReplicationMessage.RequestCompleted>(requestManagement);
            Bus.Subscribe<ReplicationMessage.CommitAck>(requestManagement);
            Bus.Subscribe<ReplicationMessage.PrepareAck>(requestManagement);
            Bus.Subscribe<ReplicationMessage.WrongExpectedVersion>(requestManagement);
            Bus.Subscribe<ReplicationMessage.InvalidTransaction>(requestManagement);
            Bus.Subscribe<ReplicationMessage.StreamDeleted>(requestManagement);
            Bus.Subscribe<ReplicationMessage.PreparePhaseTimeout>(requestManagement);
            Bus.Subscribe<ReplicationMessage.CommitPhaseTimeout>(requestManagement);

            var clientService = new ClientService();
            Bus.Subscribe<TcpMessage.ConnectionClosed>(clientService);
            Bus.Subscribe<ClientMessage.SubscribeToStream>(clientService);
            Bus.Subscribe<ClientMessage.UnsubscribeFromStream>(clientService);
            Bus.Subscribe<ClientMessage.SubscribeToAllStreams>(clientService);
            Bus.Subscribe<ClientMessage.UnsubscribeFromAllStreams>(clientService);
            Bus.Subscribe<ReplicationMessage.EventCommited>(clientService);

            //TIMER
            //var timer = new TimerService(new TimerBasedScheduler(new RealTimer(), new RealTimeProvider()));
            TimerService = new TimerService(new ThreadBasedScheduler(new RealTimeProvider()));
            Bus.Subscribe<TimerMessage.Schedule>(TimerService);

            MainQueue.Start();
            monitoringQueue.Start();
        }
        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            WriterCheckpoint = new InMemoryCheckpoint(0);
            ChaserCheckpoint = new InMemoryCheckpoint(0);

            Db = new TFChunkDb(new TFChunkDbConfig(PathName,
                                                   new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                                   10000,
                                                   0,
                                                   WriterCheckpoint,
                                                   ChaserCheckpoint,
                                                   new InMemoryCheckpoint(-1),
                                                   new InMemoryCheckpoint(-1)));

            Db.Open();
            // create db
            Writer = new TFChunkWriter(Db);
            Writer.Open();
            WriteTestScenario();
            Writer.Close();
            Writer = null;

            WriterCheckpoint.Flush();
            ChaserCheckpoint.Write(WriterCheckpoint.Read());
            ChaserCheckpoint.Flush();

            TableIndex = new TableIndex(GetFilePathFor("index"),
                                        () => new HashListMemTable(MaxEntriesInMemTable * 2),
                                        MaxEntriesInMemTable);

            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      2,
                                      2,
                                      () => new TFChunkReader(Db, Db.Config.WriterCheckpoint),
                                      TableIndex,
                                      new ByLengthHasher(),
                                      new NoLRUCache<string, StreamCacheInfo>(),
                                      additionalCommitChecks: true,
                                      metastreamMaxCount: MetastreamMaxCount);

            ReadIndex.Init(WriterCheckpoint.Read(), ChaserCheckpoint.Read());

            // scavenge must run after readIndex is built
            if (_scavenge)
            {
                if (_completeLastChunkOnScavenge)
                    Db.Manager.GetChunk(Db.Manager.ChunksCount - 1).Complete();
                _scavenger = new TFChunkScavenger(Db, ReadIndex);
                _scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: _mergeChunks);
            }
        }
Beispiel #30
0
        public SingleVNode(TFChunkDb db, 
                           SingleVNodeSettings vNodeSettings, 
                           bool dbVerifyHashes, 
                           int memTableEntryCount, 
                           params ISubsystem[] subsystems)
        {
            Ensure.NotNull(db, "db");
            Ensure.NotNull(vNodeSettings, "vNodeSettings");

            _settings = vNodeSettings;

            _mainBus = new InMemoryBus("MainBus");
            _controller = new SingleVNodeController(_mainBus, _settings.ExternalHttpEndPoint, db, this);
            _mainQueue = new QueuedHandler(_controller, "MainQueue");
            _controller.SetMainQueue(_mainQueue);

            _subsystems = subsystems;

            // MONITORING
            var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
            var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
            var monitoringQueue = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
            var monitoring = new MonitoringService(monitoringQueue,
                                                   monitoringRequestBus,
                                                   _mainQueue,
                                                   db.Config.WriterCheckpoint,
                                                   db.Config.Path,
                                                   vNodeSettings.StatsPeriod,
                                                   _settings.ExternalHttpEndPoint,
                                                   vNodeSettings.StatsStorage);
            _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.SystemInit, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.StateChangeMessage, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShutdown, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom<ClientMessage.WriteEventsCompleted, Message>());
            monitoringInnerBus.Subscribe<SystemMessage.SystemInit>(monitoring);
            monitoringInnerBus.Subscribe<SystemMessage.StateChangeMessage>(monitoring);
            monitoringInnerBus.Subscribe<SystemMessage.BecomeShuttingDown>(monitoring);
            monitoringInnerBus.Subscribe<SystemMessage.BecomeShutdown>(monitoring);
            monitoringInnerBus.Subscribe<ClientMessage.WriteEventsCompleted>(monitoring);
            monitoringInnerBus.Subscribe<MonitoringMessage.GetFreshStats>(monitoring);

            var truncPos = db.Config.TruncateCheckpoint.Read();
            if (truncPos != -1)
            {
                Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})",
                         truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read());
                var truncator = new TFChunkDbTruncator(db.Config);
                truncator.TruncateDb(truncPos);
            }

            db.Open(dbVerifyHashes);

            // STORAGE SUBSYSTEM
            var indexPath = Path.Combine(db.Config.Path, "index");
            var readerPool = new ObjectPool<ITransactionFileReader>(
                "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
                () => new TFChunkReader(db, db.Config.WriterCheckpoint));
            var tableIndex = new TableIndex(indexPath,
                                            () => new HashListMemTable(maxSize: memTableEntryCount * 2),
                                            () => new TFReaderLease(readerPool),
                                            maxSizeForMemory: memTableEntryCount,
                                            maxTablesPerLevel: 2,
                                            inMem: db.Config.InMemDb);
            var hasher = new XXHashUnsafe();
            var readIndex = new ReadIndex(_mainQueue,
                                          readerPool,
                                          tableIndex,
                                          hasher,
                                          ESConsts.StreamInfoCacheCapacity,
                                          Application.IsDefined(Application.AdditionalCommitChecks),
                                          Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1);
            var writer = new TFChunkWriter(db);
            var epochManager = new EpochManager(ESConsts.CachedEpochCount,
                                                db.Config.EpochCheckpoint,
                                                writer,
                                                initialReaderCount: 1,
                                                maxReaderCount: 5,
                                                readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint));
            epochManager.Init();

            var storageWriter = new StorageWriterService(_mainQueue, _mainBus, _settings.MinFlushDelay,
                                                         db, writer, readIndex.IndexWriter, epochManager); // subscribes internally
            monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageWriter);

            var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, ESConsts.StorageReaderThreadCount, db.Config.WriterCheckpoint);
            _mainBus.Subscribe<SystemMessage.SystemInit>(storageReader);
            _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageReader);
            _mainBus.Subscribe<SystemMessage.BecomeShutdown>(storageReader);
            monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageReader);

            var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
            var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, readIndex.IndexCommitter, epochManager);
            _mainBus.Subscribe<SystemMessage.SystemInit>(storageChaser);
            _mainBus.Subscribe<SystemMessage.SystemStart>(storageChaser);
            _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageChaser);

            var storageScavenger = new StorageScavenger(db,
                                                        tableIndex,
                                                        hasher,
                                                        readIndex,
                                                        Application.IsDefined(Application.AlwaysKeepScavenged),
                                                        mergeChunks: false /*!Application.IsDefined(Application.DisableMergeChunks)*/);
            // ReSharper disable RedundantTypeArgumentsOfMethod
            _mainBus.Subscribe<ClientMessage.ScavengeDatabase>(storageScavenger);
            // ReSharper restore RedundantTypeArgumentsOfMethod

            // MISC WORKERS
            _workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
                new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
                                watchSlowMsg: true,
                                slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray();
            _workersHandler = new MultiQueuedHandler(
                    vNodeSettings.WorkerThreads,
                    queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
                                                            string.Format("Worker #{0}", queueNum + 1),
                                                            groupName: "Workers",
                                                            watchSlowMsg: true,
                                                            slowMsgThreshold: TimeSpan.FromMilliseconds(200)));

            // AUTHENTICATION INFRASTRUCTURE
            var passwordHashAlgorithm = new Rfc2898PasswordHashAlgorithm();
            var dispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_workersHandler, crossThread: true));
            var internalAuthenticationProvider = new InternalAuthenticationProvider(dispatcher, passwordHashAlgorithm, ESConsts.CachedPrincipalCount);
            var passwordChangeNotificationReader = new PasswordChangeNotificationReader(_mainQueue, dispatcher);
            _mainBus.Subscribe<SystemMessage.SystemStart>(passwordChangeNotificationReader);
            _mainBus.Subscribe<SystemMessage.BecomeShutdown>(passwordChangeNotificationReader);
            _mainBus.Subscribe(internalAuthenticationProvider);
            SubscribeWorkers(bus =>
            {
                bus.Subscribe(dispatcher.ForwardReader);
                bus.Subscribe(dispatcher.BackwardReader);
                bus.Subscribe(dispatcher.Writer);
                bus.Subscribe(dispatcher.StreamDeleter);
                bus.Subscribe(dispatcher);
            });

            // TCP
            {
                var tcpService = new TcpService(_mainQueue, _settings.ExternalTcpEndPoint, _workersHandler,
                                                TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
                                                ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
                                                internalAuthenticationProvider, null);
                _mainBus.Subscribe<SystemMessage.SystemInit>(tcpService);
                _mainBus.Subscribe<SystemMessage.SystemStart>(tcpService);
                _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(tcpService);
            }

            // SECURE TCP
            if (vNodeSettings.ExternalSecureTcpEndPoint != null)
            {
                var secureTcpService = new TcpService(_mainQueue, _settings.ExternalSecureTcpEndPoint, _workersHandler,
                                                      TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(),
                                                      ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
                                                      internalAuthenticationProvider, _settings.Certificate);
                _mainBus.Subscribe<SystemMessage.SystemInit>(secureTcpService);
                _mainBus.Subscribe<SystemMessage.SystemStart>(secureTcpService);
                _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(secureTcpService);
            }

            SubscribeWorkers(bus =>
            {
                var tcpSendService = new TcpSendService();
                // ReSharper disable RedundantTypeArgumentsOfMethod
                bus.Subscribe<TcpMessage.TcpSend>(tcpSendService);
                // ReSharper restore RedundantTypeArgumentsOfMethod
            });

            // HTTP
            var httpAuthenticationProviders = new List<HttpAuthenticationProvider>
            {
                new BasicHttpAuthenticationProvider(internalAuthenticationProvider),
            };
            if (_settings.EnableTrustedAuth)
                httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider());
            httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider());

            var httpPipe = new HttpMessagePipe();
            var httpSendService = new HttpSendService(httpPipe, forwardRequests: false);
            _mainBus.Subscribe<SystemMessage.StateChangeMessage>(httpSendService);
            _mainBus.Subscribe(new WideningHandler<HttpMessage.SendOverHttp, Message>(_workersHandler));
            SubscribeWorkers(bus =>
            {
                bus.Subscribe<HttpMessage.HttpSend>(httpSendService);
                bus.Subscribe<HttpMessage.HttpSendPart>(httpSendService);
                bus.Subscribe<HttpMessage.HttpBeginSend>(httpSendService);
                bus.Subscribe<HttpMessage.HttpEndSend>(httpSendService);
                bus.Subscribe<HttpMessage.SendOverHttp>(httpSendService);
            });

            _httpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(), 
                                            _workersHandler, vNodeSettings.HttpPrefixes);
            _httpService.SetupController(new AdminController(_mainQueue));
            _httpService.SetupController(new PingController());
            _httpService.SetupController(new StatController(monitoringQueue, _workersHandler));
            _httpService.SetupController(new AtomController(httpSendService, _mainQueue, _workersHandler));
            _httpService.SetupController(new GuidController(_mainQueue));
            _httpService.SetupController(new UsersController(httpSendService, _mainQueue, _workersHandler));

            _mainBus.Subscribe<SystemMessage.SystemInit>(_httpService);
            _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(_httpService);
            _mainBus.Subscribe<HttpMessage.PurgeTimedOutRequests>(_httpService);

            SubscribeWorkers(bus =>
            {
                HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray());
            });

            // REQUEST MANAGEMENT
            var requestManagement = new RequestManagementService(_mainQueue, 1, 1, vNodeSettings.PrepareTimeout, vNodeSettings.CommitTimeout);
            _mainBus.Subscribe<SystemMessage.SystemInit>(requestManagement);
            _mainBus.Subscribe<ClientMessage.WriteEvents>(requestManagement);
            _mainBus.Subscribe<ClientMessage.TransactionStart>(requestManagement);
            _mainBus.Subscribe<ClientMessage.TransactionWrite>(requestManagement);
            _mainBus.Subscribe<ClientMessage.TransactionCommit>(requestManagement);
            _mainBus.Subscribe<ClientMessage.DeleteStream>(requestManagement);
            _mainBus.Subscribe<StorageMessage.RequestCompleted>(requestManagement);
            _mainBus.Subscribe<StorageMessage.CheckStreamAccessCompleted>(requestManagement);
            _mainBus.Subscribe<StorageMessage.AlreadyCommitted>(requestManagement);
            _mainBus.Subscribe<StorageMessage.CommitAck>(requestManagement);
            _mainBus.Subscribe<StorageMessage.PrepareAck>(requestManagement);
            _mainBus.Subscribe<StorageMessage.WrongExpectedVersion>(requestManagement);
            _mainBus.Subscribe<StorageMessage.InvalidTransaction>(requestManagement);
            _mainBus.Subscribe<StorageMessage.StreamDeleted>(requestManagement);
            _mainBus.Subscribe<StorageMessage.RequestManagerTimerTick>(requestManagement);

            // SUBSCRIPTIONS
            var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
            var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false);
            _mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.SystemStart, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<TcpMessage.ConnectionClosed, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.SubscribeToStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.UnsubscribeFromStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.PollStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.CheckPollTimeout, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<StorageMessage.EventCommited, Message>());

            var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex);
            subscrBus.Subscribe<SystemMessage.SystemStart>(subscription);
            subscrBus.Subscribe<SystemMessage.BecomeShuttingDown>(subscription);
            subscrBus.Subscribe<TcpMessage.ConnectionClosed>(subscription);
            subscrBus.Subscribe<ClientMessage.SubscribeToStream>(subscription);
            subscrBus.Subscribe<ClientMessage.UnsubscribeFromStream>(subscription);
            subscrBus.Subscribe<SubscriptionMessage.PollStream>(subscription);
            subscrBus.Subscribe<SubscriptionMessage.CheckPollTimeout>(subscription);
            subscrBus.Subscribe<StorageMessage.EventCommited>(subscription);

            // USER MANAGEMENT
            var ioDispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_mainQueue));
            _mainBus.Subscribe(ioDispatcher.BackwardReader);
            _mainBus.Subscribe(ioDispatcher.ForwardReader);
            _mainBus.Subscribe(ioDispatcher.Writer);
            _mainBus.Subscribe(ioDispatcher.StreamDeleter);
            _mainBus.Subscribe(ioDispatcher);

            var userManagement = new UserManagementService(
                _mainQueue, ioDispatcher, passwordHashAlgorithm, vNodeSettings.SkipInitializeStandardUsersCheck);
            _mainBus.Subscribe<UserManagementMessage.Create>(userManagement);
            _mainBus.Subscribe<UserManagementMessage.Update>(userManagement);
            _mainBus.Subscribe<UserManagementMessage.Enable>(userManagement);
            _mainBus.Subscribe<UserManagementMessage.Disable>(userManagement);
            _mainBus.Subscribe<UserManagementMessage.Delete>(userManagement);
            _mainBus.Subscribe<UserManagementMessage.ResetPassword>(userManagement);
            _mainBus.Subscribe<UserManagementMessage.ChangePassword>(userManagement);
            _mainBus.Subscribe<UserManagementMessage.Get>(userManagement);
            _mainBus.Subscribe<UserManagementMessage.GetAll>(userManagement);
            _mainBus.Subscribe<SystemMessage.BecomeMaster>(userManagement);

            // TIMER
            _timeProvider = new RealTimeProvider();
            _timerService = new TimerService(new ThreadBasedScheduler(_timeProvider));
            _mainBus.Subscribe<SystemMessage.BecomeShutdown>(_timerService);
            _mainBus.Subscribe<TimerMessage.Schedule>(_timerService);

            _workersHandler.Start();
            _mainQueue.Start();
            monitoringQueue.Start();
            subscrQueue.Start();

            if (subsystems != null)
                foreach (var subsystem in subsystems)
                    subsystem.Register(db, _mainQueue, _mainBus, _timerService, _timeProvider, httpSendService, new[]{_httpService}, _workersHandler);
        }