Ejemplo n.º 1
0
        public void SetUp(Action <IHttpService> bootstrap = null)
        {
            _bus = new InMemoryBus($"bus_{_serverEndPoint.Port}");
            var pipelineBus = InMemoryBus.CreateTest();
            var queue       = new QueuedHandlerThreadPool(pipelineBus, "Test", new QueueStatsManager(), true, TimeSpan.FromMilliseconds(50));

            _multiQueuedHandler = new MultiQueuedHandler(new IQueuedHandler[] { queue }, null);
            _multiQueuedHandler.Start();

            _service = new KestrelHttpService(ServiceAccessibility.Private, _bus, new NaiveUriRouter(),
                                              _multiQueuedHandler, false, null, 0, false, _serverEndPoint);
            _internalDispatcher = new InternalDispatcherEndpoint(queue, _multiQueuedHandler);
            _bus.Subscribe(_internalDispatcher);
            KestrelHttpService.CreateAndSubscribePipeline(pipelineBus);
            bootstrap?.Invoke(_service);
            _server = new TestServer(
                new WebHostBuilder()
                .UseStartup(new ClusterVNodeStartup(Array.Empty <ISubsystem>(), queue, _bus, _multiQueuedHandler,
                                                    new TestAuthenticationProvider(),
                                                    new IHttpAuthenticationProvider[] {
                new BasicHttpAuthenticationProvider(new TestAuthenticationProvider()),
                new AnonymousHttpAuthenticationProvider(),
            }, new TestAuthorizationProvider(), new FakeReadIndex(_ => false), 1024 * 1024, _service)));
            _httpMessageHandler = _server.CreateHandler();
            _client             = new HttpAsyncClient(_timeout, _httpMessageHandler);

            HttpBootstrap.Subscribe(_bus, _service);
        }
Ejemplo n.º 2
0
        public void of_http_requests_routing()
        {
            const int iterations = 100000;

            IPublisher inputBus           = new NoopPublisher();
            var        bus                = InMemoryBus.CreateTest();
            var        queue              = new QueuedHandlerThreadPool(bus, "Test", true, TimeSpan.FromMilliseconds(50));
            var        multiQueuedHandler = new MultiQueuedHandler(new IQueuedHandler[] { queue }, null);
            var        providers          = new HttpAuthenticationProvider[] { new AnonymousHttpAuthenticationProvider() };
            var        httpService        = new HttpService(ServiceAccessibility.Public, inputBus,
                                                            new TrieUriRouter(), multiQueuedHandler, false, "http://localhost:12345/");

            HttpService.CreateAndSubscribePipeline(bus, providers);

            var fakeController = new FakeController(iterations, null);

            httpService.SetupController(fakeController);

            httpService.Handle(new SystemMessage.SystemInit());

            var rnd = new Random();
            var sw  = Stopwatch.StartNew();

            var httpClient = new HttpAsyncClient();

            for (int i = 0; i < iterations; ++i)
            {
                var route = fakeController.BoundRoutes[rnd.Next(0, fakeController.BoundRoutes.Count)];

                switch (route.Item2)
                {
                case HttpMethod.Get:
                    httpClient.Get(route.Item1, TimeSpan.FromMilliseconds(10000), x => { }, x => { throw new Exception(); });
                    break;

                case HttpMethod.Post:
                    httpClient.Post(route.Item1, "abracadabra", ContentType.Json, TimeSpan.FromMilliseconds(10000), x => { }, x => { throw new Exception(); });
                    break;

                case HttpMethod.Delete:
                    httpClient.Delete(route.Item1, TimeSpan.FromMilliseconds(10000), x => { }, x => { throw new Exception(); });
                    break;

                case HttpMethod.Put:
                    httpClient.Put(route.Item1, "abracadabra", ContentType.Json, TimeSpan.FromMilliseconds(10000), x => { }, x => { throw new Exception(); });
                    break;

                default:
                    throw new Exception();
                }
            }

            fakeController.CountdownEvent.Wait();
            sw.Stop();

            Console.WriteLine("{0} request done in {1} ({2:0.00} per sec)", iterations, sw.Elapsed, 1000.0 * iterations / sw.ElapsedMilliseconds);

            httpService.Shutdown();
            multiQueuedHandler.Stop();
        }
        public ClusterVNodeStartup(
            ISubsystem[] subsystems,
            IPublisher mainQueue,
            ISubscriber mainBus,
            MultiQueuedHandler httpMessageHandler,
            IReadOnlyList <IHttpAuthenticationProvider> httpAuthenticationProviders,
            IAuthorizationProvider authorizationProvider,
            IReadIndex readIndex,
            int maxAppendSize,
            KestrelHttpService httpService)
        {
            if (subsystems == null)
            {
                throw new ArgumentNullException(nameof(subsystems));
            }

            if (mainQueue == null)
            {
                throw new ArgumentNullException(nameof(mainQueue));
            }

            if (httpAuthenticationProviders == null)
            {
                throw new ArgumentNullException(nameof(httpAuthenticationProviders));
            }

            if (authorizationProvider == null)
            {
                throw new ArgumentNullException(nameof(authorizationProvider));
            }

            if (readIndex == null)
            {
                throw new ArgumentNullException(nameof(readIndex));
            }

            Ensure.Positive(maxAppendSize, nameof(maxAppendSize));

            if (httpService == null)
            {
                throw new ArgumentNullException(nameof(httpService));
            }

            if (mainBus == null)
            {
                throw new ArgumentNullException(nameof(mainBus));
            }
            _subsystems                  = subsystems;
            _mainQueue                   = mainQueue;
            _mainBus                     = mainBus;
            _httpMessageHandler          = httpMessageHandler;
            _httpAuthenticationProviders = httpAuthenticationProviders;
            _authorizationProvider       = authorizationProvider;
            _readIndex                   = readIndex;
            _maxAppendSize               = maxAppendSize;
            _httpService                 = httpService;

            _statusCheck = new StatusCheck(this);
        }
Ejemplo n.º 4
0
        public StorageReaderService(
            IPublisher bus,
            ISubscriber subscriber,
            IReadIndex readIndex,
            int threadCount,
            IReadOnlyCheckpoint writerCheckpoint,
            QueueStatsManager queueStatsManager)
        {
            Ensure.NotNull(bus, "bus");
            Ensure.NotNull(subscriber, "subscriber");
            Ensure.NotNull(readIndex, "readIndex");
            Ensure.Positive(threadCount, "threadCount");
            Ensure.NotNull(writerCheckpoint, "writerCheckpoint");

            _bus         = bus;
            _readIndex   = readIndex;
            _threadCount = threadCount;
            StorageReaderWorker[] readerWorkers      = new StorageReaderWorker[threadCount];
            InMemoryBus[]         storageReaderBuses = new InMemoryBus[threadCount];
            for (var i = 0; i < threadCount; i++)
            {
                readerWorkers[i]      = new StorageReaderWorker(bus, readIndex, writerCheckpoint, i);
                storageReaderBuses[i] = new InMemoryBus("StorageReaderBus", watchSlowMsg: false);
                storageReaderBuses[i].Subscribe <ClientMessage.ReadEvent>(readerWorkers[i]);
                storageReaderBuses[i].Subscribe <ClientMessage.ReadStreamEventsBackward>(readerWorkers[i]);
                storageReaderBuses[i].Subscribe <ClientMessage.ReadStreamEventsForward>(readerWorkers[i]);
                storageReaderBuses[i].Subscribe <ClientMessage.ReadAllEventsForward>(readerWorkers[i]);
                storageReaderBuses[i].Subscribe <ClientMessage.ReadAllEventsBackward>(readerWorkers[i]);
                storageReaderBuses[i].Subscribe <ClientMessage.FilteredReadAllEventsForward>(readerWorkers[i]);
                storageReaderBuses[i].Subscribe <ClientMessage.FilteredReadAllEventsBackward>(readerWorkers[i]);
                storageReaderBuses[i].Subscribe <StorageMessage.BatchLogExpiredMessages>(readerWorkers[i]);
                storageReaderBuses[i].Subscribe <StorageMessage.EffectiveStreamAclRequest>(readerWorkers[i]);
                storageReaderBuses[i].Subscribe <StorageMessage.StreamIdFromTransactionIdRequest>(readerWorkers[i]);
            }

            _workersMultiHandler = new MultiQueuedHandler(
                _threadCount,
                queueNum => new QueuedHandlerThreadPool(storageReaderBuses[queueNum],
                                                        string.Format("StorageReaderQueue #{0}", queueNum + 1),
                                                        queueStatsManager,
                                                        groupName: "StorageReaderQueue",
                                                        watchSlowMsg: true,
                                                        slowMsgThreshold: TimeSpan.FromMilliseconds(200)));
            _workersMultiHandler.Start();

            subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadEvent, Message>());
            subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadStreamEventsBackward, Message>());
            subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadStreamEventsForward, Message>());
            subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadAllEventsForward, Message>());
            subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadAllEventsBackward, Message>());
            subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.FilteredReadAllEventsForward, Message>());
            subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.FilteredReadAllEventsBackward, Message>());
            subscriber.Subscribe(_workersMultiHandler.WidenFrom <StorageMessage.BatchLogExpiredMessages, Message>());
            subscriber.Subscribe(_workersMultiHandler.WidenFrom <StorageMessage.EffectiveStreamAclRequest, Message>());
            subscriber.Subscribe(_workersMultiHandler.WidenFrom <StorageMessage.StreamIdFromTransactionIdRequest, Message>());
        }
Ejemplo n.º 5
0
        public HttpService(ServiceAccessibility accessibility, IPublisher inputBus, IUriRouter uriRouter,
                           MultiQueuedHandler multiQueuedHandler, params string[] prefixes)
        {
            Ensure.NotNull(inputBus, "inputBus");
            Ensure.NotNull(uriRouter, "uriRouter");
            Ensure.NotNull(prefixes, "prefixes");

            _accessibility   = accessibility;
            _inputBus        = inputBus;
            _uriRouter       = uriRouter;
            _publishEnvelope = new PublishEnvelope(inputBus);

            _requestsMultiHandler = multiQueuedHandler;

            _server = new HttpAsyncServer(prefixes);
            _server.RequestReceived += RequestReceived;
        }
Ejemplo n.º 6
0
        public void SetUp()
        {
            _bus = new InMemoryBus(string.Format("bus_{0}", _serverEndPoint.Port));

            {
                var pipelineBus = InMemoryBus.CreateTest();
                var queue       = new QueuedHandlerThreadPool(pipelineBus, "Test", true, TimeSpan.FromMilliseconds(50));
                _multiQueuedHandler = new MultiQueuedHandler(new IQueuedHandler[] { queue }, null);
                var httpAuthenticationProviders = new HttpAuthenticationProvider[] { new AnonymousHttpAuthenticationProvider() };

                _service = new HttpService(ServiceAccessibility.Private, _bus, new NaiveUriRouter(),
                                           _multiQueuedHandler, false, _serverEndPoint.ToHttpUrl());
                HttpService.CreateAndSubscribePipeline(pipelineBus, httpAuthenticationProviders);
                _client = new HttpAsyncClient();
            }

            HttpBootstrap.Subscribe(_bus, _service);
        }
Ejemplo n.º 7
0
        public KestrelHttpService(ServiceAccessibility accessibility, IPublisher inputBus, IUriRouter uriRouter,
                                  MultiQueuedHandler multiQueuedHandler, bool logHttpRequests, IPAddress advertiseAsAddress,
                                  int advertiseAsPort, bool disableAuthorization, params IPEndPoint[] endPoints)
        {
            Ensure.NotNull(inputBus, nameof(inputBus));
            Ensure.NotNull(uriRouter, nameof(uriRouter));
            Ensure.NotNull(endPoints, nameof(endPoints));

            _accessibility  = accessibility;
            _inputBus       = inputBus;
            UriRouter       = uriRouter;
            LogHttpRequests = logHttpRequests;

            AdvertiseAsAddress = advertiseAsAddress;
            AdvertiseAsPort    = advertiseAsPort;


            EndPoints = endPoints;
        }
Ejemplo n.º 8
0
        public void SetUp()
        {
            _bus = new InMemoryBus(string.Format("bus_{0}", _serverEndPoint.Port));

            {

                var pipelineBus = InMemoryBus.CreateTest();
                var queue = new QueuedHandlerThreadPool(pipelineBus, "Test", true, TimeSpan.FromMilliseconds(50));
                _multiQueuedHandler = new MultiQueuedHandler(new IQueuedHandler[]{queue}, null);
                var httpAuthenticationProviders = new HttpAuthenticationProvider[] {new AnonymousHttpAuthenticationProvider()};

                _service = new HttpService(ServiceAccessibility.Private, _bus, new NaiveUriRouter(),
                                           _multiQueuedHandler, _serverEndPoint.ToHttpUrl());
                HttpService.CreateAndSubscribePipeline(pipelineBus, httpAuthenticationProviders);
                _client = new HttpAsyncClient();
            }

            HttpBootstrap.Subscribe(_bus, _service);
        }
Ejemplo n.º 9
0
        public HttpService(ServiceAccessibility accessibility, IPublisher inputBus, IUriRouter uriRouter,
                           MultiQueuedHandler multiQueuedHandler, bool logHttpRequests, IPAddress advertiseAsAddress, int advertiseAsPort, params string[] prefixes)
        {
            Ensure.NotNull(inputBus, "inputBus");
            Ensure.NotNull(uriRouter, "uriRouter");
            Ensure.NotNull(prefixes, "prefixes");

            _accessibility   = accessibility;
            _inputBus        = inputBus;
            _uriRouter       = uriRouter;
            _publishEnvelope = new PublishEnvelope(inputBus);

            _requestsMultiHandler = multiQueuedHandler;
            _logHttpRequests      = logHttpRequests;

            _server = new HttpAsyncServer(prefixes);
            _server.RequestReceived += RequestReceived;

            _advertiseAsAddress = advertiseAsAddress;
            _advertiseAsPort    = advertiseAsPort;
        }
Ejemplo n.º 10
0
        public void SetUp(Action <IHttpService> bootstrap = null)
        {
            _bus = new InMemoryBus($"bus_{_serverEndPoint.Port}");
            var pipelineBus = InMemoryBus.CreateTest();
            var queue       = new QueuedHandlerThreadPool(pipelineBus, "Test", new QueueStatsManager(), true, TimeSpan.FromMilliseconds(50));

            _multiQueuedHandler = new MultiQueuedHandler(new IQueuedHandler[] { queue }, null);
            _multiQueuedHandler.Start();

            _service = new KestrelHttpService(ServiceAccessibility.Private, _bus, new NaiveUriRouter(),
                                              _multiQueuedHandler, false, null, 0, false, _serverEndPoint);
            KestrelHttpService.CreateAndSubscribePipeline(pipelineBus);
            bootstrap?.Invoke(_service);
            _server = new TestServer(
                new WebHostBuilder()
                .UseStartup(new HttpServiceStartup(_service)));
            _httpMessageHandler = _server.CreateHandler();
            _client             = new HttpAsyncClient(_timeout, _httpMessageHandler);

            HttpBootstrap.Subscribe(_bus, _service);
        }
        protected CommandQueueSpecification(
            int queueCount = 1,
            int slowCmdMs  = 500,
            int slowAckMs  = 500)
        {
            Bus = new CommandBus(
                "Fixture Bus",
                slowCmdThreshold: TimeSpan.FromMilliseconds(slowCmdMs),
                slowMsgThreshold: TimeSpan.FromMilliseconds(slowAckMs));

            Queue = new MultiQueuedHandler(
                queueCount,
                index => new QueuedHandler(
                    new AdHocHandler <Message>(
                        msg =>
            {
                if (msg is Command)
                {
                    Bus.TryFire((Command)msg);
                }
                else
                {
                    Bus.Publish(msg);
                }
            }),
                    $"Queue {index}"
                    )
                );
            Queue.Start();
            TestQueue = new TestQueue(Bus);
            try
            {
                Given();
                When();
            }
            catch (Exception)
            {
                throw;
            }
        }
Ejemplo n.º 12
0
        public StorageReaderService(IPublisher bus, ISubscriber subscriber, IReadIndex readIndex, int threadCount, ICheckpoint writerCheckpoint)
        {
            Ensure.NotNull(bus, "bus");
            Ensure.NotNull(subscriber, "subscriber");
            Ensure.NotNull(readIndex, "readIndex");
            Ensure.Positive(threadCount, "threadCount");
            Ensure.NotNull(writerCheckpoint, "writerCheckpoint");

            _bus         = bus;
            _readIndex   = readIndex;
            _threadCount = threadCount;

            var readerWorker     = new StorageReaderWorker(bus, readIndex, writerCheckpoint);
            var storageReaderBus = new InMemoryBus("StorageReaderBus", watchSlowMsg: false);

            storageReaderBus.Subscribe <ClientMessage.ReadEvent>(readerWorker);
            storageReaderBus.Subscribe <ClientMessage.ReadStreamEventsBackward>(readerWorker);
            storageReaderBus.Subscribe <ClientMessage.ReadStreamEventsForward>(readerWorker);
            storageReaderBus.Subscribe <ClientMessage.ReadAllEventsForward>(readerWorker);
            storageReaderBus.Subscribe <ClientMessage.ReadAllEventsBackward>(readerWorker);
            storageReaderBus.Subscribe <StorageMessage.CheckStreamAccess>(readerWorker);

            _workersMultiHandler = new MultiQueuedHandler(
                _threadCount,
                queueNum => new QueuedHandlerThreadPool(storageReaderBus,
                                                        string.Format("StorageReaderQueue #{0}", queueNum + 1),
                                                        groupName: "StorageReaderQueue",
                                                        watchSlowMsg: true,
                                                        slowMsgThreshold: TimeSpan.FromMilliseconds(200)));
            _workersMultiHandler.Start();

            subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadEvent, Message>());
            subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadStreamEventsBackward, Message>());
            subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadStreamEventsForward, Message>());
            subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadAllEventsForward, Message>());
            subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadAllEventsBackward, Message>());
            subscriber.Subscribe(_workersMultiHandler.WidenFrom <StorageMessage.CheckStreamAccess, Message>());
        }
Ejemplo n.º 13
0
        public SingleVNode(TFChunkDb db, 
                           SingleVNodeSettings vNodeSettings, 
                           bool dbVerifyHashes, 
                           int memTableEntryCount, 
                           params ISubsystem[] subsystems)
        {
            Ensure.NotNull(db, "db");
            Ensure.NotNull(vNodeSettings, "vNodeSettings");

            _settings = vNodeSettings;

            _mainBus = new InMemoryBus("MainBus");
            _controller = new SingleVNodeController(_mainBus, _settings.ExternalHttpEndPoint, db, this);
            _mainQueue = new QueuedHandler(_controller, "MainQueue");
            _controller.SetMainQueue(_mainQueue);

            _subsystems = subsystems;

            // MONITORING
            var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
            var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
            var monitoringQueue = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
            var monitoring = new MonitoringService(monitoringQueue,
                                                   monitoringRequestBus,
                                                   _mainQueue,
                                                   db.Config.WriterCheckpoint,
                                                   db.Config.Path,
                                                   vNodeSettings.StatsPeriod,
                                                   _settings.ExternalHttpEndPoint,
                                                   vNodeSettings.StatsStorage);
            _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.SystemInit, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.StateChangeMessage, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShutdown, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom<ClientMessage.WriteEventsCompleted, Message>());
            monitoringInnerBus.Subscribe<SystemMessage.SystemInit>(monitoring);
            monitoringInnerBus.Subscribe<SystemMessage.StateChangeMessage>(monitoring);
            monitoringInnerBus.Subscribe<SystemMessage.BecomeShuttingDown>(monitoring);
            monitoringInnerBus.Subscribe<SystemMessage.BecomeShutdown>(monitoring);
            monitoringInnerBus.Subscribe<ClientMessage.WriteEventsCompleted>(monitoring);
            monitoringInnerBus.Subscribe<MonitoringMessage.GetFreshStats>(monitoring);

            var truncPos = db.Config.TruncateCheckpoint.Read();
            if (truncPos != -1)
            {
                Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})",
                         truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read());
                var truncator = new TFChunkDbTruncator(db.Config);
                truncator.TruncateDb(truncPos);
            }

            db.Open(dbVerifyHashes);

            // STORAGE SUBSYSTEM
            var indexPath = Path.Combine(db.Config.Path, "index");
            var readerPool = new ObjectPool<ITransactionFileReader>(
                "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
                () => new TFChunkReader(db, db.Config.WriterCheckpoint));
            var tableIndex = new TableIndex(indexPath,
                                            () => new HashListMemTable(maxSize: memTableEntryCount * 2),
                                            () => new TFReaderLease(readerPool),
                                            maxSizeForMemory: memTableEntryCount,
                                            maxTablesPerLevel: 2,
                                            inMem: db.Config.InMemDb);
            var hasher = new XXHashUnsafe();
            var readIndex = new ReadIndex(_mainQueue,
                                          readerPool,
                                          tableIndex,
                                          hasher,
                                          ESConsts.StreamInfoCacheCapacity,
                                          Application.IsDefined(Application.AdditionalCommitChecks),
                                          Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1);
            var writer = new TFChunkWriter(db);
            var epochManager = new EpochManager(ESConsts.CachedEpochCount,
                                                db.Config.EpochCheckpoint,
                                                writer,
                                                initialReaderCount: 1,
                                                maxReaderCount: 5,
                                                readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint));
            epochManager.Init();

            var storageWriter = new StorageWriterService(_mainQueue, _mainBus, _settings.MinFlushDelay,
                                                         db, writer, readIndex.IndexWriter, epochManager); // subscribes internally
            monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageWriter);

            var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, ESConsts.StorageReaderThreadCount, db.Config.WriterCheckpoint);
            _mainBus.Subscribe<SystemMessage.SystemInit>(storageReader);
            _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageReader);
            _mainBus.Subscribe<SystemMessage.BecomeShutdown>(storageReader);
            monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageReader);

            var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
            var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, readIndex.IndexCommitter, epochManager);
            _mainBus.Subscribe<SystemMessage.SystemInit>(storageChaser);
            _mainBus.Subscribe<SystemMessage.SystemStart>(storageChaser);
            _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageChaser);

            var storageScavenger = new StorageScavenger(db,
                                                        tableIndex,
                                                        hasher,
                                                        readIndex,
                                                        Application.IsDefined(Application.AlwaysKeepScavenged),
                                                        mergeChunks: false /*!Application.IsDefined(Application.DisableMergeChunks)*/);
            // ReSharper disable RedundantTypeArgumentsOfMethod
            _mainBus.Subscribe<ClientMessage.ScavengeDatabase>(storageScavenger);
            // ReSharper restore RedundantTypeArgumentsOfMethod

            // MISC WORKERS
            _workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
                new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
                                watchSlowMsg: true,
                                slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray();
            _workersHandler = new MultiQueuedHandler(
                    vNodeSettings.WorkerThreads,
                    queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
                                                            string.Format("Worker #{0}", queueNum + 1),
                                                            groupName: "Workers",
                                                            watchSlowMsg: true,
                                                            slowMsgThreshold: TimeSpan.FromMilliseconds(200)));

            // AUTHENTICATION INFRASTRUCTURE
            var passwordHashAlgorithm = new Rfc2898PasswordHashAlgorithm();
            var dispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_workersHandler, crossThread: true));
            var internalAuthenticationProvider = new InternalAuthenticationProvider(dispatcher, passwordHashAlgorithm, ESConsts.CachedPrincipalCount);
            var passwordChangeNotificationReader = new PasswordChangeNotificationReader(_mainQueue, dispatcher);
            _mainBus.Subscribe<SystemMessage.SystemStart>(passwordChangeNotificationReader);
            _mainBus.Subscribe<SystemMessage.BecomeShutdown>(passwordChangeNotificationReader);
            _mainBus.Subscribe(internalAuthenticationProvider);
            SubscribeWorkers(bus =>
            {
                bus.Subscribe(dispatcher.ForwardReader);
                bus.Subscribe(dispatcher.BackwardReader);
                bus.Subscribe(dispatcher.Writer);
                bus.Subscribe(dispatcher.StreamDeleter);
                bus.Subscribe(dispatcher);
            });

            // TCP
            {
                var tcpService = new TcpService(_mainQueue, _settings.ExternalTcpEndPoint, _workersHandler,
                                                TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
                                                ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
                                                internalAuthenticationProvider, null);
                _mainBus.Subscribe<SystemMessage.SystemInit>(tcpService);
                _mainBus.Subscribe<SystemMessage.SystemStart>(tcpService);
                _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(tcpService);
            }

            // SECURE TCP
            if (vNodeSettings.ExternalSecureTcpEndPoint != null)
            {
                var secureTcpService = new TcpService(_mainQueue, _settings.ExternalSecureTcpEndPoint, _workersHandler,
                                                      TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(),
                                                      ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
                                                      internalAuthenticationProvider, _settings.Certificate);
                _mainBus.Subscribe<SystemMessage.SystemInit>(secureTcpService);
                _mainBus.Subscribe<SystemMessage.SystemStart>(secureTcpService);
                _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(secureTcpService);
            }

            SubscribeWorkers(bus =>
            {
                var tcpSendService = new TcpSendService();
                // ReSharper disable RedundantTypeArgumentsOfMethod
                bus.Subscribe<TcpMessage.TcpSend>(tcpSendService);
                // ReSharper restore RedundantTypeArgumentsOfMethod
            });

            // HTTP
            var httpAuthenticationProviders = new List<HttpAuthenticationProvider>
            {
                new BasicHttpAuthenticationProvider(internalAuthenticationProvider),
            };
            if (_settings.EnableTrustedAuth)
                httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider());
            httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider());

            var httpPipe = new HttpMessagePipe();
            var httpSendService = new HttpSendService(httpPipe, forwardRequests: false);
            _mainBus.Subscribe<SystemMessage.StateChangeMessage>(httpSendService);
            _mainBus.Subscribe(new WideningHandler<HttpMessage.SendOverHttp, Message>(_workersHandler));
            SubscribeWorkers(bus =>
            {
                bus.Subscribe<HttpMessage.HttpSend>(httpSendService);
                bus.Subscribe<HttpMessage.HttpSendPart>(httpSendService);
                bus.Subscribe<HttpMessage.HttpBeginSend>(httpSendService);
                bus.Subscribe<HttpMessage.HttpEndSend>(httpSendService);
                bus.Subscribe<HttpMessage.SendOverHttp>(httpSendService);
            });

            _httpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(), 
                                            _workersHandler, vNodeSettings.HttpPrefixes);
            _httpService.SetupController(new AdminController(_mainQueue));
            _httpService.SetupController(new PingController());
            _httpService.SetupController(new StatController(monitoringQueue, _workersHandler));
            _httpService.SetupController(new AtomController(httpSendService, _mainQueue, _workersHandler));
            _httpService.SetupController(new GuidController(_mainQueue));
            _httpService.SetupController(new UsersController(httpSendService, _mainQueue, _workersHandler));

            _mainBus.Subscribe<SystemMessage.SystemInit>(_httpService);
            _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(_httpService);
            _mainBus.Subscribe<HttpMessage.PurgeTimedOutRequests>(_httpService);

            SubscribeWorkers(bus =>
            {
                HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray());
            });

            // REQUEST MANAGEMENT
            var requestManagement = new RequestManagementService(_mainQueue, 1, 1, vNodeSettings.PrepareTimeout, vNodeSettings.CommitTimeout);
            _mainBus.Subscribe<SystemMessage.SystemInit>(requestManagement);
            _mainBus.Subscribe<ClientMessage.WriteEvents>(requestManagement);
            _mainBus.Subscribe<ClientMessage.TransactionStart>(requestManagement);
            _mainBus.Subscribe<ClientMessage.TransactionWrite>(requestManagement);
            _mainBus.Subscribe<ClientMessage.TransactionCommit>(requestManagement);
            _mainBus.Subscribe<ClientMessage.DeleteStream>(requestManagement);
            _mainBus.Subscribe<StorageMessage.RequestCompleted>(requestManagement);
            _mainBus.Subscribe<StorageMessage.CheckStreamAccessCompleted>(requestManagement);
            _mainBus.Subscribe<StorageMessage.AlreadyCommitted>(requestManagement);
            _mainBus.Subscribe<StorageMessage.CommitAck>(requestManagement);
            _mainBus.Subscribe<StorageMessage.PrepareAck>(requestManagement);
            _mainBus.Subscribe<StorageMessage.WrongExpectedVersion>(requestManagement);
            _mainBus.Subscribe<StorageMessage.InvalidTransaction>(requestManagement);
            _mainBus.Subscribe<StorageMessage.StreamDeleted>(requestManagement);
            _mainBus.Subscribe<StorageMessage.RequestManagerTimerTick>(requestManagement);

            // SUBSCRIPTIONS
            var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
            var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false);
            _mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.SystemStart, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<TcpMessage.ConnectionClosed, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.SubscribeToStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.UnsubscribeFromStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.PollStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.CheckPollTimeout, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<StorageMessage.EventCommited, Message>());

            var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex);
            subscrBus.Subscribe<SystemMessage.SystemStart>(subscription);
            subscrBus.Subscribe<SystemMessage.BecomeShuttingDown>(subscription);
            subscrBus.Subscribe<TcpMessage.ConnectionClosed>(subscription);
            subscrBus.Subscribe<ClientMessage.SubscribeToStream>(subscription);
            subscrBus.Subscribe<ClientMessage.UnsubscribeFromStream>(subscription);
            subscrBus.Subscribe<SubscriptionMessage.PollStream>(subscription);
            subscrBus.Subscribe<SubscriptionMessage.CheckPollTimeout>(subscription);
            subscrBus.Subscribe<StorageMessage.EventCommited>(subscription);

            // USER MANAGEMENT
            var ioDispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_mainQueue));
            _mainBus.Subscribe(ioDispatcher.BackwardReader);
            _mainBus.Subscribe(ioDispatcher.ForwardReader);
            _mainBus.Subscribe(ioDispatcher.Writer);
            _mainBus.Subscribe(ioDispatcher.StreamDeleter);
            _mainBus.Subscribe(ioDispatcher);

            var userManagement = new UserManagementService(
                _mainQueue, ioDispatcher, passwordHashAlgorithm, vNodeSettings.SkipInitializeStandardUsersCheck);
            _mainBus.Subscribe<UserManagementMessage.Create>(userManagement);
            _mainBus.Subscribe<UserManagementMessage.Update>(userManagement);
            _mainBus.Subscribe<UserManagementMessage.Enable>(userManagement);
            _mainBus.Subscribe<UserManagementMessage.Disable>(userManagement);
            _mainBus.Subscribe<UserManagementMessage.Delete>(userManagement);
            _mainBus.Subscribe<UserManagementMessage.ResetPassword>(userManagement);
            _mainBus.Subscribe<UserManagementMessage.ChangePassword>(userManagement);
            _mainBus.Subscribe<UserManagementMessage.Get>(userManagement);
            _mainBus.Subscribe<UserManagementMessage.GetAll>(userManagement);
            _mainBus.Subscribe<SystemMessage.BecomeMaster>(userManagement);

            // TIMER
            _timeProvider = new RealTimeProvider();
            _timerService = new TimerService(new ThreadBasedScheduler(_timeProvider));
            _mainBus.Subscribe<SystemMessage.BecomeShutdown>(_timerService);
            _mainBus.Subscribe<TimerMessage.Schedule>(_timerService);

            _workersHandler.Start();
            _mainQueue.Start();
            monitoringQueue.Start();
            subscrQueue.Start();

            if (subsystems != null)
                foreach (var subsystem in subsystems)
                    subsystem.Register(db, _mainQueue, _mainBus, _timerService, _timeProvider, httpSendService, new[]{_httpService}, _workersHandler);
        }
Ejemplo n.º 14
0
        public SingleVNode(TFChunkDb db,
                           SingleVNodeSettings vNodeSettings,
                           bool dbVerifyHashes,
                           int memTableEntryCount,
                           params ISubsystem[] subsystems)
        {
            Ensure.NotNull(db, "db");
            Ensure.NotNull(vNodeSettings, "vNodeSettings");

            _settings = vNodeSettings;

            _mainBus    = new InMemoryBus("MainBus");
            _controller = new SingleVNodeController(_mainBus, _settings.ExternalHttpEndPoint, db, this);
            _mainQueue  = new QueuedHandler(_controller, "MainQueue");
            _controller.SetMainQueue(_mainQueue);

            _subsystems = subsystems;

            // MONITORING
            var monitoringInnerBus   = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
            var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
            var monitoringQueue      = new QueuedHandler(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
            var monitoring           = new MonitoringService(monitoringQueue,
                                                             monitoringRequestBus,
                                                             _mainQueue,
                                                             db.Config.WriterCheckpoint,
                                                             db.Config.Path,
                                                             vNodeSettings.StatsPeriod,
                                                             _settings.ExternalHttpEndPoint,
                                                             vNodeSettings.StatsStorage);

            _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.SystemInit, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom <ClientMessage.WriteEventsCompleted, Message>());
            monitoringInnerBus.Subscribe <SystemMessage.SystemInit>(monitoring);
            monitoringInnerBus.Subscribe <SystemMessage.StateChangeMessage>(monitoring);
            monitoringInnerBus.Subscribe <SystemMessage.BecomeShuttingDown>(monitoring);
            monitoringInnerBus.Subscribe <ClientMessage.WriteEventsCompleted>(monitoring);
            monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshStats>(monitoring);

            var truncPos = db.Config.TruncateCheckpoint.Read();

            if (truncPos != -1)
            {
                Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})",
                         truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read());
                var truncator = new TFChunkDbTruncator(db.Config);
                truncator.TruncateDb(truncPos);
            }

            db.Open(dbVerifyHashes);

            // STORAGE SUBSYSTEM
            var indexPath  = Path.Combine(db.Config.Path, "index");
            var tableIndex = new TableIndex(indexPath,
                                            () => new HashListMemTable(maxSize: memTableEntryCount * 2),
                                            maxSizeForMemory: memTableEntryCount,
                                            maxTablesPerLevel: 2);

            var readIndex = new ReadIndex(_mainQueue,
                                          ESConsts.PTableInitialReaderCount,
                                          ESConsts.PTableMaxReaderCount,
                                          () => new TFChunkReader(db, db.Config.WriterCheckpoint),
                                          tableIndex,
                                          new XXHashUnsafe(),
                                          new LRUCache <string, StreamCacheInfo>(ESConsts.StreamMetadataCacheCapacity),
                                          Application.IsDefined(Application.AdditionalCommitChecks),
                                          Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1);
            var writer       = new TFChunkWriter(db);
            var epochManager = new EpochManager(ESConsts.CachedEpochCount,
                                                db.Config.EpochCheckpoint,
                                                writer,
                                                initialReaderCount: 1,
                                                maxReaderCount: 5,
                                                readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint));

            epochManager.Init();

            var storageWriter = new StorageWriterService(_mainQueue, _mainBus, _settings.MinFlushDelayMs,
                                                         db, writer, readIndex, epochManager); // subscribes internally

            monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageWriter);

            var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, ESConsts.StorageReaderThreadCount, db.Config.WriterCheckpoint);

            _mainBus.Subscribe <SystemMessage.SystemInit>(storageReader);
            _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageReader);
            _mainBus.Subscribe <SystemMessage.BecomeShutdown>(storageReader);
            monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageReader);

            var chaser        = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
            var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, readIndex, epochManager);

            _mainBus.Subscribe <SystemMessage.SystemInit>(storageChaser);
            _mainBus.Subscribe <SystemMessage.SystemStart>(storageChaser);
            _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageChaser);

            var storageScavenger = new StorageScavenger(db,
                                                        readIndex,
                                                        Application.IsDefined(Application.AlwaysKeepScavenged),
                                                        mergeChunks: false /*!Application.IsDefined(Application.DisableMergeChunks)*/);

            // ReSharper disable RedundantTypeArgumentsOfMethod
            _mainBus.Subscribe <SystemMessage.ScavengeDatabase>(storageScavenger);
            // ReSharper restore RedundantTypeArgumentsOfMethod

            // MISC WORKERS
            _workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
                                                                                   new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
                                                                                                   watchSlowMsg: true,
                                                                                                   slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray();
            _workersHandler = new MultiQueuedHandler(
                vNodeSettings.WorkerThreads,
                queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
                                                        string.Format("Worker #{0}", queueNum + 1),
                                                        groupName: "Workers",
                                                        watchSlowMsg: true,
                                                        slowMsgThreshold: TimeSpan.FromMilliseconds(200)));

            // AUTHENTICATION INFRASTRUCTURE
            var passwordHashAlgorithm            = new Rfc2898PasswordHashAlgorithm();
            var dispatcher                       = new IODispatcher(_mainBus, new PublishEnvelope(_workersHandler, crossThread: true));
            var internalAuthenticationProvider   = new InternalAuthenticationProvider(dispatcher, passwordHashAlgorithm, ESConsts.CachedPrincipalCount);
            var passwordChangeNotificationReader = new PasswordChangeNotificationReader(_mainQueue, dispatcher);

            _mainBus.Subscribe <SystemMessage.SystemStart>(passwordChangeNotificationReader);
            _mainBus.Subscribe <SystemMessage.BecomeShutdown>(passwordChangeNotificationReader);
            _mainBus.Subscribe(internalAuthenticationProvider);
            SubscribeWorkers(bus =>
            {
                bus.Subscribe(dispatcher.ForwardReader);
                bus.Subscribe(dispatcher.BackwardReader);
                bus.Subscribe(dispatcher.Writer);
                bus.Subscribe(dispatcher.StreamDeleter);
                bus.Subscribe(dispatcher);
            });

            // TCP
            {
                var tcpService = new TcpService(_mainQueue, _settings.ExternalTcpEndPoint, _workersHandler,
                                                TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
                                                ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
                                                internalAuthenticationProvider, null);
                _mainBus.Subscribe <SystemMessage.SystemInit>(tcpService);
                _mainBus.Subscribe <SystemMessage.SystemStart>(tcpService);
                _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(tcpService);
            }

            // SECURE TCP
            if (vNodeSettings.ExternalSecureTcpEndPoint != null)
            {
                var secureTcpService = new TcpService(_mainQueue, _settings.ExternalSecureTcpEndPoint, _workersHandler,
                                                      TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(),
                                                      ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
                                                      internalAuthenticationProvider, _settings.Certificate);
                _mainBus.Subscribe <SystemMessage.SystemInit>(secureTcpService);
                _mainBus.Subscribe <SystemMessage.SystemStart>(secureTcpService);
                _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(secureTcpService);
            }

            SubscribeWorkers(bus =>
            {
                var tcpSendService = new TcpSendService();
                // ReSharper disable RedundantTypeArgumentsOfMethod
                bus.Subscribe <TcpMessage.TcpSend>(tcpSendService);
                // ReSharper restore RedundantTypeArgumentsOfMethod
            });

            // HTTP
            var authenticationProviders = new List <AuthenticationProvider>
            {
                new BasicHttpAuthenticationProvider(internalAuthenticationProvider),
            };

            if (_settings.EnableTrustedAuth)
            {
                authenticationProviders.Add(new TrustedAuthenticationProvider());
            }
            authenticationProviders.Add(new AnonymousAuthenticationProvider());

            var httpPipe        = new HttpMessagePipe();
            var httpSendService = new HttpSendService(httpPipe, forwardRequests: false);

            _mainBus.Subscribe <SystemMessage.StateChangeMessage>(httpSendService);
            _mainBus.Subscribe(new WideningHandler <HttpMessage.SendOverHttp, Message>(_workersHandler));
            SubscribeWorkers(bus =>
            {
                bus.Subscribe <HttpMessage.HttpSend>(httpSendService);
                bus.Subscribe <HttpMessage.HttpSendPart>(httpSendService);
                bus.Subscribe <HttpMessage.HttpBeginSend>(httpSendService);
                bus.Subscribe <HttpMessage.HttpEndSend>(httpSendService);
                bus.Subscribe <HttpMessage.SendOverHttp>(httpSendService);
            });

            _httpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(),
                                           _workersHandler, vNodeSettings.HttpPrefixes);
            _httpService.SetupController(new AdminController(_mainQueue));
            _httpService.SetupController(new PingController());
            _httpService.SetupController(new StatController(monitoringQueue, _workersHandler));
            _httpService.SetupController(new AtomController(httpSendService, _mainQueue, _workersHandler));
            _httpService.SetupController(new GuidController(_mainQueue));
            _httpService.SetupController(new UsersController(httpSendService, _mainQueue, _workersHandler));

            _mainBus.Subscribe <SystemMessage.SystemInit>(_httpService);
            _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_httpService);
            _mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_httpService);

            SubscribeWorkers(bus =>
            {
                HttpService.CreateAndSubscribePipeline(bus, authenticationProviders.ToArray());
            });

            // REQUEST MANAGEMENT
            var requestManagement = new RequestManagementService(_mainQueue, 1, 1, vNodeSettings.PrepareTimeout, vNodeSettings.CommitTimeout);

            _mainBus.Subscribe <SystemMessage.SystemInit>(requestManagement);
            _mainBus.Subscribe <ClientMessage.WriteEvents>(requestManagement);
            _mainBus.Subscribe <ClientMessage.TransactionStart>(requestManagement);
            _mainBus.Subscribe <ClientMessage.TransactionWrite>(requestManagement);
            _mainBus.Subscribe <ClientMessage.TransactionCommit>(requestManagement);
            _mainBus.Subscribe <ClientMessage.DeleteStream>(requestManagement);
            _mainBus.Subscribe <StorageMessage.RequestCompleted>(requestManagement);
            _mainBus.Subscribe <StorageMessage.CheckStreamAccessCompleted>(requestManagement);
            _mainBus.Subscribe <StorageMessage.AlreadyCommitted>(requestManagement);
            _mainBus.Subscribe <StorageMessage.CommitAck>(requestManagement);
            _mainBus.Subscribe <StorageMessage.PrepareAck>(requestManagement);
            _mainBus.Subscribe <StorageMessage.WrongExpectedVersion>(requestManagement);
            _mainBus.Subscribe <StorageMessage.InvalidTransaction>(requestManagement);
            _mainBus.Subscribe <StorageMessage.StreamDeleted>(requestManagement);
            _mainBus.Subscribe <StorageMessage.RequestManagerTimerTick>(requestManagement);

            // SUBSCRIPTIONS
            var subscrBus    = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
            var subscription = new SubscriptionsService(readIndex);

            subscrBus.Subscribe <TcpMessage.ConnectionClosed>(subscription);
            subscrBus.Subscribe <ClientMessage.SubscribeToStream>(subscription);
            subscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(subscription);
            subscrBus.Subscribe <StorageMessage.EventCommited>(subscription);

            var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false);

            _mainBus.Subscribe(subscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.SubscribeToStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <StorageMessage.EventCommited, Message>());

            // USER MANAGEMENT
            var ioDispatcher = new IODispatcher(_mainBus, new PublishEnvelope(_mainQueue));

            _mainBus.Subscribe(ioDispatcher.BackwardReader);
            _mainBus.Subscribe(ioDispatcher.ForwardReader);
            _mainBus.Subscribe(ioDispatcher.Writer);
            _mainBus.Subscribe(ioDispatcher.StreamDeleter);
            _mainBus.Subscribe(ioDispatcher);

            var userManagement = new UserManagementService(
                _mainQueue, ioDispatcher, passwordHashAlgorithm, vNodeSettings.SkipInitializeStandardUsersCheck);

            _mainBus.Subscribe <UserManagementMessage.Create>(userManagement);
            _mainBus.Subscribe <UserManagementMessage.Update>(userManagement);
            _mainBus.Subscribe <UserManagementMessage.Enable>(userManagement);
            _mainBus.Subscribe <UserManagementMessage.Disable>(userManagement);
            _mainBus.Subscribe <UserManagementMessage.Delete>(userManagement);
            _mainBus.Subscribe <UserManagementMessage.ResetPassword>(userManagement);
            _mainBus.Subscribe <UserManagementMessage.ChangePassword>(userManagement);
            _mainBus.Subscribe <UserManagementMessage.Get>(userManagement);
            _mainBus.Subscribe <UserManagementMessage.GetAll>(userManagement);
            _mainBus.Subscribe <SystemMessage.BecomeMaster>(userManagement);

            // TIMER
            _timeProvider = new RealTimeProvider();
            _timerService = new TimerService(new ThreadBasedScheduler(_timeProvider));
            // ReSharper disable RedundantTypeArgumentsOfMethod
            _mainBus.Subscribe <TimerMessage.Schedule>(_timerService);
            // ReSharper restore RedundantTypeArgumentsOfMethod

            _workersHandler.Start();
            _mainQueue.Start();
            monitoringQueue.Start();
            subscrQueue.Start();

            if (subsystems != null)
            {
                foreach (var subsystem in subsystems)
                {
                    subsystem.Register(db, _mainQueue, _mainBus, _timerService, _timeProvider, httpSendService, new[] { _httpService }, _workersHandler);
                }
            }
        }
Ejemplo n.º 15
0
        public void of_http_requests_routing()
        {
            const int iterations = 100000;

            IPublisher inputBus = new NoopPublisher();
            var bus = InMemoryBus.CreateTest();
            var queue = new QueuedHandlerThreadPool(bus, "Test", true, TimeSpan.FromMilliseconds(50));
            var multiQueuedHandler = new MultiQueuedHandler(new IQueuedHandler[]{queue}, null);
            var providers = new AuthenticationProvider[] {new AnonymousAuthenticationProvider()};
            var httpService = new HttpService(ServiceAccessibility.Public, inputBus, 
                                              new TrieUriRouter(), multiQueuedHandler, "http://localhost:12345/");
            HttpService.CreateAndSubscribePipeline(bus, providers);

            var fakeController = new FakeController(iterations, null);
            httpService.SetupController(fakeController);

            httpService.Handle(new SystemMessage.SystemInit());

            var rnd = new Random();
            var sw = Stopwatch.StartNew();

            var httpClient = new HttpAsyncClient();
            for (int i = 0; i < iterations; ++i)
            {
                var route = fakeController.BoundRoutes[rnd.Next(0, fakeController.BoundRoutes.Count)];

                switch (route.Item2)
                {
                    case HttpMethod.Get:
                        httpClient.Get(route.Item1,TimeSpan.FromMilliseconds(10000),  x => { }, x => { throw new Exception();});
                        break;
                    case HttpMethod.Post:
                        httpClient.Post(route.Item1, "abracadabra", ContentType.Json, TimeSpan.FromMilliseconds(10000), x => { }, x => { throw new Exception(); });
                        break;
                    case HttpMethod.Delete:
                        httpClient.Delete(route.Item1, TimeSpan.FromMilliseconds(10000), x => { }, x => { throw new Exception(); });
                        break;
                    case HttpMethod.Put:
                        httpClient.Put(route.Item1, "abracadabra", ContentType.Json, TimeSpan.FromMilliseconds(10000), x => { }, x => { throw new Exception(); });
                        break;
                    default:
                        throw new Exception();
                }
            }

            fakeController.CountdownEvent.Wait();
            sw.Stop();

            Console.WriteLine("{0} request done in {1} ({2:0.00} per sec)", iterations, sw.Elapsed, 1000.0 * iterations / sw.ElapsedMilliseconds);

            httpService.Shutdown();
            multiQueuedHandler.Stop();
        }
Ejemplo n.º 16
0
        public PlaygroundVNode(PlaygroundVNodeSettings vNodeSettings)
        {
            _tcpEndPoint = vNodeSettings.ExternalTcpEndPoint;
            _httpEndPoint = vNodeSettings.ExternalHttpEndPoint;

            _mainBus = new InMemoryBus("MainBus");
            _controller = new PlaygroundVNodeController(Bus, _httpEndPoint);
            _mainQueue = new QueuedHandler(_controller, "MainQueue");
            _controller.SetMainQueue(MainQueue);

            // MONITORING
            var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
            var monitoringQueue = new QueuedHandler(
                monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
            Bus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.SystemInit, Message>());
            Bus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.StateChangeMessage, Message>());
            Bus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());


            // MISC WORKERS
            _workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
                new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
                                watchSlowMsg: true,
                                slowMsgThreshold: TimeSpan.FromMilliseconds(50))).ToArray();
            _workersHandler = new MultiQueuedHandler(
                    vNodeSettings.WorkerThreads,
                    queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
                                                            string.Format("Worker #{0}", queueNum + 1),
                                                            groupName: "Workers",
                                                            watchSlowMsg: true,
                                                            slowMsgThreshold: TimeSpan.FromMilliseconds(50)));

            // AUTHENTICATION INFRASTRUCTURE
            var dispatcher = new IODispatcher(_mainBus, new PublishEnvelope(_workersHandler, crossThread: true));
            var passwordHashAlgorithm = new Rfc2898PasswordHashAlgorithm();
            var internalAuthenticationProvider = new InternalAuthenticationProvider(dispatcher, passwordHashAlgorithm, 1000);
            var passwordChangeNotificationReader = new PasswordChangeNotificationReader(_mainQueue, dispatcher);
            _mainBus.Subscribe<SystemMessage.SystemStart>(passwordChangeNotificationReader);
            _mainBus.Subscribe<SystemMessage.BecomeShutdown>(passwordChangeNotificationReader);
            _mainBus.Subscribe(internalAuthenticationProvider);
            _mainBus.Subscribe(dispatcher);

            SubscribeWorkers(bus =>
            {
                bus.Subscribe(dispatcher.ForwardReader);
                bus.Subscribe(dispatcher.BackwardReader);
                bus.Subscribe(dispatcher.Writer);
                bus.Subscribe(dispatcher.StreamDeleter);
            });

            // TCP
            var tcpService = new TcpService(
                MainQueue, _tcpEndPoint, _workersHandler, TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(), 
                ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout, internalAuthenticationProvider, null);
            Bus.Subscribe<SystemMessage.SystemInit>(tcpService);
            Bus.Subscribe<SystemMessage.SystemStart>(tcpService);
            Bus.Subscribe<SystemMessage.BecomeShuttingDown>(tcpService);

            // HTTP
            {
                var httpAuthenticationProviders = new HttpAuthenticationProvider[]
                {
                    new BasicHttpAuthenticationProvider(internalAuthenticationProvider),
                    new TrustedHttpAuthenticationProvider(),
                    new AnonymousHttpAuthenticationProvider()
                };

                var httpPipe = new HttpMessagePipe();
                var httpSendService = new HttpSendService(httpPipe, forwardRequests: false);
                _mainBus.Subscribe<SystemMessage.StateChangeMessage>(httpSendService);
                _mainBus.Subscribe(new WideningHandler<HttpMessage.SendOverHttp, Message>(_workersHandler));
                SubscribeWorkers(bus =>
                {
                    bus.Subscribe<HttpMessage.HttpSend>(httpSendService);
                    bus.Subscribe<HttpMessage.HttpSendPart>(httpSendService);
                    bus.Subscribe<HttpMessage.HttpBeginSend>(httpSendService);
                    bus.Subscribe<HttpMessage.HttpEndSend>(httpSendService);
                    bus.Subscribe<HttpMessage.SendOverHttp>(httpSendService);
                });

                _httpService = new HttpService(ServiceAccessibility.Private, _mainQueue, new TrieUriRouter(),
                                               _workersHandler, vNodeSettings.HttpPrefixes);

                _mainBus.Subscribe<SystemMessage.SystemInit>(_httpService);
                _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(_httpService);
                _mainBus.Subscribe<HttpMessage.PurgeTimedOutRequests>(_httpService);
                HttpService.SetupController(new AdminController(_mainQueue));
                HttpService.SetupController(new PingController());
                HttpService.SetupController(new StatController(monitoringQueue, _workersHandler));
                HttpService.SetupController(new AtomController(httpSendService, _mainQueue, _workersHandler));
                HttpService.SetupController(new GuidController(_mainQueue));
                HttpService.SetupController(new UsersController(httpSendService, _mainQueue, _workersHandler));

                SubscribeWorkers(bus => HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders));
            }


            // TIMER
            _timerService = new TimerService(new ThreadBasedScheduler(new RealTimeProvider()));
            Bus.Subscribe<TimerMessage.Schedule>(TimerService);

            monitoringQueue.Start();
            MainQueue.Start();
        }
Ejemplo n.º 17
0
        public ClusterVNode(TFChunkDb db,
                            ClusterVNodeSettings vNodeSettings,
                            IGossipSeedSource gossipSeedSource,
                            bool dbVerifyHashes,
                            int memTableEntryCount,
                            params ISubsystem[] subsystems)
        {
            Ensure.NotNull(db, "db");
            Ensure.NotNull(vNodeSettings, "vNodeSettings");
            Ensure.NotNull(gossipSeedSource, "gossipSeedSource");

            _nodeInfo = vNodeSettings.NodeInfo;
            _mainBus  = new InMemoryBus("MainBus");

            var forwardingProxy = new MessageForwardingProxy();

            // MISC WORKERS
            _workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
                                                                                   new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
                                                                                                   watchSlowMsg: true,
                                                                                                   slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray();
            _workersHandler = new MultiQueuedHandler(
                vNodeSettings.WorkerThreads,
                queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
                                                        string.Format("Worker #{0}", queueNum + 1),
                                                        groupName: "Workers",
                                                        watchSlowMsg: true,
                                                        slowMsgThreshold: TimeSpan.FromMilliseconds(200)));

            _controller = new ClusterVNodeController(_mainBus, _nodeInfo, db, vNodeSettings, this, forwardingProxy);
            _mainQueue  = new QueuedHandler(_controller, "MainQueue");
            _controller.SetMainQueue(_mainQueue);

            _subsystems = subsystems;

            // MONITORING
            var monitoringInnerBus   = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
            var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
            var monitoringQueue      = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
            var monitoring           = new MonitoringService(monitoringQueue,
                                                             monitoringRequestBus,
                                                             _mainQueue,
                                                             db.Config.WriterCheckpoint,
                                                             db.Config.Path,
                                                             vNodeSettings.StatsPeriod,
                                                             _nodeInfo.ExternalHttp,
                                                             vNodeSettings.StatsStorage);

            _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.SystemInit, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShutdown, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom <ClientMessage.WriteEventsCompleted, Message>());
            monitoringInnerBus.Subscribe <SystemMessage.SystemInit>(monitoring);
            monitoringInnerBus.Subscribe <SystemMessage.StateChangeMessage>(monitoring);
            monitoringInnerBus.Subscribe <SystemMessage.BecomeShuttingDown>(monitoring);
            monitoringInnerBus.Subscribe <SystemMessage.BecomeShutdown>(monitoring);
            monitoringInnerBus.Subscribe <ClientMessage.WriteEventsCompleted>(monitoring);
            monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshStats>(monitoring);

            var truncPos = db.Config.TruncateCheckpoint.Read();

            if (truncPos != -1)
            {
                Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})",
                         truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read());
                var truncator = new TFChunkDbTruncator(db.Config);
                truncator.TruncateDb(truncPos);
            }

            // STORAGE SUBSYSTEM
            db.Open(dbVerifyHashes);
            var indexPath  = Path.Combine(db.Config.Path, "index");
            var readerPool = new ObjectPool <ITransactionFileReader>(
                "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
                () => new TFChunkReader(db, db.Config.WriterCheckpoint));
            var tableIndex = new TableIndex(indexPath,
                                            () => new HashListMemTable(maxSize: memTableEntryCount * 2),
                                            () => new TFReaderLease(readerPool),
                                            maxSizeForMemory: memTableEntryCount,
                                            maxTablesPerLevel: 2,
                                            inMem: db.Config.InMemDb);
            var hash      = new XXHashUnsafe();
            var readIndex = new ReadIndex(_mainQueue,
                                          readerPool,
                                          tableIndex,
                                          hash,
                                          ESConsts.StreamInfoCacheCapacity,
                                          Application.IsDefined(Application.AdditionalCommitChecks),
                                          Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1);
            var writer       = new TFChunkWriter(db);
            var epochManager = new EpochManager(ESConsts.CachedEpochCount,
                                                db.Config.EpochCheckpoint,
                                                writer,
                                                initialReaderCount: 1,
                                                maxReaderCount: 5,
                                                readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint));

            epochManager.Init();

            var storageWriter = new ClusterStorageWriterService(_mainQueue, _mainBus, vNodeSettings.MinFlushDelay,
                                                                db, writer, readIndex.IndexWriter, epochManager,
                                                                () => readIndex.LastCommitPosition); // subscribes internally

            monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageWriter);

            var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, ESConsts.StorageReaderThreadCount, db.Config.WriterCheckpoint);

            _mainBus.Subscribe <SystemMessage.SystemInit>(storageReader);
            _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageReader);
            _mainBus.Subscribe <SystemMessage.BecomeShutdown>(storageReader);
            monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageReader);

            var chaser        = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
            var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, readIndex.IndexCommitter, epochManager);

#if DEBUG
            QueueStatsCollector.InitializeCheckpoints(
                _nodeInfo.DebugIndex, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
#endif
            _mainBus.Subscribe <SystemMessage.SystemInit>(storageChaser);
            _mainBus.Subscribe <SystemMessage.SystemStart>(storageChaser);
            _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageChaser);

            var storageScavenger = new StorageScavenger(db, tableIndex, hash, readIndex,
                                                        Application.IsDefined(Application.AlwaysKeepScavenged),
                                                        mergeChunks: !vNodeSettings.DisableScavengeMerging);

            // ReSharper disable RedundantTypeArgumentsOfMethod
            _mainBus.Subscribe <ClientMessage.ScavengeDatabase>(storageScavenger);
            // ReSharper restore RedundantTypeArgumentsOfMethod

            // AUTHENTICATION INFRASTRUCTURE - delegate to plugins
            var authenticationProvider = vNodeSettings.AuthenticationProviderFactory.BuildAuthenticationProvider(_mainQueue, _mainBus, _workersHandler, _workerBuses);

            Ensure.NotNull(authenticationProvider, "authenticationProvider");

            {
                // EXTERNAL TCP
                var extTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalTcp, _workersHandler,
                                                   TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
                                                   ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
                                                   authenticationProvider, null);
                _mainBus.Subscribe <SystemMessage.SystemInit>(extTcpService);
                _mainBus.Subscribe <SystemMessage.SystemStart>(extTcpService);
                _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extTcpService);

                // EXTERNAL SECURE TCP
                if (_nodeInfo.ExternalSecureTcp != null)
                {
                    var extSecTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalSecureTcp, _workersHandler,
                                                          TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(),
                                                          ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
                                                          authenticationProvider, vNodeSettings.Certificate);
                    _mainBus.Subscribe <SystemMessage.SystemInit>(extSecTcpService);
                    _mainBus.Subscribe <SystemMessage.SystemStart>(extSecTcpService);
                    _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extSecTcpService);
                }

                // INTERNAL TCP
                var intTcpService = new TcpService(_mainQueue, _nodeInfo.InternalTcp, _workersHandler,
                                                   TcpServiceType.Internal, TcpSecurityType.Normal,
                                                   new InternalTcpDispatcher(),
                                                   ESConsts.InternalHeartbeatInterval, ESConsts.InternalHeartbeatTimeout,
                                                   authenticationProvider, null);
                _mainBus.Subscribe <SystemMessage.SystemInit>(intTcpService);
                _mainBus.Subscribe <SystemMessage.SystemStart>(intTcpService);
                _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intTcpService);

                // INTERNAL SECURE TCP
                if (_nodeInfo.InternalSecureTcp != null)
                {
                    var intSecTcpService = new TcpService(_mainQueue, _nodeInfo.InternalSecureTcp, _workersHandler,
                                                          TcpServiceType.Internal, TcpSecurityType.Secure,
                                                          new InternalTcpDispatcher(),
                                                          ESConsts.InternalHeartbeatInterval, ESConsts.InternalHeartbeatTimeout,
                                                          authenticationProvider, vNodeSettings.Certificate);
                    _mainBus.Subscribe <SystemMessage.SystemInit>(intSecTcpService);
                    _mainBus.Subscribe <SystemMessage.SystemStart>(intSecTcpService);
                    _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intSecTcpService);
                }
            }

            SubscribeWorkers(bus =>
            {
                var tcpSendService = new TcpSendService();
                // ReSharper disable RedundantTypeArgumentsOfMethod
                bus.Subscribe <TcpMessage.TcpSend>(tcpSendService);
                // ReSharper restore RedundantTypeArgumentsOfMethod
            });

            var httpAuthenticationProviders = new List <HttpAuthenticationProvider>
            {
                new BasicHttpAuthenticationProvider(authenticationProvider),
            };
            if (vNodeSettings.EnableTrustedAuth)
            {
                httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider());
            }
            httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider());

            var httpPipe        = new HttpMessagePipe();
            var httpSendService = new HttpSendService(httpPipe, forwardRequests: true);
            _mainBus.Subscribe <SystemMessage.StateChangeMessage>(httpSendService);
            _mainBus.Subscribe(new WideningHandler <HttpMessage.SendOverHttp, Message>(_workersHandler));
            SubscribeWorkers(bus =>
            {
                bus.Subscribe <HttpMessage.HttpSend>(httpSendService);
                bus.Subscribe <HttpMessage.HttpSendPart>(httpSendService);
                bus.Subscribe <HttpMessage.HttpBeginSend>(httpSendService);
                bus.Subscribe <HttpMessage.HttpEndSend>(httpSendService);
                bus.Subscribe <HttpMessage.SendOverHttp>(httpSendService);
            });

            var adminController  = new AdminController(_mainQueue);
            var pingController   = new PingController();
            var statController   = new StatController(monitoringQueue, _workersHandler);
            var atomController   = new AtomController(httpSendService, _mainQueue, _workersHandler);
            var gossipController = new GossipController(_mainQueue, _workersHandler);
            var electController  = new ElectController(_mainQueue);

            // HTTP SENDERS
            gossipController.SubscribeSenders(httpPipe);
            electController.SubscribeSenders(httpPipe);

            // EXTERNAL HTTP
            _externalHttpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(),
                                                   _workersHandler, vNodeSettings.HttpPrefixes);
            if (vNodeSettings.AdminOnPublic)
            {
                _externalHttpService.SetupController(adminController);
            }
            _externalHttpService.SetupController(pingController);
            if (vNodeSettings.StatsOnPublic)
            {
                _externalHttpService.SetupController(statController);
            }
            _externalHttpService.SetupController(atomController);
            if (vNodeSettings.GossipOnPublic)
            {
                _externalHttpService.SetupController(gossipController);
            }

            _mainBus.Subscribe <SystemMessage.SystemInit>(_externalHttpService);
            _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_externalHttpService);
            _mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_externalHttpService);

            // INTERNAL HTTP
            _internalHttpService = new HttpService(ServiceAccessibility.Private, _mainQueue, new TrieUriRouter(),
                                                   _workersHandler, _nodeInfo.InternalHttp.ToHttpUrl());
            _internalHttpService.SetupController(adminController);
            _internalHttpService.SetupController(pingController);
            _internalHttpService.SetupController(statController);
            _internalHttpService.SetupController(atomController);
            _internalHttpService.SetupController(gossipController);
            _internalHttpService.SetupController(electController);

            // Authentication plugin HTTP
            vNodeSettings.AuthenticationProviderFactory.RegisterHttpControllers(_externalHttpService, _internalHttpService, httpSendService, _mainQueue, _workersHandler);

            _mainBus.Subscribe <SystemMessage.SystemInit>(_internalHttpService);
            _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_internalHttpService);
            _mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_internalHttpService);

            SubscribeWorkers(bus =>
            {
                HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray());
            });

            // REQUEST FORWARDING
            var forwardingService = new RequestForwardingService(_mainQueue, forwardingProxy, TimeSpan.FromSeconds(1));
            _mainBus.Subscribe <SystemMessage.SystemStart>(forwardingService);
            _mainBus.Subscribe <SystemMessage.RequestForwardingTimerTick>(forwardingService);
            _mainBus.Subscribe <ClientMessage.NotHandled>(forwardingService);
            _mainBus.Subscribe <ClientMessage.WriteEventsCompleted>(forwardingService);
            _mainBus.Subscribe <ClientMessage.TransactionStartCompleted>(forwardingService);
            _mainBus.Subscribe <ClientMessage.TransactionWriteCompleted>(forwardingService);
            _mainBus.Subscribe <ClientMessage.TransactionCommitCompleted>(forwardingService);
            _mainBus.Subscribe <ClientMessage.DeleteStreamCompleted>(forwardingService);

            // REQUEST MANAGEMENT
            var requestManagement = new RequestManagementService(_mainQueue,
                                                                 vNodeSettings.PrepareAckCount,
                                                                 vNodeSettings.CommitAckCount,
                                                                 vNodeSettings.PrepareTimeout,
                                                                 vNodeSettings.CommitTimeout);
            _mainBus.Subscribe <SystemMessage.SystemInit>(requestManagement);
            _mainBus.Subscribe <ClientMessage.WriteEvents>(requestManagement);
            _mainBus.Subscribe <ClientMessage.TransactionStart>(requestManagement);
            _mainBus.Subscribe <ClientMessage.TransactionWrite>(requestManagement);
            _mainBus.Subscribe <ClientMessage.TransactionCommit>(requestManagement);
            _mainBus.Subscribe <ClientMessage.DeleteStream>(requestManagement);
            _mainBus.Subscribe <StorageMessage.RequestCompleted>(requestManagement);
            _mainBus.Subscribe <StorageMessage.CheckStreamAccessCompleted>(requestManagement);
            _mainBus.Subscribe <StorageMessage.AlreadyCommitted>(requestManagement);
            _mainBus.Subscribe <StorageMessage.CommitAck>(requestManagement);
            _mainBus.Subscribe <StorageMessage.PrepareAck>(requestManagement);
            _mainBus.Subscribe <StorageMessage.WrongExpectedVersion>(requestManagement);
            _mainBus.Subscribe <StorageMessage.InvalidTransaction>(requestManagement);
            _mainBus.Subscribe <StorageMessage.StreamDeleted>(requestManagement);
            _mainBus.Subscribe <StorageMessage.RequestManagerTimerTick>(requestManagement);

            // SUBSCRIPTIONS
            var subscrBus   = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
            var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false);
            _mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.SystemStart, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.SubscribeToStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.PollStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.CheckPollTimeout, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <StorageMessage.EventCommitted, Message>());

            var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex);
            subscrBus.Subscribe <SystemMessage.SystemStart>(subscription);
            subscrBus.Subscribe <SystemMessage.BecomeShuttingDown>(subscription);
            subscrBus.Subscribe <TcpMessage.ConnectionClosed>(subscription);
            subscrBus.Subscribe <ClientMessage.SubscribeToStream>(subscription);
            subscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(subscription);
            subscrBus.Subscribe <SubscriptionMessage.PollStream>(subscription);
            subscrBus.Subscribe <SubscriptionMessage.CheckPollTimeout>(subscription);
            subscrBus.Subscribe <StorageMessage.EventCommitted>(subscription);

            // TIMER
            _timeProvider = new RealTimeProvider();
            _timerService = new TimerService(new ThreadBasedScheduler(_timeProvider));
            _mainBus.Subscribe <SystemMessage.BecomeShutdown>(_timerService);
            _mainBus.Subscribe <TimerMessage.Schedule>(_timerService);

            // MASTER REPLICATION
            var masterReplicationService = new MasterReplicationService(_mainQueue, _nodeInfo.InstanceId, db, _workersHandler,
                                                                        epochManager, vNodeSettings.ClusterNodeCount);
            _mainBus.Subscribe <SystemMessage.SystemStart>(masterReplicationService);
            _mainBus.Subscribe <SystemMessage.StateChangeMessage>(masterReplicationService);
            _mainBus.Subscribe <ReplicationMessage.ReplicaSubscriptionRequest>(masterReplicationService);
            _mainBus.Subscribe <ReplicationMessage.ReplicaLogPositionAck>(masterReplicationService);

            // REPLICA REPLICATION
            var replicaService = new ReplicaService(_mainQueue, db, epochManager, _workersHandler, authenticationProvider,
                                                    _nodeInfo, vNodeSettings.UseSsl, vNodeSettings.SslTargetHost, vNodeSettings.SslValidateServer);
            _mainBus.Subscribe <SystemMessage.StateChangeMessage>(replicaService);
            _mainBus.Subscribe <ReplicationMessage.ReconnectToMaster>(replicaService);
            _mainBus.Subscribe <ReplicationMessage.SubscribeToMaster>(replicaService);
            _mainBus.Subscribe <ReplicationMessage.AckLogPosition>(replicaService);
            _mainBus.Subscribe <StorageMessage.PrepareAck>(replicaService);
            _mainBus.Subscribe <StorageMessage.CommitAck>(replicaService);
            _mainBus.Subscribe <ClientMessage.TcpForwardMessage>(replicaService);

            // ELECTIONS
            var electionsService = new ElectionsService(_mainQueue, _nodeInfo, vNodeSettings.ClusterNodeCount,
                                                        db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint,
                                                        epochManager, () => readIndex.LastCommitPosition, vNodeSettings.NodePriority);
            electionsService.SubscribeMessages(_mainBus);

            // GOSSIP
            var gossip = new NodeGossipService(_mainQueue, gossipSeedSource, _nodeInfo, db.Config.WriterCheckpoint,
                                               db.Config.ChaserCheckpoint, epochManager, () => readIndex.LastCommitPosition,
                                               vNodeSettings.NodePriority);
            _mainBus.Subscribe <SystemMessage.SystemInit>(gossip);
            _mainBus.Subscribe <GossipMessage.RetrieveGossipSeedSources>(gossip);
            _mainBus.Subscribe <GossipMessage.GotGossipSeedSources>(gossip);
            _mainBus.Subscribe <GossipMessage.Gossip>(gossip);
            _mainBus.Subscribe <GossipMessage.GossipReceived>(gossip);
            _mainBus.Subscribe <SystemMessage.StateChangeMessage>(gossip);
            _mainBus.Subscribe <GossipMessage.GossipSendFailed>(gossip);
            _mainBus.Subscribe <SystemMessage.VNodeConnectionEstablished>(gossip);
            _mainBus.Subscribe <SystemMessage.VNodeConnectionLost>(gossip);

            _workersHandler.Start();
            _mainQueue.Start();
            monitoringQueue.Start();
            subscrQueue.Start();

            if (subsystems != null)
            {
                foreach (var subsystem in subsystems)
                {
                    subsystem.Register(db, _mainQueue, _mainBus, _timerService, _timeProvider,
                                       httpSendService, new[] { _internalHttpService, _externalHttpService }, _workersHandler);
                }
            }
        }
Ejemplo n.º 18
0
        public PlaygroundVNode(PlaygroundVNodeSettings vNodeSettings)
        {
            _tcpEndPoint  = vNodeSettings.ExternalTcpEndPoint;
            _httpEndPoint = vNodeSettings.ExternalHttpEndPoint;

            _mainBus    = new InMemoryBus("MainBus");
            _controller = new PlaygroundVNodeController(Bus, _httpEndPoint);
            _mainQueue  = new QueuedHandler(_controller, "MainQueue");
            _controller.SetMainQueue(MainQueue);

            // MONITORING
            var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
            var monitoringQueue    = new QueuedHandler(
                monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));

            Bus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.SystemInit, Message>());
            Bus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>());
            Bus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());


            // MISC WORKERS
            _workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
                                                                                   new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
                                                                                                   watchSlowMsg: true,
                                                                                                   slowMsgThreshold: TimeSpan.FromMilliseconds(50))).ToArray();
            _workersHandler = new MultiQueuedHandler(
                vNodeSettings.WorkerThreads,
                queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
                                                        string.Format("Worker #{0}", queueNum + 1),
                                                        groupName: "Workers",
                                                        watchSlowMsg: true,
                                                        slowMsgThreshold: TimeSpan.FromMilliseconds(50)));

            // AUTHENTICATION INFRASTRUCTURE
            var dispatcher                       = new IODispatcher(_mainBus, new PublishEnvelope(_workersHandler, crossThread: true));
            var passwordHashAlgorithm            = new Rfc2898PasswordHashAlgorithm();
            var internalAuthenticationProvider   = new InternalAuthenticationProvider(dispatcher, passwordHashAlgorithm, 1000);
            var passwordChangeNotificationReader = new PasswordChangeNotificationReader(_mainQueue, dispatcher);

            _mainBus.Subscribe <SystemMessage.SystemStart>(passwordChangeNotificationReader);
            _mainBus.Subscribe <SystemMessage.BecomeShutdown>(passwordChangeNotificationReader);
            _mainBus.Subscribe(internalAuthenticationProvider);
            _mainBus.Subscribe(dispatcher);

            SubscribeWorkers(bus =>
            {
                bus.Subscribe(dispatcher.ForwardReader);
                bus.Subscribe(dispatcher.BackwardReader);
                bus.Subscribe(dispatcher.Writer);
                bus.Subscribe(dispatcher.StreamDeleter);
            });

            // TCP
            var tcpService = new TcpService(
                MainQueue, _tcpEndPoint, _workersHandler, TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
                ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout, internalAuthenticationProvider, null);

            Bus.Subscribe <SystemMessage.SystemInit>(tcpService);
            Bus.Subscribe <SystemMessage.SystemStart>(tcpService);
            Bus.Subscribe <SystemMessage.BecomeShuttingDown>(tcpService);

            // HTTP
            {
                var authenticationProviders = new AuthenticationProvider[]
                {
                    new BasicHttpAuthenticationProvider(internalAuthenticationProvider),
                    new TrustedAuthenticationProvider(),
                    new AnonymousAuthenticationProvider()
                };

                var httpPipe        = new HttpMessagePipe();
                var httpSendService = new HttpSendService(httpPipe, forwardRequests: false);
                _mainBus.Subscribe <SystemMessage.StateChangeMessage>(httpSendService);
                _mainBus.Subscribe(new WideningHandler <HttpMessage.SendOverHttp, Message>(_workersHandler));
                SubscribeWorkers(bus =>
                {
                    bus.Subscribe <HttpMessage.HttpSend>(httpSendService);
                    bus.Subscribe <HttpMessage.HttpSendPart>(httpSendService);
                    bus.Subscribe <HttpMessage.HttpBeginSend>(httpSendService);
                    bus.Subscribe <HttpMessage.HttpEndSend>(httpSendService);
                    bus.Subscribe <HttpMessage.SendOverHttp>(httpSendService);
                });

                _httpService = new HttpService(ServiceAccessibility.Private, _mainQueue, new TrieUriRouter(),
                                               _workersHandler, vNodeSettings.HttpPrefixes);

                _mainBus.Subscribe <SystemMessage.SystemInit>(_httpService);
                _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_httpService);
                _mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_httpService);
                HttpService.SetupController(new AdminController(_mainQueue));
                HttpService.SetupController(new PingController());
                HttpService.SetupController(new StatController(monitoringQueue, _workersHandler));
                HttpService.SetupController(new AtomController(httpSendService, _mainQueue, _workersHandler));
                HttpService.SetupController(new GuidController(_mainQueue));
                HttpService.SetupController(new UsersController(httpSendService, _mainQueue, _workersHandler));

                SubscribeWorkers(bus => HttpService.CreateAndSubscribePipeline(bus, authenticationProviders));
            }


            // TIMER
            _timerService = new TimerService(new ThreadBasedScheduler(new RealTimeProvider()));
            Bus.Subscribe(TimerService);

            monitoringQueue.Start();
            MainQueue.Start();
        }
Ejemplo n.º 19
0
        public ClusterVNode(TFChunkDb db,
                            ClusterVNodeSettings vNodeSettings,
                            IGossipSeedSource gossipSeedSource,
                            bool dbVerifyHashes,
                            int memTableEntryCount,
                            params ISubsystem[] subsystems)
        {
            Ensure.NotNull(db, "db");
            Ensure.NotNull(vNodeSettings, "vNodeSettings");
            Ensure.NotNull(gossipSeedSource, "gossipSeedSource");

            _nodeInfo = vNodeSettings.NodeInfo;
            _mainBus = new InMemoryBus("MainBus");

            var forwardingProxy = new MessageForwardingProxy();
            // MISC WORKERS
            _workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
                new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
                                watchSlowMsg: true,
                                slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray();
            _workersHandler = new MultiQueuedHandler(
                    vNodeSettings.WorkerThreads,
                    queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
                                                            string.Format("Worker #{0}", queueNum + 1),
                                                            groupName: "Workers",
                                                            watchSlowMsg: true,
                                                            slowMsgThreshold: TimeSpan.FromMilliseconds(200)));

            _controller = new ClusterVNodeController(_mainBus, _nodeInfo, db, vNodeSettings, this, forwardingProxy);
            _mainQueue = new QueuedHandler(_controller, "MainQueue");
            _controller.SetMainQueue(_mainQueue);

            _subsystems = subsystems;

            // MONITORING
            var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
            var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
            var monitoringQueue = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
            var monitoring = new MonitoringService(monitoringQueue,
                                                   monitoringRequestBus,
                                                   _mainQueue,
                                                   db.Config.WriterCheckpoint,
                                                   db.Config.Path,
                                                   vNodeSettings.StatsPeriod,
                                                   _nodeInfo.ExternalHttp,
                                                   vNodeSettings.StatsStorage);
            _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.SystemInit, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.StateChangeMessage, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShutdown, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom<ClientMessage.WriteEventsCompleted, Message>());
            monitoringInnerBus.Subscribe<SystemMessage.SystemInit>(monitoring);
            monitoringInnerBus.Subscribe<SystemMessage.StateChangeMessage>(monitoring);
            monitoringInnerBus.Subscribe<SystemMessage.BecomeShuttingDown>(monitoring);
            monitoringInnerBus.Subscribe<SystemMessage.BecomeShutdown>(monitoring);
            monitoringInnerBus.Subscribe<ClientMessage.WriteEventsCompleted>(monitoring);
            monitoringInnerBus.Subscribe<MonitoringMessage.GetFreshStats>(monitoring);

            var truncPos = db.Config.TruncateCheckpoint.Read();
            if (truncPos != -1)
            {
                Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})",
                         truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read());
                var truncator = new TFChunkDbTruncator(db.Config);
                truncator.TruncateDb(truncPos);
            }

            // STORAGE SUBSYSTEM
            db.Open(dbVerifyHashes);
            var indexPath = Path.Combine(db.Config.Path, "index");
            var readerPool = new ObjectPool<ITransactionFileReader>(
                "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
                () => new TFChunkReader(db, db.Config.WriterCheckpoint));
            var tableIndex = new TableIndex(indexPath,
                                            () => new HashListMemTable(maxSize: memTableEntryCount * 2),
                                            () => new TFReaderLease(readerPool),
                                            maxSizeForMemory: memTableEntryCount,
                                            maxTablesPerLevel: 2,
                                            inMem: db.Config.InMemDb);
	        var hash = new XXHashUnsafe();
			var readIndex = new ReadIndex(_mainQueue,
                                          readerPool,
                                          tableIndex,
                                          hash,
                                          ESConsts.StreamInfoCacheCapacity,
                                          Application.IsDefined(Application.AdditionalCommitChecks),
                                          Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1);
            var writer = new TFChunkWriter(db);
            var epochManager = new EpochManager(ESConsts.CachedEpochCount,
                                                db.Config.EpochCheckpoint,
                                                writer,
                                                initialReaderCount: 1,
                                                maxReaderCount: 5,
                                                readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint));
            epochManager.Init();

            var storageWriter = new ClusterStorageWriterService(_mainQueue, _mainBus, vNodeSettings.MinFlushDelay,
                                                                db, writer, readIndex.IndexWriter, epochManager,
                                                                () => readIndex.LastCommitPosition); // subscribes internally
            monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageWriter);

            var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, ESConsts.StorageReaderThreadCount, db.Config.WriterCheckpoint);
            _mainBus.Subscribe<SystemMessage.SystemInit>(storageReader);
            _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageReader);
            _mainBus.Subscribe<SystemMessage.BecomeShutdown>(storageReader);
            monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageReader);

            var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
            var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, readIndex.IndexCommitter, epochManager);
            _mainBus.Subscribe<SystemMessage.SystemInit>(storageChaser);
            _mainBus.Subscribe<SystemMessage.SystemStart>(storageChaser);
            _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageChaser);

            var storageScavenger = new StorageScavenger(db, tableIndex, hash, readIndex,
                                                        Application.IsDefined(Application.AlwaysKeepScavenged),
                                                        mergeChunks: !vNodeSettings.DisableScavengeMerging);

			// ReSharper disable RedundantTypeArgumentsOfMethod
            _mainBus.Subscribe<ClientMessage.ScavengeDatabase>(storageScavenger);
            // ReSharper restore RedundantTypeArgumentsOfMethod

            // AUTHENTICATION INFRASTRUCTURE - delegate to plugins
	        var authenticationProvider = vNodeSettings.AuthenticationProviderFactory.BuildAuthenticationProvider(_mainQueue, _mainBus, _workersHandler, _workerBuses);
	        Ensure.NotNull(authenticationProvider, "authenticationProvider");
		
            {
                // EXTERNAL TCP
                var extTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalTcp, _workersHandler,
                                                   TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
                                                   ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
                                                   authenticationProvider, null);
                _mainBus.Subscribe<SystemMessage.SystemInit>(extTcpService);
                _mainBus.Subscribe<SystemMessage.SystemStart>(extTcpService);
                _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(extTcpService);

                // EXTERNAL SECURE TCP
                if (_nodeInfo.ExternalSecureTcp != null)
                {
                    var extSecTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalSecureTcp, _workersHandler,
                                                          TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(),
                                                          ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
                                                          authenticationProvider, vNodeSettings.Certificate);
                    _mainBus.Subscribe<SystemMessage.SystemInit>(extSecTcpService);
                    _mainBus.Subscribe<SystemMessage.SystemStart>(extSecTcpService);
                    _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(extSecTcpService);
                }

                // INTERNAL TCP
                var intTcpService = new TcpService(_mainQueue, _nodeInfo.InternalTcp, _workersHandler,
                                                   TcpServiceType.Internal, TcpSecurityType.Normal,
                                                   new InternalTcpDispatcher(),
                                                   ESConsts.InternalHeartbeatInterval, ESConsts.InternalHeartbeatTimeout,
                                                   authenticationProvider, null);
                _mainBus.Subscribe<SystemMessage.SystemInit>(intTcpService);
                _mainBus.Subscribe<SystemMessage.SystemStart>(intTcpService);
                _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(intTcpService);

                // INTERNAL SECURE TCP
                if (_nodeInfo.InternalSecureTcp != null)
                {
                    var intSecTcpService = new TcpService(_mainQueue, _nodeInfo.InternalSecureTcp, _workersHandler,
                                                           TcpServiceType.Internal, TcpSecurityType.Secure,
                                                           new InternalTcpDispatcher(),
                                                           ESConsts.InternalHeartbeatInterval, ESConsts.InternalHeartbeatTimeout,
                                                           authenticationProvider, vNodeSettings.Certificate);
                    _mainBus.Subscribe<SystemMessage.SystemInit>(intSecTcpService);
                    _mainBus.Subscribe<SystemMessage.SystemStart>(intSecTcpService);
                    _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(intSecTcpService);
                }
            }

            SubscribeWorkers(bus =>
            {
                var tcpSendService = new TcpSendService();
                // ReSharper disable RedundantTypeArgumentsOfMethod
                bus.Subscribe<TcpMessage.TcpSend>(tcpSendService);
                // ReSharper restore RedundantTypeArgumentsOfMethod
            });

            var httpAuthenticationProviders = new List<HttpAuthenticationProvider>
            {
                new BasicHttpAuthenticationProvider(authenticationProvider),
            };
            if (vNodeSettings.EnableTrustedAuth)
                httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider());
            httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider());

            var httpPipe = new HttpMessagePipe();
            var httpSendService = new HttpSendService(httpPipe, forwardRequests: true);
            _mainBus.Subscribe<SystemMessage.StateChangeMessage>(httpSendService);
            _mainBus.Subscribe(new WideningHandler<HttpMessage.SendOverHttp, Message>(_workersHandler));
            SubscribeWorkers(bus =>
            {
                bus.Subscribe<HttpMessage.HttpSend>(httpSendService);
                bus.Subscribe<HttpMessage.HttpSendPart>(httpSendService);
                bus.Subscribe<HttpMessage.HttpBeginSend>(httpSendService);
                bus.Subscribe<HttpMessage.HttpEndSend>(httpSendService);
                bus.Subscribe<HttpMessage.SendOverHttp>(httpSendService);
            });

            var adminController = new AdminController(_mainQueue);
            var pingController = new PingController();
            var statController = new StatController(monitoringQueue, _workersHandler);
            var atomController = new AtomController(httpSendService, _mainQueue, _workersHandler);
            var gossipController = new GossipController(_mainQueue, _workersHandler);
            var electController = new ElectController(_mainQueue);

            // HTTP SENDERS
            gossipController.SubscribeSenders(httpPipe);
            electController.SubscribeSenders(httpPipe);

            // EXTERNAL HTTP
            _externalHttpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(),
                                                    _workersHandler, vNodeSettings.HttpPrefixes);
            _externalHttpService.SetupController(adminController);
            _externalHttpService.SetupController(pingController);
            _externalHttpService.SetupController(statController);
            _externalHttpService.SetupController(atomController);
            _externalHttpService.SetupController(gossipController);
            
            _mainBus.Subscribe<SystemMessage.SystemInit>(_externalHttpService);
            _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(_externalHttpService);
            _mainBus.Subscribe<HttpMessage.PurgeTimedOutRequests>(_externalHttpService);

            // INTERNAL HTTP
            _internalHttpService = new HttpService(ServiceAccessibility.Private, _mainQueue, new TrieUriRouter(),
                                                    _workersHandler, _nodeInfo.InternalHttp.ToHttpUrl());
            _internalHttpService.SetupController(adminController);
            _internalHttpService.SetupController(pingController);
            _internalHttpService.SetupController(statController);
            _internalHttpService.SetupController(atomController);
            _internalHttpService.SetupController(gossipController);
            _internalHttpService.SetupController(electController);

			// Authentication plugin HTTP
	        vNodeSettings.AuthenticationProviderFactory.RegisterHttpControllers(_externalHttpService, _internalHttpService, httpSendService, _mainQueue, _workersHandler);

            _mainBus.Subscribe<SystemMessage.SystemInit>(_internalHttpService);
            _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(_internalHttpService);
            _mainBus.Subscribe<HttpMessage.PurgeTimedOutRequests>(_internalHttpService);

            SubscribeWorkers(bus =>
            {
                HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray());
            });

            // REQUEST FORWARDING
            var forwardingService = new RequestForwardingService(_mainQueue, forwardingProxy, TimeSpan.FromSeconds(1));
            _mainBus.Subscribe<SystemMessage.SystemStart>(forwardingService);
            _mainBus.Subscribe<SystemMessage.RequestForwardingTimerTick>(forwardingService);
            _mainBus.Subscribe<ClientMessage.NotHandled>(forwardingService);
            _mainBus.Subscribe<ClientMessage.WriteEventsCompleted>(forwardingService);
            _mainBus.Subscribe<ClientMessage.TransactionStartCompleted>(forwardingService);
            _mainBus.Subscribe<ClientMessage.TransactionWriteCompleted>(forwardingService);
            _mainBus.Subscribe<ClientMessage.TransactionCommitCompleted>(forwardingService);
            _mainBus.Subscribe<ClientMessage.DeleteStreamCompleted>(forwardingService);

            // REQUEST MANAGEMENT
            var requestManagement = new RequestManagementService(_mainQueue,
                                                                 vNodeSettings.PrepareAckCount,
                                                                 vNodeSettings.CommitAckCount,
                                                                 vNodeSettings.PrepareTimeout,
                                                                 vNodeSettings.CommitTimeout);
            _mainBus.Subscribe<SystemMessage.SystemInit>(requestManagement);
            _mainBus.Subscribe<ClientMessage.WriteEvents>(requestManagement);
            _mainBus.Subscribe<ClientMessage.TransactionStart>(requestManagement);
            _mainBus.Subscribe<ClientMessage.TransactionWrite>(requestManagement);
            _mainBus.Subscribe<ClientMessage.TransactionCommit>(requestManagement);
            _mainBus.Subscribe<ClientMessage.DeleteStream>(requestManagement);
            _mainBus.Subscribe<StorageMessage.RequestCompleted>(requestManagement);
            _mainBus.Subscribe<StorageMessage.CheckStreamAccessCompleted>(requestManagement);
            _mainBus.Subscribe<StorageMessage.AlreadyCommitted>(requestManagement);
            _mainBus.Subscribe<StorageMessage.CommitAck>(requestManagement);
            _mainBus.Subscribe<StorageMessage.PrepareAck>(requestManagement);
            _mainBus.Subscribe<StorageMessage.WrongExpectedVersion>(requestManagement);
            _mainBus.Subscribe<StorageMessage.InvalidTransaction>(requestManagement);
            _mainBus.Subscribe<StorageMessage.StreamDeleted>(requestManagement);
            _mainBus.Subscribe<StorageMessage.RequestManagerTimerTick>(requestManagement);

            // SUBSCRIPTIONS
            var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
            var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false);
            _mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.SystemStart, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<TcpMessage.ConnectionClosed, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.SubscribeToStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.UnsubscribeFromStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.PollStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.CheckPollTimeout, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<StorageMessage.EventCommited, Message>());

            var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex);
            subscrBus.Subscribe<SystemMessage.SystemStart>(subscription);
            subscrBus.Subscribe<SystemMessage.BecomeShuttingDown>(subscription);
            subscrBus.Subscribe<TcpMessage.ConnectionClosed>(subscription);
            subscrBus.Subscribe<ClientMessage.SubscribeToStream>(subscription);
            subscrBus.Subscribe<ClientMessage.UnsubscribeFromStream>(subscription);
            subscrBus.Subscribe<SubscriptionMessage.PollStream>(subscription);
            subscrBus.Subscribe<SubscriptionMessage.CheckPollTimeout>(subscription);
            subscrBus.Subscribe<StorageMessage.EventCommited>(subscription);

            // TIMER
            _timeProvider = new RealTimeProvider();
            _timerService = new TimerService(new ThreadBasedScheduler(_timeProvider));
            _mainBus.Subscribe<SystemMessage.BecomeShutdown>(_timerService);
            _mainBus.Subscribe<TimerMessage.Schedule>(_timerService);

            // MASTER REPLICATION
            var masterReplicationService = new MasterReplicationService(_mainQueue, _nodeInfo.InstanceId, db, _workersHandler,
                                                                        epochManager, vNodeSettings.ClusterNodeCount);
            _mainBus.Subscribe<SystemMessage.SystemStart>(masterReplicationService);
            _mainBus.Subscribe<SystemMessage.StateChangeMessage>(masterReplicationService);
            _mainBus.Subscribe<ReplicationMessage.ReplicaSubscriptionRequest>(masterReplicationService);
            _mainBus.Subscribe<ReplicationMessage.ReplicaLogPositionAck>(masterReplicationService);

            // REPLICA REPLICATION
            var replicaService = new ReplicaService(_mainQueue, db, epochManager, _workersHandler, authenticationProvider,
                                                    _nodeInfo, vNodeSettings.UseSsl, vNodeSettings.SslTargetHost, vNodeSettings.SslValidateServer);
            _mainBus.Subscribe<SystemMessage.StateChangeMessage>(replicaService);
            _mainBus.Subscribe<ReplicationMessage.ReconnectToMaster>(replicaService);
            _mainBus.Subscribe<ReplicationMessage.SubscribeToMaster>(replicaService);
            _mainBus.Subscribe<ReplicationMessage.AckLogPosition>(replicaService);
            _mainBus.Subscribe<StorageMessage.PrepareAck>(replicaService);
            _mainBus.Subscribe<StorageMessage.CommitAck>(replicaService);
            _mainBus.Subscribe<ClientMessage.TcpForwardMessage>(replicaService);

            // ELECTIONS
            var electionsService = new ElectionsService(_mainQueue, _nodeInfo, vNodeSettings.ClusterNodeCount,
                                                        db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint,
                                                        epochManager, () => readIndex.LastCommitPosition, vNodeSettings.NodePriority);
            electionsService.SubscribeMessages(_mainBus);

            // GOSSIP
            var gossip = new NodeGossipService(_mainQueue, gossipSeedSource, _nodeInfo, db.Config.WriterCheckpoint,
											   db.Config.ChaserCheckpoint, epochManager, () => readIndex.LastCommitPosition,
                                               vNodeSettings.NodePriority);
            _mainBus.Subscribe<SystemMessage.SystemInit>(gossip);
            _mainBus.Subscribe<GossipMessage.RetrieveGossipSeedSources>(gossip);
            _mainBus.Subscribe<GossipMessage.GotGossipSeedSources>(gossip);
            _mainBus.Subscribe<GossipMessage.Gossip>(gossip);
            _mainBus.Subscribe<GossipMessage.GossipReceived>(gossip);
            _mainBus.Subscribe<SystemMessage.StateChangeMessage>(gossip);
            _mainBus.Subscribe<GossipMessage.GossipSendFailed>(gossip);
            _mainBus.Subscribe<SystemMessage.VNodeConnectionEstablished>(gossip);
            _mainBus.Subscribe<SystemMessage.VNodeConnectionLost>(gossip);

            _workersHandler.Start();
            _mainQueue.Start();
            monitoringQueue.Start();
            subscrQueue.Start();

            if (subsystems != null)
            {
                foreach (var subsystem in subsystems)
                {
                    subsystem.Register(db, _mainQueue, _mainBus, _timerService, _timeProvider,
                                       httpSendService, new[] { _internalHttpService, _externalHttpService }, _workersHandler);
                }
            }
        }
Ejemplo n.º 20
0
        public ClusterVNode(TFChunkDb db,
                            ClusterVNodeSettings vNodeSettings,
                            IGossipSeedSource gossipSeedSource,
                            InfoController infoController,
                            params ISubsystem[] subsystems)
        {
            Ensure.NotNull(db, "db");
            Ensure.NotNull(vNodeSettings, "vNodeSettings");
            Ensure.NotNull(gossipSeedSource, "gossipSeedSource");

#if DEBUG
            AddTask(_taskAddedTrigger.Task);
#endif

            var isSingleNode = vNodeSettings.ClusterNodeCount == 1;
            _nodeInfo = vNodeSettings.NodeInfo;
            _mainBus  = new InMemoryBus("MainBus");

            var forwardingProxy = new MessageForwardingProxy();
            if (vNodeSettings.EnableHistograms)
            {
                HistogramService.CreateHistograms();
                //start watching jitter
                HistogramService.StartJitterMonitor();
            }
            // MISC WORKERS
            _workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
                                                                                   new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
                                                                                                   watchSlowMsg: true,
                                                                                                   slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray();
            _workersHandler = new MultiQueuedHandler(
                vNodeSettings.WorkerThreads,
                queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
                                                        string.Format("Worker #{0}", queueNum + 1),
                                                        groupName: "Workers",
                                                        watchSlowMsg: true,
                                                        slowMsgThreshold: TimeSpan.FromMilliseconds(200)));

            _subsystems = subsystems;

            _controller = new ClusterVNodeController((IPublisher)_mainBus, _nodeInfo, db, vNodeSettings, this, forwardingProxy, _subsystems);
            _mainQueue  = QueuedHandler.CreateQueuedHandler(_controller, "MainQueue");

            _controller.SetMainQueue(_mainQueue);

            //SELF
            _mainBus.Subscribe <SystemMessage.StateChangeMessage>(this);
            _mainBus.Subscribe <SystemMessage.BecomeShutdown>(this);
            // MONITORING
            var monitoringInnerBus   = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
            var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
            var monitoringQueue      = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
            var monitoring           = new MonitoringService(monitoringQueue,
                                                             monitoringRequestBus,
                                                             _mainQueue,
                                                             db.Config.WriterCheckpoint,
                                                             db.Config.Path,
                                                             vNodeSettings.StatsPeriod,
                                                             _nodeInfo.ExternalHttp,
                                                             vNodeSettings.StatsStorage,
                                                             _nodeInfo.ExternalTcp,
                                                             _nodeInfo.ExternalSecureTcp);
            _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.SystemInit, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShutdown, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom <ClientMessage.WriteEventsCompleted, Message>());
            monitoringInnerBus.Subscribe <SystemMessage.SystemInit>(monitoring);
            monitoringInnerBus.Subscribe <SystemMessage.StateChangeMessage>(monitoring);
            monitoringInnerBus.Subscribe <SystemMessage.BecomeShuttingDown>(monitoring);
            monitoringInnerBus.Subscribe <SystemMessage.BecomeShutdown>(monitoring);
            monitoringInnerBus.Subscribe <ClientMessage.WriteEventsCompleted>(monitoring);
            monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshStats>(monitoring);
            monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshTcpConnectionStats>(monitoring);

            var truncPos = db.Config.TruncateCheckpoint.Read();
            if (truncPos != -1)
            {
                Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})",
                         truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read());
                var truncator = new TFChunkDbTruncator(db.Config);
                truncator.TruncateDb(truncPos);
            }

            // STORAGE SUBSYSTEM
            db.Open(vNodeSettings.VerifyDbHash);
            var indexPath  = vNodeSettings.Index ?? Path.Combine(db.Config.Path, "index");
            var readerPool = new ObjectPool <ITransactionFileReader>(
                "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
                () => new TFChunkReader(db, db.Config.WriterCheckpoint, optimizeReadSideCache: db.Config.OptimizeReadSideCache));
            var tableIndex = new TableIndex(indexPath,
                                            new XXHashUnsafe(),
                                            new Murmur3AUnsafe(),
                                            () => new HashListMemTable(vNodeSettings.IndexBitnessVersion, maxSize: vNodeSettings.MaxMemtableEntryCount * 2),
                                            () => new TFReaderLease(readerPool),
                                            vNodeSettings.IndexBitnessVersion,
                                            maxSizeForMemory: vNodeSettings.MaxMemtableEntryCount,
                                            maxTablesPerLevel: 2,
                                            inMem: db.Config.InMemDb,
                                            skipIndexVerify: vNodeSettings.SkipIndexVerify,
                                            indexCacheDepth: vNodeSettings.IndexCacheDepth);
            var readIndex = new ReadIndex(_mainQueue,
                                          readerPool,
                                          tableIndex,
                                          ESConsts.StreamInfoCacheCapacity,
                                          Application.IsDefined(Application.AdditionalCommitChecks),
                                          Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1,
                                          vNodeSettings.HashCollisionReadLimit,
                                          vNodeSettings.SkipIndexScanOnReads,
                                          db.Config.ReplicationCheckpoint);
            var writer       = new TFChunkWriter(db);
            var epochManager = new EpochManager(_mainQueue,
                                                ESConsts.CachedEpochCount,
                                                db.Config.EpochCheckpoint,
                                                writer,
                                                initialReaderCount: 1,
                                                maxReaderCount: 5,
                                                readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint, optimizeReadSideCache: db.Config.OptimizeReadSideCache));
            epochManager.Init();

            var storageWriter = new ClusterStorageWriterService(_mainQueue, _mainBus, vNodeSettings.MinFlushDelay,
                                                                db, writer, readIndex.IndexWriter, epochManager,
                                                                () => readIndex.LastCommitPosition); // subscribes internally
            AddTasks(storageWriter.Tasks);

            monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageWriter);

            var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, vNodeSettings.ReaderThreadsCount, db.Config.WriterCheckpoint);
            _mainBus.Subscribe <SystemMessage.SystemInit>(storageReader);
            _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageReader);
            _mainBus.Subscribe <SystemMessage.BecomeShutdown>(storageReader);
            monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageReader);

            var indexCommitterService = new IndexCommitterService(readIndex.IndexCommitter, _mainQueue, db.Config.ReplicationCheckpoint, db.Config.WriterCheckpoint, vNodeSettings.CommitAckCount);
            AddTask(indexCommitterService.Task);

            _mainBus.Subscribe <SystemMessage.StateChangeMessage>(indexCommitterService);
            _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(indexCommitterService);
            _mainBus.Subscribe <StorageMessage.CommitAck>(indexCommitterService);

            var chaser        = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint, db.Config.OptimizeReadSideCache);
            var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, indexCommitterService, epochManager);
            AddTask(storageChaser.Task);

#if DEBUG
            QueueStatsCollector.InitializeCheckpoints(
                _nodeInfo.DebugIndex, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
#endif
            _mainBus.Subscribe <SystemMessage.SystemInit>(storageChaser);
            _mainBus.Subscribe <SystemMessage.SystemStart>(storageChaser);
            _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageChaser);

            // AUTHENTICATION INFRASTRUCTURE - delegate to plugins
            _internalAuthenticationProvider = vNodeSettings.AuthenticationProviderFactory.BuildAuthenticationProvider(_mainQueue, _mainBus, _workersHandler, _workerBuses);

            Ensure.NotNull(_internalAuthenticationProvider, "authenticationProvider");

            {
                // EXTERNAL TCP
                if (!vNodeSettings.DisableInsecureTCP)
                {
                    var extTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalTcp, _workersHandler,
                                                       TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
                                                       vNodeSettings.ExtTcpHeartbeatInterval, vNodeSettings.ExtTcpHeartbeatTimeout,
                                                       _internalAuthenticationProvider, null, vNodeSettings.ConnectionPendingSendBytesThreshold);
                    _mainBus.Subscribe <SystemMessage.SystemInit>(extTcpService);
                    _mainBus.Subscribe <SystemMessage.SystemStart>(extTcpService);
                    _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extTcpService);
                }

                // EXTERNAL SECURE TCP
                if (_nodeInfo.ExternalSecureTcp != null)
                {
                    var extSecTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalSecureTcp, _workersHandler,
                                                          TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(),
                                                          vNodeSettings.ExtTcpHeartbeatInterval, vNodeSettings.ExtTcpHeartbeatTimeout,
                                                          _internalAuthenticationProvider, vNodeSettings.Certificate, vNodeSettings.ConnectionPendingSendBytesThreshold);
                    _mainBus.Subscribe <SystemMessage.SystemInit>(extSecTcpService);
                    _mainBus.Subscribe <SystemMessage.SystemStart>(extSecTcpService);
                    _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extSecTcpService);
                }
                if (!isSingleNode)
                {
                    // INTERNAL TCP
                    if (!vNodeSettings.DisableInsecureTCP)
                    {
                        var intTcpService = new TcpService(_mainQueue, _nodeInfo.InternalTcp, _workersHandler,
                                                           TcpServiceType.Internal, TcpSecurityType.Normal,
                                                           new InternalTcpDispatcher(),
                                                           vNodeSettings.IntTcpHeartbeatInterval, vNodeSettings.IntTcpHeartbeatTimeout,
                                                           _internalAuthenticationProvider, null, ESConsts.UnrestrictedPendingSendBytes);
                        _mainBus.Subscribe <SystemMessage.SystemInit>(intTcpService);
                        _mainBus.Subscribe <SystemMessage.SystemStart>(intTcpService);
                        _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intTcpService);
                    }

                    // INTERNAL SECURE TCP
                    if (_nodeInfo.InternalSecureTcp != null)
                    {
                        var intSecTcpService = new TcpService(_mainQueue, _nodeInfo.InternalSecureTcp, _workersHandler,
                                                              TcpServiceType.Internal, TcpSecurityType.Secure,
                                                              new InternalTcpDispatcher(),
                                                              vNodeSettings.IntTcpHeartbeatInterval, vNodeSettings.IntTcpHeartbeatTimeout,
                                                              _internalAuthenticationProvider, vNodeSettings.Certificate, ESConsts.UnrestrictedPendingSendBytes);
                        _mainBus.Subscribe <SystemMessage.SystemInit>(intSecTcpService);
                        _mainBus.Subscribe <SystemMessage.SystemStart>(intSecTcpService);
                        _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intSecTcpService);
                    }
                }
            }

            SubscribeWorkers(bus =>
            {
                var tcpSendService = new TcpSendService();
                // ReSharper disable RedundantTypeArgumentsOfMethod
                bus.Subscribe <TcpMessage.TcpSend>(tcpSendService);
                // ReSharper restore RedundantTypeArgumentsOfMethod
            });

            var httpAuthenticationProviders = new List <HttpAuthenticationProvider>
            {
                new BasicHttpAuthenticationProvider(_internalAuthenticationProvider),
            };
            if (vNodeSettings.EnableTrustedAuth)
            {
                httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider());
            }
            httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider());

            var httpPipe        = new HttpMessagePipe();
            var httpSendService = new HttpSendService(httpPipe, forwardRequests: true);
            _mainBus.Subscribe <SystemMessage.StateChangeMessage>(httpSendService);
            _mainBus.Subscribe(new WideningHandler <HttpMessage.SendOverHttp, Message>(_workersHandler));
            SubscribeWorkers(bus =>
            {
                bus.Subscribe <HttpMessage.HttpSend>(httpSendService);
                bus.Subscribe <HttpMessage.HttpSendPart>(httpSendService);
                bus.Subscribe <HttpMessage.HttpBeginSend>(httpSendService);
                bus.Subscribe <HttpMessage.HttpEndSend>(httpSendService);
                bus.Subscribe <HttpMessage.SendOverHttp>(httpSendService);
            });

            _mainBus.Subscribe <SystemMessage.StateChangeMessage>(infoController);

            var adminController     = new AdminController(_mainQueue, _workersHandler);
            var pingController      = new PingController();
            var histogramController = new HistogramController();
            var statController      = new StatController(monitoringQueue, _workersHandler);
            var atomController      = new AtomController(httpSendService, _mainQueue, _workersHandler, vNodeSettings.DisableHTTPCaching);
            var gossipController    = new GossipController(_mainQueue, _workersHandler, vNodeSettings.GossipTimeout);
            var persistentSubscriptionController = new PersistentSubscriptionController(httpSendService, _mainQueue, _workersHandler);
            var electController = new ElectController(_mainQueue);

            // HTTP SENDERS
            gossipController.SubscribeSenders(httpPipe);
            electController.SubscribeSenders(httpPipe);

            // EXTERNAL HTTP
            _externalHttpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(),
                                                   _workersHandler, vNodeSettings.LogHttpRequests, vNodeSettings.GossipAdvertiseInfo.AdvertiseExternalIPAs,
                                                   vNodeSettings.GossipAdvertiseInfo.AdvertiseExternalHttpPortAs, vNodeSettings.ExtHttpPrefixes);
            _externalHttpService.SetupController(persistentSubscriptionController);
            if (vNodeSettings.AdminOnPublic)
            {
                _externalHttpService.SetupController(adminController);
            }
            _externalHttpService.SetupController(pingController);
            _externalHttpService.SetupController(infoController);
            if (vNodeSettings.StatsOnPublic)
            {
                _externalHttpService.SetupController(statController);
            }
            _externalHttpService.SetupController(atomController);
            if (vNodeSettings.GossipOnPublic)
            {
                _externalHttpService.SetupController(gossipController);
            }
            _externalHttpService.SetupController(histogramController);

            _mainBus.Subscribe <SystemMessage.SystemInit>(_externalHttpService);
            _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_externalHttpService);
            _mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_externalHttpService);
            // INTERNAL HTTP
            if (!isSingleNode)
            {
                _internalHttpService = new HttpService(ServiceAccessibility.Private, _mainQueue, new TrieUriRouter(),
                                                       _workersHandler, vNodeSettings.LogHttpRequests, vNodeSettings.GossipAdvertiseInfo.AdvertiseInternalIPAs,
                                                       vNodeSettings.GossipAdvertiseInfo.AdvertiseInternalHttpPortAs, vNodeSettings.IntHttpPrefixes);
                _internalHttpService.SetupController(adminController);
                _internalHttpService.SetupController(pingController);
                _internalHttpService.SetupController(infoController);
                _internalHttpService.SetupController(statController);
                _internalHttpService.SetupController(atomController);
                _internalHttpService.SetupController(gossipController);
                _internalHttpService.SetupController(electController);
                _internalHttpService.SetupController(histogramController);
                _internalHttpService.SetupController(persistentSubscriptionController);
            }
            // Authentication plugin HTTP
            vNodeSettings.AuthenticationProviderFactory.RegisterHttpControllers(_externalHttpService, _internalHttpService, httpSendService, _mainQueue, _workersHandler);
            if (_internalHttpService != null)
            {
                _mainBus.Subscribe <SystemMessage.SystemInit>(_internalHttpService);
                _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_internalHttpService);
                _mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_internalHttpService);
            }

            SubscribeWorkers(bus =>
            {
                HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray());
            });

            // REQUEST FORWARDING
            var forwardingService = new RequestForwardingService(_mainQueue, forwardingProxy, TimeSpan.FromSeconds(1));
            _mainBus.Subscribe <SystemMessage.SystemStart>(forwardingService);
            _mainBus.Subscribe <SystemMessage.RequestForwardingTimerTick>(forwardingService);
            _mainBus.Subscribe <ClientMessage.NotHandled>(forwardingService);
            _mainBus.Subscribe <ClientMessage.WriteEventsCompleted>(forwardingService);
            _mainBus.Subscribe <ClientMessage.TransactionStartCompleted>(forwardingService);
            _mainBus.Subscribe <ClientMessage.TransactionWriteCompleted>(forwardingService);
            _mainBus.Subscribe <ClientMessage.TransactionCommitCompleted>(forwardingService);
            _mainBus.Subscribe <ClientMessage.DeleteStreamCompleted>(forwardingService);

            // REQUEST MANAGEMENT
            var requestManagement = new RequestManagementService(_mainQueue,
                                                                 vNodeSettings.PrepareAckCount,
                                                                 vNodeSettings.PrepareTimeout,
                                                                 vNodeSettings.CommitTimeout,
                                                                 vNodeSettings.BetterOrdering);
            _mainBus.Subscribe <SystemMessage.SystemInit>(requestManagement);
            _mainBus.Subscribe <ClientMessage.WriteEvents>(requestManagement);
            _mainBus.Subscribe <ClientMessage.TransactionStart>(requestManagement);
            _mainBus.Subscribe <ClientMessage.TransactionWrite>(requestManagement);
            _mainBus.Subscribe <ClientMessage.TransactionCommit>(requestManagement);
            _mainBus.Subscribe <ClientMessage.DeleteStream>(requestManagement);
            _mainBus.Subscribe <StorageMessage.RequestCompleted>(requestManagement);
            _mainBus.Subscribe <StorageMessage.CheckStreamAccessCompleted>(requestManagement);
            _mainBus.Subscribe <StorageMessage.AlreadyCommitted>(requestManagement);
            _mainBus.Subscribe <StorageMessage.CommitReplicated>(requestManagement);
            _mainBus.Subscribe <StorageMessage.PrepareAck>(requestManagement);
            _mainBus.Subscribe <StorageMessage.WrongExpectedVersion>(requestManagement);
            _mainBus.Subscribe <StorageMessage.InvalidTransaction>(requestManagement);
            _mainBus.Subscribe <StorageMessage.StreamDeleted>(requestManagement);
            _mainBus.Subscribe <StorageMessage.RequestManagerTimerTick>(requestManagement);

            // SUBSCRIPTIONS
            var subscrBus   = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
            var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false);
            _mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.SystemStart, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.SubscribeToStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.PollStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.CheckPollTimeout, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom <StorageMessage.EventCommitted, Message>());

            var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex);
            subscrBus.Subscribe <SystemMessage.SystemStart>(subscription);
            subscrBus.Subscribe <SystemMessage.BecomeShuttingDown>(subscription);
            subscrBus.Subscribe <TcpMessage.ConnectionClosed>(subscription);
            subscrBus.Subscribe <ClientMessage.SubscribeToStream>(subscription);
            subscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(subscription);
            subscrBus.Subscribe <SubscriptionMessage.PollStream>(subscription);
            subscrBus.Subscribe <SubscriptionMessage.CheckPollTimeout>(subscription);
            subscrBus.Subscribe <StorageMessage.EventCommitted>(subscription);

            // PERSISTENT SUBSCRIPTIONS
            // IO DISPATCHER
            var ioDispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_mainQueue));
            _mainBus.Subscribe <ClientMessage.ReadStreamEventsBackwardCompleted>(ioDispatcher.BackwardReader);
            _mainBus.Subscribe <ClientMessage.WriteEventsCompleted>(ioDispatcher.Writer);
            _mainBus.Subscribe <ClientMessage.ReadStreamEventsForwardCompleted>(ioDispatcher.ForwardReader);
            _mainBus.Subscribe <ClientMessage.DeleteStreamCompleted>(ioDispatcher.StreamDeleter);
            _mainBus.Subscribe(ioDispatcher);
            var perSubscrBus   = new InMemoryBus("PersistentSubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
            var perSubscrQueue = new QueuedHandlerThreadPool(perSubscrBus, "PersistentSubscriptions", false);
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.CreatePersistentSubscription, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.UpdatePersistentSubscription, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.DeletePersistentSubscription, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ConnectToPersistentSubscription, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.PersistentSubscriptionAckEvents, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.PersistentSubscriptionNackEvents, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReplayAllParkedMessages, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReplayParkedMessage, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReadNextNPersistentMessages, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <StorageMessage.EventCommitted, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <MonitoringMessage.GetAllPersistentSubscriptionStats, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <MonitoringMessage.GetStreamPersistentSubscriptionStats, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <MonitoringMessage.GetPersistentSubscriptionStats, Message>());
            _mainBus.Subscribe(perSubscrQueue.WidenFrom <SubscriptionMessage.PersistentSubscriptionTimerTick, Message>());

            //TODO CC can have multiple threads working on subscription if partition
            var consumerStrategyRegistry = new PersistentSubscriptionConsumerStrategyRegistry(_mainQueue, _mainBus, vNodeSettings.AdditionalConsumerStrategies);
            var persistentSubscription   = new PersistentSubscriptionService(perSubscrQueue, readIndex, ioDispatcher, _mainQueue, consumerStrategyRegistry);
            perSubscrBus.Subscribe <SystemMessage.BecomeShuttingDown>(persistentSubscription);
            perSubscrBus.Subscribe <SystemMessage.BecomeMaster>(persistentSubscription);
            perSubscrBus.Subscribe <SystemMessage.StateChangeMessage>(persistentSubscription);
            perSubscrBus.Subscribe <TcpMessage.ConnectionClosed>(persistentSubscription);
            perSubscrBus.Subscribe <ClientMessage.ConnectToPersistentSubscription>(persistentSubscription);
            perSubscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(persistentSubscription);
            perSubscrBus.Subscribe <ClientMessage.PersistentSubscriptionAckEvents>(persistentSubscription);
            perSubscrBus.Subscribe <ClientMessage.PersistentSubscriptionNackEvents>(persistentSubscription);
            perSubscrBus.Subscribe <StorageMessage.EventCommitted>(persistentSubscription);
            perSubscrBus.Subscribe <ClientMessage.DeletePersistentSubscription>(persistentSubscription);
            perSubscrBus.Subscribe <ClientMessage.CreatePersistentSubscription>(persistentSubscription);
            perSubscrBus.Subscribe <ClientMessage.UpdatePersistentSubscription>(persistentSubscription);
            perSubscrBus.Subscribe <ClientMessage.ReplayAllParkedMessages>(persistentSubscription);
            perSubscrBus.Subscribe <ClientMessage.ReplayParkedMessage>(persistentSubscription);
            perSubscrBus.Subscribe <ClientMessage.ReadNextNPersistentMessages>(persistentSubscription);
            perSubscrBus.Subscribe <MonitoringMessage.GetAllPersistentSubscriptionStats>(persistentSubscription);
            perSubscrBus.Subscribe <MonitoringMessage.GetStreamPersistentSubscriptionStats>(persistentSubscription);
            perSubscrBus.Subscribe <MonitoringMessage.GetPersistentSubscriptionStats>(persistentSubscription);
            perSubscrBus.Subscribe <SubscriptionMessage.PersistentSubscriptionTimerTick>(persistentSubscription);

            // STORAGE SCAVENGER
            var scavengerLogManager = new TFChunkScavengerLogManager(_nodeInfo.ExternalHttp.ToString(), TimeSpan.FromDays(vNodeSettings.ScavengeHistoryMaxAge), ioDispatcher);
            var storageScavenger    = new StorageScavenger(db,
                                                           tableIndex,
                                                           readIndex,
                                                           scavengerLogManager,
                                                           vNodeSettings.AlwaysKeepScavenged,
                                                           !vNodeSettings.DisableScavengeMerging,
                                                           unsafeIgnoreHardDeletes: vNodeSettings.UnsafeIgnoreHardDeletes);

            // ReSharper disable RedundantTypeArgumentsOfMethod
            _mainBus.Subscribe <ClientMessage.ScavengeDatabase>(storageScavenger);
            _mainBus.Subscribe <ClientMessage.StopDatabaseScavenge>(storageScavenger);
            _mainBus.Subscribe <SystemMessage.StateChangeMessage>(storageScavenger);
            // ReSharper restore RedundantTypeArgumentsOfMethod


            // TIMER
            _timeProvider = new RealTimeProvider();
            var threadBasedScheduler = new ThreadBasedScheduler(_timeProvider);
            AddTask(threadBasedScheduler.Task);
            _timerService = new TimerService(threadBasedScheduler);
            _mainBus.Subscribe <SystemMessage.BecomeShutdown>(_timerService);
            _mainBus.Subscribe <TimerMessage.Schedule>(_timerService);

            var gossipInfo = new VNodeInfo(_nodeInfo.InstanceId, _nodeInfo.DebugIndex,
                                           vNodeSettings.GossipAdvertiseInfo.InternalTcp,
                                           vNodeSettings.GossipAdvertiseInfo.InternalSecureTcp,
                                           vNodeSettings.GossipAdvertiseInfo.ExternalTcp,
                                           vNodeSettings.GossipAdvertiseInfo.ExternalSecureTcp,
                                           vNodeSettings.GossipAdvertiseInfo.InternalHttp,
                                           vNodeSettings.GossipAdvertiseInfo.ExternalHttp);
            if (!isSingleNode)
            {
                // MASTER REPLICATION
                var masterReplicationService = new MasterReplicationService(_mainQueue, gossipInfo.InstanceId, db, _workersHandler,
                                                                            epochManager, vNodeSettings.ClusterNodeCount);
                AddTask(masterReplicationService.Task);
                _mainBus.Subscribe <SystemMessage.SystemStart>(masterReplicationService);
                _mainBus.Subscribe <SystemMessage.StateChangeMessage>(masterReplicationService);
                _mainBus.Subscribe <ReplicationMessage.ReplicaSubscriptionRequest>(masterReplicationService);
                _mainBus.Subscribe <ReplicationMessage.ReplicaLogPositionAck>(masterReplicationService);
                monitoringInnerBus.Subscribe <ReplicationMessage.GetReplicationStats>(masterReplicationService);

                // REPLICA REPLICATION
                var replicaService = new ReplicaService(_mainQueue, db, epochManager, _workersHandler, _internalAuthenticationProvider,
                                                        gossipInfo, vNodeSettings.UseSsl, vNodeSettings.SslTargetHost, vNodeSettings.SslValidateServer,
                                                        vNodeSettings.IntTcpHeartbeatTimeout, vNodeSettings.ExtTcpHeartbeatInterval);
                _mainBus.Subscribe <SystemMessage.StateChangeMessage>(replicaService);
                _mainBus.Subscribe <ReplicationMessage.ReconnectToMaster>(replicaService);
                _mainBus.Subscribe <ReplicationMessage.SubscribeToMaster>(replicaService);
                _mainBus.Subscribe <ReplicationMessage.AckLogPosition>(replicaService);
                _mainBus.Subscribe <StorageMessage.PrepareAck>(replicaService);
                _mainBus.Subscribe <StorageMessage.CommitAck>(replicaService);
                _mainBus.Subscribe <ClientMessage.TcpForwardMessage>(replicaService);
            }

            // ELECTIONS

            var electionsService = new ElectionsService(_mainQueue, gossipInfo, vNodeSettings.ClusterNodeCount,
                                                        db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint,
                                                        epochManager, () => readIndex.LastCommitPosition, vNodeSettings.NodePriority);
            electionsService.SubscribeMessages(_mainBus);
            if (!isSingleNode || vNodeSettings.GossipOnSingleNode)
            {
                // GOSSIP

                var gossip = new NodeGossipService(_mainQueue, gossipSeedSource, gossipInfo, db.Config.WriterCheckpoint,
                                                   db.Config.ChaserCheckpoint, epochManager, () => readIndex.LastCommitPosition,
                                                   vNodeSettings.NodePriority, vNodeSettings.GossipInterval, vNodeSettings.GossipAllowedTimeDifference);
                _mainBus.Subscribe <SystemMessage.SystemInit>(gossip);
                _mainBus.Subscribe <GossipMessage.RetrieveGossipSeedSources>(gossip);
                _mainBus.Subscribe <GossipMessage.GotGossipSeedSources>(gossip);
                _mainBus.Subscribe <GossipMessage.Gossip>(gossip);
                _mainBus.Subscribe <GossipMessage.GossipReceived>(gossip);
                _mainBus.Subscribe <SystemMessage.StateChangeMessage>(gossip);
                _mainBus.Subscribe <GossipMessage.GossipSendFailed>(gossip);
                _mainBus.Subscribe <SystemMessage.VNodeConnectionEstablished>(gossip);
                _mainBus.Subscribe <SystemMessage.VNodeConnectionLost>(gossip);
            }
            AddTasks(_workersHandler.Start());
            AddTask(_mainQueue.Start());
            AddTask(monitoringQueue.Start());
            AddTask(subscrQueue.Start());
            AddTask(perSubscrQueue.Start());

            if (subsystems != null)
            {
                foreach (var subsystem in subsystems)
                {
                    var http = isSingleNode ? new [] { _externalHttpService } : new [] { _internalHttpService, _externalHttpService };
                    subsystem.Register(new StandardComponents(db, _mainQueue, _mainBus, _timerService, _timeProvider, httpSendService, http, _workersHandler));
                }
            }
        }
 public InternalDispatcherEndpoint(IPublisher inputBus, MultiQueuedHandler requestsMultiHandler)
 {
     _inputBus             = inputBus;
     _requestsMultiHandler = requestsMultiHandler;
     _publishEnvelope      = new PublishEnvelope(inputBus);
 }