public virtual void Init()
        {
            var sendOverHttpHandler = GetSendOverHttpProcessor();

            for (int i = 0; i < InstancesCnt; ++i)
            {
                var inputBus = new InMemoryBus(string.Format("ELECTIONS-INPUT-BUS-{0}", i));
                var outputBus = new InMemoryBus(string.Format("ELECTIONS-OUTPUT-BUS-{0}", i));
                var endPoint = new IPEndPoint(BaseEndPoint.Address, BaseEndPoint.Port + i);
                var nodeInfo = new VNodeInfo(Guid.NewGuid(), endPoint, endPoint, endPoint, endPoint, endPoint, endPoint);
                _instances.Add(new ElectionsInstance(nodeInfo.InstanceId, endPoint, inputBus, outputBus));

                sendOverHttpHandler.RegisterEndPoint(endPoint, inputBus);

                var electionsService = new Core.Services.ElectionsService(outputBus,
                                                                             nodeInfo,
                                                                             InstancesCnt,
                                                                             new InMemoryCheckpoint(),
                                                                             new InMemoryCheckpoint(),
                                                                             new FakeEpochManager(),
                                                                             () => -1, 0);
                electionsService.SubscribeMessages(inputBus);

                outputBus.Subscribe(sendOverHttpHandler);
                outputBus.Subscribe(new TimerMessageProcessor(Rnd,
                                                              Runner,
                                                              endPoint,
                                                              inputBus,
                                                              _timerMinDelay,
                                                              _timerMaxDelay));
                outputBus.Subscribe(new InnerBusMessagesProcessor(Runner, endPoint, inputBus));
            }
        }
        public NodeGossipService(IPublisher bus,
                                 IGossipSeedSource gossipSeedSource,
								 VNodeInfo nodeInfo,
                                 ICheckpoint writerCheckpoint,
                                 ICheckpoint chaserCheckpoint,
                                 IEpochManager epochManager,
                                 Func<long> getLastCommitPosition,
                                 int nodePriority)
                : base(bus, gossipSeedSource, nodeInfo)
        {
            Ensure.NotNull(writerCheckpoint, "writerCheckpoint");
            Ensure.NotNull(chaserCheckpoint, "chaserCheckpoint");
            Ensure.NotNull(epochManager, "epochManager");
            Ensure.NotNull(getLastCommitPosition, "getLastCommitPosition");

            _writerCheckpoint = writerCheckpoint;
            _chaserCheckpoint = chaserCheckpoint;
            _epochManager = epochManager;
            _getLastCommitPosition = getLastCommitPosition;
            _nodePriority = nodePriority;
        }
 public BecomeSlave(Guid correlationId, VNodeInfo master): base(correlationId, VNodeState.Slave, master)
 {
 }
 public BecomeClone(Guid correlationId, VNodeInfo master): base(correlationId, VNodeState.Clone, master)
 {
 }
 public BecomeCatchingUp(Guid correlationId, VNodeInfo master): base(correlationId, VNodeState.CatchingUp, master)
 {
 }
 public BecomePreReplica(Guid correlationId, VNodeInfo master): base(correlationId, VNodeState.PreReplica, master)
 {
 }
 protected ReplicaStateMessage(Guid correlationId, VNodeState state, VNodeInfo master)
     : base(correlationId, state)
 {
     Ensure.NotNull(master, "master");
     Master = master;
 }
        public ClusterVNodeSettings(Guid instanceId, int debugIndex,
                                    IPEndPoint internalTcpEndPoint,
                                    IPEndPoint internalSecureTcpEndPoint,
                                    IPEndPoint externalTcpEndPoint,
                                    IPEndPoint externalSecureTcpEndPoint,
                                    IPEndPoint internalHttpEndPoint,
                                    IPEndPoint externalHttpEndPoint,
                                    string[] httpPrefixes,
                                    bool enableTrustedAuth,
                                    X509Certificate2 certificate,
                                    int workerThreads,
                                    bool discoverViaDns,
                                    string clusterDns,
                                    IPEndPoint[] gossipSeeds,
                                    TimeSpan minFlushDelay,
                                    int clusterNodeCount,
                                    int prepareAckCount,
                                    int commitAckCount,
                                    TimeSpan prepareTimeout,
                                    TimeSpan commitTimeout,
                                    bool useSsl,
                                    string sslTargetHost,
                                    bool sslValidateServer,
                                    TimeSpan statsPeriod,
                                    StatsStorage statsStorage,
                                    int nodePriority,
                                    IAuthenticationProviderFactory authenticationProviderFactory,
                                    bool disableScavengeMerging,
                                    bool adminOnPublic,
                                    bool statsOnPublic,
                                    bool gossipOnPublic,
                                    TimeSpan gossipInterval,
                                    TimeSpan gossipAllowedTimeDifference,
                                    TimeSpan gossipTimeout,
                                    TimeSpan tcpTimeout)
        {
            Ensure.NotEmptyGuid(instanceId, "instanceId");
            Ensure.NotNull(internalTcpEndPoint, "internalTcpEndPoint");
            Ensure.NotNull(externalTcpEndPoint, "externalTcpEndPoint");
            Ensure.NotNull(internalHttpEndPoint, "internalHttpEndPoint");
            Ensure.NotNull(externalHttpEndPoint, "externalHttpEndPoint");
            Ensure.NotNull(httpPrefixes, "httpPrefixes");
            if (internalSecureTcpEndPoint != null || externalSecureTcpEndPoint != null)
                Ensure.NotNull(certificate, "certificate");
            Ensure.Positive(workerThreads, "workerThreads");
            Ensure.NotNull(clusterDns, "clusterDns");
            Ensure.NotNull(gossipSeeds, "gossipSeeds");
            Ensure.Positive(clusterNodeCount, "clusterNodeCount");
            Ensure.Positive(prepareAckCount, "prepareAckCount");
            Ensure.Positive(commitAckCount, "commitAckCount");

            if (discoverViaDns && string.IsNullOrWhiteSpace(clusterDns))
                throw new ArgumentException("Either DNS Discovery must be disabled (and seeds specified), or a cluster DNS name must be provided.");

            if (useSsl)
                Ensure.NotNull(sslTargetHost, "sslTargetHost");

            NodeInfo = new VNodeInfo(instanceId, debugIndex,
                                     internalTcpEndPoint, internalSecureTcpEndPoint,
                                     externalTcpEndPoint, externalSecureTcpEndPoint,
                                     internalHttpEndPoint, externalHttpEndPoint);
            HttpPrefixes = httpPrefixes;
            EnableTrustedAuth = enableTrustedAuth;
            Certificate = certificate;
            WorkerThreads = workerThreads;

            DiscoverViaDns = discoverViaDns;
            ClusterDns = clusterDns;
            GossipSeeds = gossipSeeds;

            ClusterNodeCount = clusterNodeCount;
            MinFlushDelay = minFlushDelay;
            PrepareAckCount = prepareAckCount;
            CommitAckCount = commitAckCount;
            PrepareTimeout = prepareTimeout;
            CommitTimeout = commitTimeout;

            UseSsl = useSsl;
            SslTargetHost = sslTargetHost;
            SslValidateServer = sslValidateServer;

            StatsPeriod = statsPeriod;
            StatsStorage = statsStorage;

            AuthenticationProviderFactory = authenticationProviderFactory;

            NodePriority = nodePriority;
            DisableScavengeMerging = disableScavengeMerging;
            AdminOnPublic = adminOnPublic;
            StatsOnPublic = statsOnPublic;
            GossipOnPublic = gossipOnPublic;
            GossipInterval = gossipInterval;
            GossipAllowedTimeDifference = gossipAllowedTimeDifference;
            GossipTimeout = gossipTimeout;
            TcpTimeout = tcpTimeout;
        }
예제 #9
0
        public ClusterVNodeSettings(Guid instanceId, int debugIndex,
                                    IPEndPoint internalTcpEndPoint,
                                    IPEndPoint internalSecureTcpEndPoint,
                                    IPEndPoint externalTcpEndPoint,
                                    IPEndPoint externalSecureTcpEndPoint,
                                    IPEndPoint internalHttpEndPoint,
                                    IPEndPoint externalHttpEndPoint,
                                    GossipAdvertiseInfo gossipAdvertiseInfo,
                                    string[] intHttpPrefixes,
                                    string[] extHttpPrefixes,
                                    bool enableTrustedAuth,
                                    X509Certificate2 certificate,
                                    int workerThreads,
                                    bool discoverViaDns,
                                    string clusterDns,
                                    IPEndPoint[] gossipSeeds,
                                    TimeSpan minFlushDelay,
                                    int clusterNodeCount,
                                    int prepareAckCount,
                                    int commitAckCount,
                                    TimeSpan prepareTimeout,
                                    TimeSpan commitTimeout,
                                    bool useSsl,
                                    string sslTargetHost,
                                    bool sslValidateServer,
                                    TimeSpan statsPeriod,
                                    StatsStorage statsStorage,
                                    int nodePriority,
                                    IAuthenticationProviderFactory authenticationProviderFactory,
                                    bool disableScavengeMerging,
                                    bool adminOnPublic,
                                    bool statsOnPublic,
                                    bool gossipOnPublic,
                                    TimeSpan gossipInterval,
                                    TimeSpan gossipAllowedTimeDifference,
                                    TimeSpan gossipTimeout,
                                    TimeSpan intTcpHeartbeatTimeout,
                                    TimeSpan intTcpHeartbeatInterval,
                                    TimeSpan extTcpHeartbeatTimeout,
                                    TimeSpan extTcpHeartbeatInterval,
				                    bool verifyDbHash,
				                    int maxMemtableEntryCount,
                                    bool startStandardProjections,
                                    bool disableHTTPCaching, string index = null, bool enableHistograms = false,
                                    int indexCacheDepth = 16,
                                    IPersistentSubscriptionConsumerStrategyFactory[] additionalConsumerStrategies = null)
        {
            Ensure.NotEmptyGuid(instanceId, "instanceId");
            Ensure.NotNull(internalTcpEndPoint, "internalTcpEndPoint");
            Ensure.NotNull(externalTcpEndPoint, "externalTcpEndPoint");
            Ensure.NotNull(internalHttpEndPoint, "internalHttpEndPoint");
            Ensure.NotNull(externalHttpEndPoint, "externalHttpEndPoint");
            Ensure.NotNull(intHttpPrefixes, "intHttpPrefixes");
            Ensure.NotNull(extHttpPrefixes, "extHttpPrefixes");
            if (internalSecureTcpEndPoint != null || externalSecureTcpEndPoint != null)
                Ensure.NotNull(certificate, "certificate");
            Ensure.Positive(workerThreads, "workerThreads");
            Ensure.NotNull(clusterDns, "clusterDns");
            Ensure.NotNull(gossipSeeds, "gossipSeeds");
            Ensure.Positive(clusterNodeCount, "clusterNodeCount");
            Ensure.Positive(prepareAckCount, "prepareAckCount");
            Ensure.Positive(commitAckCount, "commitAckCount");
            Ensure.NotNull(gossipAdvertiseInfo, "gossipAdvertiseInfo");

            if (discoverViaDns && string.IsNullOrWhiteSpace(clusterDns))
                throw new ArgumentException("Either DNS Discovery must be disabled (and seeds specified), or a cluster DNS name must be provided.");

            if (useSsl)
                Ensure.NotNull(sslTargetHost, "sslTargetHost");

            NodeInfo = new VNodeInfo(instanceId, debugIndex,
                                     internalTcpEndPoint, internalSecureTcpEndPoint,
                                     externalTcpEndPoint, externalSecureTcpEndPoint,
                                     internalHttpEndPoint, externalHttpEndPoint);
            GossipAdvertiseInfo = gossipAdvertiseInfo;
            IntHttpPrefixes = intHttpPrefixes;
            ExtHttpPrefixes = extHttpPrefixes;
            EnableTrustedAuth = enableTrustedAuth;
            Certificate = certificate;
            WorkerThreads = workerThreads;
            StartStandardProjections = startStandardProjections;
            DisableHTTPCaching = disableHTTPCaching;
            AdditionalConsumerStrategies = additionalConsumerStrategies ?? new IPersistentSubscriptionConsumerStrategyFactory[0];

            DiscoverViaDns = discoverViaDns;
            ClusterDns = clusterDns;
            GossipSeeds = gossipSeeds;

            ClusterNodeCount = clusterNodeCount;
            MinFlushDelay = minFlushDelay;
            PrepareAckCount = prepareAckCount;
            CommitAckCount = commitAckCount;
            PrepareTimeout = prepareTimeout;
            CommitTimeout = commitTimeout;

            UseSsl = useSsl;
            SslTargetHost = sslTargetHost;
            SslValidateServer = sslValidateServer;

            StatsPeriod = statsPeriod;
            StatsStorage = statsStorage;

            AuthenticationProviderFactory = authenticationProviderFactory;

            NodePriority = nodePriority;
            DisableScavengeMerging = disableScavengeMerging;
            AdminOnPublic = adminOnPublic;
            StatsOnPublic = statsOnPublic;
            GossipOnPublic = gossipOnPublic;
            GossipInterval = gossipInterval;
            GossipAllowedTimeDifference = gossipAllowedTimeDifference;
            GossipTimeout = gossipTimeout;
            IntTcpHeartbeatTimeout = intTcpHeartbeatTimeout;
            IntTcpHeartbeatInterval = intTcpHeartbeatInterval;
            ExtTcpHeartbeatTimeout = extTcpHeartbeatTimeout;
            ExtTcpHeartbeatInterval = extTcpHeartbeatInterval;

            VerifyDbHash = verifyDbHash;
            MaxMemtableEntryCount = maxMemtableEntryCount;

            EnableHistograms = enableHistograms;
            IndexCacheDepth = indexCacheDepth;
            Index = index;
        }
예제 #10
0
        public ClusterVNode(TFChunkDb db,
                            ClusterVNodeSettings vNodeSettings,
                            IGossipSeedSource gossipSeedSource,
                            bool dbVerifyHashes,
                            int memTableEntryCount,
                            params ISubsystem[] subsystems)
        {
            Ensure.NotNull(db, "db");
            Ensure.NotNull(vNodeSettings, "vNodeSettings");
            Ensure.NotNull(gossipSeedSource, "gossipSeedSource");

            _nodeInfo = vNodeSettings.NodeInfo;
            _mainBus = new InMemoryBus("MainBus");

            var forwardingProxy = new MessageForwardingProxy();
            // MISC WORKERS
            _workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
                new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
                                watchSlowMsg: true,
                                slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray();
            _workersHandler = new MultiQueuedHandler(
                    vNodeSettings.WorkerThreads,
                    queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
                                                            string.Format("Worker #{0}", queueNum + 1),
                                                            groupName: "Workers",
                                                            watchSlowMsg: true,
                                                            slowMsgThreshold: TimeSpan.FromMilliseconds(200)));

            _controller = new ClusterVNodeController(_mainBus, _nodeInfo, db, vNodeSettings, this, forwardingProxy);
            _mainQueue = new QueuedHandler(_controller, "MainQueue");
            _controller.SetMainQueue(_mainQueue);

            _subsystems = subsystems;

            // MONITORING
            var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
            var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
            var monitoringQueue = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
            var monitoring = new MonitoringService(monitoringQueue,
                                                   monitoringRequestBus,
                                                   _mainQueue,
                                                   db.Config.WriterCheckpoint,
                                                   db.Config.Path,
                                                   vNodeSettings.StatsPeriod,
                                                   _nodeInfo.ExternalHttp,
                                                   vNodeSettings.StatsStorage);
            _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.SystemInit, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.StateChangeMessage, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShutdown, Message>());
            _mainBus.Subscribe(monitoringQueue.WidenFrom<ClientMessage.WriteEventsCompleted, Message>());
            monitoringInnerBus.Subscribe<SystemMessage.SystemInit>(monitoring);
            monitoringInnerBus.Subscribe<SystemMessage.StateChangeMessage>(monitoring);
            monitoringInnerBus.Subscribe<SystemMessage.BecomeShuttingDown>(monitoring);
            monitoringInnerBus.Subscribe<SystemMessage.BecomeShutdown>(monitoring);
            monitoringInnerBus.Subscribe<ClientMessage.WriteEventsCompleted>(monitoring);
            monitoringInnerBus.Subscribe<MonitoringMessage.GetFreshStats>(monitoring);

            var truncPos = db.Config.TruncateCheckpoint.Read();
            if (truncPos != -1)
            {
                Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})",
                         truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read());
                var truncator = new TFChunkDbTruncator(db.Config);
                truncator.TruncateDb(truncPos);
            }

            // STORAGE SUBSYSTEM
            db.Open(dbVerifyHashes);
            var indexPath = Path.Combine(db.Config.Path, "index");
            var readerPool = new ObjectPool<ITransactionFileReader>(
                "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
                () => new TFChunkReader(db, db.Config.WriterCheckpoint));
            var tableIndex = new TableIndex(indexPath,
                                            () => new HashListMemTable(maxSize: memTableEntryCount * 2),
                                            () => new TFReaderLease(readerPool),
                                            maxSizeForMemory: memTableEntryCount,
                                            maxTablesPerLevel: 2,
                                            inMem: db.Config.InMemDb);
	        var hash = new XXHashUnsafe();
			var readIndex = new ReadIndex(_mainQueue,
                                          readerPool,
                                          tableIndex,
                                          hash,
                                          ESConsts.StreamInfoCacheCapacity,
                                          Application.IsDefined(Application.AdditionalCommitChecks),
                                          Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1);
            var writer = new TFChunkWriter(db);
            var epochManager = new EpochManager(ESConsts.CachedEpochCount,
                                                db.Config.EpochCheckpoint,
                                                writer,
                                                initialReaderCount: 1,
                                                maxReaderCount: 5,
                                                readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint));
            epochManager.Init();

            var storageWriter = new ClusterStorageWriterService(_mainQueue, _mainBus, vNodeSettings.MinFlushDelay,
                                                                db, writer, readIndex.IndexWriter, epochManager,
                                                                () => readIndex.LastCommitPosition); // subscribes internally
            monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageWriter);

            var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, ESConsts.StorageReaderThreadCount, db.Config.WriterCheckpoint);
            _mainBus.Subscribe<SystemMessage.SystemInit>(storageReader);
            _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageReader);
            _mainBus.Subscribe<SystemMessage.BecomeShutdown>(storageReader);
            monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageReader);

            var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
            var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, readIndex.IndexCommitter, epochManager);
            _mainBus.Subscribe<SystemMessage.SystemInit>(storageChaser);
            _mainBus.Subscribe<SystemMessage.SystemStart>(storageChaser);
            _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageChaser);

            var storageScavenger = new StorageScavenger(db, tableIndex, hash, readIndex,
                                                        Application.IsDefined(Application.AlwaysKeepScavenged),
                                                        mergeChunks: !vNodeSettings.DisableScavengeMerging);

			// ReSharper disable RedundantTypeArgumentsOfMethod
            _mainBus.Subscribe<ClientMessage.ScavengeDatabase>(storageScavenger);
            // ReSharper restore RedundantTypeArgumentsOfMethod

            // AUTHENTICATION INFRASTRUCTURE - delegate to plugins
	        var authenticationProvider = vNodeSettings.AuthenticationProviderFactory.BuildAuthenticationProvider(_mainQueue, _mainBus, _workersHandler, _workerBuses);
	        Ensure.NotNull(authenticationProvider, "authenticationProvider");
		
            {
                // EXTERNAL TCP
                var extTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalTcp, _workersHandler,
                                                   TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
                                                   ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
                                                   authenticationProvider, null);
                _mainBus.Subscribe<SystemMessage.SystemInit>(extTcpService);
                _mainBus.Subscribe<SystemMessage.SystemStart>(extTcpService);
                _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(extTcpService);

                // EXTERNAL SECURE TCP
                if (_nodeInfo.ExternalSecureTcp != null)
                {
                    var extSecTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalSecureTcp, _workersHandler,
                                                          TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(),
                                                          ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
                                                          authenticationProvider, vNodeSettings.Certificate);
                    _mainBus.Subscribe<SystemMessage.SystemInit>(extSecTcpService);
                    _mainBus.Subscribe<SystemMessage.SystemStart>(extSecTcpService);
                    _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(extSecTcpService);
                }

                // INTERNAL TCP
                var intTcpService = new TcpService(_mainQueue, _nodeInfo.InternalTcp, _workersHandler,
                                                   TcpServiceType.Internal, TcpSecurityType.Normal,
                                                   new InternalTcpDispatcher(),
                                                   ESConsts.InternalHeartbeatInterval, ESConsts.InternalHeartbeatTimeout,
                                                   authenticationProvider, null);
                _mainBus.Subscribe<SystemMessage.SystemInit>(intTcpService);
                _mainBus.Subscribe<SystemMessage.SystemStart>(intTcpService);
                _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(intTcpService);

                // INTERNAL SECURE TCP
                if (_nodeInfo.InternalSecureTcp != null)
                {
                    var intSecTcpService = new TcpService(_mainQueue, _nodeInfo.InternalSecureTcp, _workersHandler,
                                                           TcpServiceType.Internal, TcpSecurityType.Secure,
                                                           new InternalTcpDispatcher(),
                                                           ESConsts.InternalHeartbeatInterval, ESConsts.InternalHeartbeatTimeout,
                                                           authenticationProvider, vNodeSettings.Certificate);
                    _mainBus.Subscribe<SystemMessage.SystemInit>(intSecTcpService);
                    _mainBus.Subscribe<SystemMessage.SystemStart>(intSecTcpService);
                    _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(intSecTcpService);
                }
            }

            SubscribeWorkers(bus =>
            {
                var tcpSendService = new TcpSendService();
                // ReSharper disable RedundantTypeArgumentsOfMethod
                bus.Subscribe<TcpMessage.TcpSend>(tcpSendService);
                // ReSharper restore RedundantTypeArgumentsOfMethod
            });

            var httpAuthenticationProviders = new List<HttpAuthenticationProvider>
            {
                new BasicHttpAuthenticationProvider(authenticationProvider),
            };
            if (vNodeSettings.EnableTrustedAuth)
                httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider());
            httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider());

            var httpPipe = new HttpMessagePipe();
            var httpSendService = new HttpSendService(httpPipe, forwardRequests: true);
            _mainBus.Subscribe<SystemMessage.StateChangeMessage>(httpSendService);
            _mainBus.Subscribe(new WideningHandler<HttpMessage.SendOverHttp, Message>(_workersHandler));
            SubscribeWorkers(bus =>
            {
                bus.Subscribe<HttpMessage.HttpSend>(httpSendService);
                bus.Subscribe<HttpMessage.HttpSendPart>(httpSendService);
                bus.Subscribe<HttpMessage.HttpBeginSend>(httpSendService);
                bus.Subscribe<HttpMessage.HttpEndSend>(httpSendService);
                bus.Subscribe<HttpMessage.SendOverHttp>(httpSendService);
            });

            var adminController = new AdminController(_mainQueue);
            var pingController = new PingController();
            var statController = new StatController(monitoringQueue, _workersHandler);
            var atomController = new AtomController(httpSendService, _mainQueue, _workersHandler);
            var gossipController = new GossipController(_mainQueue, _workersHandler);
            var electController = new ElectController(_mainQueue);

            // HTTP SENDERS
            gossipController.SubscribeSenders(httpPipe);
            electController.SubscribeSenders(httpPipe);

            // EXTERNAL HTTP
            _externalHttpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(),
                                                    _workersHandler, vNodeSettings.HttpPrefixes);
            _externalHttpService.SetupController(adminController);
            _externalHttpService.SetupController(pingController);
            _externalHttpService.SetupController(statController);
            _externalHttpService.SetupController(atomController);
            _externalHttpService.SetupController(gossipController);
            
            _mainBus.Subscribe<SystemMessage.SystemInit>(_externalHttpService);
            _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(_externalHttpService);
            _mainBus.Subscribe<HttpMessage.PurgeTimedOutRequests>(_externalHttpService);

            // INTERNAL HTTP
            _internalHttpService = new HttpService(ServiceAccessibility.Private, _mainQueue, new TrieUriRouter(),
                                                    _workersHandler, _nodeInfo.InternalHttp.ToHttpUrl());
            _internalHttpService.SetupController(adminController);
            _internalHttpService.SetupController(pingController);
            _internalHttpService.SetupController(statController);
            _internalHttpService.SetupController(atomController);
            _internalHttpService.SetupController(gossipController);
            _internalHttpService.SetupController(electController);

			// Authentication plugin HTTP
	        vNodeSettings.AuthenticationProviderFactory.RegisterHttpControllers(_externalHttpService, _internalHttpService, httpSendService, _mainQueue, _workersHandler);

            _mainBus.Subscribe<SystemMessage.SystemInit>(_internalHttpService);
            _mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(_internalHttpService);
            _mainBus.Subscribe<HttpMessage.PurgeTimedOutRequests>(_internalHttpService);

            SubscribeWorkers(bus =>
            {
                HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray());
            });

            // REQUEST FORWARDING
            var forwardingService = new RequestForwardingService(_mainQueue, forwardingProxy, TimeSpan.FromSeconds(1));
            _mainBus.Subscribe<SystemMessage.SystemStart>(forwardingService);
            _mainBus.Subscribe<SystemMessage.RequestForwardingTimerTick>(forwardingService);
            _mainBus.Subscribe<ClientMessage.NotHandled>(forwardingService);
            _mainBus.Subscribe<ClientMessage.WriteEventsCompleted>(forwardingService);
            _mainBus.Subscribe<ClientMessage.TransactionStartCompleted>(forwardingService);
            _mainBus.Subscribe<ClientMessage.TransactionWriteCompleted>(forwardingService);
            _mainBus.Subscribe<ClientMessage.TransactionCommitCompleted>(forwardingService);
            _mainBus.Subscribe<ClientMessage.DeleteStreamCompleted>(forwardingService);

            // REQUEST MANAGEMENT
            var requestManagement = new RequestManagementService(_mainQueue,
                                                                 vNodeSettings.PrepareAckCount,
                                                                 vNodeSettings.CommitAckCount,
                                                                 vNodeSettings.PrepareTimeout,
                                                                 vNodeSettings.CommitTimeout);
            _mainBus.Subscribe<SystemMessage.SystemInit>(requestManagement);
            _mainBus.Subscribe<ClientMessage.WriteEvents>(requestManagement);
            _mainBus.Subscribe<ClientMessage.TransactionStart>(requestManagement);
            _mainBus.Subscribe<ClientMessage.TransactionWrite>(requestManagement);
            _mainBus.Subscribe<ClientMessage.TransactionCommit>(requestManagement);
            _mainBus.Subscribe<ClientMessage.DeleteStream>(requestManagement);
            _mainBus.Subscribe<StorageMessage.RequestCompleted>(requestManagement);
            _mainBus.Subscribe<StorageMessage.CheckStreamAccessCompleted>(requestManagement);
            _mainBus.Subscribe<StorageMessage.AlreadyCommitted>(requestManagement);
            _mainBus.Subscribe<StorageMessage.CommitAck>(requestManagement);
            _mainBus.Subscribe<StorageMessage.PrepareAck>(requestManagement);
            _mainBus.Subscribe<StorageMessage.WrongExpectedVersion>(requestManagement);
            _mainBus.Subscribe<StorageMessage.InvalidTransaction>(requestManagement);
            _mainBus.Subscribe<StorageMessage.StreamDeleted>(requestManagement);
            _mainBus.Subscribe<StorageMessage.RequestManagerTimerTick>(requestManagement);

            // SUBSCRIPTIONS
            var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
            var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false);
            _mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.SystemStart, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<TcpMessage.ConnectionClosed, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.SubscribeToStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.UnsubscribeFromStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.PollStream, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.CheckPollTimeout, Message>());
            _mainBus.Subscribe(subscrQueue.WidenFrom<StorageMessage.EventCommited, Message>());

            var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex);
            subscrBus.Subscribe<SystemMessage.SystemStart>(subscription);
            subscrBus.Subscribe<SystemMessage.BecomeShuttingDown>(subscription);
            subscrBus.Subscribe<TcpMessage.ConnectionClosed>(subscription);
            subscrBus.Subscribe<ClientMessage.SubscribeToStream>(subscription);
            subscrBus.Subscribe<ClientMessage.UnsubscribeFromStream>(subscription);
            subscrBus.Subscribe<SubscriptionMessage.PollStream>(subscription);
            subscrBus.Subscribe<SubscriptionMessage.CheckPollTimeout>(subscription);
            subscrBus.Subscribe<StorageMessage.EventCommited>(subscription);

            // TIMER
            _timeProvider = new RealTimeProvider();
            _timerService = new TimerService(new ThreadBasedScheduler(_timeProvider));
            _mainBus.Subscribe<SystemMessage.BecomeShutdown>(_timerService);
            _mainBus.Subscribe<TimerMessage.Schedule>(_timerService);

            // MASTER REPLICATION
            var masterReplicationService = new MasterReplicationService(_mainQueue, _nodeInfo.InstanceId, db, _workersHandler,
                                                                        epochManager, vNodeSettings.ClusterNodeCount);
            _mainBus.Subscribe<SystemMessage.SystemStart>(masterReplicationService);
            _mainBus.Subscribe<SystemMessage.StateChangeMessage>(masterReplicationService);
            _mainBus.Subscribe<ReplicationMessage.ReplicaSubscriptionRequest>(masterReplicationService);
            _mainBus.Subscribe<ReplicationMessage.ReplicaLogPositionAck>(masterReplicationService);

            // REPLICA REPLICATION
            var replicaService = new ReplicaService(_mainQueue, db, epochManager, _workersHandler, authenticationProvider,
                                                    _nodeInfo, vNodeSettings.UseSsl, vNodeSettings.SslTargetHost, vNodeSettings.SslValidateServer);
            _mainBus.Subscribe<SystemMessage.StateChangeMessage>(replicaService);
            _mainBus.Subscribe<ReplicationMessage.ReconnectToMaster>(replicaService);
            _mainBus.Subscribe<ReplicationMessage.SubscribeToMaster>(replicaService);
            _mainBus.Subscribe<ReplicationMessage.AckLogPosition>(replicaService);
            _mainBus.Subscribe<StorageMessage.PrepareAck>(replicaService);
            _mainBus.Subscribe<StorageMessage.CommitAck>(replicaService);
            _mainBus.Subscribe<ClientMessage.TcpForwardMessage>(replicaService);

            // ELECTIONS
            var electionsService = new ElectionsService(_mainQueue, _nodeInfo, vNodeSettings.ClusterNodeCount,
                                                        db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint,
                                                        epochManager, () => readIndex.LastCommitPosition, vNodeSettings.NodePriority);
            electionsService.SubscribeMessages(_mainBus);

            // GOSSIP
            var gossip = new NodeGossipService(_mainQueue, gossipSeedSource, _nodeInfo, db.Config.WriterCheckpoint,
											   db.Config.ChaserCheckpoint, epochManager, () => readIndex.LastCommitPosition,
                                               vNodeSettings.NodePriority);
            _mainBus.Subscribe<SystemMessage.SystemInit>(gossip);
            _mainBus.Subscribe<GossipMessage.RetrieveGossipSeedSources>(gossip);
            _mainBus.Subscribe<GossipMessage.GotGossipSeedSources>(gossip);
            _mainBus.Subscribe<GossipMessage.Gossip>(gossip);
            _mainBus.Subscribe<GossipMessage.GossipReceived>(gossip);
            _mainBus.Subscribe<SystemMessage.StateChangeMessage>(gossip);
            _mainBus.Subscribe<GossipMessage.GossipSendFailed>(gossip);
            _mainBus.Subscribe<SystemMessage.VNodeConnectionEstablished>(gossip);
            _mainBus.Subscribe<SystemMessage.VNodeConnectionLost>(gossip);

            _workersHandler.Start();
            _mainQueue.Start();
            monitoringQueue.Start();
            subscrQueue.Start();

            if (subsystems != null)
            {
                foreach (var subsystem in subsystems)
                {
                    subsystem.Register(db, _mainQueue, _mainBus, _timerService, _timeProvider,
                                       httpSendService, new[] { _internalHttpService, _externalHttpService }, _workersHandler);
                }
            }
        }
 public ReconnectToMaster(Guid stateCorrelationId, VNodeInfo master)
 {
     Ensure.NotEmptyGuid(stateCorrelationId, "stateCorrelationId");
     Ensure.NotNull(master, "master");
     StateCorrelationId = stateCorrelationId;
     Master = master;
 }