public ClusterVNodeController(IPublisher outputBus, VNodeInfo nodeInfo, TFChunkDb db, ClusterVNodeSettings vnodeSettings, ClusterVNode node, MessageForwardingProxy forwardingProxy, ISubsystem[] subSystems) { Ensure.NotNull(outputBus, "outputBus"); Ensure.NotNull(nodeInfo, "nodeInfo"); Ensure.NotNull(db, "dbConfig"); Ensure.NotNull(vnodeSettings, "vnodeSettings"); Ensure.NotNull(node, "node"); Ensure.NotNull(forwardingProxy, "forwardingProxy"); _outputBus = outputBus; _nodeInfo = nodeInfo; _db = db; _node = node; _subSystems = subSystems; if (vnodeSettings.ClusterNodeCount == 1) { _serviceShutdownsToExpect = 4; } _subSystemInitsToExpect = _subSystems != null ? subSystems.Length : 0; _forwardingProxy = forwardingProxy; _forwardingTimeout = vnodeSettings.PrepareTimeout + vnodeSettings.CommitTimeout + TimeSpan.FromMilliseconds(300); _fsm = CreateFSM(); }
public MiniClusterNode( string pathname, int debugIndex, IPEndPoint internalTcp, IPEndPoint internalTcpSec, IPEndPoint internalHttp, IPEndPoint externalTcp, IPEndPoint externalTcpSec, IPEndPoint externalHttp, IPEndPoint[] gossipSeeds, ISubsystem[] subsystems = null, int?chunkSize = null, int?cachedChunkSize = null, bool enableTrustedAuth = false, bool skipInitializeStandardUsersCheck = true, int memTableSize = 1000, bool inMemDb = true, bool disableFlushToDisk = false) { RunningTime.Start(); RunCount += 1; _dbPath = Path.Combine( pathname, string.Format( "mini-cluster-node-db-{0}-{1}-{2}", externalTcp.Port, externalTcpSec.Port, externalHttp.Port)); Directory.CreateDirectory(_dbPath); FileStreamExtensions.ConfigureFlush(disableFlushToDisk); Db = new TFChunkDb( CreateDbConfig(chunkSize ?? ChunkSize, _dbPath, cachedChunkSize ?? CachedChunkSize, inMemDb)); InternalTcpEndPoint = internalTcp; InternalTcpSecEndPoint = internalTcpSec; InternalHttpEndPoint = internalHttp; ExternalTcpEndPoint = externalTcp; ExternalTcpSecEndPoint = externalTcpSec; ExternalHttpEndPoint = externalHttp; var singleVNodeSettings = new ClusterVNodeSettings( Guid.NewGuid(), debugIndex, InternalTcpEndPoint, InternalTcpSecEndPoint, ExternalTcpEndPoint, ExternalTcpSecEndPoint, InternalHttpEndPoint, ExternalHttpEndPoint, new Data.GossipAdvertiseInfo(InternalTcpEndPoint, InternalTcpSecEndPoint, ExternalTcpEndPoint, ExternalTcpSecEndPoint, InternalHttpEndPoint, ExternalHttpEndPoint), new[] { InternalHttpEndPoint.ToHttpUrl() }, new[] { ExternalHttpEndPoint.ToHttpUrl() }, enableTrustedAuth, ssl_connections.GetCertificate(), 1, false, "", gossipSeeds, TFConsts.MinFlushDelayMs, 3, 2, 2, TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(2), false, "", false, TimeSpan.FromHours(1), StatsStorage.None, 0, new InternalAuthenticationProviderFactory(), disableScavengeMerging: true, scavengeHistoryMaxAge: 30, adminOnPublic: true, statsOnPublic: true, gossipOnPublic: true, gossipInterval: TimeSpan.FromSeconds(1), gossipAllowedTimeDifference: TimeSpan.FromSeconds(1), gossipTimeout: TimeSpan.FromSeconds(1), extTcpHeartbeatTimeout: TimeSpan.FromSeconds(10), extTcpHeartbeatInterval: TimeSpan.FromSeconds(10), intTcpHeartbeatTimeout: TimeSpan.FromSeconds(10), intTcpHeartbeatInterval: TimeSpan.FromSeconds(10), verifyDbHash: false, maxMemtableEntryCount: memTableSize, startStandardProjections: false, disableHTTPCaching: false, logHttpRequests: false); Log.Info( "\n{0,-25} {1} ({2}/{3}, {4})\n" + "{5,-25} {6} ({7})\n" + "{8,-25} {9} ({10}-bit)\n" + "{11,-25} {12}\n" + "{13,-25} {14}\n" + "{15,-25} {16}\n" + "{17,-25} {18}\n" + "{19,-25} {20}\n\n", "ES VERSION:", VersionInfo.Version, VersionInfo.Branch, VersionInfo.Hashtag, VersionInfo.Timestamp, "OS:", OS.OsFlavor, Environment.OSVersion, "RUNTIME:", OS.GetRuntimeVersion(), Marshal.SizeOf(typeof(IntPtr)) * 8, "GC:", GC.MaxGeneration == 0 ? "NON-GENERATION (PROBABLY BOEHM)" : string.Format("{0} GENERATIONS", GC.MaxGeneration + 1), "DBPATH:", _dbPath, "ExTCP ENDPOINT:", ExternalTcpEndPoint, "ExTCP SECURE ENDPOINT:", ExternalTcpSecEndPoint, "ExHTTP ENDPOINT:", ExternalHttpEndPoint); Node = new ClusterVNode(Db, singleVNodeSettings, infoController: new InfoController(null, ProjectionType.None), subsystems: subsystems, gossipSeedSource: new KnownEndpointGossipSeedSource(gossipSeeds)); Node.ExternalHttpService.SetupController(new TestController(Node.MainQueue)); }
/// <summary> /// Converts an <see cref="EmbeddedVNodeBuilder"/> to a <see cref="ClusterVNode"/>. /// </summary> public ClusterVNode Build() { EnsureHttpPrefixes(); SetUpProjectionsIfNeeded(); var dbConfig = CreateDbConfig(_chunkSize, _dbPath, _chunksCacheSize, _inMemoryDb); var db = new TFChunkDb(dbConfig); var vNodeSettings = new ClusterVNodeSettings(Guid.NewGuid(), 0, _internalTcp, _internalSecureTcp, _externalTcp, _externalSecureTcp, _internalHttp, _externalHttp, new GossipAdvertiseInfo(_internalTcp, _internalSecureTcp, _externalTcp, _externalSecureTcp, _internalHttp, _externalHttp), _intHttpPrefixes.ToArray(), _extHttpPrefixes.ToArray(), _enableTrustedAuth, _certificate, _workerThreads, _discoverViaDns, _clusterDns, _gossipSeeds.ToArray(), _minFlushDelay, _clusterNodeCount, _prepareAckCount, _commitAckCount, _prepareTimeout, _commitTimeout, _useSsl, _sslTargetHost, _sslValidateServer, _statsPeriod, _statsStorage, _nodePriority, _authenticationProviderFactory, _disableScavengeMerging, _scavengeHistoryMaxAge, _adminOnPublic, _statsOnPublic, _gossipOnPublic, _gossipInterval, _gossipAllowedTimeDifference, _gossipTimeout, _intTcpHeartbeatTimeout, _intTcpHeartbeatInterval, _extTcpHeartbeatTimeout, _extTcpHeartbeatInterval, !_skipVerifyDbHashes, _maxMemtableSize, _startStandardProjections, _disableHTTPCaching, _logHttpRequests); var infoController = new InfoController(null, _projectionType); return(new ClusterVNode(db, vNodeSettings, GetGossipSource(), infoController, _subsystems.ToArray())); }
public virtual void TestFixtureSetUp() { _builder = TestVNodeBuilder.AsSingleNode() .RunInMemory(); Given(); _node = _builder.Build(); _settings = ((TestVNodeBuilder)_builder).GetSettings(); _dbConfig = ((TestVNodeBuilder)_builder).GetDbConfig(); _node.Start(); }
public virtual void TestFixtureSetUp() { _builder = TestVNodeBuilder.AsClusterMember(_clusterSize) .RunInMemory(); _quorumSize = _clusterSize / 2 + 1; Given(); _node = _builder.Build(); _settings = ((TestVNodeBuilder)_builder).GetSettings(); _dbConfig = ((TestVNodeBuilder)_builder).GetDbConfig(); _node.Start(); }
private void RegisterWebControllers(NodeSubsystems[] enabledNodeSubsystems, ClusterVNodeSettings settings) { _node.InternalHttpService.SetupController(new ClusterWebUIController(_node.MainQueue, enabledNodeSubsystems)); _node.InternalHttpService.SetupController(new UsersWebController(_node.MainQueue)); if (settings.AdminOnPublic) { _node.ExternalHttpService.SetupController( new ClusterWebUIController(_node.MainQueue, enabledNodeSubsystems)); _node.ExternalHttpService.SetupController(new UsersWebController(_node.MainQueue)); } }
public virtual void TestFixtureSetUp() { _builder = TestVNodeBuilder.AsSingleNode() .WithServerCertificate(ssl_connections.GetServerCertificate()) .RunInMemory(); Given(); _node = _builder.Build(); _settings = ((TestVNodeBuilder)_builder).GetSettings(); _dbConfig = ((TestVNodeBuilder)_builder).GetDbConfig(); _node.Start(); }
public ClusterVNodeStartup( ISubsystem[] subsystems, IQueuedHandler mainQueue, IAuthenticationProvider internalAuthenticationProvider, IReadIndex readIndex, ClusterVNodeSettings vNodeSettings, KestrelHttpService externalHttpService, KestrelHttpService internalHttpService = null) { if (subsystems == null) { throw new ArgumentNullException(nameof(subsystems)); } if (mainQueue == null) { throw new ArgumentNullException(nameof(mainQueue)); } if (internalAuthenticationProvider == null) { throw new ArgumentNullException(nameof(internalAuthenticationProvider)); } if (readIndex == null) { throw new ArgumentNullException(nameof(readIndex)); } if (vNodeSettings == null) { throw new ArgumentNullException(nameof(vNodeSettings)); } if (externalHttpService == null) { throw new ArgumentNullException(nameof(externalHttpService)); } _subsystems = subsystems; _mainQueue = mainQueue; _internalAuthenticationProvider = internalAuthenticationProvider; _readIndex = readIndex; _vNodeSettings = vNodeSettings; _externalHttpService = externalHttpService; _internalHttpService = internalHttpService; _statusCheck = new StatusCheck(this); }
public ClusterVNodeController(IPublisher outputBus, VNodeInfo nodeInfo, TFChunkDb db, ClusterVNodeSettings vnodeSettings, ClusterVNode node, MessageForwardingProxy forwardingProxy) { Ensure.NotNull(outputBus, "outputBus"); Ensure.NotNull(nodeInfo, "nodeInfo"); Ensure.NotNull(db, "dbConfig"); Ensure.NotNull(vnodeSettings, "vnodeSettings"); Ensure.NotNull(node, "node"); Ensure.NotNull(forwardingProxy, "forwardingProxy"); _outputBus = outputBus; _nodeInfo = nodeInfo; _db = db; _node = node; _forwardingProxy = forwardingProxy; _forwardingTimeout = vnodeSettings.PrepareTimeout + vnodeSettings.CommitTimeout + TimeSpan.FromMilliseconds(300); _fsm = CreateFSM(); }
public ClusterVNode(TFChunkDb db, ClusterVNodeSettings vNodeSettings, IGossipSeedSource gossipSeedSource, InfoController infoController, params ISubsystem[] subsystems) { Ensure.NotNull(db, "db"); Ensure.NotNull(vNodeSettings, "vNodeSettings"); Ensure.NotNull(gossipSeedSource, "gossipSeedSource"); #if DEBUG AddTask(_taskAddedTrigger.Task); #endif var isSingleNode = vNodeSettings.ClusterNodeCount == 1; _nodeInfo = vNodeSettings.NodeInfo; _mainBus = new InMemoryBus("MainBus"); var forwardingProxy = new MessageForwardingProxy(); if (vNodeSettings.EnableHistograms) { HistogramService.CreateHistograms(); //start watching jitter HistogramService.StartJitterMonitor(); } // MISC WORKERS _workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum => new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1), watchSlowMsg: true, slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray(); _workersHandler = new MultiQueuedHandler( vNodeSettings.WorkerThreads, queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum], string.Format("Worker #{0}", queueNum + 1), groupName: "Workers", watchSlowMsg: true, slowMsgThreshold: TimeSpan.FromMilliseconds(200))); _subsystems = subsystems; _controller = new ClusterVNodeController((IPublisher)_mainBus, _nodeInfo, db, vNodeSettings, this, forwardingProxy, _subsystems); _mainQueue = QueuedHandler.CreateQueuedHandler(_controller, "MainQueue"); _controller.SetMainQueue(_mainQueue); //SELF _mainBus.Subscribe <SystemMessage.StateChangeMessage>(this); _mainBus.Subscribe <SystemMessage.BecomeShutdown>(this); // MONITORING var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false); var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false); var monitoringQueue = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100)); var monitoring = new MonitoringService(monitoringQueue, monitoringRequestBus, _mainQueue, db.Config.WriterCheckpoint, db.Config.Path, vNodeSettings.StatsPeriod, _nodeInfo.ExternalHttp, vNodeSettings.StatsStorage, _nodeInfo.ExternalTcp, _nodeInfo.ExternalSecureTcp); _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.SystemInit, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShutdown, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom <ClientMessage.WriteEventsCompleted, Message>()); monitoringInnerBus.Subscribe <SystemMessage.SystemInit>(monitoring); monitoringInnerBus.Subscribe <SystemMessage.StateChangeMessage>(monitoring); monitoringInnerBus.Subscribe <SystemMessage.BecomeShuttingDown>(monitoring); monitoringInnerBus.Subscribe <SystemMessage.BecomeShutdown>(monitoring); monitoringInnerBus.Subscribe <ClientMessage.WriteEventsCompleted>(monitoring); monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshStats>(monitoring); monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshTcpConnectionStats>(monitoring); var truncPos = db.Config.TruncateCheckpoint.Read(); if (truncPos != -1) { Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})", truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read()); var truncator = new TFChunkDbTruncator(db.Config); truncator.TruncateDb(truncPos); } // STORAGE SUBSYSTEM db.Open(vNodeSettings.VerifyDbHash); var indexPath = vNodeSettings.Index ?? Path.Combine(db.Config.Path, "index"); var readerPool = new ObjectPool <ITransactionFileReader>( "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount, () => new TFChunkReader(db, db.Config.WriterCheckpoint, optimizeReadSideCache: db.Config.OptimizeReadSideCache)); var tableIndex = new TableIndex(indexPath, new XXHashUnsafe(), new Murmur3AUnsafe(), () => new HashListMemTable(vNodeSettings.IndexBitnessVersion, maxSize: vNodeSettings.MaxMemtableEntryCount * 2), () => new TFReaderLease(readerPool), vNodeSettings.IndexBitnessVersion, maxSizeForMemory: vNodeSettings.MaxMemtableEntryCount, maxTablesPerLevel: 2, inMem: db.Config.InMemDb, skipIndexVerify: vNodeSettings.SkipIndexVerify, indexCacheDepth: vNodeSettings.IndexCacheDepth); var readIndex = new ReadIndex(_mainQueue, readerPool, tableIndex, ESConsts.StreamInfoCacheCapacity, Application.IsDefined(Application.AdditionalCommitChecks), Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1, vNodeSettings.HashCollisionReadLimit, vNodeSettings.SkipIndexScanOnReads, db.Config.ReplicationCheckpoint); var writer = new TFChunkWriter(db); var epochManager = new EpochManager(_mainQueue, ESConsts.CachedEpochCount, db.Config.EpochCheckpoint, writer, initialReaderCount: 1, maxReaderCount: 5, readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint, optimizeReadSideCache: db.Config.OptimizeReadSideCache)); epochManager.Init(); var storageWriter = new ClusterStorageWriterService(_mainQueue, _mainBus, vNodeSettings.MinFlushDelay, db, writer, readIndex.IndexWriter, epochManager, () => readIndex.LastCommitPosition); // subscribes internally AddTasks(storageWriter.Tasks); monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageWriter); var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, vNodeSettings.ReaderThreadsCount, db.Config.WriterCheckpoint); _mainBus.Subscribe <SystemMessage.SystemInit>(storageReader); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageReader); _mainBus.Subscribe <SystemMessage.BecomeShutdown>(storageReader); monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageReader); var indexCommitterService = new IndexCommitterService(readIndex.IndexCommitter, _mainQueue, db.Config.ReplicationCheckpoint, db.Config.WriterCheckpoint, vNodeSettings.CommitAckCount); AddTask(indexCommitterService.Task); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(indexCommitterService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(indexCommitterService); _mainBus.Subscribe <StorageMessage.CommitAck>(indexCommitterService); var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint, db.Config.OptimizeReadSideCache); var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, indexCommitterService, epochManager); AddTask(storageChaser.Task); #if DEBUG QueueStatsCollector.InitializeCheckpoints( _nodeInfo.DebugIndex, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint); #endif _mainBus.Subscribe <SystemMessage.SystemInit>(storageChaser); _mainBus.Subscribe <SystemMessage.SystemStart>(storageChaser); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageChaser); // AUTHENTICATION INFRASTRUCTURE - delegate to plugins _internalAuthenticationProvider = vNodeSettings.AuthenticationProviderFactory.BuildAuthenticationProvider(_mainQueue, _mainBus, _workersHandler, _workerBuses); Ensure.NotNull(_internalAuthenticationProvider, "authenticationProvider"); { // EXTERNAL TCP if (!vNodeSettings.DisableInsecureTCP) { var extTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalTcp, _workersHandler, TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(), vNodeSettings.ExtTcpHeartbeatInterval, vNodeSettings.ExtTcpHeartbeatTimeout, _internalAuthenticationProvider, null, vNodeSettings.ConnectionPendingSendBytesThreshold); _mainBus.Subscribe <SystemMessage.SystemInit>(extTcpService); _mainBus.Subscribe <SystemMessage.SystemStart>(extTcpService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extTcpService); } // EXTERNAL SECURE TCP if (_nodeInfo.ExternalSecureTcp != null) { var extSecTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalSecureTcp, _workersHandler, TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(), vNodeSettings.ExtTcpHeartbeatInterval, vNodeSettings.ExtTcpHeartbeatTimeout, _internalAuthenticationProvider, vNodeSettings.Certificate, vNodeSettings.ConnectionPendingSendBytesThreshold); _mainBus.Subscribe <SystemMessage.SystemInit>(extSecTcpService); _mainBus.Subscribe <SystemMessage.SystemStart>(extSecTcpService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extSecTcpService); } if (!isSingleNode) { // INTERNAL TCP if (!vNodeSettings.DisableInsecureTCP) { var intTcpService = new TcpService(_mainQueue, _nodeInfo.InternalTcp, _workersHandler, TcpServiceType.Internal, TcpSecurityType.Normal, new InternalTcpDispatcher(), vNodeSettings.IntTcpHeartbeatInterval, vNodeSettings.IntTcpHeartbeatTimeout, _internalAuthenticationProvider, null, ESConsts.UnrestrictedPendingSendBytes); _mainBus.Subscribe <SystemMessage.SystemInit>(intTcpService); _mainBus.Subscribe <SystemMessage.SystemStart>(intTcpService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intTcpService); } // INTERNAL SECURE TCP if (_nodeInfo.InternalSecureTcp != null) { var intSecTcpService = new TcpService(_mainQueue, _nodeInfo.InternalSecureTcp, _workersHandler, TcpServiceType.Internal, TcpSecurityType.Secure, new InternalTcpDispatcher(), vNodeSettings.IntTcpHeartbeatInterval, vNodeSettings.IntTcpHeartbeatTimeout, _internalAuthenticationProvider, vNodeSettings.Certificate, ESConsts.UnrestrictedPendingSendBytes); _mainBus.Subscribe <SystemMessage.SystemInit>(intSecTcpService); _mainBus.Subscribe <SystemMessage.SystemStart>(intSecTcpService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intSecTcpService); } } } SubscribeWorkers(bus => { var tcpSendService = new TcpSendService(); // ReSharper disable RedundantTypeArgumentsOfMethod bus.Subscribe <TcpMessage.TcpSend>(tcpSendService); // ReSharper restore RedundantTypeArgumentsOfMethod }); var httpAuthenticationProviders = new List <HttpAuthenticationProvider> { new BasicHttpAuthenticationProvider(_internalAuthenticationProvider), }; if (vNodeSettings.EnableTrustedAuth) { httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider()); } httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider()); var httpPipe = new HttpMessagePipe(); var httpSendService = new HttpSendService(httpPipe, forwardRequests: true); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(httpSendService); _mainBus.Subscribe(new WideningHandler <HttpMessage.SendOverHttp, Message>(_workersHandler)); SubscribeWorkers(bus => { bus.Subscribe <HttpMessage.HttpSend>(httpSendService); bus.Subscribe <HttpMessage.HttpSendPart>(httpSendService); bus.Subscribe <HttpMessage.HttpBeginSend>(httpSendService); bus.Subscribe <HttpMessage.HttpEndSend>(httpSendService); bus.Subscribe <HttpMessage.SendOverHttp>(httpSendService); }); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(infoController); var adminController = new AdminController(_mainQueue, _workersHandler); var pingController = new PingController(); var histogramController = new HistogramController(); var statController = new StatController(monitoringQueue, _workersHandler); var atomController = new AtomController(httpSendService, _mainQueue, _workersHandler, vNodeSettings.DisableHTTPCaching); var gossipController = new GossipController(_mainQueue, _workersHandler, vNodeSettings.GossipTimeout); var persistentSubscriptionController = new PersistentSubscriptionController(httpSendService, _mainQueue, _workersHandler); var electController = new ElectController(_mainQueue); // HTTP SENDERS gossipController.SubscribeSenders(httpPipe); electController.SubscribeSenders(httpPipe); // EXTERNAL HTTP _externalHttpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(), _workersHandler, vNodeSettings.LogHttpRequests, vNodeSettings.GossipAdvertiseInfo.AdvertiseExternalIPAs, vNodeSettings.GossipAdvertiseInfo.AdvertiseExternalHttpPortAs, vNodeSettings.ExtHttpPrefixes); _externalHttpService.SetupController(persistentSubscriptionController); if (vNodeSettings.AdminOnPublic) { _externalHttpService.SetupController(adminController); } _externalHttpService.SetupController(pingController); _externalHttpService.SetupController(infoController); if (vNodeSettings.StatsOnPublic) { _externalHttpService.SetupController(statController); } _externalHttpService.SetupController(atomController); if (vNodeSettings.GossipOnPublic) { _externalHttpService.SetupController(gossipController); } _externalHttpService.SetupController(histogramController); _mainBus.Subscribe <SystemMessage.SystemInit>(_externalHttpService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_externalHttpService); _mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_externalHttpService); // INTERNAL HTTP if (!isSingleNode) { _internalHttpService = new HttpService(ServiceAccessibility.Private, _mainQueue, new TrieUriRouter(), _workersHandler, vNodeSettings.LogHttpRequests, vNodeSettings.GossipAdvertiseInfo.AdvertiseInternalIPAs, vNodeSettings.GossipAdvertiseInfo.AdvertiseInternalHttpPortAs, vNodeSettings.IntHttpPrefixes); _internalHttpService.SetupController(adminController); _internalHttpService.SetupController(pingController); _internalHttpService.SetupController(infoController); _internalHttpService.SetupController(statController); _internalHttpService.SetupController(atomController); _internalHttpService.SetupController(gossipController); _internalHttpService.SetupController(electController); _internalHttpService.SetupController(histogramController); _internalHttpService.SetupController(persistentSubscriptionController); } // Authentication plugin HTTP vNodeSettings.AuthenticationProviderFactory.RegisterHttpControllers(_externalHttpService, _internalHttpService, httpSendService, _mainQueue, _workersHandler); if (_internalHttpService != null) { _mainBus.Subscribe <SystemMessage.SystemInit>(_internalHttpService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_internalHttpService); _mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_internalHttpService); } SubscribeWorkers(bus => { HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray()); }); // REQUEST FORWARDING var forwardingService = new RequestForwardingService(_mainQueue, forwardingProxy, TimeSpan.FromSeconds(1)); _mainBus.Subscribe <SystemMessage.SystemStart>(forwardingService); _mainBus.Subscribe <SystemMessage.RequestForwardingTimerTick>(forwardingService); _mainBus.Subscribe <ClientMessage.NotHandled>(forwardingService); _mainBus.Subscribe <ClientMessage.WriteEventsCompleted>(forwardingService); _mainBus.Subscribe <ClientMessage.TransactionStartCompleted>(forwardingService); _mainBus.Subscribe <ClientMessage.TransactionWriteCompleted>(forwardingService); _mainBus.Subscribe <ClientMessage.TransactionCommitCompleted>(forwardingService); _mainBus.Subscribe <ClientMessage.DeleteStreamCompleted>(forwardingService); // REQUEST MANAGEMENT var requestManagement = new RequestManagementService(_mainQueue, vNodeSettings.PrepareAckCount, vNodeSettings.PrepareTimeout, vNodeSettings.CommitTimeout, vNodeSettings.BetterOrdering); _mainBus.Subscribe <SystemMessage.SystemInit>(requestManagement); _mainBus.Subscribe <ClientMessage.WriteEvents>(requestManagement); _mainBus.Subscribe <ClientMessage.TransactionStart>(requestManagement); _mainBus.Subscribe <ClientMessage.TransactionWrite>(requestManagement); _mainBus.Subscribe <ClientMessage.TransactionCommit>(requestManagement); _mainBus.Subscribe <ClientMessage.DeleteStream>(requestManagement); _mainBus.Subscribe <StorageMessage.RequestCompleted>(requestManagement); _mainBus.Subscribe <StorageMessage.CheckStreamAccessCompleted>(requestManagement); _mainBus.Subscribe <StorageMessage.AlreadyCommitted>(requestManagement); _mainBus.Subscribe <StorageMessage.CommitReplicated>(requestManagement); _mainBus.Subscribe <StorageMessage.PrepareAck>(requestManagement); _mainBus.Subscribe <StorageMessage.WrongExpectedVersion>(requestManagement); _mainBus.Subscribe <StorageMessage.InvalidTransaction>(requestManagement); _mainBus.Subscribe <StorageMessage.StreamDeleted>(requestManagement); _mainBus.Subscribe <StorageMessage.RequestManagerTimerTick>(requestManagement); // SUBSCRIPTIONS var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50)); var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false); _mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.SystemStart, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.SubscribeToStream, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.PollStream, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.CheckPollTimeout, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <StorageMessage.EventCommitted, Message>()); var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex); subscrBus.Subscribe <SystemMessage.SystemStart>(subscription); subscrBus.Subscribe <SystemMessage.BecomeShuttingDown>(subscription); subscrBus.Subscribe <TcpMessage.ConnectionClosed>(subscription); subscrBus.Subscribe <ClientMessage.SubscribeToStream>(subscription); subscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(subscription); subscrBus.Subscribe <SubscriptionMessage.PollStream>(subscription); subscrBus.Subscribe <SubscriptionMessage.CheckPollTimeout>(subscription); subscrBus.Subscribe <StorageMessage.EventCommitted>(subscription); // PERSISTENT SUBSCRIPTIONS // IO DISPATCHER var ioDispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_mainQueue)); _mainBus.Subscribe <ClientMessage.ReadStreamEventsBackwardCompleted>(ioDispatcher.BackwardReader); _mainBus.Subscribe <ClientMessage.WriteEventsCompleted>(ioDispatcher.Writer); _mainBus.Subscribe <ClientMessage.ReadStreamEventsForwardCompleted>(ioDispatcher.ForwardReader); _mainBus.Subscribe <ClientMessage.DeleteStreamCompleted>(ioDispatcher.StreamDeleter); _mainBus.Subscribe(ioDispatcher); var perSubscrBus = new InMemoryBus("PersistentSubscriptionsBus", true, TimeSpan.FromMilliseconds(50)); var perSubscrQueue = new QueuedHandlerThreadPool(perSubscrBus, "PersistentSubscriptions", false); _mainBus.Subscribe(perSubscrQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.CreatePersistentSubscription, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.UpdatePersistentSubscription, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.DeletePersistentSubscription, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ConnectToPersistentSubscription, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.PersistentSubscriptionAckEvents, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.PersistentSubscriptionNackEvents, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReplayAllParkedMessages, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReplayParkedMessage, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReadNextNPersistentMessages, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <StorageMessage.EventCommitted, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <MonitoringMessage.GetAllPersistentSubscriptionStats, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <MonitoringMessage.GetStreamPersistentSubscriptionStats, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <MonitoringMessage.GetPersistentSubscriptionStats, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <SubscriptionMessage.PersistentSubscriptionTimerTick, Message>()); //TODO CC can have multiple threads working on subscription if partition var consumerStrategyRegistry = new PersistentSubscriptionConsumerStrategyRegistry(_mainQueue, _mainBus, vNodeSettings.AdditionalConsumerStrategies); var persistentSubscription = new PersistentSubscriptionService(perSubscrQueue, readIndex, ioDispatcher, _mainQueue, consumerStrategyRegistry); perSubscrBus.Subscribe <SystemMessage.BecomeShuttingDown>(persistentSubscription); perSubscrBus.Subscribe <SystemMessage.BecomeMaster>(persistentSubscription); perSubscrBus.Subscribe <SystemMessage.StateChangeMessage>(persistentSubscription); perSubscrBus.Subscribe <TcpMessage.ConnectionClosed>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.ConnectToPersistentSubscription>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.PersistentSubscriptionAckEvents>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.PersistentSubscriptionNackEvents>(persistentSubscription); perSubscrBus.Subscribe <StorageMessage.EventCommitted>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.DeletePersistentSubscription>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.CreatePersistentSubscription>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.UpdatePersistentSubscription>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.ReplayAllParkedMessages>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.ReplayParkedMessage>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.ReadNextNPersistentMessages>(persistentSubscription); perSubscrBus.Subscribe <MonitoringMessage.GetAllPersistentSubscriptionStats>(persistentSubscription); perSubscrBus.Subscribe <MonitoringMessage.GetStreamPersistentSubscriptionStats>(persistentSubscription); perSubscrBus.Subscribe <MonitoringMessage.GetPersistentSubscriptionStats>(persistentSubscription); perSubscrBus.Subscribe <SubscriptionMessage.PersistentSubscriptionTimerTick>(persistentSubscription); // STORAGE SCAVENGER var scavengerLogManager = new TFChunkScavengerLogManager(_nodeInfo.ExternalHttp.ToString(), TimeSpan.FromDays(vNodeSettings.ScavengeHistoryMaxAge), ioDispatcher); var storageScavenger = new StorageScavenger(db, tableIndex, readIndex, scavengerLogManager, vNodeSettings.AlwaysKeepScavenged, !vNodeSettings.DisableScavengeMerging, unsafeIgnoreHardDeletes: vNodeSettings.UnsafeIgnoreHardDeletes); // ReSharper disable RedundantTypeArgumentsOfMethod _mainBus.Subscribe <ClientMessage.ScavengeDatabase>(storageScavenger); _mainBus.Subscribe <ClientMessage.StopDatabaseScavenge>(storageScavenger); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(storageScavenger); // ReSharper restore RedundantTypeArgumentsOfMethod // TIMER _timeProvider = new RealTimeProvider(); var threadBasedScheduler = new ThreadBasedScheduler(_timeProvider); AddTask(threadBasedScheduler.Task); _timerService = new TimerService(threadBasedScheduler); _mainBus.Subscribe <SystemMessage.BecomeShutdown>(_timerService); _mainBus.Subscribe <TimerMessage.Schedule>(_timerService); var gossipInfo = new VNodeInfo(_nodeInfo.InstanceId, _nodeInfo.DebugIndex, vNodeSettings.GossipAdvertiseInfo.InternalTcp, vNodeSettings.GossipAdvertiseInfo.InternalSecureTcp, vNodeSettings.GossipAdvertiseInfo.ExternalTcp, vNodeSettings.GossipAdvertiseInfo.ExternalSecureTcp, vNodeSettings.GossipAdvertiseInfo.InternalHttp, vNodeSettings.GossipAdvertiseInfo.ExternalHttp); if (!isSingleNode) { // MASTER REPLICATION var masterReplicationService = new MasterReplicationService(_mainQueue, gossipInfo.InstanceId, db, _workersHandler, epochManager, vNodeSettings.ClusterNodeCount); AddTask(masterReplicationService.Task); _mainBus.Subscribe <SystemMessage.SystemStart>(masterReplicationService); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(masterReplicationService); _mainBus.Subscribe <ReplicationMessage.ReplicaSubscriptionRequest>(masterReplicationService); _mainBus.Subscribe <ReplicationMessage.ReplicaLogPositionAck>(masterReplicationService); monitoringInnerBus.Subscribe <ReplicationMessage.GetReplicationStats>(masterReplicationService); // REPLICA REPLICATION var replicaService = new ReplicaService(_mainQueue, db, epochManager, _workersHandler, _internalAuthenticationProvider, gossipInfo, vNodeSettings.UseSsl, vNodeSettings.SslTargetHost, vNodeSettings.SslValidateServer, vNodeSettings.IntTcpHeartbeatTimeout, vNodeSettings.ExtTcpHeartbeatInterval); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(replicaService); _mainBus.Subscribe <ReplicationMessage.ReconnectToMaster>(replicaService); _mainBus.Subscribe <ReplicationMessage.SubscribeToMaster>(replicaService); _mainBus.Subscribe <ReplicationMessage.AckLogPosition>(replicaService); _mainBus.Subscribe <StorageMessage.PrepareAck>(replicaService); _mainBus.Subscribe <StorageMessage.CommitAck>(replicaService); _mainBus.Subscribe <ClientMessage.TcpForwardMessage>(replicaService); } // ELECTIONS var electionsService = new ElectionsService(_mainQueue, gossipInfo, vNodeSettings.ClusterNodeCount, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint, epochManager, () => readIndex.LastCommitPosition, vNodeSettings.NodePriority); electionsService.SubscribeMessages(_mainBus); if (!isSingleNode || vNodeSettings.GossipOnSingleNode) { // GOSSIP var gossip = new NodeGossipService(_mainQueue, gossipSeedSource, gossipInfo, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint, epochManager, () => readIndex.LastCommitPosition, vNodeSettings.NodePriority, vNodeSettings.GossipInterval, vNodeSettings.GossipAllowedTimeDifference); _mainBus.Subscribe <SystemMessage.SystemInit>(gossip); _mainBus.Subscribe <GossipMessage.RetrieveGossipSeedSources>(gossip); _mainBus.Subscribe <GossipMessage.GotGossipSeedSources>(gossip); _mainBus.Subscribe <GossipMessage.Gossip>(gossip); _mainBus.Subscribe <GossipMessage.GossipReceived>(gossip); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(gossip); _mainBus.Subscribe <GossipMessage.GossipSendFailed>(gossip); _mainBus.Subscribe <SystemMessage.VNodeConnectionEstablished>(gossip); _mainBus.Subscribe <SystemMessage.VNodeConnectionLost>(gossip); } AddTasks(_workersHandler.Start()); AddTask(_mainQueue.Start()); AddTask(monitoringQueue.Start()); AddTask(subscrQueue.Start()); AddTask(perSubscrQueue.Start()); if (subsystems != null) { foreach (var subsystem in subsystems) { var http = isSingleNode ? new [] { _externalHttpService } : new [] { _internalHttpService, _externalHttpService }; subsystem.Register(new StandardComponents(db, _mainQueue, _mainBus, _timerService, _timeProvider, httpSendService, http, _workersHandler)); } } }
public MiniClusterNode( string pathname, int debugIndex, IPEndPoint internalTcp, IPEndPoint internalTcpSec, IPEndPoint internalHttp, IPEndPoint externalTcp, IPEndPoint externalTcpSec, IPEndPoint externalHttp, IPEndPoint[] gossipSeeds, ISubsystem[] subsystems = null, int?chunkSize = null, int?cachedChunkSize = null, bool enableTrustedAuth = false, bool skipInitializeStandardUsersCheck = true, int memTableSize = 1000, bool inMemDb = true, bool disableFlushToDisk = false, bool readOnlyReplica = false) { if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true); //TODO JPB Remove this sadness when dotnet core supports kestrel + http2 on macOS } RunningTime.Start(); RunCount += 1; DebugIndex = debugIndex; _dbPath = Path.Combine( pathname, string.Format( "mini-cluster-node-db-{0}-{1}-{2}", externalTcp.Port, externalTcpSec.Port, externalHttp.Port)); Directory.CreateDirectory(_dbPath); FileStreamExtensions.ConfigureFlush(disableFlushToDisk); Db = new TFChunkDb( CreateDbConfig(chunkSize ?? ChunkSize, _dbPath, cachedChunkSize ?? CachedChunkSize, inMemDb)); InternalTcpEndPoint = internalTcp; InternalTcpSecEndPoint = internalTcpSec; InternalHttpEndPoint = internalHttp; ExternalTcpEndPoint = externalTcp; ExternalTcpSecEndPoint = externalTcpSec; ExternalHttpEndPoint = externalHttp; var certificate = ssl_connections.GetCertificate(); var disableInternalTls = !ssl_connections.IsValidCertificate(certificate); //use internal TLS only if CA certificate is installed var singleVNodeSettings = new ClusterVNodeSettings( Guid.NewGuid(), debugIndex, InternalTcpEndPoint, InternalTcpSecEndPoint, ExternalTcpEndPoint, ExternalTcpSecEndPoint, InternalHttpEndPoint, ExternalHttpEndPoint, new Data.GossipAdvertiseInfo(InternalTcpEndPoint, InternalTcpSecEndPoint, ExternalTcpEndPoint, ExternalTcpSecEndPoint, InternalHttpEndPoint, ExternalHttpEndPoint, null, null, 0, 0), enableTrustedAuth, certificate, 1, false, "", gossipSeeds, TFConsts.MinFlushDelayMs, 3, 2, 2, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10), disableInternalTls, false, TimeSpan.FromHours(1), StatsStorage.None, 0, new InternalAuthenticationProviderFactory(), new LegacyAuthorizationProviderFactory(), disableScavengeMerging: true, scavengeHistoryMaxAge: 30, adminOnPublic: true, statsOnPublic: true, gossipOnPublic: true, gossipInterval: TimeSpan.FromSeconds(2), gossipAllowedTimeDifference: TimeSpan.FromSeconds(1), gossipTimeout: TimeSpan.FromSeconds(3), extTcpHeartbeatTimeout: TimeSpan.FromSeconds(2), extTcpHeartbeatInterval: TimeSpan.FromSeconds(2), intTcpHeartbeatTimeout: TimeSpan.FromSeconds(2), intTcpHeartbeatInterval: TimeSpan.FromSeconds(2), deadMemberRemovalPeriod: TimeSpan.FromSeconds(1800), verifyDbHash: false, maxMemtableEntryCount: memTableSize, hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault, startStandardProjections: false, disableHTTPCaching: false, logHttpRequests: false, connectionPendingSendBytesThreshold: Opts.ConnectionPendingSendBytesThresholdDefault, connectionQueueSizeThreshold: Opts.ConnectionQueueSizeThresholdDefault, readOnlyReplica: readOnlyReplica, ptableMaxReaderCount: Constants.PTableMaxReaderCountDefault, enableExternalTCP: true, createHttpMessageHandler: () => new SocketsHttpHandler { SslOptions = new SslClientAuthenticationOptions { RemoteCertificateValidationCallback = delegate { return(true); } } }, gossipOverHttps: UseHttpsInternally()); _isReadOnlyReplica = readOnlyReplica; Log.Information( "\n{0,-25} {1} ({2}/{3}, {4})\n" + "{5,-25} {6} ({7})\n" + "{8,-25} {9} ({10}-bit)\n" + "{11,-25} {12}\n" + "{13,-25} {14}\n" + "{15,-25} {16}\n" + "{17,-25} {18}\n" + "{19,-25} {20}\n\n", "ES VERSION:", VersionInfo.Version, VersionInfo.Branch, VersionInfo.Hashtag, VersionInfo.Timestamp, "OS:", OS.OsFlavor, Environment.OSVersion, "RUNTIME:", OS.GetRuntimeVersion(), Marshal.SizeOf(typeof(IntPtr)) * 8, "GC:", GC.MaxGeneration == 0 ? "NON-GENERATION (PROBABLY BOEHM)" : string.Format("{0} GENERATIONS", GC.MaxGeneration + 1), "DBPATH:", _dbPath, "ExTCP ENDPOINT:", ExternalTcpEndPoint, "ExTCP SECURE ENDPOINT:", ExternalTcpSecEndPoint, "ExHTTP ENDPOINT:", ExternalHttpEndPoint); Node = new ClusterVNode(Db, singleVNodeSettings, infoController: new InfoController(null, ProjectionType.None), subsystems: subsystems, gossipSeedSource: new KnownEndpointGossipSeedSource(gossipSeeds)); Node.ExternalHttpService.SetupController(new TestController(Node.MainQueue)); _host = new WebHostBuilder() .UseKestrel(o => { o.Listen(InternalHttpEndPoint, options => { if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { options.Protocols = HttpProtocols.Http2; } else { options.UseHttps(); } }); o.Listen(ExternalHttpEndPoint); }) .UseStartup(Node.Startup) .Build(); _kestrelTestServer = new TestServer(new WebHostBuilder() .UseKestrel() .UseStartup(Node.Startup)); }
public MiniNode(string pathname, int?tcpPort = null, int?tcpSecPort = null, int?httpPort = null, ISubsystem[] subsystems = null, int?chunkSize = null, int?cachedChunkSize = null, bool enableTrustedAuth = false, bool skipInitializeStandardUsersCheck = true, int memTableSize = 1000, bool inMemDb = true, bool disableFlushToDisk = false) { if (_running) { throw new Exception("Previous MiniNode is still running!!!"); } _running = true; RunningTime.Start(); RunCount += 1; IPAddress ip = IPAddress.Loopback; //GetLocalIp(); int extTcpPort = tcpPort ?? PortsHelper.GetAvailablePort(ip); int extSecTcpPort = tcpSecPort ?? PortsHelper.GetAvailablePort(ip); int extHttpPort = httpPort ?? PortsHelper.GetAvailablePort(ip); int intTcpPort = PortsHelper.GetAvailablePort(ip); int intSecTcpPort = PortsHelper.GetAvailablePort(ip); int intHttpPort = PortsHelper.GetAvailablePort(ip); _dbPath = Path.Combine(pathname, string.Format("mini-node-db-{0}-{1}-{2}", extTcpPort, extSecTcpPort, extHttpPort)); Directory.CreateDirectory(_dbPath); FileStreamExtensions.ConfigureFlush(disableFlushToDisk); Db = new TFChunkDb(CreateDbConfig(chunkSize ?? ChunkSize, _dbPath, cachedChunkSize ?? CachedChunkSize, inMemDb)); TcpEndPoint = new IPEndPoint(ip, extTcpPort); TcpSecEndPoint = new IPEndPoint(ip, extSecTcpPort); HttpEndPoint = new IPEndPoint(ip, extHttpPort); IntTcpEndPoint = new IPEndPoint(ip, intTcpPort); IntSecTcpEndPoint = new IPEndPoint(ip, intSecTcpPort); IntHttpEndPoint = new IPEndPoint(ip, intHttpPort); var vNodeSettings = new ClusterVNodeSettings(Guid.NewGuid(), 0, IntTcpEndPoint, IntSecTcpEndPoint, TcpEndPoint, TcpSecEndPoint, IntHttpEndPoint, HttpEndPoint, new [] { HttpEndPoint.ToHttpUrl() }, enableTrustedAuth, ssl_connections.GetCertificate(), 1, false, "whatever", new IPEndPoint[] {}, TFConsts.MinFlushDelayMs, 1, 1, 1, TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(2), false, "", false, TimeSpan.FromHours(1), StatsStorage.None, 1, new InternalAuthenticationProviderFactory(), true, true, true, false, TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10), false, memTableSize); Log.Info("\n{0,-25} {1} ({2}/{3}, {4})\n" + "{5,-25} {6} ({7})\n" + "{8,-25} {9} ({10}-bit)\n" + "{11,-25} {12}\n" + "{13,-25} {14}\n" + "{15,-25} {16}\n" + "{17,-25} {18}\n" + "{19,-25} {20}\n\n", "ES VERSION:", VersionInfo.Version, VersionInfo.Branch, VersionInfo.Hashtag, VersionInfo.Timestamp, "OS:", OS.OsFlavor, Environment.OSVersion, "RUNTIME:", OS.GetRuntimeVersion(), Marshal.SizeOf(typeof(IntPtr)) * 8, "GC:", GC.MaxGeneration == 0 ? "NON-GENERATION (PROBABLY BOEHM)" : string.Format("{0} GENERATIONS", GC.MaxGeneration + 1), "DBPATH:", _dbPath, "TCP ENDPOINT:", TcpEndPoint, "TCP SECURE ENDPOINT:", TcpSecEndPoint, "HTTP ENDPOINT:", HttpEndPoint); Node = new ClusterVNode(Db, vNodeSettings, new KnownEndpointGossipSeedSource(new [] { HttpEndPoint }), new InfoController(null), subsystems); Node.ExternalHttpService.SetupController(new TestController(Node.MainQueue)); }
public ClusterVNode(TFChunkDb db, ClusterVNodeSettings vNodeSettings, IGossipSeedSource gossipSeedSource, bool dbVerifyHashes, int memTableEntryCount, params ISubsystem[] subsystems) { Ensure.NotNull(db, "db"); Ensure.NotNull(vNodeSettings, "vNodeSettings"); Ensure.NotNull(gossipSeedSource, "gossipSeedSource"); _nodeInfo = vNodeSettings.NodeInfo; _mainBus = new InMemoryBus("MainBus"); var forwardingProxy = new MessageForwardingProxy(); // MISC WORKERS _workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum => new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1), watchSlowMsg: true, slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray(); _workersHandler = new MultiQueuedHandler( vNodeSettings.WorkerThreads, queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum], string.Format("Worker #{0}", queueNum + 1), groupName: "Workers", watchSlowMsg: true, slowMsgThreshold: TimeSpan.FromMilliseconds(200))); _controller = new ClusterVNodeController(_mainBus, _nodeInfo, db, vNodeSettings, this, forwardingProxy); _mainQueue = new QueuedHandler(_controller, "MainQueue"); _controller.SetMainQueue(_mainQueue); _subsystems = subsystems; // MONITORING var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false); var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false); var monitoringQueue = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100)); var monitoring = new MonitoringService(monitoringQueue, monitoringRequestBus, _mainQueue, db.Config.WriterCheckpoint, db.Config.Path, vNodeSettings.StatsPeriod, _nodeInfo.ExternalHttp, vNodeSettings.StatsStorage); _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.SystemInit, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShutdown, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom <ClientMessage.WriteEventsCompleted, Message>()); monitoringInnerBus.Subscribe <SystemMessage.SystemInit>(monitoring); monitoringInnerBus.Subscribe <SystemMessage.StateChangeMessage>(monitoring); monitoringInnerBus.Subscribe <SystemMessage.BecomeShuttingDown>(monitoring); monitoringInnerBus.Subscribe <SystemMessage.BecomeShutdown>(monitoring); monitoringInnerBus.Subscribe <ClientMessage.WriteEventsCompleted>(monitoring); monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshStats>(monitoring); var truncPos = db.Config.TruncateCheckpoint.Read(); if (truncPos != -1) { Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})", truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read()); var truncator = new TFChunkDbTruncator(db.Config); truncator.TruncateDb(truncPos); } // STORAGE SUBSYSTEM db.Open(dbVerifyHashes); var indexPath = Path.Combine(db.Config.Path, "index"); var readerPool = new ObjectPool <ITransactionFileReader>( "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount, () => new TFChunkReader(db, db.Config.WriterCheckpoint)); var tableIndex = new TableIndex(indexPath, () => new HashListMemTable(maxSize: memTableEntryCount * 2), () => new TFReaderLease(readerPool), maxSizeForMemory: memTableEntryCount, maxTablesPerLevel: 2, inMem: db.Config.InMemDb); var hash = new XXHashUnsafe(); var readIndex = new ReadIndex(_mainQueue, readerPool, tableIndex, hash, ESConsts.StreamInfoCacheCapacity, Application.IsDefined(Application.AdditionalCommitChecks), Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1); var writer = new TFChunkWriter(db); var epochManager = new EpochManager(ESConsts.CachedEpochCount, db.Config.EpochCheckpoint, writer, initialReaderCount: 1, maxReaderCount: 5, readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint)); epochManager.Init(); var storageWriter = new ClusterStorageWriterService(_mainQueue, _mainBus, vNodeSettings.MinFlushDelay, db, writer, readIndex.IndexWriter, epochManager, () => readIndex.LastCommitPosition); // subscribes internally monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageWriter); var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, ESConsts.StorageReaderThreadCount, db.Config.WriterCheckpoint); _mainBus.Subscribe <SystemMessage.SystemInit>(storageReader); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageReader); _mainBus.Subscribe <SystemMessage.BecomeShutdown>(storageReader); monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageReader); var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint); var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, readIndex.IndexCommitter, epochManager); #if DEBUG QueueStatsCollector.InitializeCheckpoints( _nodeInfo.DebugIndex, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint); #endif _mainBus.Subscribe <SystemMessage.SystemInit>(storageChaser); _mainBus.Subscribe <SystemMessage.SystemStart>(storageChaser); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageChaser); var storageScavenger = new StorageScavenger(db, tableIndex, hash, readIndex, Application.IsDefined(Application.AlwaysKeepScavenged), mergeChunks: !vNodeSettings.DisableScavengeMerging); // ReSharper disable RedundantTypeArgumentsOfMethod _mainBus.Subscribe <ClientMessage.ScavengeDatabase>(storageScavenger); // ReSharper restore RedundantTypeArgumentsOfMethod // AUTHENTICATION INFRASTRUCTURE - delegate to plugins var authenticationProvider = vNodeSettings.AuthenticationProviderFactory.BuildAuthenticationProvider(_mainQueue, _mainBus, _workersHandler, _workerBuses); Ensure.NotNull(authenticationProvider, "authenticationProvider"); { // EXTERNAL TCP var extTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalTcp, _workersHandler, TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(), ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout, authenticationProvider, null); _mainBus.Subscribe <SystemMessage.SystemInit>(extTcpService); _mainBus.Subscribe <SystemMessage.SystemStart>(extTcpService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extTcpService); // EXTERNAL SECURE TCP if (_nodeInfo.ExternalSecureTcp != null) { var extSecTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalSecureTcp, _workersHandler, TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(), ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout, authenticationProvider, vNodeSettings.Certificate); _mainBus.Subscribe <SystemMessage.SystemInit>(extSecTcpService); _mainBus.Subscribe <SystemMessage.SystemStart>(extSecTcpService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extSecTcpService); } // INTERNAL TCP var intTcpService = new TcpService(_mainQueue, _nodeInfo.InternalTcp, _workersHandler, TcpServiceType.Internal, TcpSecurityType.Normal, new InternalTcpDispatcher(), ESConsts.InternalHeartbeatInterval, ESConsts.InternalHeartbeatTimeout, authenticationProvider, null); _mainBus.Subscribe <SystemMessage.SystemInit>(intTcpService); _mainBus.Subscribe <SystemMessage.SystemStart>(intTcpService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intTcpService); // INTERNAL SECURE TCP if (_nodeInfo.InternalSecureTcp != null) { var intSecTcpService = new TcpService(_mainQueue, _nodeInfo.InternalSecureTcp, _workersHandler, TcpServiceType.Internal, TcpSecurityType.Secure, new InternalTcpDispatcher(), ESConsts.InternalHeartbeatInterval, ESConsts.InternalHeartbeatTimeout, authenticationProvider, vNodeSettings.Certificate); _mainBus.Subscribe <SystemMessage.SystemInit>(intSecTcpService); _mainBus.Subscribe <SystemMessage.SystemStart>(intSecTcpService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intSecTcpService); } } SubscribeWorkers(bus => { var tcpSendService = new TcpSendService(); // ReSharper disable RedundantTypeArgumentsOfMethod bus.Subscribe <TcpMessage.TcpSend>(tcpSendService); // ReSharper restore RedundantTypeArgumentsOfMethod }); var httpAuthenticationProviders = new List <HttpAuthenticationProvider> { new BasicHttpAuthenticationProvider(authenticationProvider), }; if (vNodeSettings.EnableTrustedAuth) { httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider()); } httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider()); var httpPipe = new HttpMessagePipe(); var httpSendService = new HttpSendService(httpPipe, forwardRequests: true); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(httpSendService); _mainBus.Subscribe(new WideningHandler <HttpMessage.SendOverHttp, Message>(_workersHandler)); SubscribeWorkers(bus => { bus.Subscribe <HttpMessage.HttpSend>(httpSendService); bus.Subscribe <HttpMessage.HttpSendPart>(httpSendService); bus.Subscribe <HttpMessage.HttpBeginSend>(httpSendService); bus.Subscribe <HttpMessage.HttpEndSend>(httpSendService); bus.Subscribe <HttpMessage.SendOverHttp>(httpSendService); }); var adminController = new AdminController(_mainQueue); var pingController = new PingController(); var statController = new StatController(monitoringQueue, _workersHandler); var atomController = new AtomController(httpSendService, _mainQueue, _workersHandler); var gossipController = new GossipController(_mainQueue, _workersHandler); var electController = new ElectController(_mainQueue); // HTTP SENDERS gossipController.SubscribeSenders(httpPipe); electController.SubscribeSenders(httpPipe); // EXTERNAL HTTP _externalHttpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(), _workersHandler, vNodeSettings.HttpPrefixes); if (vNodeSettings.AdminOnPublic) { _externalHttpService.SetupController(adminController); } _externalHttpService.SetupController(pingController); if (vNodeSettings.StatsOnPublic) { _externalHttpService.SetupController(statController); } _externalHttpService.SetupController(atomController); if (vNodeSettings.GossipOnPublic) { _externalHttpService.SetupController(gossipController); } _mainBus.Subscribe <SystemMessage.SystemInit>(_externalHttpService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_externalHttpService); _mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_externalHttpService); // INTERNAL HTTP _internalHttpService = new HttpService(ServiceAccessibility.Private, _mainQueue, new TrieUriRouter(), _workersHandler, _nodeInfo.InternalHttp.ToHttpUrl()); _internalHttpService.SetupController(adminController); _internalHttpService.SetupController(pingController); _internalHttpService.SetupController(statController); _internalHttpService.SetupController(atomController); _internalHttpService.SetupController(gossipController); _internalHttpService.SetupController(electController); // Authentication plugin HTTP vNodeSettings.AuthenticationProviderFactory.RegisterHttpControllers(_externalHttpService, _internalHttpService, httpSendService, _mainQueue, _workersHandler); _mainBus.Subscribe <SystemMessage.SystemInit>(_internalHttpService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_internalHttpService); _mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_internalHttpService); SubscribeWorkers(bus => { HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray()); }); // REQUEST FORWARDING var forwardingService = new RequestForwardingService(_mainQueue, forwardingProxy, TimeSpan.FromSeconds(1)); _mainBus.Subscribe <SystemMessage.SystemStart>(forwardingService); _mainBus.Subscribe <SystemMessage.RequestForwardingTimerTick>(forwardingService); _mainBus.Subscribe <ClientMessage.NotHandled>(forwardingService); _mainBus.Subscribe <ClientMessage.WriteEventsCompleted>(forwardingService); _mainBus.Subscribe <ClientMessage.TransactionStartCompleted>(forwardingService); _mainBus.Subscribe <ClientMessage.TransactionWriteCompleted>(forwardingService); _mainBus.Subscribe <ClientMessage.TransactionCommitCompleted>(forwardingService); _mainBus.Subscribe <ClientMessage.DeleteStreamCompleted>(forwardingService); // REQUEST MANAGEMENT var requestManagement = new RequestManagementService(_mainQueue, vNodeSettings.PrepareAckCount, vNodeSettings.CommitAckCount, vNodeSettings.PrepareTimeout, vNodeSettings.CommitTimeout); _mainBus.Subscribe <SystemMessage.SystemInit>(requestManagement); _mainBus.Subscribe <ClientMessage.WriteEvents>(requestManagement); _mainBus.Subscribe <ClientMessage.TransactionStart>(requestManagement); _mainBus.Subscribe <ClientMessage.TransactionWrite>(requestManagement); _mainBus.Subscribe <ClientMessage.TransactionCommit>(requestManagement); _mainBus.Subscribe <ClientMessage.DeleteStream>(requestManagement); _mainBus.Subscribe <StorageMessage.RequestCompleted>(requestManagement); _mainBus.Subscribe <StorageMessage.CheckStreamAccessCompleted>(requestManagement); _mainBus.Subscribe <StorageMessage.AlreadyCommitted>(requestManagement); _mainBus.Subscribe <StorageMessage.CommitAck>(requestManagement); _mainBus.Subscribe <StorageMessage.PrepareAck>(requestManagement); _mainBus.Subscribe <StorageMessage.WrongExpectedVersion>(requestManagement); _mainBus.Subscribe <StorageMessage.InvalidTransaction>(requestManagement); _mainBus.Subscribe <StorageMessage.StreamDeleted>(requestManagement); _mainBus.Subscribe <StorageMessage.RequestManagerTimerTick>(requestManagement); // SUBSCRIPTIONS var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50)); var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false); _mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.SystemStart, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.SubscribeToStream, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.PollStream, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.CheckPollTimeout, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <StorageMessage.EventCommitted, Message>()); var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex); subscrBus.Subscribe <SystemMessage.SystemStart>(subscription); subscrBus.Subscribe <SystemMessage.BecomeShuttingDown>(subscription); subscrBus.Subscribe <TcpMessage.ConnectionClosed>(subscription); subscrBus.Subscribe <ClientMessage.SubscribeToStream>(subscription); subscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(subscription); subscrBus.Subscribe <SubscriptionMessage.PollStream>(subscription); subscrBus.Subscribe <SubscriptionMessage.CheckPollTimeout>(subscription); subscrBus.Subscribe <StorageMessage.EventCommitted>(subscription); // TIMER _timeProvider = new RealTimeProvider(); _timerService = new TimerService(new ThreadBasedScheduler(_timeProvider)); _mainBus.Subscribe <SystemMessage.BecomeShutdown>(_timerService); _mainBus.Subscribe <TimerMessage.Schedule>(_timerService); // MASTER REPLICATION var masterReplicationService = new MasterReplicationService(_mainQueue, _nodeInfo.InstanceId, db, _workersHandler, epochManager, vNodeSettings.ClusterNodeCount); _mainBus.Subscribe <SystemMessage.SystemStart>(masterReplicationService); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(masterReplicationService); _mainBus.Subscribe <ReplicationMessage.ReplicaSubscriptionRequest>(masterReplicationService); _mainBus.Subscribe <ReplicationMessage.ReplicaLogPositionAck>(masterReplicationService); // REPLICA REPLICATION var replicaService = new ReplicaService(_mainQueue, db, epochManager, _workersHandler, authenticationProvider, _nodeInfo, vNodeSettings.UseSsl, vNodeSettings.SslTargetHost, vNodeSettings.SslValidateServer); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(replicaService); _mainBus.Subscribe <ReplicationMessage.ReconnectToMaster>(replicaService); _mainBus.Subscribe <ReplicationMessage.SubscribeToMaster>(replicaService); _mainBus.Subscribe <ReplicationMessage.AckLogPosition>(replicaService); _mainBus.Subscribe <StorageMessage.PrepareAck>(replicaService); _mainBus.Subscribe <StorageMessage.CommitAck>(replicaService); _mainBus.Subscribe <ClientMessage.TcpForwardMessage>(replicaService); // ELECTIONS var electionsService = new ElectionsService(_mainQueue, _nodeInfo, vNodeSettings.ClusterNodeCount, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint, epochManager, () => readIndex.LastCommitPosition, vNodeSettings.NodePriority); electionsService.SubscribeMessages(_mainBus); // GOSSIP var gossip = new NodeGossipService(_mainQueue, gossipSeedSource, _nodeInfo, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint, epochManager, () => readIndex.LastCommitPosition, vNodeSettings.NodePriority); _mainBus.Subscribe <SystemMessage.SystemInit>(gossip); _mainBus.Subscribe <GossipMessage.RetrieveGossipSeedSources>(gossip); _mainBus.Subscribe <GossipMessage.GotGossipSeedSources>(gossip); _mainBus.Subscribe <GossipMessage.Gossip>(gossip); _mainBus.Subscribe <GossipMessage.GossipReceived>(gossip); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(gossip); _mainBus.Subscribe <GossipMessage.GossipSendFailed>(gossip); _mainBus.Subscribe <SystemMessage.VNodeConnectionEstablished>(gossip); _mainBus.Subscribe <SystemMessage.VNodeConnectionLost>(gossip); _workersHandler.Start(); _mainQueue.Start(); monitoringQueue.Start(); subscrQueue.Start(); if (subsystems != null) { foreach (var subsystem in subsystems) { subsystem.Register(db, _mainQueue, _mainBus, _timerService, _timeProvider, httpSendService, new[] { _internalHttpService, _externalHttpService }, _workersHandler); } } }