public void CanGetClientForEndpoint() { var sut = new EventStoreClusterClientCache(new FakePublisher(), EventStoreClusterClientFactory); var client = sut.Get(new IPEndPoint(IPAddress.Loopback, 1113)); Assert.AreEqual(client, sut.Get(new IPEndPoint(IPAddress.Loopback, 1113))); }
public void ShouldScheduleCacheCleanOnTimer() { var interval = TimeSpan.FromMilliseconds(1); var bus = new FakePublisher(); var sut = new EventStoreClusterClientCache(bus, EventStoreClusterClientFactory, interval, interval); sut.Handle(new SystemMessage.SystemInit()); Assert.True(bus.Messages.OfType <TimerMessage.Schedule>().Count() == 1); }
public async Task CleansCacheOnThreshold() { var interval = TimeSpan.FromMinutes(30); var oldItemThreshold = TimeSpan.FromMilliseconds(500); var sut = new EventStoreClusterClientCache(new FakePublisher(), EventStoreClusterClientFactory, interval, oldItemThreshold); var oldClient = sut.Get(new IPEndPoint(IPAddress.Loopback, 1113)); sut.Handle(new ClusterClientMessage.CleanCache()); await Task.Delay(oldItemThreshold.Add(TimeSpan.FromMilliseconds(500))); var newClient = sut.Get(new IPEndPoint(IPAddress.Loopback, 1113)); newClient = sut.Get(new IPEndPoint(IPAddress.Loopback, 1113)); Assert.AreNotEqual(oldClient, newClient); }
public async Task ShouldDisposeClientOnceEvictedFromCache() { var interval = TimeSpan.FromMinutes(30); var oldItemThreshold = TimeSpan.FromMilliseconds(500); var sut = new EventStoreClusterClientCache(new FakePublisher(), EventStoreClusterClientFactory, interval, oldItemThreshold); var client = sut.Get(new IPEndPoint(IPAddress.Loopback, 1113)); Assert.NotNull(client); await Task.Delay(oldItemThreshold.Add(TimeSpan.FromMilliseconds(500))); sut.Handle(new ClusterClientMessage.CleanCache()); // Give the cache enough time to dispose the client await Task.Delay(oldItemThreshold); Assert.True(client.Disposed); }
public GrpcSendService(EventStoreClusterClientCache eventStoreClientCache) { _eventStoreClientCache = eventStoreClientCache ?? throw new ArgumentNullException(nameof(eventStoreClientCache)); }
public ClusterVNode(TFChunkDb db, ClusterVNodeSettings vNodeSettings, IGossipSeedSource gossipSeedSource, InfoController infoController, params ISubsystem[] subsystems) { Ensure.NotNull(db, "db"); Ensure.NotNull(vNodeSettings, "vNodeSettings"); Ensure.NotNull(gossipSeedSource, "gossipSeedSource"); #if DEBUG AddTask(_taskAddedTrigger.Task); #endif var isSingleNode = vNodeSettings.ClusterNodeCount == 1; _vNodeSettings = vNodeSettings; _nodeInfo = vNodeSettings.NodeInfo; _certificate = vNodeSettings.Certificate; _mainBus = new InMemoryBus("MainBus"); _httpMessageHandler = vNodeSettings.CreateHttpMessageHandler?.Invoke(); _queueStatsManager = new QueueStatsManager(); var forwardingProxy = new MessageForwardingProxy(); if (vNodeSettings.EnableHistograms) { HistogramService.CreateHistograms(); //start watching jitter HistogramService.StartJitterMonitor(); } // MISC WORKERS _workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum => new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1), watchSlowMsg: true, slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray(); _workersHandler = new MultiQueuedHandler( vNodeSettings.WorkerThreads, queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum], string.Format("Worker #{0}", queueNum + 1), _queueStatsManager, groupName: "Workers", watchSlowMsg: true, slowMsgThreshold: TimeSpan.FromMilliseconds(200))); _subsystems = subsystems; _controller = new ClusterVNodeController((IPublisher)_mainBus, _nodeInfo, db, vNodeSettings, this, forwardingProxy, _subsystems); _mainQueue = QueuedHandler.CreateQueuedHandler(_controller, "MainQueue", _queueStatsManager); _controller.SetMainQueue(_mainQueue); _eventStoreClusterClientCache = new EventStoreClusterClientCache(_mainQueue, (endpoint, publisher) => new EventStoreClusterClient( new UriBuilder(_vNodeSettings.GossipOverHttps ? Uri.UriSchemeHttps : Uri.UriSchemeHttp, endpoint.Address.ToString(), endpoint.Port).Uri, publisher, () => _httpMessageHandler)); _mainBus.Subscribe <ClusterClientMessage.CleanCache>(_eventStoreClusterClientCache); _mainBus.Subscribe <SystemMessage.SystemInit>(_eventStoreClusterClientCache); //SELF _mainBus.Subscribe <SystemMessage.StateChangeMessage>(this); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(this); _mainBus.Subscribe <SystemMessage.BecomeShutdown>(this); // MONITORING var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false); var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false); var monitoringQueue = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", _queueStatsManager, true, TimeSpan.FromMilliseconds(800)); var monitoring = new MonitoringService(monitoringQueue, monitoringRequestBus, _mainQueue, db.Config.WriterCheckpoint, db.Config.Path, vNodeSettings.StatsPeriod, _nodeInfo.ExternalHttp, vNodeSettings.StatsStorage, _nodeInfo.ExternalTcp, _nodeInfo.ExternalSecureTcp); _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.SystemInit, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShutdown, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom <ClientMessage.WriteEventsCompleted, Message>()); monitoringInnerBus.Subscribe <SystemMessage.SystemInit>(monitoring); monitoringInnerBus.Subscribe <SystemMessage.StateChangeMessage>(monitoring); monitoringInnerBus.Subscribe <SystemMessage.BecomeShuttingDown>(monitoring); monitoringInnerBus.Subscribe <SystemMessage.BecomeShutdown>(monitoring); monitoringInnerBus.Subscribe <ClientMessage.WriteEventsCompleted>(monitoring); monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshStats>(monitoring); monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshTcpConnectionStats>(monitoring); var truncPos = db.Config.TruncateCheckpoint.Read(); var writerCheckpoint = db.Config.WriterCheckpoint.Read(); var chaserCheckpoint = db.Config.ChaserCheckpoint.Read(); var epochCheckpoint = db.Config.EpochCheckpoint.Read(); if (truncPos != -1) { Log.Information( "Truncate checkpoint is present. Truncate: {truncatePosition} (0x{truncatePosition:X}), Writer: {writerCheckpoint} (0x{writerCheckpoint:X}), Chaser: {chaserCheckpoint} (0x{chaserCheckpoint:X}), Epoch: {epochCheckpoint} (0x{epochCheckpoint:X})", truncPos, truncPos, writerCheckpoint, writerCheckpoint, chaserCheckpoint, chaserCheckpoint, epochCheckpoint, epochCheckpoint); var truncator = new TFChunkDbTruncator(db.Config); truncator.TruncateDb(truncPos); } // STORAGE SUBSYSTEM db.Open(vNodeSettings.VerifyDbHash, threads: vNodeSettings.InitializationThreads); var indexPath = vNodeSettings.Index ?? Path.Combine(db.Config.Path, "index"); var readerPool = new ObjectPool <ITransactionFileReader>( "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, vNodeSettings.PTableMaxReaderCount, () => new TFChunkReader(db, db.Config.WriterCheckpoint, optimizeReadSideCache: db.Config.OptimizeReadSideCache)); var tableIndex = new TableIndex(indexPath, new XXHashUnsafe(), new Murmur3AUnsafe(), () => new HashListMemTable(vNodeSettings.IndexBitnessVersion, maxSize: vNodeSettings.MaxMemtableEntryCount * 2), () => new TFReaderLease(readerPool), vNodeSettings.IndexBitnessVersion, maxSizeForMemory: vNodeSettings.MaxMemtableEntryCount, maxTablesPerLevel: 2, inMem: db.Config.InMemDb, skipIndexVerify: vNodeSettings.SkipIndexVerify, indexCacheDepth: vNodeSettings.IndexCacheDepth, initializationThreads: vNodeSettings.InitializationThreads, additionalReclaim: false, maxAutoMergeIndexLevel: vNodeSettings.MaxAutoMergeIndexLevel, pTableMaxReaderCount: vNodeSettings.PTableMaxReaderCount); var readIndex = new ReadIndex(_mainQueue, readerPool, tableIndex, ESConsts.StreamInfoCacheCapacity, Application.IsDefined(Application.AdditionalCommitChecks), Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1, vNodeSettings.HashCollisionReadLimit, vNodeSettings.SkipIndexScanOnReads, db.Config.ReplicationCheckpoint, db.Config.IndexCheckpoint); _readIndex = readIndex; var writer = new TFChunkWriter(db); var epochManager = new EpochManager(_mainQueue, ESConsts.CachedEpochCount, db.Config.EpochCheckpoint, writer, initialReaderCount: 1, maxReaderCount: 5, readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint, optimizeReadSideCache: db.Config.OptimizeReadSideCache)); epochManager.Init(); var storageWriter = new ClusterStorageWriterService(_mainQueue, _mainBus, vNodeSettings.MinFlushDelay, db, writer, readIndex.IndexWriter, epochManager, _queueStatsManager, () => readIndex.LastIndexedPosition); // subscribes internally AddTasks(storageWriter.Tasks); monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageWriter); var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, vNodeSettings.ReaderThreadsCount, db.Config.WriterCheckpoint, _queueStatsManager); _mainBus.Subscribe <SystemMessage.SystemInit>(storageReader); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageReader); _mainBus.Subscribe <SystemMessage.BecomeShutdown>(storageReader); monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageReader); //REPLICATION TRACKING var replicationTracker = new ReplicationTrackingService(_mainQueue, vNodeSettings.ClusterNodeCount, db.Config.ReplicationCheckpoint, db.Config.WriterCheckpoint); AddTask(replicationTracker.Task); _mainBus.Subscribe <SystemMessage.SystemInit>(replicationTracker); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(replicationTracker); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(replicationTracker); _mainBus.Subscribe <ReplicationTrackingMessage.ReplicaWriteAck>(replicationTracker); _mainBus.Subscribe <ReplicationTrackingMessage.WriterCheckpointFlushed>(replicationTracker); _mainBus.Subscribe <ReplicationTrackingMessage.LeaderReplicatedTo>(replicationTracker); _mainBus.Subscribe <SystemMessage.VNodeConnectionLost>(replicationTracker); var indexCommitterService = new IndexCommitterService(readIndex.IndexCommitter, _mainQueue, db.Config.WriterCheckpoint, db.Config.ReplicationCheckpoint, vNodeSettings.CommitAckCount, tableIndex, _queueStatsManager); AddTask(indexCommitterService.Task); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(indexCommitterService); _mainBus.Subscribe <ReplicationTrackingMessage.ReplicatedTo>(indexCommitterService); _mainBus.Subscribe <StorageMessage.CommitAck>(indexCommitterService); _mainBus.Subscribe <ClientMessage.MergeIndexes>(indexCommitterService); var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint, db.Config.OptimizeReadSideCache); var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, indexCommitterService, epochManager, _queueStatsManager); AddTask(storageChaser.Task); #if DEBUG _queueStatsManager.InitializeCheckpoints(db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint); #endif _mainBus.Subscribe <SystemMessage.SystemInit>(storageChaser); _mainBus.Subscribe <SystemMessage.SystemStart>(storageChaser); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageChaser); // AUTHENTICATION INFRASTRUCTURE - delegate to plugins _internalAuthenticationProvider = vNodeSettings.AuthenticationProviderFactory.BuildAuthenticationProvider(_mainQueue, _mainBus, _workersHandler, _workerBuses, vNodeSettings.LogFailedAuthenticationAttempts); Ensure.NotNull(_internalAuthenticationProvider, "authenticationProvider"); _authorizationProvider = vNodeSettings.AuthorizationProviderFactory.Build(_mainQueue); Ensure.NotNull(_authorizationProvider, "authorizationProvider"); AuthorizationGateway = new AuthorizationGateway(_authorizationProvider); { // EXTERNAL TCP if (_nodeInfo.ExternalTcp != null && vNodeSettings.EnableExternalTCP) { var extTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalTcp, _workersHandler, TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(), vNodeSettings.ExtTcpHeartbeatInterval, vNodeSettings.ExtTcpHeartbeatTimeout, _internalAuthenticationProvider, AuthorizationGateway, null, false, vNodeSettings.ConnectionPendingSendBytesThreshold, vNodeSettings.ConnectionQueueSizeThreshold); _mainBus.Subscribe <SystemMessage.SystemInit>(extTcpService); _mainBus.Subscribe <SystemMessage.SystemStart>(extTcpService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extTcpService); } // EXTERNAL SECURE TCP if (_nodeInfo.ExternalSecureTcp != null && vNodeSettings.EnableExternalTCP) { var extSecTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalSecureTcp, _workersHandler, TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(), vNodeSettings.ExtTcpHeartbeatInterval, vNodeSettings.ExtTcpHeartbeatTimeout, _internalAuthenticationProvider, AuthorizationGateway, vNodeSettings.Certificate, false, vNodeSettings.ConnectionPendingSendBytesThreshold, vNodeSettings.ConnectionQueueSizeThreshold); _mainBus.Subscribe <SystemMessage.SystemInit>(extSecTcpService); _mainBus.Subscribe <SystemMessage.SystemStart>(extSecTcpService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extSecTcpService); } if (!isSingleNode) { // INTERNAL TCP if (_nodeInfo.InternalTcp != null) { var intTcpService = new TcpService(_mainQueue, _nodeInfo.InternalTcp, _workersHandler, TcpServiceType.Internal, TcpSecurityType.Normal, new InternalTcpDispatcher(), vNodeSettings.IntTcpHeartbeatInterval, vNodeSettings.IntTcpHeartbeatTimeout, _internalAuthenticationProvider, AuthorizationGateway, null, false, ESConsts.UnrestrictedPendingSendBytes, ESConsts.MaxConnectionQueueSize); _mainBus.Subscribe <SystemMessage.SystemInit>(intTcpService); _mainBus.Subscribe <SystemMessage.SystemStart>(intTcpService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intTcpService); } // INTERNAL SECURE TCP if (_nodeInfo.InternalSecureTcp != null) { var intSecTcpService = new TcpService(_mainQueue, _nodeInfo.InternalSecureTcp, _workersHandler, TcpServiceType.Internal, TcpSecurityType.Secure, new InternalTcpDispatcher(), vNodeSettings.IntTcpHeartbeatInterval, vNodeSettings.IntTcpHeartbeatTimeout, _internalAuthenticationProvider, AuthorizationGateway, vNodeSettings.Certificate, true, ESConsts.UnrestrictedPendingSendBytes, ESConsts.MaxConnectionQueueSize); _mainBus.Subscribe <SystemMessage.SystemInit>(intSecTcpService); _mainBus.Subscribe <SystemMessage.SystemStart>(intSecTcpService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intSecTcpService); } } } SubscribeWorkers(bus => { var tcpSendService = new TcpSendService(); // ReSharper disable RedundantTypeArgumentsOfMethod bus.Subscribe <TcpMessage.TcpSend>(tcpSendService); // ReSharper restore RedundantTypeArgumentsOfMethod }); var httpAuthenticationProviders = new List <IHttpAuthenticationProvider> { new BasicHttpAuthenticationProvider(_internalAuthenticationProvider), }; if (vNodeSettings.EnableTrustedAuth) { httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider()); } httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider()); var httpPipe = new HttpMessagePipe(); var httpSendService = new HttpSendService(httpPipe, forwardRequests: true); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(httpSendService); SubscribeWorkers(bus => { bus.Subscribe <HttpMessage.HttpSend>(httpSendService); }); var grpcSendService = new GrpcSendService(_eventStoreClusterClientCache); _mainBus.Subscribe(new WideningHandler <GrpcMessage.SendOverGrpc, Message>(_workersHandler)); SubscribeWorkers(bus => { bus.Subscribe <GrpcMessage.SendOverGrpc>(grpcSendService); }); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(infoController); var adminController = new AdminController(_mainQueue, _workersHandler); var pingController = new PingController(); var histogramController = new HistogramController(); var statController = new StatController(monitoringQueue, _workersHandler); var atomController = new AtomController(_mainQueue, _workersHandler, vNodeSettings.DisableHTTPCaching); var gossipController = new GossipController(_mainQueue, _workersHandler, vNodeSettings.GossipTimeout, _httpMessageHandler); var persistentSubscriptionController = new PersistentSubscriptionController(httpSendService, _mainQueue, _workersHandler); // HTTP SENDERS gossipController.SubscribeSenders(httpPipe); // EXTERNAL HTTP _externalHttpService = new KestrelHttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(), _workersHandler, vNodeSettings.LogHttpRequests, vNodeSettings.GossipAdvertiseInfo.AdvertiseExternalIPAs, vNodeSettings.GossipAdvertiseInfo.AdvertiseExternalHttpPortAs, vNodeSettings.DisableFirstLevelHttpAuthorization, vNodeSettings.NodeInfo.ExternalHttp); _externalHttpService.SetupController(persistentSubscriptionController); if (vNodeSettings.AdminOnPublic) { _externalHttpService.SetupController(adminController); } _externalHttpService.SetupController(pingController); _externalHttpService.SetupController(infoController); if (vNodeSettings.StatsOnPublic) { _externalHttpService.SetupController(statController); } if (vNodeSettings.EnableAtomPubOverHTTP) { _externalHttpService.SetupController(atomController); } if (vNodeSettings.GossipOnPublic) { _externalHttpService.SetupController(gossipController); } _externalHttpService.SetupController(histogramController); _mainBus.Subscribe <SystemMessage.SystemInit>(_externalHttpService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_externalHttpService); // Authentication plugin HTTP vNodeSettings.AuthenticationProviderFactory.RegisterHttpControllers(_externalHttpService, httpSendService, _mainQueue, _workersHandler); SubscribeWorkers(KestrelHttpService.CreateAndSubscribePipeline); // REQUEST FORWARDING var forwardingService = new RequestForwardingService(_mainQueue, forwardingProxy, TimeSpan.FromSeconds(1)); _mainBus.Subscribe <SystemMessage.SystemStart>(forwardingService); _mainBus.Subscribe <SystemMessage.RequestForwardingTimerTick>(forwardingService); _mainBus.Subscribe <ClientMessage.NotHandled>(forwardingService); _mainBus.Subscribe <ClientMessage.WriteEventsCompleted>(forwardingService); _mainBus.Subscribe <ClientMessage.TransactionStartCompleted>(forwardingService); _mainBus.Subscribe <ClientMessage.TransactionWriteCompleted>(forwardingService); _mainBus.Subscribe <ClientMessage.TransactionCommitCompleted>(forwardingService); _mainBus.Subscribe <ClientMessage.DeleteStreamCompleted>(forwardingService); // REQUEST MANAGEMENT var requestManagement = new RequestManagementService( _mainQueue, vNodeSettings.PrepareTimeout, vNodeSettings.CommitTimeout); _mainBus.Subscribe <SystemMessage.SystemInit>(requestManagement); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(requestManagement); _mainBus.Subscribe <ClientMessage.WriteEvents>(requestManagement); _mainBus.Subscribe <ClientMessage.TransactionStart>(requestManagement); _mainBus.Subscribe <ClientMessage.TransactionWrite>(requestManagement); _mainBus.Subscribe <ClientMessage.TransactionCommit>(requestManagement); _mainBus.Subscribe <ClientMessage.DeleteStream>(requestManagement); _mainBus.Subscribe <StorageMessage.AlreadyCommitted>(requestManagement); _mainBus.Subscribe <StorageMessage.CommitAck>(requestManagement); _mainBus.Subscribe <StorageMessage.PrepareAck>(requestManagement); _mainBus.Subscribe <ReplicationTrackingMessage.ReplicatedTo>(requestManagement); _mainBus.Subscribe <ReplicationTrackingMessage.IndexedTo>(requestManagement); _mainBus.Subscribe <StorageMessage.RequestCompleted>(requestManagement); _mainBus.Subscribe <StorageMessage.CommitIndexed>(requestManagement); _mainBus.Subscribe <StorageMessage.WrongExpectedVersion>(requestManagement); _mainBus.Subscribe <StorageMessage.InvalidTransaction>(requestManagement); _mainBus.Subscribe <StorageMessage.StreamDeleted>(requestManagement); _mainBus.Subscribe <StorageMessage.RequestManagerTimerTick>(requestManagement); // SUBSCRIPTIONS var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50)); var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", _queueStatsManager, false); _mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.SystemStart, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.SubscribeToStream, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.FilteredSubscribeToStream, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.PollStream, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.CheckPollTimeout, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <StorageMessage.EventCommitted, Message>()); var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex); subscrBus.Subscribe <SystemMessage.SystemStart>(subscription); subscrBus.Subscribe <SystemMessage.BecomeShuttingDown>(subscription); subscrBus.Subscribe <TcpMessage.ConnectionClosed>(subscription); subscrBus.Subscribe <ClientMessage.SubscribeToStream>(subscription); subscrBus.Subscribe <ClientMessage.FilteredSubscribeToStream>(subscription); subscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(subscription); subscrBus.Subscribe <SubscriptionMessage.PollStream>(subscription); subscrBus.Subscribe <SubscriptionMessage.CheckPollTimeout>(subscription); subscrBus.Subscribe <StorageMessage.EventCommitted>(subscription); // PERSISTENT SUBSCRIPTIONS // IO DISPATCHER var ioDispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_mainQueue)); _mainBus.Subscribe <ClientMessage.ReadStreamEventsBackwardCompleted>(ioDispatcher.BackwardReader); _mainBus.Subscribe <ClientMessage.WriteEventsCompleted>(ioDispatcher.Writer); _mainBus.Subscribe <ClientMessage.ReadStreamEventsForwardCompleted>(ioDispatcher.ForwardReader); _mainBus.Subscribe <ClientMessage.DeleteStreamCompleted>(ioDispatcher.StreamDeleter); _mainBus.Subscribe(ioDispatcher); var perSubscrBus = new InMemoryBus("PersistentSubscriptionsBus", true, TimeSpan.FromMilliseconds(50)); var perSubscrQueue = new QueuedHandlerThreadPool(perSubscrBus, "PersistentSubscriptions", _queueStatsManager, false); _mainBus.Subscribe(perSubscrQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.CreatePersistentSubscription, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.UpdatePersistentSubscription, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.DeletePersistentSubscription, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ConnectToPersistentSubscription, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.PersistentSubscriptionAckEvents, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.PersistentSubscriptionNackEvents, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReplayParkedMessages, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReplayParkedMessage, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReadNextNPersistentMessages, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <StorageMessage.EventCommitted, Message>()); _mainBus.Subscribe(perSubscrQueue .WidenFrom <MonitoringMessage.GetAllPersistentSubscriptionStats, Message>()); _mainBus.Subscribe( perSubscrQueue.WidenFrom <MonitoringMessage.GetStreamPersistentSubscriptionStats, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <MonitoringMessage.GetPersistentSubscriptionStats, Message>()); _mainBus.Subscribe(perSubscrQueue .WidenFrom <SubscriptionMessage.PersistentSubscriptionTimerTick, Message>()); //TODO CC can have multiple threads working on subscription if partition var consumerStrategyRegistry = new PersistentSubscriptionConsumerStrategyRegistry(_mainQueue, _mainBus, vNodeSettings.AdditionalConsumerStrategies); var persistentSubscription = new PersistentSubscriptionService(perSubscrQueue, readIndex, ioDispatcher, _mainQueue, consumerStrategyRegistry); perSubscrBus.Subscribe <SystemMessage.BecomeShuttingDown>(persistentSubscription); perSubscrBus.Subscribe <SystemMessage.BecomeLeader>(persistentSubscription); perSubscrBus.Subscribe <SystemMessage.StateChangeMessage>(persistentSubscription); perSubscrBus.Subscribe <TcpMessage.ConnectionClosed>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.ConnectToPersistentSubscription>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.PersistentSubscriptionAckEvents>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.PersistentSubscriptionNackEvents>(persistentSubscription); perSubscrBus.Subscribe <StorageMessage.EventCommitted>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.DeletePersistentSubscription>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.CreatePersistentSubscription>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.UpdatePersistentSubscription>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.ReplayParkedMessages>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.ReplayParkedMessage>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.ReadNextNPersistentMessages>(persistentSubscription); perSubscrBus.Subscribe <MonitoringMessage.GetAllPersistentSubscriptionStats>(persistentSubscription); perSubscrBus.Subscribe <MonitoringMessage.GetStreamPersistentSubscriptionStats>(persistentSubscription); perSubscrBus.Subscribe <MonitoringMessage.GetPersistentSubscriptionStats>(persistentSubscription); perSubscrBus.Subscribe <SubscriptionMessage.PersistentSubscriptionTimerTick>(persistentSubscription); // STORAGE SCAVENGER var scavengerLogManager = new TFChunkScavengerLogManager(_nodeInfo.ExternalHttp.ToString(), TimeSpan.FromDays(vNodeSettings.ScavengeHistoryMaxAge), ioDispatcher); var storageScavenger = new StorageScavenger(db, tableIndex, readIndex, scavengerLogManager, vNodeSettings.AlwaysKeepScavenged, !vNodeSettings.DisableScavengeMerging, unsafeIgnoreHardDeletes: vNodeSettings.UnsafeIgnoreHardDeletes); // ReSharper disable RedundantTypeArgumentsOfMethod _mainBus.Subscribe <ClientMessage.ScavengeDatabase>(storageScavenger); _mainBus.Subscribe <ClientMessage.StopDatabaseScavenge>(storageScavenger); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(storageScavenger); // ReSharper restore RedundantTypeArgumentsOfMethod // TIMER _timeProvider = new RealTimeProvider(); var threadBasedScheduler = new ThreadBasedScheduler(_timeProvider, _queueStatsManager); AddTask(threadBasedScheduler.Task); _timerService = new TimerService(threadBasedScheduler); _mainBus.Subscribe <SystemMessage.BecomeShutdown>(_timerService); _mainBus.Subscribe <TimerMessage.Schedule>(_timerService); var gossipInfo = new VNodeInfo(_nodeInfo.InstanceId, _nodeInfo.DebugIndex, vNodeSettings.GossipAdvertiseInfo.InternalTcp, vNodeSettings.GossipAdvertiseInfo.InternalSecureTcp, vNodeSettings.GossipAdvertiseInfo.ExternalTcp, vNodeSettings.GossipAdvertiseInfo.ExternalSecureTcp, vNodeSettings.GossipAdvertiseInfo.InternalHttp, vNodeSettings.GossipAdvertiseInfo.ExternalHttp, vNodeSettings.ReadOnlyReplica); if (!isSingleNode) { // LEADER REPLICATION var leaderReplicationService = new LeaderReplicationService(_mainQueue, gossipInfo.InstanceId, db, _workersHandler, epochManager, vNodeSettings.ClusterNodeCount, vNodeSettings.UnsafeAllowSurplusNodes, _queueStatsManager); AddTask(leaderReplicationService.Task); _mainBus.Subscribe <SystemMessage.SystemStart>(leaderReplicationService); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(leaderReplicationService); _mainBus.Subscribe <ReplicationMessage.ReplicaSubscriptionRequest>(leaderReplicationService); _mainBus.Subscribe <ReplicationMessage.ReplicaLogPositionAck>(leaderReplicationService); _mainBus.Subscribe <ReplicationTrackingMessage.ReplicatedTo>(leaderReplicationService); monitoringInnerBus.Subscribe <ReplicationMessage.GetReplicationStats>(leaderReplicationService); // REPLICA REPLICATION var replicaService = new ReplicaService(_mainQueue, db, epochManager, _workersHandler, _internalAuthenticationProvider, AuthorizationGateway, gossipInfo, !vNodeSettings.DisableInternalTls, true, Certificate == null ? null : new X509Certificate2Collection(Certificate), vNodeSettings.IntTcpHeartbeatTimeout, vNodeSettings.ExtTcpHeartbeatInterval); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(replicaService); _mainBus.Subscribe <ReplicationMessage.ReconnectToLeader>(replicaService); _mainBus.Subscribe <ReplicationMessage.SubscribeToLeader>(replicaService); _mainBus.Subscribe <ReplicationMessage.AckLogPosition>(replicaService); _mainBus.Subscribe <ClientMessage.TcpForwardMessage>(replicaService); } // ELECTIONS var electionsService = new ElectionsService(_mainQueue, gossipInfo, vNodeSettings.ClusterNodeCount, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint, epochManager, () => readIndex.LastIndexedPosition, vNodeSettings.NodePriority, _timeProvider); electionsService.SubscribeMessages(_mainBus); if (!isSingleNode || vNodeSettings.GossipOnSingleNode) { // GOSSIP var gossip = new NodeGossipService(_mainQueue, gossipSeedSource, gossipInfo, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint, epochManager, () => readIndex.LastIndexedPosition, vNodeSettings.NodePriority, vNodeSettings.GossipInterval, vNodeSettings.GossipAllowedTimeDifference, vNodeSettings.GossipTimeout, vNodeSettings.DeadMemberRemovalPeriod, _timeProvider); _mainBus.Subscribe <SystemMessage.SystemInit>(gossip); _mainBus.Subscribe <GossipMessage.RetrieveGossipSeedSources>(gossip); _mainBus.Subscribe <GossipMessage.GotGossipSeedSources>(gossip); _mainBus.Subscribe <GossipMessage.Gossip>(gossip); _mainBus.Subscribe <GossipMessage.GossipReceived>(gossip); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(gossip); _mainBus.Subscribe <GossipMessage.GossipSendFailed>(gossip); _mainBus.Subscribe <GossipMessage.UpdateNodePriority>(gossip); _mainBus.Subscribe <SystemMessage.VNodeConnectionEstablished>(gossip); _mainBus.Subscribe <SystemMessage.VNodeConnectionLost>(gossip); _mainBus.Subscribe <GossipMessage.GetGossipFailed>(gossip); _mainBus.Subscribe <GossipMessage.GetGossipReceived>(gossip); _mainBus.Subscribe <ElectionMessage.ElectionsDone>(gossip); } // kestrel AddTasks(_workersHandler.Start()); AddTask(_mainQueue.Start()); AddTask(monitoringQueue.Start()); AddTask(subscrQueue.Start()); AddTask(perSubscrQueue.Start()); if (subsystems != null) { foreach (var subsystem in subsystems) { var http = new[] { _externalHttpService }; subsystem.Register(new StandardComponents(db, _mainQueue, _mainBus, _timerService, _timeProvider, httpSendService, http, _workersHandler, _queueStatsManager)); } } _startup = new ClusterVNodeStartup(_subsystems, _mainQueue, _mainBus, _workersHandler, httpAuthenticationProviders, _authorizationProvider, _readIndex, _vNodeSettings.MaxAppendSize, _externalHttpService); _mainBus.Subscribe <SystemMessage.SystemReady>(_startup); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_startup); }