public LeaderReplicationService(
IPublisher publisher,
Guid instanceId,
TFChunkDb db,
IPublisher tcpSendPublisher,
IEpochManager epochManager,
int clusterSize,
bool unsafeAllowSurplusNodes,
QueueStatsManager queueStatsManager)
{
Ensure.NotNull(publisher, "publisher");
Ensure.NotEmptyGuid(instanceId, "instanceId");
Ensure.NotNull(db, "db");
Ensure.NotNull(tcpSendPublisher, "tcpSendPublisher");
Ensure.NotNull(epochManager, "epochManager");
Ensure.Positive(clusterSize, "clusterSize");
_publisher = publisher;
_instanceId = instanceId;
_db = db;
_tcpSendPublisher = tcpSendPublisher;
_epochManager = epochManager;
_clusterSize = clusterSize;
_unsafeAllowSurplusNodes = unsafeAllowSurplusNodes;
_queueStats = queueStatsManager.CreateQueueStatsCollector("Leader Replication Service");
_lastRolesAssignmentTimestamp = _stopwatch.Elapsed;
_mainLoopThread = new Thread(MainLoop)
{
Name = _queueStats.Name, IsBackground = true
};
}
public StorageReaderService(
IPublisher bus,
ISubscriber subscriber,
IReadIndex readIndex,
int threadCount,
IReadOnlyCheckpoint writerCheckpoint,
QueueStatsManager queueStatsManager)
{
Ensure.NotNull(bus, "bus");
Ensure.NotNull(subscriber, "subscriber");
Ensure.NotNull(readIndex, "readIndex");
Ensure.Positive(threadCount, "threadCount");
Ensure.NotNull(writerCheckpoint, "writerCheckpoint");
_bus = bus;
_readIndex = readIndex;
_threadCount = threadCount;
StorageReaderWorker[] readerWorkers = new StorageReaderWorker[threadCount];
InMemoryBus[] storageReaderBuses = new InMemoryBus[threadCount];
for (var i = 0; i < threadCount; i++)
{
readerWorkers[i] = new StorageReaderWorker(bus, readIndex, writerCheckpoint, i);
storageReaderBuses[i] = new InMemoryBus("StorageReaderBus", watchSlowMsg: false);
storageReaderBuses[i].Subscribe <ClientMessage.ReadEvent>(readerWorkers[i]);
storageReaderBuses[i].Subscribe <ClientMessage.ReadStreamEventsBackward>(readerWorkers[i]);
storageReaderBuses[i].Subscribe <ClientMessage.ReadStreamEventsForward>(readerWorkers[i]);
storageReaderBuses[i].Subscribe <ClientMessage.ReadAllEventsForward>(readerWorkers[i]);
storageReaderBuses[i].Subscribe <ClientMessage.ReadAllEventsBackward>(readerWorkers[i]);
storageReaderBuses[i].Subscribe <ClientMessage.FilteredReadAllEventsForward>(readerWorkers[i]);
storageReaderBuses[i].Subscribe <ClientMessage.FilteredReadAllEventsBackward>(readerWorkers[i]);
storageReaderBuses[i].Subscribe <StorageMessage.BatchLogExpiredMessages>(readerWorkers[i]);
storageReaderBuses[i].Subscribe <StorageMessage.EffectiveStreamAclRequest>(readerWorkers[i]);
storageReaderBuses[i].Subscribe <StorageMessage.StreamIdFromTransactionIdRequest>(readerWorkers[i]);
}
_workersMultiHandler = new MultiQueuedHandler(
_threadCount,
queueNum => new QueuedHandlerThreadPool(storageReaderBuses[queueNum],
string.Format("StorageReaderQueue #{0}", queueNum + 1),
queueStatsManager,
groupName: "StorageReaderQueue",
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200)));
_workersMultiHandler.Start();
subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadEvent, Message>());
subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadStreamEventsBackward, Message>());
subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadStreamEventsForward, Message>());
subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadAllEventsForward, Message>());
subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.ReadAllEventsBackward, Message>());
subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.FilteredReadAllEventsForward, Message>());
subscriber.Subscribe(_workersMultiHandler.WidenFrom <ClientMessage.FilteredReadAllEventsBackward, Message>());
subscriber.Subscribe(_workersMultiHandler.WidenFrom <StorageMessage.BatchLogExpiredMessages, Message>());
subscriber.Subscribe(_workersMultiHandler.WidenFrom <StorageMessage.EffectiveStreamAclRequest, Message>());
subscriber.Subscribe(_workersMultiHandler.WidenFrom <StorageMessage.StreamIdFromTransactionIdRequest, Message>());
}
public ThreadBasedScheduler(ITimeProvider timeProvider, QueueStatsManager queueStatsManager)
{
Ensure.NotNull(timeProvider, "timeProvider");
_timeProvider = timeProvider;
_queueStats = queueStatsManager.CreateQueueStatsCollector("Timer");
_timerThread = new Thread(DoTiming);
_timerThread.IsBackground = true;
_timerThread.Name = Name;
_timerThread.Start();
}
public StorageWriterService(IPublisher bus,
ISubscriber subscribeToBus,
TimeSpan minFlushDelay,
TFChunkDb db,
TFChunkWriter writer,
IIndexWriter indexWriter,
IEpochManager epochManager,
QueueStatsManager queueStatsManager)
{
Ensure.NotNull(bus, "bus");
Ensure.NotNull(subscribeToBus, "subscribeToBus");
Ensure.NotNull(db, "db");
Ensure.NotNull(writer, "writer");
Ensure.NotNull(indexWriter, "indexWriter");
Ensure.NotNull(epochManager, "epochManager");
Bus = bus;
_subscribeToBus = subscribeToBus;
Db = db;
_indexWriter = indexWriter;
EpochManager = epochManager;
_minFlushDelay = minFlushDelay.TotalMilliseconds * TicksPerMs;
_lastFlushDelay = 0;
_lastFlushTimestamp = _watch.ElapsedTicks;
Writer = writer;
Writer.Open();
_writerBus = new InMemoryBus("StorageWriterBus", watchSlowMsg: false);
StorageWriterQueue = QueuedHandler.CreateQueuedHandler(new AdHocHandler <Message>(CommonHandle),
"StorageWriterQueue",
queueStatsManager,
true,
TimeSpan.FromMilliseconds(500));
_tasks.Add(StorageWriterQueue.Start());
SubscribeToMessage <SystemMessage.SystemInit>();
SubscribeToMessage <SystemMessage.StateChangeMessage>();
SubscribeToMessage <SystemMessage.WriteEpoch>();
SubscribeToMessage <SystemMessage.WaitForChaserToCatchUp>();
SubscribeToMessage <StorageMessage.WritePrepares>();
SubscribeToMessage <StorageMessage.WriteDelete>();
SubscribeToMessage <StorageMessage.WriteTransactionStart>();
SubscribeToMessage <StorageMessage.WriteTransactionData>();
SubscribeToMessage <StorageMessage.WriteTransactionEnd>();
SubscribeToMessage <StorageMessage.WriteCommit>();
}
public ClusterStorageWriterService(IPublisher bus,
ISubscriber subscribeToBus,
TimeSpan minFlushDelay,
TFChunkDb db,
TFChunkWriter writer,
IIndexWriter indexWriter,
IEpochManager epochManager,
QueueStatsManager queueStatsManager,
Func <long> getLastIndexedPosition)
: base(bus, subscribeToBus, minFlushDelay, db, writer, indexWriter, epochManager, queueStatsManager)
{
Ensure.NotNull(getLastIndexedPosition, "getLastCommitPosition");
_getLastIndexedPosition = getLastIndexedPosition;
_framer = new LengthPrefixSuffixFramer(OnLogRecordUnframed, TFConsts.MaxLogRecordSize);
SubscribeToMessage <ReplicationMessage.ReplicaSubscribed>();
SubscribeToMessage <ReplicationMessage.CreateChunk>();
SubscribeToMessage <ReplicationMessage.RawChunkBulk>();
SubscribeToMessage <ReplicationMessage.DataChunkBulk>();
}
public StandardComponents(
TFChunkDb db,
IQueuedHandler mainQueue,
ISubscriber mainBus,
TimerService timerService,
ITimeProvider timeProvider,
IHttpForwarder httpForwarder,
IHttpService[] httpServices,
IPublisher networkSendService,
QueueStatsManager queueStatsManager)
{
_db = db;
_mainQueue = mainQueue;
_mainBus = mainBus;
_timerService = timerService;
_timeProvider = timeProvider;
_httpForwarder = httpForwarder;
_httpServices = httpServices;
_networkSendService = networkSendService;
_queueStatsManager = queueStatsManager;
}
public IndexCommitterService(
IIndexCommitter indexCommitter,
IPublisher publisher,
ICheckpoint writerCheckpoint,
ICheckpoint replicationCheckpoint,
int commitCount,
ITableIndex tableIndex,
QueueStatsManager queueStatsManager)
{
Ensure.NotNull(indexCommitter, nameof(indexCommitter));
Ensure.NotNull(publisher, nameof(publisher));
Ensure.NotNull(writerCheckpoint, nameof(writerCheckpoint));
Ensure.NotNull(replicationCheckpoint, nameof(replicationCheckpoint));
Ensure.Positive(commitCount, nameof(commitCount));
_indexCommitter = indexCommitter;
_publisher = publisher;
_writerCheckpoint = writerCheckpoint;
_replicationCheckpoint = replicationCheckpoint;
_commitCount = commitCount;
_tableIndex = tableIndex;
_queueStats = queueStatsManager.CreateQueueStatsCollector("Index Committer");
}
public StorageChaser(IPublisher leaderBus,
IReadOnlyCheckpoint writerCheckpoint,
ITransactionFileChaser chaser,
IIndexCommitterService indexCommitterService,
IEpochManager epochManager,
QueueStatsManager queueStatsManager)
{
Ensure.NotNull(leaderBus, "leaderBus");
Ensure.NotNull(writerCheckpoint, "writerCheckpoint");
Ensure.NotNull(chaser, "chaser");
Ensure.NotNull(indexCommitterService, "indexCommitterService");
Ensure.NotNull(epochManager, "epochManager");
_leaderBus = leaderBus;
_writerCheckpoint = writerCheckpoint;
_chaser = chaser;
_indexCommitterService = indexCommitterService;
_epochManager = epochManager;
_queueStats = queueStatsManager.CreateQueueStatsCollector("Storage Chaser");
_flushDelay = 0;
_lastFlush = _watch.ElapsedTicks;
}
public ClusterVNode(TFChunkDb db,
ClusterVNodeSettings vNodeSettings,
IGossipSeedSource gossipSeedSource,
InfoController infoController,
params ISubsystem[] subsystems)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(vNodeSettings, "vNodeSettings");
Ensure.NotNull(gossipSeedSource, "gossipSeedSource");
#if DEBUG
AddTask(_taskAddedTrigger.Task);
#endif
var isSingleNode = vNodeSettings.ClusterNodeCount == 1;
_vNodeSettings = vNodeSettings;
_nodeInfo = vNodeSettings.NodeInfo;
_certificate = vNodeSettings.Certificate;
_mainBus = new InMemoryBus("MainBus");
_httpMessageHandler = vNodeSettings.CreateHttpMessageHandler?.Invoke();
_queueStatsManager = new QueueStatsManager();
var forwardingProxy = new MessageForwardingProxy();
if (vNodeSettings.EnableHistograms)
{
HistogramService.CreateHistograms();
//start watching jitter
HistogramService.StartJitterMonitor();
}
// MISC WORKERS
_workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray();
_workersHandler = new MultiQueuedHandler(
vNodeSettings.WorkerThreads,
queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
string.Format("Worker #{0}", queueNum + 1),
_queueStatsManager,
groupName: "Workers",
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200)));
_subsystems = subsystems;
_controller = new ClusterVNodeController((IPublisher)_mainBus, _nodeInfo, db, vNodeSettings, this,
forwardingProxy, _subsystems);
_mainQueue = QueuedHandler.CreateQueuedHandler(_controller, "MainQueue", _queueStatsManager);
_controller.SetMainQueue(_mainQueue);
//SELF
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(this);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(this);
_mainBus.Subscribe <SystemMessage.BecomeShutdown>(this);
// MONITORING
var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
var monitoringQueue = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", _queueStatsManager, true,
TimeSpan.FromMilliseconds(800));
var monitoring = new MonitoringService(monitoringQueue,
monitoringRequestBus,
_mainQueue,
db.Config.WriterCheckpoint,
db.Config.Path,
vNodeSettings.StatsPeriod,
_nodeInfo.ExternalHttp,
vNodeSettings.StatsStorage,
_nodeInfo.ExternalTcp,
_nodeInfo.ExternalSecureTcp);
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.SystemInit, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShutdown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <ClientMessage.WriteEventsCompleted, Message>());
monitoringInnerBus.Subscribe <SystemMessage.SystemInit>(monitoring);
monitoringInnerBus.Subscribe <SystemMessage.StateChangeMessage>(monitoring);
monitoringInnerBus.Subscribe <SystemMessage.BecomeShuttingDown>(monitoring);
monitoringInnerBus.Subscribe <SystemMessage.BecomeShutdown>(monitoring);
monitoringInnerBus.Subscribe <ClientMessage.WriteEventsCompleted>(monitoring);
monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshStats>(monitoring);
monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshTcpConnectionStats>(monitoring);
var truncPos = db.Config.TruncateCheckpoint.Read();
var writerCheckpoint = db.Config.WriterCheckpoint.Read();
var chaserCheckpoint = db.Config.ChaserCheckpoint.Read();
var epochCheckpoint = db.Config.EpochCheckpoint.Read();
if (truncPos != -1)
{
Log.Info(
"Truncate checkpoint is present. Truncate: {truncatePosition} (0x{truncatePosition:X}), Writer: {writerCheckpoint} (0x{writerCheckpoint:X}), Chaser: {chaserCheckpoint} (0x{chaserCheckpoint:X}), Epoch: {epochCheckpoint} (0x{epochCheckpoint:X})",
truncPos, truncPos, writerCheckpoint, writerCheckpoint, chaserCheckpoint, chaserCheckpoint,
epochCheckpoint, epochCheckpoint);
var truncator = new TFChunkDbTruncator(db.Config);
truncator.TruncateDb(truncPos);
}
// STORAGE SUBSYSTEM
db.Open(vNodeSettings.VerifyDbHash, threads: vNodeSettings.InitializationThreads);
var indexPath = vNodeSettings.Index ?? Path.Combine(db.Config.Path, "index");
var readerPool = new ObjectPool <ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, vNodeSettings.PTableMaxReaderCount,
() => new TFChunkReader(db, db.Config.WriterCheckpoint,
optimizeReadSideCache: db.Config.OptimizeReadSideCache));
var tableIndex = new TableIndex(indexPath,
new XXHashUnsafe(),
new Murmur3AUnsafe(),
() => new HashListMemTable(vNodeSettings.IndexBitnessVersion,
maxSize: vNodeSettings.MaxMemtableEntryCount * 2),
() => new TFReaderLease(readerPool),
vNodeSettings.IndexBitnessVersion,
maxSizeForMemory: vNodeSettings.MaxMemtableEntryCount,
maxTablesPerLevel: 2,
inMem: db.Config.InMemDb,
skipIndexVerify: vNodeSettings.SkipIndexVerify,
indexCacheDepth: vNodeSettings.IndexCacheDepth,
initializationThreads: vNodeSettings.InitializationThreads,
additionalReclaim: false,
maxAutoMergeIndexLevel: vNodeSettings.MaxAutoMergeIndexLevel,
pTableMaxReaderCount: vNodeSettings.PTableMaxReaderCount);
var readIndex = new ReadIndex(_mainQueue,
readerPool,
tableIndex,
ESConsts.StreamInfoCacheCapacity,
Application.IsDefined(Application.AdditionalCommitChecks),
Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1,
vNodeSettings.HashCollisionReadLimit,
vNodeSettings.SkipIndexScanOnReads,
db.Config.ReplicationCheckpoint);
_readIndex = readIndex;
var writer = new TFChunkWriter(db);
var epochManager = new EpochManager(_mainQueue,
ESConsts.CachedEpochCount,
db.Config.EpochCheckpoint,
writer,
initialReaderCount: 1,
maxReaderCount: 5,
readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint,
optimizeReadSideCache: db.Config.OptimizeReadSideCache));
epochManager.Init();
var storageWriter = new ClusterStorageWriterService(_mainQueue, _mainBus, vNodeSettings.MinFlushDelay,
db, writer, readIndex.IndexWriter, epochManager, _queueStatsManager,
() => readIndex.LastIndexedPosition); // subscribes internally
AddTasks(storageWriter.Tasks);
monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageWriter);
var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex,
vNodeSettings.ReaderThreadsCount, db.Config.WriterCheckpoint, _queueStatsManager);
_mainBus.Subscribe <SystemMessage.SystemInit>(storageReader);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageReader);
_mainBus.Subscribe <SystemMessage.BecomeShutdown>(storageReader);
monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageReader);
//REPLICATION TRACKING
var replicationTracker =
new ReplicationTrackingService(_mainQueue, vNodeSettings.ClusterNodeCount, db.Config.ReplicationCheckpoint, db.Config.WriterCheckpoint);
AddTask(replicationTracker.Task);
_mainBus.Subscribe <SystemMessage.SystemInit>(replicationTracker);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(replicationTracker);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(replicationTracker);
_mainBus.Subscribe <ReplicationTrackingMessage.ReplicaWriteAck>(replicationTracker);
_mainBus.Subscribe <ReplicationTrackingMessage.WriterCheckpointFlushed>(replicationTracker);
_mainBus.Subscribe <ReplicationTrackingMessage.MasterReplicatedTo>(replicationTracker);
var indexCommitterService = new IndexCommitterService(readIndex.IndexCommitter, _mainQueue,
db.Config.WriterCheckpoint, db.Config.ReplicationCheckpoint, vNodeSettings.CommitAckCount, tableIndex, _queueStatsManager);
AddTask(indexCommitterService.Task);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(indexCommitterService);
_mainBus.Subscribe <ReplicationTrackingMessage.ReplicatedTo>(indexCommitterService);
_mainBus.Subscribe <StorageMessage.CommitAck>(indexCommitterService);
_mainBus.Subscribe <ClientMessage.MergeIndexes>(indexCommitterService);
var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint,
db.Config.OptimizeReadSideCache);
var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, indexCommitterService,
epochManager, _queueStatsManager);
AddTask(storageChaser.Task);
#if DEBUG
_queueStatsManager.InitializeCheckpoints(db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
#endif
_mainBus.Subscribe <SystemMessage.SystemInit>(storageChaser);
_mainBus.Subscribe <SystemMessage.SystemStart>(storageChaser);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageChaser);
// AUTHENTICATION INFRASTRUCTURE - delegate to plugins
_internalAuthenticationProvider =
vNodeSettings.AuthenticationProviderFactory.BuildAuthenticationProvider(_mainQueue, _mainBus,
_workersHandler, _workerBuses, vNodeSettings.LogFailedAuthenticationAttempts);
Ensure.NotNull(_internalAuthenticationProvider, "authenticationProvider");
{
// EXTERNAL TCP
if (!vNodeSettings.DisableInsecureTCP)
{
var extTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalTcp, _workersHandler,
TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
vNodeSettings.ExtTcpHeartbeatInterval, vNodeSettings.ExtTcpHeartbeatTimeout,
_internalAuthenticationProvider, null, vNodeSettings.ConnectionPendingSendBytesThreshold,
vNodeSettings.ConnectionQueueSizeThreshold);
_mainBus.Subscribe <SystemMessage.SystemInit>(extTcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(extTcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extTcpService);
}
// EXTERNAL SECURE TCP
if (_nodeInfo.ExternalSecureTcp != null)
{
var extSecTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalSecureTcp, _workersHandler,
TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(),
vNodeSettings.ExtTcpHeartbeatInterval, vNodeSettings.ExtTcpHeartbeatTimeout,
_internalAuthenticationProvider, vNodeSettings.Certificate,
vNodeSettings.ConnectionPendingSendBytesThreshold, vNodeSettings.ConnectionQueueSizeThreshold);
_mainBus.Subscribe <SystemMessage.SystemInit>(extSecTcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(extSecTcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extSecTcpService);
}
if (!isSingleNode)
{
// INTERNAL TCP
if (!vNodeSettings.DisableInsecureTCP)
{
var intTcpService = new TcpService(_mainQueue, _nodeInfo.InternalTcp, _workersHandler,
TcpServiceType.Internal, TcpSecurityType.Normal,
new InternalTcpDispatcher(),
vNodeSettings.IntTcpHeartbeatInterval, vNodeSettings.IntTcpHeartbeatTimeout,
_internalAuthenticationProvider, null, ESConsts.UnrestrictedPendingSendBytes,
ESConsts.MaxConnectionQueueSize);
_mainBus.Subscribe <SystemMessage.SystemInit>(intTcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(intTcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intTcpService);
}
// INTERNAL SECURE TCP
if (_nodeInfo.InternalSecureTcp != null)
{
var intSecTcpService = new TcpService(_mainQueue, _nodeInfo.InternalSecureTcp, _workersHandler,
TcpServiceType.Internal, TcpSecurityType.Secure,
new InternalTcpDispatcher(),
vNodeSettings.IntTcpHeartbeatInterval, vNodeSettings.IntTcpHeartbeatTimeout,
_internalAuthenticationProvider, vNodeSettings.Certificate,
ESConsts.UnrestrictedPendingSendBytes,
ESConsts.MaxConnectionQueueSize);
_mainBus.Subscribe <SystemMessage.SystemInit>(intSecTcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(intSecTcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intSecTcpService);
}
}
}
SubscribeWorkers(bus => {
var tcpSendService = new TcpSendService();
// ReSharper disable RedundantTypeArgumentsOfMethod
bus.Subscribe <TcpMessage.TcpSend>(tcpSendService);
// ReSharper restore RedundantTypeArgumentsOfMethod
});
var httpAuthenticationProviders = new List <HttpAuthenticationProvider> {
new BasicHttpAuthenticationProvider(_internalAuthenticationProvider),
};
if (vNodeSettings.EnableTrustedAuth)
{
httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider());
}
httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider());
var httpPipe = new HttpMessagePipe();
var httpSendService = new HttpSendService(httpPipe, forwardRequests: true);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(httpSendService);
_mainBus.Subscribe(new WideningHandler <HttpMessage.SendOverHttp, Message>(_workersHandler));
SubscribeWorkers(bus => {
bus.Subscribe <HttpMessage.HttpSend>(httpSendService);
bus.Subscribe <HttpMessage.HttpSendPart>(httpSendService);
bus.Subscribe <HttpMessage.HttpBeginSend>(httpSendService);
bus.Subscribe <HttpMessage.HttpEndSend>(httpSendService);
bus.Subscribe <HttpMessage.SendOverHttp>(httpSendService);
});
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(infoController);
var adminController = new AdminController(_mainQueue, _workersHandler);
var pingController = new PingController();
var histogramController = new HistogramController();
var statController = new StatController(monitoringQueue, _workersHandler);
var atomController = new AtomController(httpSendService, _mainQueue, _workersHandler,
vNodeSettings.DisableHTTPCaching);
var gossipController = new GossipController(_mainQueue, _workersHandler, vNodeSettings.GossipTimeout,
_httpMessageHandler);
var persistentSubscriptionController =
new PersistentSubscriptionController(httpSendService, _mainQueue, _workersHandler);
var electController = new ElectController(_mainQueue, _httpMessageHandler);
// HTTP SENDERS
gossipController.SubscribeSenders(httpPipe);
electController.SubscribeSenders(httpPipe);
// EXTERNAL HTTP
_externalHttpService = new KestrelHttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(),
_workersHandler, vNodeSettings.LogHttpRequests,
vNodeSettings.GossipAdvertiseInfo.AdvertiseExternalIPAs,
vNodeSettings.GossipAdvertiseInfo.AdvertiseExternalHttpPortAs,
vNodeSettings.DisableFirstLevelHttpAuthorization,
vNodeSettings.NodeInfo.ExternalHttp);
_externalHttpService.SetupController(persistentSubscriptionController);
if (vNodeSettings.AdminOnPublic)
{
_externalHttpService.SetupController(adminController);
}
_externalHttpService.SetupController(pingController);
_externalHttpService.SetupController(infoController);
if (vNodeSettings.StatsOnPublic)
{
_externalHttpService.SetupController(statController);
}
_externalHttpService.SetupController(atomController);
if (vNodeSettings.GossipOnPublic)
{
_externalHttpService.SetupController(gossipController);
}
_externalHttpService.SetupController(histogramController);
_mainBus.Subscribe <SystemMessage.SystemInit>(_externalHttpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_externalHttpService);
_mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_externalHttpService);
// INTERNAL HTTP
if (!isSingleNode)
{
_internalHttpService = new KestrelHttpService(ServiceAccessibility.Private, _mainQueue,
new TrieUriRouter(),
_workersHandler, vNodeSettings.LogHttpRequests,
vNodeSettings.GossipAdvertiseInfo.AdvertiseInternalIPAs,
vNodeSettings.GossipAdvertiseInfo.AdvertiseInternalHttpPortAs,
vNodeSettings.DisableFirstLevelHttpAuthorization,
vNodeSettings.NodeInfo.InternalHttp);
_internalHttpService.SetupController(adminController);
_internalHttpService.SetupController(pingController);
_internalHttpService.SetupController(infoController);
_internalHttpService.SetupController(statController);
_internalHttpService.SetupController(atomController);
_internalHttpService.SetupController(gossipController);
_internalHttpService.SetupController(electController);
_internalHttpService.SetupController(histogramController);
_internalHttpService.SetupController(persistentSubscriptionController);
}
// Authentication plugin HTTP
vNodeSettings.AuthenticationProviderFactory.RegisterHttpControllers(_externalHttpService,
_internalHttpService, httpSendService, _mainQueue, _workersHandler);
if (_internalHttpService != null)
{
_mainBus.Subscribe <SystemMessage.SystemInit>(_internalHttpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_internalHttpService);
_mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_internalHttpService);
}
SubscribeWorkers(bus => {
KestrelHttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray());
});
// REQUEST FORWARDING
var forwardingService = new RequestForwardingService(_mainQueue, forwardingProxy, TimeSpan.FromSeconds(1));
_mainBus.Subscribe <SystemMessage.SystemStart>(forwardingService);
_mainBus.Subscribe <SystemMessage.RequestForwardingTimerTick>(forwardingService);
_mainBus.Subscribe <ClientMessage.NotHandled>(forwardingService);
_mainBus.Subscribe <ClientMessage.WriteEventsCompleted>(forwardingService);
_mainBus.Subscribe <ClientMessage.TransactionStartCompleted>(forwardingService);
_mainBus.Subscribe <ClientMessage.TransactionWriteCompleted>(forwardingService);
_mainBus.Subscribe <ClientMessage.TransactionCommitCompleted>(forwardingService);
_mainBus.Subscribe <ClientMessage.DeleteStreamCompleted>(forwardingService);
// REQUEST MANAGEMENT
var requestManagement = new RequestManagementService(
_mainQueue,
vNodeSettings.PrepareTimeout,
vNodeSettings.CommitTimeout,
vNodeSettings.BetterOrdering);
_mainBus.Subscribe <SystemMessage.SystemInit>(requestManagement);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(requestManagement);
_mainBus.Subscribe <ClientMessage.WriteEvents>(requestManagement);
_mainBus.Subscribe <ClientMessage.TransactionStart>(requestManagement);
_mainBus.Subscribe <ClientMessage.TransactionWrite>(requestManagement);
_mainBus.Subscribe <ClientMessage.TransactionCommit>(requestManagement);
_mainBus.Subscribe <ClientMessage.DeleteStream>(requestManagement);
_mainBus.Subscribe <StorageMessage.CheckStreamAccessCompleted>(requestManagement);
_mainBus.Subscribe <StorageMessage.AlreadyCommitted>(requestManagement);
_mainBus.Subscribe <StorageMessage.CommitAck>(requestManagement);
_mainBus.Subscribe <StorageMessage.PrepareAck>(requestManagement);
_mainBus.Subscribe <ReplicationTrackingMessage.ReplicatedTo>(requestManagement);
_mainBus.Subscribe <ReplicationTrackingMessage.IndexedTo>(requestManagement);
_mainBus.Subscribe <StorageMessage.RequestCompleted>(requestManagement);
_mainBus.Subscribe <StorageMessage.CommitIndexed>(requestManagement);
_mainBus.Subscribe <StorageMessage.WrongExpectedVersion>(requestManagement);
_mainBus.Subscribe <StorageMessage.InvalidTransaction>(requestManagement);
_mainBus.Subscribe <StorageMessage.StreamDeleted>(requestManagement);
_mainBus.Subscribe <StorageMessage.RequestManagerTimerTick>(requestManagement);
// SUBSCRIPTIONS
var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", _queueStatsManager, false);
_mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.SystemStart, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.SubscribeToStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.FilteredSubscribeToStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.PollStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.CheckPollTimeout, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <StorageMessage.EventCommitted, Message>());
var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex);
subscrBus.Subscribe <SystemMessage.SystemStart>(subscription);
subscrBus.Subscribe <SystemMessage.BecomeShuttingDown>(subscription);
subscrBus.Subscribe <TcpMessage.ConnectionClosed>(subscription);
subscrBus.Subscribe <ClientMessage.SubscribeToStream>(subscription);
subscrBus.Subscribe <ClientMessage.FilteredSubscribeToStream>(subscription);
subscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(subscription);
subscrBus.Subscribe <SubscriptionMessage.PollStream>(subscription);
subscrBus.Subscribe <SubscriptionMessage.CheckPollTimeout>(subscription);
subscrBus.Subscribe <StorageMessage.EventCommitted>(subscription);
// PERSISTENT SUBSCRIPTIONS
// IO DISPATCHER
var ioDispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_mainQueue));
_mainBus.Subscribe <ClientMessage.ReadStreamEventsBackwardCompleted>(ioDispatcher.BackwardReader);
_mainBus.Subscribe <ClientMessage.WriteEventsCompleted>(ioDispatcher.Writer);
_mainBus.Subscribe <ClientMessage.ReadStreamEventsForwardCompleted>(ioDispatcher.ForwardReader);
_mainBus.Subscribe <ClientMessage.DeleteStreamCompleted>(ioDispatcher.StreamDeleter);
_mainBus.Subscribe(ioDispatcher);
var perSubscrBus = new InMemoryBus("PersistentSubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
var perSubscrQueue = new QueuedHandlerThreadPool(perSubscrBus, "PersistentSubscriptions", _queueStatsManager, false);
_mainBus.Subscribe(perSubscrQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.CreatePersistentSubscription, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.UpdatePersistentSubscription, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.DeletePersistentSubscription, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ConnectToPersistentSubscription, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.PersistentSubscriptionAckEvents, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.PersistentSubscriptionNackEvents, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReplayAllParkedMessages, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReplayParkedMessage, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReadNextNPersistentMessages, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <StorageMessage.EventCommitted, Message>());
_mainBus.Subscribe(perSubscrQueue
.WidenFrom <MonitoringMessage.GetAllPersistentSubscriptionStats, Message>());
_mainBus.Subscribe(
perSubscrQueue.WidenFrom <MonitoringMessage.GetStreamPersistentSubscriptionStats, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <MonitoringMessage.GetPersistentSubscriptionStats, Message>());
_mainBus.Subscribe(perSubscrQueue
.WidenFrom <SubscriptionMessage.PersistentSubscriptionTimerTick, Message>());
//TODO CC can have multiple threads working on subscription if partition
var consumerStrategyRegistry =
new PersistentSubscriptionConsumerStrategyRegistry(_mainQueue, _mainBus,
vNodeSettings.AdditionalConsumerStrategies);
var persistentSubscription = new PersistentSubscriptionService(perSubscrQueue, readIndex, ioDispatcher,
_mainQueue, consumerStrategyRegistry);
perSubscrBus.Subscribe <SystemMessage.BecomeShuttingDown>(persistentSubscription);
perSubscrBus.Subscribe <SystemMessage.BecomeMaster>(persistentSubscription);
perSubscrBus.Subscribe <SystemMessage.StateChangeMessage>(persistentSubscription);
perSubscrBus.Subscribe <TcpMessage.ConnectionClosed>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.ConnectToPersistentSubscription>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.PersistentSubscriptionAckEvents>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.PersistentSubscriptionNackEvents>(persistentSubscription);
perSubscrBus.Subscribe <StorageMessage.EventCommitted>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.DeletePersistentSubscription>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.CreatePersistentSubscription>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.UpdatePersistentSubscription>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.ReplayAllParkedMessages>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.ReplayParkedMessage>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.ReadNextNPersistentMessages>(persistentSubscription);
perSubscrBus.Subscribe <MonitoringMessage.GetAllPersistentSubscriptionStats>(persistentSubscription);
perSubscrBus.Subscribe <MonitoringMessage.GetStreamPersistentSubscriptionStats>(persistentSubscription);
perSubscrBus.Subscribe <MonitoringMessage.GetPersistentSubscriptionStats>(persistentSubscription);
perSubscrBus.Subscribe <SubscriptionMessage.PersistentSubscriptionTimerTick>(persistentSubscription);
// STORAGE SCAVENGER
var scavengerLogManager = new TFChunkScavengerLogManager(_nodeInfo.ExternalHttp.ToString(),
TimeSpan.FromDays(vNodeSettings.ScavengeHistoryMaxAge), ioDispatcher);
var storageScavenger = new StorageScavenger(db,
tableIndex,
readIndex,
scavengerLogManager,
vNodeSettings.AlwaysKeepScavenged,
!vNodeSettings.DisableScavengeMerging,
unsafeIgnoreHardDeletes: vNodeSettings.UnsafeIgnoreHardDeletes);
// ReSharper disable RedundantTypeArgumentsOfMethod
_mainBus.Subscribe <ClientMessage.ScavengeDatabase>(storageScavenger);
_mainBus.Subscribe <ClientMessage.StopDatabaseScavenge>(storageScavenger);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(storageScavenger);
// ReSharper restore RedundantTypeArgumentsOfMethod
// TIMER
_timeProvider = new RealTimeProvider();
var threadBasedScheduler = new ThreadBasedScheduler(_timeProvider, _queueStatsManager);
AddTask(threadBasedScheduler.Task);
_timerService = new TimerService(threadBasedScheduler);
_mainBus.Subscribe <SystemMessage.BecomeShutdown>(_timerService);
_mainBus.Subscribe <TimerMessage.Schedule>(_timerService);
var gossipInfo = new VNodeInfo(_nodeInfo.InstanceId, _nodeInfo.DebugIndex,
vNodeSettings.GossipAdvertiseInfo.InternalTcp,
vNodeSettings.GossipAdvertiseInfo.InternalSecureTcp,
vNodeSettings.GossipAdvertiseInfo.ExternalTcp,
vNodeSettings.GossipAdvertiseInfo.ExternalSecureTcp,
vNodeSettings.GossipAdvertiseInfo.InternalHttp,
vNodeSettings.GossipAdvertiseInfo.ExternalHttp,
vNodeSettings.ReadOnlyReplica);
if (!isSingleNode)
{
// MASTER REPLICATION
var masterReplicationService = new MasterReplicationService(_mainQueue, gossipInfo.InstanceId, db,
_workersHandler,
epochManager, vNodeSettings.ClusterNodeCount,
vNodeSettings.UnsafeAllowSurplusNodes,
_queueStatsManager);
AddTask(masterReplicationService.Task);
_mainBus.Subscribe <SystemMessage.SystemStart>(masterReplicationService);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(masterReplicationService);
_mainBus.Subscribe <ReplicationMessage.ReplicaSubscriptionRequest>(masterReplicationService);
_mainBus.Subscribe <ReplicationMessage.ReplicaLogPositionAck>(masterReplicationService);
_mainBus.Subscribe <ReplicationTrackingMessage.ReplicatedTo>(masterReplicationService);
monitoringInnerBus.Subscribe <ReplicationMessage.GetReplicationStats>(masterReplicationService);
// REPLICA REPLICATION
var replicaService = new ReplicaService(_mainQueue, db, epochManager, _workersHandler,
_internalAuthenticationProvider,
gossipInfo, vNodeSettings.UseSsl, vNodeSettings.SslTargetHost, vNodeSettings.SslValidateServer,
vNodeSettings.IntTcpHeartbeatTimeout, vNodeSettings.ExtTcpHeartbeatInterval);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(replicaService);
_mainBus.Subscribe <ReplicationMessage.ReconnectToMaster>(replicaService);
_mainBus.Subscribe <ReplicationMessage.SubscribeToMaster>(replicaService);
_mainBus.Subscribe <ReplicationMessage.AckLogPosition>(replicaService);
_mainBus.Subscribe <ClientMessage.TcpForwardMessage>(replicaService);
}
// ELECTIONS
var electionsService = new ElectionsService(_mainQueue, gossipInfo, vNodeSettings.ClusterNodeCount,
db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint,
epochManager, () => readIndex.LastIndexedPosition, vNodeSettings.NodePriority, _timeProvider);
electionsService.SubscribeMessages(_mainBus);
if (!isSingleNode || vNodeSettings.GossipOnSingleNode)
{
// GOSSIP
var gossip = new NodeGossipService(_mainQueue, gossipSeedSource, gossipInfo, db.Config.WriterCheckpoint,
db.Config.ChaserCheckpoint, epochManager, () => readIndex.LastIndexedPosition,
vNodeSettings.NodePriority, vNodeSettings.GossipInterval,
vNodeSettings.GossipAllowedTimeDifference,
_timeProvider);
_mainBus.Subscribe <SystemMessage.SystemInit>(gossip);
_mainBus.Subscribe <GossipMessage.RetrieveGossipSeedSources>(gossip);
_mainBus.Subscribe <GossipMessage.GotGossipSeedSources>(gossip);
_mainBus.Subscribe <GossipMessage.Gossip>(gossip);
_mainBus.Subscribe <GossipMessage.GossipReceived>(gossip);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(gossip);
_mainBus.Subscribe <GossipMessage.GossipSendFailed>(gossip);
_mainBus.Subscribe <GossipMessage.UpdateNodePriority>(gossip);
_mainBus.Subscribe <SystemMessage.VNodeConnectionEstablished>(gossip);
_mainBus.Subscribe <SystemMessage.VNodeConnectionLost>(gossip);
_mainBus.Subscribe <GossipMessage.GetGossipFailed>(gossip);
_mainBus.Subscribe <GossipMessage.GetGossipReceived>(gossip);
_mainBus.Subscribe <ElectionMessage.ElectionsDone>(gossip);
}
_startup = new ClusterVNodeStartup(_subsystems, _mainQueue, _internalAuthenticationProvider, _readIndex,
_vNodeSettings, _externalHttpService, _internalHttpService);
_mainBus.Subscribe <SystemMessage.SystemReady>(_startup);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_startup);
// kestrel
AddTasks(_workersHandler.Start());
AddTask(_mainQueue.Start());
AddTask(monitoringQueue.Start());
AddTask(subscrQueue.Start());
AddTask(perSubscrQueue.Start());
if (subsystems != null)
{
foreach (var subsystem in subsystems)
{
var http = isSingleNode
? new[] { _externalHttpService }
: new[] { _internalHttpService, _externalHttpService };
subsystem.Register(new StandardComponents(db, _mainQueue, _mainBus, _timerService, _timeProvider,
httpSendService, http, _workersHandler, _queueStatsManager));
}
}
}