ReadAllEventsForward(ClientMessage.ReadAllEventsForward msg) { using (HistogramService.Measure(_readerAllRangeHistogram)) { var pos = new TFPos(msg.CommitPosition, msg.PreparePosition); var lastCommitPosition = _readIndex.LastReplicatedPosition; try { if (msg.MaxCount > MaxPageSize) { throw new ArgumentException(string.Format("Read size too big, should be less than {0} items", MaxPageSize)); } if (pos == TFPos.HeadOfTf) { var checkpoint = _writerCheckpoint.Read(); pos = new TFPos(checkpoint, checkpoint); } if (pos.CommitPosition < 0 || pos.PreparePosition < 0) { return(NoData(msg, ReadAllResult.Error, pos, lastCommitPosition, "Invalid position.")); } if (msg.ValidationTfLastCommitPosition == lastCommitPosition) { return(NoData(msg, ReadAllResult.NotModified, pos, lastCommitPosition)); } var access = _readIndex.CheckStreamAccess(SystemStreams.AllStream, StreamAccessType.Read, msg.User); if (!access.Granted) { return(NoData(msg, ReadAllResult.AccessDenied, pos, lastCommitPosition)); } var res = _readIndex.ReadAllEventsForward(pos, msg.MaxCount); var resolved = ResolveReadAllResult(res.Records, msg.ResolveLinkTos, msg.User); if (resolved == null) { return(NoData(msg, ReadAllResult.AccessDenied, pos, lastCommitPosition)); } var metadata = _readIndex.GetStreamMetadata(SystemStreams.AllStream); return(new ClientMessage.ReadAllEventsForwardCompleted( msg.CorrelationId, ReadAllResult.Success, null, resolved, metadata, access.Public, msg.MaxCount, res.CurrentPos, res.NextPos, res.PrevPos, lastCommitPosition)); } catch (Exception exc) { Log.ErrorException(exc, "Error during processing ReadAllEventsForward request."); return(NoData(msg, ReadAllResult.Error, pos, lastCommitPosition, exc.Message)); } } }
private ClientMessage.FilteredReadAllEventsBackwardCompleted FilteredReadAllEventsBackward( ClientMessage.FilteredReadAllEventsBackward msg) { using (HistogramService.Measure(ReaderAllRangeHistogram)) { var pos = new TFPos(msg.CommitPosition, msg.PreparePosition); var lastIndexedPosition = _readIndex.LastIndexedPosition; try { if (msg.MaxCount > MaxPageSize) { throw new ArgumentException($"Read size too big, should be less than {MaxPageSize} items"); } if (pos == TFPos.HeadOfTf) { var checkpoint = _writerCheckpoint.Read(); pos = new TFPos(checkpoint, checkpoint); } if (pos.CommitPosition < 0 || pos.PreparePosition < 0) { return(NoDataForFilteredCommand(msg, FilteredReadAllResult.Error, pos, lastIndexedPosition, "Invalid position.")); } if (msg.ValidationTfLastCommitPosition == lastIndexedPosition) { return(NoDataForFilteredCommand(msg, FilteredReadAllResult.NotModified, pos, lastIndexedPosition)); } var res = _readIndex.ReadAllEventsBackwardFiltered(pos, msg.MaxCount, msg.MaxSearchWindow, msg.EventFilter); var resolved = ResolveReadAllResult(res.Records, msg.ResolveLinkTos, msg.User); if (resolved == null) { return(NoDataForFilteredCommand(msg, FilteredReadAllResult.AccessDenied, pos, lastIndexedPosition)); } var metadata = _readIndex.GetStreamMetadata(SystemStreams.AllStream); return(new ClientMessage.FilteredReadAllEventsBackwardCompleted( msg.CorrelationId, FilteredReadAllResult.Success, null, resolved, metadata, false, msg.MaxCount, res.CurrentPos, res.NextPos, res.PrevPos, lastIndexedPosition, res.IsEndOfStream)); } catch (Exception exc) { Log.Error(exc, "Error during processing ReadAllEventsForwardFiltered request."); return(NoDataForFilteredCommand(msg, FilteredReadAllResult.Error, pos, lastIndexedPosition, exc.Message)); } } }
public void Handle(StorageMessage.RequestCompleted message) { Stopwatch watch = null; if (_currentTimedRequests.TryGetValue(message.CorrelationId, out watch)) { HistogramService.SetValue(_requestManagerHistogram, (long)((((double)watch.ElapsedTicks) / Stopwatch.Frequency) * 1000000000)); _currentTimedRequests.Remove(message.CorrelationId); } if (!_currentRequests.Remove(message.CorrelationId)) { throw new InvalidOperationException("Should never complete request twice."); } }
void IHandle <ClientMessage.ReadAllEventsForward> .Handle(ClientMessage.ReadAllEventsForward msg) { if (msg.Expires < DateTime.UtcNow) { Log.Debug("Read All Stream Events Forward operation has expired for C:{0}/P:{1}. Operation Expired at {2}", msg.CommitPosition, msg.PreparePosition, msg.Expires); return; } using (HistogramService.Measure(_readerAllRangeHistogram)) { var res = ReadAllEventsForward(msg); switch (res.Result) { case ReadAllResult.Success: if (msg.LongPollTimeout.HasValue && res.IsEndOfStream && res.Events.Length == 0) { _publisher.Publish(new SubscriptionMessage.PollStream( SubscriptionsService.AllStreamsSubscriptionId, res.TfLastCommitPosition, null, DateTime.UtcNow + msg.LongPollTimeout.Value, msg)); } else { msg.Envelope.ReplyWith(res); } break; case ReadAllResult.NotModified: if (msg.LongPollTimeout.HasValue && res.IsEndOfStream && res.CurrentPos.CommitPosition > res.TfLastCommitPosition) { _publisher.Publish(new SubscriptionMessage.PollStream( SubscriptionsService.AllStreamsSubscriptionId, res.TfLastCommitPosition, null, DateTime.UtcNow + msg.LongPollTimeout.Value, msg)); } else { msg.Envelope.ReplyWith(res); } break; case ReadAllResult.Error: case ReadAllResult.AccessDenied: msg.Envelope.ReplyWith(res); break; default: throw new ArgumentOutOfRangeException(string.Format("Unknown ReadAllResult: {0}", res.Result)); } } }
private ClientMessage.ReadStreamEventsBackwardCompleted ReadStreamEventsBackward(ClientMessage.ReadStreamEventsBackward msg) { using (HistogramService.Measure(_readerStreamRangeHistogram)) { var lastCommitPosition = _readIndex.LastReplicatedPosition; try { if (msg.MaxCount > MaxPageSize) { throw new ArgumentException(string.Format("Read size too big, should be less than {0} items", MaxPageSize)); } if (msg.ValidationStreamVersion.HasValue && _readIndex.GetStreamLastEventNumber(msg.EventStreamId) == msg.ValidationStreamVersion) { return(NoData(msg, ReadStreamResult.NotModified, lastCommitPosition, msg.ValidationStreamVersion.Value)); } var access = _readIndex.CheckStreamAccess(msg.EventStreamId, StreamAccessType.Read, msg.User); if (!access.Granted) { return(NoData(msg, ReadStreamResult.AccessDenied, lastCommitPosition)); } var result = _readIndex.ReadStreamEventsBackward(msg.EventStreamId, msg.FromEventNumber, msg.MaxCount); CheckEventsOrder(msg, result); var resolvedPairs = ResolveLinkToEvents(result.Records, msg.ResolveLinkTos, msg.User); if (resolvedPairs == null) { return(NoData(msg, ReadStreamResult.AccessDenied, lastCommitPosition)); } return(new ClientMessage.ReadStreamEventsBackwardCompleted( msg.CorrelationId, msg.EventStreamId, result.FromEventNumber, result.MaxCount, (ReadStreamResult)result.Result, resolvedPairs, result.Metadata, access.Public, string.Empty, result.NextEventNumber, result.LastEventNumber, result.IsEndOfStream, lastCommitPosition)); } catch (Exception exc) { Log.ErrorException(exc, "Error during processing ReadStreamEventsBackward request."); return(NoData(msg, ReadStreamResult.Error, lastCommitPosition, error: exc.Message)); } } }
void IHandle <ClientMessage.ReadStreamEventsForward> .Handle(ClientMessage.ReadStreamEventsForward msg) { if (msg.Expires < DateTime.UtcNow) { if (LogExpiredMessage(msg.Expires)) { Log.Debug( "Read Stream Events Forward operation has expired for Stream: {stream}, From Event Number: {fromEventNumber}, Max Count: {maxCount}. Operation Expired at {expiryDateTime}", msg.EventStreamId, msg.FromEventNumber, msg.MaxCount, msg.Expires); } return; } using (HistogramService.Measure(_readerStreamRangeHistogram)) { var res = ReadStreamEventsForward(msg); switch (res.Result) { case ReadStreamResult.Success: case ReadStreamResult.NoStream: case ReadStreamResult.NotModified: if (msg.LongPollTimeout.HasValue && res.FromEventNumber > res.LastEventNumber) { _publisher.Publish(new SubscriptionMessage.PollStream( msg.EventStreamId, res.TfLastCommitPosition, res.LastEventNumber, DateTime.UtcNow + msg.LongPollTimeout.Value, msg)); } else { msg.Envelope.ReplyWith(res); } break; case ReadStreamResult.StreamDeleted: case ReadStreamResult.Error: case ReadStreamResult.AccessDenied: msg.Envelope.ReplyWith(res); break; default: throw new ArgumentOutOfRangeException( string.Format("Unknown ReadStreamResult: {0}", res.Result)); } } }
private void ChaserIteration() { _queueStats.EnterBusy(); _flushSignal.Reset(); // Reset the flush signal just before a read to reduce pointless reads from [flush flush read] patterns. var result = _chaser.TryReadNext(); if (result.Success) { _queueStats.ProcessingStarted(result.LogRecord.GetType(), 0); ProcessLogRecord(result); _queueStats.ProcessingEnded(1); } var start = _watch.ElapsedTicks; if (!result.Success || start - _lastFlush >= _flushDelay + MinFlushDelay) { _queueStats.ProcessingStarted <ChaserCheckpointFlush>(0); var startflush = _watch.ElapsedTicks; _chaser.Flush(); HistogramService.SetValue(_chaserFlushHistogram, (long)((((double)_watch.ElapsedTicks - startflush) / Stopwatch.Frequency) * 1000000000)); _queueStats.ProcessingEnded(1); var end = _watch.ElapsedTicks; _flushDelay = end - start; _lastFlush = end; } if (!result.Success) { _queueStats.EnterIdle(); var startwait = _watch.ElapsedTicks; #if MONO _flushSignal.WaitOne(FlushWaitTimeout); #else _flushSignal.Wait(FlushWaitTimeout); #endif HistogramService.SetValue(_chaserWaitHistogram, (long)((((double)_watch.ElapsedTicks - startwait) / Stopwatch.Frequency) * 1000000000)); } }
protected bool Flush(bool force = false) { var start = _watch.ElapsedTicks; if (force || FlushMessagesInQueue == 0 || start - _lastFlushTimestamp >= _lastFlushDelay + _minFlushDelay) { var flushSize = Writer.Checkpoint.ReadNonFlushed() - Writer.Checkpoint.Read(); Writer.Flush(); HistogramService.SetValue(_writerFlushHistogram, (long)((((double)_watch.ElapsedTicks - start) / Stopwatch.Frequency) * 1000000000)); var end = _watch.ElapsedTicks; var flushDelay = end - start; Interlocked.Exchange(ref _lastFlushDelay, flushDelay); Interlocked.Exchange(ref _lastFlushSize, flushSize); _lastFlushTimestamp = end; if (_statCount >= LastStatsCount) { Interlocked.Add(ref _sumFlushSize, -_lastFlushSizes[_statIndex]); Interlocked.Add(ref _sumFlushDelay, -_lastFlushDelays[_statIndex]); } else { _statCount += 1; } _lastFlushSizes[_statIndex] = flushSize; _lastFlushDelays[_statIndex] = flushDelay; Interlocked.Add(ref _sumFlushSize, flushSize); Interlocked.Add(ref _sumFlushDelay, flushDelay); Interlocked.Exchange(ref _maxFlushSize, Math.Max(Interlocked.Read(ref _maxFlushSize), flushSize)); Interlocked.Exchange(ref _maxFlushDelay, Math.Max(Interlocked.Read(ref _maxFlushDelay), flushDelay)); _statIndex = (_statIndex + 1) & (LastStatsCount - 1); PurgeNotProcessedInfo(); Bus.Publish(new ReplicationTrackingMessage.WriterCheckpointFlushed()); return(true); } return(false); }
static void Main(string[] args) { var path = "../../../TextFile1.txt"; var service = new HistogramService(); try { var results = service.ProcessFile(path); foreach (var model in results) { Console.WriteLine($"\"{model.Key}\" \t : {model.Value}"); } } catch (Exception e) { Console.WriteLine($"Error: {e.Message}"); } Console.ReadLine(); }
public void Handle(HttpMessage.HttpSend message) { var deniedToHandle = message.Message as HttpMessage.DeniedToHandle; if (deniedToHandle != null) { int code; switch (deniedToHandle.Reason) { case DenialReason.ServerTooBusy: code = HttpStatusCode.ServiceUnavailable; break; default: throw new ArgumentOutOfRangeException(); } var start = _watch.ElapsedTicks; message.HttpEntityManager.ReplyStatus( code, deniedToHandle.Details, exc => Log.Debug("Error occurred while replying to HTTP with message {0}: {1}.", message.Message, exc.Message)); HistogramService.SetValue(_httpSendHistogram, (long)((((double)_watch.ElapsedTicks - start) / Stopwatch.Frequency) * 1000000000)); } else { var response = message.Data; var config = message.Configuration; var start = _watch.ElapsedTicks; message.HttpEntityManager.ReplyTextContent( response, config.Code, config.Description, config.ContentType, config.Headers, exc => Log.Debug("Error occurred while replying to HTTP with message {0}: {1}.", message.Message, exc.Message)); HistogramService.SetValue(_httpSendHistogram, (long)((((double)_watch.ElapsedTicks - start) / Stopwatch.Frequency) * 1000000000)); } }
private void ChaserIteration() { _queueStats.EnterBusy(); var result = _chaser.TryReadNext(); if (result.Success) { _queueStats.ProcessingStarted(result.LogRecord.GetType(), 0); ProcessLogRecord(result); _queueStats.ProcessingEnded(1); } var start = _watch.ElapsedTicks; if (!result.Success || start - _lastFlush >= _flushDelay + MinFlushDelay) { _queueStats.ProcessingStarted <ChaserCheckpointFlush>(0); _chaser.Flush(); var startflush = _watch.ElapsedTicks; HistogramService.SetValue(_chaserFlushHistogram, (long)((((double)_watch.ElapsedTicks - startflush) / Stopwatch.Frequency) * 1000000000)); _queueStats.ProcessingEnded(1); var end = _watch.ElapsedTicks; _flushDelay = end - start; _lastFlush = end; } if (!result.Success) { _queueStats.EnterIdle(); //Thread.Sleep(1); var startwait = _watch.ElapsedTicks; _writerCheckpoint.WaitForFlush(FlushWaitTimeout); HistogramService.SetValue(_chaserWaitHistogram, (long)((((double)_watch.ElapsedTicks - startwait) / Stopwatch.Frequency) * 1000000000)); } }
private ClientMessage.ReadEventCompleted ReadEvent(ClientMessage.ReadEvent msg) { using (HistogramService.Measure(_readerReadHistogram)) { try { var access = _readIndex.CheckStreamAccess(msg.EventStreamId, StreamAccessType.Read, msg.User); if (!access.Granted) { return(NoData(msg, ReadEventResult.AccessDenied)); } var result = _readIndex.ReadEvent(msg.EventStreamId, msg.EventNumber); var record = result.Result == ReadEventResult.Success && msg.ResolveLinkTos ? ResolveLinkToEvent(result.Record, msg.User, null) : ResolvedEvent.ForUnresolvedEvent(result.Record); if (record == null) { return(NoData(msg, ReadEventResult.AccessDenied)); } if ((result.Result == ReadEventResult.NoStream || result.Result == ReadEventResult.NotFound) && result.OriginalStreamExists && SystemStreams.IsSystemStream(msg.EventStreamId)) { return(NoData(msg, ReadEventResult.Success)); } return(new ClientMessage.ReadEventCompleted(msg.CorrelationId, msg.EventStreamId, result.Result, record.Value, result.Metadata, access.Public, null)); } catch (Exception exc) { Log.ErrorException(exc, "Error during processing ReadEvent request."); return(NoData(msg, ReadEventResult.Error, exc.Message)); } } }
public ClusterVNode(TFChunkDb db, ClusterVNodeSettings vNodeSettings, IGossipSeedSource gossipSeedSource, InfoController infoController, params ISubsystem[] subsystems) { Ensure.NotNull(db, "db"); Ensure.NotNull(vNodeSettings, "vNodeSettings"); Ensure.NotNull(gossipSeedSource, "gossipSeedSource"); #if DEBUG AddTask(_taskAddedTrigger.Task); #endif var isSingleNode = vNodeSettings.ClusterNodeCount == 1; _nodeInfo = vNodeSettings.NodeInfo; _mainBus = new InMemoryBus("MainBus"); var forwardingProxy = new MessageForwardingProxy(); if (vNodeSettings.EnableHistograms) { HistogramService.CreateHistograms(); //start watching jitter HistogramService.StartJitterMonitor(); } // MISC WORKERS _workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum => new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1), watchSlowMsg: true, slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray(); _workersHandler = new MultiQueuedHandler( vNodeSettings.WorkerThreads, queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum], string.Format("Worker #{0}", queueNum + 1), groupName: "Workers", watchSlowMsg: true, slowMsgThreshold: TimeSpan.FromMilliseconds(200))); _subsystems = subsystems; _controller = new ClusterVNodeController((IPublisher)_mainBus, _nodeInfo, db, vNodeSettings, this, forwardingProxy, _subsystems); _mainQueue = QueuedHandler.CreateQueuedHandler(_controller, "MainQueue"); _controller.SetMainQueue(_mainQueue); //SELF _mainBus.Subscribe <SystemMessage.StateChangeMessage>(this); _mainBus.Subscribe <SystemMessage.BecomeShutdown>(this); // MONITORING var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false); var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false); var monitoringQueue = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100)); var monitoring = new MonitoringService(monitoringQueue, monitoringRequestBus, _mainQueue, db.Config.WriterCheckpoint, db.Config.Path, vNodeSettings.StatsPeriod, _nodeInfo.ExternalHttp, vNodeSettings.StatsStorage, _nodeInfo.ExternalTcp, _nodeInfo.ExternalSecureTcp); _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.SystemInit, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShutdown, Message>()); _mainBus.Subscribe(monitoringQueue.WidenFrom <ClientMessage.WriteEventsCompleted, Message>()); monitoringInnerBus.Subscribe <SystemMessage.SystemInit>(monitoring); monitoringInnerBus.Subscribe <SystemMessage.StateChangeMessage>(monitoring); monitoringInnerBus.Subscribe <SystemMessage.BecomeShuttingDown>(monitoring); monitoringInnerBus.Subscribe <SystemMessage.BecomeShutdown>(monitoring); monitoringInnerBus.Subscribe <ClientMessage.WriteEventsCompleted>(monitoring); monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshStats>(monitoring); monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshTcpConnectionStats>(monitoring); var truncPos = db.Config.TruncateCheckpoint.Read(); if (truncPos != -1) { Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})", truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read()); var truncator = new TFChunkDbTruncator(db.Config); truncator.TruncateDb(truncPos); } // STORAGE SUBSYSTEM db.Open(vNodeSettings.VerifyDbHash); var indexPath = vNodeSettings.Index ?? Path.Combine(db.Config.Path, "index"); var readerPool = new ObjectPool <ITransactionFileReader>( "ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount, () => new TFChunkReader(db, db.Config.WriterCheckpoint, optimizeReadSideCache: db.Config.OptimizeReadSideCache)); var tableIndex = new TableIndex(indexPath, new XXHashUnsafe(), new Murmur3AUnsafe(), () => new HashListMemTable(vNodeSettings.IndexBitnessVersion, maxSize: vNodeSettings.MaxMemtableEntryCount * 2), () => new TFReaderLease(readerPool), vNodeSettings.IndexBitnessVersion, maxSizeForMemory: vNodeSettings.MaxMemtableEntryCount, maxTablesPerLevel: 2, inMem: db.Config.InMemDb, skipIndexVerify: vNodeSettings.SkipIndexVerify, indexCacheDepth: vNodeSettings.IndexCacheDepth); var readIndex = new ReadIndex(_mainQueue, readerPool, tableIndex, ESConsts.StreamInfoCacheCapacity, Application.IsDefined(Application.AdditionalCommitChecks), Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1, vNodeSettings.HashCollisionReadLimit, vNodeSettings.SkipIndexScanOnReads, db.Config.ReplicationCheckpoint); var writer = new TFChunkWriter(db); var epochManager = new EpochManager(_mainQueue, ESConsts.CachedEpochCount, db.Config.EpochCheckpoint, writer, initialReaderCount: 1, maxReaderCount: 5, readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint, optimizeReadSideCache: db.Config.OptimizeReadSideCache)); epochManager.Init(); var storageWriter = new ClusterStorageWriterService(_mainQueue, _mainBus, vNodeSettings.MinFlushDelay, db, writer, readIndex.IndexWriter, epochManager, () => readIndex.LastCommitPosition); // subscribes internally AddTasks(storageWriter.Tasks); monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageWriter); var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, vNodeSettings.ReaderThreadsCount, db.Config.WriterCheckpoint); _mainBus.Subscribe <SystemMessage.SystemInit>(storageReader); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageReader); _mainBus.Subscribe <SystemMessage.BecomeShutdown>(storageReader); monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageReader); var indexCommitterService = new IndexCommitterService(readIndex.IndexCommitter, _mainQueue, db.Config.ReplicationCheckpoint, db.Config.WriterCheckpoint, vNodeSettings.CommitAckCount); AddTask(indexCommitterService.Task); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(indexCommitterService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(indexCommitterService); _mainBus.Subscribe <StorageMessage.CommitAck>(indexCommitterService); var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint, db.Config.OptimizeReadSideCache); var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, indexCommitterService, epochManager); AddTask(storageChaser.Task); #if DEBUG QueueStatsCollector.InitializeCheckpoints( _nodeInfo.DebugIndex, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint); #endif _mainBus.Subscribe <SystemMessage.SystemInit>(storageChaser); _mainBus.Subscribe <SystemMessage.SystemStart>(storageChaser); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageChaser); // AUTHENTICATION INFRASTRUCTURE - delegate to plugins _internalAuthenticationProvider = vNodeSettings.AuthenticationProviderFactory.BuildAuthenticationProvider(_mainQueue, _mainBus, _workersHandler, _workerBuses); Ensure.NotNull(_internalAuthenticationProvider, "authenticationProvider"); { // EXTERNAL TCP if (!vNodeSettings.DisableInsecureTCP) { var extTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalTcp, _workersHandler, TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(), vNodeSettings.ExtTcpHeartbeatInterval, vNodeSettings.ExtTcpHeartbeatTimeout, _internalAuthenticationProvider, null, vNodeSettings.ConnectionPendingSendBytesThreshold); _mainBus.Subscribe <SystemMessage.SystemInit>(extTcpService); _mainBus.Subscribe <SystemMessage.SystemStart>(extTcpService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extTcpService); } // EXTERNAL SECURE TCP if (_nodeInfo.ExternalSecureTcp != null) { var extSecTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalSecureTcp, _workersHandler, TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(), vNodeSettings.ExtTcpHeartbeatInterval, vNodeSettings.ExtTcpHeartbeatTimeout, _internalAuthenticationProvider, vNodeSettings.Certificate, vNodeSettings.ConnectionPendingSendBytesThreshold); _mainBus.Subscribe <SystemMessage.SystemInit>(extSecTcpService); _mainBus.Subscribe <SystemMessage.SystemStart>(extSecTcpService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extSecTcpService); } if (!isSingleNode) { // INTERNAL TCP if (!vNodeSettings.DisableInsecureTCP) { var intTcpService = new TcpService(_mainQueue, _nodeInfo.InternalTcp, _workersHandler, TcpServiceType.Internal, TcpSecurityType.Normal, new InternalTcpDispatcher(), vNodeSettings.IntTcpHeartbeatInterval, vNodeSettings.IntTcpHeartbeatTimeout, _internalAuthenticationProvider, null, ESConsts.UnrestrictedPendingSendBytes); _mainBus.Subscribe <SystemMessage.SystemInit>(intTcpService); _mainBus.Subscribe <SystemMessage.SystemStart>(intTcpService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intTcpService); } // INTERNAL SECURE TCP if (_nodeInfo.InternalSecureTcp != null) { var intSecTcpService = new TcpService(_mainQueue, _nodeInfo.InternalSecureTcp, _workersHandler, TcpServiceType.Internal, TcpSecurityType.Secure, new InternalTcpDispatcher(), vNodeSettings.IntTcpHeartbeatInterval, vNodeSettings.IntTcpHeartbeatTimeout, _internalAuthenticationProvider, vNodeSettings.Certificate, ESConsts.UnrestrictedPendingSendBytes); _mainBus.Subscribe <SystemMessage.SystemInit>(intSecTcpService); _mainBus.Subscribe <SystemMessage.SystemStart>(intSecTcpService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intSecTcpService); } } } SubscribeWorkers(bus => { var tcpSendService = new TcpSendService(); // ReSharper disable RedundantTypeArgumentsOfMethod bus.Subscribe <TcpMessage.TcpSend>(tcpSendService); // ReSharper restore RedundantTypeArgumentsOfMethod }); var httpAuthenticationProviders = new List <HttpAuthenticationProvider> { new BasicHttpAuthenticationProvider(_internalAuthenticationProvider), }; if (vNodeSettings.EnableTrustedAuth) { httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider()); } httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider()); var httpPipe = new HttpMessagePipe(); var httpSendService = new HttpSendService(httpPipe, forwardRequests: true); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(httpSendService); _mainBus.Subscribe(new WideningHandler <HttpMessage.SendOverHttp, Message>(_workersHandler)); SubscribeWorkers(bus => { bus.Subscribe <HttpMessage.HttpSend>(httpSendService); bus.Subscribe <HttpMessage.HttpSendPart>(httpSendService); bus.Subscribe <HttpMessage.HttpBeginSend>(httpSendService); bus.Subscribe <HttpMessage.HttpEndSend>(httpSendService); bus.Subscribe <HttpMessage.SendOverHttp>(httpSendService); }); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(infoController); var adminController = new AdminController(_mainQueue, _workersHandler); var pingController = new PingController(); var histogramController = new HistogramController(); var statController = new StatController(monitoringQueue, _workersHandler); var atomController = new AtomController(httpSendService, _mainQueue, _workersHandler, vNodeSettings.DisableHTTPCaching); var gossipController = new GossipController(_mainQueue, _workersHandler, vNodeSettings.GossipTimeout); var persistentSubscriptionController = new PersistentSubscriptionController(httpSendService, _mainQueue, _workersHandler); var electController = new ElectController(_mainQueue); // HTTP SENDERS gossipController.SubscribeSenders(httpPipe); electController.SubscribeSenders(httpPipe); // EXTERNAL HTTP _externalHttpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(), _workersHandler, vNodeSettings.LogHttpRequests, vNodeSettings.GossipAdvertiseInfo.AdvertiseExternalIPAs, vNodeSettings.GossipAdvertiseInfo.AdvertiseExternalHttpPortAs, vNodeSettings.ExtHttpPrefixes); _externalHttpService.SetupController(persistentSubscriptionController); if (vNodeSettings.AdminOnPublic) { _externalHttpService.SetupController(adminController); } _externalHttpService.SetupController(pingController); _externalHttpService.SetupController(infoController); if (vNodeSettings.StatsOnPublic) { _externalHttpService.SetupController(statController); } _externalHttpService.SetupController(atomController); if (vNodeSettings.GossipOnPublic) { _externalHttpService.SetupController(gossipController); } _externalHttpService.SetupController(histogramController); _mainBus.Subscribe <SystemMessage.SystemInit>(_externalHttpService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_externalHttpService); _mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_externalHttpService); // INTERNAL HTTP if (!isSingleNode) { _internalHttpService = new HttpService(ServiceAccessibility.Private, _mainQueue, new TrieUriRouter(), _workersHandler, vNodeSettings.LogHttpRequests, vNodeSettings.GossipAdvertiseInfo.AdvertiseInternalIPAs, vNodeSettings.GossipAdvertiseInfo.AdvertiseInternalHttpPortAs, vNodeSettings.IntHttpPrefixes); _internalHttpService.SetupController(adminController); _internalHttpService.SetupController(pingController); _internalHttpService.SetupController(infoController); _internalHttpService.SetupController(statController); _internalHttpService.SetupController(atomController); _internalHttpService.SetupController(gossipController); _internalHttpService.SetupController(electController); _internalHttpService.SetupController(histogramController); _internalHttpService.SetupController(persistentSubscriptionController); } // Authentication plugin HTTP vNodeSettings.AuthenticationProviderFactory.RegisterHttpControllers(_externalHttpService, _internalHttpService, httpSendService, _mainQueue, _workersHandler); if (_internalHttpService != null) { _mainBus.Subscribe <SystemMessage.SystemInit>(_internalHttpService); _mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_internalHttpService); _mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_internalHttpService); } SubscribeWorkers(bus => { HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray()); }); // REQUEST FORWARDING var forwardingService = new RequestForwardingService(_mainQueue, forwardingProxy, TimeSpan.FromSeconds(1)); _mainBus.Subscribe <SystemMessage.SystemStart>(forwardingService); _mainBus.Subscribe <SystemMessage.RequestForwardingTimerTick>(forwardingService); _mainBus.Subscribe <ClientMessage.NotHandled>(forwardingService); _mainBus.Subscribe <ClientMessage.WriteEventsCompleted>(forwardingService); _mainBus.Subscribe <ClientMessage.TransactionStartCompleted>(forwardingService); _mainBus.Subscribe <ClientMessage.TransactionWriteCompleted>(forwardingService); _mainBus.Subscribe <ClientMessage.TransactionCommitCompleted>(forwardingService); _mainBus.Subscribe <ClientMessage.DeleteStreamCompleted>(forwardingService); // REQUEST MANAGEMENT var requestManagement = new RequestManagementService(_mainQueue, vNodeSettings.PrepareAckCount, vNodeSettings.PrepareTimeout, vNodeSettings.CommitTimeout, vNodeSettings.BetterOrdering); _mainBus.Subscribe <SystemMessage.SystemInit>(requestManagement); _mainBus.Subscribe <ClientMessage.WriteEvents>(requestManagement); _mainBus.Subscribe <ClientMessage.TransactionStart>(requestManagement); _mainBus.Subscribe <ClientMessage.TransactionWrite>(requestManagement); _mainBus.Subscribe <ClientMessage.TransactionCommit>(requestManagement); _mainBus.Subscribe <ClientMessage.DeleteStream>(requestManagement); _mainBus.Subscribe <StorageMessage.RequestCompleted>(requestManagement); _mainBus.Subscribe <StorageMessage.CheckStreamAccessCompleted>(requestManagement); _mainBus.Subscribe <StorageMessage.AlreadyCommitted>(requestManagement); _mainBus.Subscribe <StorageMessage.CommitReplicated>(requestManagement); _mainBus.Subscribe <StorageMessage.PrepareAck>(requestManagement); _mainBus.Subscribe <StorageMessage.WrongExpectedVersion>(requestManagement); _mainBus.Subscribe <StorageMessage.InvalidTransaction>(requestManagement); _mainBus.Subscribe <StorageMessage.StreamDeleted>(requestManagement); _mainBus.Subscribe <StorageMessage.RequestManagerTimerTick>(requestManagement); // SUBSCRIPTIONS var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50)); var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false); _mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.SystemStart, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.SubscribeToStream, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.PollStream, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.CheckPollTimeout, Message>()); _mainBus.Subscribe(subscrQueue.WidenFrom <StorageMessage.EventCommitted, Message>()); var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex); subscrBus.Subscribe <SystemMessage.SystemStart>(subscription); subscrBus.Subscribe <SystemMessage.BecomeShuttingDown>(subscription); subscrBus.Subscribe <TcpMessage.ConnectionClosed>(subscription); subscrBus.Subscribe <ClientMessage.SubscribeToStream>(subscription); subscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(subscription); subscrBus.Subscribe <SubscriptionMessage.PollStream>(subscription); subscrBus.Subscribe <SubscriptionMessage.CheckPollTimeout>(subscription); subscrBus.Subscribe <StorageMessage.EventCommitted>(subscription); // PERSISTENT SUBSCRIPTIONS // IO DISPATCHER var ioDispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_mainQueue)); _mainBus.Subscribe <ClientMessage.ReadStreamEventsBackwardCompleted>(ioDispatcher.BackwardReader); _mainBus.Subscribe <ClientMessage.WriteEventsCompleted>(ioDispatcher.Writer); _mainBus.Subscribe <ClientMessage.ReadStreamEventsForwardCompleted>(ioDispatcher.ForwardReader); _mainBus.Subscribe <ClientMessage.DeleteStreamCompleted>(ioDispatcher.StreamDeleter); _mainBus.Subscribe(ioDispatcher); var perSubscrBus = new InMemoryBus("PersistentSubscriptionsBus", true, TimeSpan.FromMilliseconds(50)); var perSubscrQueue = new QueuedHandlerThreadPool(perSubscrBus, "PersistentSubscriptions", false); _mainBus.Subscribe(perSubscrQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.CreatePersistentSubscription, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.UpdatePersistentSubscription, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.DeletePersistentSubscription, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ConnectToPersistentSubscription, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.PersistentSubscriptionAckEvents, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.PersistentSubscriptionNackEvents, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReplayAllParkedMessages, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReplayParkedMessage, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReadNextNPersistentMessages, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <StorageMessage.EventCommitted, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <MonitoringMessage.GetAllPersistentSubscriptionStats, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <MonitoringMessage.GetStreamPersistentSubscriptionStats, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <MonitoringMessage.GetPersistentSubscriptionStats, Message>()); _mainBus.Subscribe(perSubscrQueue.WidenFrom <SubscriptionMessage.PersistentSubscriptionTimerTick, Message>()); //TODO CC can have multiple threads working on subscription if partition var consumerStrategyRegistry = new PersistentSubscriptionConsumerStrategyRegistry(_mainQueue, _mainBus, vNodeSettings.AdditionalConsumerStrategies); var persistentSubscription = new PersistentSubscriptionService(perSubscrQueue, readIndex, ioDispatcher, _mainQueue, consumerStrategyRegistry); perSubscrBus.Subscribe <SystemMessage.BecomeShuttingDown>(persistentSubscription); perSubscrBus.Subscribe <SystemMessage.BecomeMaster>(persistentSubscription); perSubscrBus.Subscribe <SystemMessage.StateChangeMessage>(persistentSubscription); perSubscrBus.Subscribe <TcpMessage.ConnectionClosed>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.ConnectToPersistentSubscription>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.PersistentSubscriptionAckEvents>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.PersistentSubscriptionNackEvents>(persistentSubscription); perSubscrBus.Subscribe <StorageMessage.EventCommitted>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.DeletePersistentSubscription>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.CreatePersistentSubscription>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.UpdatePersistentSubscription>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.ReplayAllParkedMessages>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.ReplayParkedMessage>(persistentSubscription); perSubscrBus.Subscribe <ClientMessage.ReadNextNPersistentMessages>(persistentSubscription); perSubscrBus.Subscribe <MonitoringMessage.GetAllPersistentSubscriptionStats>(persistentSubscription); perSubscrBus.Subscribe <MonitoringMessage.GetStreamPersistentSubscriptionStats>(persistentSubscription); perSubscrBus.Subscribe <MonitoringMessage.GetPersistentSubscriptionStats>(persistentSubscription); perSubscrBus.Subscribe <SubscriptionMessage.PersistentSubscriptionTimerTick>(persistentSubscription); // STORAGE SCAVENGER var scavengerLogManager = new TFChunkScavengerLogManager(_nodeInfo.ExternalHttp.ToString(), TimeSpan.FromDays(vNodeSettings.ScavengeHistoryMaxAge), ioDispatcher); var storageScavenger = new StorageScavenger(db, tableIndex, readIndex, scavengerLogManager, vNodeSettings.AlwaysKeepScavenged, !vNodeSettings.DisableScavengeMerging, unsafeIgnoreHardDeletes: vNodeSettings.UnsafeIgnoreHardDeletes); // ReSharper disable RedundantTypeArgumentsOfMethod _mainBus.Subscribe <ClientMessage.ScavengeDatabase>(storageScavenger); _mainBus.Subscribe <ClientMessage.StopDatabaseScavenge>(storageScavenger); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(storageScavenger); // ReSharper restore RedundantTypeArgumentsOfMethod // TIMER _timeProvider = new RealTimeProvider(); var threadBasedScheduler = new ThreadBasedScheduler(_timeProvider); AddTask(threadBasedScheduler.Task); _timerService = new TimerService(threadBasedScheduler); _mainBus.Subscribe <SystemMessage.BecomeShutdown>(_timerService); _mainBus.Subscribe <TimerMessage.Schedule>(_timerService); var gossipInfo = new VNodeInfo(_nodeInfo.InstanceId, _nodeInfo.DebugIndex, vNodeSettings.GossipAdvertiseInfo.InternalTcp, vNodeSettings.GossipAdvertiseInfo.InternalSecureTcp, vNodeSettings.GossipAdvertiseInfo.ExternalTcp, vNodeSettings.GossipAdvertiseInfo.ExternalSecureTcp, vNodeSettings.GossipAdvertiseInfo.InternalHttp, vNodeSettings.GossipAdvertiseInfo.ExternalHttp); if (!isSingleNode) { // MASTER REPLICATION var masterReplicationService = new MasterReplicationService(_mainQueue, gossipInfo.InstanceId, db, _workersHandler, epochManager, vNodeSettings.ClusterNodeCount); AddTask(masterReplicationService.Task); _mainBus.Subscribe <SystemMessage.SystemStart>(masterReplicationService); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(masterReplicationService); _mainBus.Subscribe <ReplicationMessage.ReplicaSubscriptionRequest>(masterReplicationService); _mainBus.Subscribe <ReplicationMessage.ReplicaLogPositionAck>(masterReplicationService); monitoringInnerBus.Subscribe <ReplicationMessage.GetReplicationStats>(masterReplicationService); // REPLICA REPLICATION var replicaService = new ReplicaService(_mainQueue, db, epochManager, _workersHandler, _internalAuthenticationProvider, gossipInfo, vNodeSettings.UseSsl, vNodeSettings.SslTargetHost, vNodeSettings.SslValidateServer, vNodeSettings.IntTcpHeartbeatTimeout, vNodeSettings.ExtTcpHeartbeatInterval); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(replicaService); _mainBus.Subscribe <ReplicationMessage.ReconnectToMaster>(replicaService); _mainBus.Subscribe <ReplicationMessage.SubscribeToMaster>(replicaService); _mainBus.Subscribe <ReplicationMessage.AckLogPosition>(replicaService); _mainBus.Subscribe <StorageMessage.PrepareAck>(replicaService); _mainBus.Subscribe <StorageMessage.CommitAck>(replicaService); _mainBus.Subscribe <ClientMessage.TcpForwardMessage>(replicaService); } // ELECTIONS var electionsService = new ElectionsService(_mainQueue, gossipInfo, vNodeSettings.ClusterNodeCount, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint, epochManager, () => readIndex.LastCommitPosition, vNodeSettings.NodePriority); electionsService.SubscribeMessages(_mainBus); if (!isSingleNode || vNodeSettings.GossipOnSingleNode) { // GOSSIP var gossip = new NodeGossipService(_mainQueue, gossipSeedSource, gossipInfo, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint, epochManager, () => readIndex.LastCommitPosition, vNodeSettings.NodePriority, vNodeSettings.GossipInterval, vNodeSettings.GossipAllowedTimeDifference); _mainBus.Subscribe <SystemMessage.SystemInit>(gossip); _mainBus.Subscribe <GossipMessage.RetrieveGossipSeedSources>(gossip); _mainBus.Subscribe <GossipMessage.GotGossipSeedSources>(gossip); _mainBus.Subscribe <GossipMessage.Gossip>(gossip); _mainBus.Subscribe <GossipMessage.GossipReceived>(gossip); _mainBus.Subscribe <SystemMessage.StateChangeMessage>(gossip); _mainBus.Subscribe <GossipMessage.GossipSendFailed>(gossip); _mainBus.Subscribe <SystemMessage.VNodeConnectionEstablished>(gossip); _mainBus.Subscribe <SystemMessage.VNodeConnectionLost>(gossip); } AddTasks(_workersHandler.Start()); AddTask(_mainQueue.Start()); AddTask(monitoringQueue.Start()); AddTask(subscrQueue.Start()); AddTask(perSubscrQueue.Start()); if (subsystems != null) { foreach (var subsystem in subsystems) { var http = isSingleNode ? new [] { _externalHttpService } : new [] { _internalHttpService, _externalHttpService }; subsystem.Register(new StandardComponents(db, _mainQueue, _mainBus, _timerService, _timeProvider, httpSendService, http, _workersHandler)); } } }