private void ListenToMaintenanceWorker()
            {
                var needToWait               = false;
                var firstIteration           = true;
                var onErrorDelayTime         = _parent.Config.OnErrorDelayTime.AsTimeSpan;
                var receiveFromWorkerTimeout = _parent.Config.ReceiveFromWorkerTimeout.AsTimeSpan;
                var tcpTimeout               = _parent.Config.TcpConnectionTimeout.AsTimeSpan;

                if (tcpTimeout < receiveFromWorkerTimeout)
                {
                    if (_log.IsInfoEnabled)
                    {
                        _log.Info(
                            $"Warning: TCP timeout is lower than the receive from worker timeout ({tcpTimeout} < {receiveFromWorkerTimeout}), " +
                            "this could affect the cluster observer's decisions.");
                    }
                }

                TcpConnectionInfo tcpConnection = null;

                while (_token.IsCancellationRequested == false)
                {
                    try
                    {
                        if (needToWait)
                        {
                            needToWait = false; // avoid tight loop if there was timeout / error
                            if (firstIteration == false)
                            {
                                _token.WaitHandle.WaitOne(onErrorDelayTime);
                            }
                            firstIteration = false;
                            using (var timeout = new CancellationTokenSource(tcpTimeout))
                                using (var combined = CancellationTokenSource.CreateLinkedTokenSource(_token, timeout.Token))
                                {
                                    tcpConnection = ReplicationUtils.GetTcpInfo(Url, null, "Supervisor", _parent._server.Server.Certificate.Certificate, combined.Token);
                                }
                        }

                        if (tcpConnection == null)
                        {
                            needToWait = true;
                            continue;
                        }

                        var connection = ConnectToClientNode(tcpConnection, _parent._server.Engine.TcpConnectionTimeout);
                        var tcpClient  = connection.TcpClient;
                        var stream     = connection.Stream;
                        using (tcpClient)
                            using (_cts.Token.Register(tcpClient.Dispose))
                                using (_contextPool.AllocateOperationContext(out JsonOperationContext context))
                                    using (var timeoutEvent = new TimeoutEvent(receiveFromWorkerTimeout, $"Timeout event for: {_name}", singleShot: false))
                                    {
                                        timeoutEvent.Start(OnTimeout);
                                        while (_token.IsCancellationRequested == false)
                                        {
                                            BlittableJsonReaderObject rawReport;
                                            try
                                            {
                                                // even if there is a timeout event, we will keep waiting on the same connection until the TCP timeout occurs.
                                                rawReport = context.ReadForMemory(stream, _readStatusUpdateDebugString);
                                                timeoutEvent.Defer(_parent._leaderClusterTag);
                                            }
                                            catch (Exception e)
                                            {
                                                if (_token.IsCancellationRequested)
                                                {
                                                    return;
                                                }

                                                if (_log.IsInfoEnabled)
                                                {
                                                    _log.Info("Exception occurred while reading the report from the connection", e);
                                                }

                                                ReceivedReport = new ClusterNodeStatusReport(new Dictionary <string, DatabaseStatusReport>(),
                                                                                             ClusterNodeStatusReport.ReportStatus.Error,
                                                                                             e,
                                                                                             DateTime.UtcNow,
                                                                                             _lastSuccessfulReceivedReport);

                                                needToWait = true;
                                                break;
                                            }

                                            var report = BuildReport(rawReport);
                                            timeoutEvent.Defer(_parent._leaderClusterTag);

                                            ReceivedReport = _lastSuccessfulReceivedReport = report;
                                        }
                                    }
                    }
                    catch (Exception e)
                    {
                        if (_log.IsInfoEnabled)
                        {
                            _log.Info($"Exception was thrown while collecting info from {ClusterTag}", e);
                        }

                        ReceivedReport = new ClusterNodeStatusReport(new Dictionary <string, DatabaseStatusReport>(),
                                                                     ClusterNodeStatusReport.ReportStatus.Error,
                                                                     e,
                                                                     DateTime.UtcNow,
                                                                     _lastSuccessfulReceivedReport);

                        needToWait = true;
                    }
                }
            }
            private void ListenToMaintenanceWorker()
            {
                var firstIteration           = true;
                var onErrorDelayTime         = _parent.Config.OnErrorDelayTime.AsTimeSpan;
                var receiveFromWorkerTimeout = _parent.Config.ReceiveFromWorkerTimeout.AsTimeSpan;
                var tcpTimeout = _parent.Config.TcpConnectionTimeout.AsTimeSpan;

                if (tcpTimeout < receiveFromWorkerTimeout)
                {
                    if (_log.IsInfoEnabled)
                    {
                        _log.Info(
                            $"Warning: TCP timeout is lower than the receive from worker timeout ({tcpTimeout} < {receiveFromWorkerTimeout}), " +
                            "this could affect the cluster observer's decisions.");
                    }
                }

                while (_token.IsCancellationRequested == false)
                {
                    try
                    {
                        if (firstIteration == false)
                        {
                            // avoid tight loop if there was timeout / error
                            _token.WaitHandle.WaitOne(onErrorDelayTime);
                            if (_token.IsCancellationRequested)
                            {
                                return;
                            }
                        }
                        firstIteration = false;

                        TcpConnectionInfo tcpConnection = null;
                        using (var timeout = new CancellationTokenSource(tcpTimeout))
                            using (var combined = CancellationTokenSource.CreateLinkedTokenSource(_token, timeout.Token))
                            {
                                tcpConnection = ReplicationUtils.GetTcpInfo(Url, null, "Supervisor", _parent._server.Server.Certificate.Certificate, combined.Token);
                                if (tcpConnection == null)
                                {
                                    continue;
                                }
                            }

                        var connection = ConnectToClientNode(tcpConnection, _parent._server.Engine.TcpConnectionTimeout);
                        var tcpClient  = connection.TcpClient;
                        var stream     = connection.Stream;
                        using (tcpClient)
                            using (_cts.Token.Register(tcpClient.Dispose))
                                using (_contextPool.AllocateOperationContext(out JsonOperationContext contextForParsing))
                                    using (_contextPool.AllocateOperationContext(out JsonOperationContext contextForBuffer))
                                        using (contextForBuffer.GetMemoryBuffer(out var readBuffer))
                                            using (var timeoutEvent = new TimeoutEvent(receiveFromWorkerTimeout, $"Timeout event for: {_name}", singleShot: false))
                                            {
                                                timeoutEvent.Start(OnTimeout);
                                                var unchangedReports = new List <DatabaseStatusReport>();

                                                while (_token.IsCancellationRequested == false)
                                                {
                                                    contextForParsing.Reset();
                                                    contextForParsing.Renew();
                                                    BlittableJsonReaderObject rawReport;
                                                    try
                                                    {
                                                        // even if there is a timeout event, we will keep waiting on the same connection until the TCP timeout occurs.

                                                        rawReport = contextForParsing.Sync.ParseToMemory(stream, _readStatusUpdateDebugString, BlittableJsonDocumentBuilder.UsageMode.None, readBuffer);
                                                        timeoutEvent.Defer(_parent._leaderClusterTag);
                                                    }
                                                    catch (Exception e)
                                                    {
                                                        if (_token.IsCancellationRequested)
                                                        {
                                                            return;
                                                        }

                                                        if (_log.IsInfoEnabled)
                                                        {
                                                            _log.Info("Exception occurred while reading the report from the connection", e);
                                                        }

                                                        ReceivedReport = new ClusterNodeStatusReport(new ServerReport(), new Dictionary <string, DatabaseStatusReport>(),
                                                                                                     ClusterNodeStatusReport.ReportStatus.Error,
                                                                                                     e,
                                                                                                     DateTime.UtcNow,
                                                                                                     _lastSuccessfulReceivedReport);

                                                        break;
                                                    }

                                                    _parent.ForTestingPurposes?.BeforeReportBuildAction(this);

                                                    var nodeReport = BuildReport(rawReport, connection.SupportedFeatures);
                                                    timeoutEvent.Defer(_parent._leaderClusterTag);


                                                    UpdateNodeReportIfNeeded(nodeReport, unchangedReports);
                                                    unchangedReports.Clear();

                                                    ReceivedReport = _lastSuccessfulReceivedReport = nodeReport;
                                                    _parent.ForTestingPurposes?.AfterSettingReportAction(this);
                                                }
                                            }
                    }
                    catch (Exception e)
                    {
                        if (_token.IsCancellationRequested)
                        {
                            return;
                        }

                        if (_log.IsInfoEnabled)
                        {
                            _log.Info($"Exception was thrown while collecting info from {ClusterTag}", e);
                        }

                        ReceivedReport = new ClusterNodeStatusReport(new ServerReport(), new Dictionary <string, DatabaseStatusReport>(),
                                                                     ClusterNodeStatusReport.ReportStatus.Error,
                                                                     e,
                                                                     DateTime.UtcNow,
                                                                     _lastSuccessfulReceivedReport);
                    }
                }
            }