private void CheckConnection(DateTime now, ClientConnection connection)
        {
            if (!connection.Live)
            {
                return;
            }

            if (now - connection.LastRead > _heartbeatTimeout)
            {
                if (connection.Live)
                {
                    Logger.Warning("Heartbeat failed over the connection: " + connection);
                    DestroyConnection(connection,
                                      new TargetDisconnectedException("Heartbeat timed out to connection " + connection));
                }
            }

            if (now - connection.LastWrite > _heartbeatInterval)
            {
                if (Logger.IsFinestEnabled())
                {
                    Logger.Finest("Sending heartbeat to " + connection);
                }
                var request = ClientPingCodec.EncodeRequest();
                ((ClientInvocationService)_client.GetInvocationService()).InvokeOnConnection(request, connection);
            }
        }
Пример #2
0
        private void CheckConnection(DateTime now, Connection connection)
        {
            if (!connection.IsAlive)
            {
                return;
            }

            if (now - connection.LastRead > HeartbeatTimeout)
            {
                if (connection.IsAlive)
                {
                    Logger.Warning("Heartbeat failed over the connection: " + connection);
                    OnHeartbeatStopped(connection, "Heartbeat timed out");
                }
            }

            if (now - connection.LastWrite > _heartbeatInterval)
            {
                if (Logger.IsFinestEnabled)
                {
                    Logger.Finest("Sending heartbeat to " + connection);
                }
                var request = ClientPingCodec.EncodeRequest();
                _client.InvocationService.InvokeOnConnection(request, connection);
            }
        }
 private void HearthBeatLoop()
 {
     while (_live)
     {
         foreach (var clientConnection in _addresses.Values)
         {
             var request = ClientPingCodec.EncodeRequest();
             var task    = ((ClientInvocationService)_client.GetInvocationService()).InvokeOnConnection(request, clientConnection);
             Logger.Finest("Sending heartbeat request to " + clientConnection.GetAddress());
             try
             {
                 var response = ThreadUtil.GetResult(task, _heartBeatTimeout);
                 var result   = ClientPingCodec.DecodeResponse(response);
                 Logger.Finest("Got heartbeat response from " + clientConnection.GetAddress());
             }
             catch (Exception e)
             {
                 Logger.Warning(string.Format("Error getting heartbeat from {0}: {1}", clientConnection.GetAddress(), e));
                 var connection = clientConnection;
                 FireHeartBeatEvent((listener) => listener.HeartBeatStopped(connection));
             }
         }
         try
         {
             Thread.Sleep(_heartBeatInterval);
         }
         catch (Exception)
         {
             break;
         }
     }
 }
Пример #4
0
        // runs once on a connection to a member
        private async Task RunAsync(MemberConnection connection, DateTime now, CancellationToken cancellationToken)
        {
            // must ensure that timeout > interval ?!

            var readElapsed  = now - connection.LastReadTime;
            var writeElapsed = now - connection.LastWriteTime;

            HConsole.WriteLine(this, $"Heartbeat on {connection.Id.ToShortString()}, written {(long)(now - connection.LastWriteTime).TotalMilliseconds}ms ago, read {(long)(now - connection.LastReadTime).TotalMilliseconds}ms ago");

            // make sure we read from the client at least every 'timeout',
            // which is greater than the interval, so we *should* have
            // read from the last ping, if nothing else, so no read means
            // that the client not responsive - terminate it

            if (readElapsed > _timeout && writeElapsed < _period)
            {
                _logger.LogWarning("Heartbeat timeout for connection {ConnectionId}.", connection.Id);
                if (connection.Active)
                {
                    await connection.TerminateAsync().CfAwait();                    // does not throw;
                }
                return;
            }

            // make sure we write to the client at least every 'interval',
            // this should trigger a read when we receive the response
            if (writeElapsed > _period)
            {
                _logger.LogDebug("Ping client {ClientId}", connection.Id);

                var requestMessage = ClientPingCodec.EncodeRequest();

                try
                {
                    // ping should complete within the default invocation timeout
                    var responseMessage = await _clusterMessaging
                                          .SendToMemberAsync(requestMessage, connection, cancellationToken)
                                          .CfAwait();

                    // just to be sure everything is ok
                    _ = ClientPingCodec.DecodeResponse(responseMessage);
                }
                catch (TaskTimeoutException)
                {
                    _logger.LogWarning("Heartbeat ping timeout for connection {ConnectionId}.", connection.Id);
                    if (connection.Active)
                    {
                        await connection.TerminateAsync().CfAwait();                    // does not throw;
                    }
                }
                catch (Exception e)
                {
                    // unexpected
                    _logger.LogWarning(e, "Heartbeat has thrown an exception.");
                }
            }
        }
        // runs once on a connection to a member
        private async Task RunAsync(MemberConnection connection, DateTime now, CancellationToken cancellationToken)
        {
            // must ensure that timeout > interval ?!

            // make sure we read from the client at least every 'timeout',
            // which is greater than the interval, so we *should* have
            // read from the last ping, if nothing else, so no read means
            // that the client not responsive - terminate it
            if (now - connection.LastReadTime > _timeout)
            {
                await TerminateConnection(connection).CAF();

                return;
            }

            // make sure we write to the client at least every 'interval',
            // this should trigger a read when we receive the response
            if (now - connection.LastWriteTime > _period)
            {
                _logger.LogDebug("Ping client {ClientId}", connection.Id);

                var requestMessage = ClientPingCodec.EncodeRequest();
                var cancellation   = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);

                try
                {
                    // cannot wait forever on a ping
                    var responseMessage = await _clusterMessaging
                                          .SendToMemberAsync(requestMessage, connection, cancellation.Token)
                                          .TimeoutAfter(_pingTimeout, cancellation, true)
                                          .CAF();

                    // just to be sure everything is ok
                    _ = ClientPingCodec.DecodeResponse(responseMessage);
                }
                catch (TaskTimeoutException)
                {
                    await TerminateConnection(connection).CAF();
                }
                catch (Exception e)
                {
                    // unexpected
                    _logger.LogWarning(e, "Heartbeat has thrown an exception.");
                }
                finally
                {
                    // if .SendToClientAsync() throws before awaiting, .TimeoutAfter() is never invoked
                    // and therefore cannot dispose the cancellation = better take care of it
                    cancellation.Dispose();
                }
            }
        }
Пример #6
0
        private void Heartbeat()
        {
            if (!_live)
            {
                return;
            }

            foreach (var connection in _addresses.Values)
            {
                if (DateTime.Now - connection.LastRead > TimeSpan.FromMilliseconds(_heartbeatTimeout))
                {
                    // heartbeat timed out
                    if (connection.IsHeartBeating)
                    {
                        Logger.Warning("Heartbeat failed to connection " + connection);
                        connection.HeartbeatFailed();
                        var local = connection;
                        FireHeartBeatEvent(l => l.HeartBeatStopped(local));
                    }
                }

                if (DateTime.Now - connection.LastRead > TimeSpan.FromMilliseconds(_heartbeatInterval))
                {
                    var request = ClientPingCodec.EncodeRequest();
                    Logger.Finest("Sending heartbeat to " + connection);
                    ((ClientInvocationService)_client.GetInvocationService()).InvokeOnConnection(request,
                                                                                                 connection, true);
                }
                else
                {
                    // if heartbeat was failing for connection, restore the heartbeat
                    if (!connection.IsHeartBeating)
                    {
                        Logger.Warning("Heartbeat is back to healthy for connection: " + connection);
                        connection.HeartbeatSucceeded();
                        var local = connection;
                        FireHeartBeatEvent(l => l.HeartBeatStarted(local));
                    }
                }
            }
        }
Пример #7
0
        // runs once on a connection to a member
        private async Task RunAsync(MemberConnection connection, DateTime now, CancellationToken cancellationToken)
        {
            var readElapsed  = now - connection.LastReadTime;
            var writeElapsed = now - connection.LastWriteTime;

            HConsole.WriteLine(this, $"Heartbeat {_clusterState.ClientName} on {connection.Id.ToShortString()} to {connection.MemberId.ToShortString()} at {connection.Address}, " +
                               $"written {(int)writeElapsed.TotalSeconds}s ago, " +
                               $"read {(int)readElapsed.TotalSeconds}s ago");

            // make sure we read from the client at least every 'timeout',
            // which is greater than the interval, so we *should* have
            // read from the last ping, if nothing else, so no read means
            // that the client not responsive - terminate it

            if (readElapsed > _timeout && writeElapsed < _period)
            {
                _logger.LogWarning("Heartbeat timeout for connection {ConnectionId}, terminating.", connection.Id.ToShortString());
                if (connection.Active)
                {
                    _terminateConnections.Add(connection);
                }
                return;
            }

            // make sure we write to the client at least every 'period',
            // this should trigger a read when we receive the response
            if (writeElapsed > _period)
            {
                _logger.LogDebug("Ping client {ClientId}", connection.Id.ToShortString());

                var requestMessage = ClientPingCodec.EncodeRequest();

                try
                {
                    // ping should complete within the default invocation timeout
                    var responseMessage = await _clusterMessaging
                                          .SendToMemberAsync(requestMessage, connection, cancellationToken)
                                          .CfAwait();

                    // just to be sure everything is ok
                    _ = ClientPingCodec.DecodeResponse(responseMessage);
                }
                catch (ClientOfflineException)
                {
                    // down
                }
                catch (TaskTimeoutException)
                {
                    _logger.LogWarning("Heartbeat ping timeout for connection {ConnectionId}, terminating.", connection.Id.ToShortString());
                    if (connection.Active)
                    {
                        _terminateConnections.Add(connection);
                    }
                }
                catch (Exception e)
                {
                    // unexpected
                    _logger.LogWarning(e, "Heartbeat has thrown an exception, but will continue.");
                }
            }
        }