// runs once on a connection to a member private async Task RunAsync(MemberConnection connection, DateTime now, CancellationToken cancellationToken) { // must ensure that timeout > interval ?! var readElapsed = now - connection.LastReadTime; var writeElapsed = now - connection.LastWriteTime; HConsole.WriteLine(this, $"Heartbeat on {connection.Id.ToShortString()}, written {(long)(now - connection.LastWriteTime).TotalMilliseconds}ms ago, read {(long)(now - connection.LastReadTime).TotalMilliseconds}ms ago"); // make sure we read from the client at least every 'timeout', // which is greater than the interval, so we *should* have // read from the last ping, if nothing else, so no read means // that the client not responsive - terminate it if (readElapsed > _timeout && writeElapsed < _period) { _logger.LogWarning("Heartbeat timeout for connection {ConnectionId}.", connection.Id); if (connection.Active) { await connection.TerminateAsync().CfAwait(); // does not throw; } return; } // make sure we write to the client at least every 'interval', // this should trigger a read when we receive the response if (writeElapsed > _period) { _logger.LogDebug("Ping client {ClientId}", connection.Id); var requestMessage = ClientPingCodec.EncodeRequest(); try { // ping should complete within the default invocation timeout var responseMessage = await _clusterMessaging .SendToMemberAsync(requestMessage, connection, cancellationToken) .CfAwait(); // just to be sure everything is ok _ = ClientPingCodec.DecodeResponse(responseMessage); } catch (TaskTimeoutException) { _logger.LogWarning("Heartbeat ping timeout for connection {ConnectionId}.", connection.Id); if (connection.Active) { await connection.TerminateAsync().CfAwait(); // does not throw; } } catch (Exception e) { // unexpected _logger.LogWarning(e, "Heartbeat has thrown an exception."); } } }
private async Task TerminateConnection(MemberConnection connection) { if (!connection.Active) { return; } _logger.LogWarning("Heartbeat timeout for connection {ConnectionId}, terminating.", connection.Id); await connection.TerminateAsync().CAF(); // does not throw // TODO: original code has reasons for closing connections //connection.Close(reason, new TargetDisconnectedException($"Heartbeat timed out to connection {connection}")); }