// runs once on a connection to a member
private async Task RunAsync(MemberConnection connection, DateTime now, CancellationToken cancellationToken)
{
// must ensure that timeout > interval ?!
var readElapsed = now - connection.LastReadTime;
var writeElapsed = now - connection.LastWriteTime;
HConsole.WriteLine(this, $"Heartbeat on {connection.Id.ToShortString()}, written {(long)(now - connection.LastWriteTime).TotalMilliseconds}ms ago, read {(long)(now - connection.LastReadTime).TotalMilliseconds}ms ago");
// make sure we read from the client at least every 'timeout',
// which is greater than the interval, so we *should* have
// read from the last ping, if nothing else, so no read means
// that the client not responsive - terminate it
if (readElapsed > _timeout && writeElapsed < _period)
{
_logger.LogWarning("Heartbeat timeout for connection {ConnectionId}.", connection.Id);
if (connection.Active)
{
await connection.TerminateAsync().CfAwait(); // does not throw;
}
return;
}
// make sure we write to the client at least every 'interval',
// this should trigger a read when we receive the response
if (writeElapsed > _period)
{
_logger.LogDebug("Ping client {ClientId}", connection.Id);
var requestMessage = ClientPingCodec.EncodeRequest();
try
{
// ping should complete within the default invocation timeout
var responseMessage = await _clusterMessaging
.SendToMemberAsync(requestMessage, connection, cancellationToken)
.CfAwait();
// just to be sure everything is ok
_ = ClientPingCodec.DecodeResponse(responseMessage);
}
catch (TaskTimeoutException)
{
_logger.LogWarning("Heartbeat ping timeout for connection {ConnectionId}.", connection.Id);
if (connection.Active)
{
await connection.TerminateAsync().CfAwait(); // does not throw;
}
}
catch (Exception e)
{
// unexpected
_logger.LogWarning(e, "Heartbeat has thrown an exception.");
}
}
}
private async Task TerminateConnection(MemberConnection connection)
{
if (!connection.Active)
{
return;
}
_logger.LogWarning("Heartbeat timeout for connection {ConnectionId}, terminating.", connection.Id);
await connection.TerminateAsync().CAF(); // does not throw
// TODO: original code has reasons for closing connections
//connection.Close(reason, new TargetDisconnectedException($"Heartbeat timed out to connection {connection}"));
}
