private void HearthBeatLoop()
{
while (_live)
{
foreach (var clientConnection in _addresses.Values)
{
var request = ClientPingCodec.EncodeRequest();
var task = ((ClientInvocationService)_client.GetInvocationService()).InvokeOnConnection(request, clientConnection);
Logger.Finest("Sending heartbeat request to " + clientConnection.GetAddress());
try
{
var response = ThreadUtil.GetResult(task, _heartBeatTimeout);
var result = ClientPingCodec.DecodeResponse(response);
Logger.Finest("Got heartbeat response from " + clientConnection.GetAddress());
}
catch (Exception e)
{
Logger.Warning(string.Format("Error getting heartbeat from {0}: {1}", clientConnection.GetAddress(), e));
var connection = clientConnection;
FireHeartBeatEvent((listener) => listener.HeartBeatStopped(connection));
}
}
try
{
Thread.Sleep(_heartBeatInterval);
}
catch (Exception)
{
break;
}
}
}
// runs once on a connection to a member
private async Task RunAsync(MemberConnection connection, DateTime now, CancellationToken cancellationToken)
{
// must ensure that timeout > interval ?!
var readElapsed = now - connection.LastReadTime;
var writeElapsed = now - connection.LastWriteTime;
HConsole.WriteLine(this, $"Heartbeat on {connection.Id.ToShortString()}, written {(long)(now - connection.LastWriteTime).TotalMilliseconds}ms ago, read {(long)(now - connection.LastReadTime).TotalMilliseconds}ms ago");
// make sure we read from the client at least every 'timeout',
// which is greater than the interval, so we *should* have
// read from the last ping, if nothing else, so no read means
// that the client not responsive - terminate it
if (readElapsed > _timeout && writeElapsed < _period)
{
_logger.LogWarning("Heartbeat timeout for connection {ConnectionId}.", connection.Id);
if (connection.Active)
{
await connection.TerminateAsync().CfAwait(); // does not throw;
}
return;
}
// make sure we write to the client at least every 'interval',
// this should trigger a read when we receive the response
if (writeElapsed > _period)
{
_logger.LogDebug("Ping client {ClientId}", connection.Id);
var requestMessage = ClientPingCodec.EncodeRequest();
try
{
// ping should complete within the default invocation timeout
var responseMessage = await _clusterMessaging
.SendToMemberAsync(requestMessage, connection, cancellationToken)
.CfAwait();
// just to be sure everything is ok
_ = ClientPingCodec.DecodeResponse(responseMessage);
}
catch (TaskTimeoutException)
{
_logger.LogWarning("Heartbeat ping timeout for connection {ConnectionId}.", connection.Id);
if (connection.Active)
{
await connection.TerminateAsync().CfAwait(); // does not throw;
}
}
catch (Exception e)
{
// unexpected
_logger.LogWarning(e, "Heartbeat has thrown an exception.");
}
}
}
// runs once on a connection to a member
private async Task RunAsync(MemberConnection connection, DateTime now, CancellationToken cancellationToken)
{
// must ensure that timeout > interval ?!
// make sure we read from the client at least every 'timeout',
// which is greater than the interval, so we *should* have
// read from the last ping, if nothing else, so no read means
// that the client not responsive - terminate it
if (now - connection.LastReadTime > _timeout)
{
await TerminateConnection(connection).CAF();
return;
}
// make sure we write to the client at least every 'interval',
// this should trigger a read when we receive the response
if (now - connection.LastWriteTime > _period)
{
_logger.LogDebug("Ping client {ClientId}", connection.Id);
var requestMessage = ClientPingCodec.EncodeRequest();
var cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
try
{
// cannot wait forever on a ping
var responseMessage = await _clusterMessaging
.SendToMemberAsync(requestMessage, connection, cancellation.Token)
.TimeoutAfter(_pingTimeout, cancellation, true)
.CAF();
// just to be sure everything is ok
_ = ClientPingCodec.DecodeResponse(responseMessage);
}
catch (TaskTimeoutException)
{
await TerminateConnection(connection).CAF();
}
catch (Exception e)
{
// unexpected
_logger.LogWarning(e, "Heartbeat has thrown an exception.");
}
finally
{
// if .SendToClientAsync() throws before awaiting, .TimeoutAfter() is never invoked
// and therefore cannot dispose the cancellation = better take care of it
cancellation.Dispose();
}
}
}
// runs once on a connection to a member
private async Task RunAsync(MemberConnection connection, DateTime now, CancellationToken cancellationToken)
{
var readElapsed = now - connection.LastReadTime;
var writeElapsed = now - connection.LastWriteTime;
HConsole.WriteLine(this, $"Heartbeat {_clusterState.ClientName} on {connection.Id.ToShortString()} to {connection.MemberId.ToShortString()} at {connection.Address}, " +
$"written {(int)writeElapsed.TotalSeconds}s ago, " +
$"read {(int)readElapsed.TotalSeconds}s ago");
// make sure we read from the client at least every 'timeout',
// which is greater than the interval, so we *should* have
// read from the last ping, if nothing else, so no read means
// that the client not responsive - terminate it
if (readElapsed > _timeout && writeElapsed < _period)
{
_logger.LogWarning("Heartbeat timeout for connection {ConnectionId}, terminating.", connection.Id.ToShortString());
if (connection.Active)
{
_terminateConnections.Add(connection);
}
return;
}
// make sure we write to the client at least every 'period',
// this should trigger a read when we receive the response
if (writeElapsed > _period)
{
_logger.LogDebug("Ping client {ClientId}", connection.Id.ToShortString());
var requestMessage = ClientPingCodec.EncodeRequest();
try
{
// ping should complete within the default invocation timeout
var responseMessage = await _clusterMessaging
.SendToMemberAsync(requestMessage, connection, cancellationToken)
.CfAwait();
// just to be sure everything is ok
_ = ClientPingCodec.DecodeResponse(responseMessage);
}
catch (ClientOfflineException)
{
// down
}
catch (TaskTimeoutException)
{
_logger.LogWarning("Heartbeat ping timeout for connection {ConnectionId}, terminating.", connection.Id.ToShortString());
if (connection.Active)
{
_terminateConnections.Add(connection);
}
}
catch (Exception e)
{
// unexpected
_logger.LogWarning(e, "Heartbeat has thrown an exception, but will continue.");
}
}
}