private void CheckConnection(DateTime now, ClientConnection connection)
{
if (!connection.Live)
{
return;
}
if (now - connection.LastRead > _heartbeatTimeout)
{
if (connection.Live)
{
Logger.Warning("Heartbeat failed over the connection: " + connection);
DestroyConnection(connection,
new TargetDisconnectedException("Heartbeat timed out to connection " + connection));
}
}
if (now - connection.LastWrite > _heartbeatInterval)
{
if (Logger.IsFinestEnabled())
{
Logger.Finest("Sending heartbeat to " + connection);
}
var request = ClientPingCodec.EncodeRequest();
((ClientInvocationService)_client.GetInvocationService()).InvokeOnConnection(request, connection);
}
}
private void CheckConnection(DateTime now, Connection connection)
{
if (!connection.IsAlive)
{
return;
}
if (now - connection.LastRead > HeartbeatTimeout)
{
if (connection.IsAlive)
{
Logger.Warning("Heartbeat failed over the connection: " + connection);
OnHeartbeatStopped(connection, "Heartbeat timed out");
}
}
if (now - connection.LastWrite > _heartbeatInterval)
{
if (Logger.IsFinestEnabled)
{
Logger.Finest("Sending heartbeat to " + connection);
}
var request = ClientPingCodec.EncodeRequest();
_client.InvocationService.InvokeOnConnection(request, connection);
}
}
private void HearthBeatLoop()
{
while (_live)
{
foreach (var clientConnection in _addresses.Values)
{
var request = ClientPingCodec.EncodeRequest();
var task = ((ClientInvocationService)_client.GetInvocationService()).InvokeOnConnection(request, clientConnection);
Logger.Finest("Sending heartbeat request to " + clientConnection.GetAddress());
try
{
var response = ThreadUtil.GetResult(task, _heartBeatTimeout);
var result = ClientPingCodec.DecodeResponse(response);
Logger.Finest("Got heartbeat response from " + clientConnection.GetAddress());
}
catch (Exception e)
{
Logger.Warning(string.Format("Error getting heartbeat from {0}: {1}", clientConnection.GetAddress(), e));
var connection = clientConnection;
FireHeartBeatEvent((listener) => listener.HeartBeatStopped(connection));
}
}
try
{
Thread.Sleep(_heartBeatInterval);
}
catch (Exception)
{
break;
}
}
}
// runs once on a connection to a member
private async Task RunAsync(MemberConnection connection, DateTime now, CancellationToken cancellationToken)
{
// must ensure that timeout > interval ?!
var readElapsed = now - connection.LastReadTime;
var writeElapsed = now - connection.LastWriteTime;
HConsole.WriteLine(this, $"Heartbeat on {connection.Id.ToShortString()}, written {(long)(now - connection.LastWriteTime).TotalMilliseconds}ms ago, read {(long)(now - connection.LastReadTime).TotalMilliseconds}ms ago");
// make sure we read from the client at least every 'timeout',
// which is greater than the interval, so we *should* have
// read from the last ping, if nothing else, so no read means
// that the client not responsive - terminate it
if (readElapsed > _timeout && writeElapsed < _period)
{
_logger.LogWarning("Heartbeat timeout for connection {ConnectionId}.", connection.Id);
if (connection.Active)
{
await connection.TerminateAsync().CfAwait(); // does not throw;
}
return;
}
// make sure we write to the client at least every 'interval',
// this should trigger a read when we receive the response
if (writeElapsed > _period)
{
_logger.LogDebug("Ping client {ClientId}", connection.Id);
var requestMessage = ClientPingCodec.EncodeRequest();
try
{
// ping should complete within the default invocation timeout
var responseMessage = await _clusterMessaging
.SendToMemberAsync(requestMessage, connection, cancellationToken)
.CfAwait();
// just to be sure everything is ok
_ = ClientPingCodec.DecodeResponse(responseMessage);
}
catch (TaskTimeoutException)
{
_logger.LogWarning("Heartbeat ping timeout for connection {ConnectionId}.", connection.Id);
if (connection.Active)
{
await connection.TerminateAsync().CfAwait(); // does not throw;
}
}
catch (Exception e)
{
// unexpected
_logger.LogWarning(e, "Heartbeat has thrown an exception.");
}
}
}
// runs once on a connection to a member
private async Task RunAsync(MemberConnection connection, DateTime now, CancellationToken cancellationToken)
{
// must ensure that timeout > interval ?!
// make sure we read from the client at least every 'timeout',
// which is greater than the interval, so we *should* have
// read from the last ping, if nothing else, so no read means
// that the client not responsive - terminate it
if (now - connection.LastReadTime > _timeout)
{
await TerminateConnection(connection).CAF();
return;
}
// make sure we write to the client at least every 'interval',
// this should trigger a read when we receive the response
if (now - connection.LastWriteTime > _period)
{
_logger.LogDebug("Ping client {ClientId}", connection.Id);
var requestMessage = ClientPingCodec.EncodeRequest();
var cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
try
{
// cannot wait forever on a ping
var responseMessage = await _clusterMessaging
.SendToMemberAsync(requestMessage, connection, cancellation.Token)
.TimeoutAfter(_pingTimeout, cancellation, true)
.CAF();
// just to be sure everything is ok
_ = ClientPingCodec.DecodeResponse(responseMessage);
}
catch (TaskTimeoutException)
{
await TerminateConnection(connection).CAF();
}
catch (Exception e)
{
// unexpected
_logger.LogWarning(e, "Heartbeat has thrown an exception.");
}
finally
{
// if .SendToClientAsync() throws before awaiting, .TimeoutAfter() is never invoked
// and therefore cannot dispose the cancellation = better take care of it
cancellation.Dispose();
}
}
}
private void Heartbeat()
{
if (!_live)
{
return;
}
foreach (var connection in _addresses.Values)
{
if (DateTime.Now - connection.LastRead > TimeSpan.FromMilliseconds(_heartbeatTimeout))
{
// heartbeat timed out
if (connection.IsHeartBeating)
{
Logger.Warning("Heartbeat failed to connection " + connection);
connection.HeartbeatFailed();
var local = connection;
FireHeartBeatEvent(l => l.HeartBeatStopped(local));
}
}
if (DateTime.Now - connection.LastRead > TimeSpan.FromMilliseconds(_heartbeatInterval))
{
var request = ClientPingCodec.EncodeRequest();
Logger.Finest("Sending heartbeat to " + connection);
((ClientInvocationService)_client.GetInvocationService()).InvokeOnConnection(request,
connection, true);
}
else
{
// if heartbeat was failing for connection, restore the heartbeat
if (!connection.IsHeartBeating)
{
Logger.Warning("Heartbeat is back to healthy for connection: " + connection);
connection.HeartbeatSucceeded();
var local = connection;
FireHeartBeatEvent(l => l.HeartBeatStarted(local));
}
}
}
}
// runs once on a connection to a member
private async Task RunAsync(MemberConnection connection, DateTime now, CancellationToken cancellationToken)
{
var readElapsed = now - connection.LastReadTime;
var writeElapsed = now - connection.LastWriteTime;
HConsole.WriteLine(this, $"Heartbeat {_clusterState.ClientName} on {connection.Id.ToShortString()} to {connection.MemberId.ToShortString()} at {connection.Address}, " +
$"written {(int)writeElapsed.TotalSeconds}s ago, " +
$"read {(int)readElapsed.TotalSeconds}s ago");
// make sure we read from the client at least every 'timeout',
// which is greater than the interval, so we *should* have
// read from the last ping, if nothing else, so no read means
// that the client not responsive - terminate it
if (readElapsed > _timeout && writeElapsed < _period)
{
_logger.LogWarning("Heartbeat timeout for connection {ConnectionId}, terminating.", connection.Id.ToShortString());
if (connection.Active)
{
_terminateConnections.Add(connection);
}
return;
}
// make sure we write to the client at least every 'period',
// this should trigger a read when we receive the response
if (writeElapsed > _period)
{
_logger.LogDebug("Ping client {ClientId}", connection.Id.ToShortString());
var requestMessage = ClientPingCodec.EncodeRequest();
try
{
// ping should complete within the default invocation timeout
var responseMessage = await _clusterMessaging
.SendToMemberAsync(requestMessage, connection, cancellationToken)
.CfAwait();
// just to be sure everything is ok
_ = ClientPingCodec.DecodeResponse(responseMessage);
}
catch (ClientOfflineException)
{
// down
}
catch (TaskTimeoutException)
{
_logger.LogWarning("Heartbeat ping timeout for connection {ConnectionId}, terminating.", connection.Id.ToShortString());
if (connection.Active)
{
_terminateConnections.Add(connection);
}
}
catch (Exception e)
{
// unexpected
_logger.LogWarning(e, "Heartbeat has thrown an exception, but will continue.");
}
}
}
