// adds a subscription on one member
private async ValueTask <Attempt <InstallResult> > AddSubscriptionAsync(ClusterSubscription subscription, MemberConnection connection, CancellationToken cancellationToken)
{
// if we already know the connection is not active anymore, ignore it
// otherwise, install on this member - may throw if the connection goes away in the meantime
if (!connection.Active)
{
return(Attempt.Fail(InstallResult.ConnectionNotActive));
}
// add correlated subscription now so it is ready when the first events come
var correlationId = _clusterState.GetNextCorrelationId();
_correlatedSubscriptions[correlationId] = subscription;
// the original subscription.SubscribeRequest message may be used concurrently,
// we need a safe clone so we can use our own correlation id in a safe way.
var subscribeRequest = subscription.SubscribeRequest.CloneWithNewCorrelationId(correlationId);
// talk to the server
ClientMessage response;
try
{
response = await _clusterMessaging.SendToMemberAsync(subscribeRequest, connection, correlationId, cancellationToken).CfAwait();
}
catch (Exception e)
{
_correlatedSubscriptions.TryRemove(correlationId, out _);
return(connection.Active
? Attempt.Fail(InstallResult.Failed, e) // also if canceled
: Attempt.Fail(InstallResult.ConnectionNotActive));
}
// try to add the member subscription to the cluster subscription
// fails if the cluster subscription is not active anymore
var memberSubscription = subscription.ReadSubscriptionResponse(response, connection);
var added = subscription.TryAddMemberSubscription(memberSubscription);
if (added)
{
return(InstallResult.Success);
}
// the subscription is not active anymore
_correlatedSubscriptions.TryRemove(correlationId, out _);
CollectSubscription(memberSubscription);
return(Attempt.Fail(InstallResult.SubscriptionNotActive));
}
// runs once on a connection to a member
private async Task RunAsync(MemberConnection connection, DateTime now, CancellationToken cancellationToken)
{
// must ensure that timeout > interval ?!
var readElapsed = now - connection.LastReadTime;
var writeElapsed = now - connection.LastWriteTime;
HConsole.WriteLine(this, $"Heartbeat on {connection.Id.ToShortString()}, written {(long)(now - connection.LastWriteTime).TotalMilliseconds}ms ago, read {(long)(now - connection.LastReadTime).TotalMilliseconds}ms ago");
// make sure we read from the client at least every 'timeout',
// which is greater than the interval, so we *should* have
// read from the last ping, if nothing else, so no read means
// that the client not responsive - terminate it
if (readElapsed > _timeout && writeElapsed < _period)
{
_logger.LogWarning("Heartbeat timeout for connection {ConnectionId}.", connection.Id);
if (connection.Active)
{
await connection.TerminateAsync().CfAwait(); // does not throw;
}
return;
}
// make sure we write to the client at least every 'interval',
// this should trigger a read when we receive the response
if (writeElapsed > _period)
{
_logger.LogDebug("Ping client {ClientId}", connection.Id);
var requestMessage = ClientPingCodec.EncodeRequest();
try
{
// ping should complete within the default invocation timeout
var responseMessage = await _clusterMessaging
.SendToMemberAsync(requestMessage, connection, cancellationToken)
.CfAwait();
// just to be sure everything is ok
_ = ClientPingCodec.DecodeResponse(responseMessage);
}
catch (TaskTimeoutException)
{
_logger.LogWarning("Heartbeat ping timeout for connection {ConnectionId}.", connection.Id);
if (connection.Active)
{
await connection.TerminateAsync().CfAwait(); // does not throw;
}
}
catch (Exception e)
{
// unexpected
_logger.LogWarning(e, "Heartbeat has thrown an exception.");
}
}
}
// runs once on a connection to a member
private async Task RunAsync(MemberConnection connection, DateTime now, CancellationToken cancellationToken)
{
// must ensure that timeout > interval ?!
// make sure we read from the client at least every 'timeout',
// which is greater than the interval, so we *should* have
// read from the last ping, if nothing else, so no read means
// that the client not responsive - terminate it
if (now - connection.LastReadTime > _timeout)
{
await TerminateConnection(connection).CAF();
return;
}
// make sure we write to the client at least every 'interval',
// this should trigger a read when we receive the response
if (now - connection.LastWriteTime > _period)
{
_logger.LogDebug("Ping client {ClientId}", connection.Id);
var requestMessage = ClientPingCodec.EncodeRequest();
var cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
try
{
// cannot wait forever on a ping
var responseMessage = await _clusterMessaging
.SendToMemberAsync(requestMessage, connection, cancellation.Token)
.TimeoutAfter(_pingTimeout, cancellation, true)
.CAF();
// just to be sure everything is ok
_ = ClientPingCodec.DecodeResponse(responseMessage);
}
catch (TaskTimeoutException)
{
await TerminateConnection(connection).CAF();
}
catch (Exception e)
{
// unexpected
_logger.LogWarning(e, "Heartbeat has thrown an exception.");
}
finally
{
// if .SendToClientAsync() throws before awaiting, .TimeoutAfter() is never invoked
// and therefore cannot dispose the cancellation = better take care of it
cancellation.Dispose();
}
}
}
// runs once on a connection to a member
private async Task RunAsync(MemberConnection connection, DateTime now, CancellationToken cancellationToken)
{
var readElapsed = now - connection.LastReadTime;
var writeElapsed = now - connection.LastWriteTime;
HConsole.WriteLine(this, $"Heartbeat {_clusterState.ClientName} on {connection.Id.ToShortString()} to {connection.MemberId.ToShortString()} at {connection.Address}, " +
$"written {(int)writeElapsed.TotalSeconds}s ago, " +
$"read {(int)readElapsed.TotalSeconds}s ago");
// make sure we read from the client at least every 'timeout',
// which is greater than the interval, so we *should* have
// read from the last ping, if nothing else, so no read means
// that the client not responsive - terminate it
if (readElapsed > _timeout && writeElapsed < _period)
{
_logger.LogWarning("Heartbeat timeout for connection {ConnectionId}, terminating.", connection.Id.ToShortString());
if (connection.Active)
{
_terminateConnections.Add(connection);
}
return;
}
// make sure we write to the client at least every 'period',
// this should trigger a read when we receive the response
if (writeElapsed > _period)
{
_logger.LogDebug("Ping client {ClientId}", connection.Id.ToShortString());
var requestMessage = ClientPingCodec.EncodeRequest();
try
{
// ping should complete within the default invocation timeout
var responseMessage = await _clusterMessaging
.SendToMemberAsync(requestMessage, connection, cancellationToken)
.CfAwait();
// just to be sure everything is ok
_ = ClientPingCodec.DecodeResponse(responseMessage);
}
catch (ClientOfflineException)
{
// down
}
catch (TaskTimeoutException)
{
_logger.LogWarning("Heartbeat ping timeout for connection {ConnectionId}, terminating.", connection.Id.ToShortString());
if (connection.Active)
{
_terminateConnections.Add(connection);
}
}
catch (Exception e)
{
// unexpected
_logger.LogWarning(e, "Heartbeat has thrown an exception, but will continue.");
}
}
}
/// <summary>
/// Installs a subscription on one member.
/// </summary>
/// <param name="connection">The connection to the member.</param>
/// <param name="subscription">The subscription.</param>
/// <param name="cancellationToken">A cancellation token.</param>
/// <returns>A task that will complete when the client has subscribed to the server event.</returns>
private async ValueTask <Attempt <InstallResult> > InstallSubscriptionAsync(ClusterSubscription subscription, MemberConnection connection, CancellationToken cancellationToken)
{
// if we already know the client is not active anymore, ignore it
// otherwise, install on this client - may throw if the client goes away in the meantime
if (!connection.Active)
{
return(Attempt.Fail(InstallResult.ClientNotActive));
}
// add immediately, we don't know when the events will start to come
var correlationId = _clusterState.GetNextCorrelationId();
_correlatedSubscriptions[correlationId] = subscription;
// we do not control the original subscription.SubscribeRequest message and it may
// be used concurrently, and so it is not safe to alter its correlation identifier.
// instead, we use a safe clone of the original message
var subscribeRequest = subscription.SubscribeRequest.CloneWithNewCorrelationId(correlationId);
ClientMessage response;
try
{
// hopefully the client is still active, else this will throw
response = await _clusterMessaging.SendToMemberAsync(subscribeRequest, connection, correlationId, cancellationToken).CfAwait();
}
catch (Exception e)
{
_correlatedSubscriptions.TryRemove(correlationId, out _);
if (!connection.Active)
{
return(Attempt.Fail(InstallResult.ClientNotActive));
}
_logger.LogError(e, "Caught exception while cleaning up after failing to install a subscription.");
return(Attempt.Fail(InstallResult.Failed, e));
}
// try to add the client subscription
var(added, id) = subscription.TryAddClientSubscription(response, connection);
if (added)
{
return(InstallResult.Success);
}
// otherwise, the client subscription could not be added, which means that the
// cluster subscription is not active anymore, and so we need to undo the
// server-side subscription
// if the client is gone already it may be that the subscription has been
// removed already, in which case... just give up now
if (!_correlatedSubscriptions.TryRemove(correlationId, out _))
{
return(Attempt.Fail(InstallResult.SubscriptionNotActive));
}
var unsubscribeRequest = subscription.CreateUnsubscribeRequest(id);
try
{
var unsubscribeResponse = await _clusterMessaging.SendToMemberAsync(unsubscribeRequest, connection, cancellationToken).CfAwait();
var unsubscribed = subscription.ReadUnsubscribeResponse(unsubscribeResponse);
return(unsubscribed
? Attempt.Fail(InstallResult.SubscriptionNotActive)
: Attempt.Fail(InstallResult.ConfusedServer));
}
catch (Exception e)
{
// otherwise, we failed to undo the server-side subscription - end result is that
// the client is fine (won't handle events, we've removed the correlated subscription
// etc) but the server maybe confused.
_logger.LogError(e, "Caught exception while cleaning up after failing to install a subscription.");
return(Attempt.Fail(InstallResult.ConfusedServer, e));
}
}