// adds a subscription on one member private async ValueTask <Attempt <InstallResult> > AddSubscriptionAsync(ClusterSubscription subscription, MemberConnection connection, CancellationToken cancellationToken) { // if we already know the connection is not active anymore, ignore it // otherwise, install on this member - may throw if the connection goes away in the meantime if (!connection.Active) { return(Attempt.Fail(InstallResult.ConnectionNotActive)); } // add correlated subscription now so it is ready when the first events come var correlationId = _clusterState.GetNextCorrelationId(); _correlatedSubscriptions[correlationId] = subscription; // the original subscription.SubscribeRequest message may be used concurrently, // we need a safe clone so we can use our own correlation id in a safe way. var subscribeRequest = subscription.SubscribeRequest.CloneWithNewCorrelationId(correlationId); // talk to the server ClientMessage response; try { response = await _clusterMessaging.SendToMemberAsync(subscribeRequest, connection, correlationId, cancellationToken).CfAwait(); } catch (Exception e) { _correlatedSubscriptions.TryRemove(correlationId, out _); return(connection.Active ? Attempt.Fail(InstallResult.Failed, e) // also if canceled : Attempt.Fail(InstallResult.ConnectionNotActive)); } // try to add the member subscription to the cluster subscription // fails if the cluster subscription is not active anymore var memberSubscription = subscription.ReadSubscriptionResponse(response, connection); var added = subscription.TryAddMemberSubscription(memberSubscription); if (added) { return(InstallResult.Success); } // the subscription is not active anymore _correlatedSubscriptions.TryRemove(correlationId, out _); CollectSubscription(memberSubscription); return(Attempt.Fail(InstallResult.SubscriptionNotActive)); }
// runs once on a connection to a member private async Task RunAsync(MemberConnection connection, DateTime now, CancellationToken cancellationToken) { // must ensure that timeout > interval ?! var readElapsed = now - connection.LastReadTime; var writeElapsed = now - connection.LastWriteTime; HConsole.WriteLine(this, $"Heartbeat on {connection.Id.ToShortString()}, written {(long)(now - connection.LastWriteTime).TotalMilliseconds}ms ago, read {(long)(now - connection.LastReadTime).TotalMilliseconds}ms ago"); // make sure we read from the client at least every 'timeout', // which is greater than the interval, so we *should* have // read from the last ping, if nothing else, so no read means // that the client not responsive - terminate it if (readElapsed > _timeout && writeElapsed < _period) { _logger.LogWarning("Heartbeat timeout for connection {ConnectionId}.", connection.Id); if (connection.Active) { await connection.TerminateAsync().CfAwait(); // does not throw; } return; } // make sure we write to the client at least every 'interval', // this should trigger a read when we receive the response if (writeElapsed > _period) { _logger.LogDebug("Ping client {ClientId}", connection.Id); var requestMessage = ClientPingCodec.EncodeRequest(); try { // ping should complete within the default invocation timeout var responseMessage = await _clusterMessaging .SendToMemberAsync(requestMessage, connection, cancellationToken) .CfAwait(); // just to be sure everything is ok _ = ClientPingCodec.DecodeResponse(responseMessage); } catch (TaskTimeoutException) { _logger.LogWarning("Heartbeat ping timeout for connection {ConnectionId}.", connection.Id); if (connection.Active) { await connection.TerminateAsync().CfAwait(); // does not throw; } } catch (Exception e) { // unexpected _logger.LogWarning(e, "Heartbeat has thrown an exception."); } } }
// runs once on a connection to a member private async Task RunAsync(MemberConnection connection, DateTime now, CancellationToken cancellationToken) { // must ensure that timeout > interval ?! // make sure we read from the client at least every 'timeout', // which is greater than the interval, so we *should* have // read from the last ping, if nothing else, so no read means // that the client not responsive - terminate it if (now - connection.LastReadTime > _timeout) { await TerminateConnection(connection).CAF(); return; } // make sure we write to the client at least every 'interval', // this should trigger a read when we receive the response if (now - connection.LastWriteTime > _period) { _logger.LogDebug("Ping client {ClientId}", connection.Id); var requestMessage = ClientPingCodec.EncodeRequest(); var cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); try { // cannot wait forever on a ping var responseMessage = await _clusterMessaging .SendToMemberAsync(requestMessage, connection, cancellation.Token) .TimeoutAfter(_pingTimeout, cancellation, true) .CAF(); // just to be sure everything is ok _ = ClientPingCodec.DecodeResponse(responseMessage); } catch (TaskTimeoutException) { await TerminateConnection(connection).CAF(); } catch (Exception e) { // unexpected _logger.LogWarning(e, "Heartbeat has thrown an exception."); } finally { // if .SendToClientAsync() throws before awaiting, .TimeoutAfter() is never invoked // and therefore cannot dispose the cancellation = better take care of it cancellation.Dispose(); } } }
// runs once on a connection to a member private async Task RunAsync(MemberConnection connection, DateTime now, CancellationToken cancellationToken) { var readElapsed = now - connection.LastReadTime; var writeElapsed = now - connection.LastWriteTime; HConsole.WriteLine(this, $"Heartbeat {_clusterState.ClientName} on {connection.Id.ToShortString()} to {connection.MemberId.ToShortString()} at {connection.Address}, " + $"written {(int)writeElapsed.TotalSeconds}s ago, " + $"read {(int)readElapsed.TotalSeconds}s ago"); // make sure we read from the client at least every 'timeout', // which is greater than the interval, so we *should* have // read from the last ping, if nothing else, so no read means // that the client not responsive - terminate it if (readElapsed > _timeout && writeElapsed < _period) { _logger.LogWarning("Heartbeat timeout for connection {ConnectionId}, terminating.", connection.Id.ToShortString()); if (connection.Active) { _terminateConnections.Add(connection); } return; } // make sure we write to the client at least every 'period', // this should trigger a read when we receive the response if (writeElapsed > _period) { _logger.LogDebug("Ping client {ClientId}", connection.Id.ToShortString()); var requestMessage = ClientPingCodec.EncodeRequest(); try { // ping should complete within the default invocation timeout var responseMessage = await _clusterMessaging .SendToMemberAsync(requestMessage, connection, cancellationToken) .CfAwait(); // just to be sure everything is ok _ = ClientPingCodec.DecodeResponse(responseMessage); } catch (ClientOfflineException) { // down } catch (TaskTimeoutException) { _logger.LogWarning("Heartbeat ping timeout for connection {ConnectionId}, terminating.", connection.Id.ToShortString()); if (connection.Active) { _terminateConnections.Add(connection); } } catch (Exception e) { // unexpected _logger.LogWarning(e, "Heartbeat has thrown an exception, but will continue."); } } }
/// <summary> /// Installs a subscription on one member. /// </summary> /// <param name="connection">The connection to the member.</param> /// <param name="subscription">The subscription.</param> /// <param name="cancellationToken">A cancellation token.</param> /// <returns>A task that will complete when the client has subscribed to the server event.</returns> private async ValueTask <Attempt <InstallResult> > InstallSubscriptionAsync(ClusterSubscription subscription, MemberConnection connection, CancellationToken cancellationToken) { // if we already know the client is not active anymore, ignore it // otherwise, install on this client - may throw if the client goes away in the meantime if (!connection.Active) { return(Attempt.Fail(InstallResult.ClientNotActive)); } // add immediately, we don't know when the events will start to come var correlationId = _clusterState.GetNextCorrelationId(); _correlatedSubscriptions[correlationId] = subscription; // we do not control the original subscription.SubscribeRequest message and it may // be used concurrently, and so it is not safe to alter its correlation identifier. // instead, we use a safe clone of the original message var subscribeRequest = subscription.SubscribeRequest.CloneWithNewCorrelationId(correlationId); ClientMessage response; try { // hopefully the client is still active, else this will throw response = await _clusterMessaging.SendToMemberAsync(subscribeRequest, connection, correlationId, cancellationToken).CfAwait(); } catch (Exception e) { _correlatedSubscriptions.TryRemove(correlationId, out _); if (!connection.Active) { return(Attempt.Fail(InstallResult.ClientNotActive)); } _logger.LogError(e, "Caught exception while cleaning up after failing to install a subscription."); return(Attempt.Fail(InstallResult.Failed, e)); } // try to add the client subscription var(added, id) = subscription.TryAddClientSubscription(response, connection); if (added) { return(InstallResult.Success); } // otherwise, the client subscription could not be added, which means that the // cluster subscription is not active anymore, and so we need to undo the // server-side subscription // if the client is gone already it may be that the subscription has been // removed already, in which case... just give up now if (!_correlatedSubscriptions.TryRemove(correlationId, out _)) { return(Attempt.Fail(InstallResult.SubscriptionNotActive)); } var unsubscribeRequest = subscription.CreateUnsubscribeRequest(id); try { var unsubscribeResponse = await _clusterMessaging.SendToMemberAsync(unsubscribeRequest, connection, cancellationToken).CfAwait(); var unsubscribed = subscription.ReadUnsubscribeResponse(unsubscribeResponse); return(unsubscribed ? Attempt.Fail(InstallResult.SubscriptionNotActive) : Attempt.Fail(InstallResult.ConfusedServer)); } catch (Exception e) { // otherwise, we failed to undo the server-side subscription - end result is that // the client is fine (won't handle events, we've removed the correlated subscription // etc) but the server maybe confused. _logger.LogError(e, "Caught exception while cleaning up after failing to install a subscription."); return(Attempt.Fail(InstallResult.ConfusedServer, e)); } }