/// <summary>
/// Adds a connection.
/// </summary>
/// <param name="connection">The connection.</param>
/// <param name="isNewCluster">Whether the connection is the first connection to a new cluster.</param>
public void AddConnection(MemberConnection connection, bool isNewCluster)
{
// accept every connection, regardless of whether there is a known corresponding member,
// since the first connection is going to come before we get the first members view.
lock (_mutex)
{
// don't add the connection if it is not active - if it *is* active, it still
// could turn not-active anytime, but thanks to _mutex that will only happen
// after the connection has been added
if (!connection.Active)
{
return;
}
var contains = _connections.ContainsKey(connection.MemberId);
if (contains)
{
// we cannot accept this connection, it's a duplicate (internal error?)
_logger.LogWarning($"Cannot accept connection {connection.Id.ToShortString()} to member {connection.MemberId.ToShortString()}, a connection to that member already exists.");
_terminateConnections.Add(connection); // kill.kill.kill
return;
}
// add the connection
_connections[connection.MemberId] = connection;
if (isNewCluster)
{
// reset members
// this is safe because... isNewCluster means that this is the very first connection and there are
// no other connections yet and therefore we should not receive events and therefore no one
// should invoke SetMembers.
// TODO: what if and "old" membersUpdated event is processed?
_members = new MemberTable();
}
// if this is a true member connection
if (_members.ContainsMember(connection.MemberId))
{
// if this is the first connection to an actual member, change state & trigger event
if (!_connected)
{
// change Started | Disconnected -> Connected, ignore otherwise, it could be ShuttingDown or Shutdown
_logger.LogDebug($"Added connection {connection.Id.ToShortString()} to member {connection.MemberId.ToShortString()}, now connected.");
_clusterState.ChangeState(ClientState.Connected, ClientState.Started, ClientState.Disconnected);
_connected = true;
}
else
{
_logger.LogDebug($"Added connection {connection.Id.ToShortString()} to member {connection.MemberId.ToShortString()}.");
}
}
}
}
// (background) adds subscriptions on one member - when a connection is added
private async Task AddSubscriptionsAsync(MemberConnection connection, IReadOnlyCollection <ClusterSubscription> subscriptions, CancellationToken cancellationToken)
{
// this is a background task and therefore should never throw!
foreach (var subscription in subscriptions)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
// this never throws
var attempt = await AddSubscriptionAsync(subscription, connection, cancellationToken).CfAwait();
switch (attempt.Value)
{
case InstallResult.Success: // ok
case InstallResult.SubscriptionNotActive: // ignore it
continue;
case InstallResult.ConnectionNotActive:
// not active = has been removed = what has been done already has been undone
break; // simply exit
case InstallResult.Failed:
// failed to talk to the server - this connection is not working
_terminateConnections.Add(connection);
break; // exit
default:
continue;
}
}
// we are done now
lock (_subscribeTasksMutex) _subscribeTasks.Remove(connection);
}
// runs once on a connection to a member
private async Task RunAsync(MemberConnection connection, DateTime now, CancellationToken cancellationToken)
{
var readElapsed = now - connection.LastReadTime;
var writeElapsed = now - connection.LastWriteTime;
HConsole.WriteLine(this, $"Heartbeat {_clusterState.ClientName} on {connection.Id.ToShortString()} to {connection.MemberId.ToShortString()} at {connection.Address}, " +
$"written {(int)writeElapsed.TotalSeconds}s ago, " +
$"read {(int)readElapsed.TotalSeconds}s ago");
// make sure we read from the client at least every 'timeout',
// which is greater than the interval, so we *should* have
// read from the last ping, if nothing else, so no read means
// that the client not responsive - terminate it
if (readElapsed > _timeout && writeElapsed < _period)
{
_logger.LogWarning("Heartbeat timeout for connection {ConnectionId}, terminating.", connection.Id.ToShortString());
if (connection.Active)
{
_terminateConnections.Add(connection);
}
return;
}
// make sure we write to the client at least every 'period',
// this should trigger a read when we receive the response
if (writeElapsed > _period)
{
_logger.LogDebug("Ping client {ClientId}", connection.Id.ToShortString());
var requestMessage = ClientPingCodec.EncodeRequest();
try
{
// ping should complete within the default invocation timeout
var responseMessage = await _clusterMessaging
.SendToMemberAsync(requestMessage, connection, cancellationToken)
.CfAwait();
// just to be sure everything is ok
_ = ClientPingCodec.DecodeResponse(responseMessage);
}
catch (ClientOfflineException)
{
// down
}
catch (TaskTimeoutException)
{
_logger.LogWarning("Heartbeat ping timeout for connection {ConnectionId}, terminating.", connection.Id.ToShortString());
if (connection.Active)
{
_terminateConnections.Add(connection);
}
}
catch (Exception e)
{
// unexpected
_logger.LogWarning(e, "Heartbeat has thrown an exception, but will continue.");
}
}
}