/// <summary>
/// (thread-unsafe) Notifies that a connection has been opened.
/// </summary>
/// <param name="connection">The connection.</param>
/// <returns><c>true</c> if the connection is the first one to be established; otherwise <c>false</c>.</returns>
/// <remarks>
/// <para>This method is not thread-safe; the caller has to lock the
/// <see cref="Mutex"/> object to ensure thread-safety.</para>
/// </remarks>
public bool NotifyConnectionOpened(MemberConnection connection)
{
var isFirst = _connections.Count == 0;
#if NETSTANDARD2_0
var contains = _connections.ContainsKey(connection.MemberId);
_connections[connection.MemberId] = connection;
if (contains)
#else
if (!_connections.TryAdd(connection.MemberId, connection))
#endif
{ throw new HazelcastException("Failed to add a connection (duplicate memberId)."); }
if (_clusterId == default)
{
_clusterId = connection.ClusterId; // first cluster
}
else if (_clusterId != connection.ClusterId)
{
// see TcpClientConnectionManager java class handleSuccessfulAuth method
// does not even consider the cluster identifier when !isFirst
if (isFirst)
{
_clusterId = connection.ClusterId; // new cluster
_memberTable = new MemberTable();
}
}
return(isFirst);
}
/// <summary>
/// Adds a connection.
/// </summary>
/// <param name="connection">The connection.</param>
/// <param name="isNewCluster">Whether the connection is the first connection to a new cluster.</param>
public void AddConnection(MemberConnection connection, bool isNewCluster)
{
// accept every connection, regardless of whether there is a known corresponding member,
// since the first connection is going to come before we get the first members view.
lock (_mutex)
{
// don't add the connection if it is not active - if it *is* active, it still
// could turn not-active anytime, but thanks to _mutex that will only happen
// after the connection has been added
if (!connection.Active)
{
return;
}
var contains = _connections.ContainsKey(connection.MemberId);
if (contains)
{
// we cannot accept this connection, it's a duplicate (internal error?)
_logger.LogWarning($"Cannot accept connection {connection.Id.ToShortString()} to member {connection.MemberId.ToShortString()}, a connection to that member already exists.");
_terminateConnections.Add(connection); // kill.kill.kill
return;
}
// add the connection
_connections[connection.MemberId] = connection;
if (isNewCluster)
{
// reset members
// this is safe because... isNewCluster means that this is the very first connection and there are
// no other connections yet and therefore we should not receive events and therefore no one
// should invoke SetMembers.
// TODO: what if and "old" membersUpdated event is processed?
_members = new MemberTable();
}
// if this is a true member connection
if (_members.ContainsMember(connection.MemberId))
{
// if this is the first connection to an actual member, change state & trigger event
if (!_connected)
{
// change Started | Disconnected -> Connected, ignore otherwise, it could be ShuttingDown or Shutdown
_logger.LogDebug($"Added connection {connection.Id.ToShortString()} to member {connection.MemberId.ToShortString()}, now connected.");
_clusterState.ChangeState(ClientState.Connected, ClientState.Started, ClientState.Disconnected);
_connected = true;
}
else
{
_logger.LogDebug($"Added connection {connection.Id.ToShortString()} to member {connection.MemberId.ToShortString()}.");
}
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="ClusterMembers"/> class.
/// </summary>
/// <param name="clusterState">The cluster state.</param>
/// <param name="terminateConnections">The terminate connections task.</param>
public ClusterMembers(ClusterState clusterState, TerminateConnections terminateConnections)
{
HConsole.Configure(x => x.Configure <ClusterMembers>().SetPrefix("CLUST.MBRS"));
_clusterState = clusterState;
_terminateConnections = terminateConnections;
_loadBalancer = clusterState.Options.LoadBalancer.Service ?? new RandomLoadBalancer();
_logger = _clusterState.LoggerFactory.CreateLogger <ClusterMembers>();
_members = new MemberTable();
// members to connect
if (clusterState.IsSmartRouting)
{
_memberConnectionQueue = new MemberConnectionQueue(clusterState.LoggerFactory);
}
}
private void LogDiffs(MemberTable table, Dictionary <MemberInfo, int> diff)
{
var msg = new StringBuilder();
msg.Append("Members [");
msg.Append(table.Count);
msg.AppendLine("] {");
foreach (var member in table.Members)
{
msg.Append(" ");
msg.Append(member.Address);
msg.Append(" - ");
msg.Append(member.Id);
if (diff.TryGetValue(member, out var d) && d == 2)
{
msg.Append(" - new");
}
msg.AppendLine();
}
msg.Append('}');
_logger.LogInformation(msg.ToString());
}
/// <summary>
/// Notifies of a 'members view' event.
/// </summary>
/// <param name="version">The version.</param>
/// <param name="members">The members.</param>
public async ValueTask <MembersUpdatedEventArgs> NotifyMembersView(int version, ICollection <MemberInfo> members)
{
// FIXME could we process two of these at a time?
// get a new table
var table = new MemberTable(version, members);
// compute changes
// count 1 for old members, 2 for new members, and then the result is
// that 1=removed, 2=added, 3=unchanged
// MemberInfo overrides GetHashCode and can be used as a key here
var diff = new Dictionary <MemberInfo, int>();
if (_memberTable == null)
{
foreach (var m in table.Members.Values)
{
diff[m] = 2;
}
}
else
{
foreach (var m in _memberTable.Members.Values)
{
diff[m] = 1;
}
foreach (var m in table.Members.Values)
{
if (diff.ContainsKey(m))
{
diff[m] += 2;
}
else
{
diff[m] = 2;
}
}
}
// replace the table
_memberTable = table;
// notify the load balancer of the new list of members
_loadBalancer.NotifyMembers(members.Select(x => x.Id));
// signal once
if (Interlocked.CompareExchange(ref _firstMembersViewed, 1, 0) == 0)
{
_firstMembersView.Release();
}
// process changes, gather events
var added = new List <MemberInfo>();
var removed = new List <MemberInfo>();
foreach (var(member, status) in diff)
{
switch (status)
{
case 1: // old but not new = removed
HConsole.WriteLine(this, $"Removed member {member.Id}");
removed.Add(member);
if (_connections.TryGetValue(member.Id, out var client))
{
await client.TerminateAsync().CfAwait(); // TODO: consider dying in the background?
}
break;
case 2: // new but not old = added
HConsole.WriteLine(this, $"Added member {member.Id}");
added.Add(member);
break;
case 3: // old and new = no change
break;
default:
throw new NotSupportedException();
}
}
return(new MembersUpdatedEventArgs(added, removed, table.Members.Values));
}
/// <summary>
/// Set the members.
/// </summary>
/// <param name="version">The version.</param>
/// <param name="members">The members.</param>
/// <returns>The corresponding event arguments, if members were updated; otherwise <c>null</c>.</returns>
public MembersUpdatedEventArgs SetMembers(int version, ICollection <MemberInfo> members)
{
// skip old sets
if (version < _members.Version)
{
return(null);
}
// replace the table
var previous = _members;
var table = new MemberTable(version, members);
lock (_mutex) _members = table;
// notify the load balancer of the new list of members
// (the load balancer can always return a member that is not a member
// anymore, see note in GetMember)
_loadBalancer.SetMembers(members.Select(x => x.Id));
// compute changes
// count 1 for old members, 2 for new members, and then the result is
// 1=removed, 2=added, 3=unchanged
// MemberInfo overrides GetHashCode and can be used as a key here
var diff = new Dictionary <MemberInfo, int>();
if (previous == null)
{
foreach (var m in members)
{
diff[m] = 2;
}
}
else
{
foreach (var m in previous.Members)
{
diff[m] = 1;
}
foreach (var m in members)
{
if (diff.ContainsKey(m))
{
diff[m] += 2;
}
else
{
diff[m] = 2;
}
}
}
// log, if the members have changed (one of them at least is not 3=unchanged)
if (_logger.IsEnabled(LogLevel.Information) && diff.Any(d => d.Value != 3))
{
LogDiffs(table, diff);
}
// process changes, gather events
var added = new List <MemberInfo>();
var removed = new List <MemberInfo>();
foreach (var(member, status) in diff) // all members, old and new
{
switch (status)
{
case 1: // old but not new = removed
HConsole.WriteLine(this, $"Removed member {member.Id} at {member.Address}");
removed.Add(member);
// dequeue the member
_memberConnectionQueue?.Remove(member.Id);
break;
case 2: // new but not old = added
HConsole.WriteLine(this, $"Added member {member.Id} at {member.Address}");
added.Add(member);
// queue the member for connection
_memberConnectionQueue?.Add(member);
break;
case 3: // old and new = no change
break;
default:
throw new NotSupportedException();
}
}
var maybeDisconnected = false;
lock (_mutex)
{
// removed members need to have their connection removed and terminated
foreach (var member in removed)
{
if (_connections.TryGetValue(member.Id, out var c))
{
_connections.Remove(member.Id);
_terminateConnections.Add(c);
}
}
var isAnyMemberConnected = _members.Members.Any(x => _connections.ContainsKey(x.Id));
if (!_connected)
{
if (isAnyMemberConnected)
{
// if we were not connected and now one member happens to be connected then we are now connected
// we hold the mutex so nothing bad can happen
_logger.LogDebug($"Set members: {removed.Count} removed, {added.Count} added, {members.Count} total and at least one is connected, now connected.");
_clusterState.ChangeState(ClientState.Connected, ClientState.Started, ClientState.Disconnected);
_connected = true;
}
else
{
// remain disconnected
_logger.LogDebug($"Set members: {removed.Count} removed, {added.Count} added, {members.Count} total and none is connected, remain disconnected.");
}
}
else
{
if (isAnyMemberConnected)
{
// remain connected
_logger.LogDebug($"Set members: {removed.Count} removed, {added.Count} added, {members.Count} total and at least one is connected, remain connected.");
}
else
{
// we probably are disconnected now
// but the connection queue is running and might have re-added a member
maybeDisconnected = true;
}
}
}
// release _mutex, suspend the queue
if (maybeDisconnected)
{
_memberConnectionQueue?.Suspend();
var disconnected = false;
try
{
lock (_mutex)
{
var isAnyMemberConnected = _members.Members.Any(x => _connections.ContainsKey(x.Id));
if (!isAnyMemberConnected)
{
// no more connected member, we are now disconnected
_logger.LogDebug($"Set members: {removed.Count} removed, {added.Count} added, {members.Count} total and none connected, disconnecting.");
_clusterState.ChangeState(ClientState.Disconnected, ClientState.Connected);
_connected = false;
disconnected = true;
}
else
{
_logger.LogDebug($"Set members: {removed.Count} removed, {added.Count} added, {members.Count} total and at least one is connected, remain connected.");
}
}
}
finally
{
_memberConnectionQueue?.Resume(disconnected);
}
}
return(new MembersUpdatedEventArgs(added, removed, members.ToList()));
}
/// <summary>
/// Notifies the member service of a new cluster.
/// </summary>
/// <remarks>
/// <para>This method should be invoked within the global cluster lock.</para>
/// </remarks>
public void NotifyNewCluster()
{
// the table should be empty anyways - a new cluster means we just established
// the very first connection to a different cluster - so there were no members left
_memberTable = new MemberTable(0, Array.Empty <MemberInfo>());
}
