/// <summary> /// (thread-unsafe) Notifies that a connection has been opened. /// </summary> /// <param name="connection">The connection.</param> /// <returns><c>true</c> if the connection is the first one to be established; otherwise <c>false</c>.</returns> /// <remarks> /// <para>This method is not thread-safe; the caller has to lock the /// <see cref="Mutex"/> object to ensure thread-safety.</para> /// </remarks> public bool NotifyConnectionOpened(MemberConnection connection) { var isFirst = _connections.Count == 0; #if NETSTANDARD2_0 var contains = _connections.ContainsKey(connection.MemberId); _connections[connection.MemberId] = connection; if (contains) #else if (!_connections.TryAdd(connection.MemberId, connection)) #endif { throw new HazelcastException("Failed to add a connection (duplicate memberId)."); } if (_clusterId == default) { _clusterId = connection.ClusterId; // first cluster } else if (_clusterId != connection.ClusterId) { // see TcpClientConnectionManager java class handleSuccessfulAuth method // does not even consider the cluster identifier when !isFirst if (isFirst) { _clusterId = connection.ClusterId; // new cluster _memberTable = new MemberTable(); } } return(isFirst); }
/// <summary> /// Adds a connection. /// </summary> /// <param name="connection">The connection.</param> /// <param name="isNewCluster">Whether the connection is the first connection to a new cluster.</param> public void AddConnection(MemberConnection connection, bool isNewCluster) { // accept every connection, regardless of whether there is a known corresponding member, // since the first connection is going to come before we get the first members view. lock (_mutex) { // don't add the connection if it is not active - if it *is* active, it still // could turn not-active anytime, but thanks to _mutex that will only happen // after the connection has been added if (!connection.Active) { return; } var contains = _connections.ContainsKey(connection.MemberId); if (contains) { // we cannot accept this connection, it's a duplicate (internal error?) _logger.LogWarning($"Cannot accept connection {connection.Id.ToShortString()} to member {connection.MemberId.ToShortString()}, a connection to that member already exists."); _terminateConnections.Add(connection); // kill.kill.kill return; } // add the connection _connections[connection.MemberId] = connection; if (isNewCluster) { // reset members // this is safe because... isNewCluster means that this is the very first connection and there are // no other connections yet and therefore we should not receive events and therefore no one // should invoke SetMembers. // TODO: what if and "old" membersUpdated event is processed? _members = new MemberTable(); } // if this is a true member connection if (_members.ContainsMember(connection.MemberId)) { // if this is the first connection to an actual member, change state & trigger event if (!_connected) { // change Started | Disconnected -> Connected, ignore otherwise, it could be ShuttingDown or Shutdown _logger.LogDebug($"Added connection {connection.Id.ToShortString()} to member {connection.MemberId.ToShortString()}, now connected."); _clusterState.ChangeState(ClientState.Connected, ClientState.Started, ClientState.Disconnected); _connected = true; } else { _logger.LogDebug($"Added connection {connection.Id.ToShortString()} to member {connection.MemberId.ToShortString()}."); } } } }
/// <summary> /// Initializes a new instance of the <see cref="ClusterMembers"/> class. /// </summary> /// <param name="clusterState">The cluster state.</param> /// <param name="terminateConnections">The terminate connections task.</param> public ClusterMembers(ClusterState clusterState, TerminateConnections terminateConnections) { HConsole.Configure(x => x.Configure <ClusterMembers>().SetPrefix("CLUST.MBRS")); _clusterState = clusterState; _terminateConnections = terminateConnections; _loadBalancer = clusterState.Options.LoadBalancer.Service ?? new RandomLoadBalancer(); _logger = _clusterState.LoggerFactory.CreateLogger <ClusterMembers>(); _members = new MemberTable(); // members to connect if (clusterState.IsSmartRouting) { _memberConnectionQueue = new MemberConnectionQueue(clusterState.LoggerFactory); } }
private void LogDiffs(MemberTable table, Dictionary <MemberInfo, int> diff) { var msg = new StringBuilder(); msg.Append("Members ["); msg.Append(table.Count); msg.AppendLine("] {"); foreach (var member in table.Members) { msg.Append(" "); msg.Append(member.Address); msg.Append(" - "); msg.Append(member.Id); if (diff.TryGetValue(member, out var d) && d == 2) { msg.Append(" - new"); } msg.AppendLine(); } msg.Append('}'); _logger.LogInformation(msg.ToString()); }
/// <summary> /// Notifies of a 'members view' event. /// </summary> /// <param name="version">The version.</param> /// <param name="members">The members.</param> public async ValueTask <MembersUpdatedEventArgs> NotifyMembersView(int version, ICollection <MemberInfo> members) { // FIXME could we process two of these at a time? // get a new table var table = new MemberTable(version, members); // compute changes // count 1 for old members, 2 for new members, and then the result is // that 1=removed, 2=added, 3=unchanged // MemberInfo overrides GetHashCode and can be used as a key here var diff = new Dictionary <MemberInfo, int>(); if (_memberTable == null) { foreach (var m in table.Members.Values) { diff[m] = 2; } } else { foreach (var m in _memberTable.Members.Values) { diff[m] = 1; } foreach (var m in table.Members.Values) { if (diff.ContainsKey(m)) { diff[m] += 2; } else { diff[m] = 2; } } } // replace the table _memberTable = table; // notify the load balancer of the new list of members _loadBalancer.NotifyMembers(members.Select(x => x.Id)); // signal once if (Interlocked.CompareExchange(ref _firstMembersViewed, 1, 0) == 0) { _firstMembersView.Release(); } // process changes, gather events var added = new List <MemberInfo>(); var removed = new List <MemberInfo>(); foreach (var(member, status) in diff) { switch (status) { case 1: // old but not new = removed HConsole.WriteLine(this, $"Removed member {member.Id}"); removed.Add(member); if (_connections.TryGetValue(member.Id, out var client)) { await client.TerminateAsync().CfAwait(); // TODO: consider dying in the background? } break; case 2: // new but not old = added HConsole.WriteLine(this, $"Added member {member.Id}"); added.Add(member); break; case 3: // old and new = no change break; default: throw new NotSupportedException(); } } return(new MembersUpdatedEventArgs(added, removed, table.Members.Values)); }
/// <summary> /// Set the members. /// </summary> /// <param name="version">The version.</param> /// <param name="members">The members.</param> /// <returns>The corresponding event arguments, if members were updated; otherwise <c>null</c>.</returns> public MembersUpdatedEventArgs SetMembers(int version, ICollection <MemberInfo> members) { // skip old sets if (version < _members.Version) { return(null); } // replace the table var previous = _members; var table = new MemberTable(version, members); lock (_mutex) _members = table; // notify the load balancer of the new list of members // (the load balancer can always return a member that is not a member // anymore, see note in GetMember) _loadBalancer.SetMembers(members.Select(x => x.Id)); // compute changes // count 1 for old members, 2 for new members, and then the result is // 1=removed, 2=added, 3=unchanged // MemberInfo overrides GetHashCode and can be used as a key here var diff = new Dictionary <MemberInfo, int>(); if (previous == null) { foreach (var m in members) { diff[m] = 2; } } else { foreach (var m in previous.Members) { diff[m] = 1; } foreach (var m in members) { if (diff.ContainsKey(m)) { diff[m] += 2; } else { diff[m] = 2; } } } // log, if the members have changed (one of them at least is not 3=unchanged) if (_logger.IsEnabled(LogLevel.Information) && diff.Any(d => d.Value != 3)) { LogDiffs(table, diff); } // process changes, gather events var added = new List <MemberInfo>(); var removed = new List <MemberInfo>(); foreach (var(member, status) in diff) // all members, old and new { switch (status) { case 1: // old but not new = removed HConsole.WriteLine(this, $"Removed member {member.Id} at {member.Address}"); removed.Add(member); // dequeue the member _memberConnectionQueue?.Remove(member.Id); break; case 2: // new but not old = added HConsole.WriteLine(this, $"Added member {member.Id} at {member.Address}"); added.Add(member); // queue the member for connection _memberConnectionQueue?.Add(member); break; case 3: // old and new = no change break; default: throw new NotSupportedException(); } } var maybeDisconnected = false; lock (_mutex) { // removed members need to have their connection removed and terminated foreach (var member in removed) { if (_connections.TryGetValue(member.Id, out var c)) { _connections.Remove(member.Id); _terminateConnections.Add(c); } } var isAnyMemberConnected = _members.Members.Any(x => _connections.ContainsKey(x.Id)); if (!_connected) { if (isAnyMemberConnected) { // if we were not connected and now one member happens to be connected then we are now connected // we hold the mutex so nothing bad can happen _logger.LogDebug($"Set members: {removed.Count} removed, {added.Count} added, {members.Count} total and at least one is connected, now connected."); _clusterState.ChangeState(ClientState.Connected, ClientState.Started, ClientState.Disconnected); _connected = true; } else { // remain disconnected _logger.LogDebug($"Set members: {removed.Count} removed, {added.Count} added, {members.Count} total and none is connected, remain disconnected."); } } else { if (isAnyMemberConnected) { // remain connected _logger.LogDebug($"Set members: {removed.Count} removed, {added.Count} added, {members.Count} total and at least one is connected, remain connected."); } else { // we probably are disconnected now // but the connection queue is running and might have re-added a member maybeDisconnected = true; } } } // release _mutex, suspend the queue if (maybeDisconnected) { _memberConnectionQueue?.Suspend(); var disconnected = false; try { lock (_mutex) { var isAnyMemberConnected = _members.Members.Any(x => _connections.ContainsKey(x.Id)); if (!isAnyMemberConnected) { // no more connected member, we are now disconnected _logger.LogDebug($"Set members: {removed.Count} removed, {added.Count} added, {members.Count} total and none connected, disconnecting."); _clusterState.ChangeState(ClientState.Disconnected, ClientState.Connected); _connected = false; disconnected = true; } else { _logger.LogDebug($"Set members: {removed.Count} removed, {added.Count} added, {members.Count} total and at least one is connected, remain connected."); } } } finally { _memberConnectionQueue?.Resume(disconnected); } } return(new MembersUpdatedEventArgs(added, removed, members.ToList())); }
/// <summary> /// Notifies the member service of a new cluster. /// </summary> /// <remarks> /// <para>This method should be invoked within the global cluster lock.</para> /// </remarks> public void NotifyNewCluster() { // the table should be empty anyways - a new cluster means we just established // the very first connection to a different cluster - so there were no members left _memberTable = new MemberTable(0, Array.Empty <MemberInfo>()); }