Beispiel #1
0
        /// <summary>
        /// (thread-unsafe) Notifies that a connection has been opened.
        /// </summary>
        /// <param name="connection">The connection.</param>
        /// <returns><c>true</c> if the connection is the first one to be established; otherwise <c>false</c>.</returns>
        /// <remarks>
        /// <para>This method is not thread-safe; the caller has to lock the
        /// <see cref="Mutex"/> object to ensure thread-safety.</para>
        /// </remarks>
        public bool NotifyConnectionOpened(MemberConnection connection)
        {
            var isFirst = _connections.Count == 0;

#if NETSTANDARD2_0
            var contains = _connections.ContainsKey(connection.MemberId);
            _connections[connection.MemberId] = connection;
            if (contains)
#else
            if (!_connections.TryAdd(connection.MemberId, connection))
#endif
            { throw new HazelcastException("Failed to add a connection (duplicate memberId)."); }

            if (_clusterId == default)
            {
                _clusterId = connection.ClusterId; // first cluster
            }
            else if (_clusterId != connection.ClusterId)
            {
                // see TcpClientConnectionManager java class handleSuccessfulAuth method
                // does not even consider the cluster identifier when !isFirst
                if (isFirst)
                {
                    _clusterId   = connection.ClusterId; // new cluster
                    _memberTable = new MemberTable();
                }
            }

            return(isFirst);
        }
        /// <summary>
        /// Adds a connection.
        /// </summary>
        /// <param name="connection">The connection.</param>
        /// <param name="isNewCluster">Whether the connection is the first connection to a new cluster.</param>
        public void AddConnection(MemberConnection connection, bool isNewCluster)
        {
            // accept every connection, regardless of whether there is a known corresponding member,
            // since the first connection is going to come before we get the first members view.

            lock (_mutex)
            {
                // don't add the connection if it is not active - if it *is* active, it still
                // could turn not-active anytime, but thanks to _mutex that will only happen
                // after the connection has been added
                if (!connection.Active)
                {
                    return;
                }

                var contains = _connections.ContainsKey(connection.MemberId);

                if (contains)
                {
                    // we cannot accept this connection, it's a duplicate (internal error?)
                    _logger.LogWarning($"Cannot accept connection {connection.Id.ToShortString()} to member {connection.MemberId.ToShortString()}, a connection to that member already exists.");
                    _terminateConnections.Add(connection); // kill.kill.kill
                    return;
                }

                // add the connection
                _connections[connection.MemberId] = connection;

                if (isNewCluster)
                {
                    // reset members
                    // this is safe because... isNewCluster means that this is the very first connection and there are
                    // no other connections yet and therefore we should not receive events and therefore no one
                    // should invoke SetMembers.
                    // TODO: what if and "old" membersUpdated event is processed?
                    _members = new MemberTable();
                }

                // if this is a true member connection
                if (_members.ContainsMember(connection.MemberId))
                {
                    // if this is the first connection to an actual member, change state & trigger event
                    if (!_connected)
                    {
                        // change Started | Disconnected -> Connected, ignore otherwise, it could be ShuttingDown or Shutdown
                        _logger.LogDebug($"Added connection {connection.Id.ToShortString()} to member {connection.MemberId.ToShortString()}, now connected.");
                        _clusterState.ChangeState(ClientState.Connected, ClientState.Started, ClientState.Disconnected);
                        _connected = true;
                    }
                    else
                    {
                        _logger.LogDebug($"Added connection {connection.Id.ToShortString()} to member {connection.MemberId.ToShortString()}.");
                    }
                }
            }
        }
        /// <summary>
        /// Initializes a new instance of the <see cref="ClusterMembers"/> class.
        /// </summary>
        /// <param name="clusterState">The cluster state.</param>
        /// <param name="terminateConnections">The terminate connections task.</param>
        public ClusterMembers(ClusterState clusterState, TerminateConnections terminateConnections)
        {
            HConsole.Configure(x => x.Configure <ClusterMembers>().SetPrefix("CLUST.MBRS"));

            _clusterState         = clusterState;
            _terminateConnections = terminateConnections;
            _loadBalancer         = clusterState.Options.LoadBalancer.Service ?? new RandomLoadBalancer();

            _logger = _clusterState.LoggerFactory.CreateLogger <ClusterMembers>();

            _members = new MemberTable();

            // members to connect
            if (clusterState.IsSmartRouting)
            {
                _memberConnectionQueue = new MemberConnectionQueue(clusterState.LoggerFactory);
            }
        }
        private void LogDiffs(MemberTable table, Dictionary <MemberInfo, int> diff)
        {
            var msg = new StringBuilder();

            msg.Append("Members [");
            msg.Append(table.Count);
            msg.AppendLine("] {");
            foreach (var member in table.Members)
            {
                msg.Append("    ");
                msg.Append(member.Address);
                msg.Append(" - ");
                msg.Append(member.Id);
                if (diff.TryGetValue(member, out var d) && d == 2)
                {
                    msg.Append(" - new");
                }
                msg.AppendLine();
            }
            msg.Append('}');

            _logger.LogInformation(msg.ToString());
        }
Beispiel #5
0
        /// <summary>
        /// Notifies of a 'members view' event.
        /// </summary>
        /// <param name="version">The version.</param>
        /// <param name="members">The members.</param>
        public async ValueTask <MembersUpdatedEventArgs> NotifyMembersView(int version, ICollection <MemberInfo> members)
        {
            // FIXME could we process two of these at a time?

            // get a new table
            var table = new MemberTable(version, members);

            // compute changes
            // count 1 for old members, 2 for new members, and then the result is
            // that 1=removed, 2=added, 3=unchanged
            // MemberInfo overrides GetHashCode and can be used as a key here
            var diff = new Dictionary <MemberInfo, int>();

            if (_memberTable == null)
            {
                foreach (var m in table.Members.Values)
                {
                    diff[m] = 2;
                }
            }
            else
            {
                foreach (var m in _memberTable.Members.Values)
                {
                    diff[m] = 1;
                }
                foreach (var m in table.Members.Values)
                {
                    if (diff.ContainsKey(m))
                    {
                        diff[m] += 2;
                    }
                    else
                    {
                        diff[m] = 2;
                    }
                }
            }

            // replace the table
            _memberTable = table;

            // notify the load balancer of the new list of members
            _loadBalancer.NotifyMembers(members.Select(x => x.Id));

            // signal once
            if (Interlocked.CompareExchange(ref _firstMembersViewed, 1, 0) == 0)
            {
                _firstMembersView.Release();
            }

            // process changes, gather events
            var added   = new List <MemberInfo>();
            var removed = new List <MemberInfo>();

            foreach (var(member, status) in diff)
            {
                switch (status)
                {
                case 1:     // old but not new = removed
                    HConsole.WriteLine(this, $"Removed member {member.Id}");
                    removed.Add(member);
                    if (_connections.TryGetValue(member.Id, out var client))
                    {
                        await client.TerminateAsync().CfAwait();     // TODO: consider dying in the background?
                    }
                    break;

                case 2:     // new but not old = added
                    HConsole.WriteLine(this, $"Added member {member.Id}");
                    added.Add(member);
                    break;

                case 3:     // old and new = no change
                    break;

                default:
                    throw new NotSupportedException();
                }
            }

            return(new MembersUpdatedEventArgs(added, removed, table.Members.Values));
        }
        /// <summary>
        /// Set the members.
        /// </summary>
        /// <param name="version">The version.</param>
        /// <param name="members">The members.</param>
        /// <returns>The corresponding event arguments, if members were updated; otherwise <c>null</c>.</returns>
        public MembersUpdatedEventArgs SetMembers(int version, ICollection <MemberInfo> members)
        {
            // skip old sets
            if (version < _members.Version)
            {
                return(null);
            }

            // replace the table
            var previous = _members;
            var table    = new MemberTable(version, members);

            lock (_mutex) _members = table;

            // notify the load balancer of the new list of members
            // (the load balancer can always return a member that is not a member
            // anymore, see note in GetMember)
            _loadBalancer.SetMembers(members.Select(x => x.Id));

            // compute changes
            // count 1 for old members, 2 for new members, and then the result is
            // 1=removed, 2=added, 3=unchanged
            // MemberInfo overrides GetHashCode and can be used as a key here
            var diff = new Dictionary <MemberInfo, int>();

            if (previous == null)
            {
                foreach (var m in members)
                {
                    diff[m] = 2;
                }
            }
            else
            {
                foreach (var m in previous.Members)
                {
                    diff[m] = 1;
                }

                foreach (var m in members)
                {
                    if (diff.ContainsKey(m))
                    {
                        diff[m] += 2;
                    }
                    else
                    {
                        diff[m] = 2;
                    }
                }
            }

            // log, if the members have changed (one of them at least is not 3=unchanged)
            if (_logger.IsEnabled(LogLevel.Information) && diff.Any(d => d.Value != 3))
            {
                LogDiffs(table, diff);
            }

            // process changes, gather events
            var added   = new List <MemberInfo>();
            var removed = new List <MemberInfo>();

            foreach (var(member, status) in diff)  // all members, old and new
            {
                switch (status)
                {
                case 1:     // old but not new = removed
                    HConsole.WriteLine(this, $"Removed member {member.Id} at {member.Address}");
                    removed.Add(member);

                    // dequeue the member
                    _memberConnectionQueue?.Remove(member.Id);

                    break;

                case 2:     // new but not old = added
                    HConsole.WriteLine(this, $"Added member {member.Id} at {member.Address}");
                    added.Add(member);

                    // queue the member for connection
                    _memberConnectionQueue?.Add(member);

                    break;

                case 3:     // old and new = no change
                    break;

                default:
                    throw new NotSupportedException();
                }
            }

            var maybeDisconnected = false;

            lock (_mutex)
            {
                // removed members need to have their connection removed and terminated
                foreach (var member in removed)
                {
                    if (_connections.TryGetValue(member.Id, out var c))
                    {
                        _connections.Remove(member.Id);
                        _terminateConnections.Add(c);
                    }
                }

                var isAnyMemberConnected = _members.Members.Any(x => _connections.ContainsKey(x.Id));

                if (!_connected)
                {
                    if (isAnyMemberConnected)
                    {
                        // if we were not connected and now one member happens to be connected then we are now connected
                        // we hold the mutex so nothing bad can happen
                        _logger.LogDebug($"Set members: {removed.Count} removed, {added.Count} added, {members.Count} total and at least one is connected, now connected.");
                        _clusterState.ChangeState(ClientState.Connected, ClientState.Started, ClientState.Disconnected);
                        _connected = true;
                    }
                    else
                    {
                        // remain disconnected
                        _logger.LogDebug($"Set members: {removed.Count} removed, {added.Count} added, {members.Count} total and none is connected, remain disconnected.");
                    }
                }
                else
                {
                    if (isAnyMemberConnected)
                    {
                        // remain connected
                        _logger.LogDebug($"Set members: {removed.Count} removed, {added.Count} added, {members.Count} total and at least one is connected, remain connected.");
                    }
                    else
                    {
                        // we probably are disconnected now
                        // but the connection queue is running and might have re-added a member
                        maybeDisconnected = true;
                    }
                }
            }

            // release _mutex, suspend the queue
            if (maybeDisconnected)
            {
                _memberConnectionQueue?.Suspend();
                var disconnected = false;
                try
                {
                    lock (_mutex)
                    {
                        var isAnyMemberConnected = _members.Members.Any(x => _connections.ContainsKey(x.Id));
                        if (!isAnyMemberConnected)
                        {
                            // no more connected member, we are now disconnected
                            _logger.LogDebug($"Set members: {removed.Count} removed, {added.Count} added, {members.Count} total and none connected, disconnecting.");
                            _clusterState.ChangeState(ClientState.Disconnected, ClientState.Connected);
                            _connected   = false;
                            disconnected = true;
                        }
                        else
                        {
                            _logger.LogDebug($"Set members: {removed.Count} removed, {added.Count} added, {members.Count} total and at least one is connected, remain connected.");
                        }
                    }
                }
                finally
                {
                    _memberConnectionQueue?.Resume(disconnected);
                }
            }

            return(new MembersUpdatedEventArgs(added, removed, members.ToList()));
        }
 /// <summary>
 /// Notifies the member service of a new cluster.
 /// </summary>
 /// <remarks>
 /// <para>This method should be invoked within the global cluster lock.</para>
 /// </remarks>
 public void NotifyNewCluster()
 {
     // the table should be empty anyways - a new cluster means we just established
     // the very first connection to a different cluster - so there were no members left
     _memberTable = new MemberTable(0, Array.Empty <MemberInfo>());
 }