/// <summary>
        /// Adds a connection.
        /// </summary>
        /// <param name="connection">The connection.</param>
        /// <param name="isNewCluster">Whether the connection is the first connection to a new cluster.</param>
        public void AddConnection(MemberConnection connection, bool isNewCluster)
        {
            // accept every connection, regardless of whether there is a known corresponding member,
            // since the first connection is going to come before we get the first members view.

            lock (_mutex)
            {
                // don't add the connection if it is not active - if it *is* active, it still
                // could turn not-active anytime, but thanks to _mutex that will only happen
                // after the connection has been added
                if (!connection.Active)
                {
                    return;
                }

                var contains = _connections.ContainsKey(connection.MemberId);

                if (contains)
                {
                    // we cannot accept this connection, it's a duplicate (internal error?)
                    _logger.LogWarning($"Cannot accept connection {connection.Id.ToShortString()} to member {connection.MemberId.ToShortString()}, a connection to that member already exists.");
                    _terminateConnections.Add(connection); // kill.kill.kill
                    return;
                }

                // add the connection
                _connections[connection.MemberId] = connection;

                if (isNewCluster)
                {
                    // reset members
                    // this is safe because... isNewCluster means that this is the very first connection and there are
                    // no other connections yet and therefore we should not receive events and therefore no one
                    // should invoke SetMembers.
                    // TODO: what if and "old" membersUpdated event is processed?
                    _members = new MemberTable();
                }

                // if this is a true member connection
                if (_members.ContainsMember(connection.MemberId))
                {
                    // if this is the first connection to an actual member, change state & trigger event
                    if (!_connected)
                    {
                        // change Started | Disconnected -> Connected, ignore otherwise, it could be ShuttingDown or Shutdown
                        _logger.LogDebug($"Added connection {connection.Id.ToShortString()} to member {connection.MemberId.ToShortString()}, now connected.");
                        _clusterState.ChangeState(ClientState.Connected, ClientState.Started, ClientState.Disconnected);
                        _connected = true;
                    }
                    else
                    {
                        _logger.LogDebug($"Added connection {connection.Id.ToShortString()} to member {connection.MemberId.ToShortString()}.");
                    }
                }
            }
        }
Esempio n. 2
0
        // (background) adds subscriptions on one member - when a connection is added
        private async Task AddSubscriptionsAsync(MemberConnection connection, IReadOnlyCollection <ClusterSubscription> subscriptions, CancellationToken cancellationToken)
        {
            // this is a background task and therefore should never throw!

            foreach (var subscription in subscriptions)
            {
                if (cancellationToken.IsCancellationRequested)
                {
                    return;
                }

                // this never throws
                var attempt = await AddSubscriptionAsync(subscription, connection, cancellationToken).CfAwait();

                switch (attempt.Value)
                {
                case InstallResult.Success:               // ok
                case InstallResult.SubscriptionNotActive: // ignore it
                    continue;

                case InstallResult.ConnectionNotActive:
                    // not active = has been removed = what has been done already has been undone
                    break;     // simply exit

                case InstallResult.Failed:
                    // failed to talk to the server - this connection is not working
                    _terminateConnections.Add(connection);
                    break;     // exit

                default:
                    continue;
                }
            }

            // we are done now
            lock (_subscribeTasksMutex) _subscribeTasks.Remove(connection);
        }
Esempio n. 3
0
        // runs once on a connection to a member
        private async Task RunAsync(MemberConnection connection, DateTime now, CancellationToken cancellationToken)
        {
            var readElapsed  = now - connection.LastReadTime;
            var writeElapsed = now - connection.LastWriteTime;

            HConsole.WriteLine(this, $"Heartbeat {_clusterState.ClientName} on {connection.Id.ToShortString()} to {connection.MemberId.ToShortString()} at {connection.Address}, " +
                               $"written {(int)writeElapsed.TotalSeconds}s ago, " +
                               $"read {(int)readElapsed.TotalSeconds}s ago");

            // make sure we read from the client at least every 'timeout',
            // which is greater than the interval, so we *should* have
            // read from the last ping, if nothing else, so no read means
            // that the client not responsive - terminate it

            if (readElapsed > _timeout && writeElapsed < _period)
            {
                _logger.LogWarning("Heartbeat timeout for connection {ConnectionId}, terminating.", connection.Id.ToShortString());
                if (connection.Active)
                {
                    _terminateConnections.Add(connection);
                }
                return;
            }

            // make sure we write to the client at least every 'period',
            // this should trigger a read when we receive the response
            if (writeElapsed > _period)
            {
                _logger.LogDebug("Ping client {ClientId}", connection.Id.ToShortString());

                var requestMessage = ClientPingCodec.EncodeRequest();

                try
                {
                    // ping should complete within the default invocation timeout
                    var responseMessage = await _clusterMessaging
                                          .SendToMemberAsync(requestMessage, connection, cancellationToken)
                                          .CfAwait();

                    // just to be sure everything is ok
                    _ = ClientPingCodec.DecodeResponse(responseMessage);
                }
                catch (ClientOfflineException)
                {
                    // down
                }
                catch (TaskTimeoutException)
                {
                    _logger.LogWarning("Heartbeat ping timeout for connection {ConnectionId}, terminating.", connection.Id.ToShortString());
                    if (connection.Active)
                    {
                        _terminateConnections.Add(connection);
                    }
                }
                catch (Exception e)
                {
                    // unexpected
                    _logger.LogWarning(e, "Heartbeat has thrown an exception, but will continue.");
                }
            }
        }