/// <summary>
        /// Provides a background thread to reconnect failed node connections.
        /// </summary>
        /// <param name="state">The connection to re-open</param>
        private void ReconnectChildConnection(object state)
        {
            // Get connection object from the state object passed by the thread pool.
            VoltNodeConnection connection = state as VoltNodeConnection;

            // Try to reconnect until the connection is actively closed by the user or terminated (when all child
            // connections died and none could be re-opened)
            while (this.Status == ConnectionStatus.Connected)
            {
                try
                {
                    // Attempt to open the connection
                    connection.Open();

                    // If somebody tried to swap out the node from under us to start a different cluster, return
                    // immediately: we will not reconnect there!
                    if (
                        (this.BuildString != connection.BuildString) ||
                        (!this.LeaderIPEndPoint.Equals(connection.LeaderIPEndPoint)) ||
                        (this.ClusterStartTimeStamp != connection.ClusterStartTimeStamp)
                        )
                    {
                        connection.Terminate();
                        return;
                    }

                    // Refresh pool status to re-enlist the connection into active duty.
                    this.RefreshConnectionPoolStatus();

                    // End the thread: connection successfully re-added to the pool.
                    return;
                }
                catch
                {
                    // Sleep a while to give the node time to rejoin if needed.
                    Thread.Sleep(this.Settings.ConnectionTimeout);
                }
            }
        }
        /// <summary>
        /// Async sub-connection opening - open a child connection and add to the pool when successful.
        /// </summary>
        /// <param name="settings">Settings to use for the child connection to open.</param>
        private void OpenChildConnection(ConnectionSettings settings)
        {
            // Return immediately if there was a terminal connection failure on any of the parallel attempts.
            if (Interlocked.Read(ref this.ConnectionExceptionCount) > 0)
            {
                return;
            }

            try
            {
                // Try to open the node connection.
                VoltNodeConnection connection = new VoltNodeConnection(settings, this.CallbackExecutor);
                connection.Open();

                // Lock the connection pool for validation and addition.
                lock ((this.ConnectionPool as IList).SyncRoot)
                {
                    // Validate new node against previous connections: we will refuse any connection to an inconsistent
                    // cluster (different versions, different application, etc).
                    if (this.ConnectionPool.Count > 0)
                    {
                        if (
                            (this.BuildString != connection.BuildString) ||
                            (!this.LeaderIPEndPoint.Equals(connection.LeaderIPEndPoint)) ||
                            (this.ClusterStartTimeStamp != connection.ClusterStartTimeStamp)
                            )
                        {
                            // This node doesn't belong to the cluster - terminate the connection and throw out!
                            try
                            {
                                connection.Terminate();
                            }
                            catch {}

                            throw new VoltClusterConnectionException(
                                      Resources.InconsistentClusterTarget

                                      , string.Join("\r\n"
                                                    , this.ConnectionPool.Select(i =>
                                                                                 string.Format(
                                                                                     Resources.InconsistentClusterTargetListing
                                                                                     , i.ServerHostId
                                                                                     , i.ConnectionId
                                                                                     , i.IPEndPoint
                                                                                     , i.LeaderIPEndPoint
                                                                                     , i.ClusterStartTimeStamp
                                                                                     , i.BuildString
                                                                                     )
                                                                                 ).ToArray()
                                                    )

                                      , string.Format(
                                          Resources.InconsistentClusterTargetListing
                                          , connection.ServerHostId
                                          , connection.ConnectionId
                                          , connection.IPEndPoint
                                          , connection.LeaderIPEndPoint
                                          , connection.ClusterStartTimeStamp
                                          , connection.BuildString
                                          )

                                      );
                        }
                    }
                    else
                    {
                        // Grab cluster details after the first connection to compare future connections against.
                        this.BuildString           = connection.BuildString;
                        this.ClusterStartTimeStamp = connection.ClusterStartTimeStamp;
                        this.LeaderIPEndPoint      = connection.LeaderIPEndPoint;
                    }
                    // Connection is valid, add it to the pool
                    this.ConnectionPool.Add(connection);
                }
            }
            catch (Exception x)
            {
                // Flag terminal exceptions: in this case we do NOT want to proceed no matter if a single node ever
                // replied.
                if ((x is VoltClusterConnectionException) || this.Settings.ConnectToAllOrNone)
                {
                    Interlocked.Increment(ref this.ConnectionExceptionCount);
                }

                // Add the exception to the list for later reporting.
                lock ((this.ConnectionExceptionList as IList).SyncRoot)
                    this.ConnectionExceptionList.Add(x);
            }
        }