/// <summary> /// Provides a background thread to reconnect failed node connections. /// </summary> /// <param name="state">The connection to re-open</param> private void ReconnectChildConnection(object state) { // Get connection object from the state object passed by the thread pool. VoltNodeConnection connection = state as VoltNodeConnection; // Try to reconnect until the connection is actively closed by the user or terminated (when all child // connections died and none could be re-opened) while (this.Status == ConnectionStatus.Connected) { try { // Attempt to open the connection connection.Open(); // If somebody tried to swap out the node from under us to start a different cluster, return // immediately: we will not reconnect there! if ( (this.BuildString != connection.BuildString) || (!this.LeaderIPEndPoint.Equals(connection.LeaderIPEndPoint)) || (this.ClusterStartTimeStamp != connection.ClusterStartTimeStamp) ) { connection.Terminate(); return; } // Refresh pool status to re-enlist the connection into active duty. this.RefreshConnectionPoolStatus(); // End the thread: connection successfully re-added to the pool. return; } catch { // Sleep a while to give the node time to rejoin if needed. Thread.Sleep(this.Settings.ConnectionTimeout); } } }
/// <summary> /// Async sub-connection opening - open a child connection and add to the pool when successful. /// </summary> /// <param name="settings">Settings to use for the child connection to open.</param> private void OpenChildConnection(ConnectionSettings settings) { // Return immediately if there was a terminal connection failure on any of the parallel attempts. if (Interlocked.Read(ref this.ConnectionExceptionCount) > 0) { return; } try { // Try to open the node connection. VoltNodeConnection connection = new VoltNodeConnection(settings, this.CallbackExecutor); connection.Open(); // Lock the connection pool for validation and addition. lock ((this.ConnectionPool as IList).SyncRoot) { // Validate new node against previous connections: we will refuse any connection to an inconsistent // cluster (different versions, different application, etc). if (this.ConnectionPool.Count > 0) { if ( (this.BuildString != connection.BuildString) || (!this.LeaderIPEndPoint.Equals(connection.LeaderIPEndPoint)) || (this.ClusterStartTimeStamp != connection.ClusterStartTimeStamp) ) { // This node doesn't belong to the cluster - terminate the connection and throw out! try { connection.Terminate(); } catch {} throw new VoltClusterConnectionException( Resources.InconsistentClusterTarget , string.Join("\r\n" , this.ConnectionPool.Select(i => string.Format( Resources.InconsistentClusterTargetListing , i.ServerHostId , i.ConnectionId , i.IPEndPoint , i.LeaderIPEndPoint , i.ClusterStartTimeStamp , i.BuildString ) ).ToArray() ) , string.Format( Resources.InconsistentClusterTargetListing , connection.ServerHostId , connection.ConnectionId , connection.IPEndPoint , connection.LeaderIPEndPoint , connection.ClusterStartTimeStamp , connection.BuildString ) ); } } else { // Grab cluster details after the first connection to compare future connections against. this.BuildString = connection.BuildString; this.ClusterStartTimeStamp = connection.ClusterStartTimeStamp; this.LeaderIPEndPoint = connection.LeaderIPEndPoint; } // Connection is valid, add it to the pool this.ConnectionPool.Add(connection); } } catch (Exception x) { // Flag terminal exceptions: in this case we do NOT want to proceed no matter if a single node ever // replied. if ((x is VoltClusterConnectionException) || this.Settings.ConnectToAllOrNone) { Interlocked.Increment(ref this.ConnectionExceptionCount); } // Add the exception to the list for later reporting. lock ((this.ConnectionExceptionList as IList).SyncRoot) this.ConnectionExceptionList.Add(x); } }