public void ExecuteTransaction(Action <MessageExchangeProtocol> protocolHandler) { var retryInterval = HalibutLimits.RetryListeningSleepInterval; Exception lastError = null; // retryAllowed is also used to indicate if the error occurred before or after the connection was made var retryAllowed = true; var watch = Stopwatch.StartNew(); for (var i = 0; i < HalibutLimits.RetryCountLimit && retryAllowed && watch.Elapsed < HalibutLimits.ConnectionErrorRetryTimeout; i++) { if (i > 0) { log.Write(EventType.Error, "Retry attempt {0}", i); } try { lastError = null; IConnection connection = null; try { connection = AcquireConnection(); // Beyond this point, we have no way to be certain that the server hasn't tried to process a request; therefore, we can't retry after this point retryAllowed = false; protocolHandler(connection.Protocol); } catch { connection?.Dispose(); throw; } // Only return the connection to the pool if all went well ReleaseConnection(connection); } catch (AuthenticationException aex) { log.WriteException(EventType.Error, $"Authentication failed while setting up connection to {(serviceEndpoint == null ? "(Null EndPoint)" : serviceEndpoint.BaseUri.ToString())}", aex); lastError = aex; retryAllowed = false; break; } catch (SocketException cex) when(cex.SocketErrorCode == SocketError.ConnectionRefused) { log.Write(EventType.Error, $"The remote host at {(serviceEndpoint == null ? "(Null EndPoint)" : serviceEndpoint.BaseUri.ToString())} refused the connection, this may mean that the expected listening service is not running."); lastError = cex; } catch (SocketException sex) { log.WriteException(EventType.Error, $"Socket communication error with connection to {(serviceEndpoint == null ? "(Null EndPoint)" : serviceEndpoint.BaseUri.ToString())}", sex); lastError = sex; // When the host is not found an immediate retry isn't going to help if (sex.SocketErrorCode == SocketError.HostNotFound) { break; } } catch (ConnectionInitializationFailedException cex) { log.WriteException(EventType.Error, $"Connection initialization failed while connecting to {(serviceEndpoint == null ? "(Null EndPoint)" : serviceEndpoint.BaseUri.ToString())}", cex); lastError = cex; retryAllowed = true; // If this is the second failure, clear the pooled connections as a precaution // against all connections in the pool being bad if (i == 1) { pool.Clear(serviceEndpoint, log); } Thread.Sleep(retryInterval); } catch (IOException iox) when(iox.IsSocketConnectionReset()) { log.Write(EventType.Error, $"The remote host at {(serviceEndpoint == null ? "(Null EndPoint)" : serviceEndpoint.BaseUri.ToString())} reset the connection, this may mean that the expected listening service does not trust the thumbprint {clientCertificate.Thumbprint} or was shut down."); lastError = iox; Thread.Sleep(retryInterval); } catch (IOException iox) when(iox.IsSocketConnectionTimeout()) { // Received on a polling client when the network connection is lost. log.Write(EventType.Error, $"The connection to the host at {(serviceEndpoint == null ? "(Null EndPoint)" : serviceEndpoint.BaseUri.ToString())} timed out, there may be problems with the network, connection will be retried."); lastError = iox; Thread.Sleep(retryInterval); } catch (Exception ex) { log.WriteException(EventType.Error, "Unexpected exception executing transaction.", ex); lastError = ex; Thread.Sleep(retryInterval); } } HandleError(lastError, retryAllowed); }
public void ExecuteTransaction(Action <MessageExchangeProtocol> protocolHandler) { var retryInterval = HalibutLimits.RetryListeningSleepInterval; Exception lastError = null; // retryAllowed is also used to indicate if the error occurred before or after the connection was made var retryAllowed = true; var watch = Stopwatch.StartNew(); for (var i = 0; i < RetryCountLimit && retryAllowed && watch.Elapsed < HalibutLimits.ConnectionErrorRetryTimeout; i++) { if (i > 0) { log.Write(EventType.Error, "Retry attempt {0}", i); } try { lastError = null; var connection = AcquireConnection(); // Beyond this point, we have no way to be certain that the server hasn't tried to process a request; therefore, we can't retry after this point retryAllowed = false; protocolHandler(connection.Protocol); // Only return the connection to the pool if all went well ReleaseConnection(connection); } catch (AuthenticationException aex) { log.WriteException(EventType.Error, aex.Message, aex); lastError = aex; retryAllowed = false; break; } catch (SocketException sex) { log.WriteException(EventType.Error, sex.Message, sex); lastError = sex; // When the host is not found an immediate retry isn't going to help if (sex.SocketErrorCode == SocketError.HostNotFound) { break; } } catch (ConnectionInitializationFailedException cex) { log.WriteException(EventType.Error, cex.Message, cex); lastError = cex; retryAllowed = true; // If this is the second failure, clear the pooled connections as a precaution // against all connections in the pool being bad if (i == 1) { pool.Clear(serviceEndpoint); } Thread.Sleep(retryInterval); } catch (Exception ex) { log.WriteException(EventType.Error, ex.Message, ex); lastError = ex; Thread.Sleep(retryInterval); } } HandleError(lastError, retryAllowed); }