private async Task <T> InvokeConfiguredRequest <T>(WebRequestState state, CancellationToken cancellationToken) where T : AmazonWebServiceResponse { int currentRetries = state.RetriesAttempt; T response = null; bool shouldRetry = false; bool responseReceived = false; HttpResponseMessage responseMessage = null; HttpClientResponseData responseData = null; var requestMessage = ConfigureRequestMessage(state); try { try { SetContent(requestMessage, state); using (state.Metrics.StartEvent(Metric.HttpRequestTime)) { responseMessage = await httpClient.SendAsync(requestMessage, HttpCompletionOption.ResponseHeadersRead, cancellationToken) .ConfigureAwait(continueOnCapturedContext: false); responseReceived = true; } responseData = new HttpClientResponseData(responseMessage); if (!IsErrorResponse(responseMessage) || responseMessage.StatusCode == HttpStatusCode.NotFound && state.Request.Suppress404Exceptions) { using (state.Metrics.StartEvent(Metric.ResponseProcessingTime)) { response = (T) await HandleHttpContentAsync(state, responseMessage, responseData) .ConfigureAwait(continueOnCapturedContext: false); } } else { bool retry = await HandleHttpErrorResponseAsync(state, responseMessage, responseData, cancellationToken) .ConfigureAwait(continueOnCapturedContext: false); if (retry) { shouldRetry = true; } } } catch (HttpRequestException e) { var we = e.InnerException as WebException; if (we != null) { if (WebExceptionStatusesToThrowOn.Contains(we.Status)) { throw new AmazonServiceException("Encountered a non retryable WebException : " + we.Status, we); } // The WinRT framework doesn't break down errors, not all status values are available for WinRT. if ( #if WIN_RT (we.Status == WebExceptionStatus.UnknownError || WebExceptionStatusesToRetryOn.Contains(we.Status)) #else WebExceptionStatusesToRetryOn.Contains(we.Status) #endif ) { shouldRetry = RetryOrThrow(state, e); } } var ioe = e.InnerException as IOException; if (ioe != null) { #if !WIN_RT if (IsInnerExceptionThreadAbort(ioe)) { throw new AmazonServiceException(e); } #endif shouldRetry = RetryOrThrow(state, e); } // Check if response is null at the end as // it can be null for both WebException and IOException. if (!shouldRetry && response == null) { shouldRetry = RetryOrThrow(state, e); } // If shouldRetry is not set by any of the above checks, // re-throw the exception. if (!shouldRetry) { throw new AmazonServiceException(e); } } catch (TaskCanceledException taskCancelledException) { if (cancellationToken.IsCancellationRequested) { throw; } else { // It's a timeout exception. throw new AmazonServiceException(taskCancelledException); } } catch (OperationCanceledException operationCancelledException) { if (cancellationToken.IsCancellationRequested) { // Throw an exception with the original cancellation token. throw new OperationCanceledException(operationCancelledException.Message, cancellationToken); } else { // It's a timeout exception. throw new AmazonServiceException(operationCancelledException); } } finally { if (responseMessage != null && (response == null || !state.Unmarshaller.HasStreamingProperty)) { responseMessage.Dispose(); responseMessage = null; } } if (shouldRetry) { pauseExponentially(state); cancellationToken.ThrowIfCancellationRequested(); state.RetriesAttempt++; var retryResponse = await InvokeHelper <T>(state, cancellationToken) .ConfigureAwait(continueOnCapturedContext: false); return((T)retryResponse); } else { LogFinalMetrics(state.Metrics); } } catch (Exception e) { // On errors that are passed to the client, invoke exception handlers if (!shouldRetry) { ProcessExceptionHandlers(e, state.Request); } throw; } finally { // ProcessResponseHandlers is called only if a response was received from the server (success or error). // It's not called in case of client errors like timeout or proxy error. if (!shouldRetry && responseReceived) { ProcessResponseHandlers(response, state.Request, responseData); } } return(response); }
/// <summary> /// Determines if an AmazonServiceException is a transient error that /// should be retried. /// </summary> /// <param name="executionContext">The current execution context</param> /// <param name="exception">The current exception to check.</param> /// <returns>true if the exception is a transient error else false.</returns> public virtual bool IsTransientError(IExecutionContext executionContext, Exception exception) { // An IOException was thrown by the underlying http client. if (exception is IOException) { #if !NETSTANDARD // ThreadAbortException is not NetStandard // Don't retry IOExceptions that are caused by a ThreadAbortException if (ExceptionUtils.IsInnerException <ThreadAbortException>(exception)) { return(false); } #endif // Retry all other IOExceptions return(true); } else if (ExceptionUtils.IsInnerException <IOException>(exception)) { return(true); } //Check for AmazonServiceExceptions specifically var serviceException = exception as AmazonServiceException; if (serviceException != null) { //Check if the exception is marked retryable. if (serviceException.Retryable != null) { return(true); } //Check for specific HTTP status codes that are associated with transient //service errors as long as they are not throttling errors. if (HttpStatusCodesToRetryOn.Contains(serviceException.StatusCode) && !IsThrottlingError(exception)) { return(true); } //Check for successful responses that couldn't be unmarshalled. These should be considered //transient errors because the payload could have been corrupted after OK was sent in the //header. if (serviceException.StatusCode == HttpStatusCode.OK && serviceException is AmazonUnmarshallingException) { return(true); } } //Check for WebExceptions that are considered transient WebException webException; if (ExceptionUtils.IsInnerException(exception, out webException)) { if (WebExceptionStatusesToRetryOn.Contains(webException.Status)) { return(true); } } if (IsTransientSslError(exception)) { return(true); } #if NETSTANDARD // Version 7.35 libcurl which is the default version installed with Ubuntu 14.04 // has issues under high concurrency causing response streams being disposed // during unmarshalling. To work around this issue will add the ObjectDisposedException // to the list of exceptions to retry. if (ExceptionUtils.IsInnerException <ObjectDisposedException>(exception)) { return(true); } //If it isn't a serviceException that we already processed for StatusCode and it //is a HttpRequestException, then it is a network type error that did not reach the //service and it should be retried. if (serviceException == null && exception is System.Net.Http.HttpRequestException) { return(true); } if (exception is OperationCanceledException) { if (!executionContext.RequestContext.CancellationToken.IsCancellationRequested) { //OperationCanceledException thrown by HttpClient not the CancellationToken supplied by the user. //This exception can wrap at least IOExceptions, ObjectDisposedExceptions and should be retried return(true); } } // .NET 5 introduced changes to HttpClient for timed out requests by returning a wrapped TimeoutException. if (exception is TimeoutException) { return(true); } #endif return(false); }
private bool HandleHttpWebErrorResponse(AsyncResult asyncResult, WebException we) { asyncResult.Metrics.AddProperty(Metric.Exception, we); HttpStatusCode statusCode; AmazonServiceException errorResponseException = null; using (HttpWebResponse httpErrorResponse = we.Response as HttpWebResponse) { if (we != null && WebExceptionStatusesToThrowOn.Contains(we.Status)) { throw new AmazonServiceException("Encountered a non retryable WebException : " + we.Status, we); } if (httpErrorResponse == null || (we != null && WebExceptionStatusesToRetryOn.Contains(we.Status))) { // Abort the unsuccessful request asyncResult.RequestState.WebRequest.Abort(); if (CanRetry(asyncResult) && asyncResult.RetriesAttempt < Config.MaxErrorRetry) { pauseExponentially(asyncResult); return(true); } var errorMessage = string.Format(CultureInfo.InvariantCulture, "Encountered a WebException ({0}), the request cannot be retried. Either the maximum number of retries has been exceeded ({1}/{2}) or the request is using a non-seekable stream.", we.Status, asyncResult.RetriesAttempt, Config.MaxErrorRetry); throw new AmazonServiceException(errorMessage, we); } statusCode = httpErrorResponse.StatusCode; asyncResult.Metrics.AddProperty(Metric.StatusCode, statusCode); string redirectedLocation = httpErrorResponse.Headers[HeaderKeys.LocationHeader]; asyncResult.Metrics.AddProperty(Metric.RedirectLocation, redirectedLocation); using (httpErrorResponse) { var unmarshaller = asyncResult.Unmarshaller; var httpResponseData = new HttpWebRequestResponseData(httpErrorResponse); UnmarshallerContext errorContext = unmarshaller.CreateContext(httpResponseData, Config.LogResponse || Config.ReadEntireResponse || AWSConfigs.LoggingConfig.LogResponses != ResponseLoggingOption.Never, httpResponseData.ResponseBody.OpenResponse(), asyncResult.Metrics); errorResponseException = unmarshaller.UnmarshallException(errorContext, we, statusCode); if (Config.LogResponse || AWSConfigs.LoggingConfig.LogResponses != ResponseLoggingOption.Never) { this.logger.Error(errorResponseException, "Received error response: [{0}]", errorContext.ResponseBody); } asyncResult.Metrics.AddProperty(Metric.AWSRequestID, errorResponseException.RequestId); asyncResult.Metrics.AddProperty(Metric.AWSErrorCode, errorResponseException.ErrorCode); } asyncResult.RequestState.WebRequest.Abort(); if (CanRetry(asyncResult)) { if (isTemporaryRedirect(statusCode, redirectedLocation)) { this.logger.DebugFormat("Request {0} is being redirected to {1}.", asyncResult.RequestName, redirectedLocation); asyncResult.Request.Endpoint = new Uri(redirectedLocation); return(true); } else if (ShouldRetry(statusCode, this.Config, errorResponseException, asyncResult.RetriesAttempt)) { this.logger.DebugFormat("Retry number {0} for request {1}.", asyncResult.RetriesAttempt, asyncResult.RequestName); pauseExponentially(asyncResult); return(true); } } } if (errorResponseException != null) { this.logger.Error(errorResponseException, "Error making request {0}.", asyncResult.RequestName); throw errorResponseException; } AmazonServiceException excep = new AmazonServiceException("Unable to make request", we, statusCode); this.logger.Error(excep, "Error making request {0}.", asyncResult.RequestName); asyncResult.Metrics.AddProperty(Metric.Exception, excep); throw excep; }