private async Task <T> InvokeConfiguredRequest <T>(WebRequestState state, CancellationToken cancellationToken) where T : AmazonWebServiceResponse
{
int currentRetries = state.RetriesAttempt;
T response = null;
bool shouldRetry = false;
bool responseReceived = false;
HttpResponseMessage responseMessage = null;
HttpClientResponseData responseData = null;
var requestMessage = ConfigureRequestMessage(state);
try
{
try
{
SetContent(requestMessage, state);
using (state.Metrics.StartEvent(Metric.HttpRequestTime))
{
responseMessage = await httpClient.SendAsync(requestMessage, HttpCompletionOption.ResponseHeadersRead, cancellationToken)
.ConfigureAwait(continueOnCapturedContext: false);
responseReceived = true;
}
responseData = new HttpClientResponseData(responseMessage);
if (!IsErrorResponse(responseMessage) ||
responseMessage.StatusCode == HttpStatusCode.NotFound && state.Request.Suppress404Exceptions)
{
using (state.Metrics.StartEvent(Metric.ResponseProcessingTime))
{
response = (T) await HandleHttpContentAsync(state, responseMessage, responseData)
.ConfigureAwait(continueOnCapturedContext: false);
}
}
else
{
bool retry = await HandleHttpErrorResponseAsync(state, responseMessage, responseData, cancellationToken)
.ConfigureAwait(continueOnCapturedContext: false);
if (retry)
{
shouldRetry = true;
}
}
}
catch (HttpRequestException e)
{
var we = e.InnerException as WebException;
if (we != null)
{
if (WebExceptionStatusesToThrowOn.Contains(we.Status))
{
throw new AmazonServiceException("Encountered a non retryable WebException : " + we.Status, we);
}
// The WinRT framework doesn't break down errors, not all status values are available for WinRT.
if (
#if WIN_RT
(we.Status == WebExceptionStatus.UnknownError || WebExceptionStatusesToRetryOn.Contains(we.Status))
#else
WebExceptionStatusesToRetryOn.Contains(we.Status)
#endif
)
{
shouldRetry = RetryOrThrow(state, e);
}
}
var ioe = e.InnerException as IOException;
if (ioe != null)
{
#if !WIN_RT
if (IsInnerExceptionThreadAbort(ioe))
{
throw new AmazonServiceException(e);
}
#endif
shouldRetry = RetryOrThrow(state, e);
}
// Check if response is null at the end as
// it can be null for both WebException and IOException.
if (!shouldRetry && response == null)
{
shouldRetry = RetryOrThrow(state, e);
}
// If shouldRetry is not set by any of the above checks,
// re-throw the exception.
if (!shouldRetry)
{
throw new AmazonServiceException(e);
}
}
catch (TaskCanceledException taskCancelledException)
{
if (cancellationToken.IsCancellationRequested)
{
throw;
}
else
{
// It's a timeout exception.
throw new AmazonServiceException(taskCancelledException);
}
}
catch (OperationCanceledException operationCancelledException)
{
if (cancellationToken.IsCancellationRequested)
{
// Throw an exception with the original cancellation token.
throw new OperationCanceledException(operationCancelledException.Message, cancellationToken);
}
else
{
// It's a timeout exception.
throw new AmazonServiceException(operationCancelledException);
}
}
finally
{
if (responseMessage != null && (response == null || !state.Unmarshaller.HasStreamingProperty))
{
responseMessage.Dispose();
responseMessage = null;
}
}
if (shouldRetry)
{
pauseExponentially(state);
cancellationToken.ThrowIfCancellationRequested();
state.RetriesAttempt++;
var retryResponse = await InvokeHelper <T>(state, cancellationToken)
.ConfigureAwait(continueOnCapturedContext: false);
return((T)retryResponse);
}
else
{
LogFinalMetrics(state.Metrics);
}
}
catch (Exception e)
{
// On errors that are passed to the client, invoke exception handlers
if (!shouldRetry)
{
ProcessExceptionHandlers(e, state.Request);
}
throw;
}
finally
{
// ProcessResponseHandlers is called only if a response was received from the server (success or error).
// It's not called in case of client errors like timeout or proxy error.
if (!shouldRetry && responseReceived)
{
ProcessResponseHandlers(response, state.Request, responseData);
}
}
return(response);
}
/// <summary>
/// Determines if an AmazonServiceException is a transient error that
/// should be retried.
/// </summary>
/// <param name="executionContext">The current execution context</param>
/// <param name="exception">The current exception to check.</param>
/// <returns>true if the exception is a transient error else false.</returns>
public virtual bool IsTransientError(IExecutionContext executionContext, Exception exception)
{
// An IOException was thrown by the underlying http client.
if (exception is IOException)
{
#if !NETSTANDARD // ThreadAbortException is not NetStandard
// Don't retry IOExceptions that are caused by a ThreadAbortException
if (ExceptionUtils.IsInnerException <ThreadAbortException>(exception))
{
return(false);
}
#endif
// Retry all other IOExceptions
return(true);
}
else if (ExceptionUtils.IsInnerException <IOException>(exception))
{
return(true);
}
//Check for AmazonServiceExceptions specifically
var serviceException = exception as AmazonServiceException;
if (serviceException != null)
{
//Check if the exception is marked retryable.
if (serviceException.Retryable != null)
{
return(true);
}
//Check for specific HTTP status codes that are associated with transient
//service errors as long as they are not throttling errors.
if (HttpStatusCodesToRetryOn.Contains(serviceException.StatusCode) &&
!IsThrottlingError(exception))
{
return(true);
}
//Check for successful responses that couldn't be unmarshalled. These should be considered
//transient errors because the payload could have been corrupted after OK was sent in the
//header.
if (serviceException.StatusCode == HttpStatusCode.OK && serviceException is AmazonUnmarshallingException)
{
return(true);
}
}
//Check for WebExceptions that are considered transient
WebException webException;
if (ExceptionUtils.IsInnerException(exception, out webException))
{
if (WebExceptionStatusesToRetryOn.Contains(webException.Status))
{
return(true);
}
}
if (IsTransientSslError(exception))
{
return(true);
}
#if NETSTANDARD
// Version 7.35 libcurl which is the default version installed with Ubuntu 14.04
// has issues under high concurrency causing response streams being disposed
// during unmarshalling. To work around this issue will add the ObjectDisposedException
// to the list of exceptions to retry.
if (ExceptionUtils.IsInnerException <ObjectDisposedException>(exception))
{
return(true);
}
//If it isn't a serviceException that we already processed for StatusCode and it
//is a HttpRequestException, then it is a network type error that did not reach the
//service and it should be retried.
if (serviceException == null && exception is System.Net.Http.HttpRequestException)
{
return(true);
}
if (exception is OperationCanceledException)
{
if (!executionContext.RequestContext.CancellationToken.IsCancellationRequested)
{
//OperationCanceledException thrown by HttpClient not the CancellationToken supplied by the user.
//This exception can wrap at least IOExceptions, ObjectDisposedExceptions and should be retried
return(true);
}
}
// .NET 5 introduced changes to HttpClient for timed out requests by returning a wrapped TimeoutException.
if (exception is TimeoutException)
{
return(true);
}
#endif
return(false);
}
private bool HandleHttpWebErrorResponse(AsyncResult asyncResult, WebException we)
{
asyncResult.Metrics.AddProperty(Metric.Exception, we);
HttpStatusCode statusCode;
AmazonServiceException errorResponseException = null;
using (HttpWebResponse httpErrorResponse = we.Response as HttpWebResponse)
{
if (we != null && WebExceptionStatusesToThrowOn.Contains(we.Status))
{
throw new AmazonServiceException("Encountered a non retryable WebException : " + we.Status, we);
}
if (httpErrorResponse == null ||
(we != null && WebExceptionStatusesToRetryOn.Contains(we.Status)))
{
// Abort the unsuccessful request
asyncResult.RequestState.WebRequest.Abort();
if (CanRetry(asyncResult) && asyncResult.RetriesAttempt < Config.MaxErrorRetry)
{
pauseExponentially(asyncResult);
return(true);
}
var errorMessage = string.Format(CultureInfo.InvariantCulture,
"Encountered a WebException ({0}), the request cannot be retried. Either the maximum number of retries has been exceeded ({1}/{2}) or the request is using a non-seekable stream.",
we.Status, asyncResult.RetriesAttempt, Config.MaxErrorRetry);
throw new AmazonServiceException(errorMessage, we);
}
statusCode = httpErrorResponse.StatusCode;
asyncResult.Metrics.AddProperty(Metric.StatusCode, statusCode);
string redirectedLocation = httpErrorResponse.Headers[HeaderKeys.LocationHeader];
asyncResult.Metrics.AddProperty(Metric.RedirectLocation, redirectedLocation);
using (httpErrorResponse)
{
var unmarshaller = asyncResult.Unmarshaller;
var httpResponseData = new HttpWebRequestResponseData(httpErrorResponse);
UnmarshallerContext errorContext = unmarshaller.CreateContext(httpResponseData,
Config.LogResponse || Config.ReadEntireResponse || AWSConfigs.LoggingConfig.LogResponses != ResponseLoggingOption.Never,
httpResponseData.ResponseBody.OpenResponse(),
asyncResult.Metrics);
errorResponseException = unmarshaller.UnmarshallException(errorContext, we, statusCode);
if (Config.LogResponse || AWSConfigs.LoggingConfig.LogResponses != ResponseLoggingOption.Never)
{
this.logger.Error(errorResponseException, "Received error response: [{0}]", errorContext.ResponseBody);
}
asyncResult.Metrics.AddProperty(Metric.AWSRequestID, errorResponseException.RequestId);
asyncResult.Metrics.AddProperty(Metric.AWSErrorCode, errorResponseException.ErrorCode);
}
asyncResult.RequestState.WebRequest.Abort();
if (CanRetry(asyncResult))
{
if (isTemporaryRedirect(statusCode, redirectedLocation))
{
this.logger.DebugFormat("Request {0} is being redirected to {1}.", asyncResult.RequestName, redirectedLocation);
asyncResult.Request.Endpoint = new Uri(redirectedLocation);
return(true);
}
else if (ShouldRetry(statusCode, this.Config, errorResponseException, asyncResult.RetriesAttempt))
{
this.logger.DebugFormat("Retry number {0} for request {1}.", asyncResult.RetriesAttempt, asyncResult.RequestName);
pauseExponentially(asyncResult);
return(true);
}
}
}
if (errorResponseException != null)
{
this.logger.Error(errorResponseException, "Error making request {0}.", asyncResult.RequestName);
throw errorResponseException;
}
AmazonServiceException excep = new AmazonServiceException("Unable to make request", we, statusCode);
this.logger.Error(excep, "Error making request {0}.", asyncResult.RequestName);
asyncResult.Metrics.AddProperty(Metric.Exception, excep);
throw excep;
}
