public Task <ShouldRetryResult> ShouldRetryAsync(
Exception exception,
CancellationToken cancellationToken)
{
TimeSpan backoffTime = TimeSpan.FromSeconds(0);
if (!WebExceptionUtility.IsWebExceptionRetriable(exception))
{
// Have caller propagate original exception.
this.durationTimer.Stop();
return(Task.FromResult(ShouldRetryResult.NoRetry()));
}
// Don't penalise first retry with delay.
if (attemptCount++ > 1)
{
int remainingSeconds = WebExceptionRetryPolicy.waitTimeInSeconds - this.durationTimer.Elapsed.Seconds;
if (remainingSeconds <= 0)
{
this.durationTimer.Stop();
return(Task.FromResult(ShouldRetryResult.NoRetry()));
}
backoffTime = TimeSpan.FromSeconds(Math.Min(this.currentBackoffSeconds, remainingSeconds));
this.currentBackoffSeconds *= WebExceptionRetryPolicy.backoffMultiplier;
}
DefaultTrace.TraceWarning("Received retriable web exception, will retry, {0}", exception);
return(Task.FromResult(ShouldRetryResult.RetryAfter(backoffTime)));
}
private async Task <HttpResponseMessage> SendHttpHelperAsync(
Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
ResourceType resourceType,
CosmosDiagnosticsContext diagnosticsContext,
HttpTimeoutPolicy timeoutPolicy,
CancellationToken cancellationToken)
{
bool isDefaultCancellationToken = cancellationToken == default;
DateTime startDateTimeUtc = DateTime.UtcNow;
IEnumerator <(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> timeoutEnumerator = timeoutPolicy.TimeoutEnumerator;
timeoutEnumerator.MoveNext();
while (true)
{
(TimeSpan requestTimeout, TimeSpan delayForNextRequest) = timeoutEnumerator.Current;
using (HttpRequestMessage requestMessage = await createRequestMessageAsync())
{
// If the default cancellation token is passed then use the timeout policy
CancellationTokenSource cancellationTokenSource = null;
if (isDefaultCancellationToken)
{
cancellationTokenSource = new CancellationTokenSource();
cancellationTokenSource.CancelAfter(requestTimeout);
cancellationToken = cancellationTokenSource.Token;
}
cancellationToken.ThrowIfCancellationRequested();
try
{
using (diagnosticsContext.CreateScope(nameof(CosmosHttpClientCore.SendHttpHelperAsync)))
{
return(await this.ExecuteHttpHelperAsync(
requestMessage,
resourceType,
cancellationToken));
}
}
catch (Exception e)
{
// Log the error message
diagnosticsContext.AddDiagnosticsInternal(
new PointOperationStatistics(
activityId: Trace.CorrelationManager.ActivityId.ToString(),
statusCode: HttpStatusCode.ServiceUnavailable,
subStatusCode: SubStatusCodes.Unknown,
responseTimeUtc: DateTime.UtcNow,
requestCharge: 0,
errorMessage: e.ToString(),
method: requestMessage.Method,
requestUri: requestMessage.RequestUri.OriginalString,
requestSessionToken: null,
responseSessionToken: null));
bool isOutOfRetries = (DateTime.UtcNow - startDateTimeUtc) > timeoutPolicy.MaximumRetryTimeLimit || // Maximum of time for all retries
!timeoutEnumerator.MoveNext(); // No more retries are configured
switch (e)
{
case OperationCanceledException operationCanceledException:
// Throw if the user passed in cancellation was requested
if (!isDefaultCancellationToken && cancellationToken.IsCancellationRequested)
{
throw;
}
// Convert OperationCanceledException to 408 when the HTTP client throws it. This makes it clear that the
// the request timed out and was not user canceled operation.
if (isOutOfRetries || requestMessage.Method != HttpMethod.Get)
{
// throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
string message =
$"GatewayStoreClient Request Timeout. Start Time UTC:{startDateTimeUtc}; Total Duration:{(DateTime.UtcNow - startDateTimeUtc).TotalMilliseconds} Ms; Request Timeout {requestTimeout.TotalMilliseconds} Ms; Http Client Timeout:{this.httpClient.Timeout.TotalMilliseconds} Ms; Activity id: {Trace.CorrelationManager.ActivityId};";
throw CosmosExceptionFactory.CreateRequestTimeoutException(
message,
innerException: operationCanceledException,
diagnosticsContext: diagnosticsContext);
}
break;
case WebException webException:
if (isOutOfRetries || (requestMessage.Method != HttpMethod.Get && !WebExceptionUtility.IsWebExceptionRetriable(webException)))
{
throw;
}
break;
case HttpRequestException httpRequestException:
if (isOutOfRetries || requestMessage.Method != HttpMethod.Get)
{
throw;
}
break;
default:
throw;
}
}
}
if (delayForNextRequest != TimeSpan.Zero)
{
using (diagnosticsContext.CreateScope($"HttpRetryDelay; Delay:{delayForNextRequest} seconds; Current request timeout {requestTimeout}; TimeoutPolicy: {timeoutPolicy.TimeoutPolicyName}"))
{
await Task.Delay(delayForNextRequest);
}
}
}
}
private async Task <HttpResponseMessage> SendHttpHelperAsync(
Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
ResourceType resourceType,
HttpTimeoutPolicy timeoutPolicy,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken)
{
DateTime startDateTimeUtc = DateTime.UtcNow;
IEnumerator <(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> timeoutEnumerator = timeoutPolicy.GetTimeoutEnumerator();
timeoutEnumerator.MoveNext();
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
(TimeSpan requestTimeout, TimeSpan delayForNextRequest) = timeoutEnumerator.Current;
using (HttpRequestMessage requestMessage = await createRequestMessageAsync())
{
// If the default cancellation token is passed then use the timeout policy
using CancellationTokenSource cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
cancellationTokenSource.CancelAfter(requestTimeout);
try
{
HttpResponseMessage responseMessage = await this.ExecuteHttpHelperAsync(
requestMessage,
resourceType,
cancellationTokenSource.Token);
if (clientSideRequestStatistics is ClientSideRequestStatisticsTraceDatum datum)
{
datum.RecordHttpResponse(requestMessage, responseMessage, resourceType, DateTime.UtcNow);
}
return(responseMessage);
}
catch (Exception e)
{
if (clientSideRequestStatistics is ClientSideRequestStatisticsTraceDatum datum)
{
datum.RecordHttpException(requestMessage, e, resourceType, DateTime.UtcNow);
}
bool isOutOfRetries = (DateTime.UtcNow - startDateTimeUtc) > timeoutPolicy.MaximumRetryTimeLimit || // Maximum of time for all retries
!timeoutEnumerator.MoveNext(); // No more retries are configured
switch (e)
{
case OperationCanceledException operationCanceledException:
// Throw if the user passed in cancellation was requested
if (cancellationToken.IsCancellationRequested)
{
throw;
}
// Convert OperationCanceledException to 408 when the HTTP client throws it. This makes it clear that the
// the request timed out and was not user canceled operation.
if (isOutOfRetries || !timeoutPolicy.IsSafeToRetry(requestMessage.Method))
{
// throw current exception (caught in transport handler)
string message =
$"GatewayStoreClient Request Timeout. Start Time UTC:{startDateTimeUtc}; Total Duration:{(DateTime.UtcNow - startDateTimeUtc).TotalMilliseconds} Ms; Request Timeout {requestTimeout.TotalMilliseconds} Ms; Http Client Timeout:{this.httpClient.Timeout.TotalMilliseconds} Ms; Activity id: {System.Diagnostics.Trace.CorrelationManager.ActivityId};";
e.Data.Add("Message", message);
throw;
}
break;
case WebException webException:
if (isOutOfRetries || (!timeoutPolicy.IsSafeToRetry(requestMessage.Method) && !WebExceptionUtility.IsWebExceptionRetriable(webException)))
{
throw;
}
break;
case HttpRequestException httpRequestException:
if (isOutOfRetries || !timeoutPolicy.IsSafeToRetry(requestMessage.Method))
{
throw;
}
break;
default:
throw;
}
}
}
if (delayForNextRequest != TimeSpan.Zero)
{
await Task.Delay(delayForNextRequest);
}
}
}
public bool IsExceptionTransientRetriable(
HttpMethod httpMethod,
Exception exception,
CancellationToken cancellationToken)
{
return((!cancellationToken.IsCancellationRequested && exception is OperationCanceledException) ||
(exception is WebException webException && (httpMethod == HttpMethod.Get || WebExceptionUtility.IsWebExceptionRetriable(webException))));
}
private async Task <HttpResponseMessage> SendHttpHelperAsync(
Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
ResourceType resourceType,
HttpTimeoutPolicy timeoutPolicy,
ITrace trace,
CancellationToken cancellationToken)
{
DateTime startDateTimeUtc = DateTime.UtcNow;
IEnumerator <(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> timeoutEnumerator = timeoutPolicy.GetTimeoutEnumerator();
timeoutEnumerator.MoveNext();
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
(TimeSpan requestTimeout, TimeSpan delayForNextRequest) = timeoutEnumerator.Current;
using (HttpRequestMessage requestMessage = await createRequestMessageAsync())
{
// If the default cancellation token is passed then use the timeout policy
using CancellationTokenSource cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
cancellationTokenSource.CancelAfter(requestTimeout);
using (ITrace helperTrace = trace.StartChild($"Execute Http With Timeout Policy: {timeoutPolicy.TimeoutPolicyName}", TraceComponent.Transport, Tracing.TraceLevel.Info))
{
try
{
return(await this.ExecuteHttpHelperAsync(
requestMessage,
resourceType,
cancellationTokenSource.Token));
}
catch (Exception e)
{
// Log the error message
trace.AddDatum(
"Error",
new PointOperationStatisticsTraceDatum(
activityId: System.Diagnostics.Trace.CorrelationManager.ActivityId.ToString(),
statusCode: HttpStatusCode.ServiceUnavailable,
subStatusCode: SubStatusCodes.Unknown,
responseTimeUtc: DateTime.UtcNow,
requestCharge: 0,
errorMessage: e.ToString(),
method: requestMessage.Method,
requestUri: requestMessage.RequestUri.OriginalString,
requestSessionToken: null,
responseSessionToken: null));
bool isOutOfRetries = (DateTime.UtcNow - startDateTimeUtc) > timeoutPolicy.MaximumRetryTimeLimit || // Maximum of time for all retries
!timeoutEnumerator.MoveNext(); // No more retries are configured
switch (e)
{
case OperationCanceledException operationCanceledException:
// Throw if the user passed in cancellation was requested
if (cancellationToken.IsCancellationRequested)
{
throw;
}
// Convert OperationCanceledException to 408 when the HTTP client throws it. This makes it clear that the
// the request timed out and was not user canceled operation.
if (isOutOfRetries || !timeoutPolicy.IsSafeToRetry(requestMessage.Method))
{
// throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
string message =
$"GatewayStoreClient Request Timeout. Start Time UTC:{startDateTimeUtc}; Total Duration:{(DateTime.UtcNow - startDateTimeUtc).TotalMilliseconds} Ms; Request Timeout {requestTimeout.TotalMilliseconds} Ms; Http Client Timeout:{this.httpClient.Timeout.TotalMilliseconds} Ms; Activity id: {System.Diagnostics.Trace.CorrelationManager.ActivityId};";
throw CosmosExceptionFactory.CreateRequestTimeoutException(
message,
headers: new Headers()
{
ActivityId = System.Diagnostics.Trace.CorrelationManager.ActivityId.ToString()
},
innerException: operationCanceledException,
trace: helperTrace);
}
break;
case WebException webException:
if (isOutOfRetries || (!timeoutPolicy.IsSafeToRetry(requestMessage.Method) && !WebExceptionUtility.IsWebExceptionRetriable(webException)))
{
throw;
}
break;
case HttpRequestException httpRequestException:
if (isOutOfRetries || !timeoutPolicy.IsSafeToRetry(requestMessage.Method))
{
throw;
}
break;
default:
throw;
}
}
}
}
if (delayForNextRequest != TimeSpan.Zero)
{
using (ITrace delayTrace = trace.StartChild("Retry Delay", TraceComponent.Transport, Tracing.TraceLevel.Info))
{
await Task.Delay(delayForNextRequest);
}
}
}
}