public Task <ShouldRetryResult> ShouldRetryAsync(
            Exception exception,
            CancellationToken cancellationToken)
        {
            TimeSpan backoffTime = TimeSpan.FromSeconds(0);

            if (!WebExceptionUtility.IsWebExceptionRetriable(exception))
            {
                // Have caller propagate original exception.
                this.durationTimer.Stop();
                return(Task.FromResult(ShouldRetryResult.NoRetry()));
            }

            // Don't penalise first retry with delay.
            if (attemptCount++ > 1)
            {
                int remainingSeconds = WebExceptionRetryPolicy.waitTimeInSeconds - this.durationTimer.Elapsed.Seconds;
                if (remainingSeconds <= 0)
                {
                    this.durationTimer.Stop();
                    return(Task.FromResult(ShouldRetryResult.NoRetry()));
                }

                backoffTime = TimeSpan.FromSeconds(Math.Min(this.currentBackoffSeconds, remainingSeconds));
                this.currentBackoffSeconds *= WebExceptionRetryPolicy.backoffMultiplier;
            }

            DefaultTrace.TraceWarning("Received retriable web exception, will retry, {0}", exception);

            return(Task.FromResult(ShouldRetryResult.RetryAfter(backoffTime)));
        }
        private async Task <HttpResponseMessage> SendHttpHelperAsync(
            Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
            ResourceType resourceType,
            CosmosDiagnosticsContext diagnosticsContext,
            HttpTimeoutPolicy timeoutPolicy,
            CancellationToken cancellationToken)
        {
            bool     isDefaultCancellationToken = cancellationToken == default;
            DateTime startDateTimeUtc           = DateTime.UtcNow;
            IEnumerator <(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> timeoutEnumerator = timeoutPolicy.TimeoutEnumerator;

            timeoutEnumerator.MoveNext();
            while (true)
            {
                (TimeSpan requestTimeout, TimeSpan delayForNextRequest) = timeoutEnumerator.Current;
                using (HttpRequestMessage requestMessage = await createRequestMessageAsync())
                {
                    // If the default cancellation token is passed then use the timeout policy
                    CancellationTokenSource cancellationTokenSource = null;
                    if (isDefaultCancellationToken)
                    {
                        cancellationTokenSource = new CancellationTokenSource();
                        cancellationTokenSource.CancelAfter(requestTimeout);
                        cancellationToken = cancellationTokenSource.Token;
                    }

                    cancellationToken.ThrowIfCancellationRequested();

                    try
                    {
                        using (diagnosticsContext.CreateScope(nameof(CosmosHttpClientCore.SendHttpHelperAsync)))
                        {
                            return(await this.ExecuteHttpHelperAsync(
                                       requestMessage,
                                       resourceType,
                                       cancellationToken));
                        }
                    }
                    catch (Exception e)
                    {
                        // Log the error message
                        diagnosticsContext.AddDiagnosticsInternal(
                            new PointOperationStatistics(
                                activityId: Trace.CorrelationManager.ActivityId.ToString(),
                                statusCode: HttpStatusCode.ServiceUnavailable,
                                subStatusCode: SubStatusCodes.Unknown,
                                responseTimeUtc: DateTime.UtcNow,
                                requestCharge: 0,
                                errorMessage: e.ToString(),
                                method: requestMessage.Method,
                                requestUri: requestMessage.RequestUri.OriginalString,
                                requestSessionToken: null,
                                responseSessionToken: null));

                        bool isOutOfRetries = (DateTime.UtcNow - startDateTimeUtc) > timeoutPolicy.MaximumRetryTimeLimit || // Maximum of time for all retries
                                              !timeoutEnumerator.MoveNext();                                                // No more retries are configured

                        switch (e)
                        {
                        case OperationCanceledException operationCanceledException:
                            // Throw if the user passed in cancellation was requested
                            if (!isDefaultCancellationToken && cancellationToken.IsCancellationRequested)
                            {
                                throw;
                            }

                            // Convert OperationCanceledException to 408 when the HTTP client throws it. This makes it clear that the
                            // the request timed out and was not user canceled operation.
                            if (isOutOfRetries || requestMessage.Method != HttpMethod.Get)
                            {
                                // throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
                                string message =
                                    $"GatewayStoreClient Request Timeout. Start Time UTC:{startDateTimeUtc}; Total Duration:{(DateTime.UtcNow - startDateTimeUtc).TotalMilliseconds} Ms; Request Timeout {requestTimeout.TotalMilliseconds} Ms; Http Client Timeout:{this.httpClient.Timeout.TotalMilliseconds} Ms; Activity id: {Trace.CorrelationManager.ActivityId};";
                                throw CosmosExceptionFactory.CreateRequestTimeoutException(
                                          message,
                                          innerException: operationCanceledException,
                                          diagnosticsContext: diagnosticsContext);
                            }

                            break;

                        case WebException webException:
                            if (isOutOfRetries || (requestMessage.Method != HttpMethod.Get && !WebExceptionUtility.IsWebExceptionRetriable(webException)))
                            {
                                throw;
                            }

                            break;

                        case HttpRequestException httpRequestException:
                            if (isOutOfRetries || requestMessage.Method != HttpMethod.Get)
                            {
                                throw;
                            }

                            break;

                        default:
                            throw;
                        }
                    }
                }

                if (delayForNextRequest != TimeSpan.Zero)
                {
                    using (diagnosticsContext.CreateScope($"HttpRetryDelay; Delay:{delayForNextRequest} seconds; Current request timeout {requestTimeout}; TimeoutPolicy: {timeoutPolicy.TimeoutPolicyName}"))
                    {
                        await Task.Delay(delayForNextRequest);
                    }
                }
            }
        }
        private async Task <HttpResponseMessage> SendHttpHelperAsync(
            Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
            ResourceType resourceType,
            HttpTimeoutPolicy timeoutPolicy,
            IClientSideRequestStatistics clientSideRequestStatistics,
            CancellationToken cancellationToken)
        {
            DateTime startDateTimeUtc = DateTime.UtcNow;
            IEnumerator <(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> timeoutEnumerator = timeoutPolicy.GetTimeoutEnumerator();

            timeoutEnumerator.MoveNext();
            while (true)
            {
                cancellationToken.ThrowIfCancellationRequested();

                (TimeSpan requestTimeout, TimeSpan delayForNextRequest) = timeoutEnumerator.Current;
                using (HttpRequestMessage requestMessage = await createRequestMessageAsync())
                {
                    // If the default cancellation token is passed then use the timeout policy
                    using CancellationTokenSource cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
                    cancellationTokenSource.CancelAfter(requestTimeout);

                    try
                    {
                        HttpResponseMessage responseMessage = await this.ExecuteHttpHelperAsync(
                            requestMessage,
                            resourceType,
                            cancellationTokenSource.Token);

                        if (clientSideRequestStatistics is ClientSideRequestStatisticsTraceDatum datum)
                        {
                            datum.RecordHttpResponse(requestMessage, responseMessage, resourceType, DateTime.UtcNow);
                        }

                        return(responseMessage);
                    }
                    catch (Exception e)
                    {
                        if (clientSideRequestStatistics is ClientSideRequestStatisticsTraceDatum datum)
                        {
                            datum.RecordHttpException(requestMessage, e, resourceType, DateTime.UtcNow);
                        }

                        bool isOutOfRetries = (DateTime.UtcNow - startDateTimeUtc) > timeoutPolicy.MaximumRetryTimeLimit || // Maximum of time for all retries
                                              !timeoutEnumerator.MoveNext();                                                // No more retries are configured

                        switch (e)
                        {
                        case OperationCanceledException operationCanceledException:
                            // Throw if the user passed in cancellation was requested
                            if (cancellationToken.IsCancellationRequested)
                            {
                                throw;
                            }

                            // Convert OperationCanceledException to 408 when the HTTP client throws it. This makes it clear that the
                            // the request timed out and was not user canceled operation.
                            if (isOutOfRetries || !timeoutPolicy.IsSafeToRetry(requestMessage.Method))
                            {
                                // throw current exception (caught in transport handler)
                                string message =
                                    $"GatewayStoreClient Request Timeout. Start Time UTC:{startDateTimeUtc}; Total Duration:{(DateTime.UtcNow - startDateTimeUtc).TotalMilliseconds} Ms; Request Timeout {requestTimeout.TotalMilliseconds} Ms; Http Client Timeout:{this.httpClient.Timeout.TotalMilliseconds} Ms; Activity id: {System.Diagnostics.Trace.CorrelationManager.ActivityId};";
                                e.Data.Add("Message", message);
                                throw;
                            }

                            break;

                        case WebException webException:
                            if (isOutOfRetries || (!timeoutPolicy.IsSafeToRetry(requestMessage.Method) && !WebExceptionUtility.IsWebExceptionRetriable(webException)))
                            {
                                throw;
                            }

                            break;

                        case HttpRequestException httpRequestException:
                            if (isOutOfRetries || !timeoutPolicy.IsSafeToRetry(requestMessage.Method))
                            {
                                throw;
                            }

                            break;

                        default:
                            throw;
                        }
                    }
                }

                if (delayForNextRequest != TimeSpan.Zero)
                {
                    await Task.Delay(delayForNextRequest);
                }
            }
        }
 public bool IsExceptionTransientRetriable(
     HttpMethod httpMethod,
     Exception exception,
     CancellationToken cancellationToken)
 {
     return((!cancellationToken.IsCancellationRequested && exception is OperationCanceledException) ||
            (exception is WebException webException && (httpMethod == HttpMethod.Get || WebExceptionUtility.IsWebExceptionRetriable(webException))));
 }
        private async Task <HttpResponseMessage> SendHttpHelperAsync(
            Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
            ResourceType resourceType,
            HttpTimeoutPolicy timeoutPolicy,
            ITrace trace,
            CancellationToken cancellationToken)
        {
            DateTime startDateTimeUtc = DateTime.UtcNow;
            IEnumerator <(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> timeoutEnumerator = timeoutPolicy.GetTimeoutEnumerator();

            timeoutEnumerator.MoveNext();
            while (true)
            {
                cancellationToken.ThrowIfCancellationRequested();

                (TimeSpan requestTimeout, TimeSpan delayForNextRequest) = timeoutEnumerator.Current;
                using (HttpRequestMessage requestMessage = await createRequestMessageAsync())
                {
                    // If the default cancellation token is passed then use the timeout policy
                    using CancellationTokenSource cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
                    cancellationTokenSource.CancelAfter(requestTimeout);

                    using (ITrace helperTrace = trace.StartChild($"Execute Http With Timeout Policy: {timeoutPolicy.TimeoutPolicyName}", TraceComponent.Transport, Tracing.TraceLevel.Info))
                    {
                        try
                        {
                            return(await this.ExecuteHttpHelperAsync(
                                       requestMessage,
                                       resourceType,
                                       cancellationTokenSource.Token));
                        }
                        catch (Exception e)
                        {
                            // Log the error message
                            trace.AddDatum(
                                "Error",
                                new PointOperationStatisticsTraceDatum(
                                    activityId: System.Diagnostics.Trace.CorrelationManager.ActivityId.ToString(),
                                    statusCode: HttpStatusCode.ServiceUnavailable,
                                    subStatusCode: SubStatusCodes.Unknown,
                                    responseTimeUtc: DateTime.UtcNow,
                                    requestCharge: 0,
                                    errorMessage: e.ToString(),
                                    method: requestMessage.Method,
                                    requestUri: requestMessage.RequestUri.OriginalString,
                                    requestSessionToken: null,
                                    responseSessionToken: null));

                            bool isOutOfRetries = (DateTime.UtcNow - startDateTimeUtc) > timeoutPolicy.MaximumRetryTimeLimit || // Maximum of time for all retries
                                                  !timeoutEnumerator.MoveNext();                                                // No more retries are configured

                            switch (e)
                            {
                            case OperationCanceledException operationCanceledException:
                                // Throw if the user passed in cancellation was requested
                                if (cancellationToken.IsCancellationRequested)
                                {
                                    throw;
                                }

                                // Convert OperationCanceledException to 408 when the HTTP client throws it. This makes it clear that the
                                // the request timed out and was not user canceled operation.
                                if (isOutOfRetries || !timeoutPolicy.IsSafeToRetry(requestMessage.Method))
                                {
                                    // throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
                                    string message =
                                        $"GatewayStoreClient Request Timeout. Start Time UTC:{startDateTimeUtc}; Total Duration:{(DateTime.UtcNow - startDateTimeUtc).TotalMilliseconds} Ms; Request Timeout {requestTimeout.TotalMilliseconds} Ms; Http Client Timeout:{this.httpClient.Timeout.TotalMilliseconds} Ms; Activity id: {System.Diagnostics.Trace.CorrelationManager.ActivityId};";
                                    throw CosmosExceptionFactory.CreateRequestTimeoutException(
                                              message,
                                              headers: new Headers()
                                    {
                                        ActivityId = System.Diagnostics.Trace.CorrelationManager.ActivityId.ToString()
                                    },
                                              innerException: operationCanceledException,
                                              trace: helperTrace);
                                }

                                break;

                            case WebException webException:
                                if (isOutOfRetries || (!timeoutPolicy.IsSafeToRetry(requestMessage.Method) && !WebExceptionUtility.IsWebExceptionRetriable(webException)))
                                {
                                    throw;
                                }

                                break;

                            case HttpRequestException httpRequestException:
                                if (isOutOfRetries || !timeoutPolicy.IsSafeToRetry(requestMessage.Method))
                                {
                                    throw;
                                }

                                break;

                            default:
                                throw;
                            }
                        }
                    }
                }

                if (delayForNextRequest != TimeSpan.Zero)
                {
                    using (ITrace delayTrace = trace.StartChild("Retry Delay", TraceComponent.Transport, Tracing.TraceLevel.Info))
                    {
                        await Task.Delay(delayForNextRequest);
                    }
                }
            }
        }