Пример #1
0
 static public Task InlineIfPossibleAsync(Func <Task> function, IRetryPolicy retryPolicy, CancellationToken cancellationToken = default)
 {
     if (SynchronizationContext.Current == null)
     {
         if (retryPolicy == null)
         {
             // shortcut
             return(function());
         }
         else
         {
             return(BackoffRetryUtility <int> .ExecuteAsync(async() =>
             {
                 await function();
                 return 0;
             }, retryPolicy, cancellationToken));
         }
     }
     else
     {
         if (retryPolicy == null)
         {
             // shortcut
             return(Task.Run(function));
         }
         else
         {
             return(Task.Run(() => BackoffRetryUtility <int> .ExecuteAsync(async() =>
             {
                 await function();
                 return 0;
             }, retryPolicy, cancellationToken)));
         }
     }
 }
 static public Task <TResult> InlineIfPossible <TResult>(Func <Task <TResult> > function, IRetryPolicy retryPolicy, CancellationToken cancellationToken = default(CancellationToken))
 {
     if (SynchronizationContext.Current == null)
     {
         if (retryPolicy == null)
         {
             // shortcut
             return(function());
         }
         else
         {
             return(BackoffRetryUtility <TResult> .ExecuteAsync(() =>
             {
                 return function();
             }, retryPolicy, cancellationToken));
         }
     }
     else
     {
         if (retryPolicy == null)
         {
             // shortcut
             return(Task.Run(function));
         }
         else
         {
             return(Task.Run(() => BackoffRetryUtility <TResult> .ExecuteAsync(() =>
             {
                 return function();
             }, retryPolicy, cancellationToken)));
         }
     }
 }
Пример #3
0
#pragma warning disable VSTHRD200 // Use "Async" suffix for async methods
        static public Task <TResult> InlineIfPossible <TResult>(Func <Task <TResult> > function, IRetryPolicy retryPolicy, CancellationToken cancellationToken = default)
#pragma warning restore VSTHRD200 // Use "Async" suffix for async methods
        {
            if (SynchronizationContext.Current == null)
            {
                if (retryPolicy == null)
                {
                    // shortcut
                    return(function());
                }
                else
                {
                    return(BackoffRetryUtility <TResult> .ExecuteAsync(() =>
                    {
                        return function();
                    }, retryPolicy, cancellationToken));
                }
            }
            else
            {
                if (retryPolicy == null)
                {
                    // shortcut
                    return(Task.Run(function));
                }
                else
                {
                    return(Task.Run(() => BackoffRetryUtility <TResult> .ExecuteAsync(() =>
                    {
                        return function();
                    }, retryPolicy, cancellationToken)));
                }
            }
        }
        public async Task ValidateRetryOnWriteForbiddenExceptionAsync()
        {
            this.Initialize(
                useMultipleWriteLocations: false,
                enableEndpointDiscovery: true,
                isPreferredLocationsListEmpty: false);

            await this.endpointManager.RefreshLocationAsync(this.databaseAccount);

            ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, true, new RetryOptions());

            using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: false, isMasterResourceType: false))
            {
                int retryCount = 0;

                await BackoffRetryUtility <bool> .ExecuteAsync(
                    () =>
                {
                    retryCount++;
                    retryPolicy.OnBeforeSendRequest(request);

                    if (retryCount == 1)
                    {
                        this.mockedClient.ResetCalls();

                        Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]];

                        Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);

                        StoreRequestNameValueCollection headers       = new StoreRequestNameValueCollection();
                        headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.WriteForbidden).ToString();
                        DocumentClientException forbiddenException    = new ForbiddenException(RMResources.Forbidden, headers);

                        throw forbiddenException;
                    }
                    else if (retryCount == 2)
                    {
                        this.mockedClient.Verify(client => client.GetDatabaseAccountInternalAsync(It.IsAny <Uri>(), It.IsAny <CancellationToken>()), Times.Once);

                        // Next request must go to next preferred endpoint
                        Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]];
                        Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);

                        return(Task.FromResult(true));
                    }
                    else
                    {
                        Assert.Fail();
                    }

                    return(Task.FromResult(true));
                },
                    retryPolicy);
            }
        }
Пример #5
0
        private async Task ValidateRetryOnSessionNotAvailabeWithEndpointDiscoveryDisabled(bool isPreferredLocationsListEmpty, bool useMultipleWriteLocations, bool isReadRequest)
        {
            const bool enableEndpointDiscovery = false;

            this.Initialize(
                useMultipleWriteLocations: useMultipleWriteLocations,
                enableEndpointDiscovery: enableEndpointDiscovery,
                isPreferredLocationsListEmpty: isPreferredLocationsListEmpty);

            ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, enableEndpointDiscovery, new RetryOptions());

            using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false))
            {
                int retryCount = 0;

                try
                {
                    await BackoffRetryUtility <bool> .ExecuteAsync(
                        () =>
                    {
                        retryPolicy.OnBeforeSendRequest(request);

                        if (retryCount == 0)
                        {
                            Assert.IsFalse(request.ClearSessionTokenOnSessionReadFailure);
                            Assert.AreEqual(request.RequestContext.LocationEndpointToRoute, this.endpointManager.ReadEndpoints[0]);
                        }
                        else
                        {
                            Assert.Fail();
                        }

                        retryCount++;

                        StringKeyValueCollection headers = new StringKeyValueCollection();
                        headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.ReadSessionNotAvailable).ToString();
                        DocumentClientException notFoundException     = new NotFoundException(RMResources.NotFound, headers);

                        throw notFoundException;
                    },
                        retryPolicy);

                    Assert.Fail();
                }
                catch (NotFoundException)
                {
                    DefaultTrace.TraceInformation("Received expected notFoundException");
                    Assert.AreEqual(1, retryCount);
                }
            }
        }
        async private Task <DocumentServiceResponse> InvokeAsync(DocumentServiceRequest request, ResourceType resourceType, CancellationToken cancellationToken)
        {
            Func <Task <DocumentServiceResponse> > funcDelegate = async() =>
            {
                using (HttpRequestMessage requestMessage = await this.PrepareRequestMessageAsync(request))
                {
                    DateTime sendTimeUtc = DateTime.UtcNow;
                    Guid     localGuid   = Guid.NewGuid(); // For correlating HttpRequest and HttpResponse Traces

                    this.eventSource.Request(
                        Guid.Empty,
                        localGuid,
                        requestMessage.RequestUri.ToString(),
                        resourceType.ToResourceTypeString(),
                        requestMessage.Headers);

                    using (HttpResponseMessage responseMessage = await this.httpClient.SendAsync(requestMessage, cancellationToken))
                    {
                        DateTime receivedTimeUtc        = DateTime.UtcNow;
                        double   durationInMilliSeconds = (receivedTimeUtc - sendTimeUtc).TotalMilliseconds;

                        IEnumerable <string> headerValues;
                        Guid activityId = Guid.Empty;
                        if (responseMessage.Headers.TryGetValues(HttpConstants.HttpHeaders.ActivityId, out headerValues) &&
                            headerValues.Count() != 0)
                        {
                            activityId = new Guid(headerValues.First());
                        }

                        this.eventSource.Response(
                            activityId,
                            localGuid,
                            (short)responseMessage.StatusCode,
                            durationInMilliSeconds,
                            responseMessage.Headers);

                        return(await ClientExtensions.ParseResponseAsync(responseMessage, request.SerializerSettings ?? this.SerializerSettings, request));
                    }
                }
            };

            return(await BackoffRetryUtility <DocumentServiceResponse> .ExecuteAsync(funcDelegate, new WebExceptionRetryPolicy(), cancellationToken));
        }
Пример #7
0
 public static Task InlineIfPossible(Func <Task> function, IRetryPolicy retryPolicy, CancellationToken cancellationToken = default(CancellationToken))
 {
     if (SynchronizationContext.Current == null)
     {
         if (retryPolicy == null)
         {
             return(function());
         }
         return(BackoffRetryUtility <int> .ExecuteAsync(async delegate
         {
             await function();
             return 0;
         }, retryPolicy, cancellationToken));
     }
     if (retryPolicy == null)
     {
         return(Task.Run(function));
     }
     return(Task.Run(() => BackoffRetryUtility <int> .ExecuteAsync(async delegate
     {
         await function();
         return 0;
     }, retryPolicy, cancellationToken)));
 }
        internal static void ValidateCollection <T>(DocumentClient[] replicaClients,
                                                    string collectionId,
                                                    INameValueCollection headers = null,
                                                    bool verifyAddress           = true) where T : Resource, new()
        {
            Assert.IsTrue(replicaClients != null && replicaClients.Length > 1, "Must pass in at least two replica clients");

            Task.Delay(3000); // allow previous operations to complete and propagate, bringing more robustness to tests

            foreach (DocumentClient client in replicaClients)
            {
                client.ForceAddressRefresh(true);
                if (typeof(T) == typeof(StoredProcedure) ||
                    typeof(T) == typeof(UserDefinedFunction) ||
                    typeof(T) == typeof(Trigger))
                {
                    TestCommon.ListAllScriptDirect <T>(client, collectionId, headers ?? new StringKeyValueCollection());
                }
                else
                {
                    TestCommon.ListAll <T>(client, collectionId, headers ?? new StringKeyValueCollection(), true);
                }
            }

            try
            {
                BackoffRetryUtility <bool> .ExecuteAsync((bool isInRetry) =>
                {
                    var feeds      = new DocumentFeedResponse <T> [replicaClients.Length];
                    var allHeaders = new StringKeyValueCollection[replicaClients.Length];
                    for (int i = 0; i < replicaClients.Length; i++)
                    {
                        var header = new StringKeyValueCollection();
                        if (headers != null)
                        {
                            foreach (string key in headers)
                            {
                                header[key] = headers[key];
                            }
                        }

                        allHeaders[i] = header;
                    }

                    var continuations   = new string[replicaClients.Length];
                    var responseHeaders = new INameValueCollection[replicaClients.Length];

                    do
                    {
                        for (int i = 0; i < replicaClients.Length; i++)
                        {
                            if (!string.IsNullOrEmpty(continuations[i]))
                            {
                                allHeaders[i][HttpConstants.HttpHeaders.Continuation] = continuations[i];
                            }

                            feeds[i] = replicaClients[i].ReadFeedWithRetry <T>(collectionId, out responseHeaders[i],
                                                                               allHeaders[i]);

                            if (responseHeaders[i] != null)
                            {
                                continuations[i] = responseHeaders[i][HttpConstants.HttpHeaders.Continuation];
                            }
                            else
                            {
                                continuations[i] = null;
                            }
                        }

                        for (int i = 0; i < replicaClients.Length - 1; i++)
                        {
                            for (int j = i + 1; j < replicaClients.Length; j++)
                            {
                                Assert.AreEqual(continuations[i], continuations[j], "Collection Continuaton mismatch");
                                if (verifyAddress)
                                {
                                    var address1 = replicaClients[i].GetAddress();
                                    var address2 = replicaClients[j].GetAddress();

                                    // If the addresses match, we are in mid of reconfiguration, throw GoneException so that we retry on the entire loop
                                    // and take care of intermittent gone at the same time.
                                    if (address1.Equals(address2))
                                    {
                                        throw new GoneException("Addresses matched for multiple replicas " + address1);
                                    }
                                }
                            }
                        }

                        ReplicationTests.ValidateCollection <T>(feeds);
                    } while (!string.IsNullOrEmpty(continuations[0]));

                    return(Task.FromResult <bool>(true));
                }, new GoneAndRetryWithRetryPolicy()).Wait();
            }
            finally
            {
                //Once address are stable. dont force the cache refresh.
                foreach (DocumentClient client in replicaClients)
                {
                    client.ForceAddressRefresh(false);
                }
            }
        }
        public override Task <HttpResponseMessage> SendHttpAsync(
            Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
            ResourceType resourceType,
            CosmosDiagnosticsContext diagnosticsContext,
            CancellationToken cancellationToken)
        {
            diagnosticsContext ??= new CosmosDiagnosticsContextCore();
            HttpRequestMessage requestMessage = null;
            Func <Task <HttpResponseMessage> > funcDelegate = async() =>
            {
                using (diagnosticsContext.CreateScope(nameof(CosmosHttpClientCore.SendHttpAsync)))
                {
                    using (requestMessage = await createRequestMessageAsync())
                    {
                        DateTime sendTimeUtc = DateTime.UtcNow;
                        Guid     localGuid   = Guid.NewGuid(); // For correlating HttpRequest and HttpResponse Traces

                        Guid requestedActivityId = Trace.CorrelationManager.ActivityId;
                        this.eventSource.Request(
                            requestedActivityId,
                            localGuid,
                            requestMessage.RequestUri.ToString(),
                            resourceType.ToResourceTypeString(),
                            requestMessage.Headers);

                        // Only read the header initially. The content gets copied into a memory stream later
                        // if we read the content http client will buffer the message and then it will get buffered
                        // again when it is copied to the memory stream.
                        HttpResponseMessage responseMessage = await this.httpClient.SendAsync(
                            requestMessage,
                            HttpCompletionOption.ResponseHeadersRead,
                            cancellationToken);

                        // WebAssembly HttpClient does not set the RequestMessage property on SendAsync
                        if (responseMessage.RequestMessage == null)
                        {
                            responseMessage.RequestMessage = requestMessage;
                        }

                        DateTime receivedTimeUtc  = DateTime.UtcNow;
                        TimeSpan durationTimeSpan = receivedTimeUtc - sendTimeUtc;

                        Guid activityId = Guid.Empty;
                        if (responseMessage.Headers.TryGetValues(
                                HttpConstants.HttpHeaders.ActivityId,
                                out IEnumerable <string> headerValues) && headerValues.Any())
                        {
                            activityId = new Guid(headerValues.First());
                        }

                        this.eventSource.Response(
                            activityId,
                            localGuid,
                            (short)responseMessage.StatusCode,
                            durationTimeSpan.TotalMilliseconds,
                            responseMessage.Headers);

                        return(responseMessage);
                    }
                }
            };

            HttpRequestMessage GetHttpRequestMessage()
            {
                return(requestMessage);
            }

            return(BackoffRetryUtility <HttpResponseMessage> .ExecuteAsync(
                       callbackMethod : funcDelegate,
                       retryPolicy : new TransientHttpClientRetryPolicy(
                           getHttpRequestMessage: GetHttpRequestMessage,
                           gatewayRequestTimeout: this.httpClient.Timeout,
                           diagnosticsContext: diagnosticsContext),
                       cancellationToken : cancellationToken));
        }
Пример #10
0
        private async Task <HttpResponseMessage> InvokeClientAsync(
            DocumentServiceRequest request,
            ResourceType resourceType,
            Uri physicalAddress,
            CancellationToken cancellationToken)
        {
            Func <Task <HttpResponseMessage> > funcDelegate = async() =>
            {
                using (HttpRequestMessage requestMessage = await this.PrepareRequestMessageAsync(request, physicalAddress))
                {
                    DateTime sendTimeUtc = DateTime.UtcNow;
                    Guid     localGuid   = Guid.NewGuid(); // For correlating HttpRequest and HttpResponse Traces

                    IClientSideRequestStatistics clientSideRequestStatistics = request.RequestContext.ClientRequestStatistics;
                    if (clientSideRequestStatistics == null)
                    {
                        clientSideRequestStatistics = new CosmosClientSideRequestStatistics();
                        request.RequestContext.ClientRequestStatistics = clientSideRequestStatistics;
                    }

                    Guid requestedActivityId = Trace.CorrelationManager.ActivityId;
                    this.eventSource.Request(
                        requestedActivityId,
                        localGuid,
                        requestMessage.RequestUri.ToString(),
                        resourceType.ToResourceTypeString(),
                        requestMessage.Headers);

                    TimeSpan durationTimeSpan;
                    string   recordAddressResolutionId = clientSideRequestStatistics.RecordAddressResolutionStart(requestMessage.RequestUri);
                    try
                    {
                        HttpResponseMessage responseMessage = await this.httpClient.SendAsync(requestMessage, cancellationToken);

                        DateTime receivedTimeUtc = DateTime.UtcNow;
                        durationTimeSpan = receivedTimeUtc - sendTimeUtc;

                        IEnumerable <string> headerValues;
                        Guid activityId = Guid.Empty;
                        if (responseMessage.Headers.TryGetValues(HttpConstants.HttpHeaders.ActivityId, out headerValues) &&
                            headerValues.Count() != 0)
                        {
                            activityId = new Guid(headerValues.First());
                        }

                        this.eventSource.Response(
                            activityId,
                            localGuid,
                            (short)responseMessage.StatusCode,
                            durationTimeSpan.TotalMilliseconds,
                            responseMessage.Headers);

                        return(responseMessage);
                    }
                    catch (TaskCanceledException ex)
                    {
                        if (!cancellationToken.IsCancellationRequested)
                        {
                            // throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
                            durationTimeSpan = DateTime.UtcNow - sendTimeUtc;
                            string message = $"GatewayStoreClient Request Timeout. Start Time:{sendTimeUtc}; Total Duration:{durationTimeSpan}; Http Client Timeout:{this.httpClient.Timeout}; Activity id: {requestedActivityId}; Inner Message: {ex.Message};";
                            throw new RequestTimeoutException(message, ex, requestMessage.RequestUri);
                        }
                        else
                        {
                            throw;
                        }
                    }
                    catch (OperationCanceledException ex)
                    {
                        if (!cancellationToken.IsCancellationRequested)
                        {
                            // throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
                            durationTimeSpan = DateTime.UtcNow - sendTimeUtc;
                            string message = $"GatewayStoreClient Request Timeout. Start Time:{sendTimeUtc}; Total Duration:{durationTimeSpan}; Http Client Timeout:{this.httpClient.Timeout}; Activity id: {requestedActivityId}; Inner Message: {ex.Message};";
                            throw new RequestTimeoutException(message, ex, requestMessage.RequestUri);
                        }
                        else
                        {
                            throw;
                        }
                    }
                    finally
                    {
                        clientSideRequestStatistics.RecordAddressResolutionEnd(recordAddressResolutionId);
                    }
                }
            };

            return(await BackoffRetryUtility <HttpResponseMessage> .ExecuteAsync(funcDelegate, new WebExceptionRetryPolicy(), cancellationToken));
        }
        protected override async Task <DocumentFeedResponse <CosmosElement> > ExecuteInternalAsync(CancellationToken token)
        {
            CollectionCache collectionCache = await this.Client.GetCollectionCacheAsync();

            PartitionKeyRangeCache partitionKeyRangeCache = await this.Client.GetPartitionKeyRangeCacheAsync();

            IDocumentClientRetryPolicy retryPolicyInstance = this.Client.ResetSessionTokenRetryPolicy.GetRequestPolicy();

            retryPolicyInstance = new InvalidPartitionExceptionRetryPolicy(retryPolicyInstance);
            if (base.ResourceTypeEnum.IsPartitioned())
            {
                retryPolicyInstance = new PartitionKeyRangeGoneRetryPolicy(
                    collectionCache,
                    partitionKeyRangeCache,
                    PathsHelper.GetCollectionPath(base.ResourceLink),
                    retryPolicyInstance);
            }

            return(await BackoffRetryUtility <DocumentFeedResponse <CosmosElement> > .ExecuteAsync(
                       async() =>
            {
                this.fetchExecutionRangeAccumulator.BeginFetchRange();
                ++this.retries;
                Tuple <DocumentFeedResponse <CosmosElement>, string> responseAndPartitionIdentifier = await this.ExecuteOnceAsync(retryPolicyInstance, token);
                DocumentFeedResponse <CosmosElement> response = responseAndPartitionIdentifier.Item1;
                string partitionIdentifier = responseAndPartitionIdentifier.Item2;
                if (!string.IsNullOrEmpty(response.ResponseHeaders[HttpConstants.HttpHeaders.QueryMetrics]))
                {
                    this.fetchExecutionRangeAccumulator.EndFetchRange(
                        partitionIdentifier,
                        response.ActivityId,
                        response.Count,
                        this.retries);
                    response = new DocumentFeedResponse <CosmosElement>(
                        response,
                        response.Count,
                        response.Headers,
                        response.UseETagAsContinuation,
                        new Dictionary <string, QueryMetrics>
                    {
                        {
                            partitionIdentifier,
                            QueryMetrics.CreateFromDelimitedStringAndClientSideMetrics(
                                response.ResponseHeaders[HttpConstants.HttpHeaders.QueryMetrics],
                                response.ResponseHeaders[HttpConstants.HttpHeaders.IndexUtilization],
                                new ClientSideMetrics(
                                    this.retries,
                                    response.RequestCharge,
                                    this.fetchExecutionRangeAccumulator.GetExecutionRanges()))
                        }
                    },
                        response.RequestStatistics,
                        response.DisallowContinuationTokenMessage,
                        response.ResponseLengthBytes);
                }

                this.retries = -1;
                return response;
            },
                       retryPolicyInstance,
                       token));
        }
        private async Task ValidateRetryOnDatabaseAccountNotFoundAsync(bool enableMultipleWriteLocations, bool isReadRequest)
        {
            this.Initialize(
                useMultipleWriteLocations: enableMultipleWriteLocations,
                enableEndpointDiscovery: true,
                isPreferredLocationsListEmpty: false);

            await this.endpointManager.RefreshLocationAsync(this.databaseAccount);

            ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, true, new RetryOptions());

            int expectedRetryCount = isReadRequest || enableMultipleWriteLocations ? 2 : 1;

            using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false))
            {
                int retryCount = 0;

                try
                {
                    await BackoffRetryUtility <bool> .ExecuteAsync(
                        () =>
                    {
                        retryCount++;
                        retryPolicy.OnBeforeSendRequest(request);

                        if (retryCount == 1)
                        {
                            Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]];

                            Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);

                            StoreRequestNameValueCollection headers       = new StoreRequestNameValueCollection();
                            headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.DatabaseAccountNotFound).ToString();
                            DocumentClientException forbiddenException    = new ForbiddenException(RMResources.NotFound, headers);

                            throw forbiddenException;
                        }
                        else if (retryCount == 2)
                        {
                            // Next request must go to next preferred endpoint
                            Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]];
                            Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);

                            return(Task.FromResult(true));
                        }
                        else
                        {
                            Assert.Fail();
                        }

                        return(Task.FromResult(true));
                    },
                        retryPolicy);
                }
                catch (ForbiddenException)
                {
                    if (expectedRetryCount == 1)
                    {
                        DefaultTrace.TraceInformation("Received expected ForbiddenException");
                    }
                    else
                    {
                        Assert.Fail();
                    }
                }

                Assert.AreEqual(expectedRetryCount, retryCount);
            }
        }
        private async Task ValidateRetryOnWriteSessionNotAvailabeWithEnableMultipleWriteLocationsAsync()
        {
            const bool useMultipleWriteLocations = true;
            bool       enableEndpointDiscovery   = true;

            this.Initialize(
                useMultipleWriteLocations: useMultipleWriteLocations,
                enableEndpointDiscovery: enableEndpointDiscovery,
                isPreferredLocationsListEmpty: false);

            await this.endpointManager.RefreshLocationAsync(this.databaseAccount);

            ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, enableEndpointDiscovery, new RetryOptions());

            using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: false, isMasterResourceType: false))
            {
                int retryCount = 0;

                try
                {
                    await BackoffRetryUtility <bool> .ExecuteAsync(
                        () =>
                    {
                        retryPolicy.OnBeforeSendRequest(request);

                        if (retryCount == 0)
                        {
                            Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]];

                            Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
                        }
                        else if (retryCount == 1)
                        {
                            // Second request must go to first write endpoint
                            Uri expectedEndpoint = new Uri(this.databaseAccount.WriteLocationsInternal[0].Endpoint);

                            Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
                        }
                        else if (retryCount == 2)
                        {
                            // Second request must go to first write endpoint
                            Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]];
                            Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
                        }
                        else if (retryCount == 3)
                        {
                            // Second request must go to first write endpoint
                            Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[2]];
                            Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
                        }
                        else
                        {
                            Assert.Fail();
                        }

                        retryCount++;

                        StoreRequestNameValueCollection headers       = new StoreRequestNameValueCollection();
                        headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.ReadSessionNotAvailable).ToString();
                        DocumentClientException notFoundException     = new NotFoundException(RMResources.NotFound, headers);


                        throw notFoundException;
                    },
                        retryPolicy);

                    Assert.Fail();
                }
                catch (NotFoundException)
                {
                    DefaultTrace.TraceInformation("Received expected notFoundException");
                    Assert.AreEqual(4, retryCount);
                }
            }
        }
        private async Task <HttpResponseMessage> InvokeClientAsync(
            DocumentServiceRequest request,
            ResourceType resourceType,
            Uri physicalAddress,
            CancellationToken cancellationToken)
        {
            Func <Task <HttpResponseMessage> > funcDelegate = async() =>
            {
                using (HttpRequestMessage requestMessage = await this.PrepareRequestMessageAsync(request, physicalAddress))
                {
                    DateTime sendTimeUtc = DateTime.UtcNow;
                    Guid     localGuid   = Guid.NewGuid(); // For correlating HttpRequest and HttpResponse Traces

                    this.eventSource.Request(
                        Guid.Empty,
                        localGuid,
                        requestMessage.RequestUri.ToString(),
                        resourceType.ToResourceTypeString(),
                        requestMessage.Headers);

                    try
                    {
                        HttpResponseMessage responseMessage = await this.httpClient.SendAsync(requestMessage, cancellationToken);

                        DateTime receivedTimeUtc        = DateTime.UtcNow;
                        double   durationInMilliSeconds = (receivedTimeUtc - sendTimeUtc).TotalMilliseconds;

                        IEnumerable <string> headerValues;
                        Guid activityId = Guid.Empty;
                        if (responseMessage.Headers.TryGetValues(HttpConstants.HttpHeaders.ActivityId, out headerValues) &&
                            headerValues.Count() != 0)
                        {
                            activityId = new Guid(headerValues.First());
                        }

                        this.eventSource.Response(
                            activityId,
                            localGuid,
                            (short)responseMessage.StatusCode,
                            durationInMilliSeconds,
                            responseMessage.Headers);

                        return(responseMessage);
                    }
                    catch (TaskCanceledException ex)
                    {
                        if (!cancellationToken.IsCancellationRequested)
                        {
                            // throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
                            throw new RequestTimeoutException(ex, requestMessage.RequestUri);
                        }
                        else
                        {
                            throw;
                        }
                    }
                }
            };

            return(await BackoffRetryUtility <HttpResponseMessage> .ExecuteAsync(funcDelegate, new WebExceptionRetryPolicy(), cancellationToken));
        }
Пример #15
0
        protected override async Task <FeedResponse <dynamic> > ExecuteInternalAsync(CancellationToken cancellationToken)
        {
            CollectionCache collectionCache = await this.Client.GetCollectionCacheAsync();

            PartitionKeyRangeCache partitionKeyRangeCache = await this.Client.GetPartitionKeyRangeCache();

            IDocumentClientRetryPolicy retryPolicyInstance = this.Client.RetryPolicy.GetRequestPolicy();

            retryPolicyInstance = new InvalidPartitionExceptionRetryPolicy(collectionCache, retryPolicyInstance);
            if (base.ResourceTypeEnum.IsPartitioned())
            {
                retryPolicyInstance = new PartitionKeyRangeGoneRetryPolicy(
                    collectionCache,
                    partitionKeyRangeCache,
                    PathsHelper.GetCollectionPath(base.ResourceLink),
                    retryPolicyInstance);
            }

            return(await BackoffRetryUtility <FeedResponse <dynamic> > .ExecuteAsync(
                       async() =>
            {
                this.fetchExecutionRangeAccumulator.BeginFetchRange();
                ++this.retries;
                this.fetchSchedulingMetrics.Start();
                this.fetchExecutionRangeAccumulator.BeginFetchRange();
                FeedResponse <dynamic> response = await this.ExecuteOnceAsync(retryPolicyInstance, cancellationToken);
                this.fetchSchedulingMetrics.Stop();
                this.fetchExecutionRangeAccumulator.EndFetchRange(response.Count, this.retries);

                if (!string.IsNullOrEmpty(response.Headers[HttpConstants.HttpHeaders.QueryMetrics]))
                {
                    this.fetchExecutionRangeAccumulator.EndFetchRange(response.Count, this.retries);
                    response = new FeedResponse <dynamic>(
                        response,
                        response.Count,
                        response.Headers,
                        response.UseETagAsContinuation,
                        new Dictionary <string, QueryMetrics>
                    {
                        {
                            singlePartitionKeyId,
                            QueryMetrics.CreateFromDelimitedStringAndClientSideMetrics(
                                response.Headers[HttpConstants.HttpHeaders.QueryMetrics],
                                new ClientSideMetrics(
                                    this.retries,
                                    response.RequestCharge,
                                    this.fetchExecutionRangeAccumulator.GetExecutionRanges(),
                                    string.IsNullOrEmpty(response.ResponseContinuation) ? new List <Tuple <string, SchedulingTimeSpan> >()
                            {
                                new Tuple <string, SchedulingTimeSpan>(singlePartitionKeyId, this.fetchSchedulingMetrics.Elapsed)
                            } : new List <Tuple <string, SchedulingTimeSpan> >()),
                                Guid.Parse(response.ActivityId))
                        }
                    },
                        response.RequestStatistics,
                        response.DisallowContinuationTokenMessage,
                        response.ResponseLengthBytes);
                }

                this.retries = -1;
                return response;
            },
                       retryPolicyInstance,
                       cancellationToken));
        }
        public async Task ClientRetryPolicy_ValidateRetryOnServiceUnavailable(
            bool isReadRequest,
            bool useMultipleWriteLocations,
            bool usesPreferredLocations,
            bool shouldHaveRetried)
        {
            const bool enableEndpointDiscovery = true;

            this.Initialize(
                useMultipleWriteLocations: useMultipleWriteLocations,
                enableEndpointDiscovery: enableEndpointDiscovery,
                isPreferredLocationsListEmpty: !usesPreferredLocations);

            await this.endpointManager.RefreshLocationAsync(this.databaseAccount);

            ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, enableEndpointDiscovery, new RetryOptions());

            using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false))
            {
                int retryCount = 0;

                try
                {
                    await BackoffRetryUtility <bool> .ExecuteAsync(
                        () =>
                    {
                        retryPolicy.OnBeforeSendRequest(request);

                        if (retryCount == 1)
                        {
                            Uri expectedEndpoint = null;
                            if (usesPreferredLocations)
                            {
                                expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]];
                            }
                            else
                            {
                                if (isReadRequest)
                                {
                                    expectedEndpoint = new Uri(this.databaseAccount.ReadLocationsInternal[1].Endpoint);
                                }
                                else
                                {
                                    expectedEndpoint = new Uri(this.databaseAccount.WriteLocationsInternal[1].Endpoint);
                                }
                            }

                            Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
                        }
                        else if (retryCount > 1)
                        {
                            Assert.Fail("Should retry once");
                        }

                        retryCount++;

                        throw new ServiceUnavailableException();
                    },
                        retryPolicy);

                    Assert.Fail();
                }
                catch (ServiceUnavailableException)
                {
                    DefaultTrace.TraceInformation("Received expected ServiceUnavailableException");
                    if (shouldHaveRetried)
                    {
                        Assert.AreEqual(2, retryCount, $"Retry count {retryCount}, shouldHaveRetried {shouldHaveRetried} isReadRequest {isReadRequest} useMultipleWriteLocations {useMultipleWriteLocations} usesPreferredLocations {usesPreferredLocations}");
                    }
                    else
                    {
                        Assert.AreEqual(1, retryCount, $"Retry count {retryCount}, shouldHaveRetried {shouldHaveRetried} isReadRequest {isReadRequest} useMultipleWriteLocations {useMultipleWriteLocations} usesPreferredLocations {usesPreferredLocations}");
                    }
                }
            }
        }
        private async Task <HttpResponseMessage> InvokeClientAsync(
            DocumentServiceRequest request,
            ResourceType resourceType,
            Uri physicalAddress,
            CancellationToken cancellationToken)
        {
            Func <Task <HttpResponseMessage> > funcDelegate = async() =>
            {
                using (HttpRequestMessage requestMessage = await this.PrepareRequestMessageAsync(request, physicalAddress))
                {
                    DateTime sendTimeUtc = DateTime.UtcNow;
                    Guid     localGuid   = Guid.NewGuid(); // For correlating HttpRequest and HttpResponse Traces

                    // DEVNOTE: This is temporary until IClientSideRequestStats is modified to support gateway calls.
                    CosmosDiagnosticsContext diagnosticsContext = null;
                    if (request.RequestContext.ClientRequestStatistics is CosmosClientSideRequestStatistics clientSideRequestStatistics)
                    {
                        diagnosticsContext = clientSideRequestStatistics.DiagnosticsContext;
                    }
                    else
                    {
                        // ClientRequestStatistics are not passed in use Empty to avoid null checks.
                        // Caches do not pass in the ClientRequestStatistics
                        diagnosticsContext = EmptyCosmosDiagnosticsContext.Singleton;
                    }

                    Guid requestedActivityId = Trace.CorrelationManager.ActivityId;
                    this.eventSource.Request(
                        requestedActivityId,
                        localGuid,
                        requestMessage.RequestUri.ToString(),
                        resourceType.ToResourceTypeString(),
                        requestMessage.Headers);

                    TimeSpan durationTimeSpan;
                    try
                    {
                        HttpResponseMessage responseMessage;
                        using (diagnosticsContext.CreateScope("GatewayRequestTime"))
                        {
                            responseMessage = await this.httpClient.SendAsync(requestMessage, cancellationToken);
                        }

                        DateTime receivedTimeUtc = DateTime.UtcNow;
                        durationTimeSpan = receivedTimeUtc - sendTimeUtc;

                        IEnumerable <string> headerValues;
                        Guid activityId = Guid.Empty;
                        if (responseMessage.Headers.TryGetValues(HttpConstants.HttpHeaders.ActivityId, out headerValues) &&
                            headerValues.Count() != 0)
                        {
                            activityId = new Guid(headerValues.First());
                        }

                        this.eventSource.Response(
                            activityId,
                            localGuid,
                            (short)responseMessage.StatusCode,
                            durationTimeSpan.TotalMilliseconds,
                            responseMessage.Headers);

                        return(responseMessage);
                    }
                    catch (TaskCanceledException ex)
                    {
                        if (!cancellationToken.IsCancellationRequested)
                        {
                            // throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
                            durationTimeSpan = DateTime.UtcNow - sendTimeUtc;
                            string message = $"GatewayStoreClient Request Timeout. Start Time:{sendTimeUtc}; Total Duration:{durationTimeSpan}; Http Client Timeout:{this.httpClient.Timeout}; Activity id: {requestedActivityId}; Inner Message: {ex.Message};";
                            throw new RequestTimeoutException(message, ex, requestMessage.RequestUri);
                        }
                        else
                        {
                            throw;
                        }
                    }
                    catch (OperationCanceledException ex)
                    {
                        if (!cancellationToken.IsCancellationRequested)
                        {
                            // throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
                            durationTimeSpan = DateTime.UtcNow - sendTimeUtc;
                            string message = $"GatewayStoreClient Request Timeout. Start Time:{sendTimeUtc}; Total Duration:{durationTimeSpan}; Http Client Timeout:{this.httpClient.Timeout}; Activity id: {requestedActivityId}; Inner Message: {ex.Message};";
                            throw new RequestTimeoutException(message, ex, requestMessage.RequestUri);
                        }
                        else
                        {
                            throw;
                        }
                    }
                }
            };

            return(await BackoffRetryUtility <HttpResponseMessage> .ExecuteAsync(funcDelegate, new WebExceptionRetryPolicy(), cancellationToken));
        }
Пример #18
0
        private async Task <CollectionRoutingMap> GetRoutingMapForCollectionAsync(
            string collectionRid,
            CollectionRoutingMap previousRoutingMap,
            ITrace trace,
            IClientSideRequestStatistics clientSideRequestStatistics,
            CancellationToken cancellationToken)
        {
            List <PartitionKeyRange> ranges  = new List <PartitionKeyRange>();
            string changeFeedNextIfNoneMatch = previousRoutingMap == null ? null : previousRoutingMap.ChangeFeedNextIfNoneMatch;

            HttpStatusCode lastStatusCode = HttpStatusCode.OK;

            do
            {
                INameValueCollection headers = new StoreRequestNameValueCollection();

                headers.Set(HttpConstants.HttpHeaders.PageSize, PageSizeString);
                headers.Set(HttpConstants.HttpHeaders.A_IM, HttpConstants.A_IMHeaderValues.IncrementalFeed);
                if (changeFeedNextIfNoneMatch != null)
                {
                    headers.Set(HttpConstants.HttpHeaders.IfNoneMatch, changeFeedNextIfNoneMatch);
                }

                RetryOptions retryOptions = new RetryOptions();
                using (DocumentServiceResponse response = await BackoffRetryUtility <DocumentServiceResponse> .ExecuteAsync(
                           () => this.ExecutePartitionKeyRangeReadChangeFeedAsync(collectionRid, headers, trace, clientSideRequestStatistics),
                           new ResourceThrottleRetryPolicy(retryOptions.MaxRetryAttemptsOnThrottledRequests, retryOptions.MaxRetryWaitTimeInSeconds),
                           cancellationToken))
                {
                    lastStatusCode            = response.StatusCode;
                    changeFeedNextIfNoneMatch = response.Headers[HttpConstants.HttpHeaders.ETag];

                    FeedResource <PartitionKeyRange> feedResource = response.GetResource <FeedResource <PartitionKeyRange> >();
                    if (feedResource != null)
                    {
                        ranges.AddRange(feedResource);
                    }
                }
            }while (lastStatusCode != HttpStatusCode.NotModified);

            IEnumerable <Tuple <PartitionKeyRange, ServiceIdentity> > tuples = ranges.Select(range => Tuple.Create(range, (ServiceIdentity)null));

            CollectionRoutingMap routingMap;

            if (previousRoutingMap == null)
            {
                // Splits could have happened during change feed query and we might have a mix of gone and new ranges.
                HashSet <string> goneRanges = new HashSet <string>(ranges.SelectMany(range => range.Parents ?? Enumerable.Empty <string>()));
                routingMap = CollectionRoutingMap.TryCreateCompleteRoutingMap(
                    tuples.Where(tuple => !goneRanges.Contains(tuple.Item1.Id)),
                    string.Empty,
                    changeFeedNextIfNoneMatch);
            }
            else
            {
                routingMap = previousRoutingMap.TryCombine(tuples, changeFeedNextIfNoneMatch);
            }

            if (routingMap == null)
            {
                // Range information either doesn't exist or is not complete.
                throw new NotFoundException($"{DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture)}: GetRoutingMapForCollectionAsync(collectionRid: {collectionRid}), Range information either doesn't exist or is not complete.");
            }

            return(routingMap);
        }
        private Task <HttpResponseMessage> SendHttpAsync(
            Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
            HttpCompletionOption httpCompletionOption,
            ResourceType resourceType,
            CosmosDiagnosticsContext diagnosticsContext,
            CancellationToken cancellationToken)
        {
            diagnosticsContext ??= new CosmosDiagnosticsContextCore();
            HttpRequestMessage requestMessage = null;
            Func <Task <HttpResponseMessage> > funcDelegate = async() =>
            {
                using (diagnosticsContext.CreateScope(nameof(CosmosHttpClientCore.SendHttpAsync)))
                {
                    using (requestMessage = await createRequestMessageAsync())
                    {
                        DateTime sendTimeUtc = DateTime.UtcNow;
                        Guid     localGuid   = Guid.NewGuid(); // For correlating HttpRequest and HttpResponse Traces

                        Guid requestedActivityId = Trace.CorrelationManager.ActivityId;
                        this.eventSource.Request(
                            requestedActivityId,
                            localGuid,
                            requestMessage.RequestUri.ToString(),
                            resourceType.ToResourceTypeString(),
                            requestMessage.Headers);

                        HttpResponseMessage responseMessage = await this.httpClient.SendAsync(
                            requestMessage,
                            httpCompletionOption,
                            cancellationToken);

                        DateTime receivedTimeUtc  = DateTime.UtcNow;
                        TimeSpan durationTimeSpan = receivedTimeUtc - sendTimeUtc;

                        Guid activityId = Guid.Empty;
                        if (responseMessage.Headers.TryGetValues(
                                HttpConstants.HttpHeaders.ActivityId,
                                out IEnumerable <string> headerValues) && headerValues.Any())
                        {
                            activityId = new Guid(headerValues.First());
                        }

                        this.eventSource.Response(
                            activityId,
                            localGuid,
                            (short)responseMessage.StatusCode,
                            durationTimeSpan.TotalMilliseconds,
                            responseMessage.Headers);

                        return(responseMessage);
                    }
                }
            };

            HttpRequestMessage GetHttpRequestMessage() => requestMessage;

            return(BackoffRetryUtility <HttpResponseMessage> .ExecuteAsync(
                       callbackMethod : funcDelegate,
                       retryPolicy : new TransientHttpClientRetryPolicy(
                           getHttpRequestMessage: GetHttpRequestMessage,
                           gatewayRequestTimeout: this.httpClient.Timeout,
                           diagnosticsContext: diagnosticsContext),
                       cancellationToken : cancellationToken));
        }