public override async Task <ResponseMessage> SendAsync( RequestMessage request, CancellationToken cancellationToken) { IDocumentClientRetryPolicy retryPolicyInstance = await this.GetRetryPolicyAsync(request); request.OnBeforeSendRequestActions += retryPolicyInstance.OnBeforeSendRequest; try { return(await RetryHandler.ExecuteHttpRequestAsync( callbackMethod : () => { return base.SendAsync(request, cancellationToken); }, callShouldRetry : (cosmosResponseMessage, token) => { return retryPolicyInstance.ShouldRetryAsync(cosmosResponseMessage, cancellationToken); }, callShouldRetryException : (exception, token) => { return retryPolicyInstance.ShouldRetryAsync(exception, cancellationToken); }, diagnosticsContext : request.DiagnosticsContext, cancellationToken : cancellationToken)); } catch (DocumentClientException ex) { return(ex.ToCosmosResponseMessage(request)); } catch (CosmosException ex) { return(ex.ToCosmosResponseMessage(request)); } catch (AggregateException ex) { // TODO: because the SDK underneath this path uses ContinueWith or task.Result we need to catch AggregateExceptions here // in order to ensure that underlying DocumentClientExceptions get propagated up correctly. Once all ContinueWith and .Result // is removed this catch can be safely removed. AggregateException innerExceptions = ex.Flatten(); Exception docClientException = innerExceptions.InnerExceptions.FirstOrDefault(innerEx => innerEx is DocumentClientException); if (docClientException != null) { return(((DocumentClientException)docClientException).ToCosmosResponseMessage(request)); } throw; } finally { request.OnBeforeSendRequestActions -= retryPolicyInstance.OnBeforeSendRequest; } }
public async Task <DocumentFeedResponse <CosmosElement> > ExecuteRequestLazyAsync( DocumentServiceRequest request, IDocumentClientRetryPolicy retryPolicyInstance, CancellationToken cancellationToken) { DocumentServiceResponse documentServiceResponse = await this.ExecuteQueryRequestInternalAsync( request, retryPolicyInstance, cancellationToken); return(this.GetFeedResponse(request, documentServiceResponse)); }
public InvalidPartitionExceptionRetryPolicy( CollectionCache clientCollectionCache, IDocumentClientRetryPolicy nextPolicy) { if (clientCollectionCache == null) { throw new ArgumentNullException("clientCollectionCache"); } this.clientCollectionCache = clientCollectionCache; this.nextPolicy = nextPolicy; }
public PartitionKeyMismatchRetryPolicy( CollectionCache clientCollectionCache, IDocumentClientRetryPolicy nextRetryPolicy) { if (clientCollectionCache == null) { throw new ArgumentNullException("clientCollectionCache"); } this.clientCollectionCache = clientCollectionCache; this.nextRetryPolicy = nextRetryPolicy; }
public PartitionKeyRangeGoneRetryPolicy( CollectionCache collectionCache, PartitionKeyRangeCache partitionKeyRangeCache, string collectionLink, IDocumentClientRetryPolicy nextRetryPolicy, ITrace trace) { this.collectionCache = collectionCache; this.partitionKeyRangeCache = partitionKeyRangeCache; this.collectionLink = collectionLink; this.nextRetryPolicy = nextRetryPolicy; this.trace = trace; }
private async Task <DocumentServiceResponse> ExecuteQueryRequestInternalAsync( DocumentServiceRequest request, IDocumentClientRetryPolicy retryPolicyInstance, CancellationToken cancellationToken) { try { return(await this.Client.ExecuteQueryAsync(request, retryPolicyInstance, cancellationToken)); } finally { request.Body.Position = 0; } }
public ItemBatchOperationContext( string partitionKeyRangeId, ITrace trace, IDocumentClientRetryPolicy retryPolicy = null) { if (trace == null) { throw new ArgumentNullException(nameof(trace)); } this.PartitionKeyRangeId = partitionKeyRangeId ?? throw new ArgumentNullException(nameof(partitionKeyRangeId)); this.initialTrace = trace; this.retryPolicy = retryPolicy; }
private async Task <ContainerProperties> ReadCollectionAsync(string collectionLink, CancellationToken cancellationToken, IDocumentClientRetryPolicy retryPolicyInstance, ITrace trace) { using (ITrace childTrace = trace.StartChild("Read Collection", TraceComponent.Transport, TraceLevel.Info)) { cancellationToken.ThrowIfCancellationRequested(); using (DocumentServiceRequest request = DocumentServiceRequest.Create( OperationType.Read, ResourceType.Collection, collectionLink, AuthorizationTokenType.PrimaryMasterKey, new StoreRequestNameValueCollection())) { request.Headers[HttpConstants.HttpHeaders.XDate] = DateTime.UtcNow.ToString("r"); (string authorizationToken, string payload) = await this.tokenProvider.GetUserAuthorizationAsync( request.ResourceAddress, PathsHelper.GetResourcePath(request.ResourceType), HttpConstants.HttpMethods.Get, request.Headers, AuthorizationTokenType.PrimaryMasterKey); request.Headers[HttpConstants.HttpHeaders.Authorization] = authorizationToken; using (new ActivityScope(Guid.NewGuid())) { if (retryPolicyInstance != null) { retryPolicyInstance.OnBeforeSendRequest(request); } try { using (DocumentServiceResponse response = await this.storeModel.ProcessMessageAsync(request)) { return(CosmosResource.FromStream <ContainerProperties>(response)); } } catch (DocumentClientException ex) { childTrace.AddDatum("Exception Message", ex.Message); throw; } } } } }
public ClientRetryPolicy( GlobalEndpointManager globalEndpointManager, bool enableEndpointDiscovery, RetryOptions retryOptions) { this.throttlingRetry = new ResourceThrottleRetryPolicy( retryOptions.MaxRetryAttemptsOnThrottledRequests, retryOptions.MaxRetryWaitTimeInSeconds); this.globalEndpointManager = globalEndpointManager; this.failoverRetryCount = 0; this.enableEndpointDiscovery = enableEndpointDiscovery; this.sessionTokenRetryCount = 0; this.canUseMultipleWriteLocations = false; }
private void ScheduleFetch(IDocumentClientRetryPolicy retryPolicyInstance, TimeSpan delay = default(TimeSpan)) { // For the same DocumentProducer, the priorities of scheduled tasks monotonically decrease. // This makes sure the tasks are scheduled fairly across all DocumentProducer's. bool scheduled = this.taskScheduler.TryQueueTask( new DocumentProducerComparableTask( this, retryPolicyInstance), delay); if (!scheduled) { Interlocked.Exchange(ref this.isFetching, 0); throw new InvalidOperationException("Failed to schedule"); } }
// Extracted partiotn key might be invaild as CollectionCache might be stale. // Stale collection cache is refreshed through PartitionKeyMismatchRetryPolicy // and partition-key is extracted again. internal async Task <CosmosResponseMessage> ExtractPartitionKeyAndProcessItemStreamAsync( PartitionKey partitionKey, string itemId, Stream streamPayload, OperationType operationType, RequestOptions requestOptions, CancellationToken cancellationToken) { IDocumentClientRetryPolicy requestRetryPolicy = null; if (partitionKey == null) { requestRetryPolicy = new PartitionKeyMismatchRetryPolicy(await this.ClientContext.DocumentClient.GetCollectionCacheAsync(), requestRetryPolicy); } while (true) { CosmosResponseMessage responseMessage = await this.ProcessItemStreamAsync( partitionKey, itemId, streamPayload, operationType, requestOptions, extractPartitionKeyIfNeeded : true, cancellationToken : cancellationToken); ShouldRetryResult retryResult = ShouldRetryResult.NoRetry(); if (requestRetryPolicy != null && !responseMessage.IsSuccessStatusCode) { retryResult = await requestRetryPolicy.ShouldRetryAsync(responseMessage, cancellationToken); } if (!retryResult.ShouldRetry) { return(responseMessage); } } }
private async Task <FeedResponse <dynamic> > ExecuteOnceAsync(IDocumentClientRetryPolicy retryPolicyInstance, CancellationToken cancellationToken) { // Don't reuse request, as the rest of client SDK doesn't reuse requests between retries. // The code leaves some temporary garbage in request (in RequestContext etc.), // which shold be erased during retries. using (DocumentServiceRequest request = await this.CreateRequestAsync()) { if (retryPolicyInstance != null) { retryPolicyInstance.OnBeforeSendRequest(request); } if (!string.IsNullOrEmpty(request.Headers[HttpConstants.HttpHeaders.PartitionKey]) || !request.ResourceType.IsPartitioned()) { return(await this.ExecuteRequestAsync(request, cancellationToken)); } CollectionCache collectionCache = await this.Client.GetCollectionCacheAsync(); CosmosContainerSettings collection = await collectionCache.ResolveCollectionAsync(request, CancellationToken.None); if (!string.IsNullOrEmpty(base.PartitionKeyRangeId)) { request.RouteTo(new PartitionKeyRangeIdentity(collection.ResourceId, base.PartitionKeyRangeId)); return(await this.ExecuteRequestAsync(request, cancellationToken)); } // For non-Windows platforms(like Linux and OSX) in .NET Core SDK, we cannot use ServiceInterop for parsing the query, // so forcing the request through Gateway. We are also now by-passing this for 32-bit host process in NETFX on Windows // as the ServiceInterop dll is only available in 64-bit. if (CustomTypeExtensions.ByPassQueryParsing()) { request.UseGatewayMode = true; return(await this.ExecuteRequestAsync(request, cancellationToken)); } QueryPartitionProvider queryPartitionProvider = await this.Client.GetQueryPartitionProviderAsync(cancellationToken); IRoutingMapProvider routingMapProvider = await this.Client.GetRoutingMapProviderAsync(); List <CompositeContinuationToken> suppliedTokens; Range <string> rangeFromContinuationToken = this.partitionRoutingHelper.ExtractPartitionKeyRangeFromContinuationToken(request.Headers, out suppliedTokens); Tuple <PartitionRoutingHelper.ResolvedRangeInfo, IReadOnlyList <Range <string> > > queryRoutingInfo = await this.TryGetTargetPartitionKeyRangeAsync( request, collection, queryPartitionProvider, routingMapProvider, rangeFromContinuationToken, suppliedTokens); if (request.IsNameBased && queryRoutingInfo == null) { request.ForceNameCacheRefresh = true; collection = await collectionCache.ResolveCollectionAsync(request, CancellationToken.None); queryRoutingInfo = await this.TryGetTargetPartitionKeyRangeAsync( request, collection, queryPartitionProvider, routingMapProvider, rangeFromContinuationToken, suppliedTokens); } if (queryRoutingInfo == null) { throw new NotFoundException($"{DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture)}: Was not able to get queryRoutingInfo even after resolve collection async with force name cache refresh to the following collectionRid: {collection.ResourceId} with the supplied tokens: {JsonConvert.SerializeObject(suppliedTokens)}"); } request.RouteTo(new PartitionKeyRangeIdentity(collection.ResourceId, queryRoutingInfo.Item1.ResolvedRange.Id)); FeedResponse <dynamic> response = await this.ExecuteRequestAsync(request, cancellationToken); if (!await this.partitionRoutingHelper.TryAddPartitionKeyRangeToContinuationTokenAsync( response.Headers, providedPartitionKeyRanges: queryRoutingInfo.Item2, routingMapProvider: routingMapProvider, collectionRid: collection.ResourceId, resolvedRangeInfo: queryRoutingInfo.Item1)) { // Collection to which this request was resolved doesn't exist. // Retry policy will refresh the cache and return NotFound. throw new NotFoundException($"{DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture)}: Call to TryAddPartitionKeyRangeToContinuationTokenAsync failed to the following collectionRid: {collection.ResourceId} with the supplied tokens: {JsonConvert.SerializeObject(suppliedTokens)}"); } return(response); } }
public ClearingSessionContainerClientRetryPolicy(ISessionContainer sessionContainer, IDocumentClientRetryPolicy retryPolicy) { this.retryPolicy = retryPolicy; this.sessionContainer = sessionContainer; }
public Task <DocumentServiceResponse> ReadFeedAsync(DocumentServiceRequest request, IDocumentClientRetryPolicy retryPolicyInstance, CancellationToken cancellationToken) { return(this.innerClient.ReadFeedAsync(request, retryPolicyInstance, cancellationToken)); }
private async Task <CosmosContainerSettings> ReadCollectionAsync(string collectionLink, CancellationToken cancellationToken, IDocumentClientRetryPolicy retryPolicyInstance) { cancellationToken.ThrowIfCancellationRequested(); using (DocumentServiceRequest request = DocumentServiceRequest.Create( OperationType.Read, ResourceType.Collection, collectionLink, AuthorizationTokenType.PrimaryMasterKey, new StringKeyValueCollection())) { request.Headers[HttpConstants.HttpHeaders.XDate] = DateTime.UtcNow.ToString("r"); string authorizationToken = this.tokenProvider.GetUserAuthorizationToken( request.ResourceAddress, PathsHelper.GetResourcePath(request.ResourceType), HttpConstants.HttpMethods.Get, request.Headers, AuthorizationTokenType.PrimaryMasterKey); request.Headers[HttpConstants.HttpHeaders.Authorization] = authorizationToken; using (new ActivityScope(Guid.NewGuid())) { if (retryPolicyInstance != null) { retryPolicyInstance.OnBeforeSendRequest(request); } using (DocumentServiceResponse response = await this.storeModel.ProcessMessageAsync(request)) { return(new ResourceResponse <CosmosContainerSettings>(response).Resource); } } } }
/// <summary> /// Buffers more documents in the producer. /// </summary> /// <param name="token">The cancellation token.</param> /// <returns>A task to await on.</returns> public async Task BufferMoreDocumentsAsync(CancellationToken token) { token.ThrowIfCancellationRequested(); try { await this.fetchSemaphore.WaitAsync(); if (!this.HasMoreBackendResults) { // Just NOP return; } this.fetchSchedulingMetrics.Start(); this.fetchExecutionRangeAccumulator.BeginFetchRange(); int pageSize = (int)Math.Min(this.pageSize, (long)int.MaxValue); using (DocumentServiceRequest request = this.createRequestFunc(this.PartitionKeyRange, this.backendContinuationToken, pageSize)) { IDocumentClientRetryPolicy retryPolicy = this.createRetryPolicyFunc(); // Custom backoff and retry while (true) { int retries = 0; try { DocumentFeedResponse <CosmosElement> feedResponse = await this.executeRequestFunc(request, retryPolicy, token); this.fetchExecutionRangeAccumulator.EndFetchRange( this.PartitionKeyRange.Id, feedResponse.ActivityId, feedResponse.Count, retries); this.fetchSchedulingMetrics.Stop(); this.hasStartedFetching = true; this.backendContinuationToken = feedResponse.ResponseContinuation; this.activityId = Guid.Parse(feedResponse.ActivityId); await this.bufferedPages.AddAsync(TryMonad <DocumentFeedResponse <CosmosElement> > .FromResult(feedResponse)); Interlocked.Add(ref this.bufferedItemCount, feedResponse.Count); QueryMetrics queryMetrics = QueryMetrics.Zero; if (feedResponse.ResponseHeaders[HttpConstants.HttpHeaders.QueryMetrics] != null) { queryMetrics = QueryMetrics.CreateFromDelimitedStringAndClientSideMetrics( feedResponse.ResponseHeaders[HttpConstants.HttpHeaders.QueryMetrics], new ClientSideMetrics( retries, feedResponse.RequestCharge, this.fetchExecutionRangeAccumulator.GetExecutionRanges(), new List <Tuple <string, SchedulingTimeSpan> >())); } if (!this.HasMoreBackendResults) { queryMetrics = QueryMetrics.CreateWithSchedulingMetrics( queryMetrics, new List <Tuple <string, SchedulingTimeSpan> > { new Tuple <string, SchedulingTimeSpan>( this.PartitionKeyRange.Id, this.fetchSchedulingMetrics.Elapsed) }); } this.produceAsyncCompleteCallback( this, feedResponse.Count, feedResponse.RequestCharge, queryMetrics, feedResponse.ResponseLengthBytes, token); break; } catch (Exception exception) { // See if we need to retry or just throw ShouldRetryResult shouldRetryResult = await retryPolicy.ShouldRetryAsync(exception, token); if (!shouldRetryResult.ShouldRetry) { Exception exceptionToBuffer; if (shouldRetryResult.ExceptionToThrow != null) { exceptionToBuffer = shouldRetryResult.ExceptionToThrow; } else { // Propagate original exception. exceptionToBuffer = exception; } // Buffer the exception instead of throwing, since we don't want an unobserved exception. await this.bufferedPages.AddAsync(TryMonad <DocumentFeedResponse <CosmosElement> > .FromException(exceptionToBuffer)); // null out the backend continuation token, // so that people stop trying to buffer more on this producer. this.hasStartedFetching = true; this.backendContinuationToken = null; break; } else { await Task.Delay(shouldRetryResult.BackoffTime); retries++; } } } } } finally { this.fetchSchedulingMetrics.Stop(); this.fetchSemaphore.Release(); } }
public override async Task <ResponseMessage> SendAsync( RequestMessage request, CancellationToken cancellationToken) { using (ITrace childTrace = request.Trace.StartChild("Send Async", TraceComponent.RequestHandler, TraceLevel.Info)) { request.Trace = childTrace; IDocumentClientRetryPolicy retryPolicyInstance = await this.GetRetryPolicyAsync(request); request.OnBeforeSendRequestActions += retryPolicyInstance.OnBeforeSendRequest; try { return(await RetryHandler.ExecuteHttpRequestAsync( callbackMethod : async(trace) => { using (ITrace childTrace = trace.StartChild("Callback Method")) { request.Trace = childTrace; return await base.SendAsync(request, cancellationToken); } }, callShouldRetry : async(cosmosResponseMessage, trace, token) => { using (ITrace shouldRetryTrace = trace.StartChild("Call Should Retry")) { request.Trace = shouldRetryTrace; return await retryPolicyInstance.ShouldRetryAsync(cosmosResponseMessage, cancellationToken); } }, callShouldRetryException : async(exception, trace, token) => { using (ITrace shouldRetryTrace = trace.StartChild("Call Should Retry Exception")) { request.Trace = shouldRetryTrace; return await retryPolicyInstance.ShouldRetryAsync(exception, cancellationToken); } }, trace : request.Trace, cancellationToken : cancellationToken)); } catch (DocumentClientException ex) { return(ex.ToCosmosResponseMessage(request)); } catch (CosmosException ex) { return(ex.ToCosmosResponseMessage(request)); } catch (AggregateException ex) { // TODO: because the SDK underneath this path uses ContinueWith or task.Result we need to catch AggregateExceptions here // in order to ensure that underlying DocumentClientExceptions get propagated up correctly. Once all ContinueWith and .Result // is removed this catch can be safely removed. AggregateException innerExceptions = ex.Flatten(); Exception docClientException = innerExceptions.InnerExceptions.FirstOrDefault(innerEx => innerEx is DocumentClientException); if (docClientException != null) { return(((DocumentClientException)docClientException).ToCosmosResponseMessage(request)); } throw; } finally { request.OnBeforeSendRequestActions -= retryPolicyInstance.OnBeforeSendRequest; } } }
public RenameCollectionAwareClientRetryPolicy(ISessionContainer sessionContainer, ClientCollectionCache collectionCache, IDocumentClientRetryPolicy retryPolicy) { this.retryPolicy = retryPolicy; this.sessionContainer = sessionContainer; this.collectionCache = collectionCache; this.request = null; }
private async Task <Tuple <DocumentFeedResponse <CosmosElement>, string> > ExecuteOnceAsync( IDocumentClientRetryPolicy retryPolicyInstance, CancellationToken cancellationToken) { // Don't reuse request, as the rest of client SDK doesn't reuse requests between retries. // The code leaves some temporary garbage in request (in RequestContext etc.), // which shold be erased during retries. using (DocumentServiceRequest request = await this.CreateRequestAsync()) { DocumentFeedResponse <CosmosElement> feedRespose; string partitionIdentifier; // We need to determine how to execute the request: if (LogicalPartitionKeyProvided(request)) { feedRespose = await this.ExecuteRequestAsync(request, retryPolicyInstance, cancellationToken); partitionIdentifier = $"PKId({request.Headers[HttpConstants.HttpHeaders.PartitionKey]})"; } else if (PhysicalPartitionKeyRangeIdProvided(this)) { CollectionCache collectionCache = await this.Client.GetCollectionCacheAsync(); ContainerProperties collection = await collectionCache.ResolveCollectionAsync(request, CancellationToken.None); request.RouteTo(new PartitionKeyRangeIdentity(collection.ResourceId, base.PartitionKeyRangeId)); feedRespose = await this.ExecuteRequestAsync(request, retryPolicyInstance, cancellationToken); partitionIdentifier = base.PartitionKeyRangeId; } else { // The query is going to become a full fan out, but we go one partition at a time. if (ServiceInteropAvailable()) { // Get the routing map provider CollectionCache collectionCache = await this.Client.GetCollectionCacheAsync(); ContainerProperties collection = await collectionCache.ResolveCollectionAsync(request, CancellationToken.None); QueryPartitionProvider queryPartitionProvider = await this.Client.GetQueryPartitionProviderAsync(cancellationToken); IRoutingMapProvider routingMapProvider = await this.Client.GetRoutingMapProviderAsync(); // Figure out what partition you are going to based on the range from the continuation token // If token is null then just start at partitionKeyRangeId "0" List <CompositeContinuationToken> suppliedTokens; Range <string> rangeFromContinuationToken = this.partitionRoutingHelper.ExtractPartitionKeyRangeFromContinuationToken( request.Headers, out suppliedTokens); Tuple <PartitionRoutingHelper.ResolvedRangeInfo, IReadOnlyList <Range <string> > > queryRoutingInfo = await this.TryGetTargetPartitionKeyRangeAsync( request, collection, queryPartitionProvider, routingMapProvider, rangeFromContinuationToken, suppliedTokens); if (request.IsNameBased && queryRoutingInfo == null) { request.ForceNameCacheRefresh = true; collection = await collectionCache.ResolveCollectionAsync(request, CancellationToken.None); queryRoutingInfo = await this.TryGetTargetPartitionKeyRangeAsync( request, collection, queryPartitionProvider, routingMapProvider, rangeFromContinuationToken, suppliedTokens); } if (queryRoutingInfo == null) { throw new NotFoundException($"{DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture)}: Was not able to get queryRoutingInfo even after resolve collection async with force name cache refresh to the following collectionRid: {collection.ResourceId} with the supplied tokens: {JsonConvert.SerializeObject(suppliedTokens)}"); } request.RouteTo(new PartitionKeyRangeIdentity(collection.ResourceId, queryRoutingInfo.Item1.ResolvedRange.Id)); DocumentFeedResponse <CosmosElement> response = await this.ExecuteRequestAsync(request, retryPolicyInstance, cancellationToken); // Form a composite continuation token (range + backend continuation token). // If the backend continuation token was null for the range, // then use the next adjacent range. // This is how the default execution context serially visits every partition. if (!await this.partitionRoutingHelper.TryAddPartitionKeyRangeToContinuationTokenAsync( response.Headers, providedPartitionKeyRanges: queryRoutingInfo.Item2, routingMapProvider: routingMapProvider, collectionRid: collection.ResourceId, resolvedRangeInfo: queryRoutingInfo.Item1)) { // Collection to which this request was resolved doesn't exist. // Retry policy will refresh the cache and return NotFound. throw new NotFoundException($"{DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture)}: Call to TryAddPartitionKeyRangeToContinuationTokenAsync failed to the following collectionRid: {collection.ResourceId} with the supplied tokens: {JsonConvert.SerializeObject(suppliedTokens)}"); } feedRespose = response; partitionIdentifier = queryRoutingInfo.Item1.ResolvedRange.Id; } else { // For non-Windows platforms(like Linux and OSX) in .NET Core SDK, we cannot use ServiceInterop for parsing the query, // so forcing the request through Gateway. We are also now by-passing this for 32-bit host process in NETFX on Windows // as the ServiceInterop dll is only available in 64-bit. request.UseGatewayMode = true; feedRespose = await this.ExecuteRequestAsync(request, retryPolicyInstance, cancellationToken); partitionIdentifier = "Gateway"; } } return(new Tuple <DocumentFeedResponse <CosmosElement>, string>(feedRespose, partitionIdentifier)); } }
private async Task <DocumentProducer <T> > FetchAsync(IDocumentClientRetryPolicy retryPolicyInstance, CancellationToken cancellationToken) { // TODO: This workflow could be simplified. FetchResult exceptionFetchResult = null; try { this.fetchSchedulingMetrics.Start(); this.fetchExecutionRangeAccumulator.BeginFetchRange(); FeedResponse <T> feedResponse = null; double requestCharge = 0; long responseLengthBytes = 0; QueryMetrics queryMetrics = QueryMetrics.Zero; do { int pageSize = (int)Math.Min(this.PageSize, (long)int.MaxValue); Debug.Assert(pageSize >= 0, string.Format("pageSize was negative ... this.PageSize: {0}", this.PageSize)); using (DocumentServiceRequest request = this.createRequestFunc(this.CurrentBackendContinuationToken, pageSize)) { retryPolicyInstance = retryPolicyInstance ?? this.createRetryPolicyFunc(); retryPolicyInstance.OnBeforeSendRequest(request); // Custom backoff and retry ExceptionDispatchInfo exception = null; try { cancellationToken.ThrowIfCancellationRequested(); feedResponse = await this.executeRequestFunc(request, cancellationToken); this.fetchExecutionRangeAccumulator.EndFetchRange(feedResponse.Count, Interlocked.Read(ref this.retries)); this.ActivityId = Guid.Parse(feedResponse.ActivityId); } catch (Exception ex) { exception = ExceptionDispatchInfo.Capture(ex); } if (exception != null) { cancellationToken.ThrowIfCancellationRequested(); ShouldRetryResult shouldRetryResult = await retryPolicyInstance.ShouldRetryAsync(exception.SourceException, cancellationToken); shouldRetryResult.ThrowIfDoneTrying(exception); this.ScheduleFetch(retryPolicyInstance, shouldRetryResult.BackoffTime); Interlocked.Increment(ref this.retries); return(this); } requestCharge += feedResponse.RequestCharge; responseLengthBytes += feedResponse.ResponseLengthBytes; if (feedResponse.Headers[HttpConstants.HttpHeaders.QueryMetrics] != null) { queryMetrics = QueryMetrics.CreateFromDelimitedStringAndClientSideMetrics( feedResponse.Headers[HttpConstants.HttpHeaders.QueryMetrics], new ClientSideMetrics(this.retries, requestCharge, this.fetchExecutionRangeAccumulator.GetExecutionRanges(), new List <Tuple <string, SchedulingTimeSpan> >()), this.activityId); // Reset the counters. Interlocked.Exchange(ref this.retries, 0); } this.UpdateRequestContinuationToken(feedResponse.ResponseContinuation); retryPolicyInstance = null; this.numDocumentsFetched += feedResponse.Count; } }while (!this.FetchedAll && feedResponse.Count <= 0); await this.CompleteFetchAsync(feedResponse, cancellationToken); this.produceAsyncCompleteCallback(this, feedResponse.Count, requestCharge, queryMetrics, responseLengthBytes, cancellationToken); } catch (Exception ex) { DefaultTrace.TraceWarning(string.Format( CultureInfo.InvariantCulture, "{0}, CorrelatedActivityId: {1}, ActivityId {2} | DocumentProducer Id: {3}, Exception in FetchAsync: {4}", DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture), this.correlatedActivityId, this.ActivityId, this.targetRange.Id, ex.Message)); exceptionFetchResult = new FetchResult(ExceptionDispatchInfo.Capture(ex)); } finally { this.fetchSchedulingMetrics.Stop(); if (this.FetchedAll) { // One more callback to send the scheduling metrics this.produceAsyncCompleteCallback( producer: this, size: 0, resourceUnitUsage: 0, queryMetrics: QueryMetrics.CreateFromDelimitedStringAndClientSideMetrics( QueryMetrics.Zero.ToDelimitedString(), new ClientSideMetrics( retries: 0, requestCharge: 0, fetchExecutionRanges: new List <FetchExecutionRange>(), partitionSchedulingTimeSpans: new List <Tuple <string, SchedulingTimeSpan> > { new Tuple <string, SchedulingTimeSpan>(this.targetRange.Id, this.fetchSchedulingMetrics.Elapsed) }), Guid.Empty), responseLengthBytes: 0, token: cancellationToken); } } if (exceptionFetchResult != null) { this.UpdateRequestContinuationToken(this.CurrentBackendContinuationToken); await this.itemBuffer.AddAsync(exceptionFetchResult, cancellationToken); } return(this); }
private async Task <DocumentServiceResponse> GetFeedResponseAsync(string resourceLink, ResourceType resourceType, IDocumentClientRetryPolicy retryPolicyInstance, CancellationToken cancellationToken) { INameValueCollection headers = new DictionaryNameValueCollection(); if (this.feedOptions.MaxItemCount.HasValue) { headers.Set(HttpConstants.HttpHeaders.PageSize, this.feedOptions.MaxItemCount.ToString()); } if (this.feedOptions.SessionToken != null) { headers.Set(HttpConstants.HttpHeaders.SessionToken, this.feedOptions.SessionToken); } if (resourceType.IsPartitioned() && this.feedOptions.PartitionKeyRangeId == null && this.feedOptions.PartitionKey == null) { throw new ForbiddenException(RMResources.PartitionKeyRangeIdOrPartitionKeyMustBeSpecified); } // On REST level, change feed is using IfNoneMatch/ETag instead of continuation. if (this.nextIfNoneMatch != null) { headers.Set(HttpConstants.HttpHeaders.IfNoneMatch, this.nextIfNoneMatch); } if (this.ifModifiedSince != null) { headers.Set(HttpConstants.HttpHeaders.IfModifiedSince, this.ifModifiedSince); } headers.Set(HttpConstants.HttpHeaders.A_IM, HttpConstants.A_IMHeaderValues.IncrementalFeed); if (this.feedOptions.PartitionKey != null) { PartitionKeyInternal partitionKey = this.feedOptions.PartitionKey.InternalKey; headers.Set(HttpConstants.HttpHeaders.PartitionKey, partitionKey.ToJsonString()); } if (this.feedOptions.IncludeTentativeWrites) { headers.Set(HttpConstants.HttpHeaders.IncludeTentativeWrites, bool.TrueString); } using (DocumentServiceRequest request = this.client.CreateDocumentServiceRequest( OperationType.ReadFeed, resourceLink, resourceType, headers)) { if (resourceType.IsPartitioned() && this.feedOptions.PartitionKeyRangeId != null) { request.RouteTo(new PartitionKeyRangeIdentity(this.feedOptions.PartitionKeyRangeId)); } return(await this.client.ReadFeedAsync(request, retryPolicyInstance, cancellationToken)); } }
private async Task <DocumentFeedResponse <TResult> > ReadDocumentChangeFeedPrivateAsync <TResult>(string link, IDocumentClientRetryPolicy retryPolicyInstance, CancellationToken cancellationToken) { using (DocumentServiceResponse response = await this.GetFeedResponseAsync(link, this.resourceType, retryPolicyInstance, cancellationToken)) { this.lastStatusCode = response.StatusCode; this.nextIfNoneMatch = response.Headers[HttpConstants.HttpHeaders.ETag]; if (response.ResponseBody != null && response.ResponseBody.Length > 0) { long responseLengthInBytes = response.ResponseBody.Length; int itemCount = 0; IEnumerable <dynamic> feedResource = response.GetQueryResponse(typeof(TResource), out itemCount); DocumentFeedResponse <dynamic> feedResponse = new DocumentFeedResponse <dynamic>( feedResource, itemCount, response.Headers, true, null, response.RequestStats, responseLengthBytes: responseLengthInBytes); return((dynamic)feedResponse); } else { return(new DocumentFeedResponse <TResult>( Enumerable.Empty <TResult>(), 0, response.Headers, true, null, response.RequestStats)); } } }
protected override async Task <DocumentFeedResponse <CosmosElement> > ExecuteInternalAsync(CancellationToken token) { CollectionCache collectionCache = await this.Client.GetCollectionCacheAsync(); PartitionKeyRangeCache partitionKeyRangeCache = await this.Client.GetPartitionKeyRangeCacheAsync(); IDocumentClientRetryPolicy retryPolicyInstance = this.Client.ResetSessionTokenRetryPolicy.GetRequestPolicy(); retryPolicyInstance = new InvalidPartitionExceptionRetryPolicy(retryPolicyInstance); if (base.ResourceTypeEnum.IsPartitioned()) { retryPolicyInstance = new PartitionKeyRangeGoneRetryPolicy( collectionCache, partitionKeyRangeCache, PathsHelper.GetCollectionPath(base.ResourceLink), retryPolicyInstance); } return(await BackoffRetryUtility <DocumentFeedResponse <CosmosElement> > .ExecuteAsync( async() => { this.fetchExecutionRangeAccumulator.BeginFetchRange(); ++this.retries; Tuple <DocumentFeedResponse <CosmosElement>, string> responseAndPartitionIdentifier = await this.ExecuteOnceAsync(retryPolicyInstance, token); DocumentFeedResponse <CosmosElement> response = responseAndPartitionIdentifier.Item1; string partitionIdentifier = responseAndPartitionIdentifier.Item2; if (!string.IsNullOrEmpty(response.ResponseHeaders[HttpConstants.HttpHeaders.QueryMetrics])) { this.fetchExecutionRangeAccumulator.EndFetchRange( partitionIdentifier, response.ActivityId, response.Count, this.retries); response = new DocumentFeedResponse <CosmosElement>( response, response.Count, response.Headers, response.UseETagAsContinuation, new Dictionary <string, QueryMetrics> { { partitionIdentifier, QueryMetrics.CreateFromDelimitedStringAndClientSideMetrics( response.ResponseHeaders[HttpConstants.HttpHeaders.QueryMetrics], response.ResponseHeaders[HttpConstants.HttpHeaders.IndexUtilization], new ClientSideMetrics( this.retries, response.RequestCharge, this.fetchExecutionRangeAccumulator.GetExecutionRanges())) } }, response.RequestStatistics, response.DisallowContinuationTokenMessage, response.ResponseLengthBytes); } this.retries = -1; return response; }, retryPolicyInstance, token)); }
protected override async Task <FeedResponse <dynamic> > ExecuteInternalAsync(CancellationToken cancellationToken) { CollectionCache collectionCache = await this.Client.GetCollectionCacheAsync(); PartitionKeyRangeCache partitionKeyRangeCache = await this.Client.GetPartitionKeyRangeCache(); IDocumentClientRetryPolicy retryPolicyInstance = this.Client.RetryPolicy.GetRequestPolicy(); retryPolicyInstance = new InvalidPartitionExceptionRetryPolicy(collectionCache, retryPolicyInstance); if (base.ResourceTypeEnum.IsPartitioned()) { retryPolicyInstance = new PartitionKeyRangeGoneRetryPolicy( collectionCache, partitionKeyRangeCache, PathsHelper.GetCollectionPath(base.ResourceLink), retryPolicyInstance); } return(await BackoffRetryUtility <FeedResponse <dynamic> > .ExecuteAsync( async() => { this.fetchExecutionRangeAccumulator.BeginFetchRange(); ++this.retries; this.fetchSchedulingMetrics.Start(); this.fetchExecutionRangeAccumulator.BeginFetchRange(); FeedResponse <dynamic> response = await this.ExecuteOnceAsync(retryPolicyInstance, cancellationToken); this.fetchSchedulingMetrics.Stop(); this.fetchExecutionRangeAccumulator.EndFetchRange(response.Count, this.retries); if (!string.IsNullOrEmpty(response.Headers[HttpConstants.HttpHeaders.QueryMetrics])) { this.fetchExecutionRangeAccumulator.EndFetchRange(response.Count, this.retries); response = new FeedResponse <dynamic>( response, response.Count, response.Headers, response.UseETagAsContinuation, new Dictionary <string, QueryMetrics> { { singlePartitionKeyId, QueryMetrics.CreateFromDelimitedStringAndClientSideMetrics( response.Headers[HttpConstants.HttpHeaders.QueryMetrics], new ClientSideMetrics( this.retries, response.RequestCharge, this.fetchExecutionRangeAccumulator.GetExecutionRanges(), string.IsNullOrEmpty(response.ResponseContinuation) ? new List <Tuple <string, SchedulingTimeSpan> >() { new Tuple <string, SchedulingTimeSpan>(singlePartitionKeyId, this.fetchSchedulingMetrics.Elapsed) } : new List <Tuple <string, SchedulingTimeSpan> >()), Guid.Parse(response.ActivityId)) } }, response.RequestStatistics, response.DisallowContinuationTokenMessage, response.ResponseLengthBytes); } this.retries = -1; return response; }, retryPolicyInstance, cancellationToken)); }
public BulkPartitionKeyRangeGoneRetryPolicy(IDocumentClientRetryPolicy nextRetryPolicy) { this.nextRetryPolicy = nextRetryPolicy; }
public InvalidPartitionExceptionRetryPolicy( IDocumentClientRetryPolicy nextPolicy) { this.nextPolicy = nextPolicy; }