public override async Task <ResponseMessage> SendAsync(
RequestMessage request,
CancellationToken cancellationToken)
{
IDocumentClientRetryPolicy retryPolicyInstance = await this.GetRetryPolicyAsync(request);
request.OnBeforeSendRequestActions += retryPolicyInstance.OnBeforeSendRequest;
try
{
return(await RetryHandler.ExecuteHttpRequestAsync(
callbackMethod : () =>
{
return base.SendAsync(request, cancellationToken);
},
callShouldRetry : (cosmosResponseMessage, token) =>
{
return retryPolicyInstance.ShouldRetryAsync(cosmosResponseMessage, cancellationToken);
},
callShouldRetryException : (exception, token) =>
{
return retryPolicyInstance.ShouldRetryAsync(exception, cancellationToken);
},
diagnosticsContext : request.DiagnosticsContext,
cancellationToken : cancellationToken));
}
catch (DocumentClientException ex)
{
return(ex.ToCosmosResponseMessage(request));
}
catch (CosmosException ex)
{
return(ex.ToCosmosResponseMessage(request));
}
catch (AggregateException ex)
{
// TODO: because the SDK underneath this path uses ContinueWith or task.Result we need to catch AggregateExceptions here
// in order to ensure that underlying DocumentClientExceptions get propagated up correctly. Once all ContinueWith and .Result
// is removed this catch can be safely removed.
AggregateException innerExceptions = ex.Flatten();
Exception docClientException = innerExceptions.InnerExceptions.FirstOrDefault(innerEx => innerEx is DocumentClientException);
if (docClientException != null)
{
return(((DocumentClientException)docClientException).ToCosmosResponseMessage(request));
}
throw;
}
finally
{
request.OnBeforeSendRequestActions -= retryPolicyInstance.OnBeforeSendRequest;
}
}
public async Task <DocumentFeedResponse <CosmosElement> > ExecuteRequestLazyAsync(
DocumentServiceRequest request,
IDocumentClientRetryPolicy retryPolicyInstance,
CancellationToken cancellationToken)
{
DocumentServiceResponse documentServiceResponse = await this.ExecuteQueryRequestInternalAsync(
request,
retryPolicyInstance,
cancellationToken);
return(this.GetFeedResponse(request, documentServiceResponse));
}
public InvalidPartitionExceptionRetryPolicy(
CollectionCache clientCollectionCache,
IDocumentClientRetryPolicy nextPolicy)
{
if (clientCollectionCache == null)
{
throw new ArgumentNullException("clientCollectionCache");
}
this.clientCollectionCache = clientCollectionCache;
this.nextPolicy = nextPolicy;
}
public PartitionKeyMismatchRetryPolicy(
CollectionCache clientCollectionCache,
IDocumentClientRetryPolicy nextRetryPolicy)
{
if (clientCollectionCache == null)
{
throw new ArgumentNullException("clientCollectionCache");
}
this.clientCollectionCache = clientCollectionCache;
this.nextRetryPolicy = nextRetryPolicy;
}
public PartitionKeyRangeGoneRetryPolicy(
CollectionCache collectionCache,
PartitionKeyRangeCache partitionKeyRangeCache,
string collectionLink,
IDocumentClientRetryPolicy nextRetryPolicy,
ITrace trace)
{
this.collectionCache = collectionCache;
this.partitionKeyRangeCache = partitionKeyRangeCache;
this.collectionLink = collectionLink;
this.nextRetryPolicy = nextRetryPolicy;
this.trace = trace;
}
private async Task <DocumentServiceResponse> ExecuteQueryRequestInternalAsync(
DocumentServiceRequest request,
IDocumentClientRetryPolicy retryPolicyInstance,
CancellationToken cancellationToken)
{
try
{
return(await this.Client.ExecuteQueryAsync(request, retryPolicyInstance, cancellationToken));
}
finally
{
request.Body.Position = 0;
}
}
public ItemBatchOperationContext(
string partitionKeyRangeId,
ITrace trace,
IDocumentClientRetryPolicy retryPolicy = null)
{
if (trace == null)
{
throw new ArgumentNullException(nameof(trace));
}
this.PartitionKeyRangeId = partitionKeyRangeId ?? throw new ArgumentNullException(nameof(partitionKeyRangeId));
this.initialTrace = trace;
this.retryPolicy = retryPolicy;
}
private async Task <ContainerProperties> ReadCollectionAsync(string collectionLink,
CancellationToken cancellationToken,
IDocumentClientRetryPolicy retryPolicyInstance,
ITrace trace)
{
using (ITrace childTrace = trace.StartChild("Read Collection", TraceComponent.Transport, TraceLevel.Info))
{
cancellationToken.ThrowIfCancellationRequested();
using (DocumentServiceRequest request = DocumentServiceRequest.Create(
OperationType.Read,
ResourceType.Collection,
collectionLink,
AuthorizationTokenType.PrimaryMasterKey,
new StoreRequestNameValueCollection()))
{
request.Headers[HttpConstants.HttpHeaders.XDate] = DateTime.UtcNow.ToString("r");
(string authorizationToken, string payload) = await this.tokenProvider.GetUserAuthorizationAsync(
request.ResourceAddress,
PathsHelper.GetResourcePath(request.ResourceType),
HttpConstants.HttpMethods.Get,
request.Headers,
AuthorizationTokenType.PrimaryMasterKey);
request.Headers[HttpConstants.HttpHeaders.Authorization] = authorizationToken;
using (new ActivityScope(Guid.NewGuid()))
{
if (retryPolicyInstance != null)
{
retryPolicyInstance.OnBeforeSendRequest(request);
}
try
{
using (DocumentServiceResponse response = await this.storeModel.ProcessMessageAsync(request))
{
return(CosmosResource.FromStream <ContainerProperties>(response));
}
}
catch (DocumentClientException ex)
{
childTrace.AddDatum("Exception Message", ex.Message);
throw;
}
}
}
}
}
public ClientRetryPolicy(
GlobalEndpointManager globalEndpointManager,
bool enableEndpointDiscovery,
RetryOptions retryOptions)
{
this.throttlingRetry = new ResourceThrottleRetryPolicy(
retryOptions.MaxRetryAttemptsOnThrottledRequests,
retryOptions.MaxRetryWaitTimeInSeconds);
this.globalEndpointManager = globalEndpointManager;
this.failoverRetryCount = 0;
this.enableEndpointDiscovery = enableEndpointDiscovery;
this.sessionTokenRetryCount = 0;
this.canUseMultipleWriteLocations = false;
}
private void ScheduleFetch(IDocumentClientRetryPolicy retryPolicyInstance, TimeSpan delay = default(TimeSpan))
{
// For the same DocumentProducer, the priorities of scheduled tasks monotonically decrease.
// This makes sure the tasks are scheduled fairly across all DocumentProducer's.
bool scheduled = this.taskScheduler.TryQueueTask(
new DocumentProducerComparableTask(
this,
retryPolicyInstance),
delay);
if (!scheduled)
{
Interlocked.Exchange(ref this.isFetching, 0);
throw new InvalidOperationException("Failed to schedule");
}
}
// Extracted partiotn key might be invaild as CollectionCache might be stale.
// Stale collection cache is refreshed through PartitionKeyMismatchRetryPolicy
// and partition-key is extracted again.
internal async Task <CosmosResponseMessage> ExtractPartitionKeyAndProcessItemStreamAsync(
PartitionKey partitionKey,
string itemId,
Stream streamPayload,
OperationType operationType,
RequestOptions requestOptions,
CancellationToken cancellationToken)
{
IDocumentClientRetryPolicy requestRetryPolicy = null;
if (partitionKey == null)
{
requestRetryPolicy = new PartitionKeyMismatchRetryPolicy(await this.ClientContext.DocumentClient.GetCollectionCacheAsync(), requestRetryPolicy);
}
while (true)
{
CosmosResponseMessage responseMessage = await this.ProcessItemStreamAsync(
partitionKey,
itemId,
streamPayload,
operationType,
requestOptions,
extractPartitionKeyIfNeeded : true,
cancellationToken : cancellationToken);
ShouldRetryResult retryResult = ShouldRetryResult.NoRetry();
if (requestRetryPolicy != null &&
!responseMessage.IsSuccessStatusCode)
{
retryResult = await requestRetryPolicy.ShouldRetryAsync(responseMessage, cancellationToken);
}
if (!retryResult.ShouldRetry)
{
return(responseMessage);
}
}
}
private async Task <FeedResponse <dynamic> > ExecuteOnceAsync(IDocumentClientRetryPolicy retryPolicyInstance, CancellationToken cancellationToken)
{
// Don't reuse request, as the rest of client SDK doesn't reuse requests between retries.
// The code leaves some temporary garbage in request (in RequestContext etc.),
// which shold be erased during retries.
using (DocumentServiceRequest request = await this.CreateRequestAsync())
{
if (retryPolicyInstance != null)
{
retryPolicyInstance.OnBeforeSendRequest(request);
}
if (!string.IsNullOrEmpty(request.Headers[HttpConstants.HttpHeaders.PartitionKey]) ||
!request.ResourceType.IsPartitioned())
{
return(await this.ExecuteRequestAsync(request, cancellationToken));
}
CollectionCache collectionCache = await this.Client.GetCollectionCacheAsync();
CosmosContainerSettings collection =
await collectionCache.ResolveCollectionAsync(request, CancellationToken.None);
if (!string.IsNullOrEmpty(base.PartitionKeyRangeId))
{
request.RouteTo(new PartitionKeyRangeIdentity(collection.ResourceId, base.PartitionKeyRangeId));
return(await this.ExecuteRequestAsync(request, cancellationToken));
}
// For non-Windows platforms(like Linux and OSX) in .NET Core SDK, we cannot use ServiceInterop for parsing the query,
// so forcing the request through Gateway. We are also now by-passing this for 32-bit host process in NETFX on Windows
// as the ServiceInterop dll is only available in 64-bit.
if (CustomTypeExtensions.ByPassQueryParsing())
{
request.UseGatewayMode = true;
return(await this.ExecuteRequestAsync(request, cancellationToken));
}
QueryPartitionProvider queryPartitionProvider = await this.Client.GetQueryPartitionProviderAsync(cancellationToken);
IRoutingMapProvider routingMapProvider = await this.Client.GetRoutingMapProviderAsync();
List <CompositeContinuationToken> suppliedTokens;
Range <string> rangeFromContinuationToken =
this.partitionRoutingHelper.ExtractPartitionKeyRangeFromContinuationToken(request.Headers, out suppliedTokens);
Tuple <PartitionRoutingHelper.ResolvedRangeInfo, IReadOnlyList <Range <string> > > queryRoutingInfo =
await this.TryGetTargetPartitionKeyRangeAsync(
request,
collection,
queryPartitionProvider,
routingMapProvider,
rangeFromContinuationToken,
suppliedTokens);
if (request.IsNameBased && queryRoutingInfo == null)
{
request.ForceNameCacheRefresh = true;
collection = await collectionCache.ResolveCollectionAsync(request, CancellationToken.None);
queryRoutingInfo = await this.TryGetTargetPartitionKeyRangeAsync(
request,
collection,
queryPartitionProvider,
routingMapProvider,
rangeFromContinuationToken,
suppliedTokens);
}
if (queryRoutingInfo == null)
{
throw new NotFoundException($"{DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture)}: Was not able to get queryRoutingInfo even after resolve collection async with force name cache refresh to the following collectionRid: {collection.ResourceId} with the supplied tokens: {JsonConvert.SerializeObject(suppliedTokens)}");
}
request.RouteTo(new PartitionKeyRangeIdentity(collection.ResourceId, queryRoutingInfo.Item1.ResolvedRange.Id));
FeedResponse <dynamic> response = await this.ExecuteRequestAsync(request, cancellationToken);
if (!await this.partitionRoutingHelper.TryAddPartitionKeyRangeToContinuationTokenAsync(
response.Headers,
providedPartitionKeyRanges: queryRoutingInfo.Item2,
routingMapProvider: routingMapProvider,
collectionRid: collection.ResourceId,
resolvedRangeInfo: queryRoutingInfo.Item1))
{
// Collection to which this request was resolved doesn't exist.
// Retry policy will refresh the cache and return NotFound.
throw new NotFoundException($"{DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture)}: Call to TryAddPartitionKeyRangeToContinuationTokenAsync failed to the following collectionRid: {collection.ResourceId} with the supplied tokens: {JsonConvert.SerializeObject(suppliedTokens)}");
}
return(response);
}
}
public ClearingSessionContainerClientRetryPolicy(ISessionContainer sessionContainer, IDocumentClientRetryPolicy retryPolicy)
{
this.retryPolicy = retryPolicy;
this.sessionContainer = sessionContainer;
}
public Task <DocumentServiceResponse> ReadFeedAsync(DocumentServiceRequest request, IDocumentClientRetryPolicy retryPolicyInstance, CancellationToken cancellationToken)
{
return(this.innerClient.ReadFeedAsync(request, retryPolicyInstance, cancellationToken));
}
private async Task <CosmosContainerSettings> ReadCollectionAsync(string collectionLink, CancellationToken cancellationToken, IDocumentClientRetryPolicy retryPolicyInstance)
{
cancellationToken.ThrowIfCancellationRequested();
using (DocumentServiceRequest request = DocumentServiceRequest.Create(
OperationType.Read,
ResourceType.Collection,
collectionLink,
AuthorizationTokenType.PrimaryMasterKey,
new StringKeyValueCollection()))
{
request.Headers[HttpConstants.HttpHeaders.XDate] = DateTime.UtcNow.ToString("r");
string authorizationToken = this.tokenProvider.GetUserAuthorizationToken(
request.ResourceAddress,
PathsHelper.GetResourcePath(request.ResourceType),
HttpConstants.HttpMethods.Get,
request.Headers,
AuthorizationTokenType.PrimaryMasterKey);
request.Headers[HttpConstants.HttpHeaders.Authorization] = authorizationToken;
using (new ActivityScope(Guid.NewGuid()))
{
if (retryPolicyInstance != null)
{
retryPolicyInstance.OnBeforeSendRequest(request);
}
using (DocumentServiceResponse response = await this.storeModel.ProcessMessageAsync(request))
{
return(new ResourceResponse <CosmosContainerSettings>(response).Resource);
}
}
}
}
/// <summary>
/// Buffers more documents in the producer.
/// </summary>
/// <param name="token">The cancellation token.</param>
/// <returns>A task to await on.</returns>
public async Task BufferMoreDocumentsAsync(CancellationToken token)
{
token.ThrowIfCancellationRequested();
try
{
await this.fetchSemaphore.WaitAsync();
if (!this.HasMoreBackendResults)
{
// Just NOP
return;
}
this.fetchSchedulingMetrics.Start();
this.fetchExecutionRangeAccumulator.BeginFetchRange();
int pageSize = (int)Math.Min(this.pageSize, (long)int.MaxValue);
using (DocumentServiceRequest request = this.createRequestFunc(this.PartitionKeyRange, this.backendContinuationToken, pageSize))
{
IDocumentClientRetryPolicy retryPolicy = this.createRetryPolicyFunc();
// Custom backoff and retry
while (true)
{
int retries = 0;
try
{
DocumentFeedResponse <CosmosElement> feedResponse = await this.executeRequestFunc(request, retryPolicy, token);
this.fetchExecutionRangeAccumulator.EndFetchRange(
this.PartitionKeyRange.Id,
feedResponse.ActivityId,
feedResponse.Count,
retries);
this.fetchSchedulingMetrics.Stop();
this.hasStartedFetching = true;
this.backendContinuationToken = feedResponse.ResponseContinuation;
this.activityId = Guid.Parse(feedResponse.ActivityId);
await this.bufferedPages.AddAsync(TryMonad <DocumentFeedResponse <CosmosElement> > .FromResult(feedResponse));
Interlocked.Add(ref this.bufferedItemCount, feedResponse.Count);
QueryMetrics queryMetrics = QueryMetrics.Zero;
if (feedResponse.ResponseHeaders[HttpConstants.HttpHeaders.QueryMetrics] != null)
{
queryMetrics = QueryMetrics.CreateFromDelimitedStringAndClientSideMetrics(
feedResponse.ResponseHeaders[HttpConstants.HttpHeaders.QueryMetrics],
new ClientSideMetrics(
retries,
feedResponse.RequestCharge,
this.fetchExecutionRangeAccumulator.GetExecutionRanges(),
new List <Tuple <string, SchedulingTimeSpan> >()));
}
if (!this.HasMoreBackendResults)
{
queryMetrics = QueryMetrics.CreateWithSchedulingMetrics(
queryMetrics,
new List <Tuple <string, SchedulingTimeSpan> >
{
new Tuple <string, SchedulingTimeSpan>(
this.PartitionKeyRange.Id,
this.fetchSchedulingMetrics.Elapsed)
});
}
this.produceAsyncCompleteCallback(
this,
feedResponse.Count,
feedResponse.RequestCharge,
queryMetrics,
feedResponse.ResponseLengthBytes,
token);
break;
}
catch (Exception exception)
{
// See if we need to retry or just throw
ShouldRetryResult shouldRetryResult = await retryPolicy.ShouldRetryAsync(exception, token);
if (!shouldRetryResult.ShouldRetry)
{
Exception exceptionToBuffer;
if (shouldRetryResult.ExceptionToThrow != null)
{
exceptionToBuffer = shouldRetryResult.ExceptionToThrow;
}
else
{
// Propagate original exception.
exceptionToBuffer = exception;
}
// Buffer the exception instead of throwing, since we don't want an unobserved exception.
await this.bufferedPages.AddAsync(TryMonad <DocumentFeedResponse <CosmosElement> > .FromException(exceptionToBuffer));
// null out the backend continuation token,
// so that people stop trying to buffer more on this producer.
this.hasStartedFetching = true;
this.backendContinuationToken = null;
break;
}
else
{
await Task.Delay(shouldRetryResult.BackoffTime);
retries++;
}
}
}
}
}
finally
{
this.fetchSchedulingMetrics.Stop();
this.fetchSemaphore.Release();
}
}
public override async Task <ResponseMessage> SendAsync(
RequestMessage request,
CancellationToken cancellationToken)
{
using (ITrace childTrace = request.Trace.StartChild("Send Async", TraceComponent.RequestHandler, TraceLevel.Info))
{
request.Trace = childTrace;
IDocumentClientRetryPolicy retryPolicyInstance = await this.GetRetryPolicyAsync(request);
request.OnBeforeSendRequestActions += retryPolicyInstance.OnBeforeSendRequest;
try
{
return(await RetryHandler.ExecuteHttpRequestAsync(
callbackMethod : async(trace) =>
{
using (ITrace childTrace = trace.StartChild("Callback Method"))
{
request.Trace = childTrace;
return await base.SendAsync(request, cancellationToken);
}
},
callShouldRetry : async(cosmosResponseMessage, trace, token) =>
{
using (ITrace shouldRetryTrace = trace.StartChild("Call Should Retry"))
{
request.Trace = shouldRetryTrace;
return await retryPolicyInstance.ShouldRetryAsync(cosmosResponseMessage, cancellationToken);
}
},
callShouldRetryException : async(exception, trace, token) =>
{
using (ITrace shouldRetryTrace = trace.StartChild("Call Should Retry Exception"))
{
request.Trace = shouldRetryTrace;
return await retryPolicyInstance.ShouldRetryAsync(exception, cancellationToken);
}
},
trace : request.Trace,
cancellationToken : cancellationToken));
}
catch (DocumentClientException ex)
{
return(ex.ToCosmosResponseMessage(request));
}
catch (CosmosException ex)
{
return(ex.ToCosmosResponseMessage(request));
}
catch (AggregateException ex)
{
// TODO: because the SDK underneath this path uses ContinueWith or task.Result we need to catch AggregateExceptions here
// in order to ensure that underlying DocumentClientExceptions get propagated up correctly. Once all ContinueWith and .Result
// is removed this catch can be safely removed.
AggregateException innerExceptions = ex.Flatten();
Exception docClientException = innerExceptions.InnerExceptions.FirstOrDefault(innerEx => innerEx is DocumentClientException);
if (docClientException != null)
{
return(((DocumentClientException)docClientException).ToCosmosResponseMessage(request));
}
throw;
}
finally
{
request.OnBeforeSendRequestActions -= retryPolicyInstance.OnBeforeSendRequest;
}
}
}
public RenameCollectionAwareClientRetryPolicy(ISessionContainer sessionContainer, ClientCollectionCache collectionCache, IDocumentClientRetryPolicy retryPolicy)
{
this.retryPolicy = retryPolicy;
this.sessionContainer = sessionContainer;
this.collectionCache = collectionCache;
this.request = null;
}
private async Task <Tuple <DocumentFeedResponse <CosmosElement>, string> > ExecuteOnceAsync(
IDocumentClientRetryPolicy retryPolicyInstance,
CancellationToken cancellationToken)
{
// Don't reuse request, as the rest of client SDK doesn't reuse requests between retries.
// The code leaves some temporary garbage in request (in RequestContext etc.),
// which shold be erased during retries.
using (DocumentServiceRequest request = await this.CreateRequestAsync())
{
DocumentFeedResponse <CosmosElement> feedRespose;
string partitionIdentifier;
// We need to determine how to execute the request:
if (LogicalPartitionKeyProvided(request))
{
feedRespose = await this.ExecuteRequestAsync(request, retryPolicyInstance, cancellationToken);
partitionIdentifier = $"PKId({request.Headers[HttpConstants.HttpHeaders.PartitionKey]})";
}
else if (PhysicalPartitionKeyRangeIdProvided(this))
{
CollectionCache collectionCache = await this.Client.GetCollectionCacheAsync();
ContainerProperties collection = await collectionCache.ResolveCollectionAsync(request, CancellationToken.None);
request.RouteTo(new PartitionKeyRangeIdentity(collection.ResourceId, base.PartitionKeyRangeId));
feedRespose = await this.ExecuteRequestAsync(request, retryPolicyInstance, cancellationToken);
partitionIdentifier = base.PartitionKeyRangeId;
}
else
{
// The query is going to become a full fan out, but we go one partition at a time.
if (ServiceInteropAvailable())
{
// Get the routing map provider
CollectionCache collectionCache = await this.Client.GetCollectionCacheAsync();
ContainerProperties collection = await collectionCache.ResolveCollectionAsync(request, CancellationToken.None);
QueryPartitionProvider queryPartitionProvider = await this.Client.GetQueryPartitionProviderAsync(cancellationToken);
IRoutingMapProvider routingMapProvider = await this.Client.GetRoutingMapProviderAsync();
// Figure out what partition you are going to based on the range from the continuation token
// If token is null then just start at partitionKeyRangeId "0"
List <CompositeContinuationToken> suppliedTokens;
Range <string> rangeFromContinuationToken =
this.partitionRoutingHelper.ExtractPartitionKeyRangeFromContinuationToken(
request.Headers,
out suppliedTokens);
Tuple <PartitionRoutingHelper.ResolvedRangeInfo, IReadOnlyList <Range <string> > > queryRoutingInfo =
await this.TryGetTargetPartitionKeyRangeAsync(
request,
collection,
queryPartitionProvider,
routingMapProvider,
rangeFromContinuationToken,
suppliedTokens);
if (request.IsNameBased && queryRoutingInfo == null)
{
request.ForceNameCacheRefresh = true;
collection = await collectionCache.ResolveCollectionAsync(request, CancellationToken.None);
queryRoutingInfo = await this.TryGetTargetPartitionKeyRangeAsync(
request,
collection,
queryPartitionProvider,
routingMapProvider,
rangeFromContinuationToken,
suppliedTokens);
}
if (queryRoutingInfo == null)
{
throw new NotFoundException($"{DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture)}: Was not able to get queryRoutingInfo even after resolve collection async with force name cache refresh to the following collectionRid: {collection.ResourceId} with the supplied tokens: {JsonConvert.SerializeObject(suppliedTokens)}");
}
request.RouteTo(new PartitionKeyRangeIdentity(collection.ResourceId, queryRoutingInfo.Item1.ResolvedRange.Id));
DocumentFeedResponse <CosmosElement> response = await this.ExecuteRequestAsync(request, retryPolicyInstance, cancellationToken);
// Form a composite continuation token (range + backend continuation token).
// If the backend continuation token was null for the range,
// then use the next adjacent range.
// This is how the default execution context serially visits every partition.
if (!await this.partitionRoutingHelper.TryAddPartitionKeyRangeToContinuationTokenAsync(
response.Headers,
providedPartitionKeyRanges: queryRoutingInfo.Item2,
routingMapProvider: routingMapProvider,
collectionRid: collection.ResourceId,
resolvedRangeInfo: queryRoutingInfo.Item1))
{
// Collection to which this request was resolved doesn't exist.
// Retry policy will refresh the cache and return NotFound.
throw new NotFoundException($"{DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture)}: Call to TryAddPartitionKeyRangeToContinuationTokenAsync failed to the following collectionRid: {collection.ResourceId} with the supplied tokens: {JsonConvert.SerializeObject(suppliedTokens)}");
}
feedRespose = response;
partitionIdentifier = queryRoutingInfo.Item1.ResolvedRange.Id;
}
else
{
// For non-Windows platforms(like Linux and OSX) in .NET Core SDK, we cannot use ServiceInterop for parsing the query,
// so forcing the request through Gateway. We are also now by-passing this for 32-bit host process in NETFX on Windows
// as the ServiceInterop dll is only available in 64-bit.
request.UseGatewayMode = true;
feedRespose = await this.ExecuteRequestAsync(request, retryPolicyInstance, cancellationToken);
partitionIdentifier = "Gateway";
}
}
return(new Tuple <DocumentFeedResponse <CosmosElement>, string>(feedRespose, partitionIdentifier));
}
}
private async Task <DocumentProducer <T> > FetchAsync(IDocumentClientRetryPolicy retryPolicyInstance, CancellationToken cancellationToken)
{
// TODO: This workflow could be simplified.
FetchResult exceptionFetchResult = null;
try
{
this.fetchSchedulingMetrics.Start();
this.fetchExecutionRangeAccumulator.BeginFetchRange();
FeedResponse <T> feedResponse = null;
double requestCharge = 0;
long responseLengthBytes = 0;
QueryMetrics queryMetrics = QueryMetrics.Zero;
do
{
int pageSize = (int)Math.Min(this.PageSize, (long)int.MaxValue);
Debug.Assert(pageSize >= 0, string.Format("pageSize was negative ... this.PageSize: {0}", this.PageSize));
using (DocumentServiceRequest request = this.createRequestFunc(this.CurrentBackendContinuationToken, pageSize))
{
retryPolicyInstance = retryPolicyInstance ?? this.createRetryPolicyFunc();
retryPolicyInstance.OnBeforeSendRequest(request);
// Custom backoff and retry
ExceptionDispatchInfo exception = null;
try
{
cancellationToken.ThrowIfCancellationRequested();
feedResponse = await this.executeRequestFunc(request, cancellationToken);
this.fetchExecutionRangeAccumulator.EndFetchRange(feedResponse.Count, Interlocked.Read(ref this.retries));
this.ActivityId = Guid.Parse(feedResponse.ActivityId);
}
catch (Exception ex)
{
exception = ExceptionDispatchInfo.Capture(ex);
}
if (exception != null)
{
cancellationToken.ThrowIfCancellationRequested();
ShouldRetryResult shouldRetryResult = await retryPolicyInstance.ShouldRetryAsync(exception.SourceException, cancellationToken);
shouldRetryResult.ThrowIfDoneTrying(exception);
this.ScheduleFetch(retryPolicyInstance, shouldRetryResult.BackoffTime);
Interlocked.Increment(ref this.retries);
return(this);
}
requestCharge += feedResponse.RequestCharge;
responseLengthBytes += feedResponse.ResponseLengthBytes;
if (feedResponse.Headers[HttpConstants.HttpHeaders.QueryMetrics] != null)
{
queryMetrics = QueryMetrics.CreateFromDelimitedStringAndClientSideMetrics(
feedResponse.Headers[HttpConstants.HttpHeaders.QueryMetrics],
new ClientSideMetrics(this.retries, requestCharge,
this.fetchExecutionRangeAccumulator.GetExecutionRanges(),
new List <Tuple <string, SchedulingTimeSpan> >()),
this.activityId);
// Reset the counters.
Interlocked.Exchange(ref this.retries, 0);
}
this.UpdateRequestContinuationToken(feedResponse.ResponseContinuation);
retryPolicyInstance = null;
this.numDocumentsFetched += feedResponse.Count;
}
}while (!this.FetchedAll && feedResponse.Count <= 0);
await this.CompleteFetchAsync(feedResponse, cancellationToken);
this.produceAsyncCompleteCallback(this, feedResponse.Count, requestCharge, queryMetrics, responseLengthBytes, cancellationToken);
}
catch (Exception ex)
{
DefaultTrace.TraceWarning(string.Format(
CultureInfo.InvariantCulture,
"{0}, CorrelatedActivityId: {1}, ActivityId {2} | DocumentProducer Id: {3}, Exception in FetchAsync: {4}",
DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture),
this.correlatedActivityId,
this.ActivityId,
this.targetRange.Id,
ex.Message));
exceptionFetchResult = new FetchResult(ExceptionDispatchInfo.Capture(ex));
}
finally
{
this.fetchSchedulingMetrics.Stop();
if (this.FetchedAll)
{
// One more callback to send the scheduling metrics
this.produceAsyncCompleteCallback(
producer: this,
size: 0,
resourceUnitUsage: 0,
queryMetrics: QueryMetrics.CreateFromDelimitedStringAndClientSideMetrics(
QueryMetrics.Zero.ToDelimitedString(),
new ClientSideMetrics(
retries: 0,
requestCharge: 0,
fetchExecutionRanges: new List <FetchExecutionRange>(),
partitionSchedulingTimeSpans: new List <Tuple <string, SchedulingTimeSpan> > {
new Tuple <string, SchedulingTimeSpan>(this.targetRange.Id, this.fetchSchedulingMetrics.Elapsed)
}),
Guid.Empty),
responseLengthBytes: 0,
token: cancellationToken);
}
}
if (exceptionFetchResult != null)
{
this.UpdateRequestContinuationToken(this.CurrentBackendContinuationToken);
await this.itemBuffer.AddAsync(exceptionFetchResult, cancellationToken);
}
return(this);
}
private async Task <DocumentServiceResponse> GetFeedResponseAsync(string resourceLink, ResourceType resourceType, IDocumentClientRetryPolicy retryPolicyInstance, CancellationToken cancellationToken)
{
INameValueCollection headers = new DictionaryNameValueCollection();
if (this.feedOptions.MaxItemCount.HasValue)
{
headers.Set(HttpConstants.HttpHeaders.PageSize, this.feedOptions.MaxItemCount.ToString());
}
if (this.feedOptions.SessionToken != null)
{
headers.Set(HttpConstants.HttpHeaders.SessionToken, this.feedOptions.SessionToken);
}
if (resourceType.IsPartitioned() && this.feedOptions.PartitionKeyRangeId == null && this.feedOptions.PartitionKey == null)
{
throw new ForbiddenException(RMResources.PartitionKeyRangeIdOrPartitionKeyMustBeSpecified);
}
// On REST level, change feed is using IfNoneMatch/ETag instead of continuation.
if (this.nextIfNoneMatch != null)
{
headers.Set(HttpConstants.HttpHeaders.IfNoneMatch, this.nextIfNoneMatch);
}
if (this.ifModifiedSince != null)
{
headers.Set(HttpConstants.HttpHeaders.IfModifiedSince, this.ifModifiedSince);
}
headers.Set(HttpConstants.HttpHeaders.A_IM, HttpConstants.A_IMHeaderValues.IncrementalFeed);
if (this.feedOptions.PartitionKey != null)
{
PartitionKeyInternal partitionKey = this.feedOptions.PartitionKey.InternalKey;
headers.Set(HttpConstants.HttpHeaders.PartitionKey, partitionKey.ToJsonString());
}
if (this.feedOptions.IncludeTentativeWrites)
{
headers.Set(HttpConstants.HttpHeaders.IncludeTentativeWrites, bool.TrueString);
}
using (DocumentServiceRequest request = this.client.CreateDocumentServiceRequest(
OperationType.ReadFeed,
resourceLink,
resourceType,
headers))
{
if (resourceType.IsPartitioned() && this.feedOptions.PartitionKeyRangeId != null)
{
request.RouteTo(new PartitionKeyRangeIdentity(this.feedOptions.PartitionKeyRangeId));
}
return(await this.client.ReadFeedAsync(request, retryPolicyInstance, cancellationToken));
}
}
private async Task <DocumentFeedResponse <TResult> > ReadDocumentChangeFeedPrivateAsync <TResult>(string link, IDocumentClientRetryPolicy retryPolicyInstance, CancellationToken cancellationToken)
{
using (DocumentServiceResponse response = await this.GetFeedResponseAsync(link, this.resourceType, retryPolicyInstance, cancellationToken))
{
this.lastStatusCode = response.StatusCode;
this.nextIfNoneMatch = response.Headers[HttpConstants.HttpHeaders.ETag];
if (response.ResponseBody != null && response.ResponseBody.Length > 0)
{
long responseLengthInBytes = response.ResponseBody.Length;
int itemCount = 0;
IEnumerable <dynamic> feedResource = response.GetQueryResponse(typeof(TResource), out itemCount);
DocumentFeedResponse <dynamic> feedResponse = new DocumentFeedResponse <dynamic>(
feedResource,
itemCount,
response.Headers,
true,
null,
response.RequestStats,
responseLengthBytes: responseLengthInBytes);
return((dynamic)feedResponse);
}
else
{
return(new DocumentFeedResponse <TResult>(
Enumerable.Empty <TResult>(),
0,
response.Headers,
true,
null,
response.RequestStats));
}
}
}
protected override async Task <DocumentFeedResponse <CosmosElement> > ExecuteInternalAsync(CancellationToken token)
{
CollectionCache collectionCache = await this.Client.GetCollectionCacheAsync();
PartitionKeyRangeCache partitionKeyRangeCache = await this.Client.GetPartitionKeyRangeCacheAsync();
IDocumentClientRetryPolicy retryPolicyInstance = this.Client.ResetSessionTokenRetryPolicy.GetRequestPolicy();
retryPolicyInstance = new InvalidPartitionExceptionRetryPolicy(retryPolicyInstance);
if (base.ResourceTypeEnum.IsPartitioned())
{
retryPolicyInstance = new PartitionKeyRangeGoneRetryPolicy(
collectionCache,
partitionKeyRangeCache,
PathsHelper.GetCollectionPath(base.ResourceLink),
retryPolicyInstance);
}
return(await BackoffRetryUtility <DocumentFeedResponse <CosmosElement> > .ExecuteAsync(
async() =>
{
this.fetchExecutionRangeAccumulator.BeginFetchRange();
++this.retries;
Tuple <DocumentFeedResponse <CosmosElement>, string> responseAndPartitionIdentifier = await this.ExecuteOnceAsync(retryPolicyInstance, token);
DocumentFeedResponse <CosmosElement> response = responseAndPartitionIdentifier.Item1;
string partitionIdentifier = responseAndPartitionIdentifier.Item2;
if (!string.IsNullOrEmpty(response.ResponseHeaders[HttpConstants.HttpHeaders.QueryMetrics]))
{
this.fetchExecutionRangeAccumulator.EndFetchRange(
partitionIdentifier,
response.ActivityId,
response.Count,
this.retries);
response = new DocumentFeedResponse <CosmosElement>(
response,
response.Count,
response.Headers,
response.UseETagAsContinuation,
new Dictionary <string, QueryMetrics>
{
{
partitionIdentifier,
QueryMetrics.CreateFromDelimitedStringAndClientSideMetrics(
response.ResponseHeaders[HttpConstants.HttpHeaders.QueryMetrics],
response.ResponseHeaders[HttpConstants.HttpHeaders.IndexUtilization],
new ClientSideMetrics(
this.retries,
response.RequestCharge,
this.fetchExecutionRangeAccumulator.GetExecutionRanges()))
}
},
response.RequestStatistics,
response.DisallowContinuationTokenMessage,
response.ResponseLengthBytes);
}
this.retries = -1;
return response;
},
retryPolicyInstance,
token));
}
protected override async Task <FeedResponse <dynamic> > ExecuteInternalAsync(CancellationToken cancellationToken)
{
CollectionCache collectionCache = await this.Client.GetCollectionCacheAsync();
PartitionKeyRangeCache partitionKeyRangeCache = await this.Client.GetPartitionKeyRangeCache();
IDocumentClientRetryPolicy retryPolicyInstance = this.Client.RetryPolicy.GetRequestPolicy();
retryPolicyInstance = new InvalidPartitionExceptionRetryPolicy(collectionCache, retryPolicyInstance);
if (base.ResourceTypeEnum.IsPartitioned())
{
retryPolicyInstance = new PartitionKeyRangeGoneRetryPolicy(
collectionCache,
partitionKeyRangeCache,
PathsHelper.GetCollectionPath(base.ResourceLink),
retryPolicyInstance);
}
return(await BackoffRetryUtility <FeedResponse <dynamic> > .ExecuteAsync(
async() =>
{
this.fetchExecutionRangeAccumulator.BeginFetchRange();
++this.retries;
this.fetchSchedulingMetrics.Start();
this.fetchExecutionRangeAccumulator.BeginFetchRange();
FeedResponse <dynamic> response = await this.ExecuteOnceAsync(retryPolicyInstance, cancellationToken);
this.fetchSchedulingMetrics.Stop();
this.fetchExecutionRangeAccumulator.EndFetchRange(response.Count, this.retries);
if (!string.IsNullOrEmpty(response.Headers[HttpConstants.HttpHeaders.QueryMetrics]))
{
this.fetchExecutionRangeAccumulator.EndFetchRange(response.Count, this.retries);
response = new FeedResponse <dynamic>(
response,
response.Count,
response.Headers,
response.UseETagAsContinuation,
new Dictionary <string, QueryMetrics>
{
{
singlePartitionKeyId,
QueryMetrics.CreateFromDelimitedStringAndClientSideMetrics(
response.Headers[HttpConstants.HttpHeaders.QueryMetrics],
new ClientSideMetrics(
this.retries,
response.RequestCharge,
this.fetchExecutionRangeAccumulator.GetExecutionRanges(),
string.IsNullOrEmpty(response.ResponseContinuation) ? new List <Tuple <string, SchedulingTimeSpan> >()
{
new Tuple <string, SchedulingTimeSpan>(singlePartitionKeyId, this.fetchSchedulingMetrics.Elapsed)
} : new List <Tuple <string, SchedulingTimeSpan> >()),
Guid.Parse(response.ActivityId))
}
},
response.RequestStatistics,
response.DisallowContinuationTokenMessage,
response.ResponseLengthBytes);
}
this.retries = -1;
return response;
},
retryPolicyInstance,
cancellationToken));
}
public BulkPartitionKeyRangeGoneRetryPolicy(IDocumentClientRetryPolicy nextRetryPolicy)
{
this.nextRetryPolicy = nextRetryPolicy;
}
public InvalidPartitionExceptionRetryPolicy(
IDocumentClientRetryPolicy nextPolicy)
{
this.nextPolicy = nextPolicy;
}
