private async Task <ShouldRetryResult> ShouldRetryInternalAsync(
HttpStatusCode?statusCode,
SubStatusCodes?subStatusCode,
CancellationToken cancellationToken)
{
if (statusCode == HttpStatusCode.Gone)
{
if (subStatusCode == SubStatusCodes.PartitionKeyRangeGone ||
subStatusCode == SubStatusCodes.CompletingSplit ||
subStatusCode == SubStatusCodes.CompletingPartitionMigration)
{
PartitionKeyRangeCache partitionKeyRangeCache = await this.container.ClientContext.DocumentClient.GetPartitionKeyRangeCacheAsync();
string containerRid = await this.container.GetCachedRIDAsync(forceRefresh : false, cancellationToken : cancellationToken);
await partitionKeyRangeCache.TryGetOverlappingRangesAsync(containerRid, FeedRangeEpk.FullRange.Range, forceRefresh : true);
return(ShouldRetryResult.RetryAfter(TimeSpan.Zero));
}
if (subStatusCode == SubStatusCodes.NameCacheIsStale)
{
return(ShouldRetryResult.RetryAfter(TimeSpan.Zero));
}
}
return(null);
}
private Task <ShouldRetryResult> ShouldRetryAsyncInternal(
HttpStatusCode?statusCode,
SubStatusCodes?subStatusCode,
string resourceIdOrFullName,
Func <Task <ShouldRetryResult> > continueIfNotHandled)
{
if (!statusCode.HasValue &&
!subStatusCode.HasValue)
{
return(continueIfNotHandled());
}
if (statusCode == HttpStatusCode.Gone &&
subStatusCode == SubStatusCodes.NameCacheIsStale)
{
if (!string.IsNullOrEmpty(resourceIdOrFullName))
{
this.clientCollectionCache.Refresh(resourceIdOrFullName);
}
return(Task.FromResult(ShouldRetryResult.NoRetry(new NotFoundException())));
}
return(continueIfNotHandled());
}
private async Task <ShouldRetryResult> ShouldRetryInternalAsync(
HttpStatusCode?statusCode,
SubStatusCodes?subStatusCode,
ShouldRetryResult shouldRetryResult,
CancellationToken cancellationToken)
{
if (this.request == null)
{
// someone didn't call OnBeforeSendRequest - nothing we can do
DefaultTrace.TraceCritical("Cannot apply RenameCollectionAwareClientRetryPolicy as OnBeforeSendRequest has not been called and there is no DocumentServiceRequest context.");
return(shouldRetryResult);
}
Debug.Assert(shouldRetryResult != null);
if (!shouldRetryResult.ShouldRetry && !this.hasTriggered && statusCode.HasValue && subStatusCode.HasValue)
{
if (this.request.IsNameBased &&
statusCode.Value == HttpStatusCode.NotFound &&
subStatusCode.Value == SubStatusCodes.ReadSessionNotAvailable)
{
// Clear the session token, because the collection name might be reused.
DefaultTrace.TraceWarning("Clear the the token for named base request {0}", request.ResourceAddress);
this.sessionContainer.ClearTokenByCollectionFullname(request.ResourceAddress);
this.hasTriggered = true;
string oldCollectionRid = request.RequestContext.ResolvedCollectionRid;
request.ForceNameCacheRefresh = true;
request.RequestContext.ResolvedCollectionRid = null;
try
{
CosmosContainerProperties collectionInfo = await this.collectionCache.ResolveCollectionAsync(request, cancellationToken);
if (collectionInfo == null)
{
DefaultTrace.TraceCritical("Can't recover from session unavailable exception because resolving collection name {0} returned null", request.ResourceAddress);
}
else if (!string.IsNullOrEmpty(oldCollectionRid) && !string.IsNullOrEmpty(collectionInfo.ResourceId))
{
if (!oldCollectionRid.Equals(collectionInfo.ResourceId))
{
return(ShouldRetryResult.RetryAfter(TimeSpan.Zero));
}
}
}
catch (Exception e)
{
// When ResolveCollectionAsync throws an exception ignore it because it's an attempt to recover an existing
// error. When the recovery fails we return ShouldRetryResult.NoRetry and propaganate the original exception to the client
DefaultTrace.TraceCritical("Can't recover from session unavailable exception because resolving collection name {0} failed with {1}", request.ResourceAddress, e.ToString());
}
}
}
return(shouldRetryResult);
}
private ShouldRetryResult ShouldRetryInternal(
HttpStatusCode?statusCode,
SubStatusCodes?subStatusCode,
string resourceIdOrFullName)
{
if (!statusCode.HasValue &&
(!subStatusCode.HasValue ||
subStatusCode.Value == SubStatusCodes.Unknown))
{
return(null);
}
if (statusCode == HttpStatusCode.Gone &&
subStatusCode == SubStatusCodes.NameCacheIsStale)
{
if (!this.retried)
{
if (!string.IsNullOrEmpty(resourceIdOrFullName))
{
this.clientCollectionCache.Refresh(resourceIdOrFullName);
}
this.retried = true;
return(ShouldRetryResult.RetryAfter(TimeSpan.Zero));
}
else
{
return(ShouldRetryResult.NoRetry());
}
}
return(null);
}
private async Task <ResponseMessage> ReadNextInternalAsync(
CosmosDiagnosticsContext diagnostics,
CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
Stream stream = null;
OperationType operation = OperationType.ReadFeed;
if (this.querySpec != null)
{
stream = this.clientContext.SerializerCore.ToStreamSqlQuerySpec(this.querySpec, this.resourceType);
operation = OperationType.Query;
}
ResponseMessage responseMessage = await this.clientContext.ProcessResourceOperationStreamAsync(
resourceUri : this.containerCore.LinkUri,
resourceType : this.resourceType,
operationType : operation,
requestOptions : this.queryRequestOptions,
cosmosContainerCore : this.containerCore,
partitionKey : this.queryRequestOptions?.PartitionKey,
streamPayload : stream,
requestEnricher : request =>
{
FeedRangeVisitor feedRangeVisitor = new FeedRangeVisitor(request);
this.FeedRangeInternal.Accept(feedRangeVisitor);
this.FeedRangeContinuation.Accept(feedRangeVisitor, QueryRequestOptions.FillContinuationToken);
if (this.querySpec != null)
{
request.Headers.Add(HttpConstants.HttpHeaders.ContentType, MediaTypes.QueryJson);
request.Headers.Add(HttpConstants.HttpHeaders.IsQuery, bool.TrueString);
}
},
diagnosticsContext : diagnostics,
cancellationToken : cancellationToken);
ShouldRetryResult shouldRetryOnSplit = await this.FeedRangeContinuation.HandleSplitAsync(this.containerCore, responseMessage, cancellationToken);
if (shouldRetryOnSplit.ShouldRetry)
{
return(await this.ReadNextInternalAsync(diagnostics, cancellationToken));
}
if (responseMessage.Content != null)
{
await CosmosElementSerializer.RewriteStreamAsTextAsync(responseMessage, this.queryRequestOptions);
}
if (responseMessage.IsSuccessStatusCode)
{
this.FeedRangeContinuation.ReplaceContinuation(responseMessage.Headers.ContinuationToken);
this.hasMoreResultsInternal = !this.FeedRangeContinuation.IsDone;
return(FeedRangeResponse.CreateSuccess(responseMessage, this.FeedRangeContinuation));
}
else
{
this.hasMoreResultsInternal = false;
return(FeedRangeResponse.CreateFailure(responseMessage));
}
}
private ShouldRetryResult ShouldRetryInternal(
HttpStatusCode?statusCode,
SubStatusCodes?subStatusCode,
string resourceIdOrFullName)
{
if (!statusCode.HasValue &&
(!subStatusCode.HasValue ||
subStatusCode.Value == SubStatusCodes.Unknown))
{
return(null);
}
if (statusCode == HttpStatusCode.BadRequest &&
subStatusCode == SubStatusCodes.PartitionKeyMismatch &&
this.retriesAttempted < MaxRetries)
{
Debug.Assert(resourceIdOrFullName != null);
if (!string.IsNullOrEmpty(resourceIdOrFullName))
{
this.clientCollectionCache.Refresh(resourceIdOrFullName, HttpConstants.Versions.CurrentVersion);
}
this.retriesAttempted++;
return(ShouldRetryResult.RetryAfter(TimeSpan.Zero));
}
return(null);
}
private ShouldRetryResult ShouldRetryInternal(
HttpStatusCode?statusCode,
SubStatusCodes?subStatusCode,
string resourceIdOrFullName)
{
if (!statusCode.HasValue &&
(!subStatusCode.HasValue ||
subStatusCode.Value == SubStatusCodes.Unknown))
{
return(null);
}
if (statusCode == HttpStatusCode.Gone &&
subStatusCode == SubStatusCodes.NameCacheIsStale)
{
if (!string.IsNullOrEmpty(resourceIdOrFullName))
{
this.clientCollectionCache.Refresh(resourceIdOrFullName);
}
return(ShouldRetryResult.NoRetry(new NotFoundException()));
}
return(null);
}
private Task <ShouldRetryResult> ShouldRetryAsyncInternal(
HttpStatusCode?statusCode,
SubStatusCodes?subStatusCode,
string resourceIdOrFullName,
Func <Task <ShouldRetryResult> > continueIfNotHandled)
{
if (!statusCode.HasValue &&
!subStatusCode.HasValue)
{
return(continueIfNotHandled());
}
if (statusCode == HttpStatusCode.BadRequest &&
subStatusCode == SubStatusCodes.PartitionKeyMismatch &&
this.retriesAttempted < MaxRetries)
{
Debug.Assert(resourceIdOrFullName != null);
if (!string.IsNullOrEmpty(resourceIdOrFullName))
{
this.clientCollectionCache.Refresh(resourceIdOrFullName);
}
this.retriesAttempted++;
return(Task.FromResult(ShouldRetryResult.RetryAfter(TimeSpan.Zero)));
}
return(continueIfNotHandled());
}
public async Task RetriesOnWebException()
{
HttpMethod httpMethod = HttpMethod.Get;
HttpRequestMessage httpRequestMessageFunc() => new HttpRequestMessage(httpMethod, "https://localhost:8081");
IRetryPolicy retryPolicy = new TransientHttpClientRetryPolicy(
httpRequestMessageFunc,
TimeSpan.FromSeconds(10),
new CosmosDiagnosticsContextCore());
List <HttpMethod> httpMethods = new List <HttpMethod>()
{
HttpMethod.Get,
HttpMethod.Delete,
HttpMethod.Patch,
HttpMethod.Post,
HttpMethod.Put
};
foreach (HttpMethod method in httpMethods)
{
httpMethod = method;
ShouldRetryResult shouldRetryResult = await retryPolicy.ShouldRetryAsync(new WebException("Test", WebExceptionStatus.ConnectFailure), default(CancellationToken));
Assert.IsTrue(shouldRetryResult.ShouldRetry, method.ToString());
}
}
/// <summary>
/// Should the caller retry the operation.
/// </summary>
/// <param name="exception">Exception that occured when the operation was tried</param>
/// <param name="cancellationToken"></param>
/// <returns>True indicates caller should retry, False otherwise</returns>
public Task <ShouldRetryResult> ShouldRetryAsync(
Exception exception,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
if (exception is DocumentClientException)
{
DocumentClientException dce = (DocumentClientException)exception;
if (!this.IsValidThrottleStatusCode(dce.StatusCode))
{
DefaultTrace.TraceError(
"Operation will NOT be retried. Current attempt {0}, Status Code: {1} ",
this.currentAttemptCount,
dce.StatusCode);
return(Task.FromResult(ShouldRetryResult.NoRetry()));
}
return(this.ShouldRetryInternalAsync(dce.RetryAfter));
}
DefaultTrace.TraceError(
"Operation will NOT be retried. Current attempt {0}, Exception: {1} ",
this.currentAttemptCount,
this.GetExceptionMessage(exception));
return(Task.FromResult(ShouldRetryResult.NoRetry()));
}
private ShouldRetryResult ShouldRetryInternal(
HttpStatusCode?statusCode,
SubStatusCodes?subStatusCode,
ShouldRetryResult shouldRetryResult)
{
if (this.request == null)
{
// someone didn't call OnBeforeSendRequest - nothing we can do
return(shouldRetryResult);
}
if (!shouldRetryResult.ShouldRetry && !this.hasTriggered && statusCode.HasValue && subStatusCode.HasValue)
{
if (this.request.IsNameBased &&
statusCode.Value == HttpStatusCode.NotFound &&
subStatusCode.Value == SubStatusCodes.ReadSessionNotAvailable)
{
// Clear the session token, because the collection name might be reused.
DefaultTrace.TraceWarning("Clear the the token for named base request {0}", request.ResourceAddress);
this.sessionContainer.ClearTokenByCollectionFullname(request.ResourceAddress);
this.hasTriggered = true;
}
}
return(shouldRetryResult);
}
/// <summary>
/// Should the caller retry the operation.
/// </summary>
/// <param name="exception">Exception that occurred when the operation was tried</param>
/// <param name="cancellationToken"></param>
/// <returns>True indicates caller should retry, False otherwise</returns>
public async Task <ShouldRetryResult> ShouldRetryAsync(
Exception exception,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
this.retryContext = null;
// Received Connection error (HttpRequestException), initiate the endpoint rediscovery
if (exception is HttpRequestException _)
{
DefaultTrace.TraceWarning("Endpoint not reachable. Refresh cache and retry");
return(await this.ShouldRetryOnEndpointFailureAsync(
isReadRequest : this.isReadRequest,
forceRefresh : false,
retryOnPreferredLocations : true));
}
DocumentClientException clientException = exception as DocumentClientException;
ShouldRetryResult shouldRetryResult = await this.ShouldRetryInternalAsync(
clientException?.StatusCode,
clientException?.GetSubStatus());
if (shouldRetryResult != null)
{
return(shouldRetryResult);
}
return(await this.throttlingRetry.ShouldRetryAsync(exception, cancellationToken));
}
public async Task PartitionKeyRangeGoneRetryPolicyWithNextRetryPolicy()
{
Mock <IDocumentClientRetryPolicy> nextRetryPolicyMock = new Mock <IDocumentClientRetryPolicy>();
nextRetryPolicyMock
.Setup(m => m.ShouldRetryAsync(It.IsAny <ResponseMessage>(), It.IsAny <CancellationToken>()))
.Returns(() => Task.FromResult <ShouldRetryResult>(ShouldRetryResult.RetryAfter(TimeSpan.FromDays(1))))
.Verifiable();
nextRetryPolicyMock
.Setup(m => m.ShouldRetryAsync(It.IsAny <Exception>(), It.IsAny <CancellationToken>()))
.Returns(() => Task.FromResult <ShouldRetryResult>(ShouldRetryResult.RetryAfter(TimeSpan.FromDays(1))))
.Verifiable();
PartitionKeyRangeGoneRetryPolicy retryPolicy = new PartitionKeyRangeGoneRetryPolicy(null, null, null, nextRetryPolicyMock.Object);
ShouldRetryResult exceptionResult = await retryPolicy.ShouldRetryAsync(new Exception("", null), CancellationToken.None);
Assert.IsNotNull(exceptionResult);
Assert.IsTrue(exceptionResult.ShouldRetry);
Assert.AreEqual(TimeSpan.FromDays(1), exceptionResult.BackoffTime);
ShouldRetryResult messageResult = await retryPolicy.ShouldRetryAsync(new ResponseMessage(), CancellationToken.None);
Assert.IsNotNull(exceptionResult);
Assert.IsTrue(exceptionResult.ShouldRetry);
Assert.AreEqual(TimeSpan.FromDays(1), exceptionResult.BackoffTime);
}
/// <summary>
/// Should the caller retry the operation.
/// </summary>
/// <param name="exception">Exception that occurred when the operation was tried</param>
/// <param name="cancellationToken"></param>
/// <returns>True indicates caller should retry, False otherwise</returns>
public async Task <ShouldRetryResult> ShouldRetryAsync(
Exception exception,
CancellationToken cancellationToken)
{
this.retryContext = null;
// Received Connection error (HttpRequestException), initiate the endpoint rediscovery
if (exception is HttpRequestException _)
{
DefaultTrace.TraceWarning("ClientRetryPolicy: Gateway HttpRequestException Endpoint not reachable. Failed Location: {0}; ResourceAddress: {1}",
this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty,
this.documentServiceRequest?.ResourceAddress ?? string.Empty);
// Mark both read and write requests because it gateway exception.
// This means all requests going to the region will fail.
return(await this.ShouldRetryOnEndpointFailureAsync(
isReadRequest : this.isReadRequest,
markBothReadAndWriteAsUnavailable : true,
forceRefresh : false,
retryOnPreferredLocations : true));
}
if (exception is DocumentClientException clientException)
{
ShouldRetryResult shouldRetryResult = await this.ShouldRetryInternalAsync(
clientException?.StatusCode,
clientException?.GetSubStatus());
if (shouldRetryResult != null)
{
return(shouldRetryResult);
}
}
return(await this.throttlingRetry.ShouldRetryAsync(exception, cancellationToken));
}
public Task <ShouldRetryResult> ShouldRetryAsync(
Exception exception,
CancellationToken cancellationToken)
{
TimeSpan backoffTime = TimeSpan.FromSeconds(0);
if (!WebExceptionUtility.IsWebExceptionRetriable(exception))
{
// Have caller propagate original exception.
this.durationTimer.Stop();
return(Task.FromResult(ShouldRetryResult.NoRetry()));
}
// Don't penalise first retry with delay.
if (attemptCount++ > 1)
{
int remainingSeconds = WebExceptionRetryPolicy.waitTimeInSeconds - this.durationTimer.Elapsed.Seconds;
if (remainingSeconds <= 0)
{
this.durationTimer.Stop();
return(Task.FromResult(ShouldRetryResult.NoRetry()));
}
backoffTime = TimeSpan.FromSeconds(Math.Min(this.currentBackoffSeconds, remainingSeconds));
this.currentBackoffSeconds *= WebExceptionRetryPolicy.backoffMultiplier;
}
DefaultTrace.TraceWarning("Received retriable web exception, will retry, {0}", exception);
return(Task.FromResult(ShouldRetryResult.RetryAfter(backoffTime)));
}
private Task <ShouldRetryResult> ShouldRetryAsyncInternal(
HttpStatusCode?statusCode,
SubStatusCodes?subStatusCode,
string resourceIdOrFullName,
Func <Task <ShouldRetryResult> > continueIfNotHandled)
{
if (statusCode.HasValue &&
subStatusCode.HasValue &&
statusCode == HttpStatusCode.Gone &&
subStatusCode == SubStatusCodes.NameCacheIsStale)
{
if (!this.retried)
{
if (!string.IsNullOrEmpty(resourceIdOrFullName))
{
this.clientCollectionCache.Refresh(resourceIdOrFullName);
}
this.retried = true;
return(Task.FromResult(ShouldRetryResult.RetryAfter(TimeSpan.Zero)));
}
else
{
return(Task.FromResult(ShouldRetryResult.NoRetry()));
}
}
return(continueIfNotHandled != null?continueIfNotHandled() : Task.FromResult(ShouldRetryResult.NoRetry()));
}
private async Task <ShouldRetryResult> ShouldRetryOnEndpointFailureAsync(
bool isReadRequest,
bool forceRefresh,
bool retryOnPreferredLocations)
{
if (!this.enableEndpointDiscovery || this.failoverRetryCount > MaxRetryCount)
{
DefaultTrace.TraceInformation("ShouldRetryOnEndpointFailureAsync() Not retrying. Retry count = {0}", this.failoverRetryCount);
return(ShouldRetryResult.NoRetry());
}
this.failoverRetryCount++;
if (this.locationEndpoint != null)
{
if (isReadRequest)
{
this.globalEndpointManager.MarkEndpointUnavailableForRead(this.locationEndpoint);
}
else
{
this.globalEndpointManager.MarkEndpointUnavailableForWrite(this.locationEndpoint);
}
}
TimeSpan retryDelay = TimeSpan.Zero;
if (!isReadRequest)
{
DefaultTrace.TraceInformation("Failover happening. retryCount {0}", this.failoverRetryCount);
if (this.failoverRetryCount > 1)
{
//if retried both endpoints, follow regular retry interval.
retryDelay = TimeSpan.FromMilliseconds(ClientRetryPolicy.RetryIntervalInMS);
}
}
else
{
retryDelay = TimeSpan.FromMilliseconds(ClientRetryPolicy.RetryIntervalInMS);
}
await this.globalEndpointManager.RefreshLocationAsync(null, forceRefresh);
int retryLocationIndex = this.failoverRetryCount; // Used to generate a round-robin effect
if (retryOnPreferredLocations)
{
retryLocationIndex = 0; // When the endpoint is marked as unavailable, it is moved to the bottom of the preferrence list
}
this.retryContext = new RetryContext
{
RetryLocationIndex = retryLocationIndex,
RetryRequestOnPreferredLocations = retryOnPreferredLocations,
};
return(ShouldRetryResult.RetryAfter(retryDelay));
}
private async Task <ShouldRetryResult> ShouldRetryInternalAsync(
HttpStatusCode?statusCode,
SubStatusCodes?subStatusCode)
{
if (!statusCode.HasValue &&
(!subStatusCode.HasValue ||
subStatusCode.Value == SubStatusCodes.Unknown))
{
return(null);
}
// Received 403.3 on write region, initiate the endpoint rediscovery
if (statusCode == HttpStatusCode.Forbidden &&
subStatusCode == SubStatusCodes.WriteForbidden)
{
if (this.partitionKeyRangeLocationCache.TryMarkEndpointUnavailableForPartitionKeyRange(
this.documentServiceRequest))
{
return(ShouldRetryResult.RetryAfter(TimeSpan.Zero));
}
DefaultTrace.TraceWarning("Endpoint not writable. Refresh cache and retry");
return(await this.ShouldRetryOnEndpointFailureAsync(
isReadRequest : false,
forceRefresh : true,
retryOnPreferredLocations : false));
}
// Regional endpoint is not available yet for reads (e.g. add/ online of region is in progress)
if (statusCode == HttpStatusCode.Forbidden &&
subStatusCode == SubStatusCodes.DatabaseAccountNotFound &&
(this.isReadRequest || this.canUseMultipleWriteLocations))
{
DefaultTrace.TraceWarning("Endpoint not available for reads. Refresh cache and retry");
return(await this.ShouldRetryOnEndpointFailureAsync(
isReadRequest : this.isReadRequest,
forceRefresh : false,
retryOnPreferredLocations : false));
}
if (statusCode == HttpStatusCode.NotFound &&
subStatusCode == SubStatusCodes.ReadSessionNotAvailable)
{
return(this.ShouldRetryOnSessionNotAvailable());
}
// Received 503.0 due to client connect timeout or Gateway
if (statusCode == HttpStatusCode.ServiceUnavailable &&
subStatusCode == SubStatusCodes.Unknown)
{
this.partitionKeyRangeLocationCache.TryMarkEndpointUnavailableForPartitionKeyRange(
this.documentServiceRequest);
return(this.ShouldRetryOnServiceUnavailable());
}
return(null);
}
public async Task ShouldRetry_NoPolicy()
{
BatchOperationResult result = new BatchOperationResult(HttpStatusCode.OK);
ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, 0);
operation.AttachContext(new ItemBatchOperationContext(string.Empty));
ShouldRetryResult shouldRetryResult = await operation.Context.ShouldRetryAsync(result, default(CancellationToken));
Assert.IsFalse(shouldRetryResult.ShouldRetry);
}
public async Task RetriesOn429()
{
IDocumentClientRetryPolicy retryPolicy = new BulkPartitionKeyRangeGoneRetryPolicy(
new ResourceThrottleRetryPolicy(1));
BatchOperationResult result = new BatchOperationResult((HttpStatusCode)StatusCodes.TooManyRequests);
ShouldRetryResult shouldRetryResult = await retryPolicy.ShouldRetryAsync(result.ToResponseMessage(), default(CancellationToken));
Assert.IsTrue(shouldRetryResult.ShouldRetry);
}
public async Task ShouldRetry_NoPolicy()
{
TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.OK);
ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, 0, Cosmos.PartitionKey.Null);
operation.AttachContext(new ItemBatchOperationContext(string.Empty));
ShouldRetryResult shouldRetryResult = await operation.Context.ShouldRetryAsync(result, default);
Assert.IsFalse(shouldRetryResult.ShouldRetry);
}
public async Task NotRetryOnSuccess()
{
IDocumentClientRetryPolicy retryPolicy = new BulkPartitionKeyRangeGoneRetryPolicy(
new ResourceThrottleRetryPolicy(1));
BatchOperationResult result = new BatchOperationResult(HttpStatusCode.OK);
ShouldRetryResult shouldRetryResult = await retryPolicy.ShouldRetryAsync(result.ToResponseMessage(), default(CancellationToken));
Assert.IsFalse(shouldRetryResult.ShouldRetry);
}
private ShouldRetryResult ShouldRetryOnSessionNotAvailable()
{
this.sessionTokenRetryCount++;
if (!this.enableEndpointDiscovery)
{
// if endpoint discovery is disabled, the request cannot be retried anywhere else
return(ShouldRetryResult.NoRetry());
}
else
{
if (this.canUseMultipleWriteLocations)
{
ReadOnlyCollection <Uri> endpoints = this.isReadRequest ? this.globalEndpointManager.ReadEndpoints : this.globalEndpointManager.WriteEndpoints;
if (this.sessionTokenRetryCount > endpoints.Count)
{
// When use multiple write locations is true and the request has been tried
// on all locations, then don't retry the request
return(ShouldRetryResult.NoRetry());
}
else
{
this.retryContext = new RetryContext()
{
RetryCount = this.sessionTokenRetryCount - 1,
RetryRequestOnPreferredLocations = this.sessionTokenRetryCount > 1,
ClearSessionTokenOnSessionNotAvailable = (this.sessionTokenRetryCount == endpoints.Count) // clear on last attempt
};
return(ShouldRetryResult.RetryAfter(TimeSpan.Zero));
}
}
else
{
if (this.sessionTokenRetryCount > 1)
{
// When cannot use multiple write locations, then don't retry the request if
// we have already tried this request on the write location
return(ShouldRetryResult.NoRetry());
}
else
{
this.retryContext = new RetryContext
{
RetryCount = this.sessionTokenRetryCount - 1,
RetryRequestOnPreferredLocations = false,
ClearSessionTokenOnSessionNotAvailable = true,
};
return(ShouldRetryResult.RetryAfter(TimeSpan.Zero));
}
}
}
}
// Extracted partition key might be invalid as CollectionCache might be stale.
// Stale collection cache is refreshed through PartitionKeyMismatchRetryPolicy
// and partition-key is extracted again.
internal async Task <ResponseMessage> ExtractPartitionKeyAndProcessItemStreamAsync <T>(
PartitionKey?partitionKey,
string itemId,
T item,
OperationType operationType,
RequestOptions requestOptions,
CancellationToken cancellationToken)
{
Stream streamPayload = this.ClientContext.SerializerCore.ToStream <T>(item);
// User specified PK value, no need to extract it
if (partitionKey.HasValue)
{
return(await this.ProcessItemStreamAsync(
partitionKey,
itemId,
streamPayload,
operationType,
requestOptions,
cancellationToken : cancellationToken));
}
PartitionKeyMismatchRetryPolicy requestRetryPolicy = null;
while (true)
{
partitionKey = await this.GetPartitionKeyValueFromStreamAsync(streamPayload, cancellationToken);
ResponseMessage responseMessage = await this.ProcessItemStreamAsync(
partitionKey,
itemId,
streamPayload,
operationType,
requestOptions,
cancellationToken : cancellationToken);
if (responseMessage.IsSuccessStatusCode)
{
return(responseMessage);
}
if (requestRetryPolicy == null)
{
requestRetryPolicy = new PartitionKeyMismatchRetryPolicy(await this.ClientContext.DocumentClient.GetCollectionCacheAsync(), null);
}
ShouldRetryResult retryResult = await requestRetryPolicy.ShouldRetryAsync(responseMessage, cancellationToken);
if (!retryResult.ShouldRetry)
{
return(responseMessage);
}
}
}
public async Task NotRetryOnSuccess()
{
IDocumentClientRetryPolicy retryPolicy = new BulkPartitionKeyRangeGoneRetryPolicy(
Mock.Of <ContainerInternal>(),
new ResourceThrottleRetryPolicy(1));
TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.OK);
ShouldRetryResult shouldRetryResult = await retryPolicy.ShouldRetryAsync(result.ToResponseMessage(), default);
Assert.IsFalse(shouldRetryResult.ShouldRetry);
}
public async Task RetriesOn429()
{
IDocumentClientRetryPolicy retryPolicy = new BulkPartitionKeyRangeGoneRetryPolicy(
Mock.Of <ContainerInternal>(),
new ResourceThrottleRetryPolicy(1));
TransactionalBatchOperationResult result = new TransactionalBatchOperationResult((HttpStatusCode)StatusCodes.TooManyRequests);
ShouldRetryResult shouldRetryResult = await retryPolicy.ShouldRetryAsync(result.ToResponseMessage(), default);
Assert.IsTrue(shouldRetryResult.ShouldRetry);
}
public async Task <ShouldRetryResult> ShouldRetryAsync(
CosmosResponseMessage cosmosResponseMessage,
CancellationToken cancellationToken)
{
ShouldRetryResult shouldRetryResult = await this.retryPolicy.ShouldRetryAsync(cosmosResponseMessage, cancellationToken);
return(await this.ShouldRetryInternalAsync(
cosmosResponseMessage?.StatusCode,
cosmosResponseMessage?.Headers.SubStatusCode,
shouldRetryResult,
cancellationToken));
}
public async Task <ShouldRetryResult> ShouldRetryAsync(Exception exception, CancellationToken cancellationToken)
{
ShouldRetryResult shouldRetry = await this.retryPolicy.ShouldRetryAsync(exception, cancellationToken);
DocumentClientException clientException = exception as DocumentClientException;
return(await this.ShouldRetryInternalAsync(
clientException?.StatusCode,
clientException?.GetSubStatus(),
shouldRetry,
cancellationToken));
}
public async Task ShouldRetry_WithPolicy_On429()
{
IDocumentClientRetryPolicy retryPolicy = new BulkPartitionKeyRangeGoneRetryPolicy(
new ResourceThrottleRetryPolicy(1));
TransactionalBatchOperationResult result = new TransactionalBatchOperationResult((HttpStatusCode)StatusCodes.TooManyRequests);
ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, 0, Cosmos.PartitionKey.Null);
operation.AttachContext(new ItemBatchOperationContext(string.Empty, retryPolicy));
ShouldRetryResult shouldRetryResult = await operation.Context.ShouldRetryAsync(result, default(CancellationToken));
Assert.IsTrue(shouldRetryResult.ShouldRetry);
}
public async Task ShouldRetry_WithPolicy_OnSuccess()
{
IDocumentClientRetryPolicy retryPolicy = new BulkPartitionKeyRangeGoneRetryPolicy(
new ResourceThrottleRetryPolicy(1));
BatchOperationResult result = new BatchOperationResult(HttpStatusCode.OK);
ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, 0);
operation.AttachContext(new ItemBatchOperationContext(string.Empty, retryPolicy));
ShouldRetryResult shouldRetryResult = await operation.Context.ShouldRetryAsync(result, default(CancellationToken));
Assert.IsFalse(shouldRetryResult.ShouldRetry);
}
