private ShouldRetryResult ShouldRetryInternal(
HttpStatusCode?statusCode,
SubStatusCodes?subStatusCode,
string resourceIdOrFullName)
{
if (!statusCode.HasValue &&
(!subStatusCode.HasValue ||
subStatusCode.Value == SubStatusCodes.Unknown))
{
return(null);
}
if (statusCode == HttpStatusCode.Gone &&
subStatusCode == SubStatusCodes.NameCacheIsStale)
{
if (!string.IsNullOrEmpty(resourceIdOrFullName))
{
this.clientCollectionCache.Refresh(resourceIdOrFullName);
}
return(ShouldRetryResult.NoRetry(new NotFoundException()));
}
return(null);
}
private Task <ShouldRetryResult> ShouldRetryAsyncInternal(
HttpStatusCode?statusCode,
SubStatusCodes?subStatusCode,
string resourceIdOrFullName,
Func <Task <ShouldRetryResult> > continueIfNotHandled)
{
if (!statusCode.HasValue &&
!subStatusCode.HasValue)
{
return(continueIfNotHandled());
}
if (statusCode == HttpStatusCode.Gone &&
subStatusCode == SubStatusCodes.NameCacheIsStale)
{
if (!string.IsNullOrEmpty(resourceIdOrFullName))
{
this.clientCollectionCache.Refresh(resourceIdOrFullName);
}
return(Task.FromResult(ShouldRetryResult.NoRetry(new NotFoundException())));
}
return(continueIfNotHandled());
}
public Task <ShouldRetryResult> ShouldRetryAsync(
Exception exception,
CancellationToken cancellationToken)
{
TimeSpan backoffTime = TimeSpan.FromSeconds(0);
if (!WebExceptionUtility.IsWebExceptionRetriable(exception))
{
// Have caller propagate original exception.
this.durationTimer.Stop();
return(Task.FromResult(ShouldRetryResult.NoRetry()));
}
// Don't penalise first retry with delay.
if (attemptCount++ > 1)
{
int remainingSeconds = WebExceptionRetryPolicy.waitTimeInSeconds - this.durationTimer.Elapsed.Seconds;
if (remainingSeconds <= 0)
{
this.durationTimer.Stop();
return(Task.FromResult(ShouldRetryResult.NoRetry()));
}
backoffTime = TimeSpan.FromSeconds(Math.Min(this.currentBackoffSeconds, remainingSeconds));
this.currentBackoffSeconds *= WebExceptionRetryPolicy.backoffMultiplier;
}
DefaultTrace.TraceWarning("Received retriable web exception, will retry, {0}", exception);
return(Task.FromResult(ShouldRetryResult.RetryAfter(backoffTime)));
}
private Task <ShouldRetryResult> ShouldRetryAsyncInternal(
HttpStatusCode?statusCode,
SubStatusCodes?subStatusCode,
string resourceIdOrFullName,
Func <Task <ShouldRetryResult> > continueIfNotHandled)
{
if (statusCode.HasValue &&
subStatusCode.HasValue &&
statusCode == HttpStatusCode.Gone &&
subStatusCode == SubStatusCodes.NameCacheIsStale)
{
if (!this.retried)
{
if (!string.IsNullOrEmpty(resourceIdOrFullName))
{
this.clientCollectionCache.Refresh(resourceIdOrFullName);
}
this.retried = true;
return(Task.FromResult(ShouldRetryResult.RetryAfter(TimeSpan.Zero)));
}
else
{
return(Task.FromResult(ShouldRetryResult.NoRetry()));
}
}
return(continueIfNotHandled != null?continueIfNotHandled() : Task.FromResult(ShouldRetryResult.NoRetry()));
}
/// <summary>
/// Should the caller retry the operation.
/// </summary>
/// <param name="exception">Exception that occured when the operation was tried</param>
/// <param name="cancellationToken"></param>
/// <returns>True indicates caller should retry, False otherwise</returns>
public Task <ShouldRetryResult> ShouldRetryAsync(
Exception exception,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
if (exception is DocumentClientException)
{
DocumentClientException dce = (DocumentClientException)exception;
if (!this.IsValidThrottleStatusCode(dce.StatusCode))
{
DefaultTrace.TraceError(
"Operation will NOT be retried. Current attempt {0}, Status Code: {1} ",
this.currentAttemptCount,
dce.StatusCode);
return(Task.FromResult(ShouldRetryResult.NoRetry()));
}
return(this.ShouldRetryInternalAsync(dce.RetryAfter));
}
DefaultTrace.TraceError(
"Operation will NOT be retried. Current attempt {0}, Exception: {1} ",
this.currentAttemptCount,
this.GetExceptionMessage(exception));
return(Task.FromResult(ShouldRetryResult.NoRetry()));
}
private ShouldRetryResult ShouldRetryInternal(
HttpStatusCode?statusCode,
SubStatusCodes?subStatusCode,
string resourceIdOrFullName)
{
if (!statusCode.HasValue &&
(!subStatusCode.HasValue ||
subStatusCode.Value == SubStatusCodes.Unknown))
{
return(null);
}
if (statusCode == HttpStatusCode.Gone &&
subStatusCode == SubStatusCodes.NameCacheIsStale)
{
if (!this.retried)
{
if (!string.IsNullOrEmpty(resourceIdOrFullName))
{
this.clientCollectionCache.Refresh(resourceIdOrFullName);
}
this.retried = true;
return(ShouldRetryResult.RetryAfter(TimeSpan.Zero));
}
else
{
return(ShouldRetryResult.NoRetry());
}
}
return(null);
}
private async Task <ShouldRetryResult> ShouldRetryOnEndpointFailureAsync(
bool isReadRequest,
bool forceRefresh,
bool retryOnPreferredLocations)
{
if (!this.enableEndpointDiscovery || this.failoverRetryCount > MaxRetryCount)
{
DefaultTrace.TraceInformation("ShouldRetryOnEndpointFailureAsync() Not retrying. Retry count = {0}", this.failoverRetryCount);
return(ShouldRetryResult.NoRetry());
}
this.failoverRetryCount++;
if (this.locationEndpoint != null)
{
if (isReadRequest)
{
this.globalEndpointManager.MarkEndpointUnavailableForRead(this.locationEndpoint);
}
else
{
this.globalEndpointManager.MarkEndpointUnavailableForWrite(this.locationEndpoint);
}
}
TimeSpan retryDelay = TimeSpan.Zero;
if (!isReadRequest)
{
DefaultTrace.TraceInformation("Failover happening. retryCount {0}", this.failoverRetryCount);
if (this.failoverRetryCount > 1)
{
//if retried both endpoints, follow regular retry interval.
retryDelay = TimeSpan.FromMilliseconds(ClientRetryPolicy.RetryIntervalInMS);
}
}
else
{
retryDelay = TimeSpan.FromMilliseconds(ClientRetryPolicy.RetryIntervalInMS);
}
await this.globalEndpointManager.RefreshLocationAsync(null, forceRefresh);
int retryLocationIndex = this.failoverRetryCount; // Used to generate a round-robin effect
if (retryOnPreferredLocations)
{
retryLocationIndex = 0; // When the endpoint is marked as unavailable, it is moved to the bottom of the preferrence list
}
this.retryContext = new RetryContext
{
RetryLocationIndex = retryLocationIndex,
RetryRequestOnPreferredLocations = retryOnPreferredLocations,
};
return(ShouldRetryResult.RetryAfter(retryDelay));
}
private ShouldRetryResult ShouldRetryOnSessionNotAvailable()
{
this.sessionTokenRetryCount++;
if (!this.enableEndpointDiscovery)
{
// if endpoint discovery is disabled, the request cannot be retried anywhere else
return(ShouldRetryResult.NoRetry());
}
else
{
if (this.canUseMultipleWriteLocations)
{
ReadOnlyCollection <Uri> endpoints = this.isReadRequest ? this.globalEndpointManager.ReadEndpoints : this.globalEndpointManager.WriteEndpoints;
if (this.sessionTokenRetryCount > endpoints.Count)
{
// When use multiple write locations is true and the request has been tried
// on all locations, then don't retry the request
return(ShouldRetryResult.NoRetry());
}
else
{
this.retryContext = new RetryContext()
{
RetryCount = this.sessionTokenRetryCount - 1,
RetryRequestOnPreferredLocations = this.sessionTokenRetryCount > 1,
ClearSessionTokenOnSessionNotAvailable = (this.sessionTokenRetryCount == endpoints.Count) // clear on last attempt
};
return(ShouldRetryResult.RetryAfter(TimeSpan.Zero));
}
}
else
{
if (this.sessionTokenRetryCount > 1)
{
// When cannot use multiple write locations, then don't retry the request if
// we have already tried this request on the write location
return(ShouldRetryResult.NoRetry());
}
else
{
this.retryContext = new RetryContext
{
RetryCount = this.sessionTokenRetryCount - 1,
RetryRequestOnPreferredLocations = false,
ClearSessionTokenOnSessionNotAvailable = true,
};
return(ShouldRetryResult.RetryAfter(TimeSpan.Zero));
}
}
}
}
private async Task <ShouldRetryResult> ShouldRetryAsyncInternal(
HttpStatusCode?statusCode,
SubStatusCodes?subStatusCode,
CancellationToken cancellationToken,
Func <Task <ShouldRetryResult> > continueIfNotHandled)
{
if (statusCode.HasValue &&
subStatusCode.HasValue &&
statusCode == HttpStatusCode.Gone &&
subStatusCode == SubStatusCodes.PartitionKeyRangeGone)
{
if (this.retried)
{
return(ShouldRetryResult.NoRetry());
}
using (DocumentServiceRequest request = DocumentServiceRequest.Create(
OperationType.Read,
ResourceType.Collection,
this.collectionLink,
null,
AuthorizationTokenType.PrimaryMasterKey))
{
CosmosContainerSettings collection = await this.collectionCache.ResolveCollectionAsync(request, cancellationToken);
CollectionRoutingMap routingMap = await this.partitionKeyRangeCache.TryLookupAsync(collection.ResourceId, null, request, false, cancellationToken);
if (routingMap != null)
{
// Force refresh.
await this.partitionKeyRangeCache.TryLookupAsync(
collection.ResourceId,
routingMap,
request,
false,
cancellationToken);
}
}
this.retried = true;
return(ShouldRetryResult.RetryAfter(TimeSpan.FromSeconds(0)));
}
if (continueIfNotHandled != null)
{
return(await continueIfNotHandled() ?? ShouldRetryResult.NoRetry());
}
else
{
return(await Task.FromResult(ShouldRetryResult.NoRetry()));
}
}
private async Task <ShouldRetryResult> ShouldRetryInternalAsync(
HttpStatusCode?statusCode,
SubStatusCodes?subStatusCode,
CancellationToken cancellationToken)
{
if (!statusCode.HasValue &&
(!subStatusCode.HasValue ||
subStatusCode.Value == SubStatusCodes.Unknown))
{
return(null);
}
if (statusCode == HttpStatusCode.Gone &&
subStatusCode == SubStatusCodes.PartitionKeyRangeGone)
{
if (this.retried)
{
return(ShouldRetryResult.NoRetry());
}
using (DocumentServiceRequest request = DocumentServiceRequest.Create(
OperationType.Read,
ResourceType.Collection,
this.collectionLink,
null,
AuthorizationTokenType.PrimaryMasterKey))
{
ContainerProperties collection = await this.collectionCache.ResolveCollectionAsync(request, cancellationToken, this.trace);
CollectionRoutingMap routingMap = await this.partitionKeyRangeCache.TryLookupAsync(
collectionRid : collection.ResourceId,
previousValue : null,
request : request,
trace : this.trace);
if (routingMap != null)
{
// Force refresh.
await this.partitionKeyRangeCache.TryLookupAsync(
collectionRid : collection.ResourceId,
previousValue : routingMap,
request : request,
trace : this.trace);
}
}
this.retried = true;
return(ShouldRetryResult.RetryAfter(TimeSpan.FromSeconds(0)));
}
return(null);
}
private async Task <ShouldRetryResult> ShouldRetryInternalAsync(
HttpStatusCode?statusCode,
SubStatusCodes?subStatusCode,
CancellationToken cancellationToken)
{
if (statusCode == HttpStatusCode.Gone)
{
this.retriesOn410++;
if (this.retriesOn410 > MaxRetryOn410)
{
return(ShouldRetryResult.NoRetry());
}
if (subStatusCode == SubStatusCodes.PartitionKeyRangeGone ||
subStatusCode == SubStatusCodes.CompletingSplit ||
subStatusCode == SubStatusCodes.CompletingPartitionMigration)
{
PartitionKeyRangeCache partitionKeyRangeCache = await this.container.ClientContext.DocumentClient.GetPartitionKeyRangeCacheAsync(NoOpTrace.Singleton);
string containerRid = await this.container.GetCachedRIDAsync(
forceRefresh : false,
NoOpTrace.Singleton,
cancellationToken : cancellationToken);
await partitionKeyRangeCache.TryGetOverlappingRangesAsync(
containerRid,
FeedRangeEpk.FullRange.Range,
NoOpTrace.Singleton,
forceRefresh : true);
return(ShouldRetryResult.RetryAfter(TimeSpan.Zero));
}
if (subStatusCode == SubStatusCodes.NameCacheIsStale)
{
return(ShouldRetryResult.RetryAfter(TimeSpan.Zero));
}
}
// Batch API can return 413 which means the response is bigger than 4Mb.
// Operations that exceed the 4Mb limit are returned as 413/3402, while the operations within the 4Mb limit will be 200
if (statusCode == HttpStatusCode.RequestEntityTooLarge &&
(int)subStatusCode == SubstatusCodeBatchResponseSizeExceeded)
{
return(ShouldRetryResult.RetryAfter(TimeSpan.Zero));
}
return(null);
}
/// <summary>
/// Based on the Retry Policy, if a failed response should retry.
/// </summary>
public Task <ShouldRetryResult> ShouldRetryAsync(
BatchOperationResult batchOperationResult,
CancellationToken cancellationToken)
{
if (this.retryPolicy == null ||
batchOperationResult.IsSuccessStatusCode)
{
return(Task.FromResult(ShouldRetryResult.NoRetry()));
}
ResponseMessage responseMessage = batchOperationResult.ToResponseMessage();
return(this.retryPolicy.ShouldRetryAsync(responseMessage, cancellationToken));
}
private async Task <ShouldRetryResult> ShouldRetryOnEndpointFailureAsync(bool isReadRequest, bool forceRefresh)
{
if (!this.enableEndpointDiscovery || this.failoverRetryCount > MaxRetryCount)
{
DefaultTrace.TraceInformation("ShouldRetryOnEndpointFailureAsync() Not retrying. Retry count = {0}", this.failoverRetryCount);
return(ShouldRetryResult.NoRetry());
}
this.failoverRetryCount++;
if (this.locationEndpoint != null)
{
if (isReadRequest)
{
this.globalEndpointManager.MarkEndpointUnavailableForRead(this.locationEndpoint);
}
else
{
this.globalEndpointManager.MarkEndpointUnavailableForWrite(this.locationEndpoint);
}
}
TimeSpan retryDelay = TimeSpan.Zero;
if (!isReadRequest)
{
DefaultTrace.TraceInformation("Failover happening. retryCount {0}", this.failoverRetryCount);
if (this.failoverRetryCount > 1)
{
//if retried both endpoints, follow regular retry interval.
retryDelay = TimeSpan.FromMilliseconds(ClientRetryPolicy.RetryIntervalInMS);
}
}
else
{
retryDelay = TimeSpan.FromMilliseconds(ClientRetryPolicy.RetryIntervalInMS);
}
await this.globalEndpointManager.RefreshLocationAsync(null, forceRefresh);
this.retryContext = new RetryContext
{
RetryCount = this.failoverRetryCount,
RetryRequestOnPreferredLocations = false
};
return(ShouldRetryResult.RetryAfter(retryDelay));
}
/// <summary>
/// Should the caller retry the operation.
/// </summary>
/// <param name="cosmosResponseMessage"><see cref="ResponseMessage"/> in return of the request</param>
/// <param name="cancellationToken"></param>
/// <returns>True indicates caller should retry, False otherwise</returns>
public Task <ShouldRetryResult> ShouldRetryAsync(
ResponseMessage cosmosResponseMessage,
CancellationToken cancellationToken)
{
if (!this.IsValidThrottleStatusCode(cosmosResponseMessage?.StatusCode))
{
DefaultTrace.TraceError(
"Operation will NOT be retried. Current attempt {0}, Status Code: {1} ",
this.currentAttemptCount,
cosmosResponseMessage?.StatusCode);
return(Task.FromResult(ShouldRetryResult.NoRetry()));
}
return(this.ShouldRetryInternalAsync(cosmosResponseMessage?.Headers.RetryAfter));
}
public Task <ShouldRetryResult> ShouldRetryAsync(
Exception exception,
CancellationToken cancellationToken)
{
HttpRequestMessage httpRequestMessage = this.getHttpReqestMessage();
this.diagnosticsContext.AddDiagnosticsInternal(
new PointOperationStatistics(
activityId: Trace.CorrelationManager.ActivityId.ToString(),
statusCode: HttpStatusCode.InternalServerError,
subStatusCode: SubStatusCodes.Unknown,
responseTimeUtc: DateTime.UtcNow,
requestCharge: 0,
errorMessage: exception.ToString(),
method: httpRequestMessage.Method,
requestUri: httpRequestMessage.RequestUri.OriginalString,
requestSessionToken: null,
responseSessionToken: null));
if (!this.IsExceptionTransientRetriable(
httpRequestMessage.Method,
exception,
cancellationToken))
{
// Have caller propagate original exception.
return(Task.FromResult(ShouldRetryResult.NoRetry()));
}
TimeSpan backOffTime = TimeSpan.FromSeconds(0);
// Don't penalize first retry with delay.
if (this.attemptCount++ > 1)
{
int remainingSeconds = TransientHttpClientRetryPolicy.WaitTimeInSeconds - (int)(DateTime.UtcNow - this.startDateTimeUtc).TotalSeconds;
if (remainingSeconds <= 0)
{
ShouldRetryResult shouldRetry = this.GetShouldRetryFromException(exception);
return(Task.FromResult(shouldRetry));
}
backOffTime = TimeSpan.FromSeconds(Math.Min(this.currentBackoffSeconds, remainingSeconds));
this.currentBackoffSeconds *= TransientHttpClientRetryPolicy.BackoffMultiplier;
}
DefaultTrace.TraceWarning("Received retriable web exception, will retry, {0}", exception);
return(Task.FromResult(ShouldRetryResult.RetryAfter(backOffTime)));
}
private ShouldRetryResult GetShouldRetryFromException(
Exception exception)
{
if (exception is OperationCanceledException operationCanceledException)
{
// throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
string message =
$"GatewayStoreClient Request Timeout. Start Time Utc:{this.startDateTimeUtc}; Total Duration:{(DateTime.UtcNow - this.startDateTimeUtc).TotalMilliseconds} Ms; Http Client Timeout:{this.gatewayRequestTimeout.TotalMilliseconds} Ms; Activity id: {Trace.CorrelationManager.ActivityId.ToString()};";
return(ShouldRetryResult.NoRetry(CosmosExceptionFactory.CreateRequestTimeoutException(
message,
innerException: operationCanceledException,
diagnosticsContext: this.diagnosticsContext)));
}
return(ShouldRetryResult.NoRetry());
}
public Task <ShouldRetryResult> ShouldRetryAsync(
ResponseMessage cosmosResponseMessage,
CancellationToken cancellationToken)
{
ShouldRetryResult shouldRetryResult = this.ShouldRetryInternal(cosmosResponseMessage?.StatusCode,
cosmosResponseMessage?.Headers.SubStatusCode);
if (shouldRetryResult != null)
{
return(Task.FromResult(shouldRetryResult));
}
if (this.nextRetryPolicy == null)
{
return(Task.FromResult(ShouldRetryResult.NoRetry()));
}
return(this.nextRetryPolicy.ShouldRetryAsync(cosmosResponseMessage, cancellationToken));
}
/// <summary>
/// Based on the Retry Policy, if a failed response should retry.
/// </summary>
public async Task <ShouldRetryResult> ShouldRetryAsync(
TransactionalBatchOperationResult batchOperationResult,
CancellationToken cancellationToken)
{
if (this.retryPolicy == null ||
batchOperationResult.IsSuccessStatusCode)
{
return(ShouldRetryResult.NoRetry());
}
ResponseMessage responseMessage = batchOperationResult.ToResponseMessage();
ShouldRetryResult shouldRetry = await this.retryPolicy.ShouldRetryAsync(responseMessage, cancellationToken);
if (shouldRetry.ShouldRetry)
{
this.initialTrace.AddChild(batchOperationResult.Trace);
}
return(shouldRetry);
}
public Task <ShouldRetryResult> ShouldRetryAsync(
Exception exception,
CancellationToken cancellationToken)
{
DocumentClientException clientException = exception as DocumentClientException;
ShouldRetryResult shouldRetryResult = this.ShouldRetryInternal(
clientException?.StatusCode,
clientException?.GetSubStatus());
if (shouldRetryResult != null)
{
return(Task.FromResult(shouldRetryResult));
}
if (this.nextRetryPolicy == null)
{
return(Task.FromResult(ShouldRetryResult.NoRetry()));
}
return(this.nextRetryPolicy.ShouldRetryAsync(exception, cancellationToken));
}
// Extracted partiotn key might be invaild as CollectionCache might be stale.
// Stale collection cache is refreshed through PartitionKeyMismatchRetryPolicy
// and partition-key is extracted again.
internal async Task <CosmosResponseMessage> ExtractPartitionKeyAndProcessItemStreamAsync(
PartitionKey partitionKey,
string itemId,
Stream streamPayload,
OperationType operationType,
RequestOptions requestOptions,
CancellationToken cancellationToken)
{
IDocumentClientRetryPolicy requestRetryPolicy = null;
if (partitionKey == null)
{
requestRetryPolicy = new PartitionKeyMismatchRetryPolicy(await this.ClientContext.DocumentClient.GetCollectionCacheAsync(), requestRetryPolicy);
}
while (true)
{
CosmosResponseMessage responseMessage = await this.ProcessItemStreamAsync(
partitionKey,
itemId,
streamPayload,
operationType,
requestOptions,
extractPartitionKeyIfNeeded : true,
cancellationToken : cancellationToken);
ShouldRetryResult retryResult = ShouldRetryResult.NoRetry();
if (requestRetryPolicy != null &&
!responseMessage.IsSuccessStatusCode)
{
retryResult = await requestRetryPolicy.ShouldRetryAsync(responseMessage, cancellationToken);
}
if (!retryResult.ShouldRetry)
{
return(responseMessage);
}
}
}
private Task <ShouldRetryResult> ShouldRetryInternalAsync(TimeSpan?retryAfter)
{
TimeSpan retryDelay = TimeSpan.Zero;
if (this.currentAttemptCount < this.maxAttemptCount &&
(this.CheckIfRetryNeeded(retryAfter, out retryDelay)))
{
this.currentAttemptCount++;
DefaultTrace.TraceWarning(
"Operation will be retried after {0} milliseconds. Current attempt {1}, Cumulative delay {2}",
retryDelay.TotalMilliseconds,
this.currentAttemptCount,
this.cumulativeRetryDelay);
return(Task.FromResult(ShouldRetryResult.RetryAfter(retryDelay)));
}
else
{
DefaultTrace.TraceError(
"Operation will NOT be retried. Current attempt {0}",
this.currentAttemptCount);
return(Task.FromResult(ShouldRetryResult.NoRetry()));
}
}
/// <summary>
/// For a ServiceUnavailable (503.0) we could be having a timeout from Direct/TCP locally or a request to Gateway request with a similar response due to an endpoint not yet available.
/// We try and retry the request only if there are other regions available.
/// </summary>
private ShouldRetryResult ShouldRetryOnServiceUnavailable()
{
if (this.serviceUnavailableRetryCount++ >= ClientRetryPolicy.MaxServiceUnavailableRetryCount)
{
DefaultTrace.TraceInformation($"ShouldRetryOnServiceUnavailable() Not retrying. Retry count = {this.serviceUnavailableRetryCount}.");
return(ShouldRetryResult.NoRetry());
}
if (!this.canUseMultipleWriteLocations &&
!this.isReadRequest)
{
// Write requests on single master cannot be retried, no other regions available
return(ShouldRetryResult.NoRetry());
}
int availablePreferredLocations = this.globalEndpointManager.PreferredLocationCount;
if (availablePreferredLocations <= 1)
{
// No other regions to retry on
DefaultTrace.TraceInformation($"ShouldRetryOnServiceUnavailable() Not retrying. No other regions available for the request. AvailablePreferredLocations = {availablePreferredLocations}.");
return(ShouldRetryResult.NoRetry());
}
DefaultTrace.TraceInformation($"ShouldRetryOnServiceUnavailable() Retrying. Received on endpoint {this.locationEndpoint}, IsReadRequest = {this.isReadRequest}.");
// Retrying on second PreferredLocations
// RetryCount is used as zero-based index
this.retryContext = new RetryContext()
{
RetryLocationIndex = this.serviceUnavailableRetryCount,
RetryRequestOnPreferredLocations = true
};
return(ShouldRetryResult.RetryAfter(TimeSpan.Zero));
}
public async Task <ShouldRetryResult> ShouldRetryAsync(
Exception exception,
CancellationToken cancellationToken)
{
if (exception is CosmosException clientException)
{
ShouldRetryResult shouldRetryResult = await this.ShouldRetryInternalAsync(
clientException.StatusCode,
(SubStatusCodes)clientException.SubStatusCode,
cancellationToken);
if (shouldRetryResult != null)
{
return(shouldRetryResult);
}
if (this.nextRetryPolicy == null)
{
return(ShouldRetryResult.NoRetry());
}
}
return(await this.nextRetryPolicy.ShouldRetryAsync(exception, cancellationToken));
}
private ShouldRetryResult ShouldRetryInternal(
HttpStatusCode?statusCode,
SubStatusCodes?subStatusCode,
string resourceIdOrFullName)
{
if (!statusCode.HasValue &&
(!subStatusCode.HasValue ||
subStatusCode.Value == SubStatusCodes.Unknown))
{
return(null);
}
if (statusCode == HttpStatusCode.Gone &&
subStatusCode == SubStatusCodes.NameCacheIsStale)
{
if (!this.retried)
{
if (this.documentServiceRequest == null)
{
throw new InvalidOperationException("OnBeforeSendRequest was never called");
}
this.documentServiceRequest.ForceNameCacheRefresh = true;
this.documentServiceRequest.ClearRoutingHints();
this.retried = true;
return(ShouldRetryResult.RetryAfter(TimeSpan.Zero));
}
else
{
return(ShouldRetryResult.NoRetry());
}
}
return(null);
}
public Task <ShouldRetryResult> ShouldRetryAsync(
ResponseMessage httpResponseMessage,
CancellationToken cancellationToken)
{
ShouldRetryResult shouldRetryResult = this.ShouldRetryInternal(
httpResponseMessage.StatusCode,
httpResponseMessage.Headers.SubStatusCode,
httpResponseMessage.GetResourceAddress());
if (shouldRetryResult != null)
{
return(Task.FromResult(shouldRetryResult));
}
return(this.nextPolicy != null?this.nextPolicy.ShouldRetryAsync(httpResponseMessage, cancellationToken) : Task.FromResult(ShouldRetryResult.NoRetry()));
}
public Task <ShouldRetryResult> ShouldRetryAsync(
Exception exception,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
DocumentClientException clientException = exception as DocumentClientException;
ShouldRetryResult shouldRetryResult = this.ShouldRetryInternal(
clientException?.StatusCode,
clientException?.GetSubStatus(),
clientException?.ResourceAddress);
if (shouldRetryResult != null)
{
return(Task.FromResult(shouldRetryResult));
}
return(this.nextPolicy != null?this.nextPolicy.ShouldRetryAsync(exception, cancellationToken) : Task.FromResult(ShouldRetryResult.NoRetry()));
}
/// <summary>
/// Should the caller retry the operation.
/// </summary>
/// <param name="cosmosResponseMessage"><see cref="CosmosResponseMessage"/> in return of the request</param>
/// <param name="cancellationToken"></param>
/// <returns>True indicates caller should retry, False otherwise</returns>
public async Task <ShouldRetryResult> ShouldRetryAsync(
CosmosResponseMessage cosmosResponseMessage,
CancellationToken cancellationToken)
{
ShouldRetryResult shouldRetryResult = await this.ShouldRetryInternalAsync(cosmosResponseMessage?.StatusCode,
cosmosResponseMessage?.Headers.SubStatusCode,
cancellationToken);
if (shouldRetryResult != null)
{
return(shouldRetryResult);
}
return(this.nextRetryPolicy != null ? await this.nextRetryPolicy?.ShouldRetryAsync(cosmosResponseMessage, cancellationToken) : ShouldRetryResult.NoRetry());
}
/// <summary>
/// Should the caller retry the operation.
/// </summary>
/// <param name="exception">Exception that occured when the operation was tried</param>
/// <param name="cancellationToken"></param>
/// <returns>True indicates caller should retry, False otherwise</returns>
public async Task <ShouldRetryResult> ShouldRetryAsync(
Exception exception,
CancellationToken cancellationToken)
{
DocumentClientException clientException = exception as DocumentClientException;
ShouldRetryResult shouldRetryResult = await this.ShouldRetryInternalAsync(clientException?.StatusCode,
clientException?.GetSubStatus(),
cancellationToken);
if (shouldRetryResult != null)
{
return(shouldRetryResult);
}
return(this.nextRetryPolicy != null ? await this.nextRetryPolicy?.ShouldRetryAsync(exception, cancellationToken) : ShouldRetryResult.NoRetry());
}
