public void EnsureCorrectStatusCode()
{
string testMessage = "Test" + Guid.NewGuid().ToString();
string activityId = Guid.NewGuid().ToString();
int substatuscode = 9000;
string substatus = substatuscode.ToString();
double requestCharge = 42;
double retryAfter = 9000;
string retryAfterLiteral = retryAfter.ToString();
List <(HttpStatusCode statusCode, CosmosException exception)> exceptionsToStatusCodes = new List <(HttpStatusCode, CosmosException)>()
{
(HttpStatusCode.NotFound, CosmosExceptionFactory.CreateNotFoundException(testMessage, new Headers()
{
SubStatusCodeLiteral = substatus, ActivityId = activityId, RequestCharge = requestCharge, RetryAfterLiteral = retryAfterLiteral
})),
(HttpStatusCode.InternalServerError, CosmosExceptionFactory.CreateInternalServerErrorException(testMessage, new Headers()
{
SubStatusCodeLiteral = substatus, ActivityId = activityId, RequestCharge = requestCharge, RetryAfterLiteral = retryAfterLiteral
})),
(HttpStatusCode.BadRequest, CosmosExceptionFactory.CreateBadRequestException(testMessage, new Headers()
{
SubStatusCodeLiteral = substatus, ActivityId = activityId, RequestCharge = requestCharge, RetryAfterLiteral = retryAfterLiteral
})),
(HttpStatusCode.RequestTimeout, CosmosExceptionFactory.CreateRequestTimeoutException(testMessage, new Headers()
{
SubStatusCodeLiteral = substatus, ActivityId = activityId, RequestCharge = requestCharge, RetryAfterLiteral = retryAfterLiteral
})),
((HttpStatusCode)429, CosmosExceptionFactory.CreateThrottledException(testMessage, new Headers()
{
SubStatusCodeLiteral = substatus, ActivityId = activityId, RequestCharge = requestCharge, RetryAfterLiteral = retryAfterLiteral
})),
};
foreach ((HttpStatusCode statusCode, CosmosException exception) in exceptionsToStatusCodes)
{
this.ValidateExceptionInfo(
exception,
statusCode,
substatus,
testMessage,
activityId,
requestCharge,
retryAfter);
}
CosmosException cosmosException = CosmosExceptionFactory.CreateNotFoundException(testMessage, new Headers()
{
SubStatusCodeLiteral = ((int)SubStatusCodes.ReadSessionNotAvailable).ToString(), ActivityId = activityId, RequestCharge = requestCharge, RetryAfterLiteral = retryAfterLiteral
});
this.ValidateExceptionInfo(
cosmosException,
HttpStatusCode.NotFound,
((int)SubStatusCodes.ReadSessionNotAvailable).ToString(),
testMessage,
activityId,
requestCharge,
retryAfter);
}
public override async Task <ResponseMessage> SendAsync(
RequestMessage request,
CancellationToken cancellationToken)
{
try
{
ResponseMessage response = await this.ProcessMessageAsync(request, cancellationToken);
Debug.Assert(System.Diagnostics.Trace.CorrelationManager.ActivityId != Guid.Empty, "Trace activity id is missing");
return(response);
}
//catch DocumentClientException and exceptions that inherit it. Other exception types happen before a backend request
catch (DocumentClientException ex)
{
Debug.Assert(System.Diagnostics.Trace.CorrelationManager.ActivityId != Guid.Empty, "Trace activity id is missing");
return(ex.ToCosmosResponseMessage(request));
}
catch (CosmosException ce)
{
Debug.Assert(System.Diagnostics.Trace.CorrelationManager.ActivityId != Guid.Empty, "Trace activity id is missing");
return(ce.ToCosmosResponseMessage(request));
}
catch (OperationCanceledException ex)
{
// Catch Operation Cancelled Exception and convert to Timeout 408 if the user did not cancel it.
// Throw the exception if the user cancelled.
if (cancellationToken.IsCancellationRequested)
{
throw;
}
Debug.Assert(System.Diagnostics.Trace.CorrelationManager.ActivityId != Guid.Empty, "Trace activity id is missing");
CosmosException cosmosException = CosmosExceptionFactory.CreateRequestTimeoutException(
message: ex.Data?["Message"].ToString(),
headers: new Headers()
{
ActivityId = System.Diagnostics.Trace.CorrelationManager.ActivityId.ToString()
},
innerException: ex,
trace: request.Trace);
return(cosmosException.ToCosmosResponseMessage(request));
}
catch (AggregateException ex)
{
Debug.Assert(System.Diagnostics.Trace.CorrelationManager.ActivityId != Guid.Empty, "Trace activity id is missing");
// TODO: because the SDK underneath this path uses ContinueWith or task.Result we need to catch AggregateExceptions here
// in order to ensure that underlying DocumentClientExceptions get propagated up correctly. Once all ContinueWith and .Result
// is removed this catch can be safely removed.
ResponseMessage errorMessage = AggregateExceptionConverter(ex, request);
if (errorMessage != null)
{
return(errorMessage);
}
throw;
}
}
private ShouldRetryResult GetShouldRetryFromException(
Exception exception)
{
if (exception is OperationCanceledException operationCanceledException)
{
// throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
string message =
$"GatewayStoreClient Request Timeout. Start Time Utc:{this.startDateTimeUtc}; Total Duration:{(DateTime.UtcNow - this.startDateTimeUtc).TotalMilliseconds} Ms; Http Client Timeout:{this.gatewayRequestTimeout.TotalMilliseconds} Ms; Activity id: {Trace.CorrelationManager.ActivityId.ToString()};";
return(ShouldRetryResult.NoRetry(CosmosExceptionFactory.CreateRequestTimeoutException(
message,
innerException: operationCanceledException,
diagnosticsContext: this.diagnosticsContext)));
}
return(ShouldRetryResult.NoRetry());
}
public void EnsureCorrectStatusCode()
{
string testMessage = "Test" + Guid.NewGuid().ToString();
List <(HttpStatusCode statusCode, CosmosException exception)> exceptionsToStatusCodes = new List <(HttpStatusCode, CosmosException)>()
{
(HttpStatusCode.NotFound, CosmosExceptionFactory.CreateNotFoundException(testMessage, activityId: Guid.NewGuid().ToString())),
(HttpStatusCode.InternalServerError, CosmosExceptionFactory.CreateInternalServerErrorException(testMessage, activityId: Guid.NewGuid().ToString())),
(HttpStatusCode.BadRequest, CosmosExceptionFactory.CreateBadRequestException(testMessage, activityId: Guid.NewGuid().ToString())),
(HttpStatusCode.RequestTimeout, CosmosExceptionFactory.CreateRequestTimeoutException(testMessage, activityId: Guid.NewGuid().ToString())),
((HttpStatusCode)429, CosmosExceptionFactory.CreateThrottledException(testMessage, activityId: Guid.NewGuid().ToString())),
};
foreach ((HttpStatusCode statusCode, CosmosException exception)item in exceptionsToStatusCodes)
{
this.ValidateExceptionInfo(item.exception, item.statusCode, testMessage);
}
}
private async Task <HttpResponseMessage> SendHttpHelperAsync(
Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
ResourceType resourceType,
CosmosDiagnosticsContext diagnosticsContext,
HttpTimeoutPolicy timeoutPolicy,
CancellationToken cancellationToken)
{
bool isDefaultCancellationToken = cancellationToken == default;
DateTime startDateTimeUtc = DateTime.UtcNow;
IEnumerator <(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> timeoutEnumerator = timeoutPolicy.TimeoutEnumerator;
timeoutEnumerator.MoveNext();
while (true)
{
(TimeSpan requestTimeout, TimeSpan delayForNextRequest) = timeoutEnumerator.Current;
using (HttpRequestMessage requestMessage = await createRequestMessageAsync())
{
// If the default cancellation token is passed then use the timeout policy
CancellationTokenSource cancellationTokenSource = null;
if (isDefaultCancellationToken)
{
cancellationTokenSource = new CancellationTokenSource();
cancellationTokenSource.CancelAfter(requestTimeout);
cancellationToken = cancellationTokenSource.Token;
}
cancellationToken.ThrowIfCancellationRequested();
try
{
using (diagnosticsContext.CreateScope(nameof(CosmosHttpClientCore.SendHttpHelperAsync)))
{
return(await this.ExecuteHttpHelperAsync(
requestMessage,
resourceType,
cancellationToken));
}
}
catch (Exception e)
{
// Log the error message
diagnosticsContext.AddDiagnosticsInternal(
new PointOperationStatistics(
activityId: Trace.CorrelationManager.ActivityId.ToString(),
statusCode: HttpStatusCode.ServiceUnavailable,
subStatusCode: SubStatusCodes.Unknown,
responseTimeUtc: DateTime.UtcNow,
requestCharge: 0,
errorMessage: e.ToString(),
method: requestMessage.Method,
requestUri: requestMessage.RequestUri.OriginalString,
requestSessionToken: null,
responseSessionToken: null));
bool isOutOfRetries = (DateTime.UtcNow - startDateTimeUtc) > timeoutPolicy.MaximumRetryTimeLimit || // Maximum of time for all retries
!timeoutEnumerator.MoveNext(); // No more retries are configured
switch (e)
{
case OperationCanceledException operationCanceledException:
// Throw if the user passed in cancellation was requested
if (!isDefaultCancellationToken && cancellationToken.IsCancellationRequested)
{
throw;
}
// Convert OperationCanceledException to 408 when the HTTP client throws it. This makes it clear that the
// the request timed out and was not user canceled operation.
if (isOutOfRetries || requestMessage.Method != HttpMethod.Get)
{
// throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
string message =
$"GatewayStoreClient Request Timeout. Start Time UTC:{startDateTimeUtc}; Total Duration:{(DateTime.UtcNow - startDateTimeUtc).TotalMilliseconds} Ms; Request Timeout {requestTimeout.TotalMilliseconds} Ms; Http Client Timeout:{this.httpClient.Timeout.TotalMilliseconds} Ms; Activity id: {Trace.CorrelationManager.ActivityId};";
throw CosmosExceptionFactory.CreateRequestTimeoutException(
message,
innerException: operationCanceledException,
diagnosticsContext: diagnosticsContext);
}
break;
case WebException webException:
if (isOutOfRetries || (requestMessage.Method != HttpMethod.Get && !WebExceptionUtility.IsWebExceptionRetriable(webException)))
{
throw;
}
break;
case HttpRequestException httpRequestException:
if (isOutOfRetries || requestMessage.Method != HttpMethod.Get)
{
throw;
}
break;
default:
throw;
}
}
}
if (delayForNextRequest != TimeSpan.Zero)
{
using (diagnosticsContext.CreateScope($"HttpRetryDelay; Delay:{delayForNextRequest} seconds; Current request timeout {requestTimeout}; TimeoutPolicy: {timeoutPolicy.TimeoutPolicyName}"))
{
await Task.Delay(delayForNextRequest);
}
}
}
}
public override async Task RunAsync(CancellationToken cancellationToken)
{
string lastContinuation = this.options.StartContinuation;
while (!cancellationToken.IsCancellationRequested)
{
TimeSpan delay = this.options.FeedPollDelay;
try
{
do
{
Task <ResponseMessage> task = this.resultSetIterator.ReadNextAsync(cancellationToken);
using (CancellationTokenSource cts = new CancellationTokenSource())
{
if (!ReferenceEquals(await Task.WhenAny(task, Task.Delay(this.options.RequestTimeout, cts.Token)), task))
{
Task catchExceptionFromTask = task.ContinueWith(task => DefaultTrace.TraceInformation(
"Timed out Change Feed request failed with exception: {2}", task.Exception.InnerException),
TaskContinuationOptions.OnlyOnFaulted);
throw CosmosExceptionFactory.CreateRequestTimeoutException("Change Feed request timed out", new Headers());
}
else
{
cts.Cancel();
}
}
ResponseMessage response = await task;
if (response.StatusCode != HttpStatusCode.NotModified && !response.IsSuccessStatusCode)
{
DefaultTrace.TraceWarning("unsuccessful feed read: lease token '{0}' status code {1}. substatuscode {2}", this.options.LeaseToken, response.StatusCode, response.Headers.SubStatusCode);
this.HandleFailedRequest(response, lastContinuation);
if (response.Headers.RetryAfter.HasValue)
{
delay = response.Headers.RetryAfter.Value;
}
// Out of the loop for a retry
break;
}
lastContinuation = response.Headers.ContinuationToken;
if (this.resultSetIterator.HasMoreResults)
{
await this.DispatchChangesAsync(response, cancellationToken).ConfigureAwait(false);
}
}while (this.resultSetIterator.HasMoreResults && !cancellationToken.IsCancellationRequested);
}
catch (OperationCanceledException canceledException)
{
if (cancellationToken.IsCancellationRequested)
{
throw;
}
Extensions.TraceException(canceledException);
DefaultTrace.TraceWarning("exception: lease token '{0}'", this.options.LeaseToken);
// ignore as it is caused by Cosmos DB client when StopAsync is called
}
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
}
}
private async ValueTask RefreshCachedTokenWithRetryHelperAsync(ITrace trace)
{
// A different thread is already updating the access token. Count starts off at 1.
bool skipRefreshBecause = this.backgroundRefreshLock.CurrentCount != 1;
await this.backgroundRefreshLock.WaitAsync();
try
{
// Token was already refreshed successfully from another thread.
if (skipRefreshBecause && this.cachedAccessToken.ExpiresOn > DateTime.UtcNow)
{
return;
}
Exception lastException = null;
const int totalRetryCount = 3;
for (int retry = 0; retry < totalRetryCount; retry++)
{
if (this.cancellationToken.IsCancellationRequested)
{
DefaultTrace.TraceInformation(
"Stop RefreshTokenWithIndefiniteRetries because cancellation is requested");
break;
}
using (ITrace getTokenTrace = trace.StartChild(
name: nameof(this.RefreshCachedTokenWithRetryHelperAsync),
component: TraceComponent.Authorization,
level: Tracing.TraceLevel.Info))
{
try
{
await this.ExecuteGetTokenWithRequestTimeoutAsync();
return;
}
catch (RequestFailedException requestFailedException)
{
lastException = requestFailedException;
getTokenTrace.AddDatum(
$"RequestFailedException at {DateTime.UtcNow.ToString(CultureInfo.InvariantCulture)}",
requestFailedException);
DefaultTrace.TraceError($"TokenCredential.GetToken() failed with RequestFailedException. scope = {string.Join(";", this.tokenRequestContext.Scopes)}, retry = {retry}, Exception = {lastException}");
// Don't retry on auth failures
if (requestFailedException.Status == (int)HttpStatusCode.Unauthorized ||
requestFailedException.Status == (int)HttpStatusCode.Forbidden)
{
this.cachedAccessToken = default;
throw;
}
}
catch (OperationCanceledException operationCancelled)
{
lastException = operationCancelled;
getTokenTrace.AddDatum(
$"OperationCanceledException at {DateTime.UtcNow.ToString(CultureInfo.InvariantCulture)}",
operationCancelled);
DefaultTrace.TraceError(
$"TokenCredential.GetTokenAsync() failed. scope = {string.Join(";", this.tokenRequestContext.Scopes)}, retry = {retry}, Exception = {lastException}");
throw CosmosExceptionFactory.CreateRequestTimeoutException(
message: ClientResources.FailedToGetAadToken,
headers: new Headers()
{
SubStatusCode = SubStatusCodes.FailedToGetAadToken,
},
innerException: lastException,
trace: getTokenTrace);
}
catch (Exception exception)
{
lastException = exception;
getTokenTrace.AddDatum(
$"Exception at {DateTime.UtcNow.ToString(CultureInfo.InvariantCulture)}",
exception);
DefaultTrace.TraceError(
$"TokenCredential.GetTokenAsync() failed. scope = {string.Join(";", this.tokenRequestContext.Scopes)}, retry = {retry}, Exception = {lastException}");
}
}
DefaultTrace.TraceError(
$"TokenCredential.GetTokenAsync() failed. scope = {string.Join(";", this.tokenRequestContext.Scopes)}, retry = {retry}, Exception = {lastException}");
}
throw CosmosExceptionFactory.CreateUnauthorizedException(
message: ClientResources.FailedToGetAadToken,
headers: new Headers()
{
SubStatusCode = SubStatusCodes.FailedToGetAadToken,
},
innerException: lastException,
trace: trace);
}
finally
{
this.backgroundRefreshLock.Release();
}
}
private async Task <HttpResponseMessage> SendHttpHelperAsync(
Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
ResourceType resourceType,
HttpTimeoutPolicy timeoutPolicy,
ITrace trace,
CancellationToken cancellationToken)
{
DateTime startDateTimeUtc = DateTime.UtcNow;
IEnumerator <(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> timeoutEnumerator = timeoutPolicy.GetTimeoutEnumerator();
timeoutEnumerator.MoveNext();
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
(TimeSpan requestTimeout, TimeSpan delayForNextRequest) = timeoutEnumerator.Current;
using (HttpRequestMessage requestMessage = await createRequestMessageAsync())
{
// If the default cancellation token is passed then use the timeout policy
using CancellationTokenSource cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
cancellationTokenSource.CancelAfter(requestTimeout);
using (ITrace helperTrace = trace.StartChild($"Execute Http With Timeout Policy: {timeoutPolicy.TimeoutPolicyName}", TraceComponent.Transport, Tracing.TraceLevel.Info))
{
try
{
return(await this.ExecuteHttpHelperAsync(
requestMessage,
resourceType,
cancellationTokenSource.Token));
}
catch (Exception e)
{
// Log the error message
trace.AddDatum(
"Error",
new PointOperationStatisticsTraceDatum(
activityId: System.Diagnostics.Trace.CorrelationManager.ActivityId.ToString(),
statusCode: HttpStatusCode.ServiceUnavailable,
subStatusCode: SubStatusCodes.Unknown,
responseTimeUtc: DateTime.UtcNow,
requestCharge: 0,
errorMessage: e.ToString(),
method: requestMessage.Method,
requestUri: requestMessage.RequestUri.OriginalString,
requestSessionToken: null,
responseSessionToken: null));
bool isOutOfRetries = (DateTime.UtcNow - startDateTimeUtc) > timeoutPolicy.MaximumRetryTimeLimit || // Maximum of time for all retries
!timeoutEnumerator.MoveNext(); // No more retries are configured
switch (e)
{
case OperationCanceledException operationCanceledException:
// Throw if the user passed in cancellation was requested
if (cancellationToken.IsCancellationRequested)
{
throw;
}
// Convert OperationCanceledException to 408 when the HTTP client throws it. This makes it clear that the
// the request timed out and was not user canceled operation.
if (isOutOfRetries || !timeoutPolicy.IsSafeToRetry(requestMessage.Method))
{
// throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
string message =
$"GatewayStoreClient Request Timeout. Start Time UTC:{startDateTimeUtc}; Total Duration:{(DateTime.UtcNow - startDateTimeUtc).TotalMilliseconds} Ms; Request Timeout {requestTimeout.TotalMilliseconds} Ms; Http Client Timeout:{this.httpClient.Timeout.TotalMilliseconds} Ms; Activity id: {System.Diagnostics.Trace.CorrelationManager.ActivityId};";
throw CosmosExceptionFactory.CreateRequestTimeoutException(
message,
headers: new Headers()
{
ActivityId = System.Diagnostics.Trace.CorrelationManager.ActivityId.ToString()
},
innerException: operationCanceledException,
trace: helperTrace);
}
break;
case WebException webException:
if (isOutOfRetries || (!timeoutPolicy.IsSafeToRetry(requestMessage.Method) && !WebExceptionUtility.IsWebExceptionRetriable(webException)))
{
throw;
}
break;
case HttpRequestException httpRequestException:
if (isOutOfRetries || !timeoutPolicy.IsSafeToRetry(requestMessage.Method))
{
throw;
}
break;
default:
throw;
}
}
}
}
if (delayForNextRequest != TimeSpan.Zero)
{
using (ITrace delayTrace = trace.StartChild("Retry Delay", TraceComponent.Transport, Tracing.TraceLevel.Info))
{
await Task.Delay(delayForNextRequest);
}
}
}
}
