/// <summary>
/// Get the next set of results from the cosmos service
/// </summary>
/// <param name="cancellationToken">(Optional) <see cref="CancellationToken"/> representing request cancellation.</param>
/// <returns>A query response from cosmos service</returns>
public override async Task <ResponseMessage> ReadNextAsync(CancellationToken cancellationToken = default)
{
CosmosDiagnosticsContext diagnostics = CosmosDiagnosticsContext.Create(this.changeFeedOptions);
using (diagnostics.GetOverallScope())
{
diagnostics.AddDiagnosticsInternal(new FeedRangeStatistics(this.changeFeedOptions.FeedRange));
if (!this.lazyContainerRid.ValueInitialized)
{
using (diagnostics.CreateScope("InitializeContainerResourceId"))
{
TryCatch <string> tryInitializeContainerRId = await this.lazyContainerRid.GetValueAsync(cancellationToken);
if (!tryInitializeContainerRId.Succeeded)
{
if (!(tryInitializeContainerRId.Exception.InnerException is CosmosException cosmosException))
{
throw new InvalidOperationException("Failed to convert to CosmosException.");
}
return(cosmosException.ToCosmosResponseMessage(
new RequestMessage(
method: null,
requestUri: null,
diagnosticsContext: diagnostics)));
}
}
if (this.FeedRangeContinuation == null)
{
using (diagnostics.CreateScope("InitializeContinuation"))
{
await this.InitializeFeedContinuationAsync(cancellationToken);
}
}
TryCatch validateContainer = this.FeedRangeContinuation.ValidateContainer(this.lazyContainerRid.Result.Result);
if (!validateContainer.Succeeded)
{
return(CosmosExceptionFactory
.CreateBadRequestException(
message: validateContainer.Exception.InnerException.Message,
innerException: validateContainer.Exception.InnerException,
diagnosticsContext: diagnostics)
.ToCosmosResponseMessage(
new RequestMessage(
method: null,
requestUri: null,
diagnosticsContext: diagnostics)));
}
}
return(await this.ReadNextInternalAsync(diagnostics, cancellationToken));
}
}
internal static ResponseMessage ToCosmosResponseMessage(this DocumentClientException documentClientException, RequestMessage requestMessage)
{
CosmosDiagnosticsContext diagnosticsContext = requestMessage?.DiagnosticsContext;
if (diagnosticsContext == null)
{
diagnosticsContext = CosmosDiagnosticsContextCore.Create();
}
CosmosException cosmosException = CosmosExceptionFactory.Create(
documentClientException,
diagnosticsContext);
PointOperationStatistics pointOperationStatistics = new PointOperationStatistics(
activityId: cosmosException.Headers.ActivityId,
statusCode: cosmosException.StatusCode,
subStatusCode: (int)SubStatusCodes.Unknown,
requestCharge: cosmosException.Headers.RequestCharge,
errorMessage: cosmosException.Message,
method: requestMessage?.Method,
requestUri: requestMessage?.RequestUri,
requestSessionToken: requestMessage?.Headers?.Session,
responseSessionToken: cosmosException.Headers.Session,
clientSideRequestStatistics: documentClientException.RequestStatistics as CosmosClientSideRequestStatistics);
diagnosticsContext.AddDiagnosticsInternal(pointOperationStatistics);
// if StatusCode is null it is a client business logic error and it never hit the backend, so throw
if (documentClientException.StatusCode == null)
{
throw cosmosException;
}
// if there is a status code then it came from the backend, return error as http error instead of throwing the exception
ResponseMessage responseMessage = cosmosException.ToCosmosResponseMessage(requestMessage);
if (requestMessage != null)
{
requestMessage.Properties.Remove(nameof(DocumentClientException));
requestMessage.Properties.Add(nameof(DocumentClientException), documentClientException);
}
return(responseMessage);
}
private async Task <HttpResponseMessage> SendHttpHelperAsync(
Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
ResourceType resourceType,
CosmosDiagnosticsContext diagnosticsContext,
HttpTimeoutPolicy timeoutPolicy,
CancellationToken cancellationToken)
{
bool isDefaultCancellationToken = cancellationToken == default;
DateTime startDateTimeUtc = DateTime.UtcNow;
IEnumerator <(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> timeoutEnumerator = timeoutPolicy.TimeoutEnumerator;
timeoutEnumerator.MoveNext();
while (true)
{
(TimeSpan requestTimeout, TimeSpan delayForNextRequest) = timeoutEnumerator.Current;
using (HttpRequestMessage requestMessage = await createRequestMessageAsync())
{
// If the default cancellation token is passed then use the timeout policy
CancellationTokenSource cancellationTokenSource = null;
if (isDefaultCancellationToken)
{
cancellationTokenSource = new CancellationTokenSource();
cancellationTokenSource.CancelAfter(requestTimeout);
cancellationToken = cancellationTokenSource.Token;
}
cancellationToken.ThrowIfCancellationRequested();
try
{
using (diagnosticsContext.CreateScope(nameof(CosmosHttpClientCore.SendHttpHelperAsync)))
{
return(await this.ExecuteHttpHelperAsync(
requestMessage,
resourceType,
cancellationToken));
}
}
catch (Exception e)
{
// Log the error message
diagnosticsContext.AddDiagnosticsInternal(
new PointOperationStatistics(
activityId: Trace.CorrelationManager.ActivityId.ToString(),
statusCode: HttpStatusCode.ServiceUnavailable,
subStatusCode: SubStatusCodes.Unknown,
responseTimeUtc: DateTime.UtcNow,
requestCharge: 0,
errorMessage: e.ToString(),
method: requestMessage.Method,
requestUri: requestMessage.RequestUri.OriginalString,
requestSessionToken: null,
responseSessionToken: null));
bool isOutOfRetries = (DateTime.UtcNow - startDateTimeUtc) > timeoutPolicy.MaximumRetryTimeLimit || // Maximum of time for all retries
!timeoutEnumerator.MoveNext(); // No more retries are configured
switch (e)
{
case OperationCanceledException operationCanceledException:
// Throw if the user passed in cancellation was requested
if (!isDefaultCancellationToken && cancellationToken.IsCancellationRequested)
{
throw;
}
// Convert OperationCanceledException to 408 when the HTTP client throws it. This makes it clear that the
// the request timed out and was not user canceled operation.
if (isOutOfRetries || requestMessage.Method != HttpMethod.Get)
{
// throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
string message =
$"GatewayStoreClient Request Timeout. Start Time UTC:{startDateTimeUtc}; Total Duration:{(DateTime.UtcNow - startDateTimeUtc).TotalMilliseconds} Ms; Request Timeout {requestTimeout.TotalMilliseconds} Ms; Http Client Timeout:{this.httpClient.Timeout.TotalMilliseconds} Ms; Activity id: {Trace.CorrelationManager.ActivityId};";
throw CosmosExceptionFactory.CreateRequestTimeoutException(
message,
innerException: operationCanceledException,
diagnosticsContext: diagnosticsContext);
}
break;
case WebException webException:
if (isOutOfRetries || (requestMessage.Method != HttpMethod.Get && !WebExceptionUtility.IsWebExceptionRetriable(webException)))
{
throw;
}
break;
case HttpRequestException httpRequestException:
if (isOutOfRetries || requestMessage.Method != HttpMethod.Get)
{
throw;
}
break;
default:
throw;
}
}
}
if (delayForNextRequest != TimeSpan.Zero)
{
using (diagnosticsContext.CreateScope($"HttpRetryDelay; Delay:{delayForNextRequest} seconds; Current request timeout {requestTimeout}; TimeoutPolicy: {timeoutPolicy.TimeoutPolicyName}"))
{
await Task.Delay(delayForNextRequest);
}
}
}
}