public abstract Task <HttpResponseMessage> GetAsync(
Uri uri,
INameValueCollection additionalHeaders,
ResourceType resourceType,
HttpTimeoutPolicy timeoutPolicy,
ITrace trace,
CancellationToken cancellationToken);
public abstract Task <HttpResponseMessage> GetAsync(
Uri uri,
INameValueCollection additionalHeaders,
ResourceType resourceType,
HttpTimeoutPolicy timeoutPolicy,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken);
public abstract Task <HttpResponseMessage> GetAsync(
Uri uri,
INameValueCollection additionalHeaders,
ResourceType resourceType,
HttpTimeoutPolicy timeoutPolicy,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken);
internal Task <HttpResponseMessage> SendHttpAsync(
Func <ValueTask <HttpRequestMessage> > requestMessage,
ResourceType resourceType,
HttpTimeoutPolicy timeoutPolicy,
CancellationToken cancellationToken = default)
{
return(this.httpClient.SendHttpAsync(
createRequestMessageAsync: requestMessage,
resourceType: resourceType,
timeoutPolicy: timeoutPolicy,
diagnosticsContext: null,
cancellationToken: cancellationToken));
}
internal Task <HttpResponseMessage> SendHttpAsync(
Func <ValueTask <HttpRequestMessage> > requestMessage,
ResourceType resourceType,
HttpTimeoutPolicy timeoutPolicy,
CancellationToken cancellationToken = default)
{
return(this.httpClient.SendHttpAsync(
createRequestMessageAsync: requestMessage,
resourceType: resourceType,
timeoutPolicy: timeoutPolicy,
trace: NoOpTrace.Singleton,
cancellationToken: cancellationToken));
}
private Task <HttpResponseMessage> InvokeClientAsync(
DocumentServiceRequest request,
ResourceType resourceType,
Uri physicalAddress,
CancellationToken cancellationToken)
{
return(this.httpClient.SendHttpAsync(
() => this.PrepareRequestMessageAsync(request, physicalAddress),
resourceType,
HttpTimeoutPolicy.GetTimeoutPolicy(request),
request.RequestContext.ClientRequestStatistics,
cancellationToken));
}
internal Task <HttpResponseMessage> SendHttpAsync(
Func <ValueTask <HttpRequestMessage> > requestMessage,
ResourceType resourceType,
HttpTimeoutPolicy timeoutPolicy,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken = default)
{
return(this.httpClient.SendHttpAsync(
createRequestMessageAsync: requestMessage,
resourceType: resourceType,
timeoutPolicy: timeoutPolicy,
clientSideRequestStatistics: clientSideRequestStatistics,
cancellationToken: cancellationToken));
}
public override Task <HttpResponseMessage> SendHttpAsync(
Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
ResourceType resourceType,
HttpTimeoutPolicy timeoutPolicy,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken)
{
if (createRequestMessageAsync == null)
{
throw new ArgumentNullException(nameof(createRequestMessageAsync));
}
return(this.SendHttpHelperAsync(
createRequestMessageAsync,
resourceType,
diagnosticsContext ?? new CosmosDiagnosticsContextCore(),
timeoutPolicy,
cancellationToken));
}
public override Task <HttpResponseMessage> SendHttpAsync(
Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
ResourceType resourceType,
HttpTimeoutPolicy timeoutPolicy,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken)
{
if (createRequestMessageAsync == null)
{
throw new ArgumentNullException(nameof(createRequestMessageAsync));
}
return(this.SendHttpHelperAsync(
createRequestMessageAsync,
resourceType,
timeoutPolicy,
clientSideRequestStatistics,
cancellationToken));
}
private Task <HttpResponseMessage> InvokeClientAsync(
DocumentServiceRequest request,
ResourceType resourceType,
Uri physicalAddress,
CancellationToken cancellationToken)
{
CosmosDiagnosticsContext diagnosticsContext = null;
if (request?.RequestContext?.ClientRequestStatistics is CosmosClientSideRequestStatistics cosmosClientSideRequestStatistics)
{
diagnosticsContext = cosmosClientSideRequestStatistics.DiagnosticsContext;
}
return(this.httpClient.SendHttpAsync(
() => this.PrepareRequestMessageAsync(request, physicalAddress),
resourceType,
HttpTimeoutPolicy.GetTimeoutPolicy(request),
diagnosticsContext,
cancellationToken));
}
public override Task <HttpResponseMessage> GetAsync(
Uri uri,
INameValueCollection additionalHeaders,
ResourceType resourceType,
HttpTimeoutPolicy timeoutPolicy,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken)
{
if (uri == null)
{
throw new ArgumentNullException(nameof(uri));
}
// GetAsync doesn't let clients to pass in additional headers. So, we are
// internally using SendAsync and add the additional headers to requestMessage.
ValueTask <HttpRequestMessage> CreateRequestMessage()
{
HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Get, uri);
if (additionalHeaders != null)
{
foreach (string header in additionalHeaders)
{
if (GatewayStoreClient.IsAllowedRequestHeader(header))
{
requestMessage.Headers.TryAddWithoutValidation(header, additionalHeaders[header]);
}
}
}
return(new ValueTask <HttpRequestMessage>(requestMessage));
}
return(this.SendHttpAsync(
CreateRequestMessage,
resourceType,
timeoutPolicy,
diagnosticsContext,
cancellationToken));
}
private async Task <HttpResponseMessage> SendHttpHelperAsync(
Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
ResourceType resourceType,
CosmosDiagnosticsContext diagnosticsContext,
HttpTimeoutPolicy timeoutPolicy,
CancellationToken cancellationToken)
{
bool isDefaultCancellationToken = cancellationToken == default;
DateTime startDateTimeUtc = DateTime.UtcNow;
IEnumerator <(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> timeoutEnumerator = timeoutPolicy.TimeoutEnumerator;
timeoutEnumerator.MoveNext();
while (true)
{
(TimeSpan requestTimeout, TimeSpan delayForNextRequest) = timeoutEnumerator.Current;
using (HttpRequestMessage requestMessage = await createRequestMessageAsync())
{
// If the default cancellation token is passed then use the timeout policy
CancellationTokenSource cancellationTokenSource = null;
if (isDefaultCancellationToken)
{
cancellationTokenSource = new CancellationTokenSource();
cancellationTokenSource.CancelAfter(requestTimeout);
cancellationToken = cancellationTokenSource.Token;
}
cancellationToken.ThrowIfCancellationRequested();
try
{
using (diagnosticsContext.CreateScope(nameof(CosmosHttpClientCore.SendHttpHelperAsync)))
{
return(await this.ExecuteHttpHelperAsync(
requestMessage,
resourceType,
cancellationToken));
}
}
catch (Exception e)
{
// Log the error message
diagnosticsContext.AddDiagnosticsInternal(
new PointOperationStatistics(
activityId: Trace.CorrelationManager.ActivityId.ToString(),
statusCode: HttpStatusCode.ServiceUnavailable,
subStatusCode: SubStatusCodes.Unknown,
responseTimeUtc: DateTime.UtcNow,
requestCharge: 0,
errorMessage: e.ToString(),
method: requestMessage.Method,
requestUri: requestMessage.RequestUri.OriginalString,
requestSessionToken: null,
responseSessionToken: null));
bool isOutOfRetries = (DateTime.UtcNow - startDateTimeUtc) > timeoutPolicy.MaximumRetryTimeLimit || // Maximum of time for all retries
!timeoutEnumerator.MoveNext(); // No more retries are configured
switch (e)
{
case OperationCanceledException operationCanceledException:
// Throw if the user passed in cancellation was requested
if (!isDefaultCancellationToken && cancellationToken.IsCancellationRequested)
{
throw;
}
// Convert OperationCanceledException to 408 when the HTTP client throws it. This makes it clear that the
// the request timed out and was not user canceled operation.
if (isOutOfRetries || requestMessage.Method != HttpMethod.Get)
{
// throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
string message =
$"GatewayStoreClient Request Timeout. Start Time UTC:{startDateTimeUtc}; Total Duration:{(DateTime.UtcNow - startDateTimeUtc).TotalMilliseconds} Ms; Request Timeout {requestTimeout.TotalMilliseconds} Ms; Http Client Timeout:{this.httpClient.Timeout.TotalMilliseconds} Ms; Activity id: {Trace.CorrelationManager.ActivityId};";
throw CosmosExceptionFactory.CreateRequestTimeoutException(
message,
innerException: operationCanceledException,
diagnosticsContext: diagnosticsContext);
}
break;
case WebException webException:
if (isOutOfRetries || (requestMessage.Method != HttpMethod.Get && !WebExceptionUtility.IsWebExceptionRetriable(webException)))
{
throw;
}
break;
case HttpRequestException httpRequestException:
if (isOutOfRetries || requestMessage.Method != HttpMethod.Get)
{
throw;
}
break;
default:
throw;
}
}
}
if (delayForNextRequest != TimeSpan.Zero)
{
using (diagnosticsContext.CreateScope($"HttpRetryDelay; Delay:{delayForNextRequest} seconds; Current request timeout {requestTimeout}; TimeoutPolicy: {timeoutPolicy.TimeoutPolicyName}"))
{
await Task.Delay(delayForNextRequest);
}
}
}
}
public abstract Task <HttpResponseMessage> SendHttpAsync(
Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
ResourceType resourceType,
HttpTimeoutPolicy timeoutPolicy,
ITrace trace,
CancellationToken cancellationToken);
private async Task <HttpResponseMessage> SendHttpHelperAsync(
Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
ResourceType resourceType,
HttpTimeoutPolicy timeoutPolicy,
ITrace trace,
CancellationToken cancellationToken)
{
DateTime startDateTimeUtc = DateTime.UtcNow;
IEnumerator <(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> timeoutEnumerator = timeoutPolicy.GetTimeoutEnumerator();
timeoutEnumerator.MoveNext();
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
(TimeSpan requestTimeout, TimeSpan delayForNextRequest) = timeoutEnumerator.Current;
using (HttpRequestMessage requestMessage = await createRequestMessageAsync())
{
// If the default cancellation token is passed then use the timeout policy
using CancellationTokenSource cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
cancellationTokenSource.CancelAfter(requestTimeout);
using (ITrace helperTrace = trace.StartChild($"Execute Http With Timeout Policy: {timeoutPolicy.TimeoutPolicyName}", TraceComponent.Transport, Tracing.TraceLevel.Info))
{
try
{
return(await this.ExecuteHttpHelperAsync(
requestMessage,
resourceType,
cancellationTokenSource.Token));
}
catch (Exception e)
{
// Log the error message
trace.AddDatum(
"Error",
new PointOperationStatisticsTraceDatum(
activityId: System.Diagnostics.Trace.CorrelationManager.ActivityId.ToString(),
statusCode: HttpStatusCode.ServiceUnavailable,
subStatusCode: SubStatusCodes.Unknown,
responseTimeUtc: DateTime.UtcNow,
requestCharge: 0,
errorMessage: e.ToString(),
method: requestMessage.Method,
requestUri: requestMessage.RequestUri.OriginalString,
requestSessionToken: null,
responseSessionToken: null));
bool isOutOfRetries = (DateTime.UtcNow - startDateTimeUtc) > timeoutPolicy.MaximumRetryTimeLimit || // Maximum of time for all retries
!timeoutEnumerator.MoveNext(); // No more retries are configured
switch (e)
{
case OperationCanceledException operationCanceledException:
// Throw if the user passed in cancellation was requested
if (cancellationToken.IsCancellationRequested)
{
throw;
}
// Convert OperationCanceledException to 408 when the HTTP client throws it. This makes it clear that the
// the request timed out and was not user canceled operation.
if (isOutOfRetries || !timeoutPolicy.IsSafeToRetry(requestMessage.Method))
{
// throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
string message =
$"GatewayStoreClient Request Timeout. Start Time UTC:{startDateTimeUtc}; Total Duration:{(DateTime.UtcNow - startDateTimeUtc).TotalMilliseconds} Ms; Request Timeout {requestTimeout.TotalMilliseconds} Ms; Http Client Timeout:{this.httpClient.Timeout.TotalMilliseconds} Ms; Activity id: {System.Diagnostics.Trace.CorrelationManager.ActivityId};";
throw CosmosExceptionFactory.CreateRequestTimeoutException(
message,
headers: new Headers()
{
ActivityId = System.Diagnostics.Trace.CorrelationManager.ActivityId.ToString()
},
innerException: operationCanceledException,
trace: helperTrace);
}
break;
case WebException webException:
if (isOutOfRetries || (!timeoutPolicy.IsSafeToRetry(requestMessage.Method) && !WebExceptionUtility.IsWebExceptionRetriable(webException)))
{
throw;
}
break;
case HttpRequestException httpRequestException:
if (isOutOfRetries || !timeoutPolicy.IsSafeToRetry(requestMessage.Method))
{
throw;
}
break;
default:
throw;
}
}
}
}
if (delayForNextRequest != TimeSpan.Zero)
{
using (ITrace delayTrace = trace.StartChild("Retry Delay", TraceComponent.Transport, Tracing.TraceLevel.Info))
{
await Task.Delay(delayForNextRequest);
}
}
}
}
private async Task <HttpResponseMessage> SendHttpHelperAsync(
Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
ResourceType resourceType,
HttpTimeoutPolicy timeoutPolicy,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken)
{
DateTime startDateTimeUtc = DateTime.UtcNow;
IEnumerator <(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> timeoutEnumerator = timeoutPolicy.GetTimeoutEnumerator();
timeoutEnumerator.MoveNext();
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
(TimeSpan requestTimeout, TimeSpan delayForNextRequest) = timeoutEnumerator.Current;
using (HttpRequestMessage requestMessage = await createRequestMessageAsync())
{
// If the default cancellation token is passed then use the timeout policy
using CancellationTokenSource cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
cancellationTokenSource.CancelAfter(requestTimeout);
try
{
HttpResponseMessage responseMessage = await this.ExecuteHttpHelperAsync(
requestMessage,
resourceType,
cancellationTokenSource.Token);
if (clientSideRequestStatistics is ClientSideRequestStatisticsTraceDatum datum)
{
datum.RecordHttpResponse(requestMessage, responseMessage, resourceType, DateTime.UtcNow);
}
return(responseMessage);
}
catch (Exception e)
{
if (clientSideRequestStatistics is ClientSideRequestStatisticsTraceDatum datum)
{
datum.RecordHttpException(requestMessage, e, resourceType, DateTime.UtcNow);
}
bool isOutOfRetries = (DateTime.UtcNow - startDateTimeUtc) > timeoutPolicy.MaximumRetryTimeLimit || // Maximum of time for all retries
!timeoutEnumerator.MoveNext(); // No more retries are configured
switch (e)
{
case OperationCanceledException operationCanceledException:
// Throw if the user passed in cancellation was requested
if (cancellationToken.IsCancellationRequested)
{
throw;
}
// Convert OperationCanceledException to 408 when the HTTP client throws it. This makes it clear that the
// the request timed out and was not user canceled operation.
if (isOutOfRetries || !timeoutPolicy.IsSafeToRetry(requestMessage.Method))
{
// throw current exception (caught in transport handler)
string message =
$"GatewayStoreClient Request Timeout. Start Time UTC:{startDateTimeUtc}; Total Duration:{(DateTime.UtcNow - startDateTimeUtc).TotalMilliseconds} Ms; Request Timeout {requestTimeout.TotalMilliseconds} Ms; Http Client Timeout:{this.httpClient.Timeout.TotalMilliseconds} Ms; Activity id: {System.Diagnostics.Trace.CorrelationManager.ActivityId};";
e.Data.Add("Message", message);
throw;
}
break;
case WebException webException:
if (isOutOfRetries || (!timeoutPolicy.IsSafeToRetry(requestMessage.Method) && !WebExceptionUtility.IsWebExceptionRetriable(webException)))
{
throw;
}
break;
case HttpRequestException httpRequestException:
if (isOutOfRetries || !timeoutPolicy.IsSafeToRetry(requestMessage.Method))
{
throw;
}
break;
default:
throw;
}
}
}
if (delayForNextRequest != TimeSpan.Zero)
{
await Task.Delay(delayForNextRequest);
}
}
}
public abstract Task <HttpResponseMessage> SendHttpAsync(
Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
ResourceType resourceType,
HttpTimeoutPolicy timeoutPolicy,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken);
private static bool IsOutOfRetries(
HttpTimeoutPolicy timeoutPolicy,
DateTime startDateTimeUtc,
IEnumerator <(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> timeoutEnumerator)
