static public Task InlineIfPossibleAsync(Func <Task> function, IRetryPolicy retryPolicy, CancellationToken cancellationToken = default)
{
if (SynchronizationContext.Current == null)
{
if (retryPolicy == null)
{
// shortcut
return(function());
}
else
{
return(BackoffRetryUtility <int> .ExecuteAsync(async() =>
{
await function();
return 0;
}, retryPolicy, cancellationToken));
}
}
else
{
if (retryPolicy == null)
{
// shortcut
return(Task.Run(function));
}
else
{
return(Task.Run(() => BackoffRetryUtility <int> .ExecuteAsync(async() =>
{
await function();
return 0;
}, retryPolicy, cancellationToken)));
}
}
}
static public Task <TResult> InlineIfPossible <TResult>(Func <Task <TResult> > function, IRetryPolicy retryPolicy, CancellationToken cancellationToken = default(CancellationToken))
{
if (SynchronizationContext.Current == null)
{
if (retryPolicy == null)
{
// shortcut
return(function());
}
else
{
return(BackoffRetryUtility <TResult> .ExecuteAsync(() =>
{
return function();
}, retryPolicy, cancellationToken));
}
}
else
{
if (retryPolicy == null)
{
// shortcut
return(Task.Run(function));
}
else
{
return(Task.Run(() => BackoffRetryUtility <TResult> .ExecuteAsync(() =>
{
return function();
}, retryPolicy, cancellationToken)));
}
}
}
#pragma warning disable VSTHRD200 // Use "Async" suffix for async methods
static public Task <TResult> InlineIfPossible <TResult>(Func <Task <TResult> > function, IRetryPolicy retryPolicy, CancellationToken cancellationToken = default)
#pragma warning restore VSTHRD200 // Use "Async" suffix for async methods
{
if (SynchronizationContext.Current == null)
{
if (retryPolicy == null)
{
// shortcut
return(function());
}
else
{
return(BackoffRetryUtility <TResult> .ExecuteAsync(() =>
{
return function();
}, retryPolicy, cancellationToken));
}
}
else
{
if (retryPolicy == null)
{
// shortcut
return(Task.Run(function));
}
else
{
return(Task.Run(() => BackoffRetryUtility <TResult> .ExecuteAsync(() =>
{
return function();
}, retryPolicy, cancellationToken)));
}
}
}
public async Task ValidateRetryOnWriteForbiddenExceptionAsync()
{
this.Initialize(
useMultipleWriteLocations: false,
enableEndpointDiscovery: true,
isPreferredLocationsListEmpty: false);
await this.endpointManager.RefreshLocationAsync(this.databaseAccount);
ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, true, new RetryOptions());
using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: false, isMasterResourceType: false))
{
int retryCount = 0;
await BackoffRetryUtility <bool> .ExecuteAsync(
() =>
{
retryCount++;
retryPolicy.OnBeforeSendRequest(request);
if (retryCount == 1)
{
this.mockedClient.ResetCalls();
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.WriteForbidden).ToString();
DocumentClientException forbiddenException = new ForbiddenException(RMResources.Forbidden, headers);
throw forbiddenException;
}
else if (retryCount == 2)
{
this.mockedClient.Verify(client => client.GetDatabaseAccountInternalAsync(It.IsAny <Uri>(), It.IsAny <CancellationToken>()), Times.Once);
// Next request must go to next preferred endpoint
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
return(Task.FromResult(true));
}
else
{
Assert.Fail();
}
return(Task.FromResult(true));
},
retryPolicy);
}
}
private async Task ValidateRetryOnSessionNotAvailabeWithEndpointDiscoveryDisabled(bool isPreferredLocationsListEmpty, bool useMultipleWriteLocations, bool isReadRequest)
{
const bool enableEndpointDiscovery = false;
this.Initialize(
useMultipleWriteLocations: useMultipleWriteLocations,
enableEndpointDiscovery: enableEndpointDiscovery,
isPreferredLocationsListEmpty: isPreferredLocationsListEmpty);
ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, enableEndpointDiscovery, new RetryOptions());
using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false))
{
int retryCount = 0;
try
{
await BackoffRetryUtility <bool> .ExecuteAsync(
() =>
{
retryPolicy.OnBeforeSendRequest(request);
if (retryCount == 0)
{
Assert.IsFalse(request.ClearSessionTokenOnSessionReadFailure);
Assert.AreEqual(request.RequestContext.LocationEndpointToRoute, this.endpointManager.ReadEndpoints[0]);
}
else
{
Assert.Fail();
}
retryCount++;
StringKeyValueCollection headers = new StringKeyValueCollection();
headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.ReadSessionNotAvailable).ToString();
DocumentClientException notFoundException = new NotFoundException(RMResources.NotFound, headers);
throw notFoundException;
},
retryPolicy);
Assert.Fail();
}
catch (NotFoundException)
{
DefaultTrace.TraceInformation("Received expected notFoundException");
Assert.AreEqual(1, retryCount);
}
}
}
async private Task <DocumentServiceResponse> InvokeAsync(DocumentServiceRequest request, ResourceType resourceType, CancellationToken cancellationToken)
{
Func <Task <DocumentServiceResponse> > funcDelegate = async() =>
{
using (HttpRequestMessage requestMessage = await this.PrepareRequestMessageAsync(request))
{
DateTime sendTimeUtc = DateTime.UtcNow;
Guid localGuid = Guid.NewGuid(); // For correlating HttpRequest and HttpResponse Traces
this.eventSource.Request(
Guid.Empty,
localGuid,
requestMessage.RequestUri.ToString(),
resourceType.ToResourceTypeString(),
requestMessage.Headers);
using (HttpResponseMessage responseMessage = await this.httpClient.SendAsync(requestMessage, cancellationToken))
{
DateTime receivedTimeUtc = DateTime.UtcNow;
double durationInMilliSeconds = (receivedTimeUtc - sendTimeUtc).TotalMilliseconds;
IEnumerable <string> headerValues;
Guid activityId = Guid.Empty;
if (responseMessage.Headers.TryGetValues(HttpConstants.HttpHeaders.ActivityId, out headerValues) &&
headerValues.Count() != 0)
{
activityId = new Guid(headerValues.First());
}
this.eventSource.Response(
activityId,
localGuid,
(short)responseMessage.StatusCode,
durationInMilliSeconds,
responseMessage.Headers);
return(await ClientExtensions.ParseResponseAsync(responseMessage, request.SerializerSettings ?? this.SerializerSettings, request));
}
}
};
return(await BackoffRetryUtility <DocumentServiceResponse> .ExecuteAsync(funcDelegate, new WebExceptionRetryPolicy(), cancellationToken));
}
public static Task InlineIfPossible(Func <Task> function, IRetryPolicy retryPolicy, CancellationToken cancellationToken = default(CancellationToken))
{
if (SynchronizationContext.Current == null)
{
if (retryPolicy == null)
{
return(function());
}
return(BackoffRetryUtility <int> .ExecuteAsync(async delegate
{
await function();
return 0;
}, retryPolicy, cancellationToken));
}
if (retryPolicy == null)
{
return(Task.Run(function));
}
return(Task.Run(() => BackoffRetryUtility <int> .ExecuteAsync(async delegate
{
await function();
return 0;
}, retryPolicy, cancellationToken)));
}
internal static void ValidateCollection <T>(DocumentClient[] replicaClients,
string collectionId,
INameValueCollection headers = null,
bool verifyAddress = true) where T : Resource, new()
{
Assert.IsTrue(replicaClients != null && replicaClients.Length > 1, "Must pass in at least two replica clients");
Task.Delay(3000); // allow previous operations to complete and propagate, bringing more robustness to tests
foreach (DocumentClient client in replicaClients)
{
client.ForceAddressRefresh(true);
if (typeof(T) == typeof(StoredProcedure) ||
typeof(T) == typeof(UserDefinedFunction) ||
typeof(T) == typeof(Trigger))
{
TestCommon.ListAllScriptDirect <T>(client, collectionId, headers ?? new StringKeyValueCollection());
}
else
{
TestCommon.ListAll <T>(client, collectionId, headers ?? new StringKeyValueCollection(), true);
}
}
try
{
BackoffRetryUtility <bool> .ExecuteAsync((bool isInRetry) =>
{
var feeds = new DocumentFeedResponse <T> [replicaClients.Length];
var allHeaders = new StringKeyValueCollection[replicaClients.Length];
for (int i = 0; i < replicaClients.Length; i++)
{
var header = new StringKeyValueCollection();
if (headers != null)
{
foreach (string key in headers)
{
header[key] = headers[key];
}
}
allHeaders[i] = header;
}
var continuations = new string[replicaClients.Length];
var responseHeaders = new INameValueCollection[replicaClients.Length];
do
{
for (int i = 0; i < replicaClients.Length; i++)
{
if (!string.IsNullOrEmpty(continuations[i]))
{
allHeaders[i][HttpConstants.HttpHeaders.Continuation] = continuations[i];
}
feeds[i] = replicaClients[i].ReadFeedWithRetry <T>(collectionId, out responseHeaders[i],
allHeaders[i]);
if (responseHeaders[i] != null)
{
continuations[i] = responseHeaders[i][HttpConstants.HttpHeaders.Continuation];
}
else
{
continuations[i] = null;
}
}
for (int i = 0; i < replicaClients.Length - 1; i++)
{
for (int j = i + 1; j < replicaClients.Length; j++)
{
Assert.AreEqual(continuations[i], continuations[j], "Collection Continuaton mismatch");
if (verifyAddress)
{
var address1 = replicaClients[i].GetAddress();
var address2 = replicaClients[j].GetAddress();
// If the addresses match, we are in mid of reconfiguration, throw GoneException so that we retry on the entire loop
// and take care of intermittent gone at the same time.
if (address1.Equals(address2))
{
throw new GoneException("Addresses matched for multiple replicas " + address1);
}
}
}
}
ReplicationTests.ValidateCollection <T>(feeds);
} while (!string.IsNullOrEmpty(continuations[0]));
return(Task.FromResult <bool>(true));
}, new GoneAndRetryWithRetryPolicy()).Wait();
}
finally
{
//Once address are stable. dont force the cache refresh.
foreach (DocumentClient client in replicaClients)
{
client.ForceAddressRefresh(false);
}
}
}
public override Task <HttpResponseMessage> SendHttpAsync(
Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
ResourceType resourceType,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken)
{
diagnosticsContext ??= new CosmosDiagnosticsContextCore();
HttpRequestMessage requestMessage = null;
Func <Task <HttpResponseMessage> > funcDelegate = async() =>
{
using (diagnosticsContext.CreateScope(nameof(CosmosHttpClientCore.SendHttpAsync)))
{
using (requestMessage = await createRequestMessageAsync())
{
DateTime sendTimeUtc = DateTime.UtcNow;
Guid localGuid = Guid.NewGuid(); // For correlating HttpRequest and HttpResponse Traces
Guid requestedActivityId = Trace.CorrelationManager.ActivityId;
this.eventSource.Request(
requestedActivityId,
localGuid,
requestMessage.RequestUri.ToString(),
resourceType.ToResourceTypeString(),
requestMessage.Headers);
// Only read the header initially. The content gets copied into a memory stream later
// if we read the content http client will buffer the message and then it will get buffered
// again when it is copied to the memory stream.
HttpResponseMessage responseMessage = await this.httpClient.SendAsync(
requestMessage,
HttpCompletionOption.ResponseHeadersRead,
cancellationToken);
// WebAssembly HttpClient does not set the RequestMessage property on SendAsync
if (responseMessage.RequestMessage == null)
{
responseMessage.RequestMessage = requestMessage;
}
DateTime receivedTimeUtc = DateTime.UtcNow;
TimeSpan durationTimeSpan = receivedTimeUtc - sendTimeUtc;
Guid activityId = Guid.Empty;
if (responseMessage.Headers.TryGetValues(
HttpConstants.HttpHeaders.ActivityId,
out IEnumerable <string> headerValues) && headerValues.Any())
{
activityId = new Guid(headerValues.First());
}
this.eventSource.Response(
activityId,
localGuid,
(short)responseMessage.StatusCode,
durationTimeSpan.TotalMilliseconds,
responseMessage.Headers);
return(responseMessage);
}
}
};
HttpRequestMessage GetHttpRequestMessage()
{
return(requestMessage);
}
return(BackoffRetryUtility <HttpResponseMessage> .ExecuteAsync(
callbackMethod : funcDelegate,
retryPolicy : new TransientHttpClientRetryPolicy(
getHttpRequestMessage: GetHttpRequestMessage,
gatewayRequestTimeout: this.httpClient.Timeout,
diagnosticsContext: diagnosticsContext),
cancellationToken : cancellationToken));
}
private async Task <HttpResponseMessage> InvokeClientAsync(
DocumentServiceRequest request,
ResourceType resourceType,
Uri physicalAddress,
CancellationToken cancellationToken)
{
Func <Task <HttpResponseMessage> > funcDelegate = async() =>
{
using (HttpRequestMessage requestMessage = await this.PrepareRequestMessageAsync(request, physicalAddress))
{
DateTime sendTimeUtc = DateTime.UtcNow;
Guid localGuid = Guid.NewGuid(); // For correlating HttpRequest and HttpResponse Traces
IClientSideRequestStatistics clientSideRequestStatistics = request.RequestContext.ClientRequestStatistics;
if (clientSideRequestStatistics == null)
{
clientSideRequestStatistics = new CosmosClientSideRequestStatistics();
request.RequestContext.ClientRequestStatistics = clientSideRequestStatistics;
}
Guid requestedActivityId = Trace.CorrelationManager.ActivityId;
this.eventSource.Request(
requestedActivityId,
localGuid,
requestMessage.RequestUri.ToString(),
resourceType.ToResourceTypeString(),
requestMessage.Headers);
TimeSpan durationTimeSpan;
string recordAddressResolutionId = clientSideRequestStatistics.RecordAddressResolutionStart(requestMessage.RequestUri);
try
{
HttpResponseMessage responseMessage = await this.httpClient.SendAsync(requestMessage, cancellationToken);
DateTime receivedTimeUtc = DateTime.UtcNow;
durationTimeSpan = receivedTimeUtc - sendTimeUtc;
IEnumerable <string> headerValues;
Guid activityId = Guid.Empty;
if (responseMessage.Headers.TryGetValues(HttpConstants.HttpHeaders.ActivityId, out headerValues) &&
headerValues.Count() != 0)
{
activityId = new Guid(headerValues.First());
}
this.eventSource.Response(
activityId,
localGuid,
(short)responseMessage.StatusCode,
durationTimeSpan.TotalMilliseconds,
responseMessage.Headers);
return(responseMessage);
}
catch (TaskCanceledException ex)
{
if (!cancellationToken.IsCancellationRequested)
{
// throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
durationTimeSpan = DateTime.UtcNow - sendTimeUtc;
string message = $"GatewayStoreClient Request Timeout. Start Time:{sendTimeUtc}; Total Duration:{durationTimeSpan}; Http Client Timeout:{this.httpClient.Timeout}; Activity id: {requestedActivityId}; Inner Message: {ex.Message};";
throw new RequestTimeoutException(message, ex, requestMessage.RequestUri);
}
else
{
throw;
}
}
catch (OperationCanceledException ex)
{
if (!cancellationToken.IsCancellationRequested)
{
// throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
durationTimeSpan = DateTime.UtcNow - sendTimeUtc;
string message = $"GatewayStoreClient Request Timeout. Start Time:{sendTimeUtc}; Total Duration:{durationTimeSpan}; Http Client Timeout:{this.httpClient.Timeout}; Activity id: {requestedActivityId}; Inner Message: {ex.Message};";
throw new RequestTimeoutException(message, ex, requestMessage.RequestUri);
}
else
{
throw;
}
}
finally
{
clientSideRequestStatistics.RecordAddressResolutionEnd(recordAddressResolutionId);
}
}
};
return(await BackoffRetryUtility <HttpResponseMessage> .ExecuteAsync(funcDelegate, new WebExceptionRetryPolicy(), cancellationToken));
}
protected override async Task <DocumentFeedResponse <CosmosElement> > ExecuteInternalAsync(CancellationToken token)
{
CollectionCache collectionCache = await this.Client.GetCollectionCacheAsync();
PartitionKeyRangeCache partitionKeyRangeCache = await this.Client.GetPartitionKeyRangeCacheAsync();
IDocumentClientRetryPolicy retryPolicyInstance = this.Client.ResetSessionTokenRetryPolicy.GetRequestPolicy();
retryPolicyInstance = new InvalidPartitionExceptionRetryPolicy(retryPolicyInstance);
if (base.ResourceTypeEnum.IsPartitioned())
{
retryPolicyInstance = new PartitionKeyRangeGoneRetryPolicy(
collectionCache,
partitionKeyRangeCache,
PathsHelper.GetCollectionPath(base.ResourceLink),
retryPolicyInstance);
}
return(await BackoffRetryUtility <DocumentFeedResponse <CosmosElement> > .ExecuteAsync(
async() =>
{
this.fetchExecutionRangeAccumulator.BeginFetchRange();
++this.retries;
Tuple <DocumentFeedResponse <CosmosElement>, string> responseAndPartitionIdentifier = await this.ExecuteOnceAsync(retryPolicyInstance, token);
DocumentFeedResponse <CosmosElement> response = responseAndPartitionIdentifier.Item1;
string partitionIdentifier = responseAndPartitionIdentifier.Item2;
if (!string.IsNullOrEmpty(response.ResponseHeaders[HttpConstants.HttpHeaders.QueryMetrics]))
{
this.fetchExecutionRangeAccumulator.EndFetchRange(
partitionIdentifier,
response.ActivityId,
response.Count,
this.retries);
response = new DocumentFeedResponse <CosmosElement>(
response,
response.Count,
response.Headers,
response.UseETagAsContinuation,
new Dictionary <string, QueryMetrics>
{
{
partitionIdentifier,
QueryMetrics.CreateFromDelimitedStringAndClientSideMetrics(
response.ResponseHeaders[HttpConstants.HttpHeaders.QueryMetrics],
response.ResponseHeaders[HttpConstants.HttpHeaders.IndexUtilization],
new ClientSideMetrics(
this.retries,
response.RequestCharge,
this.fetchExecutionRangeAccumulator.GetExecutionRanges()))
}
},
response.RequestStatistics,
response.DisallowContinuationTokenMessage,
response.ResponseLengthBytes);
}
this.retries = -1;
return response;
},
retryPolicyInstance,
token));
}
private async Task ValidateRetryOnDatabaseAccountNotFoundAsync(bool enableMultipleWriteLocations, bool isReadRequest)
{
this.Initialize(
useMultipleWriteLocations: enableMultipleWriteLocations,
enableEndpointDiscovery: true,
isPreferredLocationsListEmpty: false);
await this.endpointManager.RefreshLocationAsync(this.databaseAccount);
ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, true, new RetryOptions());
int expectedRetryCount = isReadRequest || enableMultipleWriteLocations ? 2 : 1;
using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false))
{
int retryCount = 0;
try
{
await BackoffRetryUtility <bool> .ExecuteAsync(
() =>
{
retryCount++;
retryPolicy.OnBeforeSendRequest(request);
if (retryCount == 1)
{
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.DatabaseAccountNotFound).ToString();
DocumentClientException forbiddenException = new ForbiddenException(RMResources.NotFound, headers);
throw forbiddenException;
}
else if (retryCount == 2)
{
// Next request must go to next preferred endpoint
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
return(Task.FromResult(true));
}
else
{
Assert.Fail();
}
return(Task.FromResult(true));
},
retryPolicy);
}
catch (ForbiddenException)
{
if (expectedRetryCount == 1)
{
DefaultTrace.TraceInformation("Received expected ForbiddenException");
}
else
{
Assert.Fail();
}
}
Assert.AreEqual(expectedRetryCount, retryCount);
}
}
private async Task ValidateRetryOnWriteSessionNotAvailabeWithEnableMultipleWriteLocationsAsync()
{
const bool useMultipleWriteLocations = true;
bool enableEndpointDiscovery = true;
this.Initialize(
useMultipleWriteLocations: useMultipleWriteLocations,
enableEndpointDiscovery: enableEndpointDiscovery,
isPreferredLocationsListEmpty: false);
await this.endpointManager.RefreshLocationAsync(this.databaseAccount);
ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, enableEndpointDiscovery, new RetryOptions());
using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: false, isMasterResourceType: false))
{
int retryCount = 0;
try
{
await BackoffRetryUtility <bool> .ExecuteAsync(
() =>
{
retryPolicy.OnBeforeSendRequest(request);
if (retryCount == 0)
{
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
}
else if (retryCount == 1)
{
// Second request must go to first write endpoint
Uri expectedEndpoint = new Uri(this.databaseAccount.WriteLocationsInternal[0].Endpoint);
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
}
else if (retryCount == 2)
{
// Second request must go to first write endpoint
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
}
else if (retryCount == 3)
{
// Second request must go to first write endpoint
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[2]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
}
else
{
Assert.Fail();
}
retryCount++;
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.ReadSessionNotAvailable).ToString();
DocumentClientException notFoundException = new NotFoundException(RMResources.NotFound, headers);
throw notFoundException;
},
retryPolicy);
Assert.Fail();
}
catch (NotFoundException)
{
DefaultTrace.TraceInformation("Received expected notFoundException");
Assert.AreEqual(4, retryCount);
}
}
}
private async Task <HttpResponseMessage> InvokeClientAsync(
DocumentServiceRequest request,
ResourceType resourceType,
Uri physicalAddress,
CancellationToken cancellationToken)
{
Func <Task <HttpResponseMessage> > funcDelegate = async() =>
{
using (HttpRequestMessage requestMessage = await this.PrepareRequestMessageAsync(request, physicalAddress))
{
DateTime sendTimeUtc = DateTime.UtcNow;
Guid localGuid = Guid.NewGuid(); // For correlating HttpRequest and HttpResponse Traces
this.eventSource.Request(
Guid.Empty,
localGuid,
requestMessage.RequestUri.ToString(),
resourceType.ToResourceTypeString(),
requestMessage.Headers);
try
{
HttpResponseMessage responseMessage = await this.httpClient.SendAsync(requestMessage, cancellationToken);
DateTime receivedTimeUtc = DateTime.UtcNow;
double durationInMilliSeconds = (receivedTimeUtc - sendTimeUtc).TotalMilliseconds;
IEnumerable <string> headerValues;
Guid activityId = Guid.Empty;
if (responseMessage.Headers.TryGetValues(HttpConstants.HttpHeaders.ActivityId, out headerValues) &&
headerValues.Count() != 0)
{
activityId = new Guid(headerValues.First());
}
this.eventSource.Response(
activityId,
localGuid,
(short)responseMessage.StatusCode,
durationInMilliSeconds,
responseMessage.Headers);
return(responseMessage);
}
catch (TaskCanceledException ex)
{
if (!cancellationToken.IsCancellationRequested)
{
// throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
throw new RequestTimeoutException(ex, requestMessage.RequestUri);
}
else
{
throw;
}
}
}
};
return(await BackoffRetryUtility <HttpResponseMessage> .ExecuteAsync(funcDelegate, new WebExceptionRetryPolicy(), cancellationToken));
}
protected override async Task <FeedResponse <dynamic> > ExecuteInternalAsync(CancellationToken cancellationToken)
{
CollectionCache collectionCache = await this.Client.GetCollectionCacheAsync();
PartitionKeyRangeCache partitionKeyRangeCache = await this.Client.GetPartitionKeyRangeCache();
IDocumentClientRetryPolicy retryPolicyInstance = this.Client.RetryPolicy.GetRequestPolicy();
retryPolicyInstance = new InvalidPartitionExceptionRetryPolicy(collectionCache, retryPolicyInstance);
if (base.ResourceTypeEnum.IsPartitioned())
{
retryPolicyInstance = new PartitionKeyRangeGoneRetryPolicy(
collectionCache,
partitionKeyRangeCache,
PathsHelper.GetCollectionPath(base.ResourceLink),
retryPolicyInstance);
}
return(await BackoffRetryUtility <FeedResponse <dynamic> > .ExecuteAsync(
async() =>
{
this.fetchExecutionRangeAccumulator.BeginFetchRange();
++this.retries;
this.fetchSchedulingMetrics.Start();
this.fetchExecutionRangeAccumulator.BeginFetchRange();
FeedResponse <dynamic> response = await this.ExecuteOnceAsync(retryPolicyInstance, cancellationToken);
this.fetchSchedulingMetrics.Stop();
this.fetchExecutionRangeAccumulator.EndFetchRange(response.Count, this.retries);
if (!string.IsNullOrEmpty(response.Headers[HttpConstants.HttpHeaders.QueryMetrics]))
{
this.fetchExecutionRangeAccumulator.EndFetchRange(response.Count, this.retries);
response = new FeedResponse <dynamic>(
response,
response.Count,
response.Headers,
response.UseETagAsContinuation,
new Dictionary <string, QueryMetrics>
{
{
singlePartitionKeyId,
QueryMetrics.CreateFromDelimitedStringAndClientSideMetrics(
response.Headers[HttpConstants.HttpHeaders.QueryMetrics],
new ClientSideMetrics(
this.retries,
response.RequestCharge,
this.fetchExecutionRangeAccumulator.GetExecutionRanges(),
string.IsNullOrEmpty(response.ResponseContinuation) ? new List <Tuple <string, SchedulingTimeSpan> >()
{
new Tuple <string, SchedulingTimeSpan>(singlePartitionKeyId, this.fetchSchedulingMetrics.Elapsed)
} : new List <Tuple <string, SchedulingTimeSpan> >()),
Guid.Parse(response.ActivityId))
}
},
response.RequestStatistics,
response.DisallowContinuationTokenMessage,
response.ResponseLengthBytes);
}
this.retries = -1;
return response;
},
retryPolicyInstance,
cancellationToken));
}
public async Task ClientRetryPolicy_ValidateRetryOnServiceUnavailable(
bool isReadRequest,
bool useMultipleWriteLocations,
bool usesPreferredLocations,
bool shouldHaveRetried)
{
const bool enableEndpointDiscovery = true;
this.Initialize(
useMultipleWriteLocations: useMultipleWriteLocations,
enableEndpointDiscovery: enableEndpointDiscovery,
isPreferredLocationsListEmpty: !usesPreferredLocations);
await this.endpointManager.RefreshLocationAsync(this.databaseAccount);
ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, enableEndpointDiscovery, new RetryOptions());
using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false))
{
int retryCount = 0;
try
{
await BackoffRetryUtility <bool> .ExecuteAsync(
() =>
{
retryPolicy.OnBeforeSendRequest(request);
if (retryCount == 1)
{
Uri expectedEndpoint = null;
if (usesPreferredLocations)
{
expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]];
}
else
{
if (isReadRequest)
{
expectedEndpoint = new Uri(this.databaseAccount.ReadLocationsInternal[1].Endpoint);
}
else
{
expectedEndpoint = new Uri(this.databaseAccount.WriteLocationsInternal[1].Endpoint);
}
}
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
}
else if (retryCount > 1)
{
Assert.Fail("Should retry once");
}
retryCount++;
throw new ServiceUnavailableException();
},
retryPolicy);
Assert.Fail();
}
catch (ServiceUnavailableException)
{
DefaultTrace.TraceInformation("Received expected ServiceUnavailableException");
if (shouldHaveRetried)
{
Assert.AreEqual(2, retryCount, $"Retry count {retryCount}, shouldHaveRetried {shouldHaveRetried} isReadRequest {isReadRequest} useMultipleWriteLocations {useMultipleWriteLocations} usesPreferredLocations {usesPreferredLocations}");
}
else
{
Assert.AreEqual(1, retryCount, $"Retry count {retryCount}, shouldHaveRetried {shouldHaveRetried} isReadRequest {isReadRequest} useMultipleWriteLocations {useMultipleWriteLocations} usesPreferredLocations {usesPreferredLocations}");
}
}
}
}
private async Task <HttpResponseMessage> InvokeClientAsync(
DocumentServiceRequest request,
ResourceType resourceType,
Uri physicalAddress,
CancellationToken cancellationToken)
{
Func <Task <HttpResponseMessage> > funcDelegate = async() =>
{
using (HttpRequestMessage requestMessage = await this.PrepareRequestMessageAsync(request, physicalAddress))
{
DateTime sendTimeUtc = DateTime.UtcNow;
Guid localGuid = Guid.NewGuid(); // For correlating HttpRequest and HttpResponse Traces
// DEVNOTE: This is temporary until IClientSideRequestStats is modified to support gateway calls.
CosmosDiagnosticsContext diagnosticsContext = null;
if (request.RequestContext.ClientRequestStatistics is CosmosClientSideRequestStatistics clientSideRequestStatistics)
{
diagnosticsContext = clientSideRequestStatistics.DiagnosticsContext;
}
else
{
// ClientRequestStatistics are not passed in use Empty to avoid null checks.
// Caches do not pass in the ClientRequestStatistics
diagnosticsContext = EmptyCosmosDiagnosticsContext.Singleton;
}
Guid requestedActivityId = Trace.CorrelationManager.ActivityId;
this.eventSource.Request(
requestedActivityId,
localGuid,
requestMessage.RequestUri.ToString(),
resourceType.ToResourceTypeString(),
requestMessage.Headers);
TimeSpan durationTimeSpan;
try
{
HttpResponseMessage responseMessage;
using (diagnosticsContext.CreateScope("GatewayRequestTime"))
{
responseMessage = await this.httpClient.SendAsync(requestMessage, cancellationToken);
}
DateTime receivedTimeUtc = DateTime.UtcNow;
durationTimeSpan = receivedTimeUtc - sendTimeUtc;
IEnumerable <string> headerValues;
Guid activityId = Guid.Empty;
if (responseMessage.Headers.TryGetValues(HttpConstants.HttpHeaders.ActivityId, out headerValues) &&
headerValues.Count() != 0)
{
activityId = new Guid(headerValues.First());
}
this.eventSource.Response(
activityId,
localGuid,
(short)responseMessage.StatusCode,
durationTimeSpan.TotalMilliseconds,
responseMessage.Headers);
return(responseMessage);
}
catch (TaskCanceledException ex)
{
if (!cancellationToken.IsCancellationRequested)
{
// throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
durationTimeSpan = DateTime.UtcNow - sendTimeUtc;
string message = $"GatewayStoreClient Request Timeout. Start Time:{sendTimeUtc}; Total Duration:{durationTimeSpan}; Http Client Timeout:{this.httpClient.Timeout}; Activity id: {requestedActivityId}; Inner Message: {ex.Message};";
throw new RequestTimeoutException(message, ex, requestMessage.RequestUri);
}
else
{
throw;
}
}
catch (OperationCanceledException ex)
{
if (!cancellationToken.IsCancellationRequested)
{
// throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
durationTimeSpan = DateTime.UtcNow - sendTimeUtc;
string message = $"GatewayStoreClient Request Timeout. Start Time:{sendTimeUtc}; Total Duration:{durationTimeSpan}; Http Client Timeout:{this.httpClient.Timeout}; Activity id: {requestedActivityId}; Inner Message: {ex.Message};";
throw new RequestTimeoutException(message, ex, requestMessage.RequestUri);
}
else
{
throw;
}
}
}
};
return(await BackoffRetryUtility <HttpResponseMessage> .ExecuteAsync(funcDelegate, new WebExceptionRetryPolicy(), cancellationToken));
}
private async Task <CollectionRoutingMap> GetRoutingMapForCollectionAsync(
string collectionRid,
CollectionRoutingMap previousRoutingMap,
ITrace trace,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken)
{
List <PartitionKeyRange> ranges = new List <PartitionKeyRange>();
string changeFeedNextIfNoneMatch = previousRoutingMap == null ? null : previousRoutingMap.ChangeFeedNextIfNoneMatch;
HttpStatusCode lastStatusCode = HttpStatusCode.OK;
do
{
INameValueCollection headers = new StoreRequestNameValueCollection();
headers.Set(HttpConstants.HttpHeaders.PageSize, PageSizeString);
headers.Set(HttpConstants.HttpHeaders.A_IM, HttpConstants.A_IMHeaderValues.IncrementalFeed);
if (changeFeedNextIfNoneMatch != null)
{
headers.Set(HttpConstants.HttpHeaders.IfNoneMatch, changeFeedNextIfNoneMatch);
}
RetryOptions retryOptions = new RetryOptions();
using (DocumentServiceResponse response = await BackoffRetryUtility <DocumentServiceResponse> .ExecuteAsync(
() => this.ExecutePartitionKeyRangeReadChangeFeedAsync(collectionRid, headers, trace, clientSideRequestStatistics),
new ResourceThrottleRetryPolicy(retryOptions.MaxRetryAttemptsOnThrottledRequests, retryOptions.MaxRetryWaitTimeInSeconds),
cancellationToken))
{
lastStatusCode = response.StatusCode;
changeFeedNextIfNoneMatch = response.Headers[HttpConstants.HttpHeaders.ETag];
FeedResource <PartitionKeyRange> feedResource = response.GetResource <FeedResource <PartitionKeyRange> >();
if (feedResource != null)
{
ranges.AddRange(feedResource);
}
}
}while (lastStatusCode != HttpStatusCode.NotModified);
IEnumerable <Tuple <PartitionKeyRange, ServiceIdentity> > tuples = ranges.Select(range => Tuple.Create(range, (ServiceIdentity)null));
CollectionRoutingMap routingMap;
if (previousRoutingMap == null)
{
// Splits could have happened during change feed query and we might have a mix of gone and new ranges.
HashSet <string> goneRanges = new HashSet <string>(ranges.SelectMany(range => range.Parents ?? Enumerable.Empty <string>()));
routingMap = CollectionRoutingMap.TryCreateCompleteRoutingMap(
tuples.Where(tuple => !goneRanges.Contains(tuple.Item1.Id)),
string.Empty,
changeFeedNextIfNoneMatch);
}
else
{
routingMap = previousRoutingMap.TryCombine(tuples, changeFeedNextIfNoneMatch);
}
if (routingMap == null)
{
// Range information either doesn't exist or is not complete.
throw new NotFoundException($"{DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture)}: GetRoutingMapForCollectionAsync(collectionRid: {collectionRid}), Range information either doesn't exist or is not complete.");
}
return(routingMap);
}
private Task <HttpResponseMessage> SendHttpAsync(
Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
HttpCompletionOption httpCompletionOption,
ResourceType resourceType,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken)
{
diagnosticsContext ??= new CosmosDiagnosticsContextCore();
HttpRequestMessage requestMessage = null;
Func <Task <HttpResponseMessage> > funcDelegate = async() =>
{
using (diagnosticsContext.CreateScope(nameof(CosmosHttpClientCore.SendHttpAsync)))
{
using (requestMessage = await createRequestMessageAsync())
{
DateTime sendTimeUtc = DateTime.UtcNow;
Guid localGuid = Guid.NewGuid(); // For correlating HttpRequest and HttpResponse Traces
Guid requestedActivityId = Trace.CorrelationManager.ActivityId;
this.eventSource.Request(
requestedActivityId,
localGuid,
requestMessage.RequestUri.ToString(),
resourceType.ToResourceTypeString(),
requestMessage.Headers);
HttpResponseMessage responseMessage = await this.httpClient.SendAsync(
requestMessage,
httpCompletionOption,
cancellationToken);
DateTime receivedTimeUtc = DateTime.UtcNow;
TimeSpan durationTimeSpan = receivedTimeUtc - sendTimeUtc;
Guid activityId = Guid.Empty;
if (responseMessage.Headers.TryGetValues(
HttpConstants.HttpHeaders.ActivityId,
out IEnumerable <string> headerValues) && headerValues.Any())
{
activityId = new Guid(headerValues.First());
}
this.eventSource.Response(
activityId,
localGuid,
(short)responseMessage.StatusCode,
durationTimeSpan.TotalMilliseconds,
responseMessage.Headers);
return(responseMessage);
}
}
};
HttpRequestMessage GetHttpRequestMessage() => requestMessage;
return(BackoffRetryUtility <HttpResponseMessage> .ExecuteAsync(
callbackMethod : funcDelegate,
retryPolicy : new TransientHttpClientRetryPolicy(
getHttpRequestMessage: GetHttpRequestMessage,
gatewayRequestTimeout: this.httpClient.Timeout,
diagnosticsContext: diagnosticsContext),
cancellationToken : cancellationToken));
}
