public async Task ValidateRetryOnWriteForbiddenExceptionAsync()
{
this.Initialize(
useMultipleWriteLocations: false,
enableEndpointDiscovery: true,
isPreferredLocationsListEmpty: false);
await this.endpointManager.RefreshLocationAsync(this.databaseAccount);
ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, true, new RetryOptions());
using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: false, isMasterResourceType: false))
{
int retryCount = 0;
await BackoffRetryUtility <bool> .ExecuteAsync(
() =>
{
retryCount++;
retryPolicy.OnBeforeSendRequest(request);
if (retryCount == 1)
{
this.mockedClient.ResetCalls();
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.WriteForbidden).ToString();
DocumentClientException forbiddenException = new ForbiddenException(RMResources.Forbidden, headers);
throw forbiddenException;
}
else if (retryCount == 2)
{
this.mockedClient.Verify(client => client.GetDatabaseAccountInternalAsync(It.IsAny <Uri>(), It.IsAny <CancellationToken>()), Times.Once);
// Next request must go to next preferred endpoint
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
return(Task.FromResult(true));
}
else
{
Assert.Fail();
}
return(Task.FromResult(true));
},
retryPolicy);
}
}
private async Task ValidateRetryOnSessionNotAvailabeWithEndpointDiscoveryDisabled(bool isPreferredLocationsListEmpty, bool useMultipleWriteLocations, bool isReadRequest)
{
const bool enableEndpointDiscovery = false;
this.Initialize(
useMultipleWriteLocations: useMultipleWriteLocations,
enableEndpointDiscovery: enableEndpointDiscovery,
isPreferredLocationsListEmpty: isPreferredLocationsListEmpty);
ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, enableEndpointDiscovery, new RetryOptions());
using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false))
{
int retryCount = 0;
try
{
await BackoffRetryUtility <bool> .ExecuteAsync(
() =>
{
retryPolicy.OnBeforeSendRequest(request);
if (retryCount == 0)
{
Assert.IsFalse(request.ClearSessionTokenOnSessionReadFailure);
Assert.AreEqual(request.RequestContext.LocationEndpointToRoute, this.endpointManager.ReadEndpoints[0]);
}
else
{
Assert.Fail();
}
retryCount++;
StringKeyValueCollection headers = new StringKeyValueCollection();
headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.ReadSessionNotAvailable).ToString();
DocumentClientException notFoundException = new NotFoundException(RMResources.NotFound, headers);
throw notFoundException;
},
retryPolicy);
Assert.Fail();
}
catch (NotFoundException)
{
DefaultTrace.TraceInformation("Received expected notFoundException");
Assert.AreEqual(1, retryCount);
}
}
}
public async Task ClientRetryPolicy_ValidateRetryOnServiceUnavailable(
bool isReadRequest,
bool useMultipleWriteLocations,
bool usesPreferredLocations,
bool shouldHaveRetried)
{
const bool enableEndpointDiscovery = true;
this.Initialize(
useMultipleWriteLocations: useMultipleWriteLocations,
enableEndpointDiscovery: enableEndpointDiscovery,
isPreferredLocationsListEmpty: !usesPreferredLocations);
await this.endpointManager.RefreshLocationAsync(this.databaseAccount);
ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, enableEndpointDiscovery, new RetryOptions());
using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false))
{
int retryCount = 0;
try
{
await BackoffRetryUtility <bool> .ExecuteAsync(
() =>
{
retryPolicy.OnBeforeSendRequest(request);
if (retryCount == 1)
{
Uri expectedEndpoint = null;
if (usesPreferredLocations)
{
expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]];
}
else
{
if (isReadRequest)
{
expectedEndpoint = new Uri(this.databaseAccount.ReadLocationsInternal[1].Endpoint);
}
else
{
expectedEndpoint = new Uri(this.databaseAccount.WriteLocationsInternal[1].Endpoint);
}
}
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
}
else if (retryCount > 1)
{
Assert.Fail("Should retry once");
}
retryCount++;
throw new ServiceUnavailableException();
},
retryPolicy);
Assert.Fail();
}
catch (ServiceUnavailableException)
{
DefaultTrace.TraceInformation("Received expected ServiceUnavailableException");
if (shouldHaveRetried)
{
Assert.AreEqual(2, retryCount, $"Retry count {retryCount}, shouldHaveRetried {shouldHaveRetried} isReadRequest {isReadRequest} useMultipleWriteLocations {useMultipleWriteLocations} usesPreferredLocations {usesPreferredLocations}");
}
else
{
Assert.AreEqual(1, retryCount, $"Retry count {retryCount}, shouldHaveRetried {shouldHaveRetried} isReadRequest {isReadRequest} useMultipleWriteLocations {useMultipleWriteLocations} usesPreferredLocations {usesPreferredLocations}");
}
}
}
}
private async Task ValidateRetryOnDatabaseAccountNotFoundAsync(bool enableMultipleWriteLocations, bool isReadRequest)
{
this.Initialize(
useMultipleWriteLocations: enableMultipleWriteLocations,
enableEndpointDiscovery: true,
isPreferredLocationsListEmpty: false);
await this.endpointManager.RefreshLocationAsync(this.databaseAccount);
ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, true, new RetryOptions());
int expectedRetryCount = isReadRequest || enableMultipleWriteLocations ? 2 : 1;
using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false))
{
int retryCount = 0;
try
{
await BackoffRetryUtility <bool> .ExecuteAsync(
() =>
{
retryCount++;
retryPolicy.OnBeforeSendRequest(request);
if (retryCount == 1)
{
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.DatabaseAccountNotFound).ToString();
DocumentClientException forbiddenException = new ForbiddenException(RMResources.NotFound, headers);
throw forbiddenException;
}
else if (retryCount == 2)
{
// Next request must go to next preferred endpoint
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
return(Task.FromResult(true));
}
else
{
Assert.Fail();
}
return(Task.FromResult(true));
},
retryPolicy);
}
catch (ForbiddenException)
{
if (expectedRetryCount == 1)
{
DefaultTrace.TraceInformation("Received expected ForbiddenException");
}
else
{
Assert.Fail();
}
}
Assert.AreEqual(expectedRetryCount, retryCount);
}
}
private async Task ValidateRetryOnWriteSessionNotAvailabeWithEnableMultipleWriteLocationsAsync()
{
const bool useMultipleWriteLocations = true;
bool enableEndpointDiscovery = true;
this.Initialize(
useMultipleWriteLocations: useMultipleWriteLocations,
enableEndpointDiscovery: enableEndpointDiscovery,
isPreferredLocationsListEmpty: false);
await this.endpointManager.RefreshLocationAsync(this.databaseAccount);
ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, enableEndpointDiscovery, new RetryOptions());
using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: false, isMasterResourceType: false))
{
int retryCount = 0;
try
{
await BackoffRetryUtility <bool> .ExecuteAsync(
() =>
{
retryPolicy.OnBeforeSendRequest(request);
if (retryCount == 0)
{
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
}
else if (retryCount == 1)
{
// Second request must go to first write endpoint
Uri expectedEndpoint = new Uri(this.databaseAccount.WriteLocationsInternal[0].Endpoint);
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
}
else if (retryCount == 2)
{
// Second request must go to first write endpoint
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
}
else if (retryCount == 3)
{
// Second request must go to first write endpoint
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[2]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
}
else
{
Assert.Fail();
}
retryCount++;
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.ReadSessionNotAvailable).ToString();
DocumentClientException notFoundException = new NotFoundException(RMResources.NotFound, headers);
throw notFoundException;
},
retryPolicy);
Assert.Fail();
}
catch (NotFoundException)
{
DefaultTrace.TraceInformation("Received expected notFoundException");
Assert.AreEqual(4, retryCount);
}
}
}