public async Task ValidateRetryOnWriteForbiddenExceptionAsync() { this.Initialize( useMultipleWriteLocations: false, enableEndpointDiscovery: true, isPreferredLocationsListEmpty: false); await this.endpointManager.RefreshLocationAsync(this.databaseAccount); ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, true, new RetryOptions()); using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: false, isMasterResourceType: false)) { int retryCount = 0; await BackoffRetryUtility <bool> .ExecuteAsync( () => { retryCount++; retryPolicy.OnBeforeSendRequest(request); if (retryCount == 1) { this.mockedClient.ResetCalls(); Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]]; Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection(); headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.WriteForbidden).ToString(); DocumentClientException forbiddenException = new ForbiddenException(RMResources.Forbidden, headers); throw forbiddenException; } else if (retryCount == 2) { this.mockedClient.Verify(client => client.GetDatabaseAccountInternalAsync(It.IsAny <Uri>(), It.IsAny <CancellationToken>()), Times.Once); // Next request must go to next preferred endpoint Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]]; Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); return(Task.FromResult(true)); } else { Assert.Fail(); } return(Task.FromResult(true)); }, retryPolicy); } }
private async Task ValidateRetryOnSessionNotAvailabeWithEndpointDiscoveryDisabled(bool isPreferredLocationsListEmpty, bool useMultipleWriteLocations, bool isReadRequest) { const bool enableEndpointDiscovery = false; this.Initialize( useMultipleWriteLocations: useMultipleWriteLocations, enableEndpointDiscovery: enableEndpointDiscovery, isPreferredLocationsListEmpty: isPreferredLocationsListEmpty); ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, enableEndpointDiscovery, new RetryOptions()); using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false)) { int retryCount = 0; try { await BackoffRetryUtility <bool> .ExecuteAsync( () => { retryPolicy.OnBeforeSendRequest(request); if (retryCount == 0) { Assert.IsFalse(request.ClearSessionTokenOnSessionReadFailure); Assert.AreEqual(request.RequestContext.LocationEndpointToRoute, this.endpointManager.ReadEndpoints[0]); } else { Assert.Fail(); } retryCount++; StringKeyValueCollection headers = new StringKeyValueCollection(); headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.ReadSessionNotAvailable).ToString(); DocumentClientException notFoundException = new NotFoundException(RMResources.NotFound, headers); throw notFoundException; }, retryPolicy); Assert.Fail(); } catch (NotFoundException) { DefaultTrace.TraceInformation("Received expected notFoundException"); Assert.AreEqual(1, retryCount); } } }
public async Task ClientRetryPolicy_ValidateRetryOnServiceUnavailable( bool isReadRequest, bool useMultipleWriteLocations, bool usesPreferredLocations, bool shouldHaveRetried) { const bool enableEndpointDiscovery = true; this.Initialize( useMultipleWriteLocations: useMultipleWriteLocations, enableEndpointDiscovery: enableEndpointDiscovery, isPreferredLocationsListEmpty: !usesPreferredLocations); await this.endpointManager.RefreshLocationAsync(this.databaseAccount); ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, enableEndpointDiscovery, new RetryOptions()); using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false)) { int retryCount = 0; try { await BackoffRetryUtility <bool> .ExecuteAsync( () => { retryPolicy.OnBeforeSendRequest(request); if (retryCount == 1) { Uri expectedEndpoint = null; if (usesPreferredLocations) { expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]]; } else { if (isReadRequest) { expectedEndpoint = new Uri(this.databaseAccount.ReadLocationsInternal[1].Endpoint); } else { expectedEndpoint = new Uri(this.databaseAccount.WriteLocationsInternal[1].Endpoint); } } Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); } else if (retryCount > 1) { Assert.Fail("Should retry once"); } retryCount++; throw new ServiceUnavailableException(); }, retryPolicy); Assert.Fail(); } catch (ServiceUnavailableException) { DefaultTrace.TraceInformation("Received expected ServiceUnavailableException"); if (shouldHaveRetried) { Assert.AreEqual(2, retryCount, $"Retry count {retryCount}, shouldHaveRetried {shouldHaveRetried} isReadRequest {isReadRequest} useMultipleWriteLocations {useMultipleWriteLocations} usesPreferredLocations {usesPreferredLocations}"); } else { Assert.AreEqual(1, retryCount, $"Retry count {retryCount}, shouldHaveRetried {shouldHaveRetried} isReadRequest {isReadRequest} useMultipleWriteLocations {useMultipleWriteLocations} usesPreferredLocations {usesPreferredLocations}"); } } } }
private async Task ValidateRetryOnDatabaseAccountNotFoundAsync(bool enableMultipleWriteLocations, bool isReadRequest) { this.Initialize( useMultipleWriteLocations: enableMultipleWriteLocations, enableEndpointDiscovery: true, isPreferredLocationsListEmpty: false); await this.endpointManager.RefreshLocationAsync(this.databaseAccount); ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, true, new RetryOptions()); int expectedRetryCount = isReadRequest || enableMultipleWriteLocations ? 2 : 1; using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false)) { int retryCount = 0; try { await BackoffRetryUtility <bool> .ExecuteAsync( () => { retryCount++; retryPolicy.OnBeforeSendRequest(request); if (retryCount == 1) { Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]]; Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection(); headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.DatabaseAccountNotFound).ToString(); DocumentClientException forbiddenException = new ForbiddenException(RMResources.NotFound, headers); throw forbiddenException; } else if (retryCount == 2) { // Next request must go to next preferred endpoint Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]]; Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); return(Task.FromResult(true)); } else { Assert.Fail(); } return(Task.FromResult(true)); }, retryPolicy); } catch (ForbiddenException) { if (expectedRetryCount == 1) { DefaultTrace.TraceInformation("Received expected ForbiddenException"); } else { Assert.Fail(); } } Assert.AreEqual(expectedRetryCount, retryCount); } }
private async Task ValidateRetryOnWriteSessionNotAvailabeWithEnableMultipleWriteLocationsAsync() { const bool useMultipleWriteLocations = true; bool enableEndpointDiscovery = true; this.Initialize( useMultipleWriteLocations: useMultipleWriteLocations, enableEndpointDiscovery: enableEndpointDiscovery, isPreferredLocationsListEmpty: false); await this.endpointManager.RefreshLocationAsync(this.databaseAccount); ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, enableEndpointDiscovery, new RetryOptions()); using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: false, isMasterResourceType: false)) { int retryCount = 0; try { await BackoffRetryUtility <bool> .ExecuteAsync( () => { retryPolicy.OnBeforeSendRequest(request); if (retryCount == 0) { Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]]; Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); } else if (retryCount == 1) { // Second request must go to first write endpoint Uri expectedEndpoint = new Uri(this.databaseAccount.WriteLocationsInternal[0].Endpoint); Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); } else if (retryCount == 2) { // Second request must go to first write endpoint Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]]; Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); } else if (retryCount == 3) { // Second request must go to first write endpoint Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[2]]; Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); } else { Assert.Fail(); } retryCount++; StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection(); headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.ReadSessionNotAvailable).ToString(); DocumentClientException notFoundException = new NotFoundException(RMResources.NotFound, headers); throw notFoundException; }, retryPolicy); Assert.Fail(); } catch (NotFoundException) { DefaultTrace.TraceInformation("Received expected notFoundException"); Assert.AreEqual(4, retryCount); } } }