public async Task TestHttpRequestExceptionScenarioAsync()
{
// testhost.dll.config sets it to 2 seconds which causes it to always expire before retrying. Remove the override.
System.Configuration.ConfigurationManager.AppSettings["UnavailableLocationsExpirationTimeInSeconds"] = "500";
string accountName = nameof(TestHttpRequestExceptionScenarioAsync);
string primaryRegionNameForUri = "eastus";
string secondaryRegionNameForUri = "westus";
string globalEndpoint = $"https://{accountName}.documents.azure.com:443/";
Uri globalEndpointUri = new Uri(globalEndpoint);
string primaryRegionEndpoint = $"https://{accountName}-{primaryRegionNameForUri}.documents.azure.com";
string secondaryRegionEndpiont = $"https://{accountName}-{secondaryRegionNameForUri}.documents.azure.com";
string databaseName = "testDb";
string containerName = "testContainer";
string containerRid = "ccZ1ANCszwk=";
ResourceId containerResourceId = ResourceId.Parse(containerRid);
List <AccountRegion> writeRegion = new List <AccountRegion>()
{
new AccountRegion()
{
Name = "East US",
Endpoint = $"{primaryRegionEndpoint}:443/"
}
};
List <AccountRegion> readRegions = new List <AccountRegion>()
{
new AccountRegion()
{
Name = "East US",
Endpoint = $"{primaryRegionEndpoint}:443/"
},
new AccountRegion()
{
Name = "West US",
Endpoint = $"{secondaryRegionEndpiont}:443/"
}
};
List <AccountRegion> writeRegionFailedOver = new List <AccountRegion>()
{
new AccountRegion()
{
Name = "West US",
Endpoint = $"{secondaryRegionEndpiont}:443/"
}
};
List <AccountRegion> readRegionsFailedOver = new List <AccountRegion>()
{
new AccountRegion()
{
Name = "West US",
Endpoint = $"{secondaryRegionEndpiont}:443/"
},
new AccountRegion()
{
Name = "East US",
Endpoint = $"{primaryRegionEndpoint}:443/"
},
};
// Create a mock http handler to inject gateway responses.
// MockBehavior.Strict ensures that only the mocked APIs get called
Mock <IHttpHandler> mockHttpHandler = new Mock <IHttpHandler>(MockBehavior.Strict);
mockHttpHandler.Setup(x => x.SendAsync(
It.Is <HttpRequestMessage>(m => m.RequestUri == globalEndpointUri || m.RequestUri.ToString().Contains(primaryRegionNameForUri)),
It.IsAny <CancellationToken>())).Throws(new HttpRequestException("Mock HttpRequestException to simulate region being down"));
int count = 0;
mockHttpHandler.Setup(x => x.SendAsync(
It.Is <HttpRequestMessage>(x => x.RequestUri == new Uri(secondaryRegionEndpiont)),
It.IsAny <CancellationToken>()))
.Returns <HttpRequestMessage, CancellationToken>((request, cancellationToken) =>
{
// Simulate the legacy gateway being down. After 40 requests simulate the write region pointing to new location.
count++;
if (count < 2)
{
return(Task.FromResult(MockSetupsHelper.CreateStrongAccount(accountName, writeRegion, readRegions)));
}
else
{
return(Task.FromResult(MockSetupsHelper.CreateStrongAccount(accountName, writeRegionFailedOver, readRegionsFailedOver)));
}
});
MockSetupsHelper.SetupContainerProperties(
mockHttpHandler: mockHttpHandler,
regionEndpoint: secondaryRegionEndpiont,
databaseName: databaseName,
containerName: containerName,
containerRid: containerRid);
MockSetupsHelper.SetupPartitionKeyRanges(
mockHttpHandler: mockHttpHandler,
regionEndpoint: secondaryRegionEndpiont,
containerResourceId: containerResourceId,
partitionKeyRangeIds: out IReadOnlyList <string> secondaryRegionPartitionKeyRangeIds);
MockSetupsHelper.SetupAddresses(
mockHttpHandler: mockHttpHandler,
partitionKeyRangeId: secondaryRegionPartitionKeyRangeIds.First(),
regionEndpoint: secondaryRegionEndpiont,
regionName: secondaryRegionNameForUri,
containerResourceId: containerResourceId,
primaryReplicaUri: out TransportAddressUri secondaryRegionprimaryReplicaUri);
Mock <TransportClient> mockTransport = new Mock <TransportClient>(MockBehavior.Strict);
MockSetupsHelper.SetupRequestTimeoutException(
mockTransport,
secondaryRegionprimaryReplicaUri);
// Partition key ranges are the same in both regions so the SDK
// does not need to go the secondary to get the partition key ranges.
// Only the addresses need to be mocked on the secondary
MockSetupsHelper.SetupAddresses(
mockHttpHandler: mockHttpHandler,
partitionKeyRangeId: secondaryRegionPartitionKeyRangeIds.First(),
regionEndpoint: secondaryRegionEndpiont,
regionName: secondaryRegionNameForUri,
containerResourceId: containerResourceId,
primaryReplicaUri: out TransportAddressUri secondaryRegionPrimaryReplicaUri);
MockSetupsHelper.SetupCreateItemResponse(
mockTransport,
secondaryRegionPrimaryReplicaUri);
CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
{
EnablePartitionLevelFailover = true,
ConsistencyLevel = Cosmos.ConsistencyLevel.Strong,
ApplicationPreferredRegions = new List <string>()
{
Regions.EastUS,
Regions.WestUS
},
HttpClientFactory = () => new HttpClient(new HttpHandlerHelper(mockHttpHandler.Object)),
TransportClientHandlerFactory = (original) => mockTransport.Object,
};
using (CosmosClient customClient = new CosmosClient(
globalEndpoint,
Convert.ToBase64String(Encoding.UTF8.GetBytes(Guid.NewGuid().ToString())),
cosmosClientOptions))
{
Container container = customClient.GetContainer(databaseName, containerName);
ToDoActivity toDoActivity = new ToDoActivity()
{
Id = "TestItem",
Pk = "TestPk"
};
ItemResponse <ToDoActivity> response = await container.CreateItemAsync(toDoActivity, new Cosmos.PartitionKey(toDoActivity.Pk));
Assert.AreEqual(HttpStatusCode.Created, response.StatusCode);
mockTransport.VerifyAll();
mockHttpHandler.VerifyAll();
// Clears all the setups. No network calls should be done on the next operation.
mockHttpHandler.Reset();
mockTransport.Reset();
MockSetupsHelper.SetupCreateItemResponse(
mockTransport,
secondaryRegionPrimaryReplicaUri);
ToDoActivity toDoActivity2 = new ToDoActivity()
{
Id = "TestItem2",
Pk = "TestPk"
};
response = await container.CreateItemAsync(toDoActivity2, new Cosmos.PartitionKey(toDoActivity2.Pk));
Assert.AreEqual(HttpStatusCode.Created, response.StatusCode);
mockTransport.Setup(x => x.Dispose());
// Reset it back to the override to avoid impacting other tests.
System.Configuration.ConfigurationManager.AppSettings["UnavailableLocationsExpirationTimeInSeconds"] = "2";
}
await Task.Delay(TimeSpan.FromMinutes(2));
}
private static void SetupAccountAndCacheOperations(
out string secondaryRegionNameForUri,
out string globalEndpoint,
out string secondaryRegionEndpiont,
out string databaseName,
out string containerName,
out ResourceId containerResourceId,
out Mock <IHttpHandler> mockHttpHandler,
out IReadOnlyList <string> primaryRegionPartitionKeyRangeIds,
out TransportAddressUri primaryRegionprimaryReplicaUri)
{
string accountName = "testAccount";
string primaryRegionNameForUri = "eastus";
secondaryRegionNameForUri = "westus";
globalEndpoint = $"https://{accountName}.documents.azure.com:443/";
Uri globalEndpointUri = new Uri(globalEndpoint);
string primaryRegionEndpoint = $"https://{accountName}-{primaryRegionNameForUri}.documents.azure.com";
secondaryRegionEndpiont = $"https://{accountName}-{secondaryRegionNameForUri}.documents.azure.com";
databaseName = "testDb";
containerName = "testContainer";
string containerRid = "ccZ1ANCszwk=";
containerResourceId = ResourceId.Parse(containerRid);
List <AccountRegion> writeRegion = new List <AccountRegion>()
{
new AccountRegion()
{
Name = "East US",
Endpoint = $"{primaryRegionEndpoint}:443/"
}
};
List <AccountRegion> readRegions = new List <AccountRegion>()
{
new AccountRegion()
{
Name = "East US",
Endpoint = $"{primaryRegionEndpoint}:443/"
},
new AccountRegion()
{
Name = "West US",
Endpoint = $"{secondaryRegionEndpiont}:443/"
}
};
// Create a mock http handler to inject gateway responses.
// MockBehavior.Strict ensures that only the mocked APIs get called
mockHttpHandler = new Mock <IHttpHandler>(MockBehavior.Strict);
MockSetupsHelper.SetupStrongAccountProperties(
mockHttpClientHandler: mockHttpHandler,
endpoint: globalEndpointUri.ToString(),
accountName: accountName,
writeRegions: writeRegion,
readRegions: readRegions);
MockSetupsHelper.SetupContainerProperties(
mockHttpHandler: mockHttpHandler,
regionEndpoint: primaryRegionEndpoint,
databaseName: databaseName,
containerName: containerName,
containerRid: containerRid);
MockSetupsHelper.SetupPartitionKeyRanges(
mockHttpHandler: mockHttpHandler,
regionEndpoint: primaryRegionEndpoint,
containerResourceId: containerResourceId,
partitionKeyRangeIds: out primaryRegionPartitionKeyRangeIds);
MockSetupsHelper.SetupAddresses(
mockHttpHandler: mockHttpHandler,
partitionKeyRangeId: primaryRegionPartitionKeyRangeIds.First(),
regionEndpoint: primaryRegionEndpoint,
regionName: primaryRegionNameForUri,
containerResourceId: containerResourceId,
primaryReplicaUri: out primaryRegionprimaryReplicaUri);
}
public async Task TestGoneFromServiceScenarioAsync()
{
Mock <IHttpHandler> mockHttpHandler = new Mock <IHttpHandler>(MockBehavior.Strict);
Uri endpoint = MockSetupsHelper.SetupSingleRegionAccount(
"mockAccountInfo",
consistencyLevel: ConsistencyLevel.Session,
mockHttpHandler,
out string primaryRegionEndpoint);
string databaseName = "mockDbName";
string containerName = "mockContainerName";
string containerRid = "ccZ1ANCszwk=";
Documents.ResourceId cRid = Documents.ResourceId.Parse(containerRid);
MockSetupsHelper.SetupContainerProperties(
mockHttpHandler: mockHttpHandler,
regionEndpoint: primaryRegionEndpoint,
databaseName: databaseName,
containerName: containerName,
containerRid: containerRid);
MockSetupsHelper.SetupSinglePartitionKeyRange(
mockHttpHandler,
primaryRegionEndpoint,
cRid,
out IReadOnlyList <string> partitionKeyRanges);
List <string> replicaIds1 = new List <string>()
{
"11111111111111111",
"22222222222222222",
"33333333333333333",
"44444444444444444",
};
HttpResponseMessage replicaSet1 = MockSetupsHelper.CreateAddresses(
replicaIds1,
partitionKeyRanges.First(),
"eastus",
cRid);
// One replica changed on the refresh
List <string> replicaIds2 = new List <string>()
{
"11111111111111111",
"22222222222222222",
"33333333333333333",
"55555555555555555",
};
HttpResponseMessage replicaSet2 = MockSetupsHelper.CreateAddresses(
replicaIds2,
partitionKeyRanges.First(),
"eastus",
cRid);
bool delayCacheRefresh = true;
bool delayRefreshUnblocked = false;
mockHttpHandler.SetupSequence(x => x.SendAsync(
It.Is <HttpRequestMessage>(r => r.RequestUri.ToString().Contains("addresses")), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(replicaSet1))
.Returns(async() =>
{
//block cache refresh to verify bad replica is not visited during refresh
while (delayCacheRefresh)
{
await Task.Delay(TimeSpan.FromMilliseconds(20));
}
delayRefreshUnblocked = true;
return(replicaSet2);
});
int callBack = 0;
List <Documents.TransportAddressUri> urisVisited = new List <Documents.TransportAddressUri>();
Mock <Documents.TransportClient> mockTransportClient = new Mock <Documents.TransportClient>(MockBehavior.Strict);
mockTransportClient.Setup(x => x.InvokeResourceOperationAsync(It.IsAny <Documents.TransportAddressUri>(), It.IsAny <Documents.DocumentServiceRequest>()))
.Callback <Documents.TransportAddressUri, Documents.DocumentServiceRequest>((t, _) => urisVisited.Add(t))
.Returns(() =>
{
callBack++;
if (callBack == 1)
{
throw Documents.Rntbd.TransportExceptions.GetGoneException(
new Uri("https://localhost:8081"),
Guid.NewGuid(),
new Documents.TransportException(Documents.TransportErrorCode.ConnectionBroken,
null,
Guid.NewGuid(),
new Uri("https://localhost:8081"),
"Mock",
userPayload: true,
payloadSent: false));
}
return(Task.FromResult(new Documents.StoreResponse()
{
Status = 200,
Headers = new Documents.Collections.StoreResponseNameValueCollection()
{
ActivityId = Guid.NewGuid().ToString(),
LSN = "12345",
PartitionKeyRangeId = "0",
GlobalCommittedLSN = "12345",
SessionToken = "1#12345#1=12345"
},
ResponseBody = new MemoryStream()
}));
});
CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
{
ConsistencyLevel = Cosmos.ConsistencyLevel.Session,
HttpClientFactory = () => new HttpClient(new HttpHandlerHelper(mockHttpHandler.Object)),
TransportClientHandlerFactory = (original) => mockTransportClient.Object,
};
using (CosmosClient customClient = new CosmosClient(
endpoint.ToString(),
Convert.ToBase64String(Encoding.UTF8.GetBytes(Guid.NewGuid().ToString())),
cosmosClientOptions))
{
try
{
Container container = customClient.GetContainer(databaseName, containerName);
for (int i = 0; i < 20; i++)
{
ResponseMessage response = await container.ReadItemStreamAsync(Guid.NewGuid().ToString(), new Cosmos.PartitionKey(Guid.NewGuid().ToString()));
Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
}
mockTransportClient.VerifyAll();
mockHttpHandler.VerifyAll();
Documents.TransportAddressUri failedReplica = urisVisited.First();
Assert.AreEqual(1, urisVisited.Count(x => x.Equals(failedReplica)));
urisVisited.Clear();
delayCacheRefresh = false;
do
{
await Task.Delay(TimeSpan.FromMilliseconds(100));
}while (!delayRefreshUnblocked);
for (int i = 0; i < 20; i++)
{
ResponseMessage response = await container.ReadItemStreamAsync(Guid.NewGuid().ToString(), new Cosmos.PartitionKey(Guid.NewGuid().ToString()));
Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
}
Assert.AreEqual(4, urisVisited.ToHashSet().Count());
// Clears all the setups. No network calls should be done on the next operation.
mockHttpHandler.Reset();
mockTransportClient.Reset();
}
finally
{
mockTransportClient.Setup(x => x.Dispose());
}
}
}