public virtual async Task <AccountProperties> GetDatabaseAccountAsync(Func <ValueTask <HttpRequestMessage> > requestMessage, CancellationToken cancellationToken = default)
{
AccountProperties databaseAccount = null;
// Get the ServiceDocumentResource from the gateway.
using (HttpResponseMessage responseMessage = await this.gatewayStoreClient.SendHttpAsync(
requestMessage,
ResourceType.DatabaseAccount,
cancellationToken))
{
using (DocumentServiceResponse documentServiceResponse = await ClientExtensions.ParseResponseAsync(responseMessage))
{
databaseAccount = CosmosResource.FromStream <AccountProperties>(documentServiceResponse);
}
long longValue;
IEnumerable <string> headerValues;
if (responseMessage.Headers.TryGetValues(HttpConstants.HttpHeaders.MaxMediaStorageUsageInMB, out headerValues) &&
(headerValues.Count() != 0))
{
if (long.TryParse(headerValues.First(), out longValue))
{
databaseAccount.MaxMediaStorageUsageInMB = longValue;
}
}
if (responseMessage.Headers.TryGetValues(HttpConstants.HttpHeaders.CurrentMediaStorageUsageInMB, out headerValues) &&
(headerValues.Count() != 0))
{
if (long.TryParse(headerValues.First(), out longValue))
{
databaseAccount.MediaStorageUsageInMB = longValue;
}
}
if (responseMessage.Headers.TryGetValues(HttpConstants.HttpHeaders.DatabaseAccountConsumedDocumentStorageInMB, out headerValues) &&
(headerValues.Count() != 0))
{
if (long.TryParse(headerValues.First(), out longValue))
{
databaseAccount.ConsumedDocumentStorageInMB = longValue;
}
}
if (responseMessage.Headers.TryGetValues(HttpConstants.HttpHeaders.DatabaseAccountProvisionedDocumentStorageInMB, out headerValues) &&
(headerValues.Count() != 0))
{
if (long.TryParse(headerValues.First(), out longValue))
{
databaseAccount.ProvisionedDocumentStorageInMB = longValue;
}
}
if (responseMessage.Headers.TryGetValues(HttpConstants.HttpHeaders.DatabaseAccountReservedDocumentStorageInMB, out headerValues) &&
(headerValues.Count() != 0))
{
if (long.TryParse(headerValues.First(), out longValue))
{
databaseAccount.ReservedDocumentStorageInMB = longValue;
}
}
}
return(databaseAccount);
}
public void ReadLocationRemoveAndAddMockTest()
{
string originalConfigValue = Environment.GetEnvironmentVariable("MinimumIntervalForNonForceRefreshLocationInMS");
Environment.SetEnvironmentVariable("MinimumIntervalForNonForceRefreshLocationInMS", "1000");
// Setup dummpy read locations for the database account
Collection <AccountRegion> readableLocations = new Collection <AccountRegion>();
AccountRegion writeLocation = new AccountRegion();
writeLocation.Name = "WriteLocation";
writeLocation.Endpoint = "https://writeendpoint.net/";
AccountRegion readLocation1 = new AccountRegion();
readLocation1.Name = "ReadLocation1";
readLocation1.Endpoint = "https://readendpoint1.net/";
AccountRegion readLocation2 = new AccountRegion();
readLocation2.Name = "ReadLocation2";
readLocation2.Endpoint = "https://readendpoint2.net/";
readableLocations.Add(writeLocation);
readableLocations.Add(readLocation1);
readableLocations.Add(readLocation2);
AccountProperties databaseAccount = new AccountProperties();
databaseAccount.ReadLocationsInternal = readableLocations;
//Setup mock owner "document client"
Mock <IDocumentClientInternal> mockOwner = new Mock <IDocumentClientInternal>();
mockOwner.Setup(owner => owner.ServiceEndpoint).Returns(new Uri("https://defaultendpoint.net/"));
mockOwner.Setup(owner => owner.GetDatabaseAccountInternalAsync(It.IsAny <Uri>(), It.IsAny <CancellationToken>())).ReturnsAsync(databaseAccount);
//Create connection policy and populate preferred locations
ConnectionPolicy connectionPolicy = new ConnectionPolicy();
connectionPolicy.PreferredLocations.Add("ReadLocation1");
connectionPolicy.PreferredLocations.Add("ReadLocation2");
GlobalEndpointManager globalEndpointManager = new GlobalEndpointManager(mockOwner.Object, connectionPolicy);
globalEndpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(databaseAccount);
Assert.AreEqual(globalEndpointManager.ReadEndpoints[0], new Uri(readLocation1.Endpoint));
//Remove location 1 from read locations and validate that the read endpoint switches to the next preferred location
readableLocations.Remove(readLocation1);
databaseAccount.ReadLocationsInternal = readableLocations;
globalEndpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(databaseAccount);
Assert.AreEqual(globalEndpointManager.ReadEndpoints[0], new Uri(readLocation2.Endpoint));
//Add location 1 back to read locations and validate that location 1 becomes the read endpoint again.
readableLocations.Add(readLocation1);
databaseAccount.ReadLocationsInternal = readableLocations;
//Sleep a bit for the refresh timer to kick in and rediscover location 1
Thread.Sleep(2000);
Assert.AreEqual(globalEndpointManager.ReadEndpoints[0], new Uri(readLocation1.Endpoint));
Environment.SetEnvironmentVariable("MinimumIntervalForNonForceRefreshLocationInMS", "1000");
}
public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync()
{
AccountProperties databaseAccount = new AccountProperties
{
ReadLocationsInternal = new Collection <AccountRegion>()
{
new AccountRegion
{
Name = "Location1",
Endpoint = "https://testfailover-westus.documents-test.windows-int.net/"
},
new AccountRegion
{
Name = "Location2",
Endpoint = "https://testfailover-southeastasia.documents-test.windows-int.net/"
},
new AccountRegion
{
Name = "Location3",
Endpoint = "https://testfailover-northcentralus.documents-test.windows-int.net/"
},
}
};
Uri defaultEndpoint = new Uri("https://testfailover.documents-test.windows-int.net/");
GetAccountRequestInjector slowPrimaryRegionHelper = new GetAccountRequestInjector()
{
ShouldFailRequest = (uri) => false,
ShouldDelayRequest = (uri) => false,
SuccessResponse = databaseAccount,
};
// Happy path where global succeeds
AccountProperties globalEndpointResult = await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync(
defaultEndpoint : defaultEndpoint,
locations : new List <string>()
{
"westus",
"southeastasia",
"northcentralus"
},
getDatabaseAccountFn : (uri) => slowPrimaryRegionHelper.RequestHelper(uri));
Assert.AreEqual(globalEndpointResult, databaseAccount);
Assert.AreEqual(0, slowPrimaryRegionHelper.FailedEndpointCount);
Assert.AreEqual(0, slowPrimaryRegionHelper.SlowEndpointCount);
Assert.IsTrue(slowPrimaryRegionHelper.ReturnedSuccess);
// global and primary slow and fail
{
slowPrimaryRegionHelper.Reset();
slowPrimaryRegionHelper.ShouldDelayRequest = (uri) => uri == defaultEndpoint || uri == new Uri(databaseAccount.ReadLocationsInternal.First().Endpoint);
slowPrimaryRegionHelper.ShouldFailRequest = slowPrimaryRegionHelper.ShouldDelayRequest;
Stopwatch stopwatch = Stopwatch.StartNew();
globalEndpointResult = await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync(
defaultEndpoint : defaultEndpoint,
locations : new List <string>()
{
"westus",
"southeastasia",
"northcentralus"
},
getDatabaseAccountFn : (uri) => slowPrimaryRegionHelper.RequestHelper(uri));
stopwatch.Stop();
Assert.AreEqual(globalEndpointResult, databaseAccount);
Assert.AreEqual(2, slowPrimaryRegionHelper.SlowEndpointCount);
Assert.IsTrue(slowPrimaryRegionHelper.ReturnedSuccess);
Assert.IsTrue(stopwatch.Elapsed > TimeSpan.FromSeconds(5));
Assert.IsTrue(stopwatch.Elapsed < TimeSpan.FromSeconds(10));
}
// All but the last URI succeeds
{
slowPrimaryRegionHelper.Reset();
slowPrimaryRegionHelper.ShouldDelayRequest = (uri) => false;
slowPrimaryRegionHelper.ShouldFailRequest = (uri) => uri != new Uri(databaseAccount.ReadLocationsInternal.Last().Endpoint);
globalEndpointResult = await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync(
defaultEndpoint : defaultEndpoint,
locations : new List <string>()
{
"westus",
"southeastasia",
"northcentralus"
},
getDatabaseAccountFn : (uri) => slowPrimaryRegionHelper.RequestHelper(uri));
Assert.AreEqual(globalEndpointResult, databaseAccount);
Assert.AreEqual(3, slowPrimaryRegionHelper.FailedEndpointCount);
Assert.IsTrue(slowPrimaryRegionHelper.ReturnedSuccess);
}
// All request but middle is delayed
{
slowPrimaryRegionHelper.Reset();
slowPrimaryRegionHelper.ShouldDelayRequest = (uri) => uri != new Uri(databaseAccount.ReadLocationsInternal[1].Endpoint);
slowPrimaryRegionHelper.ShouldFailRequest = (uri) => false;
globalEndpointResult = await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync(
defaultEndpoint : defaultEndpoint,
locations : new List <string>()
{
"westus",
"southeastasia",
"northcentralus"
},
getDatabaseAccountFn : (uri) => slowPrimaryRegionHelper.RequestHelper(uri));
Assert.AreEqual(globalEndpointResult, databaseAccount);
Assert.AreEqual(0, slowPrimaryRegionHelper.FailedEndpointCount);
Assert.IsTrue(slowPrimaryRegionHelper.ReturnedSuccess);
}
// Delay global and primary region, then only last region should succeed.
{
slowPrimaryRegionHelper.Reset();
slowPrimaryRegionHelper.ShouldFailRequest = (uri) => !uri.ToString().Contains("westus7");
slowPrimaryRegionHelper.ShouldDelayRequest = (uri) => uri == defaultEndpoint || uri == new Uri(databaseAccount.ReadLocationsInternal.First().Endpoint);
globalEndpointResult = await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync(
defaultEndpoint : defaultEndpoint,
locations : new List <string>()
{
"westus",
"westus2",
"westus3",
"westus4",
"westus5",
"westus6",
"westus7",
},
getDatabaseAccountFn : (uri) => slowPrimaryRegionHelper.RequestHelper(uri));
Assert.AreEqual(globalEndpointResult, databaseAccount);
Assert.AreEqual(5, slowPrimaryRegionHelper.FailedEndpointCount);
Assert.IsTrue(slowPrimaryRegionHelper.ReturnedSuccess);
}
}
public async Task EndpointFailureMockTest()
{
// Setup dummpy read locations for the database account
Collection <AccountRegion> readableLocations = new Collection <AccountRegion>();
AccountRegion writeLocation = new AccountRegion();
writeLocation.Name = "WriteLocation";
writeLocation.Endpoint = "https://writeendpoint.net/";
AccountRegion readLocation1 = new AccountRegion();
readLocation1.Name = "ReadLocation1";
readLocation1.Endpoint = "https://readendpoint1.net/";
AccountRegion readLocation2 = new AccountRegion();
readLocation2.Name = "ReadLocation2";
readLocation2.Endpoint = "https://readendpoint2.net/";
readableLocations.Add(writeLocation);
readableLocations.Add(readLocation1);
readableLocations.Add(readLocation2);
AccountProperties databaseAccount = new AccountProperties();
databaseAccount.ReadLocationsInternal = readableLocations;
//Setup mock owner "document client"
Mock <IDocumentClientInternal> mockOwner = new Mock <IDocumentClientInternal>();
mockOwner.Setup(owner => owner.ServiceEndpoint).Returns(new Uri("https://defaultendpoint.net/"));
mockOwner.Setup(owner => owner.GetDatabaseAccountInternalAsync(It.IsAny <Uri>(), It.IsAny <CancellationToken>())).ReturnsAsync(databaseAccount);
//Create connection policy and populate preferred locations
ConnectionPolicy connectionPolicy = new ConnectionPolicy();
connectionPolicy.PreferredLocations.Add("ReadLocation1");
connectionPolicy.PreferredLocations.Add("ReadLocation2");
GlobalEndpointManager globalEndpointManager = new GlobalEndpointManager(mockOwner.Object, connectionPolicy);
globalEndpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(databaseAccount);
Assert.AreEqual(globalEndpointManager.ReadEndpoints[0], new Uri(readLocation1.Endpoint));
//Mark each of the read locations as unavailable and validate that the read endpoint switches to the next preferred region / default endpoint.
globalEndpointManager.MarkEndpointUnavailableForRead(globalEndpointManager.ReadEndpoints[0]);
globalEndpointManager.RefreshLocationAsync().Wait();
Assert.AreEqual(globalEndpointManager.ReadEndpoints[0], new Uri(readLocation2.Endpoint));
globalEndpointManager.MarkEndpointUnavailableForRead(globalEndpointManager.ReadEndpoints[0]);
await globalEndpointManager.RefreshLocationAsync();
Assert.AreEqual(globalEndpointManager.ReadEndpoints[0], globalEndpointManager.WriteEndpoints[0]);
//Sleep a second for the unavailable endpoint entry to expire and background refresh timer to kick in
Thread.Sleep(3000);
await globalEndpointManager.RefreshLocationAsync();
Assert.AreEqual(globalEndpointManager.ReadEndpoints[0], new Uri(readLocation1.Endpoint));
}
public async Task EndpointFailureMockTest()
{
Environment.SetEnvironmentVariable("MinimumIntervalForNonForceRefreshLocationInMS", "100");
try
{
// Setup dummpy read locations for the database account
Collection <AccountRegion> readableLocations = new Collection <AccountRegion>();
AccountRegion writeLocation = new AccountRegion();
writeLocation.Name = "WriteLocation";
writeLocation.Endpoint = "https://writeendpoint.net/";
AccountRegion readLocation1 = new AccountRegion();
readLocation1.Name = "ReadLocation1";
readLocation1.Endpoint = "https://readendpoint1.net/";
AccountRegion readLocation2 = new AccountRegion();
readLocation2.Name = "ReadLocation2";
readLocation2.Endpoint = "https://readendpoint2.net/";
readableLocations.Add(writeLocation);
readableLocations.Add(readLocation1);
readableLocations.Add(readLocation2);
AccountProperties databaseAccount = new AccountProperties();
databaseAccount.ReadLocationsInternal = readableLocations;
//Setup mock owner "document client"
Mock <IDocumentClientInternal> mockOwner = new Mock <IDocumentClientInternal>();
mockOwner.Setup(owner => owner.ServiceEndpoint).Returns(new Uri("https://defaultendpoint.net/"));
int getAccountInfoCount = 0;
mockOwner.Setup(owner => owner.GetDatabaseAccountInternalAsync(It.IsAny <Uri>(), It.IsAny <CancellationToken>()))
.Callback(() => getAccountInfoCount++)
.ReturnsAsync(databaseAccount);
//Create connection policy and populate preferred locations
ConnectionPolicy connectionPolicy = new ConnectionPolicy();
connectionPolicy.PreferredLocations.Add("ReadLocation1");
connectionPolicy.PreferredLocations.Add("ReadLocation2");
using (GlobalEndpointManager globalEndpointManager = new GlobalEndpointManager(mockOwner.Object, connectionPolicy))
{
globalEndpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(databaseAccount);
Assert.AreEqual(globalEndpointManager.ReadEndpoints[0], new Uri(readLocation1.Endpoint));
//Mark each of the read locations as unavailable and validate that the read endpoint switches to the next preferred region / default endpoint.
globalEndpointManager.MarkEndpointUnavailableForRead(globalEndpointManager.ReadEndpoints[0]);
await globalEndpointManager.RefreshLocationAsync();
Assert.AreEqual(globalEndpointManager.ReadEndpoints[0], new Uri(readLocation2.Endpoint));
globalEndpointManager.MarkEndpointUnavailableForRead(globalEndpointManager.ReadEndpoints[0]);
await globalEndpointManager.RefreshLocationAsync();
Assert.AreEqual(globalEndpointManager.ReadEndpoints[0], globalEndpointManager.WriteEndpoints[0]);
getAccountInfoCount = 0;
//Sleep a second for the unavailable endpoint entry to expire and background refresh timer to kick in
await Task.Delay(TimeSpan.FromSeconds(3));
Assert.IsTrue(getAccountInfoCount > 0, "Callback is not working. There should be at least one call in this time frame.");
await globalEndpointManager.RefreshLocationAsync();
Assert.AreEqual(globalEndpointManager.ReadEndpoints[0], new Uri(readLocation1.Endpoint));
}
Assert.IsTrue(getAccountInfoCount > 0, "Callback is not working. There should be at least one call in this time frame.");
getAccountInfoCount = 0;
Thread.Sleep(TimeSpan.FromSeconds(3));
Assert.AreEqual(0, getAccountInfoCount, "There should be no more account calls after the GlobalEndpointManager is disposed");
}
finally
{
Environment.SetEnvironmentVariable("MinimumIntervalForNonForceRefreshLocationInMS", null);
}
}
