public async Task <CosmosAccountSettings> InitializeReaderAsync()
{
CosmosAccountSettings databaseAccount = await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync(
this.serviceEndpoint, this.connectionPolicy.PreferredLocations, this.GetDatabaseAccountAsync);
return(databaseAccount);
}
private GatewayStoreModel GetGatewayStoreModelForConsistencyTest()
{
Func <HttpRequestMessage, Task <HttpResponseMessage> > messageHandler = async request =>
{
String content = await request.Content.ReadAsStringAsync();
if (content.Equals("document"))
{
IEnumerable <string> sessionTokens = request.Headers.GetValues("x-ms-session-token");
string sessionToken = "";
foreach (string singleToken in sessionTokens)
{
sessionToken = singleToken;
break;
}
Assert.AreEqual(sessionToken, "0:1#100#1=20#2=5#3=30");
}
else
{
IEnumerable <string> enumerable;
Assert.IsFalse(request.Headers.TryGetValues("x-ms-session-token", out enumerable));
}
return(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent("Response")
});
};
Mock <IDocumentClientInternal> mockDocumentClient = new Mock <IDocumentClientInternal>();
mockDocumentClient.Setup(client => client.ServiceEndpoint).Returns(new Uri("https://foo"));
mockDocumentClient.Setup(client => client.ConsistencyLevel).Returns(Documents.ConsistencyLevel.Session);
GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockDocumentClient.Object, new ConnectionPolicy());
SessionContainer sessionContainer = new SessionContainer(string.Empty);
sessionContainer.SetSessionToken(
ResourceId.NewDocumentCollectionId(42, 129).DocumentCollectionId.ToString(),
"dbs/db1/colls/coll1",
new DictionaryNameValueCollection()
{
{ HttpConstants.HttpHeaders.SessionToken, "0:1#100#1=20#2=5#3=30" }
});
DocumentClientEventSource eventSource = new DocumentClientEventSource();
HttpMessageHandler httpMessageHandler = new MockMessageHandler(messageHandler);
GatewayStoreModel storeModel = new GatewayStoreModel(
endpointManager,
sessionContainer,
TimeSpan.FromSeconds(50),
ConsistencyLevel.Session,
eventSource,
new UserAgentContainer(),
ApiType.None,
httpMessageHandler);
return(storeModel);
}
public void GatewayStoreModel_HttpClientFactory_IfNull()
{
HttpClient staticHttpClient = null;
Mock <Func <HttpClient> > mockFactory = new Mock <Func <HttpClient> >();
mockFactory.Setup(f => f()).Returns(staticHttpClient);
Mock <IDocumentClientInternal> mockDocumentClient = new Mock <IDocumentClientInternal>();
mockDocumentClient.Setup(client => client.ServiceEndpoint).Returns(new Uri("https://foo"));
GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockDocumentClient.Object, new ConnectionPolicy());
SessionContainer sessionContainer = new SessionContainer(string.Empty);
DocumentClientEventSource eventSource = DocumentClientEventSource.Instance;
GatewayStoreModel storeModel = new GatewayStoreModel(
endpointManager,
sessionContainer,
TimeSpan.FromSeconds(5),
ConsistencyLevel.Eventual,
eventSource,
null,
new UserAgentContainer(),
ApiType.None,
mockFactory.Object);
}
public void ReadLocationRemoveAndAddMockTest()
{
// Setup dummpy read locations for the database account
Collection <CosmosAccountLocation> readableLocations = new Collection <CosmosAccountLocation>();
CosmosAccountLocation writeLocation = new CosmosAccountLocation();
writeLocation.Name = "WriteLocation";
writeLocation.DatabaseAccountEndpoint = "https://writeendpoint.net/";
CosmosAccountLocation readLocation1 = new CosmosAccountLocation();
readLocation1.Name = "ReadLocation1";
readLocation1.DatabaseAccountEndpoint = "https://readendpoint1.net/";
CosmosAccountLocation readLocation2 = new CosmosAccountLocation();
readLocation2.Name = "ReadLocation2";
readLocation2.DatabaseAccountEndpoint = "https://readendpoint2.net/";
readableLocations.Add(writeLocation);
readableLocations.Add(readLocation1);
readableLocations.Add(readLocation2);
CosmosAccountSettings databaseAccount = new CosmosAccountSettings();
databaseAccount.ReadLocationsInternal = readableLocations;
//Setup mock owner "document client"
Mock <IDocumentClientInternal> mockOwner = new Mock <IDocumentClientInternal>();
mockOwner.Setup(owner => owner.ServiceEndpoint).Returns(new Uri("https://defaultendpoint.net/"));
mockOwner.Setup(owner => owner.GetDatabaseAccountInternalAsync(It.IsAny <Uri>(), It.IsAny <CancellationToken>())).ReturnsAsync(databaseAccount);
//Create connection policy and populate preferred locations
ConnectionPolicy connectionPolicy = new ConnectionPolicy();
connectionPolicy.PreferredLocations.Add("ReadLocation1");
connectionPolicy.PreferredLocations.Add("ReadLocation2");
GlobalEndpointManager globalEndpointManager = new GlobalEndpointManager(mockOwner.Object, connectionPolicy);
globalEndpointManager.RefreshLocationAsync(databaseAccount).Wait();
Assert.IsTrue(globalEndpointManager.ReadEndpoints[0] == new Uri(readLocation1.DatabaseAccountEndpoint));
//Remove location 1 from read locations and validate that the read endpoint switches to the next preferred location
readableLocations.Remove(readLocation1);
databaseAccount.ReadLocationsInternal = readableLocations;
globalEndpointManager.RefreshLocationAsync(databaseAccount).Wait();
Assert.IsTrue(globalEndpointManager.ReadEndpoints[0] == new Uri(readLocation2.DatabaseAccountEndpoint));
//Add location 1 back to read locations and validate that location 1 becomes the read endpoint again.
readableLocations.Add(readLocation1);
databaseAccount.ReadLocationsInternal = readableLocations;
//Sleep a bit for the refresh timer to kick in and rediscover location 1
Thread.Sleep(2000);
Assert.IsTrue(globalEndpointManager.ReadEndpoints[0] == new Uri(readLocation1.DatabaseAccountEndpoint));
}
public GatewayStoreModel(
GlobalEndpointManager endpointManager,
ISessionContainer sessionContainer,
TimeSpan requestTimeout,
ConsistencyLevel defaultConsistencyLevel,
DocumentClientEventSource eventSource,
JsonSerializerSettings serializerSettings,
UserAgentContainer userAgent,
ApiType apiType,
Func <HttpClient> httpClientFactory)
: this(endpointManager,
sessionContainer,
defaultConsistencyLevel,
eventSource)
{
HttpClient httpClient = httpClientFactory();
if (httpClient == null)
{
throw new InvalidOperationException("HttpClientFactory did not produce an HttpClient");
}
this.InitializeGatewayStoreClient(
requestTimeout,
serializerSettings,
userAgent,
apiType,
httpClient);
}
public async Task ValidateCancellationTokenLogicForGetDatabaseAccountFromAnyLocationAsync()
{
Uri defaultEndpoint = new Uri("https://testfailover.documents-test.windows-int.net/");
using CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
cancellationTokenSource.Cancel();
try
{
await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync(
defaultEndpoint,
locations : new List <string>()
{
"westus",
"southeastasia",
"northcentralus"
},
getDatabaseAccountFn : (uri) => throw new Exception("The operation should be canceled and never make the network call."),
cancellationTokenSource.Token);
Assert.Fail("Previous call should have failed");
}
catch (OperationCanceledException op)
{
Assert.IsTrue(op.Message.Contains("GlobalEndpointManager"));
}
}
private async Task GatewayStoreModel_Exception_UpdateSessionTokenOnKnownException(Exception ex)
{
const string originalSessionToken = "0:1#100#1=20#2=5#3=30";
const string updatedSessionToken = "0:1#100#1=20#2=5#3=31";
Func <HttpRequestMessage, Task <HttpResponseMessage> > sendFunc = request =>
{
throw ex;
};
Mock <IDocumentClientInternal> mockDocumentClient = new Mock <IDocumentClientInternal>();
mockDocumentClient.Setup(client => client.ServiceEndpoint).Returns(new Uri("https://foo"));
GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockDocumentClient.Object, new ConnectionPolicy());
SessionContainer sessionContainer = new SessionContainer(string.Empty);
DocumentClientEventSource eventSource = DocumentClientEventSource.Instance;
HttpMessageHandler messageHandler = new MockMessageHandler(sendFunc);
GatewayStoreModel storeModel = new GatewayStoreModel(
endpointManager,
sessionContainer,
TimeSpan.FromSeconds(5),
ConsistencyLevel.Eventual,
eventSource,
null,
new UserAgentContainer(),
ApiType.None,
messageHandler);
INameValueCollection headers = new DictionaryNameValueCollection();
headers.Set(HttpConstants.HttpHeaders.ConsistencyLevel, ConsistencyLevel.Session.ToString());
headers.Set(HttpConstants.HttpHeaders.SessionToken, originalSessionToken);
headers.Set(WFConstants.BackendHeaders.PartitionKeyRangeId, "0");
using (new ActivityScope(Guid.NewGuid()))
{
using (DocumentServiceRequest request = DocumentServiceRequest.Create(
OperationType.Read,
ResourceType.Document,
"dbs/OVJwAA==/colls/OVJwAOcMtA0=/docs/OVJwAOcMtA0BAAAAAAAAAA==/",
AuthorizationTokenType.PrimaryMasterKey,
headers))
{
request.UseStatusCodeFor429 = true;
request.UseStatusCodeForFailures = true;
try
{
DocumentServiceResponse response = await storeModel.ProcessMessageAsync(request);
Assert.Fail("Should had thrown exception");
}
catch (Exception)
{
// Expecting exception
}
Assert.AreEqual(updatedSessionToken, sessionContainer.GetSessionToken("dbs/OVJwAA==/colls/OVJwAOcMtA0="));
}
}
}
public async Task GatewayProcessMessageAsyncCancels()
{
using (CancellationTokenSource source = new CancellationTokenSource())
{
CancellationToken cancellationToken = source.Token;
int run = 0;
Func <HttpRequestMessage, Task <HttpResponseMessage> > sendFunc = async request =>
{
string content = await request.Content.ReadAsStringAsync();
Assert.AreEqual("content1", content);
if (run == 0)
{
// We force a retry but cancel the token to verify if the retry mechanism cancels inbetween
source.Cancel();
throw new WebException("", WebExceptionStatus.ConnectFailure);
}
return(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent("Response")
});
};
Mock <IDocumentClientInternal> mockDocumentClient = new Mock <IDocumentClientInternal>();
mockDocumentClient.Setup(client => client.ServiceEndpoint).Returns(new Uri("https://foo"));
GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockDocumentClient.Object, new ConnectionPolicy());
ISessionContainer sessionContainer = new SessionContainer(string.Empty);
DocumentClientEventSource eventSource = DocumentClientEventSource.Instance;
HttpMessageHandler messageHandler = new MockMessageHandler(sendFunc);
GatewayStoreModel storeModel = new GatewayStoreModel(
endpointManager,
sessionContainer,
ConsistencyLevel.Eventual,
eventSource,
null,
new HttpClient(messageHandler));
using (new ActivityScope(Guid.NewGuid()))
{
using (DocumentServiceRequest request =
DocumentServiceRequest.Create(
Documents.OperationType.Query,
Documents.ResourceType.Document,
new Uri("https://foo.com/dbs/db1/colls/coll1", UriKind.Absolute),
new MemoryStream(Encoding.UTF8.GetBytes("content1")),
AuthorizationTokenType.PrimaryMasterKey,
null))
{
await storeModel.ProcessMessageAsync(request, cancellationToken);
}
}
Assert.Fail();
}
}
public void EndpointFailureMockTest()
{
// Setup dummpy read locations for the database account
Collection <CosmosAccountLocation> readableLocations = new Collection <CosmosAccountLocation>();
CosmosAccountLocation writeLocation = new CosmosAccountLocation();
writeLocation.Name = "WriteLocation";
writeLocation.DatabaseAccountEndpoint = "https://writeendpoint.net/";
CosmosAccountLocation readLocation1 = new CosmosAccountLocation();
readLocation1.Name = "ReadLocation1";
readLocation1.DatabaseAccountEndpoint = "https://readendpoint1.net/";
CosmosAccountLocation readLocation2 = new CosmosAccountLocation();
readLocation2.Name = "ReadLocation2";
readLocation2.DatabaseAccountEndpoint = "https://readendpoint2.net/";
readableLocations.Add(writeLocation);
readableLocations.Add(readLocation1);
readableLocations.Add(readLocation2);
CosmosAccountSettings databaseAccount = new CosmosAccountSettings();
databaseAccount.ReadLocationsInternal = readableLocations;
//Setup mock owner "document client"
Mock <IDocumentClientInternal> mockOwner = new Mock <IDocumentClientInternal>();
mockOwner.Setup(owner => owner.ServiceEndpoint).Returns(new Uri("https://defaultendpoint.net/"));
mockOwner.Setup(owner => owner.GetDatabaseAccountInternalAsync(It.IsAny <Uri>(), It.IsAny <CancellationToken>())).ReturnsAsync(databaseAccount);
//Create connection policy and populate preferred locations
ConnectionPolicy connectionPolicy = new ConnectionPolicy();
connectionPolicy.PreferredLocations.Add("ReadLocation1");
connectionPolicy.PreferredLocations.Add("ReadLocation2");
GlobalEndpointManager globalEndpointManager = new GlobalEndpointManager(mockOwner.Object, connectionPolicy);
globalEndpointManager.RefreshLocationAsync(databaseAccount).Wait();
Assert.IsTrue(globalEndpointManager.ReadEndpoints[0] == new Uri(readLocation1.DatabaseAccountEndpoint));
//Mark each of the read locations as unavailable and validate that the read endpoint switches to the next preferred region / default endpoint.
globalEndpointManager.MarkEndpointUnavailableForRead(globalEndpointManager.ReadEndpoints[0]);
globalEndpointManager.RefreshLocationAsync(null).Wait();
Assert.IsTrue(globalEndpointManager.ReadEndpoints[0] == new Uri(readLocation2.DatabaseAccountEndpoint));
globalEndpointManager.MarkEndpointUnavailableForRead(globalEndpointManager.ReadEndpoints[0]);
globalEndpointManager.RefreshLocationAsync(null).Wait();
Assert.IsTrue(globalEndpointManager.ReadEndpoints[0] == globalEndpointManager.WriteEndpoints[0]);
//Sleep a second for the unavailable endpoint entry to expire and background refresh timer to kick in
Thread.Sleep(1000);
globalEndpointManager.RefreshLocationAsync(null).Wait();
Assert.IsTrue(globalEndpointManager.ReadEndpoints[0] == new Uri(readLocation1.DatabaseAccountEndpoint));
}
public async Task TestRetries()
{
int run = 0;
Func <HttpRequestMessage, Task <HttpResponseMessage> > sendFunc = async request =>
{
string content = await request.Content.ReadAsStringAsync();
Assert.AreEqual("content1", content);
if (run == 0)
{
run++;
throw new WebException("", WebExceptionStatus.ConnectFailure);
}
else
{
return(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent("Response")
});
}
};
Mock <IDocumentClientInternal> mockDocumentClient = new Mock <IDocumentClientInternal>();
mockDocumentClient.Setup(client => client.ServiceEndpoint).Returns(new Uri("https://foo"));
GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockDocumentClient.Object, new ConnectionPolicy());
ISessionContainer sessionContainer = new SessionContainer(string.Empty);
DocumentClientEventSource eventSource = DocumentClientEventSource.Instance;
HttpMessageHandler messageHandler = new MockMessageHandler(sendFunc);
GatewayStoreModel storeModel = new GatewayStoreModel(
endpointManager,
sessionContainer,
TimeSpan.FromSeconds(5),
ConsistencyLevel.Eventual,
eventSource,
null,
new UserAgentContainer(),
ApiType.None,
messageHandler);
using (new ActivityScope(Guid.NewGuid()))
{
using (DocumentServiceRequest request =
DocumentServiceRequest.Create(
Documents.OperationType.Query,
Documents.ResourceType.Document,
new Uri("https://foo.com/dbs/db1/colls/coll1", UriKind.Absolute),
new MemoryStream(Encoding.UTF8.GetBytes("content1")),
AuthorizationTokenType.PrimaryMasterKey,
null))
{
await storeModel.ProcessMessageAsync(request);
}
}
Assert.IsTrue(run > 0);
}
private GatewayStoreModel GetGatewayStoreModelForConsistencyTest()
{
Func <HttpRequestMessage, Task <HttpResponseMessage> > messageHandler = async request =>
{
String content = await request.Content.ReadAsStringAsync();
if (content.Equals("document"))
{
IEnumerable <string> sessionTokens = request.Headers.GetValues("x-ms-session-token");
string sessionToken = "";
foreach (string singleToken in sessionTokens)
{
sessionToken = singleToken;
break;
}
Assert.AreEqual(sessionToken, "0:9");
}
else
{
IEnumerable <string> enumerable;
Assert.IsFalse(request.Headers.TryGetValues("x-ms-session-token", out enumerable));
}
return(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent("Response")
});
};
Mock <IDocumentClientInternal> mockDocumentClient = new Mock <IDocumentClientInternal>();
mockDocumentClient.Setup(client => client.ServiceEndpoint).Returns(new Uri("https://foo"));
mockDocumentClient.Setup(client => client.ConsistencyLevel).Returns(ConsistencyLevel.Session);
GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockDocumentClient.Object, new ConnectionPolicy());
ConcurrentDictionary <string, ISessionToken> token = new ConcurrentDictionary <string, ISessionToken>();
token["0"] = new SimpleSessionToken(9);
ConcurrentDictionary <string, ConcurrentDictionary <string, ISessionToken> > sessionTokensNameBased = new ConcurrentDictionary <string, ConcurrentDictionary <string, ISessionToken> >();
sessionTokensNameBased["dbs/db1/colls/coll1"] = token;
ISessionContainer sessionContainer = new SessionContainer(string.Empty, null, sessionTokensNameBased);
DocumentClientEventSource eventSource = new DocumentClientEventSource();
HttpMessageHandler httpMessageHandler = new MockMessageHandler(messageHandler);
GatewayStoreModel storeModel = new GatewayStoreModel(
endpointManager,
sessionContainer,
TimeSpan.FromSeconds(50),
ConsistencyLevel.Session,
eventSource,
new UserAgentContainer(),
ApiType.None,
httpMessageHandler);
return(storeModel);
}
/// <summary>
/// Initialize the instance of the RetryPolicy class
/// </summary>
public RetryPolicy(
GlobalEndpointManager globalEndpointManager,
ConnectionPolicy connectionPolicy,
GlobalPartitionEndpointManager partitionKeyRangeLocationCache)
{
this.enableEndpointDiscovery = connectionPolicy.EnableEndpointDiscovery;
this.globalEndpointManager = globalEndpointManager;
this.retryOptions = connectionPolicy.RetryOptions;
this.partitionKeyRangeLocationCache = partitionKeyRangeLocationCache;
}
public async Task GatewayProcessMessageAsyncCancelsOnDeadline()
{
// Cancellation deadline is before Request timeout
using (CancellationTokenSource source = new CancellationTokenSource(TimeSpan.FromSeconds(2)))
{
CancellationToken cancellationToken = source.Token;
Func <HttpRequestMessage, Task <HttpResponseMessage> > sendFunc = async request =>
{
string content = await request.Content.ReadAsStringAsync();
Assert.AreEqual("content1", content);
// Wait until CancellationTokenSource deadline expires
Thread.Sleep(2000);
// Force retries
throw new WebException("", WebExceptionStatus.ConnectFailure);
};
Mock <IDocumentClientInternal> mockDocumentClient = new Mock <IDocumentClientInternal>();
mockDocumentClient.Setup(client => client.ServiceEndpoint).Returns(new Uri("https://foo"));
GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockDocumentClient.Object, new ConnectionPolicy());
ISessionContainer sessionContainer = new SessionContainer(string.Empty);
DocumentClientEventSource eventSource = DocumentClientEventSource.Instance;
HttpMessageHandler messageHandler = new MockMessageHandler(sendFunc);
GatewayStoreModel storeModel = new GatewayStoreModel(
endpointManager,
sessionContainer,
TimeSpan.FromSeconds(5),
ConsistencyLevel.Eventual,
eventSource,
null,
new UserAgentContainer(),
ApiType.None,
messageHandler);
using (new ActivityScope(Guid.NewGuid()))
{
using (DocumentServiceRequest request =
DocumentServiceRequest.Create(
Documents.OperationType.Query,
Documents.ResourceType.Document,
new Uri("https://foo.com/dbs/db1/colls/coll1", UriKind.Absolute),
new MemoryStream(Encoding.UTF8.GetBytes("content1")),
AuthorizationTokenType.PrimaryMasterKey,
null))
{
await storeModel.ProcessMessageAsync(request, cancellationToken);
}
}
Assert.Fail();
}
}
public async Task TestErrorResponsesProvideBody()
{
string testContent = "Content";
Func <HttpRequestMessage, Task <HttpResponseMessage> > sendFunc = request =>
{
return(Task.FromResult(new HttpResponseMessage(HttpStatusCode.Conflict)
{
Content = new StringContent(testContent)
}));
};
Mock <IDocumentClientInternal> mockDocumentClient = new Mock <IDocumentClientInternal>();
mockDocumentClient.Setup(client => client.ServiceEndpoint).Returns(new Uri("https://foo"));
GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockDocumentClient.Object, new ConnectionPolicy());
ISessionContainer sessionContainer = new SessionContainer(string.Empty);
DocumentClientEventSource eventSource = DocumentClientEventSource.Instance;
HttpMessageHandler messageHandler = new MockMessageHandler(sendFunc);
GatewayStoreModel storeModel = new GatewayStoreModel(
endpointManager,
sessionContainer,
TimeSpan.FromSeconds(5),
ConsistencyLevel.Eventual,
eventSource,
null,
new UserAgentContainer(),
ApiType.None,
messageHandler);
using (new ActivityScope(Guid.NewGuid()))
{
using (DocumentServiceRequest request =
DocumentServiceRequest.Create(
Documents.OperationType.Query,
Documents.ResourceType.Document,
new Uri("https://foo.com/dbs/db1/colls/coll1", UriKind.Absolute),
new MemoryStream(Encoding.UTF8.GetBytes("content1")),
AuthorizationTokenType.PrimaryMasterKey,
null))
{
request.UseStatusCodeForFailures = true;
request.UseStatusCodeFor429 = true;
DocumentServiceResponse response = await storeModel.ProcessMessageAsync(request);
Assert.IsNotNull(response.ResponseBody);
using (StreamReader reader = new StreamReader(response.ResponseBody))
{
Assert.AreEqual(testContent, await reader.ReadToEndAsync());
}
}
}
}
private async Task GatewayStoreModel_Exceptionless_NotUpdateSessionTokenOnKnownResponses(ResourceType resourceType, HttpStatusCode httpStatusCode, SubStatusCodes subStatusCode = SubStatusCodes.Unknown)
{
const string originalSessionToken = "0:1#100#1=20#2=5#3=30";
const string updatedSessionToken = "0:1#100#1=20#2=5#3=31";
Func <HttpRequestMessage, Task <HttpResponseMessage> > sendFunc = request =>
{
HttpResponseMessage response = new HttpResponseMessage(httpStatusCode);
response.Headers.Add(HttpConstants.HttpHeaders.SessionToken, updatedSessionToken);
response.Headers.Add(WFConstants.BackendHeaders.SubStatus, subStatusCode.ToString());
return(Task.FromResult(response));
};
Mock <IDocumentClientInternal> mockDocumentClient = new Mock <IDocumentClientInternal>();
mockDocumentClient.Setup(client => client.ServiceEndpoint).Returns(new Uri("https://foo"));
using GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockDocumentClient.Object, new ConnectionPolicy());
SessionContainer sessionContainer = new SessionContainer(string.Empty);
DocumentClientEventSource eventSource = DocumentClientEventSource.Instance;
HttpMessageHandler messageHandler = new MockMessageHandler(sendFunc);
using GatewayStoreModel storeModel = new GatewayStoreModel(
endpointManager,
sessionContainer,
ConsistencyLevel.Eventual,
eventSource,
null,
MockCosmosUtil.CreateCosmosHttpClient(() => new HttpClient(messageHandler)));
INameValueCollection headers = new StoreRequestNameValueCollection();
headers.Set(HttpConstants.HttpHeaders.ConsistencyLevel, ConsistencyLevel.Session.ToString());
headers.Set(HttpConstants.HttpHeaders.SessionToken, originalSessionToken);
headers.Set(WFConstants.BackendHeaders.PartitionKeyRangeId, "0");
using (new ActivityScope(Guid.NewGuid()))
{
using (DocumentServiceRequest request = DocumentServiceRequest.Create(
OperationType.Read,
resourceType,
"dbs/OVJwAA==/colls/OVJwAOcMtA0=/docs/OVJwAOcMtA0BAAAAAAAAAA==/",
AuthorizationTokenType.PrimaryMasterKey,
headers))
{
request.UseStatusCodeFor429 = true;
request.UseStatusCodeForFailures = true;
DocumentServiceResponse response = await storeModel.ProcessMessageAsync(request);
Assert.AreEqual(string.Empty, sessionContainer.GetSessionToken("dbs/OVJwAA==/colls/OVJwAOcMtA0="));
}
}
}
private GatewayStoreModel(
GlobalEndpointManager endpointManager,
ISessionContainer sessionContainer,
ConsistencyLevel defaultConsistencyLevel,
DocumentClientEventSource eventSource)
{
// CookieContainer is not really required, but is helpful in debugging.
this.cookieJar = new CookieContainer();
this.endpointManager = endpointManager;
this.sessionContainer = sessionContainer;
this.defaultConsistencyLevel = defaultConsistencyLevel;
this.eventSource = eventSource;
}
public ClientRetryPolicy(
GlobalEndpointManager globalEndpointManager,
bool enableEndpointDiscovery,
RetryOptions retryOptions)
{
this.throttlingRetry = new ResourceThrottleRetryPolicy(
retryOptions.MaxRetryAttemptsOnThrottledRequests,
retryOptions.MaxRetryWaitTimeInSeconds);
this.globalEndpointManager = globalEndpointManager;
this.failoverRetryCount = 0;
this.enableEndpointDiscovery = enableEndpointDiscovery;
this.sessionTokenRetryCount = 0;
this.canUseMultipleWriteLocations = false;
}
public void GlobalAddressResolverUpdateAsyncSynchronizationTest()
{
SynchronizationContext prevContext = SynchronizationContext.Current;
try
{
TestSynchronizationContext syncContext = new TestSynchronizationContext();
SynchronizationContext.SetSynchronizationContext(syncContext);
syncContext.Post(_ =>
{
UserAgentContainer container = new UserAgentContainer(clientId: 0);
FakeMessageHandler messageHandler = new FakeMessageHandler();
AccountProperties databaseAccount = new AccountProperties();
Mock <IDocumentClientInternal> mockDocumentClient = new Mock <IDocumentClientInternal>();
mockDocumentClient.Setup(owner => owner.ServiceEndpoint).Returns(new Uri("https://blabla.com/"));
mockDocumentClient.Setup(owner => owner.GetDatabaseAccountInternalAsync(It.IsAny <Uri>(), It.IsAny <CancellationToken>())).ReturnsAsync(databaseAccount);
GlobalEndpointManager globalEndpointManager = new GlobalEndpointManager(mockDocumentClient.Object, new ConnectionPolicy());
GlobalPartitionEndpointManager partitionKeyRangeLocationCache = new GlobalPartitionEndpointManagerCore(globalEndpointManager);
ConnectionPolicy connectionPolicy = new ConnectionPolicy
{
RequestTimeout = TimeSpan.FromSeconds(10)
};
GlobalAddressResolver globalAddressResolver = new GlobalAddressResolver(
endpointManager: globalEndpointManager,
partitionKeyRangeLocationCache: partitionKeyRangeLocationCache,
protocol: Documents.Client.Protocol.Tcp,
tokenProvider: this.mockTokenProvider.Object,
collectionCache: null,
routingMapProvider: null,
serviceConfigReader: this.mockServiceConfigReader.Object,
connectionPolicy: connectionPolicy,
httpClient: MockCosmosUtil.CreateCosmosHttpClient(() => new HttpClient(messageHandler)));
ConnectionStateListener connectionStateListener = new ConnectionStateListener(globalAddressResolver);
connectionStateListener.OnConnectionEvent(ConnectionEvent.ReadEof, DateTime.Now, new Documents.Rntbd.ServerKey(new Uri("https://endpoint.azure.com:4040/")));
}, state: null);
}
finally
{
SynchronizationContext.SetSynchronizationContext(prevContext);
}
}
public GatewayStoreModel(
GlobalEndpointManager endpointManager,
ISessionContainer sessionContainer,
TimeSpan requestTimeout,
ConsistencyLevel defaultConsistencyLevel,
DocumentClientEventSource eventSource,
JsonSerializerSettings serializerSettings,
UserAgentContainer userAgent,
ApiType apiType = ApiType.None,
HttpMessageHandler messageHandler = null)
{
// CookieContainer is not really required, but is helpful in debugging.
this.cookieJar = new CookieContainer();
this.endpointManager = endpointManager;
HttpClient httpClient = new HttpClient(messageHandler ?? new HttpClientHandler {
CookieContainer = this.cookieJar
});
this.sessionContainer = sessionContainer;
this.defaultConsistencyLevel = defaultConsistencyLevel;
// Use max of client specified and our own request timeout value when sending
// requests to gateway. Otherwise, we will have gateway's transient
// error hiding retries are of no use.
httpClient.Timeout = (requestTimeout > this.requestTimeout) ? requestTimeout : this.requestTimeout;
httpClient.DefaultRequestHeaders.CacheControl = new CacheControlHeaderValue {
NoCache = true
};
httpClient.AddUserAgentHeader(userAgent);
httpClient.AddApiTypeHeader(apiType);
// Set requested API version header that can be used for
// version enforcement.
httpClient.DefaultRequestHeaders.Add(HttpConstants.HttpHeaders.Version,
HttpConstants.Versions.CurrentVersion);
httpClient.DefaultRequestHeaders.Add(HttpConstants.HttpHeaders.Accept, RuntimeConstants.MediaTypes.Json);
this.eventSource = eventSource;
this.gatewayStoreClient = new GatewayStoreClient(
httpClient,
this.eventSource,
serializerSettings);
}
private static GatewayStoreModel MockGatewayStoreModel(Func <HttpRequestMessage, Task <HttpResponseMessage> > sendFunc)
{
Mock <IDocumentClientInternal> mockDocumentClient = new Mock <IDocumentClientInternal>();
mockDocumentClient.Setup(client => client.ServiceEndpoint).Returns(new Uri("https://foo"));
GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockDocumentClient.Object, new ConnectionPolicy());
ISessionContainer sessionContainer = new SessionContainer(string.Empty);
HttpMessageHandler messageHandler = new MockMessageHandler(sendFunc);
return(new GatewayStoreModel(
endpointManager,
sessionContainer,
Cosmos.ConsistencyLevel.Eventual,
new DocumentClientEventSource(),
new JsonSerializerSettings(),
MockCosmosUtil.CreateCosmosHttpClient(() => new HttpClient(messageHandler))));
}
public GatewayStoreModel(
GlobalEndpointManager endpointManager,
ISessionContainer sessionContainer,
ConsistencyLevel defaultConsistencyLevel,
DocumentClientEventSource eventSource,
JsonSerializerSettings serializerSettings,
HttpClient httpClient)
{
this.endpointManager = endpointManager;
this.sessionContainer = sessionContainer;
this.defaultConsistencyLevel = defaultConsistencyLevel;
this.eventSource = eventSource;
this.gatewayStoreClient = new GatewayStoreClient(
httpClient,
this.eventSource,
serializerSettings);
}
public async Task ClientRetryPolicyShouldCancel()
{
CancellationToken notCancelledToken = new CancellationToken();
CancellationToken cancelledToken = new CancellationToken(true);
Mock <IDocumentClientInternal> mockDocumentClient = new Mock <IDocumentClientInternal>();
mockDocumentClient.Setup(client => client.ServiceEndpoint).Returns(new Uri("https://foo"));
GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockDocumentClient.Object, new ConnectionPolicy());
ClientRetryPolicy clientRetryPolicy = new ClientRetryPolicy(endpointManager, true, new RetryOptions());
DocumentClientException ex = new DocumentClientException("", (HttpStatusCode)StatusCodes.TooManyRequests, SubStatusCodes.OwnerResourceNotFound);
ShouldRetryResult result = await clientRetryPolicy.ShouldRetryAsync(ex, notCancelledToken);
Assert.IsTrue(result.ShouldRetry);
await clientRetryPolicy.ShouldRetryAsync(ex, cancelledToken);
}
private static GatewayStoreModel MockGatewayStoreModel(Func <HttpRequestMessage, Task <HttpResponseMessage> > sendFunc)
{
Mock <IDocumentClientInternal> mockDocumentClient = new Mock <IDocumentClientInternal>();
mockDocumentClient.Setup(client => client.ServiceEndpoint).Returns(new Uri("https://foo"));
GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockDocumentClient.Object, new ConnectionPolicy());
ISessionContainer sessionContainer = new SessionContainer(string.Empty);
HttpMessageHandler messageHandler = new MockMessageHandler(sendFunc);
return(new GatewayStoreModel(
endpointManager,
sessionContainer,
TimeSpan.FromSeconds(5),
ConsistencyLevel.Eventual,
new DocumentClientEventSource(),
new UserAgentContainer(),
ApiType.None,
messageHandler));
}
private void Initialize(
bool useMultipleWriteLocations,
bool enableEndpointDiscovery,
bool isPreferredLocationsListEmpty)
{
this.databaseAccount = LocationCacheTests.CreateDatabaseAccount(useMultipleWriteLocations);
this.preferredLocations = isPreferredLocationsListEmpty ? new List <string>().AsReadOnly() : new List <string>()
{
"location1",
"location2",
"location3"
}.AsReadOnly();
this.cache = new LocationCache(
this.preferredLocations,
LocationCacheTests.DefaultEndpoint,
enableEndpointDiscovery,
10,
useMultipleWriteLocations);
this.cache.OnDatabaseAccountRead(this.databaseAccount);
this.mockedClient = new Mock <IDocumentClientInternal>();
mockedClient.Setup(owner => owner.ServiceEndpoint).Returns(LocationCacheTests.DefaultEndpoint);
mockedClient.Setup(owner => owner.GetDatabaseAccountInternalAsync(It.IsAny <Uri>(), It.IsAny <CancellationToken>())).ReturnsAsync(this.databaseAccount);
ConnectionPolicy connectionPolicy = new ConnectionPolicy()
{
EnableEndpointDiscovery = enableEndpointDiscovery,
UseMultipleWriteLocations = useMultipleWriteLocations,
};
foreach (string preferredLocation in this.preferredLocations)
{
connectionPolicy.PreferredLocations.Add(preferredLocation);
}
this.endpointManager = new GlobalEndpointManager(mockedClient.Object, connectionPolicy);
}
public GatewayStoreModel(
GlobalEndpointManager endpointManager,
ISessionContainer sessionContainer,
TimeSpan requestTimeout,
ConsistencyLevel defaultConsistencyLevel,
DocumentClientEventSource eventSource,
JsonSerializerSettings serializerSettings,
UserAgentContainer userAgent,
ApiType apiType,
HttpMessageHandler messageHandler)
: this(endpointManager,
sessionContainer,
defaultConsistencyLevel,
eventSource)
{
this.InitializeGatewayStoreClient(
requestTimeout,
serializerSettings,
userAgent,
apiType,
new HttpClient(messageHandler ?? new HttpClientHandler {
CookieContainer = this.cookieJar
}));
}
/// <summary>
/// Initialize the instance of the RetryPolicy class
/// </summary>
public RetryPolicy(GlobalEndpointManager globalEndpointManager, ConnectionPolicy connectionPolicy)
{
this.enableEndpointDiscovery = connectionPolicy.EnableEndpointDiscovery;
this.globalEndpointManager = globalEndpointManager;
this.retryOptions = connectionPolicy.RetryOptions;
}
private GatewayStoreModel GetGatewayStoreModelForConsistencyTest()
{
Func <HttpRequestMessage, Task <HttpResponseMessage> > messageHandler = async request =>
{
String content = await request.Content.ReadAsStringAsync();
if (content.Equals("document"))
{
IEnumerable <string> sessionTokens = request.Headers.GetValues("x-ms-session-token");
string sessionToken = "";
foreach (string singleToken in sessionTokens)
{
sessionToken = singleToken;
break;
}
Assert.AreEqual(sessionToken, "range_0:1#9#4=8#5=7");
}
else
{
IEnumerable <string> enumerable;
Assert.IsFalse(request.Headers.TryGetValues("x-ms-session-token", out enumerable));
}
return(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent("Response")
});
};
Mock <IDocumentClientInternal> mockDocumentClient = new Mock <IDocumentClientInternal>();
mockDocumentClient.Setup(client => client.ServiceEndpoint).Returns(new Uri("https://foo"));
mockDocumentClient.Setup(client => client.ConsistencyLevel).Returns(Documents.ConsistencyLevel.Session);
GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockDocumentClient.Object, new ConnectionPolicy());
SessionContainer sessionContainer = new SessionContainer(string.Empty);
sessionContainer.SetSessionToken(
ResourceId.NewDocumentCollectionId(42, 129).DocumentCollectionId.ToString(),
"dbs/db1/colls/coll1",
new StoreRequestNameValueCollection()
{
{ HttpConstants.HttpHeaders.SessionToken, "range_0:1#9#4=8#5=7" }
});
DocumentClientEventSource eventSource = DocumentClientEventSource.Instance;
HttpMessageHandler httpMessageHandler = new MockMessageHandler(messageHandler);
GatewayStoreModel storeModel = new GatewayStoreModel(
endpointManager,
sessionContainer,
ConsistencyLevel.Eventual,
eventSource,
null,
MockCosmosUtil.CreateCosmosHttpClient(() => new HttpClient(httpMessageHandler)));
ClientCollectionCache clientCollectionCache = new Mock <ClientCollectionCache>(new SessionContainer("testhost"), storeModel, null, null).Object;
PartitionKeyRangeCache partitionKeyRangeCache = new Mock <PartitionKeyRangeCache>(null, storeModel, clientCollectionCache).Object;
storeModel.SetCaches(partitionKeyRangeCache, clientCollectionCache);
return(storeModel);
}
public async Task EndpointFailureMockTest()
{
Environment.SetEnvironmentVariable("MinimumIntervalForNonForceRefreshLocationInMS", "100");
try
{
// Setup dummpy read locations for the database account
Collection <AccountRegion> readableLocations = new Collection <AccountRegion>();
AccountRegion writeLocation = new AccountRegion();
writeLocation.Name = "WriteLocation";
writeLocation.Endpoint = "https://writeendpoint.net/";
AccountRegion readLocation1 = new AccountRegion();
readLocation1.Name = "ReadLocation1";
readLocation1.Endpoint = "https://readendpoint1.net/";
AccountRegion readLocation2 = new AccountRegion();
readLocation2.Name = "ReadLocation2";
readLocation2.Endpoint = "https://readendpoint2.net/";
readableLocations.Add(writeLocation);
readableLocations.Add(readLocation1);
readableLocations.Add(readLocation2);
AccountProperties databaseAccount = new AccountProperties();
databaseAccount.ReadLocationsInternal = readableLocations;
//Setup mock owner "document client"
Mock <IDocumentClientInternal> mockOwner = new Mock <IDocumentClientInternal>();
mockOwner.Setup(owner => owner.ServiceEndpoint).Returns(new Uri("https://defaultendpoint.net/"));
int getAccountInfoCount = 0;
mockOwner.Setup(owner => owner.GetDatabaseAccountInternalAsync(It.IsAny <Uri>(), It.IsAny <CancellationToken>()))
.Callback(() => getAccountInfoCount++)
.ReturnsAsync(databaseAccount);
//Create connection policy and populate preferred locations
ConnectionPolicy connectionPolicy = new ConnectionPolicy();
connectionPolicy.PreferredLocations.Add("ReadLocation1");
connectionPolicy.PreferredLocations.Add("ReadLocation2");
using (GlobalEndpointManager globalEndpointManager = new GlobalEndpointManager(mockOwner.Object, connectionPolicy))
{
globalEndpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(databaseAccount);
Assert.AreEqual(globalEndpointManager.ReadEndpoints[0], new Uri(readLocation1.Endpoint));
//Mark each of the read locations as unavailable and validate that the read endpoint switches to the next preferred region / default endpoint.
globalEndpointManager.MarkEndpointUnavailableForRead(globalEndpointManager.ReadEndpoints[0]);
await globalEndpointManager.RefreshLocationAsync();
Assert.AreEqual(globalEndpointManager.ReadEndpoints[0], new Uri(readLocation2.Endpoint));
globalEndpointManager.MarkEndpointUnavailableForRead(globalEndpointManager.ReadEndpoints[0]);
await globalEndpointManager.RefreshLocationAsync();
Assert.AreEqual(globalEndpointManager.ReadEndpoints[0], globalEndpointManager.WriteEndpoints[0]);
getAccountInfoCount = 0;
//Sleep a second for the unavailable endpoint entry to expire and background refresh timer to kick in
await Task.Delay(TimeSpan.FromSeconds(3));
Assert.IsTrue(getAccountInfoCount > 0, "Callback is not working. There should be at least one call in this time frame.");
await globalEndpointManager.RefreshLocationAsync();
Assert.AreEqual(globalEndpointManager.ReadEndpoints[0], new Uri(readLocation1.Endpoint));
}
Assert.IsTrue(getAccountInfoCount > 0, "Callback is not working. There should be at least one call in this time frame.");
getAccountInfoCount = 0;
Thread.Sleep(TimeSpan.FromSeconds(3));
Assert.AreEqual(0, getAccountInfoCount, "There should be no more account calls after the GlobalEndpointManager is disposed");
}
finally
{
Environment.SetEnvironmentVariable("MinimumIntervalForNonForceRefreshLocationInMS", null);
}
}
public void ReadLocationRemoveAndAddMockTest()
{
string originalConfigValue = Environment.GetEnvironmentVariable("MinimumIntervalForNonForceRefreshLocationInMS");
Environment.SetEnvironmentVariable("MinimumIntervalForNonForceRefreshLocationInMS", "1000");
// Setup dummpy read locations for the database account
Collection <AccountRegion> readableLocations = new Collection <AccountRegion>();
AccountRegion writeLocation = new AccountRegion();
writeLocation.Name = "WriteLocation";
writeLocation.Endpoint = "https://writeendpoint.net/";
AccountRegion readLocation1 = new AccountRegion();
readLocation1.Name = "ReadLocation1";
readLocation1.Endpoint = "https://readendpoint1.net/";
AccountRegion readLocation2 = new AccountRegion();
readLocation2.Name = "ReadLocation2";
readLocation2.Endpoint = "https://readendpoint2.net/";
readableLocations.Add(writeLocation);
readableLocations.Add(readLocation1);
readableLocations.Add(readLocation2);
AccountProperties databaseAccount = new AccountProperties();
databaseAccount.ReadLocationsInternal = readableLocations;
//Setup mock owner "document client"
Mock <IDocumentClientInternal> mockOwner = new Mock <IDocumentClientInternal>();
mockOwner.Setup(owner => owner.ServiceEndpoint).Returns(new Uri("https://defaultendpoint.net/"));
mockOwner.Setup(owner => owner.GetDatabaseAccountInternalAsync(It.IsAny <Uri>(), It.IsAny <CancellationToken>())).ReturnsAsync(databaseAccount);
//Create connection policy and populate preferred locations
ConnectionPolicy connectionPolicy = new ConnectionPolicy();
connectionPolicy.PreferredLocations.Add("ReadLocation1");
connectionPolicy.PreferredLocations.Add("ReadLocation2");
GlobalEndpointManager globalEndpointManager = new GlobalEndpointManager(mockOwner.Object, connectionPolicy);
globalEndpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(databaseAccount);
Assert.AreEqual(globalEndpointManager.ReadEndpoints[0], new Uri(readLocation1.Endpoint));
//Remove location 1 from read locations and validate that the read endpoint switches to the next preferred location
readableLocations.Remove(readLocation1);
databaseAccount.ReadLocationsInternal = readableLocations;
globalEndpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(databaseAccount);
Assert.AreEqual(globalEndpointManager.ReadEndpoints[0], new Uri(readLocation2.Endpoint));
//Add location 1 back to read locations and validate that location 1 becomes the read endpoint again.
readableLocations.Add(readLocation1);
databaseAccount.ReadLocationsInternal = readableLocations;
//Sleep a bit for the refresh timer to kick in and rediscover location 1
Thread.Sleep(2000);
Assert.AreEqual(globalEndpointManager.ReadEndpoints[0], new Uri(readLocation1.Endpoint));
Environment.SetEnvironmentVariable("MinimumIntervalForNonForceRefreshLocationInMS", "1000");
}
public async Task GetDatabaseAccountFromAnyLocationsMockNegativeTestAsync()
{
Uri defaultEndpoint = new Uri("https://testfailover.documents-test.windows-int.net/");
int count = 0;
try
{
await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync(
defaultEndpoint : defaultEndpoint,
locations : new List <string>()
{
"westus",
"southeastasia",
"northcentralus"
},
getDatabaseAccountFn : (uri) =>
{
count++;
if (uri == defaultEndpoint)
{
throw new Microsoft.Azure.Documents.UnauthorizedException("Mock failed exception");
}
throw new Exception("This should never be hit since it should stop after the global endpoint hit the nonretriable exception");
});
Assert.Fail("Should throw the UnauthorizedException");
}
catch (Microsoft.Azure.Documents.UnauthorizedException)
{
Assert.AreEqual(1, count, "Only request should be made");
}
int countDelayRequests = 0;
count = 0;
try
{
await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync(
defaultEndpoint : defaultEndpoint,
locations : new List <string>()
{
"westus",
"southeastasia",
"northcentralus"
},
getDatabaseAccountFn : async(uri) =>
{
count++;
if (uri == defaultEndpoint)
{
countDelayRequests++;
await Task.Delay(TimeSpan.FromMinutes(1));
}
throw new Microsoft.Azure.Documents.UnauthorizedException("Mock failed exception");
});
Assert.Fail("Should throw the UnauthorizedException");
}
catch (Microsoft.Azure.Documents.UnauthorizedException)
{
Assert.IsTrue(count <= 3, "Global endpoint is 1, 2 tasks going to regions parallel");
Assert.AreEqual(2, count, "Only request should be made");
}
}
