public void AddressCacheMockTest()
{
// create a real document service request
DocumentServiceRequest entity = DocumentServiceRequest.Create(OperationType.Read, ResourceType.Document, AuthorizationTokenType.PrimaryMasterKey);
// setup mocks for address information
AddressInformation[] addressInformation = new AddressInformation[3];
for (int i = 0; i < 3; i++)
{
addressInformation[i] = new AddressInformation
{
PhysicalUri = "http://replica-" + i.ToString("G", CultureInfo.CurrentCulture)
};
}
// Address Selector is an internal sealed class that can't be mocked, but its dependency
// AddressCache can be mocked.
Mock <IAddressResolver> mockAddressCache = new Mock <IAddressResolver>();
mockAddressCache.Setup(
cache => cache.ResolveAsync(
It.IsAny <DocumentServiceRequest>(),
false /*forceRefresh*/,
new CancellationToken()))
.ReturnsAsync(new PartitionAddressInformation(addressInformation, null, null));
// validate that the mock works
PartitionAddressInformation addressInfo = mockAddressCache.Object.ResolveAsync(entity, false, new CancellationToken()).Result;
Assert.IsTrue(addressInfo.AllAddresses[0] == addressInformation[0]);
}
private static void SetTransportAddressUrisToUnhealthy(
PartitionAddressInformation stalePartitionAddressInformation,
Lazy <HashSet <TransportAddressUri> > failedEndpoints)
{
if (stalePartitionAddressInformation == null ||
failedEndpoints == null ||
!failedEndpoints.IsValueCreated)
{
return;
}
IReadOnlyList <TransportAddressUri> perProtocolPartitionAddressInformation = stalePartitionAddressInformation.Get(Protocol.Tcp)?.ReplicaTransportAddressUris;
if (perProtocolPartitionAddressInformation == null)
{
return;
}
foreach (TransportAddressUri failed in perProtocolPartitionAddressInformation)
{
if (failedEndpoints.Value.Contains(failed))
{
failed.SetUnhealthy();
}
}
}
private static void LogPartitionCacheRefresh(
IClientSideRequestStatistics clientSideRequestStatistics,
PartitionAddressInformation old,
PartitionAddressInformation updated)
{
if (clientSideRequestStatistics is ClientSideRequestStatisticsTraceDatum traceDatum)
{
traceDatum.RecordAddressCachRefreshContent(old, updated);
}
}
public async Task GatewayAddressCacheInNetworkRequestTestAsync()
{
FakeMessageHandler messageHandler = new FakeMessageHandler();
HttpClient httpClient = new HttpClient(messageHandler);
httpClient.Timeout = TimeSpan.FromSeconds(120);
GatewayAddressCache cache = new GatewayAddressCache(
new Uri(GatewayAddressCacheTests.DatabaseAccountApiEndpoint),
Documents.Client.Protocol.Https,
this.mockTokenProvider.Object,
this.mockServiceConfigReader.Object,
MockCosmosUtil.CreateCosmosHttpClient(() => httpClient),
suboptimalPartitionForceRefreshIntervalInSeconds: 2,
enableTcpConnectionEndpointRediscovery: true);
// No header should be present.
PartitionAddressInformation legacyRequest = await cache.TryGetAddressesAsync(
DocumentServiceRequest.Create(OperationType.Invalid, ResourceType.Address, AuthorizationTokenType.Invalid),
this.testPartitionKeyRangeIdentity,
this.serviceIdentity,
false,
CancellationToken.None);
Assert.IsFalse(legacyRequest.IsLocalRegion);
// Header indicates the request is from the same azure region.
messageHandler.Headers[HttpConstants.HttpHeaders.LocalRegionRequest] = "true";
PartitionAddressInformation inNetworkAddresses = await cache.TryGetAddressesAsync(
DocumentServiceRequest.Create(OperationType.Invalid, ResourceType.Address, AuthorizationTokenType.Invalid),
this.testPartitionKeyRangeIdentity,
this.serviceIdentity,
true,
CancellationToken.None);
Assert.IsTrue(inNetworkAddresses.IsLocalRegion);
// Header indicates the request is not from the same azure region.
messageHandler.Headers[HttpConstants.HttpHeaders.LocalRegionRequest] = "false";
PartitionAddressInformation outOfNetworkAddresses = await cache.TryGetAddressesAsync(
DocumentServiceRequest.Create(OperationType.Invalid, ResourceType.Address, AuthorizationTokenType.Invalid),
this.testPartitionKeyRangeIdentity,
this.serviceIdentity,
true,
CancellationToken.None);
Assert.IsFalse(outOfNetworkAddresses.IsLocalRegion);
}
public async Task <PartitionAddressInformation> ResolveAsync(
DocumentServiceRequest request,
bool forceRefresh,
CancellationToken cancellationToken)
{
IAddressResolver resolver = this.GetAddressResolver(request);
PartitionAddressInformation partitionAddressInformation = await resolver.ResolveAsync(request, forceRefresh, cancellationToken);
if (!this.partitionKeyRangeLocationCache.TryAddPartitionLevelLocationOverride(request))
{
return(partitionAddressInformation);
}
resolver = this.GetAddressResolver(request);
return(await resolver.ResolveAsync(request, forceRefresh, cancellationToken));
}
public ResolutionResult(
PartitionAddressInformation addresses,
ServiceIdentity serviceIdentity)
{
if (addresses == null)
{
throw new ArgumentNullException("addresses");
}
if (serviceIdentity == null)
{
throw new ArgumentNullException("serviceIdentity");
}
this.Addresses = addresses;
this.TargetServiceIdentity = serviceIdentity;
}
private async Task <DocumentServiceResponse> ProcessMessageForRead(DocumentClient client, DocumentServiceRequest request)
{
Uri requestEndpointUri = client.GlobalEndpointManager.ResolveServiceEndpoint(request);
Assert.IsNotNull(requestEndpointUri);
PartitionAddressInformation partitionAddress = await client.AddressResolver.ResolveAsync(request, false, new CancellationToken());
AddressInformation[] addressList = partitionAddress.AllAddresses;
Assert.IsNotNull(addressList);
Assert.IsTrue(addressList.Length > 0);
if (!request.ClearSessionTokenOnSessionReadFailure)
{
Assert.AreEqual(8081, requestEndpointUri.Port);
throw this.CreateReadSessionNotAvailableException();
}
Assert.AreEqual(8081, requestEndpointUri.Port);
return(await this.CreateEmptyDocumentServiceResponse());
}
public async Task TestGatewayAddressCacheUpdateOnConnectionResetAsync()
{
FakeMessageHandler messageHandler = new FakeMessageHandler();
HttpClient httpClient = new HttpClient(messageHandler);
httpClient.Timeout = TimeSpan.FromSeconds(120);
GatewayAddressCache cache = new GatewayAddressCache(
new Uri(GatewayAddressCacheTests.DatabaseAccountApiEndpoint),
Documents.Client.Protocol.Https,
this.mockTokenProvider.Object,
this.mockServiceConfigReader.Object,
MockCosmosUtil.CreateCosmosHttpClient(() => httpClient),
suboptimalPartitionForceRefreshIntervalInSeconds: 2,
enableTcpConnectionEndpointRediscovery: true);
PartitionAddressInformation addresses = cache.TryGetAddressesAsync(
DocumentServiceRequest.Create(OperationType.Invalid, ResourceType.Address, AuthorizationTokenType.Invalid),
this.testPartitionKeyRangeIdentity,
this.serviceIdentity,
false,
CancellationToken.None).Result;
Assert.IsNotNull(addresses.AllAddresses.Select(address => address.PhysicalUri == "https://blabla.com"));
// call updateAddress
await cache.TryRemoveAddressesAsync(new Documents.Rntbd.ServerKey(new Uri("https://blabla.com")), CancellationToken.None);
// check if the addresss is updated
addresses = cache.TryGetAddressesAsync(
DocumentServiceRequest.Create(OperationType.Invalid, ResourceType.Address, AuthorizationTokenType.Invalid),
this.testPartitionKeyRangeIdentity,
this.serviceIdentity,
false,
CancellationToken.None).Result;
Assert.IsNotNull(addresses.AllAddresses.Select(address => address.PhysicalUri == "https://blabla5.com"));
}
private async Task <ResolutionResult> TryResolveServerPartitionByPartitionKeyRangeIdAsync(
DocumentServiceRequest request,
ContainerProperties collection,
CollectionRoutingMap routingMap,
bool collectionCacheIsUpToDate,
bool routingMapCacheIsUpToDate,
bool forceRefreshPartitionAddresses,
CancellationToken cancellationToken)
{
PartitionKeyRange partitionKeyRange = routingMap.TryGetRangeByPartitionKeyRangeId(request.PartitionKeyRangeIdentity.PartitionKeyRangeId);
if (partitionKeyRange == null)
{
DefaultTrace.TraceInformation("Cannot resolve range '{0}'", request.PartitionKeyRangeIdentity.ToHeader());
return(this.HandleRangeAddressResolutionFailure(request, collectionCacheIsUpToDate, routingMapCacheIsUpToDate, routingMap));
}
ServiceIdentity identity = routingMap.TryGetInfoByPartitionKeyRangeId(request.PartitionKeyRangeIdentity.PartitionKeyRangeId);
PartitionAddressInformation addresses = await this.addressCache.TryGetAddressesAsync(
request,
new PartitionKeyRangeIdentity(collection.ResourceId, request.PartitionKeyRangeIdentity.PartitionKeyRangeId),
identity,
forceRefreshPartitionAddresses,
cancellationToken);
if (addresses == null)
{
DefaultTrace.TraceInformation("Cannot resolve addresses for range '{0}'", request.PartitionKeyRangeIdentity.ToHeader());
return(this.HandleRangeAddressResolutionFailure(request, collectionCacheIsUpToDate, routingMapCacheIsUpToDate, routingMap));
}
return(new ResolutionResult(partitionKeyRange, addresses, identity));
}
private async Task <ResolutionResult> TryResolveServerPartitionAsync(
DocumentServiceRequest request,
ContainerProperties collection,
CollectionRoutingMap routingMap,
bool collectionCacheIsUptodate,
bool collectionRoutingMapCacheIsUptodate,
bool forceRefreshPartitionAddresses,
CancellationToken cancellationToken)
{
// Check if this request partitionkeyrange-aware routing logic. We cannot retry here in this case
// and need to bubble up errors.
if (request.PartitionKeyRangeIdentity != null)
{
return(await this.TryResolveServerPartitionByPartitionKeyRangeIdAsync(
request,
collection,
routingMap,
collectionCacheIsUptodate,
collectionRoutingMapCacheIsUptodate,
forceRefreshPartitionAddresses,
cancellationToken));
}
if (!request.ResourceType.IsPartitioned() &&
!(request.ResourceType == ResourceType.StoredProcedure && request.OperationType == OperationType.ExecuteJavaScript) &&
// Collection head is sent internally for strong consistency given routing hints from original requst, which is for partitioned resource.
!(request.ResourceType == ResourceType.Collection && request.OperationType == OperationType.Head))
{
DefaultTrace.TraceCritical(
"Shouldn't come here for non partitioned resources. resourceType : {0}, operationtype:{1}, resourceaddress:{2}",
request.ResourceType,
request.OperationType,
request.ResourceAddress);
throw new InternalServerErrorException(RMResources.InternalServerError)
{
ResourceAddress = request.ResourceAddress
};
}
PartitionKeyRange range;
string partitionKeyString = request.Headers[HttpConstants.HttpHeaders.PartitionKey];
object effectivePartitionKeyStringObject = null;
if (partitionKeyString != null)
{
range = this.TryResolveServerPartitionByPartitionKey(
request,
partitionKeyString,
collectionCacheIsUptodate,
collection,
routingMap);
}
else if (request.Properties != null && request.Properties.TryGetValue(
WFConstants.BackendHeaders.EffectivePartitionKeyString,
out effectivePartitionKeyStringObject))
{
// Allow EPK only for partitioned collection (excluding migrated fixed collections)
if (!collection.HasPartitionKey || collection.PartitionKey.IsSystemKey.GetValueOrDefault(false))
{
throw new ArgumentOutOfRangeException(nameof(collection));
}
string effectivePartitionKeyString = effectivePartitionKeyStringObject as string;
if (string.IsNullOrEmpty(effectivePartitionKeyString))
{
throw new ArgumentOutOfRangeException(nameof(effectivePartitionKeyString));
}
range = routingMap.GetRangeByEffectivePartitionKey(effectivePartitionKeyString);
}
else
{
range = this.TryResolveSinglePartitionCollection(request, routingMap, collectionCacheIsUptodate);
}
if (range == null)
{
// Collection cache or routing map cache is potentially outdated. Return null -
// upper logic will refresh cache and retry.
return(null);
}
ServiceIdentity serviceIdentity = routingMap.TryGetInfoByPartitionKeyRangeId(range.Id);
PartitionAddressInformation addresses = await this.addressCache.TryGetAddressesAsync(
request,
new PartitionKeyRangeIdentity(collection.ResourceId, range.Id),
serviceIdentity,
forceRefreshPartitionAddresses,
cancellationToken);
if (addresses == null)
{
DefaultTrace.TraceVerbose(
"Could not resolve addresses for identity {0}/{1}. Potentially collection cache or routing map cache is outdated. Return null - upper logic will refresh and retry. ",
new PartitionKeyRangeIdentity(collection.ResourceId, range.Id),
serviceIdentity);
return(null);
}
return(new ResolutionResult(range, addresses, serviceIdentity));
}
/// <summary>
/// Resolves the endpoint of the partition for the given request
/// </summary>
/// <param name="request">Request for which the partition endpoint resolution is to be performed</param>
/// <param name="forceRefreshPartitionAddresses">Force refresh the partition's endpoint</param>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>An instance of <see cref="ResolutionResult"/>.</returns>
private async Task <ResolutionResult> ResolveAddressesAndIdentityAsync(
DocumentServiceRequest request,
bool forceRefreshPartitionAddresses,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
if (request.ServiceIdentity != null)
{
// In this case we don't populate request.RequestContext.ResolvedPartitionKeyRangeId,
// which is needed for session token.
// The assumption is that:
// 1. Master requests never use session consistency.
// 2. Service requests (like collection create etc.) don't use session consistency.
// 3. Requests which target specific partition of an existing collection will use x-ms-documentdb-partitionkeyrangeid header
// to send request to specific partition and will not set request.ServiceIdentity
ServiceIdentity identity = request.ServiceIdentity;
PartitionAddressInformation addresses = await this.addressCache.TryGetAddressesAsync(request, null, identity, forceRefreshPartitionAddresses, cancellationToken);
if (addresses == null)
{
DefaultTrace.TraceInformation("Could not get addresses for explicitly specified ServiceIdentity {0}", identity);
throw new NotFoundException()
{
ResourceAddress = request.ResourceAddress
};
}
return(new ResolutionResult(addresses, identity));
}
if (ReplicatedResourceClient.IsReadingFromMaster(request.ResourceType, request.OperationType) && request.PartitionKeyRangeIdentity == null)
{
DefaultTrace.TraceInformation("Resolving Master service address, forceMasterRefresh: {0}, currentMaster: {1}",
request.ForceMasterRefresh,
this.masterServiceIdentityProvider?.MasterServiceIdentity);
// Client implementation, GlobalAddressResolver passes in a null IMasterServiceIdentityProvider, because it doesn't actually use the serviceIdentity
// in the addressCache.TryGetAddresses method. In GatewayAddressCache.cs, the master address is resolved by making a call to Gateway AddressFeed,
// not using the serviceIdentity that is passed in
if (request.ForceMasterRefresh && this.masterServiceIdentityProvider != null)
{
ServiceIdentity previousMasterService = this.masterServiceIdentityProvider.MasterServiceIdentity;
await this.masterServiceIdentityProvider.RefreshAsync(previousMasterService, cancellationToken);
}
ServiceIdentity serviceIdentity = this.masterServiceIdentityProvider?.MasterServiceIdentity;
PartitionKeyRangeIdentity partitionKeyRangeIdentity = this.masterPartitionKeyRangeIdentity;
PartitionAddressInformation addresses = await this.addressCache.TryGetAddressesAsync(
request,
partitionKeyRangeIdentity,
serviceIdentity,
forceRefreshPartitionAddresses,
cancellationToken);
if (addresses == null)
{
// This shouldn't really happen.
DefaultTrace.TraceCritical("Could not get addresses for master partition {0}", serviceIdentity);
throw new NotFoundException()
{
ResourceAddress = request.ResourceAddress
};
}
PartitionKeyRange partitionKeyRange = new PartitionKeyRange {
Id = PartitionKeyRange.MasterPartitionKeyRangeId
};
return(new ResolutionResult(partitionKeyRange, addresses, serviceIdentity));
}
bool collectionCacheIsUptoDate = !request.IsNameBased ||
(request.PartitionKeyRangeIdentity != null && request.PartitionKeyRangeIdentity.CollectionRid != null);
bool collectionRoutingMapCacheIsUptoDate = false;
ContainerProperties collection = await this.collectionCache.ResolveCollectionAsync(request, cancellationToken);
CollectionRoutingMap routingMap = await this.collectionRoutingMapCache.TryLookupAsync(
collection.ResourceId, null, request, cancellationToken);
if (routingMap != null && request.ForceCollectionRoutingMapRefresh)
{
DefaultTrace.TraceInformation(
"AddressResolver.ResolveAddressesAndIdentityAsync ForceCollectionRoutingMapRefresh collection.ResourceId = {0}",
collection.ResourceId);
routingMap = await this.collectionRoutingMapCache.TryLookupAsync(collection.ResourceId, routingMap, request, cancellationToken);
}
if (request.ForcePartitionKeyRangeRefresh)
{
collectionRoutingMapCacheIsUptoDate = true;
request.ForcePartitionKeyRangeRefresh = false;
if (routingMap != null)
{
routingMap = await this.collectionRoutingMapCache.TryLookupAsync(collection.ResourceId, routingMap, request, cancellationToken);
}
}
if (routingMap == null && !collectionCacheIsUptoDate)
{
// Routing map was not found by resolved collection rid. Maybe collection rid is outdated.
// Refresh collection cache and reresolve routing map.
request.ForceNameCacheRefresh = true;
collectionCacheIsUptoDate = true;
collectionRoutingMapCacheIsUptoDate = false;
collection = await this.collectionCache.ResolveCollectionAsync(request, cancellationToken);
routingMap = await this.collectionRoutingMapCache.TryLookupAsync(
collection.ResourceId,
previousValue : null,
request : request,
cancellationToken : cancellationToken);
}
AddressResolver.EnsureRoutingMapPresent(request, routingMap, collection);
// At this point we have both collection and routingMap.
ResolutionResult result = await this.TryResolveServerPartitionAsync(
request,
collection,
routingMap,
collectionCacheIsUptoDate,
collectionRoutingMapCacheIsUptodate : collectionRoutingMapCacheIsUptoDate,
forceRefreshPartitionAddresses : forceRefreshPartitionAddresses,
cancellationToken : cancellationToken);
if (result == null)
{
// Couldn't resolve server partition or its addresses.
// Either collection cache is outdated or routing map cache is outdated.
if (!collectionCacheIsUptoDate)
{
request.ForceNameCacheRefresh = true;
collectionCacheIsUptoDate = true;
collection = await this.collectionCache.ResolveCollectionAsync(request, cancellationToken);
if (collection.ResourceId != routingMap.CollectionUniqueId)
{
// Collection cache was stale. We resolved to new Rid. routing map cache is potentially stale
// for this new collection rid. Mark it as such.
collectionRoutingMapCacheIsUptoDate = false;
routingMap = await this.collectionRoutingMapCache.TryLookupAsync(
collection.ResourceId,
previousValue : null,
request : request,
cancellationToken : cancellationToken);
}
}
if (!collectionRoutingMapCacheIsUptoDate)
{
collectionRoutingMapCacheIsUptoDate = true;
routingMap = await this.collectionRoutingMapCache.TryLookupAsync(
collection.ResourceId,
previousValue : routingMap,
request : request,
cancellationToken : cancellationToken);
}
AddressResolver.EnsureRoutingMapPresent(request, routingMap, collection);
result = await this.TryResolveServerPartitionAsync(
request,
collection,
routingMap,
collectionCacheIsUptodate : true,
collectionRoutingMapCacheIsUptodate : true,
forceRefreshPartitionAddresses : forceRefreshPartitionAddresses,
cancellationToken : cancellationToken);
}
if (result == null)
{
DefaultTrace.TraceInformation("Couldn't route partitionkeyrange-oblivious request after retry/cache refresh. Collection doesn't exist.");
// At this point collection cache and routing map caches are refreshed.
// The only reason we will get here is if collection doesn't exist.
// Case when partitionkeyrange doesn't exist is handled in the corresponding method.
throw new NotFoundException()
{
ResourceAddress = request.ResourceAddress
};
}
if (request.IsNameBased)
{
// Append collection rid.
// If we resolved collection rid incorrectly because of outdated cache, this can lead
// to incorrect routing decisions. But backend will validate collection rid and throw
// InvalidPartitionException if we reach wrong collection.
// Also this header will be used by backend to inject collection rid into metrics for
// throttled requests.
request.Headers[WFConstants.BackendHeaders.CollectionRid] = collection.ResourceId;
}
return(result);
}
public async Task <PartitionAddressInformation> TryGetAddresses(
DocumentServiceRequest request,
PartitionKeyRangeIdentity partitionKeyRangeIdentity,
ServiceIdentity serviceIdentity,
bool forceRefreshPartitionAddresses,
CancellationToken cancellationToken)
{
if (request == null)
{
throw new ArgumentNullException("request");
}
if (partitionKeyRangeIdentity == null)
{
throw new ArgumentNullException("partitionKeyRangeIdentity");
}
try
{
if (partitionKeyRangeIdentity.PartitionKeyRangeId == PartitionKeyRange.MasterPartitionKeyRangeId)
{
return((await this.ResolveMasterAsync(request, forceRefreshPartitionAddresses)).Item2);
}
DateTime suboptimalServerPartitionTimestamp;
if (this.suboptimalServerPartitionTimestamps.TryGetValue(partitionKeyRangeIdentity, out suboptimalServerPartitionTimestamp))
{
bool forceRefreshDueToSuboptimalPartitionReplicaSet =
DateTime.UtcNow.Subtract(suboptimalServerPartitionTimestamp) > TimeSpan.FromSeconds(this.suboptimalPartitionForceRefreshIntervalInSeconds);
if (forceRefreshDueToSuboptimalPartitionReplicaSet && this.suboptimalServerPartitionTimestamps.TryUpdate(partitionKeyRangeIdentity, DateTime.MaxValue, suboptimalServerPartitionTimestamp))
{
forceRefreshPartitionAddresses = true;
}
}
if (forceRefreshPartitionAddresses || request.ForceCollectionRoutingMapRefresh)
{
this.serverPartitionAddressCache.Refresh(
partitionKeyRangeIdentity,
async() => await this.GetAddressesForRangeId(
request,
partitionKeyRangeIdentity.CollectionRid,
partitionKeyRangeIdentity.PartitionKeyRangeId,
forceRefresh: forceRefreshPartitionAddresses),
cancellationToken);
DateTime ignoreDateTime;
this.suboptimalServerPartitionTimestamps.TryRemove(partitionKeyRangeIdentity, out ignoreDateTime);
}
PartitionAddressInformation addresses = await this.serverPartitionAddressCache.GetAsync(
partitionKeyRangeIdentity,
null,
async() => await this.GetAddressesForRangeId(
request,
partitionKeyRangeIdentity.CollectionRid,
partitionKeyRangeIdentity.PartitionKeyRangeId,
forceRefresh: false),
cancellationToken);
int targetReplicaSetSize = this.serviceConfigReader.UserReplicationPolicy.MaxReplicaSetSize;
if (addresses.AllAddresses.Count() < targetReplicaSetSize)
{
this.suboptimalServerPartitionTimestamps.TryAdd(partitionKeyRangeIdentity, DateTime.UtcNow);
}
return(addresses);
}
catch (DocumentClientException ex)
{
if ((ex.StatusCode == HttpStatusCode.NotFound) ||
(ex.StatusCode == HttpStatusCode.Gone && ex.GetSubStatus() == SubStatusCodes.PartitionKeyRangeGone))
{
//remove from suboptimal cache in case the the collection+pKeyRangeId combo is gone.
DateTime ignoreDateTime;
this.suboptimalServerPartitionTimestamps.TryRemove(partitionKeyRangeIdentity, out ignoreDateTime);
return(null);
}
throw;
}
catch (Exception)
{
if (forceRefreshPartitionAddresses)
{
DateTime ignoreDateTime;
this.suboptimalServerPartitionTimestamps.TryRemove(partitionKeyRangeIdentity, out ignoreDateTime);
}
throw;
}
}
public async Task TestGatewayAddressCacheAvoidCacheRefresWhenAlreadyUpdatedAsync()
{
Mock <IHttpHandler> mockHttpHandler = new Mock <IHttpHandler>(MockBehavior.Strict);
string oldAddress = "rntbd://dummytenant.documents.azure.com:14003/apps/APPGUID/services/SERVICEGUID/partitions/PARTITIONGUID/replicas/4s";
string newAddress = "rntbd://dummytenant.documents.azure.com:14003/apps/APPGUID/services/SERVICEGUID/partitions/PARTITIONGUID/replicas/5s";
mockHttpHandler.SetupSequence(x => x.SendAsync(
It.IsAny <HttpRequestMessage>(),
It.IsAny <CancellationToken>()))
.Returns(MockCosmosUtil.CreateHttpResponseOfAddresses(new List <string>()
{
"rntbd://dummytenant.documents.azure.com:14003/apps/APPGUID/services/SERVICEGUID/partitions/PARTITIONGUID/replicas/1p",
"rntbd://dummytenant.documents.azure.com:14003/apps/APPGUID/services/SERVICEGUID/partitions/PARTITIONGUID/replicas/2s",
"rntbd://dummytenant.documents.azure.com:14003/apps/APPGUID/services/SERVICEGUID/partitions/PARTITIONGUID/replicas/3s",
oldAddress,
}))
.Returns(MockCosmosUtil.CreateHttpResponseOfAddresses(new List <string>()
{
"rntbd://dummytenant.documents.azure.com:14003/apps/APPGUID/services/SERVICEGUID/partitions/PARTITIONGUID/replicas/1p",
"rntbd://dummytenant.documents.azure.com:14003/apps/APPGUID/services/SERVICEGUID/partitions/PARTITIONGUID/replicas/2s",
"rntbd://dummytenant.documents.azure.com:14003/apps/APPGUID/services/SERVICEGUID/partitions/PARTITIONGUID/replicas/3s",
newAddress,
}));
HttpClient httpClient = new HttpClient(new HttpHandlerHelper(mockHttpHandler.Object));
GatewayAddressCache cache = new GatewayAddressCache(
new Uri(GatewayAddressCacheTests.DatabaseAccountApiEndpoint),
Documents.Client.Protocol.Tcp,
this.mockTokenProvider.Object,
this.mockServiceConfigReader.Object,
MockCosmosUtil.CreateCosmosHttpClient(() => httpClient),
suboptimalPartitionForceRefreshIntervalInSeconds: 2,
enableTcpConnectionEndpointRediscovery: true);
DocumentServiceRequest request1 = DocumentServiceRequest.Create(OperationType.Invalid, ResourceType.Address, AuthorizationTokenType.Invalid);
DocumentServiceRequest request2 = DocumentServiceRequest.Create(OperationType.Invalid, ResourceType.Address, AuthorizationTokenType.Invalid);
PartitionAddressInformation request1Addresses = await cache.TryGetAddressesAsync(
request : request1,
partitionKeyRangeIdentity : this.testPartitionKeyRangeIdentity,
serviceIdentity : this.serviceIdentity,
forceRefreshPartitionAddresses : false,
cancellationToken : CancellationToken.None);
PartitionAddressInformation request2Addresses = await cache.TryGetAddressesAsync(
request : request2,
partitionKeyRangeIdentity : this.testPartitionKeyRangeIdentity,
serviceIdentity : this.serviceIdentity,
forceRefreshPartitionAddresses : false,
cancellationToken : CancellationToken.None);
Assert.AreEqual(request1Addresses, request2Addresses);
Assert.AreEqual(4, request1Addresses.AllAddresses.Count());
Assert.AreEqual(1, request1Addresses.AllAddresses.Count(x => x.PhysicalUri == oldAddress));
Assert.AreEqual(0, request1Addresses.AllAddresses.Count(x => x.PhysicalUri == newAddress));
// check if the addresss is updated
request1Addresses = await cache.TryGetAddressesAsync(
request : request1,
partitionKeyRangeIdentity : this.testPartitionKeyRangeIdentity,
serviceIdentity : this.serviceIdentity,
forceRefreshPartitionAddresses : true,
cancellationToken : CancellationToken.None);
// Even though force refresh is true it will just use the new cache
// value rather than doing a gateway call to do another refresh since the value
// already changed from the last cache access
request2Addresses = await cache.TryGetAddressesAsync(
request : request2,
partitionKeyRangeIdentity : this.testPartitionKeyRangeIdentity,
serviceIdentity : this.serviceIdentity,
forceRefreshPartitionAddresses : true,
cancellationToken : CancellationToken.None);
Assert.AreEqual(request1Addresses, request2Addresses);
Assert.AreEqual(4, request1Addresses.AllAddresses.Count());
Assert.AreEqual(0, request1Addresses.AllAddresses.Count(x => x.PhysicalUri == oldAddress));
Assert.AreEqual(1, request1Addresses.AllAddresses.Count(x => x.PhysicalUri == newAddress));
mockHttpHandler.VerifyAll();
}
public void GlobalStrongConsistencyMockTest()
{
// create a real document service request (with auth token level = god)
DocumentServiceRequest entity = DocumentServiceRequest.Create(OperationType.Read, ResourceType.Document, AuthorizationTokenType.SystemAll);
// set request charge tracker - this is referenced in store reader (ReadMultipleReplicaAsync)
DocumentServiceRequestContext requestContext = new DocumentServiceRequestContext
{
RequestChargeTracker = new RequestChargeTracker()
};
entity.RequestContext = requestContext;
// set a dummy resource id on the request.
entity.ResourceId = "1-MxAPlgMgA=";
// set consistency level on the request to Bounded Staleness
entity.Headers[HttpConstants.HttpHeaders.ConsistencyLevel] = ConsistencyLevel.BoundedStaleness.ToString();
// also setup timeout helper, used in store reader
entity.RequestContext.TimeoutHelper = new TimeoutHelper(new TimeSpan(2, 2, 2));
// when the store reader throws Invalid Partition exception, the higher layer should
// clear this target identity.
entity.RequestContext.TargetIdentity = new ServiceIdentity("dummyTargetIdentity1", new Uri("http://dummyTargetIdentity1"), false);
entity.RequestContext.ResolvedPartitionKeyRange = new PartitionKeyRange();
AddressInformation[] addressInformation = this.GetMockAddressInformationDuringUpgrade();
Mock <IAddressResolver> mockAddressCache = this.GetMockAddressCache(addressInformation);
// validate that the mock works
PartitionAddressInformation partitionAddressInformation = mockAddressCache.Object.ResolveAsync(entity, false, new CancellationToken()).Result;
IReadOnlyList <AddressInformation> addressInfo = partitionAddressInformation.AllAddresses;
Assert.IsTrue(addressInfo[0] == addressInformation[0]);
AddressSelector addressSelector = new AddressSelector(mockAddressCache.Object, Protocol.Tcp);
Tuple <Uri, AddressCacheToken> primaryAddress = addressSelector.ResolvePrimaryUriAsync(entity, false /*forceAddressRefresh*/).Result;
// check if the address return from Address Selector matches the original address info
Assert.IsTrue(primaryAddress.Item1.Equals(addressInformation[0].PhysicalUri));
// Quorum Met scenario
{
// get mock transport client that returns a sequence of responses to simulate upgrade
TransportClient mockTransportClient = this.GetMockTransportClientForGlobalStrongReads(addressInformation, ReadQuorumResultKind.QuorumMet);
// create a real session container - we don't need session for this test anyway
ISessionContainer sessionContainer = new SessionContainer(string.Empty);
ConnectionStateListener connectionStateListener = new ConnectionStateListener(null);
// create store reader with mock transport client, real address selector (that has mock address cache), and real session container
StoreReader storeReader =
new StoreReader(mockTransportClient,
addressSelector,
sessionContainer,
connectionStateListener);
Mock <IAuthorizationTokenProvider> mockAuthorizationTokenProvider = new Mock <IAuthorizationTokenProvider>();
mockAuthorizationTokenProvider.Setup(provider => provider.AddSystemAuthorizationHeaderAsync(
It.IsAny <DocumentServiceRequest>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>()))
.Returns(Task.FromResult(0));
// setup max replica set size on the config reader
ReplicationPolicy replicationPolicy = new ReplicationPolicy
{
MaxReplicaSetSize = 4
};
Mock <IServiceConfigurationReader> mockServiceConfigReader = new Mock <IServiceConfigurationReader>();
mockServiceConfigReader.SetupGet(x => x.UserReplicationPolicy).Returns(replicationPolicy);
QuorumReader reader =
new QuorumReader(mockTransportClient, addressSelector, storeReader, mockServiceConfigReader.Object, mockAuthorizationTokenProvider.Object);
entity.RequestContext.OriginalRequestConsistencyLevel = Documents.ConsistencyLevel.Strong;
entity.RequestContext.ClientRequestStatistics = new ClientSideRequestStatistics();
StoreResponse result = reader.ReadStrongAsync(entity, 2, ReadMode.Strong).Result;
Assert.IsTrue(result.LSN == 100);
result.TryGetHeaderValue(WFConstants.BackendHeaders.GlobalCommittedLSN, out string globalCommitedLSN);
long nGlobalCommitedLSN = long.Parse(globalCommitedLSN, CultureInfo.InvariantCulture);
Assert.IsTrue(nGlobalCommitedLSN == 90);
}
// Quorum Selected scenario
{
// get mock transport client that returns a sequence of responses to simulate upgrade
TransportClient mockTransportClient = this.GetMockTransportClientForGlobalStrongReads(addressInformation, ReadQuorumResultKind.QuorumSelected);
// create a real session container - we don't need session for this test anyway
ISessionContainer sessionContainer = new SessionContainer(string.Empty);
ConnectionStateListener connectionStateListener = new ConnectionStateListener(null);
// create store reader with mock transport client, real address selector (that has mock address cache), and real session container
StoreReader storeReader =
new StoreReader(mockTransportClient,
addressSelector,
sessionContainer,
connectionStateListener);
Mock <IAuthorizationTokenProvider> mockAuthorizationTokenProvider = new Mock <IAuthorizationTokenProvider>();
mockAuthorizationTokenProvider.Setup(provider => provider.AddSystemAuthorizationHeaderAsync(
It.IsAny <DocumentServiceRequest>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>()))
.Returns(Task.FromResult(0));
// setup max replica set size on the config reader
ReplicationPolicy replicationPolicy = new ReplicationPolicy
{
MaxReplicaSetSize = 4
};
Mock <IServiceConfigurationReader> mockServiceConfigReader = new Mock <IServiceConfigurationReader>();
mockServiceConfigReader.SetupGet(x => x.UserReplicationPolicy).Returns(replicationPolicy);
QuorumReader reader =
new QuorumReader(mockTransportClient, addressSelector, storeReader, mockServiceConfigReader.Object, mockAuthorizationTokenProvider.Object);
entity.RequestContext.OriginalRequestConsistencyLevel = Documents.ConsistencyLevel.Strong;
entity.RequestContext.ClientRequestStatistics = new ClientSideRequestStatistics();
entity.RequestContext.QuorumSelectedLSN = -1;
entity.RequestContext.GlobalCommittedSelectedLSN = -1;
try
{
StoreResponse result = reader.ReadStrongAsync(entity, 2, ReadMode.Strong).Result;
Assert.IsTrue(false);
}
catch (AggregateException ex)
{
if (ex.InnerException is GoneException)
{
DefaultTrace.TraceInformation("Gone exception expected!");
}
}
Assert.IsTrue(entity.RequestContext.QuorumSelectedLSN == 100);
Assert.IsTrue(entity.RequestContext.GlobalCommittedSelectedLSN == 100);
}
// Quorum not met scenario
{
// get mock transport client that returns a sequence of responses to simulate upgrade
TransportClient mockTransportClient = this.GetMockTransportClientForGlobalStrongReads(addressInformation, ReadQuorumResultKind.QuorumNotSelected);
// create a real session container - we don't need session for this test anyway
ISessionContainer sessionContainer = new SessionContainer(string.Empty);
ConnectionStateListener connectionStateListener = new ConnectionStateListener(null);
// create store reader with mock transport client, real address selector (that has mock address cache), and real session container
StoreReader storeReader =
new StoreReader(mockTransportClient,
addressSelector,
sessionContainer,
connectionStateListener);
Mock <IAuthorizationTokenProvider> mockAuthorizationTokenProvider = new Mock <IAuthorizationTokenProvider>();
mockAuthorizationTokenProvider.Setup(provider => provider.AddSystemAuthorizationHeaderAsync(
It.IsAny <DocumentServiceRequest>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>()))
.Returns(Task.FromResult(0));
// setup max replica set size on the config reader
ReplicationPolicy replicationPolicy = new ReplicationPolicy
{
MaxReplicaSetSize = 4
};
Mock <IServiceConfigurationReader> mockServiceConfigReader = new Mock <IServiceConfigurationReader>();
mockServiceConfigReader.SetupGet(x => x.UserReplicationPolicy).Returns(replicationPolicy);
QuorumReader reader =
new QuorumReader(mockTransportClient, addressSelector, storeReader, mockServiceConfigReader.Object, mockAuthorizationTokenProvider.Object);
entity.RequestContext.PerformLocalRefreshOnGoneException = true;
entity.RequestContext.OriginalRequestConsistencyLevel = Documents.ConsistencyLevel.Strong;
entity.RequestContext.ClientRequestStatistics = new ClientSideRequestStatistics();
StoreResponse result = reader.ReadStrongAsync(entity, 2, ReadMode.Strong).Result;
Assert.IsTrue(result.LSN == 100);
result.TryGetHeaderValue(WFConstants.BackendHeaders.GlobalCommittedLSN, out string globalCommitedLSN);
long nGlobalCommitedLSN = long.Parse(globalCommitedLSN, CultureInfo.InvariantCulture);
Assert.IsTrue(nGlobalCommitedLSN == 90);
}
}
public void GlobalStrongConsistentWriteMockTest()
{
// create a real document service request (with auth token level = god)
DocumentServiceRequest entity = DocumentServiceRequest.Create(OperationType.Create, ResourceType.Document, AuthorizationTokenType.SystemAll);
// set request charge tracker - this is referenced in store reader (ReadMultipleReplicaAsync)
DocumentServiceRequestContext requestContext = new DocumentServiceRequestContext
{
RequestChargeTracker = new RequestChargeTracker()
};
entity.RequestContext = requestContext;
// set a dummy resource id on the request.
entity.ResourceId = "1-MxAPlgMgA=";
// set consistency level on the request to Bounded Staleness
entity.Headers[HttpConstants.HttpHeaders.ConsistencyLevel] = ConsistencyLevel.Strong.ToString();
// also setup timeout helper, used in store reader
entity.RequestContext.TimeoutHelper = new TimeoutHelper(new TimeSpan(2, 2, 2));
// when the store reader throws Invalid Partition exception, the higher layer should
// clear this target identity.
entity.RequestContext.TargetIdentity = new ServiceIdentity("dummyTargetIdentity1", new Uri("http://dummyTargetIdentity1"), false);
entity.RequestContext.ResolvedPartitionKeyRange = new PartitionKeyRange();
AddressInformation[] addressInformation = this.GetMockAddressInformationDuringUpgrade();
Mock <IAddressResolver> mockAddressCache = this.GetMockAddressCache(addressInformation);
// validate that the mock works
PartitionAddressInformation partitionAddressInformation = mockAddressCache.Object.ResolveAsync(entity, false, new CancellationToken()).Result;
IReadOnlyList <AddressInformation> addressInfo = partitionAddressInformation.AllAddresses;
Assert.IsTrue(addressInfo[0] == addressInformation[0]);
AddressSelector addressSelector = new AddressSelector(mockAddressCache.Object, Protocol.Tcp);
Tuple <Uri, AddressCacheToken> primaryAddress = addressSelector.ResolvePrimaryUriAsync(entity, false /*forceAddressRefresh*/).Result;
// check if the address return from Address Selector matches the original address info
Assert.IsTrue(primaryAddress.Item1.Equals(addressInformation[0].PhysicalUri));
// create a real session container - we don't need session for this test anyway
ISessionContainer sessionContainer = new SessionContainer(string.Empty);
ConnectionStateListener connectionStateListener = new ConnectionStateListener(null);
Mock <IServiceConfigurationReader> mockServiceConfigReader = new Mock <IServiceConfigurationReader>();
Mock <IAuthorizationTokenProvider> mockAuthorizationTokenProvider = new Mock <IAuthorizationTokenProvider>();
mockAuthorizationTokenProvider.Setup(provider => provider.AddSystemAuthorizationHeaderAsync(
It.IsAny <DocumentServiceRequest>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>()))
.Returns(Task.FromResult(0));
for (int i = 0; i < addressInformation.Length; i++)
{
TransportClient mockTransportClient = this.GetMockTransportClientForGlobalStrongWrites(addressInformation, i, false, false, false);
StoreReader storeReader = new StoreReader(mockTransportClient, addressSelector, sessionContainer, connectionStateListener);
ConsistencyWriter consistencyWriter = new ConsistencyWriter(addressSelector, sessionContainer, mockTransportClient, mockServiceConfigReader.Object, mockAuthorizationTokenProvider.Object, connectionStateListener, false);
StoreResponse response = consistencyWriter.WriteAsync(entity, new TimeoutHelper(TimeSpan.FromSeconds(30)), false).Result;
Assert.AreEqual(100, response.LSN);
//globalCommittedLsn never catches up in this case
mockTransportClient = this.GetMockTransportClientForGlobalStrongWrites(addressInformation, i, true, false, false);
consistencyWriter = new ConsistencyWriter(addressSelector, sessionContainer, mockTransportClient, mockServiceConfigReader.Object, mockAuthorizationTokenProvider.Object, connectionStateListener, false);
try
{
response = consistencyWriter.WriteAsync(entity, new TimeoutHelper(TimeSpan.FromSeconds(30)), false).Result;
Assert.Fail();
}
catch (Exception)
{
}
mockTransportClient = this.GetMockTransportClientForGlobalStrongWrites(addressInformation, i, false, true, false);
consistencyWriter = new ConsistencyWriter(addressSelector, sessionContainer, mockTransportClient, mockServiceConfigReader.Object, mockAuthorizationTokenProvider.Object, connectionStateListener, false);
response = consistencyWriter.WriteAsync(entity, new TimeoutHelper(TimeSpan.FromSeconds(30)), false).Result;
Assert.AreEqual(100, response.LSN);
mockTransportClient = this.GetMockTransportClientForGlobalStrongWrites(addressInformation, i, false, true, true);
consistencyWriter = new ConsistencyWriter(addressSelector, sessionContainer, mockTransportClient, mockServiceConfigReader.Object, mockAuthorizationTokenProvider.Object, connectionStateListener, false);
response = consistencyWriter.WriteAsync(entity, new TimeoutHelper(TimeSpan.FromSeconds(30)), false).Result;
Assert.AreEqual(100, response.LSN);
mockTransportClient = this.GetMockTransportClientForGlobalStrongWrites(addressInformation, i, false, false, true);
consistencyWriter = new ConsistencyWriter(addressSelector, sessionContainer, mockTransportClient, mockServiceConfigReader.Object, mockAuthorizationTokenProvider.Object, connectionStateListener, false);
response = consistencyWriter.WriteAsync(entity, new TimeoutHelper(TimeSpan.FromSeconds(30)), false).Result;
Assert.AreEqual(100, response.LSN);
}
}
public void StoreReaderBarrierTest()
{
// create a real document service request
DocumentServiceRequest entity = DocumentServiceRequest.Create(OperationType.Read, ResourceType.Document, AuthorizationTokenType.PrimaryMasterKey);
// set request charge tracker - this is referenced in store reader (ReadMultipleReplicaAsync)
DocumentServiceRequestContext requestContext = new DocumentServiceRequestContext
{
ClientRequestStatistics = new ClientSideRequestStatistics(),
RequestChargeTracker = new RequestChargeTracker()
};
entity.RequestContext = requestContext;
// also setup timeout helper, used in store reader
entity.RequestContext.TimeoutHelper = new TimeoutHelper(new TimeSpan(2, 2, 2));
// when the store reader throws Invalid Partition exception, the higher layer should
// clear this target identity.
entity.RequestContext.TargetIdentity = new ServiceIdentity("dummyTargetIdentity1", new Uri("http://dummyTargetIdentity1"), false);
entity.RequestContext.ResolvedPartitionKeyRange = new PartitionKeyRange();
AddressInformation[] addressInformation = this.GetMockAddressInformationDuringUpgrade();
Mock <IAddressResolver> mockAddressCache = this.GetMockAddressCache(addressInformation);
// validate that the mock works
PartitionAddressInformation partitionAddressInformation = mockAddressCache.Object.ResolveAsync(entity, false, new CancellationToken()).Result;
IReadOnlyList <AddressInformation> addressInfo = partitionAddressInformation.AllAddresses;
Assert.IsTrue(addressInfo[0] == addressInformation[0]);
AddressSelector addressSelector = new AddressSelector(mockAddressCache.Object, Protocol.Tcp);
Tuple <Uri, AddressCacheToken> primaryAddress = addressSelector.ResolvePrimaryUriAsync(entity, false /*forceAddressRefresh*/).Result;
// check if the address return from Address Selector matches the original address info
Assert.IsTrue(primaryAddress.Item1.Equals(addressInformation[0].PhysicalUri));
// get mock transport client that returns a sequence of responses to simulate upgrade
TransportClient mockTransportClient = this.GetMockTransportClientDuringUpgrade(addressInformation);
// get response from mock object
StoreResponse response = mockTransportClient.InvokeResourceOperationAsync(new Uri(addressInformation[0].PhysicalUri), entity).Result;
// validate that the LSN matches
Assert.IsTrue(response.LSN == 50);
response.TryGetHeaderValue(WFConstants.BackendHeaders.ActivityId, out string activityId);
// validate that the ActivityId Matches
Assert.IsTrue(activityId == "ACTIVITYID1_1");
// create a real session container - we don't need session for this test anyway
ISessionContainer sessionContainer = new SessionContainer(string.Empty);
ConnectionStateListener connectionStateListener = new ConnectionStateListener(null);
// create store reader with mock transport client, real address selector (that has mock address cache), and real session container
StoreReader storeReader =
new StoreReader(mockTransportClient,
addressSelector,
sessionContainer,
connectionStateListener);
// reads always go to read quorum (2) replicas
int replicaCountToRead = 2;
IList <StoreResult> result = storeReader.ReadMultipleReplicaAsync(
entity,
false /*includePrimary*/,
replicaCountToRead,
true /*requiresValidLSN*/,
false /*useSessionToken*/,
ReadMode.Strong).Result;
// make sure we got 2 responses from the store reader
Assert.IsTrue(result.Count == 2);
}
public async Task <PartitionAddressInformation> TryGetAddressesAsync(
DocumentServiceRequest request,
PartitionKeyRangeIdentity partitionKeyRangeIdentity,
ServiceIdentity serviceIdentity,
bool forceRefreshPartitionAddresses,
CancellationToken cancellationToken)
{
if (request == null)
{
throw new ArgumentNullException(nameof(request));
}
if (partitionKeyRangeIdentity == null)
{
throw new ArgumentNullException(nameof(partitionKeyRangeIdentity));
}
try
{
if (partitionKeyRangeIdentity.PartitionKeyRangeId == PartitionKeyRange.MasterPartitionKeyRangeId)
{
return((await this.ResolveMasterAsync(request, forceRefreshPartitionAddresses)).Item2);
}
if (this.suboptimalServerPartitionTimestamps.TryGetValue(partitionKeyRangeIdentity, out DateTime suboptimalServerPartitionTimestamp))
{
bool forceRefreshDueToSuboptimalPartitionReplicaSet =
DateTime.UtcNow.Subtract(suboptimalServerPartitionTimestamp) > TimeSpan.FromSeconds(this.suboptimalPartitionForceRefreshIntervalInSeconds);
if (forceRefreshDueToSuboptimalPartitionReplicaSet && this.suboptimalServerPartitionTimestamps.TryUpdate(partitionKeyRangeIdentity, DateTime.MaxValue, suboptimalServerPartitionTimestamp))
{
forceRefreshPartitionAddresses = true;
}
}
PartitionAddressInformation addresses;
PartitionAddressInformation staleAddressInfo = null;
if (forceRefreshPartitionAddresses || request.ForceCollectionRoutingMapRefresh)
{
addresses = await this.serverPartitionAddressCache.GetAsync(
key : partitionKeyRangeIdentity,
singleValueInitFunc : (currentCachedValue) =>
{
staleAddressInfo = currentCachedValue;
GatewayAddressCache.SetTransportAddressUrisToUnhealthy(
currentCachedValue,
request?.RequestContext?.FailedEndpoints);
return(this.GetAddressesForRangeIdAsync(
request,
partitionKeyRangeIdentity.CollectionRid,
partitionKeyRangeIdentity.PartitionKeyRangeId,
forceRefresh: forceRefreshPartitionAddresses));
},
forceRefresh : (currentCachedValue) =>
{
int cachedHashCode = request?.RequestContext?.LastPartitionAddressInformationHashCode ?? 0;
if (cachedHashCode == 0)
{
return(true);
}
// The cached value is different then the previous access hash then assume
// another request already updated the cache since there is a new value in the cache
return(currentCachedValue.GetHashCode() == cachedHashCode);
});
if (staleAddressInfo != null)
{
GatewayAddressCache.LogPartitionCacheRefresh(request.RequestContext.ClientRequestStatistics, staleAddressInfo, addresses);
}
this.suboptimalServerPartitionTimestamps.TryRemove(partitionKeyRangeIdentity, out DateTime ignoreDateTime);
}
else
{
addresses = await this.serverPartitionAddressCache.GetAsync(
key : partitionKeyRangeIdentity,
singleValueInitFunc : (_) => this.GetAddressesForRangeIdAsync(
request,
partitionKeyRangeIdentity.CollectionRid,
partitionKeyRangeIdentity.PartitionKeyRangeId,
forceRefresh: false),
forceRefresh : (_) => false);
}
// Always save the hash code. This is used to determine if another request already updated the cache.
// This helps reduce latency by avoiding uncessary cache refreshes.
if (request?.RequestContext != null)
{
request.RequestContext.LastPartitionAddressInformationHashCode = addresses.GetHashCode();
}
int targetReplicaSetSize = this.serviceConfigReader.UserReplicationPolicy.MaxReplicaSetSize;
if (addresses.AllAddresses.Count() < targetReplicaSetSize)
{
this.suboptimalServerPartitionTimestamps.TryAdd(partitionKeyRangeIdentity, DateTime.UtcNow);
}
return(addresses);
}
catch (DocumentClientException ex)
{
if ((ex.StatusCode == HttpStatusCode.NotFound) ||
(ex.StatusCode == HttpStatusCode.Gone && ex.GetSubStatus() == SubStatusCodes.PartitionKeyRangeGone))
{
//remove from suboptimal cache in case the the collection+pKeyRangeId combo is gone.
this.suboptimalServerPartitionTimestamps.TryRemove(partitionKeyRangeIdentity, out _);
return(null);
}
throw;
}
catch (Exception)
{
if (forceRefreshPartitionAddresses)
{
this.suboptimalServerPartitionTimestamps.TryRemove(partitionKeyRangeIdentity, out _);
}
throw;
}
}
public void MockStoreClientTest()
{
// create a real document service request (with auth token level = god)
DocumentServiceRequest entity = DocumentServiceRequest.Create(OperationType.Read, ResourceType.Document, AuthorizationTokenType.PrimaryMasterKey);
// set request charge tracker - this is referenced in store reader (ReadMultipleReplicaAsync)
var requestContext = new DocumentServiceRequestContext();
requestContext.RequestChargeTracker = new RequestChargeTracker();
entity.RequestContext = requestContext;
// set a dummy resource id on the request.
entity.ResourceId = "1-MxAPlgMgA=";
// set consistency level on the request to Bounded Staleness
entity.Headers[HttpConstants.HttpHeaders.ConsistencyLevel] = ConsistencyLevel.BoundedStaleness.ToString();
// also setup timeout helper, used in store reader
entity.RequestContext.TimeoutHelper = new TimeoutHelper(new TimeSpan(2, 2, 2));
// when the store reader throws Invalid Partition exception, the higher layer should
// clear this target identity.
entity.RequestContext.TargetIdentity = new ServiceIdentity("dummyTargetIdentity1", new Uri("http://dummyTargetIdentity1"), false);
entity.RequestContext.ResolvedPartitionKeyRange = new PartitionKeyRange();
AddressInformation[] addressInformation = GetMockAddressInformationDuringUpgrade();
var mockAddressCache = GetMockAddressCache(addressInformation);
// validate that the mock works
PartitionAddressInformation partitionAddressInformation = mockAddressCache.Object.ResolveAsync(entity, false, new CancellationToken()).Result;
var addressInfo = partitionAddressInformation.AllAddresses;
Assert.IsTrue(addressInfo[0] == addressInformation[0]);
AddressSelector addressSelector = new AddressSelector(mockAddressCache.Object, Protocol.Tcp);
var primaryAddress = addressSelector.ResolvePrimaryUriAsync(entity, false /*forceAddressRefresh*/).Result;
// check if the address return from Address Selector matches the original address info
Assert.IsTrue(primaryAddress.Equals(addressInformation[0].PhysicalUri));
// get mock transport client that returns a sequence of responses to simulate upgrade
var mockTransportClient = GetMockTransportClientDuringUpgrade(addressInformation);
// get response from mock object
var response = mockTransportClient.InvokeResourceOperationAsync(new Uri(addressInformation[0].PhysicalUri), entity).Result;
// validate that the LSN matches
Assert.IsTrue(response.LSN == 50);
string activityId;
response.TryGetHeaderValue(WFConstants.BackendHeaders.ActivityId, out activityId);
// validate that the ActivityId Matches
Assert.IsTrue(activityId == "ACTIVITYID1_1");
// create a real session container - we don't need session for this test anyway
ISessionContainer sessionContainer = new SessionContainer(string.Empty);
// create store reader with mock transport client, real address selector (that has mock address cache), and real session container
StoreReader storeReader =
new StoreReader(mockTransportClient,
addressSelector,
sessionContainer);
Mock <IAuthorizationTokenProvider> mockAuthorizationTokenProvider = new Mock <IAuthorizationTokenProvider>();
// setup max replica set size on the config reader
ReplicationPolicy replicationPolicy = new ReplicationPolicy();
replicationPolicy.MaxReplicaSetSize = 4;
Mock <IServiceConfigurationReader> mockServiceConfigReader = new Mock <IServiceConfigurationReader>();
mockServiceConfigReader.SetupGet(x => x.UserReplicationPolicy).Returns(replicationPolicy);
try
{
StoreClient storeClient =
new StoreClient(
mockAddressCache.Object,
sessionContainer,
mockServiceConfigReader.Object,
mockAuthorizationTokenProvider.Object,
Protocol.Tcp,
mockTransportClient);
ServerStoreModel storeModel = new ServerStoreModel(storeClient);
var result = storeModel.ProcessMessageAsync(entity).Result;
// if we have reached this point, there was a successful request.
// validate if the target identity has been cleared out.
// If the target identity is null and the request still succeeded, it means
// that the very first read succeeded without a barrier request.
Assert.IsNull(entity.RequestContext.TargetIdentity);
Assert.IsNull(entity.RequestContext.ResolvedPartitionKeyRange);
}
catch (Exception e)
{
Assert.IsTrue(e.InnerException is ServiceUnavailableException ||
e.InnerException is ArgumentNullException ||
e.InnerException is ServiceUnavailableException ||
e.InnerException is NullReferenceException);
}
}
