public GatewayAddressCache(
Uri serviceEndpoint,
Protocol protocol,
IAuthorizationTokenProvider tokenProvider,
IServiceConfigurationReader serviceConfigReader,
CosmosHttpClient httpClient,
long suboptimalPartitionForceRefreshIntervalInSeconds = 600,
bool enableTcpConnectionEndpointRediscovery = false)
{
this.addressEndpoint = new Uri(serviceEndpoint + "/" + Paths.AddressPathSegment);
this.protocol = protocol;
this.tokenProvider = tokenProvider;
this.serviceEndpoint = serviceEndpoint;
this.serviceConfigReader = serviceConfigReader;
this.serverPartitionAddressCache = new AsyncCache <PartitionKeyRangeIdentity, PartitionAddressInformation>();
this.suboptimalServerPartitionTimestamps = new ConcurrentDictionary <PartitionKeyRangeIdentity, DateTime>();
this.serverPartitionAddressToPkRangeIdMap = new ConcurrentDictionary <ServerKey, HashSet <PartitionKeyRangeIdentity> >();
this.suboptimalMasterPartitionTimestamp = DateTime.MaxValue;
this.enableTcpConnectionEndpointRediscovery = enableTcpConnectionEndpointRediscovery;
this.suboptimalPartitionForceRefreshIntervalInSeconds = suboptimalPartitionForceRefreshIntervalInSeconds;
this.httpClient = httpClient;
this.protocolFilter =
string.Format(CultureInfo.InvariantCulture,
GatewayAddressCache.protocolFilterFormat,
Constants.Properties.Protocol,
GatewayAddressCache.ProtocolString(this.protocol));
}
public GatewayAddressCache(
Uri serviceEndpoint,
Protocol protocol,
IAuthorizationTokenProvider tokenProvider,
UserAgentContainer userAgent,
IServiceConfigurationReader serviceConfigReader,
long suboptimalPartitionForceRefreshIntervalInSeconds = 600,
HttpMessageHandler messageHandler = null,
ApiType apiType = ApiType.None)
{
this.addressEndpoint = new Uri(serviceEndpoint + "/" + Paths.AddressPathSegment);
this.protocol = protocol;
this.tokenProvider = tokenProvider;
this.serviceEndpoint = serviceEndpoint;
this.serviceConfigReader = serviceConfigReader;
this.serverPartitionAddressCache = new AsyncCache <PartitionKeyRangeIdentity, PartitionAddressInformation>();
this.suboptimalServerPartitionTimestamps = new ConcurrentDictionary <PartitionKeyRangeIdentity, DateTime>();
this.suboptimalMasterPartitionTimestamp = DateTime.MaxValue;
this.suboptimalPartitionForceRefreshIntervalInSeconds = suboptimalPartitionForceRefreshIntervalInSeconds;
this.httpClient = messageHandler == null ? new HttpClient() : new HttpClient(messageHandler);
this.protocolFilter =
string.Format(CultureInfo.InvariantCulture,
GatewayAddressCache.protocolFilterFormat,
Constants.Properties.Protocol,
GatewayAddressCache.ProtocolString(this.protocol));
// Set requested API version header for version enforcement.
this.httpClient.DefaultRequestHeaders.Add(HttpConstants.HttpHeaders.Version,
HttpConstants.Versions.CurrentVersion);
this.httpClient.AddUserAgentHeader(userAgent);
this.httpClient.AddApiTypeHeader(apiType);
}
private async Task <FeedResource <Address> > GetMasterAddressesViaGatewayAsync(
DocumentServiceRequest request,
ResourceType resourceType,
string resourceAddress,
string entryUrl,
bool forceRefresh,
bool useMasterCollectionResolver)
{
INameValueCollection addressQuery = new DictionaryNameValueCollection(StringComparer.Ordinal);
addressQuery.Add(HttpConstants.QueryStrings.Url, HttpUtility.UrlEncode(entryUrl));
INameValueCollection headers = new DictionaryNameValueCollection(StringComparer.Ordinal);
if (forceRefresh)
{
headers.Set(HttpConstants.HttpHeaders.ForceRefresh, bool.TrueString);
}
if (useMasterCollectionResolver)
{
headers.Set(HttpConstants.HttpHeaders.UseMasterCollectionResolver, bool.TrueString);
}
if (request.ForceCollectionRoutingMapRefresh)
{
headers.Set(HttpConstants.HttpHeaders.ForceCollectionRoutingMapRefresh, bool.TrueString);
}
addressQuery.Add(HttpConstants.QueryStrings.Filter, this.protocolFilter);
string resourceTypeToSign = PathsHelper.GetResourcePath(resourceType);
headers.Set(HttpConstants.HttpHeaders.XDate, DateTime.UtcNow.ToString("r", CultureInfo.InvariantCulture));
string token = this.tokenProvider.GetUserAuthorizationToken(
resourceAddress,
resourceTypeToSign,
HttpConstants.HttpMethods.Get,
headers,
AuthorizationTokenType.PrimaryMasterKey,
payload: out _);
headers.Set(HttpConstants.HttpHeaders.Authorization, token);
Uri targetEndpoint = UrlUtility.SetQuery(this.addressEndpoint, UrlUtility.CreateQuery(addressQuery));
string identifier = GatewayAddressCache.LogAddressResolutionStart(request, targetEndpoint);
using (HttpResponseMessage httpResponseMessage = await this.httpClient.GetAsync(targetEndpoint, headers))
{
using (DocumentServiceResponse documentServiceResponse =
await ClientExtensions.ParseResponseAsync(httpResponseMessage))
{
GatewayAddressCache.LogAddressResolutionEnd(request, identifier);
return(documentServiceResponse.GetResource <FeedResource <Address> >());
}
}
}
private async Task <DocumentServiceResponse> GetMasterAddressesViaGatewayAsync(
DocumentServiceRequest request,
ResourceType resourceType,
string resourceAddress,
string entryUrl,
bool forceRefresh,
bool useMasterCollectionResolver)
{
INameValueCollection addressQuery = new RequestNameValueCollection
{
{ HttpConstants.QueryStrings.Url, HttpUtility.UrlEncode(entryUrl) }
};
INameValueCollection headers = new RequestNameValueCollection();
if (forceRefresh)
{
headers.Set(HttpConstants.HttpHeaders.ForceRefresh, bool.TrueString);
}
if (useMasterCollectionResolver)
{
headers.Set(HttpConstants.HttpHeaders.UseMasterCollectionResolver, bool.TrueString);
}
if (request.ForceCollectionRoutingMapRefresh)
{
headers.Set(HttpConstants.HttpHeaders.ForceCollectionRoutingMapRefresh, bool.TrueString);
}
addressQuery.Add(HttpConstants.QueryStrings.Filter, this.protocolFilter);
string resourceTypeToSign = PathsHelper.GetResourcePath(resourceType);
headers.Set(HttpConstants.HttpHeaders.XDate, Rfc1123DateTimeCache.UtcNow());
using (ITrace trace = Trace.GetRootTrace(nameof(GetMasterAddressesViaGatewayAsync), TraceComponent.Authorization, TraceLevel.Info))
{
string token = await this.tokenProvider.GetUserAuthorizationTokenAsync(
resourceAddress,
resourceTypeToSign,
HttpConstants.HttpMethods.Get,
headers,
AuthorizationTokenType.PrimaryMasterKey,
trace);
headers.Set(HttpConstants.HttpHeaders.Authorization, token);
Uri targetEndpoint = UrlUtility.SetQuery(this.addressEndpoint, UrlUtility.CreateQuery(addressQuery));
string identifier = GatewayAddressCache.LogAddressResolutionStart(request, targetEndpoint);
using (HttpResponseMessage httpResponseMessage = await this.httpClient.GetAsync(
uri: targetEndpoint,
additionalHeaders: headers,
resourceType: resourceType,
timeoutPolicy: HttpTimeoutPolicyControlPlaneRetriableHotPath.Instance,
clientSideRequestStatistics: request.RequestContext?.ClientRequestStatistics,
cancellationToken: default))
private async Task <FeedResource <Address> > GetMasterAddressesViaGatewayAsync(
DocumentServiceRequest request,
ResourceType resourceType,
string resourceAddress,
string entryUrl,
bool forceRefresh,
bool useMasterCollectionResolver)
{
INameValueCollection addressQuery = new StoreRequestNameValueCollection
{
{ HttpConstants.QueryStrings.Url, HttpUtility.UrlEncode(entryUrl) }
};
INameValueCollection headers = new StoreRequestNameValueCollection();
if (forceRefresh)
{
headers.Set(HttpConstants.HttpHeaders.ForceRefresh, bool.TrueString);
}
if (useMasterCollectionResolver)
{
headers.Set(HttpConstants.HttpHeaders.UseMasterCollectionResolver, bool.TrueString);
}
if (request.ForceCollectionRoutingMapRefresh)
{
headers.Set(HttpConstants.HttpHeaders.ForceCollectionRoutingMapRefresh, bool.TrueString);
}
addressQuery.Add(HttpConstants.QueryStrings.Filter, this.protocolFilter);
string resourceTypeToSign = PathsHelper.GetResourcePath(resourceType);
headers.Set(HttpConstants.HttpHeaders.XDate, DateTime.UtcNow.ToString("r", CultureInfo.InvariantCulture));
(string token, string _) = await this.tokenProvider.GetUserAuthorizationAsync(
resourceAddress,
resourceTypeToSign,
HttpConstants.HttpMethods.Get,
headers,
AuthorizationTokenType.PrimaryMasterKey);
headers.Set(HttpConstants.HttpHeaders.Authorization, token);
Uri targetEndpoint = UrlUtility.SetQuery(this.addressEndpoint, UrlUtility.CreateQuery(addressQuery));
string identifier = GatewayAddressCache.LogAddressResolutionStart(request, targetEndpoint);
using (HttpResponseMessage httpResponseMessage = await this.httpClient.GetAsync(
uri: targetEndpoint,
additionalHeaders: headers,
resourceType: resourceType,
timeoutPolicy: HttpTimeoutPolicyControlPlaneHotPath.Instance,
diagnosticsContext: null,
cancellationToken: default))
private EndpointCache GetOrAddEndpoint(Uri endpoint)
{
// The GetorAdd is followed by a call to .Count which in a ConcurrentDictionary
// will acquire all locks for all buckets. This is really expensive. Since the check
// there is only to see if we've exceeded the count of endpoints, we can simply
// avoid that check altogether if we are not adding any more endpoints.
if (this.addressCacheByEndpoint.TryGetValue(endpoint, out EndpointCache existingCache))
{
return(existingCache);
}
EndpointCache endpointCache = this.addressCacheByEndpoint.GetOrAdd(
endpoint,
(Uri resolvedEndpoint) =>
{
GatewayAddressCache gatewayAddressCache = new GatewayAddressCache(
resolvedEndpoint,
this.protocol,
this.tokenProvider,
this.serviceConfigReader,
this.httpClient,
enableTcpConnectionEndpointRediscovery: this.enableTcpConnectionEndpointRediscovery);
string location = this.endpointManager.GetLocation(endpoint);
AddressResolver addressResolver = new AddressResolver(null, new NullRequestSigner(), location);
addressResolver.InitializeCaches(this.collectionCache, this.routingMapProvider, gatewayAddressCache);
return(new EndpointCache()
{
AddressCache = gatewayAddressCache,
AddressResolver = addressResolver,
});
});
if (this.addressCacheByEndpoint.Count > this.maxEndpoints)
{
IEnumerable <Uri> allEndpoints = this.endpointManager.WriteEndpoints.Union(this.endpointManager.ReadEndpoints);
Queue <Uri> endpoints = new Queue <Uri>(allEndpoints.Reverse());
while (this.addressCacheByEndpoint.Count > this.maxEndpoints)
{
if (endpoints.Count > 0)
{
EndpointCache removedEntry;
this.addressCacheByEndpoint.TryRemove(endpoints.Dequeue(), out removedEntry);
}
else
{
break;
}
}
}
return(endpointCache);
}
private EndpointCache GetOrAddEndpoint(Uri endpoint)
{
EndpointCache endpointCache = this.addressCacheByEndpoint.GetOrAdd(
endpoint,
(Uri resolvedEndpoint) =>
{
GatewayAddressCache gatewayAddressCache = new GatewayAddressCache(
resolvedEndpoint,
this.protocol,
this.tokenProvider,
this.userAgentContainer,
this.serviceConfigReader,
this.requestTimeout,
messageHandler: this.messageHandler,
apiType: this.apiType,
enableTcpConnectionEndpointRediscovery: this.enableTcpConnectionEndpointRediscovery);
string location = this.endpointManager.GetLocation(endpoint);
AddressResolver addressResolver = new AddressResolver(null, new NullRequestSigner(), location);
addressResolver.InitializeCaches(this.collectionCache, this.routingMapProvider, gatewayAddressCache);
return(new EndpointCache()
{
AddressCache = gatewayAddressCache,
AddressResolver = addressResolver,
});
});
if (this.addressCacheByEndpoint.Count > this.maxEndpoints)
{
IEnumerable <Uri> allEndpoints = this.endpointManager.WriteEndpoints.Union(this.endpointManager.ReadEndpoints);
Queue <Uri> endpoints = new Queue <Uri>(allEndpoints.Reverse());
while (this.addressCacheByEndpoint.Count > this.maxEndpoints)
{
if (endpoints.Count > 0)
{
EndpointCache removedEntry;
this.addressCacheByEndpoint.TryRemove(endpoints.Dequeue(), out removedEntry);
}
else
{
break;
}
}
}
return(endpointCache);
}
private async Task <FeedResource <Address> > GetServerAddressesViaGatewayAsync(
DocumentServiceRequest request,
string collectionRid,
IEnumerable <string> partitionKeyRangeIds,
bool forceRefresh)
{
string entryUrl = PathsHelper.GeneratePath(ResourceType.Document, collectionRid, true);
INameValueCollection addressQuery = new DictionaryNameValueCollection();
addressQuery.Add(HttpConstants.QueryStrings.Url, HttpUtility.UrlEncode(entryUrl));
INameValueCollection headers = new DictionaryNameValueCollection();
if (forceRefresh)
{
headers.Set(HttpConstants.HttpHeaders.ForceRefresh, bool.TrueString);
}
if (request.ForceCollectionRoutingMapRefresh)
{
headers.Set(HttpConstants.HttpHeaders.ForceCollectionRoutingMapRefresh, bool.TrueString);
}
addressQuery.Add(HttpConstants.QueryStrings.Filter, this.protocolFilter);
addressQuery.Add(HttpConstants.QueryStrings.PartitionKeyRangeIds, string.Join(",", partitionKeyRangeIds));
string resourceTypeToSign = PathsHelper.GetResourcePath(ResourceType.Document);
headers.Set(HttpConstants.HttpHeaders.XDate, DateTime.UtcNow.ToString("r", CultureInfo.InvariantCulture));
string token = null;
try
{
token = this.tokenProvider.GetUserAuthorizationToken(
collectionRid,
resourceTypeToSign,
HttpConstants.HttpMethods.Get,
headers,
AuthorizationTokenType.PrimaryMasterKey);
}
catch (UnauthorizedException)
{
}
if (token == null && request.IsNameBased)
{
// User doesn't have rid based resource token. Maybe he has name based.
string collectionAltLink = PathsHelper.GetCollectionPath(request.ResourceAddress);
token = this.tokenProvider.GetUserAuthorizationToken(
collectionAltLink,
resourceTypeToSign,
HttpConstants.HttpMethods.Get,
headers,
AuthorizationTokenType.PrimaryMasterKey);
}
headers.Set(HttpConstants.HttpHeaders.Authorization, token);
Uri targetEndpoint = UrlUtility.SetQuery(this.addressEndpoint, UrlUtility.CreateQuery(addressQuery));
string identifier = GatewayAddressCache.LogAddressResolutionStart(request, targetEndpoint);
using (HttpResponseMessage httpResponseMessage = await this.httpClient.GetAsync(targetEndpoint, headers))
{
using (DocumentServiceResponse documentServiceResponse =
await ClientExtensions.ParseResponseAsync(httpResponseMessage))
{
GatewayAddressCache.LogAddressResolutionEnd(request, identifier);
return(documentServiceResponse.GetResource <FeedResource <Address> >());
}
}
}
public async Task <PartitionAddressInformation> TryGetAddressesAsync(
DocumentServiceRequest request,
PartitionKeyRangeIdentity partitionKeyRangeIdentity,
ServiceIdentity serviceIdentity,
bool forceRefreshPartitionAddresses,
CancellationToken cancellationToken)
{
if (request == null)
{
throw new ArgumentNullException(nameof(request));
}
if (partitionKeyRangeIdentity == null)
{
throw new ArgumentNullException(nameof(partitionKeyRangeIdentity));
}
try
{
if (partitionKeyRangeIdentity.PartitionKeyRangeId == PartitionKeyRange.MasterPartitionKeyRangeId)
{
return((await this.ResolveMasterAsync(request, forceRefreshPartitionAddresses)).Item2);
}
if (this.suboptimalServerPartitionTimestamps.TryGetValue(partitionKeyRangeIdentity, out DateTime suboptimalServerPartitionTimestamp))
{
bool forceRefreshDueToSuboptimalPartitionReplicaSet =
DateTime.UtcNow.Subtract(suboptimalServerPartitionTimestamp) > TimeSpan.FromSeconds(this.suboptimalPartitionForceRefreshIntervalInSeconds);
if (forceRefreshDueToSuboptimalPartitionReplicaSet && this.suboptimalServerPartitionTimestamps.TryUpdate(partitionKeyRangeIdentity, DateTime.MaxValue, suboptimalServerPartitionTimestamp))
{
forceRefreshPartitionAddresses = true;
}
}
PartitionAddressInformation addresses;
PartitionAddressInformation staleAddressInfo = null;
if (forceRefreshPartitionAddresses || request.ForceCollectionRoutingMapRefresh)
{
addresses = await this.serverPartitionAddressCache.GetAsync(
key : partitionKeyRangeIdentity,
singleValueInitFunc : (currentCachedValue) =>
{
staleAddressInfo = currentCachedValue;
GatewayAddressCache.SetTransportAddressUrisToUnhealthy(
currentCachedValue,
request?.RequestContext?.FailedEndpoints);
return(this.GetAddressesForRangeIdAsync(
request,
partitionKeyRangeIdentity.CollectionRid,
partitionKeyRangeIdentity.PartitionKeyRangeId,
forceRefresh: forceRefreshPartitionAddresses));
},
forceRefresh : (currentCachedValue) =>
{
int cachedHashCode = request?.RequestContext?.LastPartitionAddressInformationHashCode ?? 0;
if (cachedHashCode == 0)
{
return(true);
}
// The cached value is different then the previous access hash then assume
// another request already updated the cache since there is a new value in the cache
return(currentCachedValue.GetHashCode() == cachedHashCode);
});
if (staleAddressInfo != null)
{
GatewayAddressCache.LogPartitionCacheRefresh(request.RequestContext.ClientRequestStatistics, staleAddressInfo, addresses);
}
this.suboptimalServerPartitionTimestamps.TryRemove(partitionKeyRangeIdentity, out DateTime ignoreDateTime);
}
else
{
addresses = await this.serverPartitionAddressCache.GetAsync(
key : partitionKeyRangeIdentity,
singleValueInitFunc : (_) => this.GetAddressesForRangeIdAsync(
request,
partitionKeyRangeIdentity.CollectionRid,
partitionKeyRangeIdentity.PartitionKeyRangeId,
forceRefresh: false),
forceRefresh : (_) => false);
}
// Always save the hash code. This is used to determine if another request already updated the cache.
// This helps reduce latency by avoiding uncessary cache refreshes.
if (request?.RequestContext != null)
{
request.RequestContext.LastPartitionAddressInformationHashCode = addresses.GetHashCode();
}
int targetReplicaSetSize = this.serviceConfigReader.UserReplicationPolicy.MaxReplicaSetSize;
if (addresses.AllAddresses.Count() < targetReplicaSetSize)
{
this.suboptimalServerPartitionTimestamps.TryAdd(partitionKeyRangeIdentity, DateTime.UtcNow);
}
return(addresses);
}
catch (DocumentClientException ex)
{
if ((ex.StatusCode == HttpStatusCode.NotFound) ||
(ex.StatusCode == HttpStatusCode.Gone && ex.GetSubStatus() == SubStatusCodes.PartitionKeyRangeGone))
{
//remove from suboptimal cache in case the the collection+pKeyRangeId combo is gone.
this.suboptimalServerPartitionTimestamps.TryRemove(partitionKeyRangeIdentity, out _);
return(null);
}
throw;
}
catch (Exception)
{
if (forceRefreshPartitionAddresses)
{
this.suboptimalServerPartitionTimestamps.TryRemove(partitionKeyRangeIdentity, out _);
}
throw;
}
}
public async Task <PartitionAddressInformation> TryGetAddressesAsync(
DocumentServiceRequest request,
PartitionKeyRangeIdentity partitionKeyRangeIdentity,
ServiceIdentity serviceIdentity,
bool forceRefreshPartitionAddresses,
CancellationToken cancellationToken)
{
if (request == null)
{
throw new ArgumentNullException(nameof(request));
}
if (partitionKeyRangeIdentity == null)
{
throw new ArgumentNullException(nameof(partitionKeyRangeIdentity));
}
try
{
if (partitionKeyRangeIdentity.PartitionKeyRangeId == PartitionKeyRange.MasterPartitionKeyRangeId)
{
return((await this.ResolveMasterAsync(request, forceRefreshPartitionAddresses)).Item2);
}
if (this.suboptimalServerPartitionTimestamps.TryGetValue(partitionKeyRangeIdentity, out DateTime suboptimalServerPartitionTimestamp))
{
bool forceRefreshDueToSuboptimalPartitionReplicaSet =
DateTime.UtcNow.Subtract(suboptimalServerPartitionTimestamp) > TimeSpan.FromSeconds(this.suboptimalPartitionForceRefreshIntervalInSeconds);
if (forceRefreshDueToSuboptimalPartitionReplicaSet && this.suboptimalServerPartitionTimestamps.TryUpdate(partitionKeyRangeIdentity, DateTime.MaxValue, suboptimalServerPartitionTimestamp))
{
forceRefreshPartitionAddresses = true;
}
}
PartitionAddressInformation addresses;
if (forceRefreshPartitionAddresses || request.ForceCollectionRoutingMapRefresh)
{
addresses = await this.serverPartitionAddressCache.GetAsync(
key : partitionKeyRangeIdentity,
singleValueInitFunc : () => this.GetAddressesForRangeIdAsync(
request,
partitionKeyRangeIdentity.CollectionRid,
partitionKeyRangeIdentity.PartitionKeyRangeId,
forceRefresh: forceRefreshPartitionAddresses),
forceRefresh : true,
callBackOnForceRefresh : (old, updated) => GatewayAddressCache.LogPartitionCacheRefresh(request.RequestContext.ClientRequestStatistics, old, updated));
this.suboptimalServerPartitionTimestamps.TryRemove(partitionKeyRangeIdentity, out DateTime ignoreDateTime);
}
else
{
addresses = await this.serverPartitionAddressCache.GetAsync(
key : partitionKeyRangeIdentity,
singleValueInitFunc : () => this.GetAddressesForRangeIdAsync(
request,
partitionKeyRangeIdentity.CollectionRid,
partitionKeyRangeIdentity.PartitionKeyRangeId,
forceRefresh: false),
forceRefresh : false,
callBackOnForceRefresh : (old, updated) => GatewayAddressCache.LogPartitionCacheRefresh(request.RequestContext.ClientRequestStatistics, old, updated));
}
int targetReplicaSetSize = this.serviceConfigReader.UserReplicationPolicy.MaxReplicaSetSize;
if (addresses.AllAddresses.Count() < targetReplicaSetSize)
{
this.suboptimalServerPartitionTimestamps.TryAdd(partitionKeyRangeIdentity, DateTime.UtcNow);
}
return(addresses);
}
catch (DocumentClientException ex)
{
if ((ex.StatusCode == HttpStatusCode.NotFound) ||
(ex.StatusCode == HttpStatusCode.Gone && ex.GetSubStatus() == SubStatusCodes.PartitionKeyRangeGone))
{
//remove from suboptimal cache in case the the collection+pKeyRangeId combo is gone.
this.suboptimalServerPartitionTimestamps.TryRemove(partitionKeyRangeIdentity, out _);
return(null);
}
throw;
}
catch (Exception)
{
if (forceRefreshPartitionAddresses)
{
this.suboptimalServerPartitionTimestamps.TryRemove(partitionKeyRangeIdentity, out _);
}
throw;
}
}
