/// <summary>
/// Resolves request to service endpoint.
/// 1. If this is a write request
/// (a) If UseMultipleWriteLocations = true
/// (i) For document writes, resolve to most preferred and available write endpoint.
/// Once the endpoint is marked unavailable, it is moved to the end of available write endpoint. Current request will
/// be retried on next preferred available write endpoint.
/// (ii) For all other resources, always resolve to first/second (regardless of preferred locations)
/// write endpoint in <see cref="AccountProperties.WritableRegions"/>.
/// Endpoint of first write location in <see cref="AccountProperties.WritableRegions"/> is the only endpoint that supports
/// write operation on all resource types (except during that region's failover).
/// Only during manual failover, client would retry write on second write location in <see cref="AccountProperties.WritableRegions"/>.
/// (b) Else resolve the request to first write endpoint in <see cref="AccountProperties.writeRegions"/> OR
/// second write endpoint in <see cref="AccountProperties.WritableRegions"/> in case of manual failover of that location.
/// 2. Else resolve the request to most preferred available read endpoint (automatic failover for read requests)
/// </summary>
/// <param name="request">Request for which endpoint is to be resolved</param>
/// <returns>Resolved endpoint</returns>
public Uri ResolveServiceEndpoint(DocumentServiceRequest request)
{
if (request.RequestContext != null && request.RequestContext.LocationEndpointToRoute != null)
{
return(request.RequestContext.LocationEndpointToRoute);
}
int locationIndex = request.RequestContext.LocationIndexToRoute.GetValueOrDefault(0);
Uri locationEndpointToRoute = this.defaultEndpoint;
if (!request.RequestContext.UsePreferredLocations.GetValueOrDefault(true) || // Should not use preferred location ?
(request.OperationType.IsWriteOperation() && !this.CanUseMultipleWriteLocations(request)))
{
// For non-document resource types in case of client can use multiple write locations
// or when client cannot use multiple write locations, flip-flop between the
// first and the second writable region in DatabaseAccount (for manual failover)
DatabaseAccountLocationsInfo currentLocationInfo = this.locationInfo;
if (this.enableEndpointDiscovery && currentLocationInfo.AvailableWriteLocations.Count > 0)
{
locationIndex = Math.Min(locationIndex % 2, currentLocationInfo.AvailableWriteLocations.Count - 1);
string writeLocation = currentLocationInfo.AvailableWriteLocations[locationIndex];
locationEndpointToRoute = currentLocationInfo.AvailableWriteEndpointByLocation[writeLocation];
}
}
else
{
ReadOnlyCollection <Uri> endpoints = request.OperationType.IsWriteOperation() ? this.WriteEndpoints : this.ReadEndpoints;
locationEndpointToRoute = endpoints[locationIndex % endpoints.Count];
}
request.RequestContext.RouteToLocation(locationEndpointToRoute);
return(locationEndpointToRoute);
}
public DatabaseAccountLocationsInfo(DatabaseAccountLocationsInfo other)
{
this.PreferredLocations = other.PreferredLocations;
this.AvailableWriteLocations = other.AvailableWriteLocations;
this.AvailableReadLocations = other.AvailableReadLocations;
this.AvailableWriteEndpointByLocation = other.AvailableWriteEndpointByLocation;
this.AvailableReadEndpointByLocation = other.AvailableReadEndpointByLocation;
this.WriteEndpoints = other.WriteEndpoints;
this.ReadEndpoints = other.ReadEndpoints;
}
private void UpdateLocationCache(
IEnumerable <AccountRegion> writeLocations = null,
IEnumerable <AccountRegion> readLocations = null,
ReadOnlyCollection <string> preferenceList = null,
bool?enableMultipleWriteLocations = null)
{
lock (this.lockObject)
{
DatabaseAccountLocationsInfo nextLocationInfo = new DatabaseAccountLocationsInfo(this.locationInfo);
if (preferenceList != null)
{
nextLocationInfo.PreferredLocations = preferenceList;
}
if (enableMultipleWriteLocations.HasValue)
{
this.enableMultipleWriteLocations = enableMultipleWriteLocations.Value;
}
this.ClearStaleEndpointUnavailabilityInfo();
if (readLocations != null)
{
ReadOnlyCollection <string> availableReadLocations;
nextLocationInfo.AvailableReadEndpointByLocation = this.GetEndpointByLocation(readLocations, out availableReadLocations);
nextLocationInfo.AvailableReadLocations = availableReadLocations;
}
if (writeLocations != null)
{
ReadOnlyCollection <string> availableWriteLocations;
nextLocationInfo.AvailableWriteEndpointByLocation = this.GetEndpointByLocation(writeLocations, out availableWriteLocations);
nextLocationInfo.AvailableWriteLocations = availableWriteLocations;
}
nextLocationInfo.WriteEndpoints = this.GetPreferredAvailableEndpoints(nextLocationInfo.AvailableWriteEndpointByLocation, nextLocationInfo.AvailableWriteLocations, OperationType.Write, this.defaultEndpoint);
nextLocationInfo.ReadEndpoints = this.GetPreferredAvailableEndpoints(nextLocationInfo.AvailableReadEndpointByLocation, nextLocationInfo.AvailableReadLocations, OperationType.Read, nextLocationInfo.WriteEndpoints[0]);
this.lastCacheUpdateTimestamp = DateTime.UtcNow;
DefaultTrace.TraceInformation("Current WriteEndpoints = ({0}) ReadEndpoints = ({1})",
string.Join(", ", nextLocationInfo.WriteEndpoints.Select(endpoint => endpoint.ToString())),
string.Join(", ", nextLocationInfo.ReadEndpoints.Select(endpoint => endpoint.ToString())));
this.locationInfo = nextLocationInfo;
}
}
public LocationCache(
ReadOnlyCollection <string> preferredLocations,
Uri defaultEndpoint,
bool enableEndpointDiscovery,
int connectionLimit,
bool useMultipleWriteLocations)
{
this.locationInfo = new DatabaseAccountLocationsInfo(preferredLocations, defaultEndpoint);
this.defaultEndpoint = defaultEndpoint;
this.enableEndpointDiscovery = enableEndpointDiscovery;
this.useMultipleWriteLocations = useMultipleWriteLocations;
this.connectionLimit = connectionLimit;
this.lockObject = new object();
this.locationUnavailablityInfoByEndpoint = new ConcurrentDictionary <Uri, LocationUnavailabilityInfo>();
this.lastCacheUpdateTimestamp = DateTime.MinValue;
this.enableMultipleWriteLocations = false;
this.unavailableLocationsExpirationTime = TimeSpan.FromSeconds(LocationCache.DefaultUnavailableLocationsExpirationTimeInSeconds);
#if !(NETSTANDARD15 || NETSTANDARD16)
#if NETSTANDARD20
// GetEntryAssembly returns null when loaded from native netstandard2.0
if (System.Reflection.Assembly.GetEntryAssembly() != null)
{
#endif
string unavailableLocationsExpirationTimeInSecondsConfig = System.Configuration.ConfigurationManager.AppSettings[LocationCache.UnavailableLocationsExpirationTimeInSeconds];
if (!string.IsNullOrEmpty(unavailableLocationsExpirationTimeInSecondsConfig))
{
int unavailableLocationsExpirationTimeinSecondsConfigValue;
if (!int.TryParse(unavailableLocationsExpirationTimeInSecondsConfig, out unavailableLocationsExpirationTimeinSecondsConfigValue))
{
this.unavailableLocationsExpirationTime = TimeSpan.FromSeconds(LocationCache.DefaultUnavailableLocationsExpirationTimeInSeconds);
}
else
{
this.unavailableLocationsExpirationTime = TimeSpan.FromSeconds(unavailableLocationsExpirationTimeinSecondsConfigValue);
}
}
#if NETSTANDARD20
}
#endif
#endif
}
private ReadOnlyCollection <Uri> GetPreferredAvailableEndpoints(ReadOnlyDictionary <string, Uri> endpointsByLocation, ReadOnlyCollection <string> orderedLocations, OperationType expectedAvailableOperation, Uri fallbackEndpoint)
{
List <Uri> endpoints = new List <Uri>();
DatabaseAccountLocationsInfo currentLocationInfo = this.locationInfo;
// if enableEndpointDiscovery is false, we always use the defaultEndpoint that user passed in during documentClient init
if (this.enableEndpointDiscovery)
{
if (this.CanUseMultipleWriteLocations() || expectedAvailableOperation.HasFlag(OperationType.Read))
{
List <Uri> unavailableEndpoints = new List <Uri>();
// When client can not use multiple write locations, preferred locations list should only be used
// determining read endpoints order.
// If client can use multiple write locations, preferred locations list should be used for determining
// both read and write endpoints order.
foreach (string location in currentLocationInfo.PreferredLocations)
{
Uri endpoint;
if (endpointsByLocation.TryGetValue(location, out endpoint))
{
if (this.IsEndpointUnavailable(endpoint, expectedAvailableOperation))
{
unavailableEndpoints.Add(endpoint);
}
else
{
endpoints.Add(endpoint);
}
}
}
if (endpoints.Count == 0)
{
endpoints.Add(fallbackEndpoint);
unavailableEndpoints.Remove(fallbackEndpoint);
}
endpoints.AddRange(unavailableEndpoints);
}
else
{
foreach (string location in orderedLocations)
{
Uri endpoint;
if (!string.IsNullOrEmpty(location) && // location is empty during manual failover
endpointsByLocation.TryGetValue(location, out endpoint))
{
endpoints.Add(endpoint);
}
}
}
}
if (endpoints.Count == 0)
{
endpoints.Add(fallbackEndpoint);
}
return(endpoints.AsReadOnly());
}
public bool ShouldRefreshEndpoints(out bool canRefreshInBackground)
{
canRefreshInBackground = true;
DatabaseAccountLocationsInfo currentLocationInfo = this.locationInfo;
string mostPreferredLocation = currentLocationInfo.PreferredLocations.FirstOrDefault();
// we should schedule refresh in background if we are unable to target the user's most preferredLocation.
if (this.enableEndpointDiscovery)
{
// Refresh if client opts-in to useMultipleWriteLocations but server-side setting is disabled
bool shouldRefresh = this.useMultipleWriteLocations && !this.enableMultipleWriteLocations;
ReadOnlyCollection <Uri> readLocationEndpoints = currentLocationInfo.ReadEndpoints;
if (this.IsEndpointUnavailable(readLocationEndpoints[0], OperationType.Read))
{
canRefreshInBackground = readLocationEndpoints.Count > 1;
DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since the first read endpoint {0} is not available for read. canRefreshInBackground = {1}",
readLocationEndpoints[0],
canRefreshInBackground);
return(true);
}
if (!string.IsNullOrEmpty(mostPreferredLocation))
{
Uri mostPreferredReadEndpoint;
if (currentLocationInfo.AvailableReadEndpointByLocation.TryGetValue(mostPreferredLocation, out mostPreferredReadEndpoint))
{
if (mostPreferredReadEndpoint != readLocationEndpoints[0])
{
// For reads, we can always refresh in background as we can alternate to
// other available read endpoints
DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since most preferred location {0} is not available for read.", mostPreferredLocation);
return(true);
}
}
else
{
DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since most preferred location {0} is not in available read locations.", mostPreferredLocation);
return(true);
}
}
Uri mostPreferredWriteEndpoint;
ReadOnlyCollection <Uri> writeLocationEndpoints = currentLocationInfo.WriteEndpoints;
if (!this.CanUseMultipleWriteLocations())
{
if (this.IsEndpointUnavailable(writeLocationEndpoints[0], OperationType.Write))
{
// Since most preferred write endpoint is unavailable, we can only refresh in background if
// we have an alternate write endpoint
canRefreshInBackground = writeLocationEndpoints.Count > 1;
DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since most preferred location {0} endpoint {1} is not available for write. canRefreshInBackground = {2}",
mostPreferredLocation,
writeLocationEndpoints[0],
canRefreshInBackground);
return(true);
}
else
{
return(shouldRefresh);
}
}
else if (!string.IsNullOrEmpty(mostPreferredLocation))
{
if (currentLocationInfo.AvailableWriteEndpointByLocation.TryGetValue(mostPreferredLocation, out mostPreferredWriteEndpoint))
{
shouldRefresh |= mostPreferredWriteEndpoint != writeLocationEndpoints[0];
DefaultTrace.TraceInformation("ShouldRefreshEndpoints = {0} since most preferred location {1} is not available for write.", shouldRefresh, mostPreferredLocation);
return(shouldRefresh);
}
else
{
DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since most preferred location {0} is not in available write locations", mostPreferredLocation);
return(true);
}
}
else
{
return(shouldRefresh);
}
}
else
{
return(false);
}
}
