private async Task <DocumentServiceResponse> GetMasterAddressesViaGatewayAsync(
DocumentServiceRequest request,
ResourceType resourceType,
string resourceAddress,
string entryUrl,
bool forceRefresh,
bool useMasterCollectionResolver)
{
INameValueCollection addressQuery = new RequestNameValueCollection
{
{ HttpConstants.QueryStrings.Url, HttpUtility.UrlEncode(entryUrl) }
};
INameValueCollection headers = new RequestNameValueCollection();
if (forceRefresh)
{
headers.Set(HttpConstants.HttpHeaders.ForceRefresh, bool.TrueString);
}
if (useMasterCollectionResolver)
{
headers.Set(HttpConstants.HttpHeaders.UseMasterCollectionResolver, bool.TrueString);
}
if (request.ForceCollectionRoutingMapRefresh)
{
headers.Set(HttpConstants.HttpHeaders.ForceCollectionRoutingMapRefresh, bool.TrueString);
}
addressQuery.Add(HttpConstants.QueryStrings.Filter, this.protocolFilter);
string resourceTypeToSign = PathsHelper.GetResourcePath(resourceType);
headers.Set(HttpConstants.HttpHeaders.XDate, Rfc1123DateTimeCache.UtcNow());
using (ITrace trace = Trace.GetRootTrace(nameof(GetMasterAddressesViaGatewayAsync), TraceComponent.Authorization, TraceLevel.Info))
{
string token = await this.tokenProvider.GetUserAuthorizationTokenAsync(
resourceAddress,
resourceTypeToSign,
HttpConstants.HttpMethods.Get,
headers,
AuthorizationTokenType.PrimaryMasterKey,
trace);
headers.Set(HttpConstants.HttpHeaders.Authorization, token);
Uri targetEndpoint = UrlUtility.SetQuery(this.addressEndpoint, UrlUtility.CreateQuery(addressQuery));
string identifier = GatewayAddressCache.LogAddressResolutionStart(request, targetEndpoint);
using (HttpResponseMessage httpResponseMessage = await this.httpClient.GetAsync(
uri: targetEndpoint,
additionalHeaders: headers,
resourceType: resourceType,
timeoutPolicy: HttpTimeoutPolicyControlPlaneRetriableHotPath.Instance,
clientSideRequestStatistics: request.RequestContext?.ClientRequestStatistics,
cancellationToken: default))
private async Task <DocumentServiceResponse> GetFeedResponseAsync(string resourceLink, ResourceType resourceType, IDocumentClientRetryPolicy retryPolicyInstance, CancellationToken cancellationToken)
{
RequestNameValueCollection headers = new RequestNameValueCollection();
if (this.feedOptions.MaxItemCount.HasValue)
{
headers.PageSize = this.feedOptions.MaxItemCount.ToString();
}
if (this.feedOptions.SessionToken != null)
{
headers.SessionToken = this.feedOptions.SessionToken;
}
if (resourceType.IsPartitioned() && this.feedOptions.PartitionKeyRangeId == null && this.feedOptions.PartitionKey == null)
{
throw new ForbiddenException(RMResources.PartitionKeyRangeIdOrPartitionKeyMustBeSpecified);
}
// On REST level, change feed is using IfNoneMatch/ETag instead of continuation.
if (this.nextIfNoneMatch != null)
{
headers.IfNoneMatch = this.nextIfNoneMatch;
}
if (this.ifModifiedSince != null)
{
headers.IfModifiedSince = this.ifModifiedSince;
}
headers.Set(HttpConstants.HttpHeaders.A_IM, HttpConstants.A_IMHeaderValues.IncrementalFeed);
if (this.feedOptions.PartitionKey != null)
{
PartitionKeyInternal partitionKey = this.feedOptions.PartitionKey.InternalKey;
headers.PartitionKey = partitionKey.ToJsonString();
}
if (this.feedOptions.IncludeTentativeWrites)
{
headers.IncludeTentativeWrites = bool.TrueString;
}
using (DocumentServiceRequest request = this.client.CreateDocumentServiceRequest(
OperationType.ReadFeed,
resourceLink,
resourceType,
headers))
{
if (resourceType.IsPartitioned() && this.feedOptions.PartitionKeyRangeId != null)
{
request.RouteTo(new PartitionKeyRangeIdentity(this.feedOptions.PartitionKeyRangeId));
}
return(await this.client.ReadFeedAsync(request, retryPolicyInstance, cancellationToken));
}
}
private async Task <CollectionRoutingMap> GetRoutingMapForCollectionAsync(
string collectionRid,
CollectionRoutingMap previousRoutingMap,
ITrace trace,
IClientSideRequestStatistics clientSideRequestStatistics)
{
List <PartitionKeyRange> ranges = new List <PartitionKeyRange>();
string changeFeedNextIfNoneMatch = previousRoutingMap?.ChangeFeedNextIfNoneMatch;
HttpStatusCode lastStatusCode = HttpStatusCode.OK;
do
{
INameValueCollection headers = new RequestNameValueCollection();
headers.Set(HttpConstants.HttpHeaders.PageSize, PageSizeString);
headers.Set(HttpConstants.HttpHeaders.A_IM, HttpConstants.A_IMHeaderValues.IncrementalFeed);
if (changeFeedNextIfNoneMatch != null)
{
headers.Set(HttpConstants.HttpHeaders.IfNoneMatch, changeFeedNextIfNoneMatch);
}
RetryOptions retryOptions = new RetryOptions();
using (DocumentServiceResponse response = await BackoffRetryUtility <DocumentServiceResponse> .ExecuteAsync(
() => this.ExecutePartitionKeyRangeReadChangeFeedAsync(collectionRid, headers, trace, clientSideRequestStatistics),
new ResourceThrottleRetryPolicy(retryOptions.MaxRetryAttemptsOnThrottledRequests, retryOptions.MaxRetryWaitTimeInSeconds)))
{
lastStatusCode = response.StatusCode;
changeFeedNextIfNoneMatch = response.Headers[HttpConstants.HttpHeaders.ETag];
FeedResource <PartitionKeyRange> feedResource = response.GetResource <FeedResource <PartitionKeyRange> >();
if (feedResource != null)
{
ranges.AddRange(feedResource);
}
}
}while (lastStatusCode != HttpStatusCode.NotModified);
IEnumerable <Tuple <PartitionKeyRange, ServiceIdentity> > tuples = ranges.Select(range => Tuple.Create(range, (ServiceIdentity)null));
CollectionRoutingMap routingMap;
if (previousRoutingMap == null)
{
// Splits could have happened during change feed query and we might have a mix of gone and new ranges.
HashSet <string> goneRanges = new HashSet <string>(ranges.SelectMany(range => range.Parents ?? Enumerable.Empty <string>()));
routingMap = CollectionRoutingMap.TryCreateCompleteRoutingMap(
tuples.Where(tuple => !goneRanges.Contains(tuple.Item1.Id)),
string.Empty,
changeFeedNextIfNoneMatch);
}
else
{
routingMap = previousRoutingMap.TryCombine(tuples, changeFeedNextIfNoneMatch);
}
if (routingMap == null)
{
// Range information either doesn't exist or is not complete.
throw new NotFoundException($"{DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture)}: GetRoutingMapForCollectionAsync(collectionRid: {collectionRid}), Range information either doesn't exist or is not complete.");
}
trace.AddDatum($"PKRangeCache Info({DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture)})",
new PartitionKeyRangeCacheTraceDatum(
previousContinuationToken: previousRoutingMap?.ChangeFeedNextIfNoneMatch,
continuationToken: routingMap.ChangeFeedNextIfNoneMatch));
return(routingMap);
}
public async Task <INameValueCollection> CreateCommonHeadersAsync(FeedOptions feedOptions)
{
INameValueCollection requestHeaders = new RequestNameValueCollection();
Cosmos.ConsistencyLevel defaultConsistencyLevel = (Cosmos.ConsistencyLevel) await this.Client.GetDefaultConsistencyLevelAsync();
Cosmos.ConsistencyLevel?desiredConsistencyLevel = (Cosmos.ConsistencyLevel?) await this.Client.GetDesiredConsistencyLevelAsync();
if (!string.IsNullOrEmpty(feedOptions.SessionToken) && !ReplicatedResourceClient.IsReadingFromMaster(this.ResourceTypeEnum, OperationType.ReadFeed))
{
if (defaultConsistencyLevel == Cosmos.ConsistencyLevel.Session || (desiredConsistencyLevel.HasValue && desiredConsistencyLevel.Value == Cosmos.ConsistencyLevel.Session))
{
// Query across partitions is not supported today. Master resources (for e.g., database)
// can span across partitions, whereas server resources (viz: collection, document and attachment)
// don't span across partitions. Hence, session token returned by one partition should not be used
// when quering resources from another partition.
// Since master resources can span across partitions, don't send session token to the backend.
// As master resources are sync replicated, we should always get consistent query result for master resources,
// irrespective of the chosen replica.
// For server resources, which don't span partitions, specify the session token
// for correct replica to be chosen for servicing the query result.
requestHeaders[HttpConstants.HttpHeaders.SessionToken] = feedOptions.SessionToken;
}
}
requestHeaders[HttpConstants.HttpHeaders.Continuation] = feedOptions.RequestContinuationToken;
requestHeaders[HttpConstants.HttpHeaders.IsQuery] = bool.TrueString;
// Flow the pageSize only when we are not doing client eval
if (feedOptions.MaxItemCount.HasValue)
{
requestHeaders[HttpConstants.HttpHeaders.PageSize] = feedOptions.MaxItemCount.ToString();
}
requestHeaders[HttpConstants.HttpHeaders.EnableCrossPartitionQuery] = feedOptions.EnableCrossPartitionQuery.ToString();
if (feedOptions.MaxDegreeOfParallelism != 0)
{
requestHeaders[HttpConstants.HttpHeaders.ParallelizeCrossPartitionQuery] = bool.TrueString;
}
if (this.feedOptions.EnableScanInQuery != null)
{
requestHeaders[HttpConstants.HttpHeaders.EnableScanInQuery] = this.feedOptions.EnableScanInQuery.ToString();
}
if (this.feedOptions.EmitVerboseTracesInQuery != null)
{
requestHeaders[HttpConstants.HttpHeaders.EmitVerboseTracesInQuery] = this.feedOptions.EmitVerboseTracesInQuery.ToString();
}
if (this.feedOptions.EnableLowPrecisionOrderBy != null)
{
requestHeaders[HttpConstants.HttpHeaders.EnableLowPrecisionOrderBy] = this.feedOptions.EnableLowPrecisionOrderBy.ToString();
}
if (!string.IsNullOrEmpty(this.feedOptions.FilterBySchemaResourceId))
{
requestHeaders[HttpConstants.HttpHeaders.FilterBySchemaResourceId] = this.feedOptions.FilterBySchemaResourceId;
}
if (this.feedOptions.ResponseContinuationTokenLimitInKb != null)
{
requestHeaders[HttpConstants.HttpHeaders.ResponseContinuationTokenLimitInKB] = this.feedOptions.ResponseContinuationTokenLimitInKb.ToString();
}
if (this.feedOptions.ConsistencyLevel.HasValue)
{
await this.Client.EnsureValidOverwriteAsync(
(Documents.ConsistencyLevel) feedOptions.ConsistencyLevel.Value,
OperationType.ReadFeed,
this.ResourceTypeEnum);
requestHeaders.Set(HttpConstants.HttpHeaders.ConsistencyLevel, this.feedOptions.ConsistencyLevel.Value.ToString());
}
else if (desiredConsistencyLevel.HasValue)
{
requestHeaders.Set(HttpConstants.HttpHeaders.ConsistencyLevel, desiredConsistencyLevel.Value.ToString());
}
if (this.feedOptions.EnumerationDirection.HasValue)
{
requestHeaders.Set(HttpConstants.HttpHeaders.EnumerationDirection, this.feedOptions.EnumerationDirection.Value.ToString());
}
if (this.feedOptions.ReadFeedKeyType.HasValue)
{
requestHeaders.Set(HttpConstants.HttpHeaders.ReadFeedKeyType, this.feedOptions.ReadFeedKeyType.Value.ToString());
}
if (this.feedOptions.StartId != null)
{
requestHeaders.Set(HttpConstants.HttpHeaders.StartId, this.feedOptions.StartId);
}
if (this.feedOptions.EndId != null)
{
requestHeaders.Set(HttpConstants.HttpHeaders.EndId, this.feedOptions.EndId);
}
if (this.feedOptions.StartEpk != null)
{
requestHeaders.Set(HttpConstants.HttpHeaders.StartEpk, this.feedOptions.StartEpk);
}
if (this.feedOptions.EndEpk != null)
{
requestHeaders.Set(HttpConstants.HttpHeaders.EndEpk, this.feedOptions.EndEpk);
}
if (this.feedOptions.PopulateQueryMetrics)
{
requestHeaders[HttpConstants.HttpHeaders.PopulateQueryMetrics] = bool.TrueString;
}
if (this.feedOptions.ForceQueryScan)
{
requestHeaders[HttpConstants.HttpHeaders.ForceQueryScan] = bool.TrueString;
}
if (this.feedOptions.MergeStaticId != null)
{
requestHeaders.Set(HttpConstants.HttpHeaders.MergeStaticId, this.feedOptions.MergeStaticId);
}
if (this.feedOptions.CosmosSerializationFormatOptions != null)
{
requestHeaders[HttpConstants.HttpHeaders.ContentSerializationFormat] = this.feedOptions.CosmosSerializationFormatOptions.ContentSerializationFormat;
}
else if (this.feedOptions.ContentSerializationFormat.HasValue)
{
requestHeaders[HttpConstants.HttpHeaders.ContentSerializationFormat] = this.feedOptions.ContentSerializationFormat.Value.ToString();
}
return(requestHeaders);
}
