// used in Compute
public static void ParseAuthorizationToken(
string authorizationTokenString,
out ReadOnlyMemory <char> typeOutput,
out ReadOnlyMemory <char> versionOutput,
out ReadOnlyMemory <char> tokenOutput)
{
typeOutput = default;
versionOutput = default;
tokenOutput = default;
if (string.IsNullOrEmpty(authorizationTokenString))
{
DefaultTrace.TraceError("Auth token missing");
throw new UnauthorizedException(RMResources.MissingAuthHeader);
}
if (authorizationTokenString.Length > AuthorizationHelper.MaxAuthorizationHeaderSize)
{
throw new UnauthorizedException(RMResources.InvalidAuthHeaderFormat);
}
authorizationTokenString = HttpUtility.UrlDecode(authorizationTokenString);
// Format of the token being deciphered is
// type=<master/resource/system>&ver=<version>&sig=<base64encodedstring>
// Step 1. split the tokens into type/ver/token.
// when parsing for the last token, I use , as a separator to skip any redundant authorization headers
ReadOnlyMemory <char> authorizationToken = authorizationTokenString.AsMemory();
int typeSeparatorPosition = authorizationToken.Span.IndexOf('&');
if (typeSeparatorPosition == -1)
{
throw new UnauthorizedException(RMResources.InvalidAuthHeaderFormat);
}
ReadOnlyMemory <char> authType = authorizationToken.Slice(0, typeSeparatorPosition);
authorizationToken = authorizationToken.Slice(typeSeparatorPosition + 1, authorizationToken.Length - typeSeparatorPosition - 1);
int versionSepartorPosition = authorizationToken.Span.IndexOf('&');
if (versionSepartorPosition == -1)
{
throw new UnauthorizedException(RMResources.InvalidAuthHeaderFormat);
}
ReadOnlyMemory <char> version = authorizationToken.Slice(0, versionSepartorPosition);
authorizationToken = authorizationToken.Slice(versionSepartorPosition + 1, authorizationToken.Length - versionSepartorPosition - 1);
ReadOnlyMemory <char> token = authorizationToken;
int tokenSeparatorPosition = authorizationToken.Span.IndexOf(',');
if (tokenSeparatorPosition != -1)
{
token = authorizationToken.Slice(0, tokenSeparatorPosition);
}
// Step 2. For each token, split to get the right half of '='
// Additionally check for the left half to be the expected scheme type
int typeKeyValueSepartorPosition = authType.Span.IndexOf('=');
if (typeKeyValueSepartorPosition == -1 ||
!authType.Span.Slice(0, typeKeyValueSepartorPosition).SequenceEqual(Constants.Properties.AuthSchemaType.AsSpan()) ||
!authType.Span.Slice(0, typeKeyValueSepartorPosition).ToString().Equals(Constants.Properties.AuthSchemaType, StringComparison.OrdinalIgnoreCase))
{
throw new UnauthorizedException(RMResources.InvalidAuthHeaderFormat);
}
ReadOnlyMemory <char> authTypeValue = authType.Slice(typeKeyValueSepartorPosition + 1);
int versionKeyValueSeparatorPosition = version.Span.IndexOf('=');
if (versionKeyValueSeparatorPosition == -1 ||
!version.Span.Slice(0, versionKeyValueSeparatorPosition).SequenceEqual(Constants.Properties.AuthVersion.AsSpan()) ||
!version.Slice(0, versionKeyValueSeparatorPosition).ToString().Equals(Constants.Properties.AuthVersion, StringComparison.OrdinalIgnoreCase))
{
throw new UnauthorizedException(RMResources.InvalidAuthHeaderFormat);
}
ReadOnlyMemory <char> versionValue = version.Slice(versionKeyValueSeparatorPosition + 1);
int tokenKeyValueSeparatorPosition = token.Span.IndexOf('=');
if (tokenKeyValueSeparatorPosition == -1 ||
!token.Slice(0, tokenKeyValueSeparatorPosition).Span.SequenceEqual(Constants.Properties.AuthSignature.AsSpan()) ||
!token.Slice(0, tokenKeyValueSeparatorPosition).ToString().Equals(Constants.Properties.AuthSignature, StringComparison.OrdinalIgnoreCase))
{
throw new UnauthorizedException(RMResources.InvalidAuthHeaderFormat);
}
ReadOnlyMemory <char> tokenValue = token.Slice(tokenKeyValueSeparatorPosition + 1);
if (authTypeValue.IsEmpty ||
versionValue.IsEmpty ||
tokenValue.IsEmpty)
{
throw new UnauthorizedException(RMResources.InvalidAuthHeaderFormat);
}
typeOutput = authTypeValue;
versionOutput = versionValue;
tokenOutput = tokenValue;
}
private async Task <ResolutionResult> TryResolveServerPartitionAsync(
DocumentServiceRequest request,
CosmosContainerSettings collection,
CollectionRoutingMap routingMap,
bool collectionCacheIsUptodate,
bool collectionRoutingMapCacheIsUptodate,
bool forceRefreshPartitionAddresses,
CancellationToken cancellationToken)
{
// Check if this request partitionkeyrange-aware routing logic. We cannot retry here in this case
// and need to bubble up errors.
if (request.PartitionKeyRangeIdentity != null)
{
return(await this.TryResolveServerPartitionByPartitionKeyRangeIdAsync(
request,
collection,
routingMap,
collectionCacheIsUptodate,
collectionRoutingMapCacheIsUptodate,
forceRefreshPartitionAddresses,
cancellationToken));
}
if (!request.ResourceType.IsPartitioned() &&
!(request.ResourceType == ResourceType.StoredProcedure && request.OperationType == OperationType.ExecuteJavaScript) &&
// Collection head is sent internally for strong consistency given routing hints from original requst, which is for partitioned resource.
!(request.ResourceType == ResourceType.Collection && request.OperationType == OperationType.Head))
{
DefaultTrace.TraceCritical(
"Shouldn't come here for non partitioned resources. resourceType : {0}, operationtype:{1}, resourceaddress:{2}",
request.ResourceType,
request.OperationType,
request.ResourceAddress);
throw new InternalServerErrorException(RMResources.InternalServerError)
{
ResourceAddress = request.ResourceAddress
};
}
PartitionKeyRange range;
string partitionKeyString = request.Headers[HttpConstants.HttpHeaders.PartitionKey];
if (partitionKeyString != null)
{
range = this.TryResolveServerPartitionByPartitionKey(
request,
partitionKeyString,
collectionCacheIsUptodate,
collection,
routingMap);
}
else if (request.Properties != null && request.Properties.TryGetValue(
WFConstants.BackendHeaders.EffectivePartitionKeyString,
out object effectivePartitionKeyStringObject))
{
string effectivePartitionKeyString = effectivePartitionKeyStringObject as string;
if (string.IsNullOrEmpty(effectivePartitionKeyString))
{
throw new ArgumentOutOfRangeException(nameof(effectivePartitionKeyString));
}
range = routingMap.GetRangeByEffectivePartitionKey(effectivePartitionKeyString);
}
else
{
range = this.TryResolveSinglePartitionCollection(request, routingMap, collectionCacheIsUptodate);
}
if (range == null)
{
// Collection cache or routing map cache is potentially outdated. Return null -
// upper logic will refresh cache and retry.
return(null);
}
ServiceIdentity serviceIdentity = routingMap.TryGetInfoByPartitionKeyRangeId(range.Id);
PartitionAddressInformation addresses = await this.addressCache.TryGetAddresses(
request,
new PartitionKeyRangeIdentity(collection.ResourceId, range.Id),
serviceIdentity,
forceRefreshPartitionAddresses,
cancellationToken);
if (addresses == null)
{
DefaultTrace.TraceVerbose(
"Could not resolve addresses for identity {0}/{1}. Potentially collection cache or routing map cache is outdated. Return null - upper logic will refresh and retry. ",
new PartitionKeyRangeIdentity(collection.ResourceId, range.Id),
serviceIdentity);
return(null);
}
return(new ResolutionResult(range, addresses, serviceIdentity));
}
/// <summary>
/// A method to create the cosmos client
/// </summary>
/// <remarks>
/// Setting this property after sending any request won't have any effect.
/// </remarks>
/// <returns>An instance of <see cref="CosmosClient"/>.</returns>
public virtual CosmosClient Build()
{
DefaultTrace.TraceInformation($"CosmosClientBuilder.Build with configuration: {this.clientOptions.GetSerializedConfiguration()}");
return(new CosmosClient(this.clientOptions));
}
public static IReadOnlyList <Range <string> > GetProvidedPartitionKeyRanges(
string querySpecJsonString,
bool enableCrossPartitionQuery,
bool parallelizeCrossPartitionQuery,
bool isContinuationExpected,
bool hasLogicalPartitionKey,
bool allowDCount,
bool allowNonValueAggregates,
bool useSystemPrefix,
PartitionKeyDefinition partitionKeyDefinition,
QueryPartitionProvider queryPartitionProvider,
string clientApiVersion,
out QueryInfo queryInfo)
{
if (querySpecJsonString == null)
{
throw new ArgumentNullException(nameof(querySpecJsonString));
}
if (partitionKeyDefinition == null)
{
throw new ArgumentNullException(nameof(partitionKeyDefinition));
}
if (queryPartitionProvider == null)
{
throw new ArgumentNullException(nameof(queryPartitionProvider));
}
TryCatch <PartitionedQueryExecutionInfo> tryGetPartitionQueryExecutionInfo = queryPartitionProvider.TryGetPartitionedQueryExecutionInfo(
querySpecJsonString: querySpecJsonString,
partitionKeyDefinition: partitionKeyDefinition,
requireFormattableOrderByQuery: VersionUtility.IsLaterThan(clientApiVersion, HttpConstants.VersionDates.v2016_11_14),
isContinuationExpected: isContinuationExpected,
allowNonValueAggregateQuery: allowNonValueAggregates,
hasLogicalPartitionKey: hasLogicalPartitionKey,
allowDCount: allowDCount,
useSystemPrefix: useSystemPrefix);
if (!tryGetPartitionQueryExecutionInfo.Succeeded)
{
throw new BadRequestException(tryGetPartitionQueryExecutionInfo.Exception);
}
PartitionedQueryExecutionInfo queryExecutionInfo = tryGetPartitionQueryExecutionInfo.Result;
if (queryExecutionInfo?.QueryRanges == null ||
queryExecutionInfo.QueryInfo == null ||
queryExecutionInfo.QueryRanges.Any(range => range.Min == null || range.Max == null))
{
DefaultTrace.TraceInformation("QueryPartitionProvider returned bad query info");
}
bool isSinglePartitionQuery = queryExecutionInfo.QueryRanges.Count == 1 && queryExecutionInfo.QueryRanges[0].IsSingleValue;
bool queryFansOutToMultiplePartitions = partitionKeyDefinition.Paths.Count > 0 && !isSinglePartitionQuery;
if (queryFansOutToMultiplePartitions)
{
if (!enableCrossPartitionQuery)
{
BadRequestException exception = new BadRequestException(RMResources.CrossPartitionQueryDisabled);
exception.Error.AdditionalErrorInfo = JsonConvert.SerializeObject(queryExecutionInfo);
throw exception;
}
else
{
bool queryNotServiceableByGateway = parallelizeCrossPartitionQuery ||
queryExecutionInfo.QueryInfo.HasTop ||
queryExecutionInfo.QueryInfo.HasOrderBy ||
queryExecutionInfo.QueryInfo.HasAggregates ||
queryExecutionInfo.QueryInfo.HasDistinct ||
queryExecutionInfo.QueryInfo.HasOffset ||
queryExecutionInfo.QueryInfo.HasLimit ||
queryExecutionInfo.QueryInfo.HasGroupBy;
if (queryNotServiceableByGateway)
{
if (!IsSupportedPartitionedQueryExecutionInfo(queryExecutionInfo, clientApiVersion))
{
BadRequestException exception = new BadRequestException(RMResources.UnsupportedCrossPartitionQuery);
exception.Error.AdditionalErrorInfo = JsonConvert.SerializeObject(queryExecutionInfo);
throw exception;
}
else if (queryExecutionInfo.QueryInfo.HasAggregates && !IsAggregateSupportedApiVersion(clientApiVersion))
{
BadRequestException exception = new BadRequestException(RMResources.UnsupportedCrossPartitionQueryWithAggregate);
exception.Error.AdditionalErrorInfo = JsonConvert.SerializeObject(queryExecutionInfo);
throw exception;
}
else
{
DocumentClientException exception = new DocumentClientException(
RMResources.UnsupportedCrossPartitionQuery,
HttpStatusCode.BadRequest,
SubStatusCodes.CrossPartitionQueryNotServable);
exception.Error.AdditionalErrorInfo = JsonConvert.SerializeObject(queryExecutionInfo);
throw exception;
}
}
}
}
else
{
if (queryExecutionInfo.QueryInfo.HasAggregates && !isContinuationExpected)
{
// For single partition query with aggregate functions and no continuation expected,
// we would try to accumulate the results for them on the SDK, if supported.
if (IsAggregateSupportedApiVersion(clientApiVersion))
{
DocumentClientException exception = new DocumentClientException(
RMResources.UnsupportedQueryWithFullResultAggregate,
HttpStatusCode.BadRequest,
SubStatusCodes.CrossPartitionQueryNotServable);
exception.Error.AdditionalErrorInfo = JsonConvert.SerializeObject(queryExecutionInfo);
throw exception;
}
else
{
throw new BadRequestException(RMResources.UnsupportedQueryWithFullResultAggregate);
}
}
else if (queryExecutionInfo.QueryInfo.HasDistinct)
{
// If the query has DISTINCT then we have to reject it since the backend only returns
// elements that are DISTINCT within a page and we need the client to do post distinct processing
DocumentClientException exception = new DocumentClientException(
RMResources.UnsupportedCrossPartitionQuery,
HttpStatusCode.BadRequest,
SubStatusCodes.CrossPartitionQueryNotServable);
exception.Error.AdditionalErrorInfo = JsonConvert.SerializeObject(queryExecutionInfo);
throw exception;
}
else if (queryExecutionInfo.QueryInfo.HasGroupBy)
{
// If the query has GROUP BY then we have to reject it since the backend only returns
// elements that are grouped within a page and we need the client to merge the groupings
DocumentClientException exception = new DocumentClientException(
RMResources.UnsupportedCrossPartitionQuery,
HttpStatusCode.BadRequest,
SubStatusCodes.CrossPartitionQueryNotServable);
exception.Error.AdditionalErrorInfo = JsonConvert.SerializeObject(queryExecutionInfo);
throw exception;
}
}
queryInfo = queryExecutionInfo.QueryInfo;
return(queryExecutionInfo.QueryRanges);
}
/// <summary>
/// A method to create the cosmos client
/// </summary>
/// <remarks>
/// Setting this property after sending any request won't have any effect.
/// </remarks>
/// <returns>An instance of <see cref="CosmosClient"/>.</returns>
public CosmosClient Build()
{
DefaultTrace.TraceInformation($"CosmosClientBuilder.Build with configuration: {this.clientOptions.GetSerializedConfiguration()}");
return(new CosmosClient(this.accountEndpoint, this.accountKey, this.clientOptions));
}
public virtual async Task DispatchAsync(
BatchPartitionMetric partitionMetric,
CancellationToken cancellationToken = default(CancellationToken))
{
this.interlockIncrementCheck.EnterLockCheck();
PartitionKeyRangeServerBatchRequest serverRequest = null;
ArraySegment <ItemBatchOperation> pendingOperations;
try
{
try
{
// HybridRow serialization might leave some pending operations out of the batch
Tuple <PartitionKeyRangeServerBatchRequest, ArraySegment <ItemBatchOperation> > createRequestResponse = await this.CreateServerRequestAsync(cancellationToken);
serverRequest = createRequestResponse.Item1;
pendingOperations = createRequestResponse.Item2;
// Any overflow goes to a new batch
foreach (ItemBatchOperation operation in pendingOperations)
{
await this.retrier(operation, cancellationToken);
}
}
catch (Exception ex)
{
// Exceptions happening during request creation, fail the entire list
foreach (ItemBatchOperation itemBatchOperation in this.batchOperations)
{
itemBatchOperation.Context.Fail(this, ex);
}
throw;
}
try
{
Stopwatch stopwatch = Stopwatch.StartNew();
PartitionKeyRangeBatchExecutionResult result = await this.executor(serverRequest, cancellationToken);
int numThrottle = result.ServerResponse.Any(r => r.StatusCode == (System.Net.HttpStatusCode)StatusCodes.TooManyRequests) ? 1 : 0;
partitionMetric.Add(
numberOfDocumentsOperatedOn: result.ServerResponse.Count,
timeTakenInMilliseconds: stopwatch.ElapsedMilliseconds,
numberOfThrottles: numThrottle);
using (PartitionKeyRangeBatchResponse batchResponse = new PartitionKeyRangeBatchResponse(serverRequest.Operations.Count, result.ServerResponse, this.serializerCore))
{
foreach (ItemBatchOperation itemBatchOperation in batchResponse.Operations)
{
TransactionalBatchOperationResult response = batchResponse[itemBatchOperation.OperationIndex];
// Bulk has diagnostics per a item operation.
// Batch has a single diagnostics for the execute operation
if (itemBatchOperation.DiagnosticsContext != null)
{
response.DiagnosticsContext = itemBatchOperation.DiagnosticsContext;
response.DiagnosticsContext.AddDiagnosticsInternal(batchResponse.DiagnosticsContext);
}
else
{
response.DiagnosticsContext = batchResponse.DiagnosticsContext;
}
if (!response.IsSuccessStatusCode)
{
Documents.ShouldRetryResult shouldRetry = await itemBatchOperation.Context.ShouldRetryAsync(response, cancellationToken);
if (shouldRetry.ShouldRetry)
{
await this.retrier(itemBatchOperation, cancellationToken);
continue;
}
}
itemBatchOperation.Context.Complete(this, response);
}
}
}
catch (Exception ex)
{
// Exceptions happening during execution fail all the Tasks part of the request (excluding overflow)
foreach (ItemBatchOperation itemBatchOperation in serverRequest.Operations)
{
itemBatchOperation.Context.Fail(this, ex);
}
throw;
}
}
catch (Exception ex)
{
DefaultTrace.TraceError("Exception during BatchAsyncBatcher: {0}", ex);
}
finally
{
this.batchOperations.Clear();
this.dispatched = true;
}
}
private PartitionKeyRange TryResolveSinglePartitionCollection(
DocumentServiceRequest request,
ContainerProperties collection,
CollectionRoutingMap routingMap,
bool collectionCacheIsUptoDate)
{
// Neither partitionkey nor partitionkeyrangeid is specified.
// Three options here:
// * This is non-partitioned collection and old client SDK which doesn't send partition key. In
// this case there's single entry in routing map. But can be multiple entries if before that
// existed partitioned collection with same name.
// * This is partitioned collection and old client SDK which doesn't send partition key.
// In this case there can be multiple ranges in routing map.
// * This is partitioned collection and this is custom written REST sdk, which has a bug and doesn't send
// partition key.
// We cannot know for sure whether this is partitioned collection or not, because
// partition key definition cache can be outdated.
// So we route request to the first partition. If this is non-partitioned collection - request will succeed.
// If it is partitioned collection - backend will return bad request as partition key header is required in this case.
if (routingMap.OrderedPartitionKeyRanges.Count == 1)
{
return(routingMap.OrderedPartitionKeyRanges.Single());
}
if (collectionCacheIsUptoDate)
{
// If the current collection is user-partitioned collection
if (collection.PartitionKey.Paths.Count >= 1 &&
!collection.PartitionKey.IsSystemKey.GetValueOrDefault(false))
{
throw new BadRequestException(RMResources.MissingPartitionKeyValue)
{
ResourceAddress = request.ResourceAddress
};
}
else if (routingMap.OrderedPartitionKeyRanges.Count > 1)
{
// With migrated-fixed-collection, it is possible to have multiple partition key ranges
// due to parallel usage of V3 SDK and a possible storage or throughput split
// The current client might be legacy and not aware of this.
// In such case route the request to the first partition
return(this.TryResolveServerPartitionByPartitionKey(
request,
"[]", // This corresponds to first partition
collectionCacheIsUptoDate,
collection,
routingMap));
}
else
{
// routingMap.OrderedPartitionKeyRanges.Count == 0
// Should never come here.
DefaultTrace.TraceCritical(
"No Partition Key ranges present for the collection {0}", collection.ResourceId);
throw new InternalServerErrorException(RMResources.InternalServerError)
{
ResourceAddress = request.ResourceAddress
};
}
}
else
{
return(null);
}
}
public virtual void MarkEndpointUnavailableForRead(Uri endpoint)
{
DefaultTrace.TraceInformation("Marking endpoint {0} unavailable for read", endpoint);
this.locationCache.MarkEndpointUnavailableForRead(endpoint);
}
public virtual async Task DispatchAsync(CancellationToken cancellationToken = default(CancellationToken))
{
this.interlockIncrementCheck.EnterLockCheck();
PartitionKeyRangeServerBatchRequest serverRequest = null;
ArraySegment <ItemBatchOperation> pendingOperations;
try
{
try
{
// HybridRow serialization might leave some pending operations out of the batch
Tuple <PartitionKeyRangeServerBatchRequest, ArraySegment <ItemBatchOperation> > createRequestResponse = await this.CreateServerRequestAsync(cancellationToken);
serverRequest = createRequestResponse.Item1;
pendingOperations = createRequestResponse.Item2;
// Any overflow goes to a new batch
foreach (ItemBatchOperation operation in pendingOperations)
{
await this.retrier(operation, cancellationToken);
}
}
catch (Exception ex)
{
// Exceptions happening during request creation, fail the entire list
foreach (ItemBatchOperation itemBatchOperation in this.batchOperations)
{
itemBatchOperation.Context.Fail(this, ex);
}
throw;
}
try
{
PartitionKeyRangeBatchExecutionResult result = await this.executor(serverRequest, cancellationToken);
using (PartitionKeyRangeBatchResponse batchResponse = new PartitionKeyRangeBatchResponse(serverRequest.Operations.Count, result.ServerResponse, this.cosmosSerializer))
{
foreach (ItemBatchOperation itemBatchOperation in batchResponse.Operations)
{
BatchOperationResult response = batchResponse[itemBatchOperation.OperationIndex];
itemBatchOperation.Context.Diagnostics.AppendDiagnostics(batchResponse.Diagnostics);
if (!response.IsSuccessStatusCode)
{
Documents.ShouldRetryResult shouldRetry = await itemBatchOperation.Context.ShouldRetryAsync(response, cancellationToken);
if (shouldRetry.ShouldRetry)
{
await this.retrier(itemBatchOperation, cancellationToken);
continue;
}
}
itemBatchOperation.Context.Complete(this, response);
}
}
}
catch (Exception ex)
{
// Exceptions happening during execution fail all the Tasks part of the request (excluding overflow)
foreach (ItemBatchOperation itemBatchOperation in serverRequest.Operations)
{
itemBatchOperation.Context.Fail(this, ex);
}
throw;
}
}
catch (Exception ex)
{
DefaultTrace.TraceError("Exception during BatchAsyncBatcher: {0}", ex);
}
finally
{
this.batchOperations.Clear();
this.dispached = true;
}
}
/// <summary>
/// Traces a verbose message with the proper formatting.
/// </summary>
/// <param name="message">The message to trace.</param>
protected void TraceVerbose(string message)
{
DefaultTrace.TraceVerbose(this.GetTrace(message));
}
/// <summary>
/// Traces information with the proper formatting.
/// </summary>
/// <param name="message">The message to trace.</param>
protected void TraceInformation(string message)
{
DefaultTrace.TraceInformation(this.GetTrace(message));
}
/// <summary>
/// Traces a warning with the proper formatting.
/// </summary>
/// <param name="message">The message to trace.</param>
protected void TraceWarning(string message)
{
DefaultTrace.TraceWarning(this.GetTrace(message));
}
private async Task <ShouldRetryResult> ShouldRetryOnEndpointFailureAsync(
bool isReadRequest,
bool markBothReadAndWriteAsUnavailable,
bool forceRefresh,
bool retryOnPreferredLocations)
{
if (!this.enableEndpointDiscovery || this.failoverRetryCount > MaxRetryCount)
{
DefaultTrace.TraceInformation("ClientRetryPolicy: ShouldRetryOnEndpointFailureAsync() Not retrying. Retry count = {0}, Endpoint = {1}",
this.failoverRetryCount,
this.locationEndpoint?.ToString() ?? string.Empty);
return(ShouldRetryResult.NoRetry());
}
this.failoverRetryCount++;
if (this.locationEndpoint != null)
{
if (isReadRequest || markBothReadAndWriteAsUnavailable)
{
this.globalEndpointManager.MarkEndpointUnavailableForRead(this.locationEndpoint);
}
if (!isReadRequest || markBothReadAndWriteAsUnavailable)
{
this.globalEndpointManager.MarkEndpointUnavailableForWrite(this.locationEndpoint);
}
}
TimeSpan retryDelay = TimeSpan.Zero;
if (!isReadRequest)
{
DefaultTrace.TraceInformation("ClientRetryPolicy: Failover happening. retryCount {0}", this.failoverRetryCount);
if (this.failoverRetryCount > 1)
{
//if retried both endpoints, follow regular retry interval.
retryDelay = TimeSpan.FromMilliseconds(ClientRetryPolicy.RetryIntervalInMS);
}
}
else
{
retryDelay = TimeSpan.FromMilliseconds(ClientRetryPolicy.RetryIntervalInMS);
}
await this.globalEndpointManager.RefreshLocationAsync(null, forceRefresh);
int retryLocationIndex = this.failoverRetryCount; // Used to generate a round-robin effect
if (retryOnPreferredLocations)
{
retryLocationIndex = 0; // When the endpoint is marked as unavailable, it is moved to the bottom of the preferrence list
}
this.retryContext = new RetryContext
{
RetryLocationIndex = retryLocationIndex,
RetryRequestOnPreferredLocations = retryOnPreferredLocations,
};
return(ShouldRetryResult.RetryAfter(retryDelay));
}
private async Task <ShouldRetryResult> ShouldRetryInternalAsync(
HttpStatusCode?statusCode,
SubStatusCodes?subStatusCode)
{
if (!statusCode.HasValue &&
(!subStatusCode.HasValue ||
subStatusCode.Value == SubStatusCodes.Unknown))
{
return(null);
}
// Received request timeout
if (statusCode == HttpStatusCode.RequestTimeout)
{
DefaultTrace.TraceWarning("ClientRetryPolicy: RequestTimeout. Failed Location: {0}; ResourceAddress: {1}",
this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty,
this.documentServiceRequest?.ResourceAddress ?? string.Empty);
// Mark the partition key range as unavailable to retry future request on a new region.
this.partitionKeyRangeLocationCache.TryMarkEndpointUnavailableForPartitionKeyRange(
this.documentServiceRequest);
}
// Received 403.3 on write region, initiate the endpoint rediscovery
if (statusCode == HttpStatusCode.Forbidden &&
subStatusCode == SubStatusCodes.WriteForbidden)
{
// It's a write forbidden so it safe to retry
if (this.partitionKeyRangeLocationCache.TryMarkEndpointUnavailableForPartitionKeyRange(
this.documentServiceRequest))
{
return(ShouldRetryResult.RetryAfter(TimeSpan.Zero));
}
DefaultTrace.TraceWarning("ClientRetryPolicy: Endpoint not writable. Refresh cache and retry. Failed Location: {0}; ResourceAddress: {1}",
this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty,
this.documentServiceRequest?.ResourceAddress ?? string.Empty);
return(await this.ShouldRetryOnEndpointFailureAsync(
isReadRequest : false,
markBothReadAndWriteAsUnavailable : false,
forceRefresh : true,
retryOnPreferredLocations : false));
}
// Regional endpoint is not available yet for reads (e.g. add/ online of region is in progress)
if (statusCode == HttpStatusCode.Forbidden &&
subStatusCode == SubStatusCodes.DatabaseAccountNotFound &&
(this.isReadRequest || this.canUseMultipleWriteLocations))
{
DefaultTrace.TraceWarning("ClientRetryPolicy: Endpoint not available for reads. Refresh cache and retry. Failed Location: {0}; ResourceAddress: {1}",
this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty,
this.documentServiceRequest?.ResourceAddress ?? string.Empty);
return(await this.ShouldRetryOnEndpointFailureAsync(
isReadRequest : this.isReadRequest,
markBothReadAndWriteAsUnavailable : false,
forceRefresh : false,
retryOnPreferredLocations : false));
}
if (statusCode == HttpStatusCode.NotFound &&
subStatusCode == SubStatusCodes.ReadSessionNotAvailable)
{
return(this.ShouldRetryOnSessionNotAvailable());
}
// Received 503.0 due to client connect timeout or Gateway
if (statusCode == HttpStatusCode.ServiceUnavailable &&
subStatusCode == SubStatusCodes.Unknown)
{
DefaultTrace.TraceWarning("ClientRetryPolicy: ServiceUnavailable. Refresh cache and retry. Failed Location: {0}; ResourceAddress: {1}",
this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty,
this.documentServiceRequest?.ResourceAddress ?? string.Empty);
// Mark the partition as unavailable.
// Let the ClientRetry logic decide if the request should be retried
this.partitionKeyRangeLocationCache.TryMarkEndpointUnavailableForPartitionKeyRange(
this.documentServiceRequest);
return(this.ShouldRetryOnServiceUnavailable());
}
return(null);
}
/// <summary>
/// If a query encounters split up resuming using continuation, we need to regenerate the continuation tokens.
/// Specifically, since after split we will have new set of ranges, we need to remove continuation token for the
/// parent partition and introduce continuation token for the child partitions.
///
/// This function does that. Also in that process, we also check validity of the input continuation tokens. For example,
/// even after split the boundary ranges of the child partitions should match with the parent partitions. If the Min and Max
/// range of a target partition in the continuation token was Min1 and Max1. Then the Min and Max range info for the two
/// corresponding child partitions C1Min, C1Max, C2Min, and C2Max should follow the constrain below:
/// PMax = C2Max > C2Min > C1Max > C1Min = PMin.
///
/// Note that,
/// this is assuming the fact that the target partition was split once. But, in reality, the target partition might be split
/// multiple times
/// </summary>
/// <Remarks>
/// The code assumes that merge doesn't happen
/// </Remarks>
protected int FindTargetRangeAndExtractContinuationTokens <TContinuationToken>(
List <PartitionKeyRange> partitionKeyRanges,
IEnumerable <Tuple <TContinuationToken, Range <string> > > suppliedContinuationTokens,
out Dictionary <string, TContinuationToken> targetRangeToContinuationTokenMap)
{
targetRangeToContinuationTokenMap = new Dictionary <string, TContinuationToken>();
bool foundInitialRange = false;
int index = 0;
int minIndex = -1;
foreach (Tuple <TContinuationToken, Range <string> > tuple in suppliedContinuationTokens)
{
if (!foundInitialRange)
{
PartitionKeyRange targetRange = new PartitionKeyRange
{
MinInclusive = tuple.Item2.Min,
MaxExclusive = tuple.Item2.Max
};
minIndex = partitionKeyRanges.BinarySearch(
targetRange,
Comparer <PartitionKeyRange> .Create((range1, range2) => string.CompareOrdinal(range1.MinInclusive, range2.MinInclusive)));
if (minIndex < 0)
{
DefaultTrace.TraceWarning(
string.Format(
CultureInfo.InvariantCulture,
"{0}, CorrelatedActivityId: {2} | Invalid format for continuation token {1} for OrderBy~Context.",
DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture),
tuple.Item1.ToString(),
this.CorrelatedActivityId));
throw new BadRequestException(RMResources.InvalidContinuationToken);
}
index = minIndex;
foundInitialRange = true;
}
if (partitionKeyRanges[index].ToRange().Equals(tuple.Item2))
{
targetRangeToContinuationTokenMap.Add(partitionKeyRanges[index++].Id, tuple.Item1);
}
else
{
bool canConsume = true;
if (string.CompareOrdinal(partitionKeyRanges[index].MinInclusive, tuple.Item2.Min) == 0 &&
string.CompareOrdinal(tuple.Item2.Max, partitionKeyRanges[index].MaxExclusive) > 0)
{
while (index < partitionKeyRanges.Count &&
string.CompareOrdinal(partitionKeyRanges[index].MaxExclusive, tuple.Item2.Max) <= 0)
{
targetRangeToContinuationTokenMap.Add(partitionKeyRanges[index++].Id, tuple.Item1);
}
if (index > 0 && string.CompareOrdinal(partitionKeyRanges[index - 1].MaxExclusive, tuple.Item2.Max) != 0)
{
canConsume = false;
}
}
else
{
canConsume = false;
}
if (!canConsume)
{
DefaultTrace.TraceWarning(
string.Format(
CultureInfo.InvariantCulture,
"{0}, CorrelatedActivityId: {1} | Invalid format for continuation token {2} for OrderBy~Context.",
DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture),
this.CorrelatedActivityId,
tuple.Item1.ToString()));
throw new BadRequestException(RMResources.InvalidContinuationToken);
}
}
if (index >= partitionKeyRanges.Count)
{
break;
}
}
return(minIndex);
}
private void SetSessionToken(ResourceId resourceId, string collectionName, string encodedToken)
{
string partitionKeyRangeId;
ISessionToken token;
if (VersionUtility.IsLaterThan(HttpConstants.Versions.CurrentVersion, HttpConstants.Versions.v2015_12_16))
{
string[] tokenParts = encodedToken.Split(':');
partitionKeyRangeId = tokenParts[0];
token = SessionTokenHelper.Parse(tokenParts[1], HttpConstants.Versions.CurrentVersion);
}
else
{
//todo: elasticcollections remove after first upgrade.
partitionKeyRangeId = "0";
token = SessionTokenHelper.Parse(encodedToken, HttpConstants.Versions.CurrentVersion);
}
DefaultTrace.TraceVerbose("Update Session token {0} {1} {2}", resourceId.UniqueDocumentCollectionId, collectionName, token);
bool isKnownCollection = false;
this.rwlock.EnterReadLock();
try
{
ulong resolvedCollectionResourceId;
string resolvedCollectionName;
isKnownCollection = this.collectionNameByResourceId.TryGetValue(collectionName, out resolvedCollectionResourceId) &&
this.collectionResourceIdByName.TryGetValue(resourceId.UniqueDocumentCollectionId, out resolvedCollectionName) &&
resolvedCollectionResourceId == resourceId.UniqueDocumentCollectionId &&
resolvedCollectionName == collectionName;
if (isKnownCollection)
{
this.AddSessionToken(resourceId.UniqueDocumentCollectionId, partitionKeyRangeId, token);
}
}
finally
{
this.rwlock.ExitReadLock();
}
if (!isKnownCollection)
{
this.rwlock.EnterWriteLock();
try
{
ulong resolvedCollectionResourceId;
if (this.collectionNameByResourceId.TryGetValue(collectionName, out resolvedCollectionResourceId))
{
string ignoreString;
ConcurrentDictionary <string, ISessionToken> ignored;
this.sessionTokensRIDBased.TryRemove(resolvedCollectionResourceId, out ignored);
this.collectionResourceIdByName.TryRemove(resolvedCollectionResourceId, out ignoreString);
}
this.collectionNameByResourceId[collectionName] = resourceId.UniqueDocumentCollectionId;
this.collectionResourceIdByName[resourceId.UniqueDocumentCollectionId] = collectionName;
this.AddSessionToken(resourceId.UniqueDocumentCollectionId, partitionKeyRangeId, token);
}
finally
{
this.rwlock.ExitWriteLock();
}
}
}
private void OnDocumentProducerCompleteFetching(
DocumentProducer <T> producer,
int size,
double resourceUnitUsage,
QueryMetrics queryMetrics,
long responseLengthBytes,
CancellationToken token)
{
// Update charge and states
this.chargeTracker.AddCharge(resourceUnitUsage);
Interlocked.Add(ref this.totalBufferedItems, size);
Interlocked.Increment(ref this.totalRequestRoundTrips);
this.IncrementResponseLengthBytes(responseLengthBytes);
this.partitionedQueryMetrics.Add(Tuple.Create(producer.TargetRange.Id, queryMetrics));
//Check to see if we can buffer more item
long countToAdd = size - this.FreeItemSpace;
if (countToAdd > 0 &&
this.actualMaxBufferedItemCount < MaxixmumDynamicMaxBufferedItemCountValue - countToAdd)
{
DefaultTrace.TraceVerbose(string.Format(
CultureInfo.InvariantCulture,
"{0}, CorrelatedActivityId: {4} | Id: {1}, increasing MaxBufferedItemCount {2} by {3}.",
DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture),
producer.TargetRange.Id,
this.actualMaxBufferedItemCount,
countToAdd,
this.CorrelatedActivityId));
countToAdd += this.actualMaxBufferedItemCount;
}
// Adjust max DoP if necessary
this.AdjustTaskSchedulerMaximumConcurrencyLevel();
// Fetch again if necessary
if (!producer.FetchedAll)
{
if (producer.PageSize < this.actualMaxPageSize)
{
producer.PageSize = Math.Min((long)(producer.PageSize * DynamicPageSizeAdjustmentFactor), this.actualMaxPageSize);
Debug.Assert(producer.PageSize >= 0 && producer.PageSize <= int.MaxValue, string.Format("producer.PageSize is invalid at {0}", producer.PageSize));
}
if (this.ShouldPrefetch &&
this.FreeItemSpace - producer.NormalizedPageSize > 0)
{
producer.TryScheduleFetch();
}
}
DefaultTrace.TraceVerbose(string.Format(
CultureInfo.InvariantCulture,
"{0}, CorrelatedActivityId: {5} | Id: {1}, size: {2}, resourceUnitUsage: {3}, taskScheduler.CurrentRunningTaskCount: {4}",
DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture),
producer.TargetRange.Id,
size,
resourceUnitUsage,
this.TaskScheduler.CurrentRunningTaskCount,
this.CorrelatedActivityId));
}
public void MarkEndpointUnavailableForWrite(Uri endpoint)
{
DefaultTrace.TraceInformation("Marking endpoint {0} unavailable for Write", endpoint);
this.locationCache.MarkEndpointUnavailableForWrite(endpoint);
}
public bool ShouldRefreshEndpoints(out bool canRefreshInBackground)
{
canRefreshInBackground = true;
DatabaseAccountLocationsInfo currentLocationInfo = this.locationInfo;
string mostPreferredLocation = currentLocationInfo.PreferredLocations.FirstOrDefault();
// we should schedule refresh in background if we are unable to target the user's most preferredLocation.
if (this.enableEndpointDiscovery)
{
// Refresh if client opts-in to useMultipleWriteLocations but server-side setting is disabled
bool shouldRefresh = this.useMultipleWriteLocations && !this.enableMultipleWriteLocations;
ReadOnlyCollection <Uri> readLocationEndpoints = currentLocationInfo.ReadEndpoints;
if (this.IsEndpointUnavailable(readLocationEndpoints[0], OperationType.Read))
{
canRefreshInBackground = readLocationEndpoints.Count > 1;
DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since the first read endpoint {0} is not available for read. canRefreshInBackground = {1}",
readLocationEndpoints[0],
canRefreshInBackground);
return(true);
}
if (!string.IsNullOrEmpty(mostPreferredLocation))
{
Uri mostPreferredReadEndpoint;
if (currentLocationInfo.AvailableReadEndpointByLocation.TryGetValue(mostPreferredLocation, out mostPreferredReadEndpoint))
{
if (mostPreferredReadEndpoint != readLocationEndpoints[0])
{
// For reads, we can always refresh in background as we can alternate to
// other available read endpoints
DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since most preferred location {0} is not available for read.", mostPreferredLocation);
return(true);
}
}
else
{
DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since most preferred location {0} is not in available read locations.", mostPreferredLocation);
return(true);
}
}
Uri mostPreferredWriteEndpoint;
ReadOnlyCollection <Uri> writeLocationEndpoints = currentLocationInfo.WriteEndpoints;
if (!this.CanUseMultipleWriteLocations())
{
if (this.IsEndpointUnavailable(writeLocationEndpoints[0], OperationType.Write))
{
// Since most preferred write endpoint is unavailable, we can only refresh in background if
// we have an alternate write endpoint
canRefreshInBackground = writeLocationEndpoints.Count > 1;
DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since most preferred location {0} endpoint {1} is not available for write. canRefreshInBackground = {2}",
mostPreferredLocation,
writeLocationEndpoints[0],
canRefreshInBackground);
return(true);
}
else
{
return(shouldRefresh);
}
}
else if (!string.IsNullOrEmpty(mostPreferredLocation))
{
if (currentLocationInfo.AvailableWriteEndpointByLocation.TryGetValue(mostPreferredLocation, out mostPreferredWriteEndpoint))
{
shouldRefresh |= mostPreferredWriteEndpoint != writeLocationEndpoints[0];
DefaultTrace.TraceInformation("ShouldRefreshEndpoints = {0} since most preferred location {1} is not available for write.", shouldRefresh, mostPreferredLocation);
return(shouldRefresh);
}
else
{
DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since most preferred location {0} is not in available write locations", mostPreferredLocation);
return(true);
}
}
else
{
return(shouldRefresh);
}
}
else
{
return(false);
}
}
private async Task ValidateRetryOnDatabaseAccountNotFoundAsync(bool enableMultipleWriteLocations, bool isReadRequest)
{
this.Initialize(
useMultipleWriteLocations: enableMultipleWriteLocations,
enableEndpointDiscovery: true,
isPreferredLocationsListEmpty: false);
await this.endpointManager.RefreshLocationAsync(this.databaseAccount);
ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, true, new RetryOptions());
int expectedRetryCount = isReadRequest || enableMultipleWriteLocations ? 2 : 1;
using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false))
{
int retryCount = 0;
try
{
await BackoffRetryUtility <bool> .ExecuteAsync(
() =>
{
retryCount++;
retryPolicy.OnBeforeSendRequest(request);
if (retryCount == 1)
{
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
StringKeyValueCollection headers = new StringKeyValueCollection();
headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.DatabaseAccountNotFound).ToString();
DocumentClientException forbiddenException = new ForbiddenException(RMResources.NotFound, headers);
throw forbiddenException;
}
else if (retryCount == 2)
{
// Next request must go to next preferred endpoint
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
return(Task.FromResult(true));
}
else
{
Assert.Fail();
}
return(Task.FromResult(true));
},
retryPolicy);
}
catch (ForbiddenException)
{
if (expectedRetryCount == 1)
{
DefaultTrace.TraceInformation("Received expected ForbiddenException");
}
else
{
Assert.Fail();
}
}
Assert.AreEqual(expectedRetryCount, retryCount);
}
}
public void GlobalStrongConsistencyMockTest()
{
// create a real document service request (with auth token level = god)
DocumentServiceRequest entity = DocumentServiceRequest.Create(OperationType.Read, ResourceType.Document, AuthorizationTokenType.SystemAll);
// set request charge tracker - this is referenced in store reader (ReadMultipleReplicaAsync)
var requestContext = new DocumentServiceRequestContext();
requestContext.RequestChargeTracker = new RequestChargeTracker();
entity.RequestContext = requestContext;
// set a dummy resource id on the request.
entity.ResourceId = "1-MxAPlgMgA=";
// set consistency level on the request to Bounded Staleness
entity.Headers[HttpConstants.HttpHeaders.ConsistencyLevel] = ConsistencyLevel.BoundedStaleness.ToString();
// also setup timeout helper, used in store reader
entity.RequestContext.TimeoutHelper = new TimeoutHelper(new TimeSpan(2, 2, 2));
// when the store reader throws Invalid Partition exception, the higher layer should
// clear this target identity.
entity.RequestContext.TargetIdentity = new ServiceIdentity("dummyTargetIdentity1", new Uri("http://dummyTargetIdentity1"), false);
entity.RequestContext.ResolvedPartitionKeyRange = new PartitionKeyRange();
AddressInformation[] addressInformation = GetMockAddressInformationDuringUpgrade();
var mockAddressCache = GetMockAddressCache(addressInformation);
// validate that the mock works
PartitionAddressInformation partitionAddressInformation = mockAddressCache.Object.ResolveAsync(entity, false, new CancellationToken()).Result;
var addressInfo = partitionAddressInformation.AllAddresses;
Assert.IsTrue(addressInfo[0] == addressInformation[0]);
AddressSelector addressSelector = new AddressSelector(mockAddressCache.Object, Protocol.Tcp);
var primaryAddress = addressSelector.ResolvePrimaryUriAsync(entity, false /*forceAddressRefresh*/).Result;
// check if the address return from Address Selector matches the original address info
Assert.IsTrue(primaryAddress.Equals(addressInformation[0].PhysicalUri));
// Quorum Met scenario
{
// get mock transport client that returns a sequence of responses to simulate upgrade
var mockTransportClient = GetMockTransportClientForGlobalStrongReads(addressInformation, ReadQuorumResultKind.QuorumMet);
// create a real session container - we don't need session for this test anyway
ISessionContainer sessionContainer = new SessionContainer(string.Empty);
// create store reader with mock transport client, real address selector (that has mock address cache), and real session container
StoreReader storeReader =
new StoreReader(mockTransportClient,
addressSelector,
sessionContainer);
Mock <IAuthorizationTokenProvider> mockAuthorizationTokenProvider = new Mock <IAuthorizationTokenProvider>();
// setup max replica set size on the config reader
ReplicationPolicy replicationPolicy = new ReplicationPolicy();
replicationPolicy.MaxReplicaSetSize = 4;
Mock <IServiceConfigurationReader> mockServiceConfigReader = new Mock <IServiceConfigurationReader>();
mockServiceConfigReader.SetupGet(x => x.UserReplicationPolicy).Returns(replicationPolicy);
QuorumReader reader =
new QuorumReader(mockTransportClient, addressSelector, storeReader, mockServiceConfigReader.Object, mockAuthorizationTokenProvider.Object);
entity.RequestContext.OriginalRequestConsistencyLevel = Documents.ConsistencyLevel.Strong;
entity.RequestContext.ClientRequestStatistics = new ClientSideRequestStatistics();
StoreResponse result = reader.ReadStrongAsync(entity, 2, ReadMode.Strong).Result;
Assert.IsTrue(result.LSN == 100);
string globalCommitedLSN;
result.TryGetHeaderValue(WFConstants.BackendHeaders.GlobalCommittedLSN, out globalCommitedLSN);
long nGlobalCommitedLSN = long.Parse(globalCommitedLSN, CultureInfo.InvariantCulture);
Assert.IsTrue(nGlobalCommitedLSN == 90);
}
// Quorum Selected scenario
{
// get mock transport client that returns a sequence of responses to simulate upgrade
var mockTransportClient = GetMockTransportClientForGlobalStrongReads(addressInformation, ReadQuorumResultKind.QuorumSelected);
// create a real session container - we don't need session for this test anyway
ISessionContainer sessionContainer = new SessionContainer(string.Empty);
// create store reader with mock transport client, real address selector (that has mock address cache), and real session container
StoreReader storeReader =
new StoreReader(mockTransportClient,
addressSelector,
sessionContainer);
Mock <IAuthorizationTokenProvider> mockAuthorizationTokenProvider = new Mock <IAuthorizationTokenProvider>();
// setup max replica set size on the config reader
ReplicationPolicy replicationPolicy = new ReplicationPolicy();
replicationPolicy.MaxReplicaSetSize = 4;
Mock <IServiceConfigurationReader> mockServiceConfigReader = new Mock <IServiceConfigurationReader>();
mockServiceConfigReader.SetupGet(x => x.UserReplicationPolicy).Returns(replicationPolicy);
QuorumReader reader =
new QuorumReader(mockTransportClient, addressSelector, storeReader, mockServiceConfigReader.Object, mockAuthorizationTokenProvider.Object);
entity.RequestContext.OriginalRequestConsistencyLevel = Documents.ConsistencyLevel.Strong;
entity.RequestContext.ClientRequestStatistics = new ClientSideRequestStatistics();
entity.RequestContext.QuorumSelectedLSN = -1;
entity.RequestContext.GlobalCommittedSelectedLSN = -1;
try
{
StoreResponse result = reader.ReadStrongAsync(entity, 2, ReadMode.Strong).Result;
Assert.IsTrue(false);
}
catch (AggregateException ex)
{
if (ex.InnerException is GoneException)
{
DefaultTrace.TraceInformation("Gone exception expected!");
}
}
Assert.IsTrue(entity.RequestContext.QuorumSelectedLSN == 100);
Assert.IsTrue(entity.RequestContext.GlobalCommittedSelectedLSN == 100);
}
// Quorum not met scenario
{
// get mock transport client that returns a sequence of responses to simulate upgrade
var mockTransportClient = GetMockTransportClientForGlobalStrongReads(addressInformation, ReadQuorumResultKind.QuorumNotSelected);
// create a real session container - we don't need session for this test anyway
ISessionContainer sessionContainer = new SessionContainer(string.Empty);
// create store reader with mock transport client, real address selector (that has mock address cache), and real session container
StoreReader storeReader =
new StoreReader(mockTransportClient,
addressSelector,
sessionContainer);
Mock <IAuthorizationTokenProvider> mockAuthorizationTokenProvider = new Mock <IAuthorizationTokenProvider>();
// setup max replica set size on the config reader
ReplicationPolicy replicationPolicy = new ReplicationPolicy();
replicationPolicy.MaxReplicaSetSize = 4;
Mock <IServiceConfigurationReader> mockServiceConfigReader = new Mock <IServiceConfigurationReader>();
mockServiceConfigReader.SetupGet(x => x.UserReplicationPolicy).Returns(replicationPolicy);
QuorumReader reader =
new QuorumReader(mockTransportClient, addressSelector, storeReader, mockServiceConfigReader.Object, mockAuthorizationTokenProvider.Object);
entity.RequestContext.PerformLocalRefreshOnGoneException = true;
entity.RequestContext.OriginalRequestConsistencyLevel = Documents.ConsistencyLevel.Strong;
entity.RequestContext.ClientRequestStatistics = new ClientSideRequestStatistics();
StoreResponse result = reader.ReadStrongAsync(entity, 2, ReadMode.Strong).Result;
Assert.IsTrue(result.LSN == 100);
string globalCommitedLSN;
result.TryGetHeaderValue(WFConstants.BackendHeaders.GlobalCommittedLSN, out globalCommitedLSN);
long nGlobalCommitedLSN = long.Parse(globalCommitedLSN, CultureInfo.InvariantCulture);
Assert.IsTrue(nGlobalCommitedLSN == 90);
}
}
internal static CosmosClientContext Create(
CosmosClient cosmosClient,
DocumentClient documentClient,
CosmosClientOptions clientOptions,
RequestInvokerHandler requestInvokerHandler = null)
{
if (cosmosClient == null)
{
throw new ArgumentNullException(nameof(cosmosClient));
}
if (documentClient == null)
{
throw new ArgumentNullException(nameof(documentClient));
}
clientOptions = ClientContextCore.CreateOrCloneClientOptions(clientOptions);
ConnectionPolicy connectionPolicy = clientOptions.GetConnectionPolicy(cosmosClient.ClientId);
ClientTelemetry telemetry = null;
if (connectionPolicy.EnableClientTelemetry)
{
telemetry = ClientTelemetry.CreateAndStartBackgroundTelemetry(
documentClient: documentClient,
userAgent: connectionPolicy.UserAgentContainer.UserAgent,
connectionMode: connectionPolicy.ConnectionMode,
authorizationTokenProvider: cosmosClient.AuthorizationTokenProvider,
diagnosticsHelper: DiagnosticsHandlerHelper.Instance,
preferredRegions: clientOptions.ApplicationPreferredRegions);
}
else
{
DefaultTrace.TraceInformation("Telemetry Disabled.");
}
if (requestInvokerHandler == null)
{
//Request pipeline
ClientPipelineBuilder clientPipelineBuilder = new ClientPipelineBuilder(
cosmosClient,
clientOptions.ConsistencyLevel,
clientOptions.CustomHandlers,
telemetry: telemetry);
requestInvokerHandler = clientPipelineBuilder.Build();
}
CosmosSerializerCore serializerCore = CosmosSerializerCore.Create(
clientOptions.Serializer,
clientOptions.SerializerOptions);
// This sets the serializer on client options which gives users access to it if a custom one is not configured.
clientOptions.SetSerializerIfNotConfigured(serializerCore.GetCustomOrDefaultSerializer());
CosmosResponseFactoryInternal responseFactory = new CosmosResponseFactoryCore(serializerCore);
return(new ClientContextCore(
client: cosmosClient,
clientOptions: clientOptions,
serializerCore: serializerCore,
cosmosResponseFactory: responseFactory,
requestHandler: requestInvokerHandler,
documentClient: documentClient,
userAgent: documentClient.ConnectionPolicy.UserAgentContainer.UserAgent,
batchExecutorCache: new BatchAsyncContainerExecutorCache(),
telemetry: telemetry));
}
public virtual Range <string> ExtractPartitionKeyRangeFromContinuationToken(INameValueCollection headers, out List <CompositeContinuationToken> compositeContinuationTokens)
{
if (headers == null)
{
throw new ArgumentNullException("headers");
}
compositeContinuationTokens = null;
Range <string> range = Range <string> .GetEmptyRange(PartitionKeyInternal.MinimumInclusiveEffectivePartitionKey);
if (string.IsNullOrEmpty(headers[HttpConstants.HttpHeaders.Continuation]))
{
return(range);
}
string providedContinuation = headers[HttpConstants.HttpHeaders.Continuation];
CompositeContinuationToken initialContinuationToken = null;
if (!string.IsNullOrEmpty(providedContinuation))
{
try
{
if (providedContinuation.Trim().StartsWith("[", StringComparison.Ordinal))
{
compositeContinuationTokens = JsonConvert.DeserializeObject <List <CompositeContinuationToken> >(providedContinuation);
if (compositeContinuationTokens != null && compositeContinuationTokens.Count > 0)
{
headers[HttpConstants.HttpHeaders.Continuation] = compositeContinuationTokens[0].Token;
initialContinuationToken = compositeContinuationTokens[0];
}
else
{
headers.Remove(HttpConstants.HttpHeaders.Continuation);
}
}
else
{
// TODO: Remove the else logic after the gateway deployment is complete
initialContinuationToken = JsonConvert.DeserializeObject <CompositeContinuationToken>(providedContinuation);
if (initialContinuationToken != null)
{
compositeContinuationTokens = new List <CompositeContinuationToken> {
initialContinuationToken
};
}
else
{
throw new BadRequestException(RMResources.InvalidContinuationToken);
}
}
if (initialContinuationToken?.Range != null)
{
range = initialContinuationToken.Range;
}
if (initialContinuationToken != null && !string.IsNullOrEmpty(initialContinuationToken.Token))
{
headers[HttpConstants.HttpHeaders.Continuation] = initialContinuationToken.Token;
}
else
{
headers.Remove(HttpConstants.HttpHeaders.Continuation);
}
}
catch (JsonException ex)
{
DefaultTrace.TraceWarning(
string.Format(
CultureInfo.InvariantCulture,
"{0} Invalid JSON in the continuation token {1}",
DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture),
providedContinuation));
throw new BadRequestException(RMResources.InvalidContinuationToken, ex);
}
}
else
{
headers.Remove(HttpConstants.HttpHeaders.Continuation);
}
return(range);
}
private async Task ValidateRetryOnWriteSessionNotAvailabeWithEnableMultipleWriteLocationsAsync()
{
const bool useMultipleWriteLocations = true;
bool enableEndpointDiscovery = true;
this.Initialize(
useMultipleWriteLocations: useMultipleWriteLocations,
enableEndpointDiscovery: enableEndpointDiscovery,
isPreferredLocationsListEmpty: false);
await this.endpointManager.RefreshLocationAsync(this.databaseAccount);
ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, enableEndpointDiscovery, new RetryOptions());
using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: false, isMasterResourceType: false))
{
int retryCount = 0;
try
{
await BackoffRetryUtility <bool> .ExecuteAsync(
() =>
{
retryPolicy.OnBeforeSendRequest(request);
if (retryCount == 0)
{
Assert.IsFalse(request.ClearSessionTokenOnSessionReadFailure);
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
}
else if (retryCount == 1)
{
Assert.IsFalse(request.ClearSessionTokenOnSessionReadFailure);
// Second request must go to first write endpoint
Uri expectedEndpoint = new Uri(this.databaseAccount.WriteLocationsInternal[0].DatabaseAccountEndpoint);
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
}
else if (retryCount == 2)
{
Assert.IsFalse(request.ClearSessionTokenOnSessionReadFailure);
// Second request must go to first write endpoint
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
}
else if (retryCount == 3)
{
Assert.IsTrue(request.ClearSessionTokenOnSessionReadFailure);
// Second request must go to first write endpoint
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[2]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
}
else
{
Assert.Fail();
}
retryCount++;
StringKeyValueCollection headers = new StringKeyValueCollection();
headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.ReadSessionNotAvailable).ToString();
DocumentClientException notFoundException = new NotFoundException(RMResources.NotFound, headers);
throw notFoundException;
},
retryPolicy);
Assert.Fail();
}
catch (NotFoundException)
{
DefaultTrace.TraceInformation("Received expected notFoundException");
Assert.AreEqual(4, retryCount);
}
}
}
/// <summary>
/// Resolves the endpoint of the partition for the given request
/// </summary>
/// <param name="request">Request for which the partition endpoint resolution is to be performed</param>
/// <param name="forceRefreshPartitionAddresses">Force refresh the partition's endpoint</param>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns></returns>
private async Task <ResolutionResult> ResolveAddressesAndIdentityAsync(
DocumentServiceRequest request,
bool forceRefreshPartitionAddresses,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
if (request.ServiceIdentity != null)
{
// In this case we don't populate request.RequestContext.ResolvedPartitionKeyRangeId,
// which is needed for session token.
// The assumption is that:
// 1. Master requests never use session consistency.
// 2. Service requests (like collection create etc.) don't use session consistency.
// 3. Requests which target specific partition of an existing collection will use x-ms-documentdb-partitionkeyrangeid header
// to send request to specific partition and will not set request.ServiceIdentity
ServiceIdentity identity = request.ServiceIdentity;
PartitionAddressInformation addresses = await this.addressCache.TryGetAddresses(request, null, identity, forceRefreshPartitionAddresses, cancellationToken);
if (addresses == null)
{
DefaultTrace.TraceInformation("Could not get addresses for explicitly specified ServiceIdentity {0}", identity);
throw new NotFoundException()
{
ResourceAddress = request.ResourceAddress
};
}
return(new ResolutionResult(addresses, identity));
}
if (ReplicatedResourceClient.IsReadingFromMaster(request.ResourceType, request.OperationType) && request.PartitionKeyRangeIdentity == null)
{
// Client implementation, GlobalAddressResolver passes in a null IMasterServiceIdentityProvider, because it does't actually use the serviceIdentity
// in the addressCache.TryGetAddresses method. In GatewayAddressCache.cs, the master address is resolved by making a call to Gateway AddressFeed,
// not using the serviceIdentity that is passed in
ServiceIdentity serviceIdentity = this.masterServiceIdentityProvider?.MasterServiceIdentity;
PartitionKeyRangeIdentity partitionKeyRangeIdentity = this.masterPartitionKeyRangeIdentity;
PartitionAddressInformation addresses = await this.addressCache.TryGetAddresses(
request,
partitionKeyRangeIdentity,
serviceIdentity,
forceRefreshPartitionAddresses,
cancellationToken);
if (addresses == null)
{
// This shouldn't really happen.
DefaultTrace.TraceCritical("Could not get addresses for master partition {0}", serviceIdentity);
throw new NotFoundException()
{
ResourceAddress = request.ResourceAddress
};
}
PartitionKeyRange partitionKeyRange = new PartitionKeyRange {
Id = PartitionKeyRange.MasterPartitionKeyRangeId
};
return(new ResolutionResult(partitionKeyRange, addresses, serviceIdentity));
}
bool collectionCacheIsUptoDate = !request.IsNameBased ||
(request.PartitionKeyRangeIdentity != null && request.PartitionKeyRangeIdentity.CollectionRid != null);
bool collectionRoutingMapCacheIsUptoDate = false;
var collection = await this.collectionCache.ResolveCollectionAsync(request, cancellationToken);
CollectionRoutingMap routingMap = await this.collectionRoutingMapCache.TryLookupAsync(
collection.ResourceId, null, request, request.ForceCollectionRoutingMapRefresh, cancellationToken);
if (request.ForcePartitionKeyRangeRefresh)
{
collectionRoutingMapCacheIsUptoDate = true;
request.ForcePartitionKeyRangeRefresh = false;
if (routingMap != null)
{
routingMap = await this.collectionRoutingMapCache.TryLookupAsync(collection.ResourceId, routingMap, request, false, cancellationToken);
}
}
if (routingMap == null && !collectionCacheIsUptoDate)
{
// Routing map was not found by resolved collection rid. Maybe collection rid is outdated.
// Refresh collection cache and reresolve routing map.
request.ForceNameCacheRefresh = true;
collectionCacheIsUptoDate = true;
collectionRoutingMapCacheIsUptoDate = false;
collection = await this.collectionCache.ResolveCollectionAsync(request, cancellationToken);
routingMap = await this.collectionRoutingMapCache.TryLookupAsync(
collection.ResourceId,
previousValue : null,
request : request,
forceRefreshCollectionRoutingMap : false,
cancellationToken : cancellationToken);
}
AddressResolver.EnsureRoutingMapPresent(request, routingMap, collection);
// At this point we have both collection and routingMap.
ResolutionResult result = await this.TryResolveServerPartitionAsync(
request,
collection,
routingMap,
collectionCacheIsUptoDate,
collectionRoutingMapCacheIsUptodate : collectionRoutingMapCacheIsUptoDate,
forceRefreshPartitionAddresses : forceRefreshPartitionAddresses,
cancellationToken : cancellationToken);
if (result == null)
{
// Couldn't resolve server partition or its addresses.
// Either collection cache is outdated or routing map cache is outdated.
if (!collectionCacheIsUptoDate)
{
request.ForceNameCacheRefresh = true;
collectionCacheIsUptoDate = true;
collection = await this.collectionCache.ResolveCollectionAsync(request, cancellationToken);
if (collection.ResourceId != routingMap.CollectionUniqueId)
{
// Collection cache was stale. We resolved to new Rid. routing map cache is potentially stale
// for this new collection rid. Mark it as such.
collectionRoutingMapCacheIsUptoDate = false;
routingMap = await this.collectionRoutingMapCache.TryLookupAsync(
collection.ResourceId,
previousValue : null,
request : request,
forceRefreshCollectionRoutingMap : false,
cancellationToken : cancellationToken);
}
}
if (!collectionRoutingMapCacheIsUptoDate)
{
collectionRoutingMapCacheIsUptoDate = true;
routingMap = await this.collectionRoutingMapCache.TryLookupAsync(
collection.ResourceId,
previousValue : routingMap,
request : request,
forceRefreshCollectionRoutingMap : false,
cancellationToken : cancellationToken);
}
AddressResolver.EnsureRoutingMapPresent(request, routingMap, collection);
result = await this.TryResolveServerPartitionAsync(
request,
collection,
routingMap,
collectionCacheIsUptodate : true,
collectionRoutingMapCacheIsUptodate : true,
forceRefreshPartitionAddresses : forceRefreshPartitionAddresses,
cancellationToken : cancellationToken);
}
if (result == null)
{
DefaultTrace.TraceInformation("Couldn't route partitionkeyrange-oblivious request after retry/cache refresh. Collection doesn't exist.");
// At this point collection cache and routing map caches are refreshed.
// The only reason we will get here is if collection doesn't exist.
// Case when partitionkeyrange doesn't exist is handled in the corresponding method.
throw new NotFoundException()
{
ResourceAddress = request.ResourceAddress
};
}
if (request.IsNameBased)
{
// Append collection rid.
// If we resolved collection rid incorrectly because of outdated cache, this can lead
// to incorrect routing decisions. But backend will validate collection rid and throw
// InvalidPartitionException if we reach wrong collection.
// Also this header will be used by backend to inject collection rid into metrics for
// throttled requests.
request.Headers[WFConstants.BackendHeaders.CollectionRid] = collection.ResourceId;
}
return(result);
}
internal TryCatch <PartitionedQueryExecutionInfoInternal> TryGetPartitionedQueryExecutionInfoInternal(
SqlQuerySpec querySpec,
PartitionKeyDefinition partitionKeyDefinition,
bool requireFormattableOrderByQuery,
bool isContinuationExpected,
bool allowNonValueAggregateQuery,
bool hasLogicalPartitionKey)
{
if (querySpec == null || partitionKeyDefinition == null)
{
return(TryCatch <PartitionedQueryExecutionInfoInternal> .FromResult(DefaultInfoInternal));
}
string queryText = JsonConvert.SerializeObject(querySpec);
List <string> paths = new List <string>(partitionKeyDefinition.Paths);
List <string[]> pathParts = new List <string[]>();
paths.ForEach(path =>
{
pathParts.Add(PathParser.GetPathParts(path).ToArray());
});
string[] allParts = pathParts.SelectMany(parts => parts).ToArray();
uint[] partsLengths = pathParts.Select(parts => (uint)parts.Length).ToArray();
PartitionKind partitionKind = partitionKeyDefinition.Kind;
this.Initialize();
byte[] buffer = new byte[InitialBufferSize];
uint errorCode;
uint serializedQueryExecutionInfoResultLength;
unsafe
{
fixed(byte *bytePtr = buffer)
{
errorCode = ServiceInteropWrapper.GetPartitionKeyRangesFromQuery(
this.serviceProvider,
queryText,
requireFormattableOrderByQuery,
isContinuationExpected,
allowNonValueAggregateQuery,
hasLogicalPartitionKey,
allParts,
partsLengths,
(uint)partitionKeyDefinition.Paths.Count,
partitionKind,
new IntPtr(bytePtr),
(uint)buffer.Length,
out serializedQueryExecutionInfoResultLength);
if (errorCode == DISP_E_BUFFERTOOSMALL)
{
buffer = new byte[serializedQueryExecutionInfoResultLength];
fixed(byte *bytePtr2 = buffer)
{
errorCode = ServiceInteropWrapper.GetPartitionKeyRangesFromQuery(
this.serviceProvider,
queryText,
requireFormattableOrderByQuery,
isContinuationExpected,
allowNonValueAggregateQuery,
hasLogicalPartitionKey, // has logical partition key
allParts,
partsLengths,
(uint)partitionKeyDefinition.Paths.Count,
partitionKind,
new IntPtr(bytePtr2),
(uint)buffer.Length,
out serializedQueryExecutionInfoResultLength);
}
}
}
}
string serializedQueryExecutionInfo = Encoding.UTF8.GetString(buffer, 0, (int)serializedQueryExecutionInfoResultLength);
Exception exception = Marshal.GetExceptionForHR((int)errorCode);
if (exception != null)
{
DefaultTrace.TraceInformation("QueryEngineConfiguration: " + this.queryengineConfiguration);
string errorMessage;
if (string.IsNullOrEmpty(serializedQueryExecutionInfo))
{
errorMessage = $"Message: Query service interop parsing hit an unexpected exception: {exception.ToString()}";
}
else
{
errorMessage = "Message: " + serializedQueryExecutionInfo;
}
return(TryCatch <PartitionedQueryExecutionInfoInternal> .FromException(new Exception(errorMessage)));
}
PartitionedQueryExecutionInfoInternal queryInfoInternal =
JsonConvert.DeserializeObject <PartitionedQueryExecutionInfoInternal>(
serializedQueryExecutionInfo,
new JsonSerializerSettings
{
DateParseHandling = DateParseHandling.None
});
return(TryCatch <PartitionedQueryExecutionInfoInternal> .FromResult(queryInfoInternal));
}
private PartitionKeyRange TryResolveServerPartitionByPartitionKey(
DocumentServiceRequest request,
string partitionKeyString,
bool collectionCacheUptoDate,
CosmosContainerSettings collection,
CollectionRoutingMap routingMap)
{
if (request == null)
{
throw new ArgumentNullException("request");
}
if (partitionKeyString == null)
{
throw new ArgumentNullException("partitionKeyString");
}
if (collection == null)
{
throw new ArgumentNullException("collection");
}
if (routingMap == null)
{
throw new ArgumentNullException("routingMap");
}
PartitionKeyInternal partitionKey;
try
{
partitionKey = PartitionKeyInternal.FromJsonString(partitionKeyString);
}
catch (JsonException ex)
{
throw new BadRequestException(
string.Format(CultureInfo.InvariantCulture, RMResources.InvalidPartitionKey, partitionKeyString),
ex)
{
ResourceAddress = request.ResourceAddress
};
}
if (partitionKey == null)
{
throw new InternalServerErrorException(string.Format(CultureInfo.InvariantCulture, "partition key is null '{0}'", partitionKeyString));
}
if (partitionKey.Components.Count == collection.PartitionKey.Paths.Count)
{
// Although we can compute effective partition key here, in general case this Gateway can have outdated
// partition key definition cached - like if collection with same name but with Range partitioning is created.
// In this case server will not pass x-ms-documentdb-collection-rid check and will return back InvalidPartitionException.
// Gateway will refresh its cache and retry.
string effectivePartitionKey = partitionKey.GetEffectivePartitionKeyString(collection.PartitionKey);
// There should be exactly one range which contains a partition key. Always.
return(routingMap.GetRangeByEffectivePartitionKey(effectivePartitionKey));
}
if (collectionCacheUptoDate)
{
BadRequestException badRequestException = new BadRequestException(RMResources.PartitionKeyMismatch)
{
ResourceAddress = request.ResourceAddress
};
badRequestException.Headers[WFConstants.BackendHeaders.SubStatus] =
((uint)SubStatusCodes.PartitionKeyMismatch).ToString(CultureInfo.InvariantCulture);
throw badRequestException;
}
// Partition key supplied has different number paths than locally cached partition key definition.
// Three things can happen:
// 1. User supplied wrong partition key.
// 2. Client SDK has outdated partition key definition cache and extracted wrong value from the document.
// 3. Gateway's cache is outdated.
//
// What we will do is append x-ms-documentdb-collection-rid header and forward it to random collection partition.
// * If collection rid matches, server will send back 400.1001, because it also will not be able to compute
// effective partition key. Gateway will forward this status code to client - client will handle it.
// * If collection rid doesn't match, server will send back InvalidPartiitonException and Gateway will
// refresh name routing cache - this will refresh partition key definition as well, and retry.
DefaultTrace.TraceInformation(
"Cannot compute effective partition key. Definition has '{0}' paths, values supplied has '{1}' paths. Will refresh cache and retry.",
collection.PartitionKey.Paths.Count,
partitionKey.Components.Count);
return(null);
}
protected async Task RepairContextAsync(
string collectionRid,
int currentDocumentProducerIndex,
Func <DocumentProducer <T>, int> taskPriorityFunc,
IReadOnlyList <PartitionKeyRange> replacementRanges,
SqlQuerySpec querySpecForRepair,
Action callback = null)
{
CollectionCache collectionCache = await this.Client.GetCollectionCacheAsync();
INameValueCollection requestHeaders = await this.CreateCommonHeadersAsync(this.GetFeedOptions(null));
this.DocumentProducers.Capacity = this.DocumentProducers.Count + replacementRanges.Count - 1;
DocumentProducer <T> replacedDocumentProducer = this.DocumentProducers[currentDocumentProducerIndex];
DefaultTrace.TraceInformation(string.Format(
CultureInfo.InvariantCulture,
"{0}, CorrelatedActivityId: {5} | Parallel~ContextBase.RepairContextAsync, MaxBufferedItemCount: {1}, Replacement PartitionKeyRange Count: {2}, MaximumConcurrencyLevel: {3}, DocumentProducer Initial Page Size {4}",
DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture),
this.actualMaxBufferedItemCount,
replacementRanges.Count,
this.TaskScheduler.MaximumConcurrencyLevel,
replacedDocumentProducer.PageSize,
this.CorrelatedActivityId));
int index = currentDocumentProducerIndex + 1;
foreach (PartitionKeyRange range in replacementRanges)
{
this.DocumentProducers.Insert(
index++,
new DocumentProducer <T>(
this.TaskScheduler,
(continuationToken, pageSize) =>
{
INameValueCollection headers = requestHeaders.Clone();
headers[HttpConstants.HttpHeaders.Continuation] = continuationToken;
headers[HttpConstants.HttpHeaders.PageSize] = pageSize.ToString(CultureInfo.InvariantCulture);
return(this.CreateDocumentServiceRequest(
headers,
querySpecForRepair,
range,
collectionRid));
},
range,
taskPriorityFunc,
this.ExecuteRequestAsync <T>,
() => new NonRetriableInvalidPartitionExceptionRetryPolicy(collectionCache, this.Client.RetryPolicy.GetRequestPolicy()),
this.OnDocumentProducerCompleteFetching,
this.CorrelatedActivityId,
replacedDocumentProducer.PageSize,
replacedDocumentProducer.CurrentBackendContinuationToken));
}
this.DocumentProducers.RemoveAt(currentDocumentProducerIndex);
if (callback != null)
{
callback();
}
if (this.ShouldPrefetch)
{
for (int i = 0; i < replacementRanges.Count; i++)
{
this.DocumentProducers[i + currentDocumentProducerIndex].TryScheduleFetch();
}
}
if (this.CurrentContinuationTokens.Remove(replacedDocumentProducer))
{
for (int i = 0; i < replacementRanges.Count; ++i)
{
this.CurrentContinuationTokens[this.DocumentProducers[currentDocumentProducerIndex + i]] = replacedDocumentProducer.CurrentBackendContinuationToken;
}
}
}
/// <summary>
/// A method to create the cosmos client
/// </summary>
/// <remarks>
/// Setting this property after sending any request won't have any effect.
/// </remarks>
internal virtual CosmosClient Build(DocumentClient documentClient)
{
DefaultTrace.TraceInformation($"CosmosClientBuilder.Build(DocumentClient) with configuration: {this.clientOptions.GetSerializedConfiguration()}");
return(new CosmosClient(this.clientOptions, documentClient));
}
/// <summary>
/// Resolves a request to a collection in a sticky manner.
/// Unless request.ForceNameCacheRefresh is equal to true, it will return the same collection.
/// </summary>
/// <param name="request">Request to resolve.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <param name="trace">The trace.</param>
/// <returns>Instance of <see cref="ContainerProperties"/>.</returns>
public virtual async Task <ContainerProperties> ResolveCollectionAsync(
DocumentServiceRequest request,
CancellationToken cancellationToken,
ITrace trace)
{
IClientSideRequestStatistics clientSideRequestStatistics = request.RequestContext?.ClientRequestStatistics;
if (request.IsNameBased)
{
if (request.ForceNameCacheRefresh)
{
await this.RefreshAsync(request, trace, clientSideRequestStatistics, cancellationToken);
request.ForceNameCacheRefresh = false;
}
ContainerProperties collectionInfo = await this.ResolveByPartitionKeyRangeIdentityAsync(
request.Headers[HttpConstants.HttpHeaders.Version],
request.PartitionKeyRangeIdentity,
trace,
clientSideRequestStatistics,
cancellationToken);
if (collectionInfo != null)
{
return(collectionInfo);
}
if (request.RequestContext.ResolvedCollectionRid == null)
{
collectionInfo =
await this.ResolveByNameAsync(
apiVersion : request.Headers[HttpConstants.HttpHeaders.Version],
resourceAddress : request.ResourceAddress,
forceRefesh : false,
trace : trace,
clientSideRequestStatistics : clientSideRequestStatistics,
cancellationToken : cancellationToken);
if (collectionInfo != null)
{
DefaultTrace.TraceVerbose(
"Mapped resourceName {0} to resourceId {1}. '{2}'",
request.ResourceAddress,
collectionInfo.ResourceId,
System.Diagnostics.Trace.CorrelationManager.ActivityId);
request.ResourceId = collectionInfo.ResourceId;
request.RequestContext.ResolvedCollectionRid = collectionInfo.ResourceId;
}
else
{
DefaultTrace.TraceVerbose(
"Collection with resourceName {0} not found. '{1}'",
request.ResourceAddress,
System.Diagnostics.Trace.CorrelationManager.ActivityId);
}
return(collectionInfo);
}
else
{
return(await this.ResolveByRidAsync(request.Headers[HttpConstants.HttpHeaders.Version], request.RequestContext.ResolvedCollectionRid, trace, clientSideRequestStatistics, cancellationToken));
}
}
else
{
return(await this.ResolveByPartitionKeyRangeIdentityAsync(request.Headers[HttpConstants.HttpHeaders.Version], request.PartitionKeyRangeIdentity, trace, clientSideRequestStatistics, cancellationToken) ??
await this.ResolveByRidAsync(request.Headers[HttpConstants.HttpHeaders.Version], request.ResourceAddress, trace, clientSideRequestStatistics, cancellationToken));
}
}
