private async Task ValidateRetryOnDatabaseAccountNotFoundAsync(bool enableMultipleWriteLocations, bool isReadRequest)
{
this.Initialize(
useMultipleWriteLocations: enableMultipleWriteLocations,
enableEndpointDiscovery: true,
isPreferredLocationsListEmpty: false);
await this.endpointManager.RefreshLocationAsync(this.databaseAccount);
ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, true, new RetryOptions());
int expectedRetryCount = isReadRequest || enableMultipleWriteLocations ? 2 : 1;
using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false))
{
int retryCount = 0;
try
{
await BackoffRetryUtility <bool> .ExecuteAsync(
() =>
{
retryCount++;
retryPolicy.OnBeforeSendRequest(request);
if (retryCount == 1)
{
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
StringKeyValueCollection headers = new StringKeyValueCollection();
headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.DatabaseAccountNotFound).ToString();
DocumentClientException forbiddenException = new ForbiddenException(RMResources.NotFound, headers);
throw forbiddenException;
}
else if (retryCount == 2)
{
// Next request must go to next preferred endpoint
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
return(Task.FromResult(true));
}
else
{
Assert.Fail();
}
return(Task.FromResult(true));
},
retryPolicy);
}
catch (ForbiddenException)
{
if (expectedRetryCount == 1)
{
DefaultTrace.TraceInformation("Received expected ForbiddenException");
}
else
{
Assert.Fail();
}
}
Assert.AreEqual(expectedRetryCount, retryCount);
}
}
private async Task <Tuple <DocumentFeedResponse <CosmosElement>, string> > ExecuteOnceAsync(
IDocumentClientRetryPolicy retryPolicyInstance,
CancellationToken cancellationToken)
{
// Don't reuse request, as the rest of client SDK doesn't reuse requests between retries.
// The code leaves some temporary garbage in request (in RequestContext etc.),
// which shold be erased during retries.
using (DocumentServiceRequest request = await this.CreateRequestAsync())
{
DocumentFeedResponse <CosmosElement> feedRespose;
string partitionIdentifier;
// We need to determine how to execute the request:
if (LogicalPartitionKeyProvided(request))
{
feedRespose = await this.ExecuteRequestAsync(request, retryPolicyInstance, cancellationToken);
partitionIdentifier = $"PKId({request.Headers[HttpConstants.HttpHeaders.PartitionKey]})";
}
else if (PhysicalPartitionKeyRangeIdProvided(this))
{
CollectionCache collectionCache = await this.Client.GetCollectionCacheAsync();
ContainerProperties collection = await collectionCache.ResolveCollectionAsync(request, CancellationToken.None, NoOpTrace.Singleton);
request.RouteTo(new PartitionKeyRangeIdentity(collection.ResourceId, base.PartitionKeyRangeId));
feedRespose = await this.ExecuteRequestAsync(request, retryPolicyInstance, cancellationToken);
partitionIdentifier = base.PartitionKeyRangeId;
}
else
{
// The query is going to become a full fan out, but we go one partition at a time.
if (ServiceInteropAvailable())
{
// Get the routing map provider
CollectionCache collectionCache = await this.Client.GetCollectionCacheAsync();
ContainerProperties collection = await collectionCache.ResolveCollectionAsync(request, CancellationToken.None, NoOpTrace.Singleton);
QueryPartitionProvider queryPartitionProvider = await this.Client.GetQueryPartitionProviderAsync();
IRoutingMapProvider routingMapProvider = await this.Client.GetRoutingMapProviderAsync();
// Figure out what partition you are going to based on the range from the continuation token
// If token is null then just start at partitionKeyRangeId "0"
List <CompositeContinuationToken> suppliedTokens;
Range <string> rangeFromContinuationToken =
this.partitionRoutingHelper.ExtractPartitionKeyRangeFromContinuationToken(
request.Headers,
out suppliedTokens);
Tuple <PartitionRoutingHelper.ResolvedRangeInfo, IReadOnlyList <Range <string> > > queryRoutingInfo =
await this.TryGetTargetPartitionKeyRangeAsync(
request,
collection,
queryPartitionProvider,
routingMapProvider,
rangeFromContinuationToken,
suppliedTokens);
if (request.IsNameBased && queryRoutingInfo == null)
{
request.ForceNameCacheRefresh = true;
collection = await collectionCache.ResolveCollectionAsync(request, CancellationToken.None, NoOpTrace.Singleton);
queryRoutingInfo = await this.TryGetTargetPartitionKeyRangeAsync(
request,
collection,
queryPartitionProvider,
routingMapProvider,
rangeFromContinuationToken,
suppliedTokens);
}
if (queryRoutingInfo == null)
{
throw new NotFoundException($"{DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture)}: Was not able to get queryRoutingInfo even after resolve collection async with force name cache refresh to the following collectionRid: {collection.ResourceId} with the supplied tokens: {JsonConvert.SerializeObject(suppliedTokens)}");
}
request.RouteTo(new PartitionKeyRangeIdentity(collection.ResourceId, queryRoutingInfo.Item1.ResolvedRange.Id));
DocumentFeedResponse <CosmosElement> response = await this.ExecuteRequestAsync(request, retryPolicyInstance, cancellationToken);
// Form a composite continuation token (range + backend continuation token).
// If the backend continuation token was null for the range,
// then use the next adjacent range.
// This is how the default execution context serially visits every partition.
if (!await this.partitionRoutingHelper.TryAddPartitionKeyRangeToContinuationTokenAsync(
response.Headers,
providedPartitionKeyRanges: queryRoutingInfo.Item2,
routingMapProvider: routingMapProvider,
collectionRid: collection.ResourceId,
resolvedRangeInfo: queryRoutingInfo.Item1,
trace: NoOpTrace.Singleton))
{
// Collection to which this request was resolved doesn't exist.
// Retry policy will refresh the cache and return NotFound.
throw new NotFoundException($"{DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture)}: Call to TryAddPartitionKeyRangeToContinuationTokenAsync failed to the following collectionRid: {collection.ResourceId} with the supplied tokens: {JsonConvert.SerializeObject(suppliedTokens)}");
}
feedRespose = response;
partitionIdentifier = queryRoutingInfo.Item1.ResolvedRange.Id;
}
else
{
// For non-Windows platforms(like Linux and OSX) in .NET Core SDK, we cannot use ServiceInterop for parsing the query,
// so forcing the request through Gateway. We are also now by-passing this for 32-bit host process in NETFX on Windows
// as the ServiceInterop dll is only available in 64-bit.
request.UseGatewayMode = true;
feedRespose = await this.ExecuteRequestAsync(request, retryPolicyInstance, cancellationToken);
partitionIdentifier = "Gateway";
}
}
return(new Tuple <DocumentFeedResponse <CosmosElement>, string>(feedRespose, partitionIdentifier));
}
}
private async Task <Tuple <PartitionRoutingHelper.ResolvedRangeInfo, IReadOnlyList <Range <string> > > > TryGetTargetPartitionKeyRangeAsync(
DocumentServiceRequest request,
ContainerProperties collection,
QueryPartitionProvider queryPartitionProvider,
IRoutingMapProvider routingMapProvider,
Range <string> rangeFromContinuationToken,
List <CompositeContinuationToken> suppliedTokens)
{
string version = request.Headers[HttpConstants.HttpHeaders.Version];
version = string.IsNullOrEmpty(version) ? HttpConstants.Versions.CurrentVersion : version;
bool enableCrossPartitionQuery = false;
string enableCrossPartitionQueryHeader = request.Headers[HttpConstants.HttpHeaders.EnableCrossPartitionQuery];
if (enableCrossPartitionQueryHeader != null)
{
if (!bool.TryParse(enableCrossPartitionQueryHeader, out enableCrossPartitionQuery))
{
throw new BadRequestException(
string.Format(
CultureInfo.InvariantCulture,
RMResources.InvalidHeaderValue,
enableCrossPartitionQueryHeader,
HttpConstants.HttpHeaders.EnableCrossPartitionQuery));
}
}
IReadOnlyList <Range <string> > providedRanges;
if (!this.providedRangesCache.TryGetValue(collection.ResourceId, out providedRanges))
{
if (this.ShouldExecuteQueryRequest)
{
FeedOptions feedOptions = this.GetFeedOptions(null);
PartitionKeyDefinition partitionKeyDefinition;
if ((feedOptions.Properties != null) && feedOptions.Properties.TryGetValue(
DefaultDocumentQueryExecutionContext.InternalPartitionKeyDefinitionProperty,
out object partitionKeyDefinitionObject))
{
if (partitionKeyDefinitionObject is PartitionKeyDefinition definition)
{
partitionKeyDefinition = definition;
}
else
{
throw new ArgumentException(
"partitionkeydefinition has invalid type",
nameof(partitionKeyDefinitionObject));
}
}
else
{
partitionKeyDefinition = collection.PartitionKey;
}
providedRanges = PartitionRoutingHelper.GetProvidedPartitionKeyRanges(
querySpecJsonString: JsonConvert.SerializeObject(this.QuerySpec),
enableCrossPartitionQuery: enableCrossPartitionQuery,
parallelizeCrossPartitionQuery: false,
isContinuationExpected: this.isContinuationExpected,
hasLogicalPartitionKey: false,
allowDCount: false,
allowNonValueAggregates: false,
useSystemPrefix: false,
partitionKeyDefinition: partitionKeyDefinition,
queryPartitionProvider: queryPartitionProvider,
clientApiVersion: version,
out QueryInfo _);
}
else if (request.Properties != null && request.Properties.TryGetValue(
WFConstants.BackendHeaders.EffectivePartitionKeyString,
out object effectivePartitionKey))
{
if (effectivePartitionKey is string effectivePartitionKeyString)
{
providedRanges = new List <Range <string> >()
{
Range <string> .GetPointRange(effectivePartitionKeyString),
};
}
else
{
throw new ArgumentException(
"EffectivePartitionKey must be a string",
WFConstants.BackendHeaders.EffectivePartitionKeyString);
}
}
else
{
providedRanges = new List <Range <string> >
{
new Range <string>(
PartitionKeyInternal.MinimumInclusiveEffectivePartitionKey,
PartitionKeyInternal.MaximumExclusiveEffectivePartitionKey,
true,
false)
};
}
this.providedRangesCache[collection.ResourceId] = providedRanges;
}
PartitionRoutingHelper.ResolvedRangeInfo resolvedRangeInfo = await this.partitionRoutingHelper.TryGetTargetRangeFromContinuationTokenRangeAsync(
providedRanges,
routingMapProvider,
collection.ResourceId,
rangeFromContinuationToken,
suppliedTokens,
NoOpTrace.Singleton);
if (resolvedRangeInfo.ResolvedRange == null)
{
return(null);
}
else
{
return(Tuple.Create(resolvedRangeInfo, providedRanges));
}
}
private async Task <ResolutionResult> TryResolveServerPartitionAsync(
DocumentServiceRequest request,
ContainerProperties collection,
CollectionRoutingMap routingMap,
bool collectionCacheIsUptodate,
bool collectionRoutingMapCacheIsUptodate,
bool forceRefreshPartitionAddresses,
CancellationToken cancellationToken)
{
// Check if this request partitionkeyrange-aware routing logic. We cannot retry here in this case
// and need to bubble up errors.
if (request.PartitionKeyRangeIdentity != null)
{
return(await this.TryResolveServerPartitionByPartitionKeyRangeIdAsync(
request,
collection,
routingMap,
collectionCacheIsUptodate,
collectionRoutingMapCacheIsUptodate,
forceRefreshPartitionAddresses,
cancellationToken));
}
if (!request.ResourceType.IsPartitioned() &&
!(request.ResourceType == ResourceType.StoredProcedure && request.OperationType == OperationType.ExecuteJavaScript) &&
// Collection head is sent internally for strong consistency given routing hints from original requst, which is for partitioned resource.
!(request.ResourceType == ResourceType.Collection && request.OperationType == OperationType.Head))
{
DefaultTrace.TraceCritical(
"Shouldn't come here for non partitioned resources. resourceType : {0}, operationtype:{1}, resourceaddress:{2}",
request.ResourceType,
request.OperationType,
request.ResourceAddress);
throw new InternalServerErrorException(RMResources.InternalServerError)
{
ResourceAddress = request.ResourceAddress
};
}
PartitionKeyRange range;
string partitionKeyString = request.Headers[HttpConstants.HttpHeaders.PartitionKey];
object effectivePartitionKeyStringObject = null;
if (partitionKeyString != null)
{
range = this.TryResolveServerPartitionByPartitionKey(
request,
partitionKeyString,
collectionCacheIsUptodate,
collection,
routingMap);
}
else if (request.Properties != null && request.Properties.TryGetValue(
WFConstants.BackendHeaders.EffectivePartitionKeyString,
out effectivePartitionKeyStringObject))
{
// Allow EPK only for partitioned collection (excluding migrated fixed collections)
if (!collection.HasPartitionKey || collection.PartitionKey.IsSystemKey.GetValueOrDefault(false))
{
throw new ArgumentOutOfRangeException(nameof(collection));
}
string effectivePartitionKeyString = effectivePartitionKeyStringObject as string;
if (string.IsNullOrEmpty(effectivePartitionKeyString))
{
throw new ArgumentOutOfRangeException(nameof(effectivePartitionKeyString));
}
range = routingMap.GetRangeByEffectivePartitionKey(effectivePartitionKeyString);
}
else
{
range = this.TryResolveSinglePartitionCollection(request, collection, routingMap, collectionCacheIsUptodate);
}
if (range == null)
{
// Collection cache or routing map cache is potentially outdated. Return null -
// upper logic will refresh cache and retry.
return(null);
}
ServiceIdentity serviceIdentity = routingMap.TryGetInfoByPartitionKeyRangeId(range.Id);
PartitionAddressInformation addresses = await this.addressCache.TryGetAddressesAsync(
request,
new PartitionKeyRangeIdentity(collection.ResourceId, range.Id),
serviceIdentity,
forceRefreshPartitionAddresses,
cancellationToken);
if (addresses == null)
{
DefaultTrace.TraceVerbose(
"Could not resolve addresses for identity {0}/{1}. Potentially collection cache or routing map cache is outdated. Return null - upper logic will refresh and retry. ",
new PartitionKeyRangeIdentity(collection.ResourceId, range.Id),
serviceIdentity);
return(null);
}
return(new ResolutionResult(range, addresses, serviceIdentity));
}
private PartitionKeyRange TryResolveServerPartitionByPartitionKey(
DocumentServiceRequest request,
string partitionKeyString,
bool collectionCacheUptoDate,
ContainerProperties collection,
CollectionRoutingMap routingMap)
{
if (request == null)
{
throw new ArgumentNullException("request");
}
if (partitionKeyString == null)
{
throw new ArgumentNullException("partitionKeyString");
}
if (collection == null)
{
throw new ArgumentNullException("collection");
}
if (routingMap == null)
{
throw new ArgumentNullException("routingMap");
}
PartitionKeyInternal partitionKey;
try
{
partitionKey = PartitionKeyInternal.FromJsonString(partitionKeyString);
}
catch (JsonException ex)
{
throw new BadRequestException(
string.Format(CultureInfo.InvariantCulture, RMResources.InvalidPartitionKey, partitionKeyString),
ex)
{
ResourceAddress = request.ResourceAddress
};
}
if (partitionKey == null)
{
throw new InternalServerErrorException(string.Format(CultureInfo.InvariantCulture, "partition key is null '{0}'", partitionKeyString));
}
if (partitionKey.Equals(PartitionKeyInternal.Empty) || partitionKey.Components.Count == collection.PartitionKey.Paths.Count)
{
// Although we can compute effective partition key here, in general case this Gateway can have outdated
// partition key definition cached - like if collection with same name but with Range partitioning is created.
// In this case server will not pass x-ms-documentdb-collection-rid check and will return back InvalidPartitionException.
// Gateway will refresh its cache and retry.
string effectivePartitionKey = partitionKey.GetEffectivePartitionKeyString(collection.PartitionKey);
// There should be exactly one range which contains a partition key. Always.
return(routingMap.GetRangeByEffectivePartitionKey(effectivePartitionKey));
}
if (collectionCacheUptoDate)
{
BadRequestException badRequestException = new BadRequestException(RMResources.PartitionKeyMismatch)
{
ResourceAddress = request.ResourceAddress
};
badRequestException.Headers[WFConstants.BackendHeaders.SubStatus] =
((uint)SubStatusCodes.PartitionKeyMismatch).ToString(CultureInfo.InvariantCulture);
throw badRequestException;
}
// Partition key supplied has different number paths than locally cached partition key definition.
// Three things can happen:
// 1. User supplied wrong partition key.
// 2. Client SDK has outdated partition key definition cache and extracted wrong value from the document.
// 3. Gateway's cache is outdated.
//
// What we will do is append x-ms-documentdb-collection-rid header and forward it to random collection partition.
// * If collection rid matches, server will send back 400.1001, because it also will not be able to compute
// effective partition key. Gateway will forward this status code to client - client will handle it.
// * If collection rid doesn't match, server will send back InvalidPartiitonException and Gateway will
// refresh name routing cache - this will refresh partition key definition as well, and retry.
DefaultTrace.TraceInformation(
"Cannot compute effective partition key. Definition has '{0}' paths, values supplied has '{1}' paths. Will refresh cache and retry.",
collection.PartitionKey.Paths.Count,
partitionKey.Components.Count);
return(null);
}
public void StoreReaderBarrierTest()
{
// create a real document service request
DocumentServiceRequest entity = DocumentServiceRequest.Create(OperationType.Read, ResourceType.Document, AuthorizationTokenType.PrimaryMasterKey);
// set request charge tracker - this is referenced in store reader (ReadMultipleReplicaAsync)
DocumentServiceRequestContext requestContext = new DocumentServiceRequestContext
{
ClientRequestStatistics = new ClientSideRequestStatistics(),
RequestChargeTracker = new RequestChargeTracker()
};
entity.RequestContext = requestContext;
// also setup timeout helper, used in store reader
entity.RequestContext.TimeoutHelper = new TimeoutHelper(new TimeSpan(2, 2, 2));
// when the store reader throws Invalid Partition exception, the higher layer should
// clear this target identity.
entity.RequestContext.TargetIdentity = new ServiceIdentity("dummyTargetIdentity1", new Uri("http://dummyTargetIdentity1"), false);
entity.RequestContext.ResolvedPartitionKeyRange = new PartitionKeyRange();
AddressInformation[] addressInformation = this.GetMockAddressInformationDuringUpgrade();
Mock <IAddressResolver> mockAddressCache = this.GetMockAddressCache(addressInformation);
// validate that the mock works
PartitionAddressInformation partitionAddressInformation = mockAddressCache.Object.ResolveAsync(entity, false, new CancellationToken()).Result;
IReadOnlyList <AddressInformation> addressInfo = partitionAddressInformation.AllAddresses;
Assert.IsTrue(addressInfo[0] == addressInformation[0]);
AddressSelector addressSelector = new AddressSelector(mockAddressCache.Object, Protocol.Tcp);
Uri primaryAddress = addressSelector.ResolvePrimaryTransportAddressUriAsync(entity, false /*forceAddressRefresh*/).Result.Uri;
// check if the address return from Address Selector matches the original address info
Assert.IsTrue(primaryAddress.Equals(addressInformation[0].PhysicalUri));
// get mock transport client that returns a sequence of responses to simulate upgrade
TransportClient mockTransportClient = this.GetMockTransportClientDuringUpgrade(addressInformation);
// get response from mock object
StoreResponse response = mockTransportClient.InvokeResourceOperationAsync(new TransportAddressUri(new Uri(addressInformation[0].PhysicalUri)), entity).Result;
// validate that the LSN matches
Assert.IsTrue(response.LSN == 50);
response.TryGetHeaderValue(WFConstants.BackendHeaders.ActivityId, out string activityId);
// validate that the ActivityId Matches
Assert.IsTrue(activityId == "ACTIVITYID1_1");
// create a real session container - we don't need session for this test anyway
ISessionContainer sessionContainer = new SessionContainer(string.Empty);
// create store reader with mock transport client, real address selector (that has mock address cache), and real session container
StoreReader storeReader =
new StoreReader(mockTransportClient,
addressSelector,
new AddressEnumerator(),
sessionContainer);
// reads always go to read quorum (2) replicas
int replicaCountToRead = 2;
IList <StoreResult> result = storeReader.ReadMultipleReplicaAsync(
entity,
false /*includePrimary*/,
replicaCountToRead,
true /*requiresValidLSN*/,
false /*useSessionToken*/,
ReadMode.Strong).Result;
// make sure we got 2 responses from the store reader
Assert.IsTrue(result.Count == 2);
}
public void GlobalStrongConsistencyMockTest()
{
// create a real document service request (with auth token level = god)
DocumentServiceRequest entity = DocumentServiceRequest.Create(OperationType.Read, ResourceType.Document, AuthorizationTokenType.SystemAll);
// set request charge tracker - this is referenced in store reader (ReadMultipleReplicaAsync)
DocumentServiceRequestContext requestContext = new DocumentServiceRequestContext
{
RequestChargeTracker = new RequestChargeTracker()
};
entity.RequestContext = requestContext;
// set a dummy resource id on the request.
entity.ResourceId = "1-MxAPlgMgA=";
// set consistency level on the request to Bounded Staleness
entity.Headers[HttpConstants.HttpHeaders.ConsistencyLevel] = ConsistencyLevel.BoundedStaleness.ToString();
// also setup timeout helper, used in store reader
entity.RequestContext.TimeoutHelper = new TimeoutHelper(new TimeSpan(2, 2, 2));
// when the store reader throws Invalid Partition exception, the higher layer should
// clear this target identity.
entity.RequestContext.TargetIdentity = new ServiceIdentity("dummyTargetIdentity1", new Uri("http://dummyTargetIdentity1"), false);
entity.RequestContext.ResolvedPartitionKeyRange = new PartitionKeyRange();
AddressInformation[] addressInformation = this.GetMockAddressInformationDuringUpgrade();
Mock <IAddressResolver> mockAddressCache = this.GetMockAddressCache(addressInformation);
// validate that the mock works
PartitionAddressInformation partitionAddressInformation = mockAddressCache.Object.ResolveAsync(entity, false, new CancellationToken()).Result;
IReadOnlyList <AddressInformation> addressInfo = partitionAddressInformation.AllAddresses;
Assert.IsTrue(addressInfo[0] == addressInformation[0]);
AddressSelector addressSelector = new AddressSelector(mockAddressCache.Object, Protocol.Tcp);
Uri primaryAddress = addressSelector.ResolvePrimaryTransportAddressUriAsync(entity, false /*forceAddressRefresh*/).Result.Uri;
// check if the address return from Address Selector matches the original address info
Assert.IsTrue(primaryAddress.Equals(addressInformation[0].PhysicalUri));
// Quorum Met scenario
{
// get mock transport client that returns a sequence of responses to simulate upgrade
TransportClient mockTransportClient = this.GetMockTransportClientForGlobalStrongReads(addressInformation, ReadQuorumResultKind.QuorumMet);
// create a real session container - we don't need session for this test anyway
ISessionContainer sessionContainer = new SessionContainer(string.Empty);
// create store reader with mock transport client, real address selector (that has mock address cache), and real session container
StoreReader storeReader =
new StoreReader(mockTransportClient,
addressSelector,
new AddressEnumerator(),
sessionContainer);
Mock <IAuthorizationTokenProvider> mockAuthorizationTokenProvider = new Mock <IAuthorizationTokenProvider>();
mockAuthorizationTokenProvider.Setup(provider => provider.AddSystemAuthorizationHeaderAsync(
It.IsAny <DocumentServiceRequest>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>()))
.Returns(Task.FromResult(0));
// setup max replica set size on the config reader
ReplicationPolicy replicationPolicy = new ReplicationPolicy
{
MaxReplicaSetSize = 4
};
Mock <IServiceConfigurationReader> mockServiceConfigReader = new Mock <IServiceConfigurationReader>();
mockServiceConfigReader.SetupGet(x => x.UserReplicationPolicy).Returns(replicationPolicy);
QuorumReader reader =
new QuorumReader(mockTransportClient, addressSelector, storeReader, mockServiceConfigReader.Object, mockAuthorizationTokenProvider.Object);
entity.RequestContext.OriginalRequestConsistencyLevel = Documents.ConsistencyLevel.Strong;
entity.RequestContext.ClientRequestStatistics = new ClientSideRequestStatistics();
StoreResponse result = reader.ReadStrongAsync(entity, 2, ReadMode.Strong).Result;
Assert.IsTrue(result.LSN == 100);
result.TryGetHeaderValue(WFConstants.BackendHeaders.GlobalCommittedLSN, out string globalCommitedLSN);
long nGlobalCommitedLSN = long.Parse(globalCommitedLSN, CultureInfo.InvariantCulture);
Assert.IsTrue(nGlobalCommitedLSN == 90);
}
// Quorum Selected scenario
{
// get mock transport client that returns a sequence of responses to simulate upgrade
TransportClient mockTransportClient = this.GetMockTransportClientForGlobalStrongReads(addressInformation, ReadQuorumResultKind.QuorumSelected);
// create a real session container - we don't need session for this test anyway
ISessionContainer sessionContainer = new SessionContainer(string.Empty);
// create store reader with mock transport client, real address selector (that has mock address cache), and real session container
StoreReader storeReader =
new StoreReader(mockTransportClient,
addressSelector,
new AddressEnumerator(),
sessionContainer);
Mock <IAuthorizationTokenProvider> mockAuthorizationTokenProvider = new Mock <IAuthorizationTokenProvider>();
mockAuthorizationTokenProvider.Setup(provider => provider.AddSystemAuthorizationHeaderAsync(
It.IsAny <DocumentServiceRequest>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>()))
.Returns(Task.FromResult(0));
// setup max replica set size on the config reader
ReplicationPolicy replicationPolicy = new ReplicationPolicy
{
MaxReplicaSetSize = 4
};
Mock <IServiceConfigurationReader> mockServiceConfigReader = new Mock <IServiceConfigurationReader>();
mockServiceConfigReader.SetupGet(x => x.UserReplicationPolicy).Returns(replicationPolicy);
QuorumReader reader =
new QuorumReader(mockTransportClient, addressSelector, storeReader, mockServiceConfigReader.Object, mockAuthorizationTokenProvider.Object);
entity.RequestContext.OriginalRequestConsistencyLevel = Documents.ConsistencyLevel.Strong;
entity.RequestContext.ClientRequestStatistics = new ClientSideRequestStatistics();
entity.RequestContext.QuorumSelectedLSN = -1;
entity.RequestContext.GlobalCommittedSelectedLSN = -1;
try
{
StoreResponse result = reader.ReadStrongAsync(entity, 2, ReadMode.Strong).Result;
Assert.IsTrue(false);
}
catch (AggregateException ex)
{
if (ex.InnerException is GoneException)
{
DefaultTrace.TraceInformation("Gone exception expected!");
}
}
Assert.IsTrue(entity.RequestContext.QuorumSelectedLSN == 100);
Assert.IsTrue(entity.RequestContext.GlobalCommittedSelectedLSN == 100);
}
// Quorum not met scenario
{
// get mock transport client that returns a sequence of responses to simulate upgrade
TransportClient mockTransportClient = this.GetMockTransportClientForGlobalStrongReads(addressInformation, ReadQuorumResultKind.QuorumNotSelected);
// create a real session container - we don't need session for this test anyway
ISessionContainer sessionContainer = new SessionContainer(string.Empty);
// create store reader with mock transport client, real address selector (that has mock address cache), and real session container
StoreReader storeReader =
new StoreReader(mockTransportClient,
addressSelector,
new AddressEnumerator(),
sessionContainer);
Mock <IAuthorizationTokenProvider> mockAuthorizationTokenProvider = new Mock <IAuthorizationTokenProvider>();
mockAuthorizationTokenProvider.Setup(provider => provider.AddSystemAuthorizationHeaderAsync(
It.IsAny <DocumentServiceRequest>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>()))
.Returns(Task.FromResult(0));
// setup max replica set size on the config reader
ReplicationPolicy replicationPolicy = new ReplicationPolicy
{
MaxReplicaSetSize = 4
};
Mock <IServiceConfigurationReader> mockServiceConfigReader = new Mock <IServiceConfigurationReader>();
mockServiceConfigReader.SetupGet(x => x.UserReplicationPolicy).Returns(replicationPolicy);
QuorumReader reader =
new QuorumReader(mockTransportClient, addressSelector, storeReader, mockServiceConfigReader.Object, mockAuthorizationTokenProvider.Object);
entity.RequestContext.PerformLocalRefreshOnGoneException = true;
entity.RequestContext.OriginalRequestConsistencyLevel = Documents.ConsistencyLevel.Strong;
entity.RequestContext.ClientRequestStatistics = new ClientSideRequestStatistics();
StoreResponse result = reader.ReadStrongAsync(entity, 2, ReadMode.Strong).Result;
Assert.IsTrue(result.LSN == 100);
result.TryGetHeaderValue(WFConstants.BackendHeaders.GlobalCommittedLSN, out string globalCommitedLSN);
long nGlobalCommitedLSN = long.Parse(globalCommitedLSN, CultureInfo.InvariantCulture);
Assert.IsTrue(nGlobalCommitedLSN == 90);
}
}
public void MockStoreClientTest()
{
// create a real document service request (with auth token level = god)
DocumentServiceRequest entity = DocumentServiceRequest.Create(OperationType.Read, ResourceType.Document, AuthorizationTokenType.PrimaryMasterKey);
// set request charge tracker - this is referenced in store reader (ReadMultipleReplicaAsync)
var requestContext = new DocumentServiceRequestContext();
requestContext.RequestChargeTracker = new RequestChargeTracker();
entity.RequestContext = requestContext;
// set a dummy resource id on the request.
entity.ResourceId = "1-MxAPlgMgA=";
// set consistency level on the request to Bounded Staleness
entity.Headers[HttpConstants.HttpHeaders.ConsistencyLevel] = ConsistencyLevel.BoundedStaleness.ToString();
// also setup timeout helper, used in store reader
entity.RequestContext.TimeoutHelper = new TimeoutHelper(new TimeSpan(2, 2, 2));
// when the store reader throws Invalid Partition exception, the higher layer should
// clear this target identity.
entity.RequestContext.TargetIdentity = new ServiceIdentity("dummyTargetIdentity1", new Uri("http://dummyTargetIdentity1"), false);
entity.RequestContext.ResolvedPartitionKeyRange = new PartitionKeyRange();
AddressInformation[] addressInformation = GetMockAddressInformationDuringUpgrade();
var mockAddressCache = GetMockAddressCache(addressInformation);
// validate that the mock works
PartitionAddressInformation partitionAddressInformation = mockAddressCache.Object.ResolveAsync(entity, false, new CancellationToken()).Result;
var addressInfo = partitionAddressInformation.AllAddresses;
Assert.IsTrue(addressInfo[0] == addressInformation[0]);
AddressSelector addressSelector = new AddressSelector(mockAddressCache.Object, Protocol.Tcp);
var primaryAddress = addressSelector.ResolvePrimaryUriAsync(entity, false /*forceAddressRefresh*/).Result;
// check if the address return from Address Selector matches the original address info
Assert.IsTrue(primaryAddress.Equals(addressInformation[0].PhysicalUri));
// get mock transport client that returns a sequence of responses to simulate upgrade
var mockTransportClient = GetMockTransportClientDuringUpgrade(addressInformation);
// get response from mock object
var response = mockTransportClient.InvokeResourceOperationAsync(new Uri(addressInformation[0].PhysicalUri), entity).Result;
// validate that the LSN matches
Assert.IsTrue(response.LSN == 50);
string activityId;
response.TryGetHeaderValue(WFConstants.BackendHeaders.ActivityId, out activityId);
// validate that the ActivityId Matches
Assert.IsTrue(activityId == "ACTIVITYID1_1");
// create a real session container - we don't need session for this test anyway
ISessionContainer sessionContainer = new SessionContainer(string.Empty);
// create store reader with mock transport client, real address selector (that has mock address cache), and real session container
StoreReader storeReader =
new StoreReader(mockTransportClient,
addressSelector,
sessionContainer);
Mock<IAuthorizationTokenProvider> mockAuthorizationTokenProvider = new Mock<IAuthorizationTokenProvider>();
mockAuthorizationTokenProvider.Setup(provider => provider.GetSystemAuthorizationTokenAsync(
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<INameValueCollection>(), It.IsAny<AuthorizationTokenType>()))
.ReturnsAsync("dummyauthtoken");
// setup max replica set size on the config reader
ReplicationPolicy replicationPolicy = new ReplicationPolicy();
replicationPolicy.MaxReplicaSetSize = 4;
Mock<IServiceConfigurationReader> mockServiceConfigReader = new Mock<IServiceConfigurationReader>();
mockServiceConfigReader.SetupGet(x => x.UserReplicationPolicy).Returns(replicationPolicy);
try
{
StoreClient storeClient =
new StoreClient(
mockAddressCache.Object,
sessionContainer,
mockServiceConfigReader.Object,
mockAuthorizationTokenProvider.Object,
Protocol.Tcp,
mockTransportClient);
ServerStoreModel storeModel = new ServerStoreModel(storeClient);
var result = storeModel.ProcessMessageAsync(entity).Result;
// if we have reached this point, there was a successful request.
// validate if the target identity has been cleared out.
// If the target identity is null and the request still succeeded, it means
// that the very first read succeeded without a barrier request.
Assert.IsNull(entity.RequestContext.TargetIdentity);
Assert.IsNull(entity.RequestContext.ResolvedPartitionKeyRange);
}
catch(Exception e)
{
Assert.IsTrue(e.InnerException is ServiceUnavailableException
|| e.InnerException is ArgumentNullException
|| e.InnerException is ServiceUnavailableException
|| e.InnerException is NullReferenceException);
}
}
private async Task TestGatewayStoreModelProcessMessageAsync(GatewayStoreModel storeModel, DocumentServiceRequest request)
{
request.Headers["x-ms-session-token"] = "0:1#100#1=20#2=5#3=30";
await storeModel.ProcessMessageAsync(request);
request.Headers.Remove("x-ms-session-token");
request.Headers["x-ms-consistency-level"] = "Session";
await storeModel.ProcessMessageAsync(request);
}
private async Task ValidateRetryOnWriteSessionNotAvailabeWithEnableMultipleWriteLocationsAsync()
{
const bool useMultipleWriteLocations = true;
bool enableEndpointDiscovery = true;
this.Initialize(
useMultipleWriteLocations: useMultipleWriteLocations,
enableEndpointDiscovery: enableEndpointDiscovery,
isPreferredLocationsListEmpty: false);
await this.endpointManager.RefreshLocationAsync(this.databaseAccount);
ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, enableEndpointDiscovery, new RetryOptions());
using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: false, isMasterResourceType: false))
{
int retryCount = 0;
try
{
await BackoffRetryUtility <bool> .ExecuteAsync(
() =>
{
retryPolicy.OnBeforeSendRequest(request);
if (retryCount == 0)
{
Assert.IsFalse(request.ClearSessionTokenOnSessionReadFailure);
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
}
else if (retryCount == 1)
{
Assert.IsFalse(request.ClearSessionTokenOnSessionReadFailure);
// Second request must go to first write endpoint
Uri expectedEndpoint = new Uri(this.databaseAccount.WriteLocationsInternal[0].DatabaseAccountEndpoint);
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
}
else if (retryCount == 2)
{
Assert.IsFalse(request.ClearSessionTokenOnSessionReadFailure);
// Second request must go to first write endpoint
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
}
else if (retryCount == 3)
{
Assert.IsTrue(request.ClearSessionTokenOnSessionReadFailure);
// Second request must go to first write endpoint
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[2]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
}
else
{
Assert.Fail();
}
retryCount++;
StringKeyValueCollection headers = new StringKeyValueCollection();
headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.ReadSessionNotAvailable).ToString();
DocumentClientException notFoundException = new NotFoundException(RMResources.NotFound, headers);
throw notFoundException;
},
retryPolicy);
Assert.Fail();
}
catch (NotFoundException)
{
DefaultTrace.TraceInformation("Received expected notFoundException");
Assert.AreEqual(4, retryCount);
}
}
}
public override async Task <CosmosResponseMessage> SendAsync(
CosmosRequestMessage request,
CancellationToken cancellationToken)
{
CosmosResponseMessage response = null;
string originalContinuation = request.Headers.Continuation;
try
{
RntdbEnumerationDirection rntdbEnumerationDirection = RntdbEnumerationDirection.Forward;
object direction;
if (request.Properties.TryGetValue(HttpConstants.HttpHeaders.EnumerationDirection, out direction))
{
rntdbEnumerationDirection = (byte)RntdbEnumerationDirection.Reverse == (byte)direction ? RntdbEnumerationDirection.Reverse : RntdbEnumerationDirection.Forward;
}
request.Headers.Remove(HttpConstants.HttpHeaders.IsContinuationExpected);
request.Headers.Add(HttpConstants.HttpHeaders.IsContinuationExpected, bool.TrueString);
object startEpk;
object endEpk;
if (!request.Properties.TryGetValue(HandlerConstants.StartEpkString, out startEpk))
{
startEpk = PartitionKeyInternal.MinimumInclusiveEffectivePartitionKey;
}
if (!request.Properties.TryGetValue(HandlerConstants.EndEpkString, out endEpk))
{
endEpk = PartitionKeyInternal.MaximumExclusiveEffectivePartitionKey;
}
startEpk = startEpk ?? PartitionKeyInternal.MinimumInclusiveEffectivePartitionKey;
endEpk = endEpk ?? PartitionKeyInternal.MaximumExclusiveEffectivePartitionKey;
List <Range <string> > providedRanges = new List <Range <string> > {
new Range <string>(
(string)startEpk,
(string)endEpk,
isMinInclusive: true,
isMaxInclusive: false)
};
DocumentServiceRequest serviceRequest = request.ToDocumentServiceRequest();
PartitionKeyRangeCache routingMapProvider = await this.client.DocumentClient.GetPartitionKeyRangeCacheAsync();
CollectionCache collectionCache = await this.client.DocumentClient.GetCollectionCacheAsync();
CosmosContainerSettings collectionFromCache =
await collectionCache.ResolveCollectionAsync(serviceRequest, CancellationToken.None);
List <CompositeContinuationToken> suppliedTokens;
//direction is not expected to change between continuations.
Range <string> rangeFromContinuationToken =
this.partitionRoutingHelper.ExtractPartitionKeyRangeFromContinuationToken(serviceRequest.Headers, out suppliedTokens);
ResolvedRangeInfo resolvedRangeInfo =
await this.partitionRoutingHelper.TryGetTargetRangeFromContinuationTokenRange(
providedPartitionKeyRanges : providedRanges,
routingMapProvider : routingMapProvider,
collectionRid : collectionFromCache.ResourceId,
rangeFromContinuationToken : rangeFromContinuationToken,
suppliedTokens : suppliedTokens,
direction : rntdbEnumerationDirection);
if (serviceRequest.IsNameBased && resolvedRangeInfo.ResolvedRange == null && resolvedRangeInfo.ContinuationTokens == null)
{
serviceRequest.ForceNameCacheRefresh = true;
collectionFromCache = await collectionCache.ResolveCollectionAsync(serviceRequest, CancellationToken.None);
resolvedRangeInfo = await this.partitionRoutingHelper.TryGetTargetRangeFromContinuationTokenRange(
providedPartitionKeyRanges : providedRanges,
routingMapProvider : routingMapProvider,
collectionRid : collectionFromCache.ResourceId,
rangeFromContinuationToken : rangeFromContinuationToken,
suppliedTokens : suppliedTokens,
direction : rntdbEnumerationDirection);
}
if (resolvedRangeInfo.ResolvedRange == null && resolvedRangeInfo.ContinuationTokens == null)
{
return(((DocumentClientException)
new NotFoundException(
$"{DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture)}: Was not able to get queryRoutingInfo even after resolve collection async with force name cache refresh to the following collectionRid: {collectionFromCache.ResourceId} with the supplied tokens: {JsonConvert.SerializeObject(suppliedTokens)}")
).ToCosmosResponseMessage(request));
}
serviceRequest.RouteTo(new PartitionKeyRangeIdentity(collectionFromCache.ResourceId, resolvedRangeInfo.ResolvedRange.Id));
response = await base.SendAsync(request, cancellationToken);
if (!response.IsSuccessStatusCode)
{
this.SetOriginalContinuationToken(request, response, originalContinuation);
}
else
{
if (!await this.partitionRoutingHelper.TryAddPartitionKeyRangeToContinuationTokenAsync(
response.Headers.CosmosMessageHeaders,
providedPartitionKeyRanges: providedRanges,
routingMapProvider: routingMapProvider,
collectionRid: collectionFromCache.ResourceId,
resolvedRangeInfo: resolvedRangeInfo,
direction: rntdbEnumerationDirection))
{
return(((DocumentClientException)
new NotFoundException(
$"{DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture)}: Call to TryAddPartitionKeyRangeToContinuationTokenAsync failed to the following collectionRid: {collectionFromCache.ResourceId} with the supplied tokens: {JsonConvert.SerializeObject(suppliedTokens)}")
).ToCosmosResponseMessage(request));
}
}
return(response);
}
catch (DocumentClientException ex)
{
CosmosResponseMessage errorResponse = ex.ToCosmosResponseMessage(request);
this.SetOriginalContinuationToken(request, errorResponse, originalContinuation);
return(errorResponse);
}
catch (AggregateException ex)
{
this.SetOriginalContinuationToken(request, response, originalContinuation);
// TODO: because the SDK underneath this path uses ContinueWith or task.Result we need to catch AggregateExceptions here
// in order to ensure that underlying DocumentClientExceptions get propagated up correctly. Once all ContinueWith and .Result
// is removed this catch can be safely removed.
AggregateException innerExceptions = ex.Flatten();
Exception docClientException = innerExceptions.InnerExceptions.FirstOrDefault(innerEx => innerEx is DocumentClientException);
if (docClientException != null)
{
return(((DocumentClientException)docClientException).ToCosmosResponseMessage(request));
}
throw;
}
}
internal static async Task <DocumentServiceResponse> ParseResponseAsync(HttpResponseMessage responseMessage, JsonSerializerSettings serializerSettings = null, DocumentServiceRequest request = null)
{
using (responseMessage)
{
IClientSideRequestStatistics requestStatistics = request?.RequestContext?.ClientRequestStatistics;
if ((int)responseMessage.StatusCode < 400)
{
INameValueCollection headers = GatewayStoreClient.ExtractResponseHeaders(responseMessage);
Stream contentStream = await GatewayStoreClient.BufferContentIfAvailableAsync(responseMessage);
return(new DocumentServiceResponse(
body: contentStream,
headers: headers,
statusCode: responseMessage.StatusCode,
clientSideRequestStatistics: requestStatistics,
serializerSettings: serializerSettings));
}
else if (request != null &&
request.IsValidStatusCodeForExceptionlessRetry((int)responseMessage.StatusCode))
{
INameValueCollection headers = GatewayStoreClient.ExtractResponseHeaders(responseMessage);
Stream contentStream = await GatewayStoreClient.BufferContentIfAvailableAsync(responseMessage);
return(new DocumentServiceResponse(
body: contentStream,
headers: headers,
statusCode: responseMessage.StatusCode,
clientSideRequestStatistics: requestStatistics,
serializerSettings: serializerSettings));
}
else
{
throw await GatewayStoreClient.CreateDocumentClientExceptionAsync(responseMessage, requestStatistics);
}
}
}
internal override async Task <StoreResponse> InvokeStoreAsync(Uri baseAddress, ResourceOperation resourceOperation, DocumentServiceRequest request)
{
Uri physicalAddress = GatewayStoreClient.IsFeedRequest(request.OperationType) ?
HttpTransportClient.GetResourceFeedUri(resourceOperation.resourceType, baseAddress, request) :
HttpTransportClient.GetResourceEntryUri(resourceOperation.resourceType, baseAddress, request);
using (HttpResponseMessage responseMessage = await this.InvokeClientAsync(request, resourceOperation.resourceType, physicalAddress, default))
{
return(await HttpTransportClient.ProcessHttpResponse(request.ResourceAddress, string.Empty, responseMessage, physicalAddress, request));
}
}
private async ValueTask <HttpRequestMessage> PrepareRequestMessageAsync(
DocumentServiceRequest request,
Uri physicalAddress)
{
HttpMethod httpMethod = HttpMethod.Head;
if (request.OperationType == OperationType.Create ||
request.OperationType == OperationType.Upsert ||
request.OperationType == OperationType.Query ||
request.OperationType == OperationType.SqlQuery ||
request.OperationType == OperationType.Batch ||
request.OperationType == OperationType.ExecuteJavaScript ||
request.OperationType == OperationType.QueryPlan ||
(request.ResourceType == ResourceType.PartitionKey && request.OperationType == OperationType.Delete))
{
httpMethod = HttpMethod.Post;
}
else if (request.OperationType == OperationType.Read ||
request.OperationType == OperationType.ReadFeed)
{
httpMethod = HttpMethod.Get;
}
else if ((request.OperationType == OperationType.Replace) ||
(request.OperationType == OperationType.CollectionTruncate))
{
httpMethod = HttpMethod.Put;
}
else if (request.OperationType == OperationType.Delete)
{
httpMethod = HttpMethod.Delete;
}
else if (request.OperationType == OperationType.Patch)
{
// There isn't support for PATCH method in .NetStandard 2.0
httpMethod = httpPatchMethod;
}
else
{
throw new NotImplementedException();
}
HttpRequestMessage requestMessage = new HttpRequestMessage(httpMethod, physicalAddress);
// The StreamContent created below will own and dispose its underlying stream, but we may need to reuse the stream on the
// DocumentServiceRequest for future requests. Hence we need to clone without incurring copy cost, so that when
// HttpRequestMessage -> StreamContent -> MemoryStream all get disposed, the original stream will be left open.
if (request.Body != null)
{
await request.EnsureBufferedBodyAsync();
MemoryStream clonedStream = new MemoryStream();
// WriteTo doesn't use and update Position of source stream. No point in setting/restoring it.
request.CloneableBody.WriteTo(clonedStream);
clonedStream.Position = 0;
requestMessage.Content = new StreamContent(clonedStream);
}
if (request.Headers != null)
{
foreach (string key in request.Headers)
{
if (GatewayStoreClient.IsAllowedRequestHeader(key))
{
if (key.Equals(HttpConstants.HttpHeaders.ContentType, StringComparison.OrdinalIgnoreCase))
{
requestMessage.Content.Headers.ContentType = new MediaTypeHeaderValue(request.Headers[key]);
}
else
{
requestMessage.Headers.TryAddWithoutValidation(key, request.Headers[key]);
}
}
}
}
// add activityId
Guid activityId = System.Diagnostics.Trace.CorrelationManager.ActivityId;
Debug.Assert(activityId != Guid.Empty);
requestMessage.Headers.Add(HttpConstants.HttpHeaders.ActivityId, activityId.ToString());
return(requestMessage);
}
private async Task <Tuple <PartitionKeyRangeIdentity, PartitionAddressInformation> > ResolveMasterAsync(DocumentServiceRequest request, bool forceRefresh)
{
Tuple <PartitionKeyRangeIdentity, PartitionAddressInformation> masterAddressAndRange = this.masterPartitionAddressCache;
int targetReplicaSetSize = this.serviceConfigReader.SystemReplicationPolicy.MaxReplicaSetSize;
forceRefresh = forceRefresh ||
(masterAddressAndRange != null &&
masterAddressAndRange.Item2.AllAddresses.Count() < targetReplicaSetSize &&
DateTime.UtcNow.Subtract(this.suboptimalMasterPartitionTimestamp) > TimeSpan.FromSeconds(this.suboptimalPartitionForceRefreshIntervalInSeconds));
if (forceRefresh || this.masterPartitionAddressCache == null)
{
string entryUrl = PathsHelper.GeneratePath(
ResourceType.Database,
string.Empty,
true);
try
{
FeedResource <Address> masterAddresses = await this.GetMasterAddressesViaGatewayAsync(
request,
ResourceType.Database,
null,
entryUrl,
forceRefresh,
false);
masterAddressAndRange = this.ToPartitionAddressAndRange(string.Empty, masterAddresses.ToList());
this.masterPartitionAddressCache = masterAddressAndRange;
this.suboptimalMasterPartitionTimestamp = DateTime.MaxValue;
}
catch (Exception)
{
this.suboptimalMasterPartitionTimestamp = DateTime.MaxValue;
throw;
}
}
if (masterAddressAndRange.Item2.AllAddresses.Count() < targetReplicaSetSize && this.suboptimalMasterPartitionTimestamp.Equals(DateTime.MaxValue))
{
this.suboptimalMasterPartitionTimestamp = DateTime.UtcNow;
}
return(masterAddressAndRange);
}
private DocumentFeedResponse <CosmosElement> GetFeedResponse(
DocumentServiceRequest documentServiceRequest,
DocumentServiceResponse documentServiceResponse)
{
// Execute the callback an each element of the page
// For example just could get a response like this
// {
// "_rid": "qHVdAImeKAQ=",
// "Documents": [{
// "id": "03230",
// "_rid": "qHVdAImeKAQBAAAAAAAAAA==",
// "_self": "dbs\/qHVdAA==\/colls\/qHVdAImeKAQ=\/docs\/qHVdAImeKAQBAAAAAAAAAA==\/",
// "_etag": "\"410000b0-0000-0000-0000-597916b00000\"",
// "_attachments": "attachments\/",
// "_ts": 1501107886
// }],
// "_count": 1
// }
// And you should execute the callback on each document in "Documents".
MemoryStream memoryStream = new MemoryStream();
documentServiceResponse.ResponseBody.CopyTo(memoryStream);
long responseLengthBytes = memoryStream.Length;
byte[] content = memoryStream.ToArray();
IJsonNavigator jsonNavigator = null;
// Use the users custom navigator first. If it returns null back try the
// internal navigator.
if (this.feedOptions.CosmosSerializationOptions != null)
{
jsonNavigator = this.feedOptions.CosmosSerializationOptions.CreateCustomNavigatorCallback(content);
if (jsonNavigator == null)
{
throw new InvalidOperationException("The CosmosSerializationOptions did not return a JSON navigator.");
}
}
else
{
jsonNavigator = JsonNavigator.Create(content);
}
string resourceName = this.GetRootNodeName(documentServiceRequest.ResourceType);
if (!jsonNavigator.TryGetObjectProperty(
jsonNavigator.GetRootNode(),
resourceName,
out ObjectProperty objectProperty))
{
throw new InvalidOperationException($"Response Body Contract was violated. QueryResponse did not have property: {resourceName}");
}
IJsonNavigatorNode cosmosElements = objectProperty.ValueNode;
if (!(CosmosElement.Dispatch(
jsonNavigator,
cosmosElements) is CosmosArray cosmosArray))
{
throw new InvalidOperationException($"QueryResponse did not have an array of : {resourceName}");
}
int itemCount = cosmosArray.Count;
return(new DocumentFeedResponse <CosmosElement>(
cosmosArray,
itemCount,
documentServiceResponse.Headers,
documentServiceResponse.RequestStats,
responseLengthBytes));
}
private async Task <FeedResource <Address> > GetServerAddressesViaGatewayAsync(
DocumentServiceRequest request,
string collectionRid,
IEnumerable <string> partitionKeyRangeIds,
bool forceRefresh)
{
string entryUrl = PathsHelper.GeneratePath(ResourceType.Document, collectionRid, true);
INameValueCollection addressQuery = new StringKeyValueCollection();
addressQuery.Add(HttpConstants.QueryStrings.Url, HttpUtility.UrlEncode(entryUrl));
INameValueCollection headers = new StringKeyValueCollection();
if (forceRefresh)
{
headers.Set(HttpConstants.HttpHeaders.ForceRefresh, bool.TrueString);
}
if (request.ForceCollectionRoutingMapRefresh)
{
headers.Set(HttpConstants.HttpHeaders.ForceCollectionRoutingMapRefresh, bool.TrueString);
}
addressQuery.Add(HttpConstants.QueryStrings.Filter, this.protocolFilter);
addressQuery.Add(HttpConstants.QueryStrings.PartitionKeyRangeIds, string.Join(",", partitionKeyRangeIds));
string resourceTypeToSign = PathsHelper.GetResourcePath(ResourceType.Document);
headers.Set(HttpConstants.HttpHeaders.XDate, DateTime.UtcNow.ToString("r", CultureInfo.InvariantCulture));
string token = null;
try
{
token = this.tokenProvider.GetUserAuthorizationToken(
collectionRid,
resourceTypeToSign,
HttpConstants.HttpMethods.Get,
headers,
AuthorizationTokenType.PrimaryMasterKey);
}
catch (UnauthorizedException)
{
}
if (token == null && request.IsNameBased)
{
// User doesn't have rid based resource token. Maybe he has name based.
string collectionAltLink = PathsHelper.GetCollectionPath(request.ResourceAddress);
token = this.tokenProvider.GetUserAuthorizationToken(
collectionAltLink,
resourceTypeToSign,
HttpConstants.HttpMethods.Get,
headers,
AuthorizationTokenType.PrimaryMasterKey);
}
headers.Set(HttpConstants.HttpHeaders.Authorization, token);
Uri targetEndpoint = UrlUtility.SetQuery(this.addressEndpoint, UrlUtility.CreateQuery(addressQuery));
string identifier = GatewayAddressCache.LogAddressResolutionStart(request, targetEndpoint);
using (HttpResponseMessage httpResponseMessage = await this.httpClient.GetAsync(targetEndpoint, headers))
{
using (DocumentServiceResponse documentServiceResponse =
await ClientExtensions.ParseResponseAsync(httpResponseMessage))
{
GatewayAddressCache.LogAddressResolutionEnd(request, identifier);
return(documentServiceResponse.GetResource <FeedResource <Address> >());
}
}
}
/// <summary>
/// Buffers more documents in the producer.
/// </summary>
/// <param name="token">The cancellation token.</param>
/// <returns>A task to await on.</returns>
public async Task BufferMoreDocuments(CancellationToken token)
{
token.ThrowIfCancellationRequested();
try
{
await this.fetchSemaphore.WaitAsync();
if (!this.HasMoreBackendResults)
{
// Just NOP
return;
}
this.fetchSchedulingMetrics.Start();
this.fetchExecutionRangeAccumulator.BeginFetchRange();
int pageSize = (int)Math.Min(this.pageSize, (long)int.MaxValue);
using (DocumentServiceRequest request = this.createRequestFunc(this.PartitionKeyRange, this.backendContinuationToken, pageSize))
{
// BUG: retryPolicyInstance shound not be shared betweet different requests
IDocumentClientRetryPolicy retryPolicy = this.createRetryPolicyFunc();
retryPolicy.OnBeforeSendRequest(request);
// Custom backoff and retry
while (true)
{
int retries = 0;
try
{
FeedResponse <CosmosElement> feedResponse = await this.executeRequestFunc(request, token);
this.fetchExecutionRangeAccumulator.EndFetchRange(
feedResponse.ActivityId,
feedResponse.Count,
retries);
this.fetchSchedulingMetrics.Stop();
this.hasStartedFetching = true;
this.backendContinuationToken = feedResponse.ResponseContinuation;
this.activityId = Guid.Parse(feedResponse.ActivityId);
await this.bufferedPages.AddAsync(feedResponse);
Interlocked.Add(ref this.bufferedItemCount, feedResponse.Count);
QueryMetrics queryMetrics = QueryMetrics.Zero;
if (feedResponse.ResponseHeaders[HttpConstants.HttpHeaders.QueryMetrics] != null)
{
queryMetrics = QueryMetrics.CreateFromDelimitedStringAndClientSideMetrics(
feedResponse.ResponseHeaders[HttpConstants.HttpHeaders.QueryMetrics],
new ClientSideMetrics(
retries,
feedResponse.RequestCharge,
this.fetchExecutionRangeAccumulator.GetExecutionRanges(),
new List <Tuple <string, SchedulingTimeSpan> >()));
}
if (!this.HasMoreBackendResults)
{
queryMetrics = QueryMetrics.CreateWithSchedulingMetrics(
queryMetrics,
new List <Tuple <string, SchedulingTimeSpan> >
{
new Tuple <string, SchedulingTimeSpan>(
this.PartitionKeyRange.Id,
this.fetchSchedulingMetrics.Elapsed)
});
}
this.produceAsyncCompleteCallback(
this,
feedResponse.Count,
feedResponse.RequestCharge,
queryMetrics,
feedResponse.ResponseLengthBytes,
token);
break;
}
catch (Exception exception)
{
// See if we need to retry or just throw
ShouldRetryResult shouldRetryResult = await retryPolicy.ShouldRetryAsync(exception, token);
if (!shouldRetryResult.ShouldRetry)
{
if (shouldRetryResult.ExceptionToThrow != null)
{
throw shouldRetryResult.ExceptionToThrow;
}
else
{
// Propagate original exception.
throw;
}
}
else
{
await Task.Delay(shouldRetryResult.BackoffTime);
retries++;
}
}
}
}
}
finally
{
this.fetchSchedulingMetrics.Stop();
this.fetchSemaphore.Release();
}
}
public void GlobalStrongConsistentWriteMockTest()
{
// create a real document service request (with auth token level = god)
DocumentServiceRequest entity = DocumentServiceRequest.Create(OperationType.Create, ResourceType.Document, AuthorizationTokenType.SystemAll);
// set request charge tracker - this is referenced in store reader (ReadMultipleReplicaAsync)
DocumentServiceRequestContext requestContext = new DocumentServiceRequestContext
{
RequestChargeTracker = new RequestChargeTracker()
};
entity.RequestContext = requestContext;
// set a dummy resource id on the request.
entity.ResourceId = "1-MxAPlgMgA=";
// set consistency level on the request to Bounded Staleness
entity.Headers[HttpConstants.HttpHeaders.ConsistencyLevel] = ConsistencyLevel.Strong.ToString();
// also setup timeout helper, used in store reader
entity.RequestContext.TimeoutHelper = new TimeoutHelper(new TimeSpan(2, 2, 2));
// when the store reader throws Invalid Partition exception, the higher layer should
// clear this target identity.
entity.RequestContext.TargetIdentity = new ServiceIdentity("dummyTargetIdentity1", new Uri("http://dummyTargetIdentity1"), false);
entity.RequestContext.ResolvedPartitionKeyRange = new PartitionKeyRange();
AddressInformation[] addressInformation = this.GetMockAddressInformationDuringUpgrade();
Mock <IAddressResolver> mockAddressCache = this.GetMockAddressCache(addressInformation);
// validate that the mock works
PartitionAddressInformation partitionAddressInformation = mockAddressCache.Object.ResolveAsync(entity, false, new CancellationToken()).Result;
IReadOnlyList <AddressInformation> addressInfo = partitionAddressInformation.AllAddresses;
Assert.IsTrue(addressInfo[0] == addressInformation[0]);
AddressSelector addressSelector = new AddressSelector(mockAddressCache.Object, Protocol.Tcp);
Uri primaryAddress = addressSelector.ResolvePrimaryTransportAddressUriAsync(entity, false /*forceAddressRefresh*/).Result.Uri;
// check if the address return from Address Selector matches the original address info
Assert.IsTrue(primaryAddress.Equals(addressInformation[0].PhysicalUri));
// create a real session container - we don't need session for this test anyway
ISessionContainer sessionContainer = new SessionContainer(string.Empty);
Mock <IServiceConfigurationReader> mockServiceConfigReader = new Mock <IServiceConfigurationReader>();
Mock <IAuthorizationTokenProvider> mockAuthorizationTokenProvider = new Mock <IAuthorizationTokenProvider>();
mockAuthorizationTokenProvider.Setup(provider => provider.AddSystemAuthorizationHeaderAsync(
It.IsAny <DocumentServiceRequest>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>()))
.Returns(Task.FromResult(0));
for (int i = 0; i < addressInformation.Length; i++)
{
TransportClient mockTransportClient = this.GetMockTransportClientForGlobalStrongWrites(addressInformation, i, false, false, false);
StoreReader storeReader = new StoreReader(mockTransportClient, addressSelector, new AddressEnumerator(), sessionContainer);
ConsistencyWriter consistencyWriter = new ConsistencyWriter(addressSelector, sessionContainer, mockTransportClient, mockServiceConfigReader.Object, mockAuthorizationTokenProvider.Object, false);
StoreResponse response = consistencyWriter.WriteAsync(entity, new TimeoutHelper(TimeSpan.FromSeconds(30)), false).Result;
Assert.AreEqual(100, response.LSN);
//globalCommittedLsn never catches up in this case
mockTransportClient = this.GetMockTransportClientForGlobalStrongWrites(addressInformation, i, true, false, false);
consistencyWriter = new ConsistencyWriter(addressSelector, sessionContainer, mockTransportClient, mockServiceConfigReader.Object, mockAuthorizationTokenProvider.Object, false);
try
{
response = consistencyWriter.WriteAsync(entity, new TimeoutHelper(TimeSpan.FromSeconds(30)), false).Result;
Assert.Fail();
}
catch (Exception)
{
}
mockTransportClient = this.GetMockTransportClientForGlobalStrongWrites(addressInformation, i, false, true, false);
consistencyWriter = new ConsistencyWriter(addressSelector, sessionContainer, mockTransportClient, mockServiceConfigReader.Object, mockAuthorizationTokenProvider.Object, false);
response = consistencyWriter.WriteAsync(entity, new TimeoutHelper(TimeSpan.FromSeconds(30)), false).Result;
Assert.AreEqual(100, response.LSN);
mockTransportClient = this.GetMockTransportClientForGlobalStrongWrites(addressInformation, i, false, true, true);
consistencyWriter = new ConsistencyWriter(addressSelector, sessionContainer, mockTransportClient, mockServiceConfigReader.Object, mockAuthorizationTokenProvider.Object, false);
response = consistencyWriter.WriteAsync(entity, new TimeoutHelper(TimeSpan.FromSeconds(30)), false).Result;
Assert.AreEqual(100, response.LSN);
mockTransportClient = this.GetMockTransportClientForGlobalStrongWrites(addressInformation, i, false, false, true);
consistencyWriter = new ConsistencyWriter(addressSelector, sessionContainer, mockTransportClient, mockServiceConfigReader.Object, mockAuthorizationTokenProvider.Object, false);
response = consistencyWriter.WriteAsync(entity, new TimeoutHelper(TimeSpan.FromSeconds(30)), false).Result;
Assert.AreEqual(100, response.LSN);
}
}
public Task <DocumentServiceResponse> ReadFeedAsync(DocumentServiceRequest request, IDocumentClientRetryPolicy retryPolicyInstance, CancellationToken cancellationToken)
{
return(this.innerClient.ReadFeedAsync(request, retryPolicyInstance, cancellationToken));
}
/// <summary>
/// Resolves the endpoint of the partition for the given request
/// </summary>
/// <param name="request">Request for which the partition endpoint resolution is to be performed</param>
/// <param name="forceRefreshPartitionAddresses">Force refresh the partition's endpoint</param>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>An instance of <see cref="ResolutionResult"/>.</returns>
private async Task <ResolutionResult> ResolveAddressesAndIdentityAsync(
DocumentServiceRequest request,
bool forceRefreshPartitionAddresses,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
if (request.ServiceIdentity != null)
{
if (request.ServiceIdentity.IsMasterService &&
request.ForceMasterRefresh &&
this.masterServiceIdentityProvider != null)
{
await this.masterServiceIdentityProvider.RefreshAsync(request.ServiceIdentity, cancellationToken);
ServiceIdentity newMasterServiceIdentity = this.masterServiceIdentityProvider.MasterServiceIdentity;
bool masterServiceIdentityChanged = newMasterServiceIdentity != null &&
!newMasterServiceIdentity.Equals(request.ServiceIdentity);
DefaultTrace.TraceInformation(
"Refreshed master service identity. masterServiceIdentityChanged = {0}, " +
"previousRequestServiceIdentity = {1}, newMasterServiceIdentity = {2}",
masterServiceIdentityChanged,
request.ServiceIdentity,
newMasterServiceIdentity);
if (masterServiceIdentityChanged)
{
request.RouteTo(newMasterServiceIdentity);
}
}
// In this case we don't populate request.RequestContext.ResolvedPartitionKeyRangeId,
// which is needed for session token.
// The assumption is that:
// 1. Master requests never use session consistency.
// 2. Service requests (like collection create etc.) don't use session consistency.
// 3. Requests which target specific partition of an existing collection will use x-ms-documentdb-partitionkeyrangeid header
// to send request to specific partition and will not set request.ServiceIdentity
ServiceIdentity identity = request.ServiceIdentity;
PartitionAddressInformation addresses = await this.addressCache.TryGetAddressesAsync(request, null, identity, forceRefreshPartitionAddresses, cancellationToken);
if (addresses == null && identity.IsMasterService && this.masterServiceIdentityProvider != null)
{
DefaultTrace.TraceWarning("Could not get addresses for MasterServiceIdentity {0}. will refresh masterServiceIdentity and retry", identity);
await this.masterServiceIdentityProvider.RefreshAsync(identity, cancellationToken);
identity = this.masterServiceIdentityProvider.MasterServiceIdentity;
addresses = await this.addressCache.TryGetAddressesAsync(request, null, identity, forceRefreshPartitionAddresses, cancellationToken);
}
if (addresses == null)
{
DefaultTrace.TraceInformation("Could not get addresses for explicitly specified ServiceIdentity {0}", identity);
throw new NotFoundException()
{
ResourceAddress = request.ResourceAddress
};
}
return(new ResolutionResult(addresses, identity));
}
if (ReplicatedResourceClient.IsReadingFromMaster(request.ResourceType, request.OperationType) && request.PartitionKeyRangeIdentity == null)
{
DefaultTrace.TraceInformation("Resolving Master service address, forceMasterRefresh: {0}, currentMaster: {1}",
request.ForceMasterRefresh,
this.masterServiceIdentityProvider?.MasterServiceIdentity);
// Client implementation, GlobalAddressResolver passes in a null IMasterServiceIdentityProvider, because it doesn't actually use the serviceIdentity
// in the addressCache.TryGetAddresses method. In GatewayAddressCache.cs, the master address is resolved by making a call to Gateway AddressFeed,
// not using the serviceIdentity that is passed in
if (request.ForceMasterRefresh && this.masterServiceIdentityProvider != null)
{
ServiceIdentity previousMasterService = this.masterServiceIdentityProvider.MasterServiceIdentity;
await this.masterServiceIdentityProvider.RefreshAsync(previousMasterService, cancellationToken);
}
ServiceIdentity serviceIdentity = this.masterServiceIdentityProvider?.MasterServiceIdentity;
PartitionKeyRangeIdentity partitionKeyRangeIdentity = this.masterPartitionKeyRangeIdentity;
PartitionAddressInformation addresses = await this.addressCache.TryGetAddressesAsync(
request,
partitionKeyRangeIdentity,
serviceIdentity,
forceRefreshPartitionAddresses,
cancellationToken);
if (addresses == null)
{
// This shouldn't really happen.
DefaultTrace.TraceCritical("Could not get addresses for master partition {0}", serviceIdentity);
throw new NotFoundException()
{
ResourceAddress = request.ResourceAddress
};
}
PartitionKeyRange partitionKeyRange = new PartitionKeyRange {
Id = PartitionKeyRange.MasterPartitionKeyRangeId
};
return(new ResolutionResult(partitionKeyRange, addresses, serviceIdentity));
}
bool collectionCacheIsUptoDate = !request.IsNameBased ||
(request.PartitionKeyRangeIdentity != null && request.PartitionKeyRangeIdentity.CollectionRid != null);
bool collectionRoutingMapCacheIsUptoDate = false;
ContainerProperties collection = await this.collectionCache.ResolveCollectionAsync(request, cancellationToken);
CollectionRoutingMap routingMap = await this.collectionRoutingMapCache.TryLookupAsync(
collection.ResourceId, null, request, cancellationToken);
if (routingMap != null && request.ForceCollectionRoutingMapRefresh)
{
DefaultTrace.TraceInformation(
"AddressResolver.ResolveAddressesAndIdentityAsync ForceCollectionRoutingMapRefresh collection.ResourceId = {0}",
collection.ResourceId);
routingMap = await this.collectionRoutingMapCache.TryLookupAsync(collection.ResourceId, routingMap, request, cancellationToken);
}
if (request.ForcePartitionKeyRangeRefresh)
{
collectionRoutingMapCacheIsUptoDate = true;
request.ForcePartitionKeyRangeRefresh = false;
if (routingMap != null)
{
routingMap = await this.collectionRoutingMapCache.TryLookupAsync(collection.ResourceId, routingMap, request, cancellationToken);
}
}
if (routingMap == null && !collectionCacheIsUptoDate)
{
// Routing map was not found by resolved collection rid. Maybe collection rid is outdated.
// Refresh collection cache and reresolve routing map.
request.ForceNameCacheRefresh = true;
collectionCacheIsUptoDate = true;
collectionRoutingMapCacheIsUptoDate = false;
collection = await this.collectionCache.ResolveCollectionAsync(request, cancellationToken);
routingMap = await this.collectionRoutingMapCache.TryLookupAsync(
collection.ResourceId,
previousValue : null,
request : request,
cancellationToken : cancellationToken);
}
AddressResolver.EnsureRoutingMapPresent(request, routingMap, collection);
// At this point we have both collection and routingMap.
ResolutionResult result = await this.TryResolveServerPartitionAsync(
request,
collection,
routingMap,
collectionCacheIsUptoDate,
collectionRoutingMapCacheIsUptodate : collectionRoutingMapCacheIsUptoDate,
forceRefreshPartitionAddresses : forceRefreshPartitionAddresses,
cancellationToken : cancellationToken);
if (result == null)
{
// Couldn't resolve server partition or its addresses.
// Either collection cache is outdated or routing map cache is outdated.
if (!collectionCacheIsUptoDate)
{
request.ForceNameCacheRefresh = true;
collectionCacheIsUptoDate = true;
collection = await this.collectionCache.ResolveCollectionAsync(request, cancellationToken);
if (collection.ResourceId != routingMap.CollectionUniqueId)
{
// Collection cache was stale. We resolved to new Rid. routing map cache is potentially stale
// for this new collection rid. Mark it as such.
collectionRoutingMapCacheIsUptoDate = false;
routingMap = await this.collectionRoutingMapCache.TryLookupAsync(
collection.ResourceId,
previousValue : null,
request : request,
cancellationToken : cancellationToken);
}
}
if (!collectionRoutingMapCacheIsUptoDate)
{
collectionRoutingMapCacheIsUptoDate = true;
routingMap = await this.collectionRoutingMapCache.TryLookupAsync(
collection.ResourceId,
previousValue : routingMap,
request : request,
cancellationToken : cancellationToken);
}
AddressResolver.EnsureRoutingMapPresent(request, routingMap, collection);
result = await this.TryResolveServerPartitionAsync(
request,
collection,
routingMap,
collectionCacheIsUptodate : true,
collectionRoutingMapCacheIsUptodate : true,
forceRefreshPartitionAddresses : forceRefreshPartitionAddresses,
cancellationToken : cancellationToken);
}
if (result == null)
{
DefaultTrace.TraceInformation("Couldn't route partitionkeyrange-oblivious request after retry/cache refresh. Collection doesn't exist.");
// At this point collection cache and routing map caches are refreshed.
// The only reason we will get here is if collection doesn't exist.
// Case when partitionkeyrange doesn't exist is handled in the corresponding method.
throw new NotFoundException()
{
ResourceAddress = request.ResourceAddress
};
}
if (request.IsNameBased)
{
// Append collection rid.
// If we resolved collection rid incorrectly because of outdated cache, this can lead
// to incorrect routing decisions. But backend will validate collection rid and throw
// InvalidPartitionException if we reach wrong collection.
// Also this header will be used by backend to inject collection rid into metrics for
// throttled requests.
request.Headers[WFConstants.BackendHeaders.CollectionRid] = collection.ResourceId;
}
return(result);
}
public void OnBeforeSendRequest(DocumentServiceRequest request)
{
this.nextRetryPolicy?.OnBeforeSendRequest(request);
}
private PartitionKeyRange TryResolveSinglePartitionCollection(
DocumentServiceRequest request,
ContainerProperties collection,
CollectionRoutingMap routingMap,
bool collectionCacheIsUptoDate)
{
// Neither partitionkey nor partitionkeyrangeid is specified.
// Three options here:
// * This is non-partitioned collection and old client SDK which doesn't send partition key. In
// this case there's single entry in routing map. But can be multiple entries if before that
// existed partitioned collection with same name.
// * This is partitioned collection and old client SDK which doesn't send partition key.
// In this case there can be multiple ranges in routing map.
// * This is partitioned collection and this is custom written REST sdk, which has a bug and doesn't send
// partition key.
// We cannot know for sure whether this is partitioned collection or not, because
// partition key definition cache can be outdated.
// So we route request to the first partition. If this is non-partitioned collection - request will succeed.
// If it is partitioned collection - backend will return bad request as partition key header is required in this case.
if (routingMap.OrderedPartitionKeyRanges.Count == 1)
{
return(routingMap.OrderedPartitionKeyRanges.Single());
}
if (collectionCacheIsUptoDate)
{
// If the current collection is user-partitioned collection
if (collection.PartitionKey.Paths.Count >= 1 &&
!collection.PartitionKey.IsSystemKey.GetValueOrDefault(false))
{
throw new BadRequestException(RMResources.MissingPartitionKeyValue)
{
ResourceAddress = request.ResourceAddress
};
}
else if (routingMap.OrderedPartitionKeyRanges.Count > 1)
{
// With migrated-fixed-collection, it is possible to have multiple partition key ranges
// due to parallel usage of V3 SDK and a possible storage or throughput split
// The current client might be legacy and not aware of this.
// In such case route the request to the first partition
return(this.TryResolveServerPartitionByPartitionKey(
request,
"[]", // This corresponds to first partition
collectionCacheIsUptoDate,
collection,
routingMap));
}
else
{
// routingMap.OrderedPartitionKeyRanges.Count == 0
// Should never come here.
DefaultTrace.TraceCritical(
"No Partition Key ranges present for the collection {0}", collection.ResourceId);
throw new InternalServerErrorException(RMResources.InternalServerError)
{
ResourceAddress = request.ResourceAddress
};
}
}
else
{
return(null);
}
}
public void OnBeforeSendRequest(DocumentServiceRequest request)
{
this.documentServiceRequest = request;
this.nextPolicy?.OnBeforeSendRequest(request);
}
/// <summary>
/// ReplicatedResourceClient will use this API to get the direct connectivity AddressCache for given request.
/// </summary>
/// <param name="request"></param>
private IAddressResolver GetAddressResolver(DocumentServiceRequest request)
{
Uri endpoint = this.endpointManager.ResolveServiceEndpoint(request);
return(this.GetOrAddEndpoint(endpoint).AddressResolver);
}
public bool CanUseMultipleWriteLocations(DocumentServiceRequest request)
{
return(this.CanUseMultipleWriteLocations() &&
(request.ResourceType == ResourceType.Document ||
(request.ResourceType == ResourceType.StoredProcedure && request.OperationType == Documents.OperationType.ExecuteJavaScript)));
}
private static bool LogicalPartitionKeyProvided(DocumentServiceRequest request)
{
return(!string.IsNullOrEmpty(request.Headers[HttpConstants.HttpHeaders.PartitionKey]) ||
!request.ResourceType.IsPartitioned());
}
public async Task <PartitionAddressInformation> TryGetAddresses(
DocumentServiceRequest request,
PartitionKeyRangeIdentity partitionKeyRangeIdentity,
ServiceIdentity serviceIdentity,
bool forceRefreshPartitionAddresses,
CancellationToken cancellationToken)
{
if (request == null)
{
throw new ArgumentNullException("request");
}
if (partitionKeyRangeIdentity == null)
{
throw new ArgumentNullException("partitionKeyRangeIdentity");
}
try
{
if (partitionKeyRangeIdentity.PartitionKeyRangeId == PartitionKeyRange.MasterPartitionKeyRangeId)
{
return((await this.ResolveMasterAsync(request, forceRefreshPartitionAddresses)).Item2);
}
DateTime suboptimalServerPartitionTimestamp;
if (this.suboptimalServerPartitionTimestamps.TryGetValue(partitionKeyRangeIdentity, out suboptimalServerPartitionTimestamp))
{
bool forceRefreshDueToSuboptimalPartitionReplicaSet =
DateTime.UtcNow.Subtract(suboptimalServerPartitionTimestamp) > TimeSpan.FromSeconds(this.suboptimalPartitionForceRefreshIntervalInSeconds);
if (forceRefreshDueToSuboptimalPartitionReplicaSet && this.suboptimalServerPartitionTimestamps.TryUpdate(partitionKeyRangeIdentity, DateTime.MaxValue, suboptimalServerPartitionTimestamp))
{
forceRefreshPartitionAddresses = true;
}
}
if (forceRefreshPartitionAddresses)
{
this.serverPartitionAddressCache.Refresh(
partitionKeyRangeIdentity,
async() => await this.GetAddressesForRangeId(
request,
partitionKeyRangeIdentity.CollectionRid,
partitionKeyRangeIdentity.PartitionKeyRangeId,
forceRefresh: true),
cancellationToken);
DateTime ignoreDateTime;
this.suboptimalServerPartitionTimestamps.TryRemove(partitionKeyRangeIdentity, out ignoreDateTime);
}
PartitionAddressInformation addresses = await this.serverPartitionAddressCache.GetAsync(
partitionKeyRangeIdentity,
null,
() => this.GetAddressesForRangeId(
request,
partitionKeyRangeIdentity.CollectionRid,
partitionKeyRangeIdentity.PartitionKeyRangeId,
forceRefresh : false),
cancellationToken);
int targetReplicaSetSize = this.serviceConfigReader.UserReplicationPolicy.MaxReplicaSetSize;
if (addresses.AllAddresses.Count() < targetReplicaSetSize)
{
this.suboptimalServerPartitionTimestamps.TryAdd(partitionKeyRangeIdentity, DateTime.UtcNow);
}
return(addresses);
}
catch (DocumentClientException ex)
{
if ((ex.StatusCode == HttpStatusCode.NotFound) ||
(ex.StatusCode == HttpStatusCode.Gone && ex.GetSubStatus() == SubStatusCodes.PartitionKeyRangeGone))
{
//remove from suboptimal cache in case the the collection+pKeyRangeId combo is gone.
DateTime ignoreDateTime;
this.suboptimalServerPartitionTimestamps.TryRemove(partitionKeyRangeIdentity, out ignoreDateTime);
return(null);
}
throw;
}
catch (Exception)
{
if (forceRefreshPartitionAddresses)
{
DateTime ignoreDateTime;
this.suboptimalServerPartitionTimestamps.TryRemove(partitionKeyRangeIdentity, out ignoreDateTime);
}
throw;
}
}
private async Task <DocumentServiceResponse> ProcessUpsertAsync(IStoreModel storeProxy, DocumentServiceRequest serviceRequest, CancellationToken cancellationToken)
{
DocumentServiceResponse response = await storeProxy.ProcessMessageAsync(serviceRequest, cancellationToken);
this.client.DocumentClient.CaptureSessionToken(serviceRequest, response);
return(response);
}
public Task ReauthorizeSystemKeySignedRequestAsync(DocumentServiceRequest request, CancellationToken cancellationToken)
{
return(Task.FromResult(0));
}
