public async Task DoesNotRecalculatePartitionKeyRangeOnNoSplits() { ItemBatchOperation itemBatchOperation = CreateItem("test"); Mock <CosmosClientContext> mockedContext = new Mock <CosmosClientContext>(); mockedContext.Setup(c => c.ClientOptions).Returns(new CosmosClientOptions()); mockedContext .Setup(c => c.ProcessResourceOperationStreamAsync( It.IsAny <string>(), It.IsAny <ResourceType>(), It.IsAny <OperationType>(), It.IsAny <RequestOptions>(), It.IsAny <ContainerInternal>(), It.IsAny <Cosmos.PartitionKey?>(), It.IsAny <Stream>(), It.IsAny <Action <RequestMessage> >(), It.IsAny <CosmosDiagnosticsContext>(), It.IsAny <CancellationToken>())) .Returns(this.GenerateOkResponseAsync(itemBatchOperation)); mockedContext.Setup(c => c.SerializerCore).Returns(MockCosmosUtil.Serializer); string link = "/dbs/db/colls/colls"; Mock <ContainerInternal> mockContainer = new Mock <ContainerInternal>(); mockContainer.Setup(x => x.LinkUri).Returns(link); mockContainer.Setup(x => x.GetPartitionKeyDefinitionAsync(It.IsAny <CancellationToken>())).Returns(Task.FromResult(new PartitionKeyDefinition() { Paths = new Collection <string>() { "/id" } })); CollectionRoutingMap routingMap = CollectionRoutingMap.TryCreateCompleteRoutingMap( new[] { Tuple.Create(new PartitionKeyRange { Id = "0", MinInclusive = "", MaxExclusive = "FF" }, (ServiceIdentity)null) }, string.Empty); mockContainer.Setup(x => x.GetRoutingMapAsync(It.IsAny <CancellationToken>())).Returns(Task.FromResult(routingMap)); BatchAsyncContainerExecutor executor = new BatchAsyncContainerExecutor(mockContainer.Object, mockedContext.Object, 20, BatchAsyncContainerExecutorCache.DefaultMaxBulkRequestBodySizeInBytes); TransactionalBatchOperationResult result = await executor.AddAsync(itemBatchOperation); Mock.Get(mockContainer.Object) .Verify(x => x.GetPartitionKeyDefinitionAsync(It.IsAny <CancellationToken>()), Times.Once); Mock.Get(mockedContext.Object) .Verify(c => c.ProcessResourceOperationStreamAsync( It.IsAny <string>(), It.IsAny <ResourceType>(), It.IsAny <OperationType>(), It.IsAny <RequestOptions>(), It.IsAny <ContainerInternal>(), It.IsAny <Cosmos.PartitionKey?>(), It.IsAny <Stream>(), It.IsAny <Action <RequestMessage> >(), It.IsAny <CosmosDiagnosticsContext>(), It.IsAny <CancellationToken>()), Times.Once); Assert.AreEqual(HttpStatusCode.OK, result.StatusCode); }
public void TestCollectionRoutingMap() { ServiceIdentity serviceIdentity0 = new ServiceIdentity("1", new Uri("http://1"), false); ServiceIdentity serviceIdentity1 = new ServiceIdentity("2", new Uri("http://2"), false); ServiceIdentity serviceIdentity2 = new ServiceIdentity("3", new Uri("http://3"), false); ServiceIdentity serviceIdentity3 = new ServiceIdentity("4", new Uri("http://4"), false); CollectionRoutingMap routingMap = CollectionRoutingMap.TryCreateCompleteRoutingMap( new[] { Tuple.Create( new PartitionKeyRange { Id = "2", MinInclusive = "0000000050", MaxExclusive = "0000000070" }, serviceIdentity2), Tuple.Create( new PartitionKeyRange { Id = "0", MinInclusive = "", MaxExclusive = "0000000030" }, serviceIdentity0), Tuple.Create( new PartitionKeyRange { Id = "1", MinInclusive = "0000000030", MaxExclusive = "0000000050" }, serviceIdentity1), Tuple.Create( new PartitionKeyRange { Id = "3", MinInclusive = "0000000070", MaxExclusive = "FF" }, serviceIdentity3), }, string.Empty); Assert.AreEqual("0", routingMap.OrderedPartitionKeyRanges[0].Id); Assert.AreEqual("1", routingMap.OrderedPartitionKeyRanges[1].Id); Assert.AreEqual("2", routingMap.OrderedPartitionKeyRanges[2].Id); Assert.AreEqual("3", routingMap.OrderedPartitionKeyRanges[3].Id); Assert.AreEqual(serviceIdentity0, routingMap.TryGetInfoByPartitionKeyRangeId("0")); Assert.AreEqual(serviceIdentity1, routingMap.TryGetInfoByPartitionKeyRangeId("1")); Assert.AreEqual(serviceIdentity2, routingMap.TryGetInfoByPartitionKeyRangeId("2")); Assert.AreEqual(serviceIdentity3, routingMap.TryGetInfoByPartitionKeyRangeId("3")); Assert.AreEqual("0", routingMap.GetRangeByEffectivePartitionKey("").Id); Assert.AreEqual("0", routingMap.GetRangeByEffectivePartitionKey("0000000000").Id); Assert.AreEqual("1", routingMap.GetRangeByEffectivePartitionKey("0000000030").Id); Assert.AreEqual("1", routingMap.GetRangeByEffectivePartitionKey("0000000031").Id); Assert.AreEqual("3", routingMap.GetRangeByEffectivePartitionKey("0000000071").Id); Assert.AreEqual("0", routingMap.TryGetRangeByPartitionKeyRangeId("0").Id); Assert.AreEqual("1", routingMap.TryGetRangeByPartitionKeyRangeId("1").Id); Assert.AreEqual(4, routingMap.GetOverlappingRanges(new[] { new Range <string>(PartitionKeyInternal.MinimumInclusiveEffectivePartitionKey, PartitionKeyInternal.MaximumExclusiveEffectivePartitionKey, true, false) }).Count); Assert.AreEqual(0, routingMap.GetOverlappingRanges(new[] { new Range <string>(PartitionKeyInternal.MinimumInclusiveEffectivePartitionKey, PartitionKeyInternal.MinimumInclusiveEffectivePartitionKey, false, false) }).Count); IReadOnlyList <PartitionKeyRange> partitionKeyRanges = routingMap.GetOverlappingRanges(new[] { new Range <string>( "0000000040", "0000000040", true, true) }); Assert.AreEqual(1, partitionKeyRanges.Count); Assert.AreEqual("1", partitionKeyRanges.ElementAt(0).Id); IReadOnlyList <PartitionKeyRange> partitionKeyRanges1 = routingMap.GetOverlappingRanges(new[] { new Range <string>( "0000000040", "0000000045", true, true), new Range <string>( "0000000045", "0000000046", true, true), new Range <string>( "0000000046", "0000000050", true, true) }); Assert.AreEqual(2, partitionKeyRanges1.Count); Assert.AreEqual("1", partitionKeyRanges1.ElementAt(0).Id); Assert.AreEqual("2", partitionKeyRanges1.ElementAt(1).Id); }
public void TestTryCombineRanges() { CollectionRoutingMap routingMap = CollectionRoutingMap.TryCreateCompleteRoutingMap( new[] { Tuple.Create( new PartitionKeyRange { Id = "2", MinInclusive = "0000000050", MaxExclusive = "0000000070" }, (ServiceIdentity)null), Tuple.Create( new PartitionKeyRange { Id = "0", MinInclusive = "", MaxExclusive = "0000000030" }, (ServiceIdentity)null), Tuple.Create( new PartitionKeyRange { Id = "1", MinInclusive = "0000000030", MaxExclusive = "0000000050" }, (ServiceIdentity)null), Tuple.Create( new PartitionKeyRange { Id = "3", MinInclusive = "0000000070", MaxExclusive = "FF" }, (ServiceIdentity)null), }, string.Empty); CollectionRoutingMap newRoutingMap = routingMap.TryCombine( new[] { Tuple.Create( new PartitionKeyRange { Id = "4", Parents = new Collection <string> { "0" }, MinInclusive = "", MaxExclusive = "0000000010" }, (ServiceIdentity)null), Tuple.Create( new PartitionKeyRange { Id = "5", Parents = new Collection <string> { "0" }, MinInclusive = "0000000010", MaxExclusive = "0000000030" }, (ServiceIdentity)null), }, null); Assert.IsNotNull(newRoutingMap); newRoutingMap = routingMap.TryCombine( new[] { Tuple.Create( new PartitionKeyRange { Id = "6", Parents = new Collection <string> { "0", "4" }, MinInclusive = "", MaxExclusive = "0000000005" }, (ServiceIdentity)null), Tuple.Create( new PartitionKeyRange { Id = "7", Parents = new Collection <string> { "0", "4" }, MinInclusive = "0000000005", MaxExclusive = "0000000010" }, (ServiceIdentity)null), Tuple.Create( new PartitionKeyRange { Id = "8", Parents = new Collection <string> { "0", "5" }, MinInclusive = "0000000010", MaxExclusive = "0000000015" }, (ServiceIdentity)null), Tuple.Create( new PartitionKeyRange { Id = "9", Parents = new Collection <string> { "0", "5" }, MinInclusive = "0000000015", MaxExclusive = "0000000030" }, (ServiceIdentity)null), }, null); Assert.IsNotNull(newRoutingMap); newRoutingMap = routingMap.TryCombine( new[] { Tuple.Create( new PartitionKeyRange { Id = "10", Parents = new Collection <string> { "0", "4", "6" }, MinInclusive = "", MaxExclusive = "0000000002" }, (ServiceIdentity)null), }, null); Assert.IsNull(newRoutingMap); }
public RoutingMapProvider(CollectionRoutingMap collectionRoutingMap) { this.collectionRoutingMap = collectionRoutingMap; }
public async Task RetryOnNameStale() { ItemBatchOperation itemBatchOperation = CreateItem("test"); Mock <CosmosClientContext> mockedContext = new Mock <CosmosClientContext>(); mockedContext.Setup(c => c.ClientOptions).Returns(new CosmosClientOptions()); mockedContext .SetupSequence(c => c.ProcessResourceOperationStreamAsync( It.IsAny <Uri>(), It.IsAny <ResourceType>(), It.IsAny <OperationType>(), It.IsAny <RequestOptions>(), It.IsAny <ContainerInternal>(), It.IsAny <Cosmos.PartitionKey?>(), It.IsAny <Stream>(), It.IsAny <Action <RequestMessage> >(), It.IsAny <CosmosDiagnosticsContext>(), It.IsAny <CancellationToken>())) .Returns(this.GenerateCacheStaleResponseAsync(itemBatchOperation)) .Returns(this.GenerateOkResponseAsync(itemBatchOperation)); mockedContext.Setup(c => c.SerializerCore).Returns(MockCosmosUtil.Serializer); Uri link = new Uri($"/dbs/db/colls/colls", UriKind.Relative); Mock <ContainerInternal> mockContainer = new Mock <ContainerInternal>(); mockContainer.Setup(x => x.LinkUri).Returns(link); mockContainer.Setup(x => x.GetPartitionKeyDefinitionAsync(It.IsAny <CancellationToken>())).Returns(Task.FromResult(new PartitionKeyDefinition() { Paths = new Collection <string>() { "/id" } })); CollectionRoutingMap routingMap = CollectionRoutingMap.TryCreateCompleteRoutingMap( new[] { Tuple.Create(new PartitionKeyRange { Id = "0", MinInclusive = "", MaxExclusive = "FF" }, (ServiceIdentity)null) }, string.Empty); mockContainer.Setup(x => x.GetRoutingMapAsync(It.IsAny <CancellationToken>())).Returns(Task.FromResult(routingMap)); BatchAsyncContainerExecutor executor = new BatchAsyncContainerExecutor(mockContainer.Object, mockedContext.Object, 20, Constants.MaxDirectModeBatchRequestBodySizeInBytes, 1); TransactionalBatchOperationResult result = await executor.AddAsync(itemBatchOperation); Mock.Get(mockContainer.Object) .Verify(x => x.GetPartitionKeyDefinitionAsync(It.IsAny <CancellationToken>()), Times.Exactly(2)); Mock.Get(mockedContext.Object) .Verify(c => c.ProcessResourceOperationStreamAsync( It.IsAny <Uri>(), It.IsAny <ResourceType>(), It.IsAny <OperationType>(), It.IsAny <RequestOptions>(), It.IsAny <ContainerInternal>(), It.IsAny <Cosmos.PartitionKey?>(), It.IsAny <Stream>(), It.IsAny <Action <RequestMessage> >(), It.IsAny <CosmosDiagnosticsContext>(), It.IsAny <CancellationToken>()), Times.Exactly(2)); Assert.AreEqual(HttpStatusCode.OK, result.StatusCode); Assert.IsNotNull(result.DiagnosticsContext); string diagnosticsString = result.DiagnosticsContext.ToString(); Assert.IsTrue(diagnosticsString.Contains("PointOperationStatistics"), "Diagnostics might be missing"); }
public static string GetPartitionKeyRangeId(PartitionKeyInternal partitionKeyInternal, PartitionKeyDefinition partitionKeyDefinition, CollectionRoutingMap collectionRoutingMap) { string effectivePartitionKey = partitionKeyInternal.GetEffectivePartitionKeyString(partitionKeyDefinition); return(collectionRoutingMap.GetRangeByEffectivePartitionKey(effectivePartitionKey).Id); }
private void Init() { this.collectionCache = new Mock <ClientCollectionCache>(null, new ServerStoreModel(null), null, null); ContainerProperties containerProperties = ContainerProperties.CreateWithResourceId("test"); containerProperties.PartitionKey = partitionKeyDefinition; this.collectionCache.Setup (m => m.ResolveCollectionAsync( It.IsAny <DocumentServiceRequest>(), It.IsAny <CancellationToken>(), It.IsAny <ITrace>() ) ).Returns(Task.FromResult(containerProperties)); this.collectionCache.Setup(x => x.ResolveByNameAsync( It.IsAny <string>(), It.IsAny <string>(), It.IsAny <bool>(), It.IsAny <ITrace>(), It.IsAny <IClientSideRequestStatistics>(), It.IsAny <CancellationToken>())).Returns(Task.FromResult(containerProperties)); CollectionRoutingMap routingMap = CollectionRoutingMap.TryCreateCompleteRoutingMap( new[] { Tuple.Create(new PartitionKeyRange { Id = "0", MinInclusive = "", MaxExclusive = "FF" }, (ServiceIdentity)null) }, string.Empty); this.partitionKeyRangeCache = new Mock <PartitionKeyRangeCache>(null, null, null); this.partitionKeyRangeCache.Setup( m => m.TryLookupAsync( It.IsAny <string>(), It.IsAny <CollectionRoutingMap>(), It.IsAny <DocumentServiceRequest>(), It.IsAny <CancellationToken>(), It.IsAny <ITrace>() ) ).Returns(Task.FromResult <CollectionRoutingMap>(routingMap)); List <PartitionKeyRange> result = new List <PartitionKeyRange> { new PartitionKeyRange() { MinInclusive = Documents.Routing.PartitionKeyInternal.MinimumInclusiveEffectivePartitionKey, MaxExclusive = Documents.Routing.PartitionKeyInternal.MaximumExclusiveEffectivePartitionKey, Id = "0" } }; this.partitionKeyRangeCache .Setup(m => m.TryGetOverlappingRangesAsync( It.IsAny <string>(), It.IsAny <Documents.Routing.Range <string> >(), It.IsAny <ITrace>(), It.IsAny <bool>())) .Returns(Task.FromResult((IReadOnlyList <PartitionKeyRange>)result)); this.globalEndpointManager = new Mock <GlobalEndpointManager>(this, new ConnectionPolicy()); this.InitStoreModels(); }
private async Task <ResolutionResult> TryResolveServerPartitionAsync( DocumentServiceRequest request, CosmosContainerSettings collection, CollectionRoutingMap routingMap, bool collectionCacheIsUptodate, bool collectionRoutingMapCacheIsUptodate, bool forceRefreshPartitionAddresses, CancellationToken cancellationToken) { // Check if this request partitionkeyrange-aware routing logic. We cannot retry here in this case // and need to bubble up errors. if (request.PartitionKeyRangeIdentity != null) { return(await this.TryResolveServerPartitionByPartitionKeyRangeIdAsync( request, collection, routingMap, collectionCacheIsUptodate, collectionRoutingMapCacheIsUptodate, forceRefreshPartitionAddresses, cancellationToken)); } if (!request.ResourceType.IsPartitioned() && !(request.ResourceType == ResourceType.StoredProcedure && request.OperationType == OperationType.ExecuteJavaScript) && // Collection head is sent internally for strong consistency given routing hints from original requst, which is for partitioned resource. !(request.ResourceType == ResourceType.Collection && request.OperationType == OperationType.Head)) { DefaultTrace.TraceCritical( "Shouldn't come here for non partitioned resources. resourceType : {0}, operationtype:{1}, resourceaddress:{2}", request.ResourceType, request.OperationType, request.ResourceAddress); throw new InternalServerErrorException(RMResources.InternalServerError) { ResourceAddress = request.ResourceAddress }; } PartitionKeyRange range; string partitionKeyString = request.Headers[HttpConstants.HttpHeaders.PartitionKey]; object effectivePartitionKeyStringObject = null; if (partitionKeyString != null) { range = this.TryResolveServerPartitionByPartitionKey( request, partitionKeyString, collectionCacheIsUptodate, collection, routingMap); } else if (request.Properties != null && request.Properties.TryGetValue( WFConstants.BackendHeaders.EffectivePartitionKeyString, out effectivePartitionKeyStringObject)) { // Allow EPK only for partitioned collection (excluding migrated fixed collections) if (!collection.HasPartitionKey || collection.PartitionKey.IsSystemKey) { throw new ArgumentOutOfRangeException(nameof(collection)); } string effectivePartitionKeyString = effectivePartitionKeyStringObject as string; if (string.IsNullOrEmpty(effectivePartitionKeyString)) { throw new ArgumentOutOfRangeException(nameof(effectivePartitionKeyString)); } range = routingMap.GetRangeByEffectivePartitionKey(effectivePartitionKeyString); } else { range = this.TryResolveSinglePartitionCollection(request, routingMap, collectionCacheIsUptodate); } if (range == null) { // Collection cache or routing map cache is potentially outdated. Return null - // upper logic will refresh cache and retry. return(null); } ServiceIdentity serviceIdentity = routingMap.TryGetInfoByPartitionKeyRangeId(range.Id); PartitionAddressInformation addresses = await this.addressCache.TryGetAddressesAsync( request, new PartitionKeyRangeIdentity(collection.ResourceId, range.Id), serviceIdentity, forceRefreshPartitionAddresses, cancellationToken); if (addresses == null) { DefaultTrace.TraceVerbose( "Could not resolve addresses for identity {0}/{1}. Potentially collection cache or routing map cache is outdated. Return null - upper logic will refresh and retry. ", new PartitionKeyRangeIdentity(collection.ResourceId, range.Id), serviceIdentity); return(null); } return(new ResolutionResult(range, addresses, serviceIdentity)); }
private PartitionKeyRange TryResolveServerPartitionByPartitionKey( DocumentServiceRequest request, string partitionKeyString, bool collectionCacheUptoDate, CosmosContainerSettings collection, CollectionRoutingMap routingMap) { if (request == null) { throw new ArgumentNullException("request"); } if (partitionKeyString == null) { throw new ArgumentNullException("partitionKeyString"); } if (collection == null) { throw new ArgumentNullException("collection"); } if (routingMap == null) { throw new ArgumentNullException("routingMap"); } PartitionKeyInternal partitionKey; try { partitionKey = PartitionKeyInternal.FromJsonString(partitionKeyString); } catch (JsonException ex) { throw new BadRequestException( string.Format(CultureInfo.InvariantCulture, RMResources.InvalidPartitionKey, partitionKeyString), ex) { ResourceAddress = request.ResourceAddress }; } if (partitionKey == null) { throw new InternalServerErrorException(string.Format(CultureInfo.InvariantCulture, "partition key is null '{0}'", partitionKeyString)); } if (partitionKey.Equals(PartitionKeyInternal.Empty) || partitionKey.Components.Count == collection.PartitionKey.Paths.Count) { // Although we can compute effective partition key here, in general case this Gateway can have outdated // partition key definition cached - like if collection with same name but with Range partitioning is created. // In this case server will not pass x-ms-documentdb-collection-rid check and will return back InvalidPartitionException. // Gateway will refresh its cache and retry. string effectivePartitionKey = partitionKey.GetEffectivePartitionKeyString(collection.PartitionKey); // There should be exactly one range which contains a partition key. Always. return(routingMap.GetRangeByEffectivePartitionKey(effectivePartitionKey)); } if (collectionCacheUptoDate) { BadRequestException badRequestException = new BadRequestException(RMResources.PartitionKeyMismatch) { ResourceAddress = request.ResourceAddress }; badRequestException.Headers[WFConstants.BackendHeaders.SubStatus] = ((uint)SubStatusCodes.PartitionKeyMismatch).ToString(CultureInfo.InvariantCulture); throw badRequestException; } // Partition key supplied has different number paths than locally cached partition key definition. // Three things can happen: // 1. User supplied wrong partition key. // 2. Client SDK has outdated partition key definition cache and extracted wrong value from the document. // 3. Gateway's cache is outdated. // // What we will do is append x-ms-documentdb-collection-rid header and forward it to random collection partition. // * If collection rid matches, server will send back 400.1001, because it also will not be able to compute // effective partition key. Gateway will forward this status code to client - client will handle it. // * If collection rid doesn't match, server will send back InvalidPartiitonException and Gateway will // refresh name routing cache - this will refresh partition key definition as well, and retry. DefaultTrace.TraceInformation( "Cannot compute effective partition key. Definition has '{0}' paths, values supplied has '{1}' paths. Will refresh cache and retry.", collection.PartitionKey.Paths.Count, partitionKey.Components.Count); return(null); }
/// <summary> /// Resolves the endpoint of the partition for the given request /// </summary> /// <param name="request">Request for which the partition endpoint resolution is to be performed</param> /// <param name="forceRefreshPartitionAddresses">Force refresh the partition's endpoint</param> /// <param name="cancellationToken">Cancellation token</param> /// <returns>An instance of <see cref="ResolutionResult"/>.</returns> private async Task <ResolutionResult> ResolveAddressesAndIdentityAsync( DocumentServiceRequest request, bool forceRefreshPartitionAddresses, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); if (request.ServiceIdentity != null) { // In this case we don't populate request.RequestContext.ResolvedPartitionKeyRangeId, // which is needed for session token. // The assumption is that: // 1. Master requests never use session consistency. // 2. Service requests (like collection create etc.) don't use session consistency. // 3. Requests which target specific partition of an existing collection will use x-ms-documentdb-partitionkeyrangeid header // to send request to specific partition and will not set request.ServiceIdentity ServiceIdentity identity = request.ServiceIdentity; PartitionAddressInformation addresses = await this.addressCache.TryGetAddressesAsync(request, null, identity, forceRefreshPartitionAddresses, cancellationToken); if (addresses == null) { DefaultTrace.TraceInformation("Could not get addresses for explicitly specified ServiceIdentity {0}", identity); throw new NotFoundException() { ResourceAddress = request.ResourceAddress }; } return(new ResolutionResult(addresses, identity)); } if (ReplicatedResourceClient.IsReadingFromMaster(request.ResourceType, request.OperationType) && request.PartitionKeyRangeIdentity == null) { DefaultTrace.TraceInformation("Resolving Master service address, forceMasterRefresh: {0}, currentMaster: {1}", request.ForceMasterRefresh, this.masterServiceIdentityProvider?.MasterServiceIdentity); // Client implementation, GlobalAddressResolver passes in a null IMasterServiceIdentityProvider, because it doesn't actually use the serviceIdentity // in the addressCache.TryGetAddresses method. In GatewayAddressCache.cs, the master address is resolved by making a call to Gateway AddressFeed, // not using the serviceIdentity that is passed in if (request.ForceMasterRefresh && this.masterServiceIdentityProvider != null) { ServiceIdentity previousMasterService = this.masterServiceIdentityProvider.MasterServiceIdentity; await this.masterServiceIdentityProvider.RefreshAsync(previousMasterService, cancellationToken); } ServiceIdentity serviceIdentity = this.masterServiceIdentityProvider?.MasterServiceIdentity; PartitionKeyRangeIdentity partitionKeyRangeIdentity = this.masterPartitionKeyRangeIdentity; PartitionAddressInformation addresses = await this.addressCache.TryGetAddressesAsync( request, partitionKeyRangeIdentity, serviceIdentity, forceRefreshPartitionAddresses, cancellationToken); if (addresses == null) { // This shouldn't really happen. DefaultTrace.TraceCritical("Could not get addresses for master partition {0}", serviceIdentity); throw new NotFoundException() { ResourceAddress = request.ResourceAddress }; } PartitionKeyRange partitionKeyRange = new PartitionKeyRange { Id = PartitionKeyRange.MasterPartitionKeyRangeId }; return(new ResolutionResult(partitionKeyRange, addresses, serviceIdentity)); } bool collectionCacheIsUptoDate = !request.IsNameBased || (request.PartitionKeyRangeIdentity != null && request.PartitionKeyRangeIdentity.CollectionRid != null); bool collectionRoutingMapCacheIsUptoDate = false; CosmosContainerSettings collection = await this.collectionCache.ResolveCollectionAsync(request, cancellationToken); CollectionRoutingMap routingMap = await this.collectionRoutingMapCache.TryLookupAsync( collection.ResourceId, null, request, cancellationToken); if (routingMap != null && request.ForceCollectionRoutingMapRefresh) { DefaultTrace.TraceInformation( "AddressResolver.ResolveAddressesAndIdentityAsync ForceCollectionRoutingMapRefresh collection.ResourceId = {0}", collection.ResourceId); routingMap = await this.collectionRoutingMapCache.TryLookupAsync(collection.ResourceId, routingMap, request, cancellationToken); } if (request.ForcePartitionKeyRangeRefresh) { collectionRoutingMapCacheIsUptoDate = true; request.ForcePartitionKeyRangeRefresh = false; if (routingMap != null) { routingMap = await this.collectionRoutingMapCache.TryLookupAsync(collection.ResourceId, routingMap, request, cancellationToken); } } if (routingMap == null && !collectionCacheIsUptoDate) { // Routing map was not found by resolved collection rid. Maybe collection rid is outdated. // Refresh collection cache and reresolve routing map. request.ForceNameCacheRefresh = true; collectionCacheIsUptoDate = true; collectionRoutingMapCacheIsUptoDate = false; collection = await this.collectionCache.ResolveCollectionAsync(request, cancellationToken); routingMap = await this.collectionRoutingMapCache.TryLookupAsync( collection.ResourceId, previousValue : null, request : request, cancellationToken : cancellationToken); } AddressResolver.EnsureRoutingMapPresent(request, routingMap, collection); // At this point we have both collection and routingMap. ResolutionResult result = await this.TryResolveServerPartitionAsync( request, collection, routingMap, collectionCacheIsUptoDate, collectionRoutingMapCacheIsUptodate : collectionRoutingMapCacheIsUptoDate, forceRefreshPartitionAddresses : forceRefreshPartitionAddresses, cancellationToken : cancellationToken); if (result == null) { // Couldn't resolve server partition or its addresses. // Either collection cache is outdated or routing map cache is outdated. if (!collectionCacheIsUptoDate) { request.ForceNameCacheRefresh = true; collectionCacheIsUptoDate = true; collection = await this.collectionCache.ResolveCollectionAsync(request, cancellationToken); if (collection.ResourceId != routingMap.CollectionUniqueId) { // Collection cache was stale. We resolved to new Rid. routing map cache is potentially stale // for this new collection rid. Mark it as such. collectionRoutingMapCacheIsUptoDate = false; routingMap = await this.collectionRoutingMapCache.TryLookupAsync( collection.ResourceId, previousValue : null, request : request, cancellationToken : cancellationToken); } } if (!collectionRoutingMapCacheIsUptoDate) { collectionRoutingMapCacheIsUptoDate = true; routingMap = await this.collectionRoutingMapCache.TryLookupAsync( collection.ResourceId, previousValue : routingMap, request : request, cancellationToken : cancellationToken); } AddressResolver.EnsureRoutingMapPresent(request, routingMap, collection); result = await this.TryResolveServerPartitionAsync( request, collection, routingMap, collectionCacheIsUptodate : true, collectionRoutingMapCacheIsUptodate : true, forceRefreshPartitionAddresses : forceRefreshPartitionAddresses, cancellationToken : cancellationToken); } if (result == null) { DefaultTrace.TraceInformation("Couldn't route partitionkeyrange-oblivious request after retry/cache refresh. Collection doesn't exist."); // At this point collection cache and routing map caches are refreshed. // The only reason we will get here is if collection doesn't exist. // Case when partitionkeyrange doesn't exist is handled in the corresponding method. throw new NotFoundException() { ResourceAddress = request.ResourceAddress }; } if (request.IsNameBased) { // Append collection rid. // If we resolved collection rid incorrectly because of outdated cache, this can lead // to incorrect routing decisions. But backend will validate collection rid and throw // InvalidPartitionException if we reach wrong collection. // Also this header will be used by backend to inject collection rid into metrics for // throttled requests. request.Headers[WFConstants.BackendHeaders.CollectionRid] = collection.ResourceId; } return(result); }
private static async Task <Tuple <bool, PartitionKeyRange> > TryResolvePartitionKeyRangeAsync( DocumentServiceRequest request, ISessionContainer sessionContainer, PartitionKeyRangeCache partitionKeyRangeCache, CollectionCache clientCollectionCache, bool refreshCache) { if (refreshCache) { request.ForceMasterRefresh = true; request.ForceNameCacheRefresh = true; } PartitionKeyRange partitonKeyRange = null; ContainerProperties collection = await clientCollectionCache.ResolveCollectionAsync( request, CancellationToken.None, NoOpTrace.Singleton); string partitionKeyString = request.Headers[HttpConstants.HttpHeaders.PartitionKey]; if (partitionKeyString != null) { CollectionRoutingMap collectionRoutingMap = await partitionKeyRangeCache.TryLookupAsync( collectionRid : collection.ResourceId, previousValue : null, request : request, cancellationToken : CancellationToken.None, NoOpTrace.Singleton); if (refreshCache && collectionRoutingMap != null) { collectionRoutingMap = await partitionKeyRangeCache.TryLookupAsync( collectionRid : collection.ResourceId, previousValue : collectionRoutingMap, request : request, cancellationToken : CancellationToken.None, NoOpTrace.Singleton); } partitonKeyRange = AddressResolver.TryResolveServerPartitionByPartitionKey( request: request, partitionKeyString: partitionKeyString, collectionCacheUptoDate: false, collection: collection, routingMap: collectionRoutingMap); } else if (request.PartitionKeyRangeIdentity != null) { PartitionKeyRangeIdentity partitionKeyRangeId = request.PartitionKeyRangeIdentity; partitonKeyRange = await partitionKeyRangeCache.TryGetPartitionKeyRangeByIdAsync( collection.ResourceId, partitionKeyRangeId.PartitionKeyRangeId, NoOpTrace.Singleton, refreshCache); } else if (request.RequestContext.ResolvedPartitionKeyRange != null) { partitonKeyRange = request.RequestContext.ResolvedPartitionKeyRange; } if (partitonKeyRange == null) { if (refreshCache) { return(new Tuple <bool, PartitionKeyRange>(false, null)); } // need to refresh cache. Maybe split happened. return(await GatewayStoreModel.TryResolvePartitionKeyRangeAsync( request : request, sessionContainer : sessionContainer, partitionKeyRangeCache : partitionKeyRangeCache, clientCollectionCache : clientCollectionCache, refreshCache : true)); } return(new Tuple <bool, PartitionKeyRange>(true, partitonKeyRange)); }
private PartitionKeyRange TryResolveSinglePartitionCollection( DocumentServiceRequest request, ContainerProperties collection, CollectionRoutingMap routingMap, bool collectionCacheIsUptoDate) { // Neither partitionkey nor partitionkeyrangeid is specified. // Three options here: // * This is non-partitioned collection and old client SDK which doesn't send partition key. In // this case there's single entry in routing map. But can be multiple entries if before that // existed partitioned collection with same name. // * This is partitioned collection and old client SDK which doesn't send partition key. // In this case there can be multiple ranges in routing map. // * This is partitioned collection and this is custom written REST sdk, which has a bug and doesn't send // partition key. // We cannot know for sure whether this is partitioned collection or not, because // partition key definition cache can be outdated. // So we route request to the first partition. If this is non-partitioned collection - request will succeed. // If it is partitioned collection - backend will return bad request as partition key header is required in this case. if (routingMap.OrderedPartitionKeyRanges.Count == 1) { return(routingMap.OrderedPartitionKeyRanges.Single()); } if (collectionCacheIsUptoDate) { // If the current collection is user-partitioned collection if (collection.PartitionKey.Paths.Count >= 1 && !collection.PartitionKey.IsSystemKey.GetValueOrDefault(false)) { throw new BadRequestException(RMResources.MissingPartitionKeyValue) { ResourceAddress = request.ResourceAddress }; } else if (routingMap.OrderedPartitionKeyRanges.Count > 1) { // With migrated-fixed-collection, it is possible to have multiple partition key ranges // due to parallel usage of V3 SDK and a possible storage or throughput split // The current client might be legacy and not aware of this. // In such case route the request to the first partition return(this.TryResolveServerPartitionByPartitionKey( request, "[]", // This corresponds to first partition collectionCacheIsUptoDate, collection, routingMap)); } else { // routingMap.OrderedPartitionKeyRanges.Count == 0 // Should never come here. DefaultTrace.TraceCritical( "No Partition Key ranges present for the collection {0}", collection.ResourceId); throw new InternalServerErrorException(RMResources.InternalServerError) { ResourceAddress = request.ResourceAddress }; } } else { return(null); } }
public async Task RetryOnSplit() { ItemBatchOperation itemBatchOperation = CreateItem("test"); Mock <CosmosClientContext> mockedContext = this.MockClientContext(); mockedContext.Setup(c => c.ClientOptions).Returns(new CosmosClientOptions()); mockedContext .SetupSequence(c => c.ProcessResourceOperationStreamAsync( It.IsAny <string>(), It.IsAny <ResourceType>(), It.IsAny <OperationType>(), It.IsAny <RequestOptions>(), It.IsAny <ContainerInternal>(), It.IsAny <Cosmos.FeedRange>(), It.IsAny <Stream>(), It.IsAny <Action <RequestMessage> >(), It.IsAny <ITrace>(), It.IsAny <CancellationToken>())) .Returns(GenerateSplitResponseAsync(itemBatchOperation)) .Returns(GenerateOkResponseAsync(itemBatchOperation)); mockedContext.Setup(c => c.SerializerCore).Returns(MockCosmosUtil.Serializer); string link = "/dbs/db/colls/colls"; Mock <ContainerInternal> mockContainer = new Mock <ContainerInternal>(); mockContainer.Setup(x => x.LinkUri).Returns(link); mockContainer.Setup(x => x.GetPartitionKeyDefinitionAsync(It.IsAny <CancellationToken>())).Returns(Task.FromResult(new PartitionKeyDefinition() { Paths = new Collection <string>() { "/id" } })); mockContainer.Setup(c => c.GetCachedRIDAsync(It.IsAny <bool>(), It.IsAny <ITrace>(), It.IsAny <CancellationToken>())).ReturnsAsync(Guid.NewGuid().ToString()); Mock <CosmosClientContext> context = this.MockClientContext(); mockContainer.Setup(c => c.ClientContext).Returns(context.Object); context.Setup(c => c.DocumentClient).Returns(new ClientWithSplitDetection()); CollectionRoutingMap routingMap = CollectionRoutingMap.TryCreateCompleteRoutingMap( new[] { Tuple.Create(new PartitionKeyRange { Id = "0", MinInclusive = "", MaxExclusive = "FF" }, (ServiceIdentity)null) }, string.Empty); mockContainer.Setup(x => x.GetRoutingMapAsync(It.IsAny <CancellationToken>())).Returns(Task.FromResult(routingMap)); BatchAsyncContainerExecutor executor = new BatchAsyncContainerExecutor(mockContainer.Object, mockedContext.Object, 20, BatchAsyncContainerExecutorCache.DefaultMaxBulkRequestBodySizeInBytes); TransactionalBatchOperationResult result = await executor.AddAsync(itemBatchOperation); Mock.Get(mockContainer.Object) .Verify(x => x.GetPartitionKeyDefinitionAsync(It.IsAny <CancellationToken>()), Times.Exactly(2)); Mock.Get(mockedContext.Object) .Verify(c => c.ProcessResourceOperationStreamAsync( It.IsAny <string>(), It.IsAny <ResourceType>(), It.IsAny <OperationType>(), It.IsAny <RequestOptions>(), It.IsAny <ContainerInternal>(), It.IsAny <Cosmos.FeedRange>(), It.IsAny <Stream>(), It.IsAny <Action <RequestMessage> >(), It.IsAny <ITrace>(), It.IsAny <CancellationToken>()), Times.Exactly(2)); Assert.AreEqual(HttpStatusCode.OK, result.StatusCode); Assert.IsNotNull(result.ToResponseMessage().Trace); }
public MockRoutingMapProvider(IList <PartitionKeyRange> ranges) { this.routingMap = CollectionRoutingMap.TryCreateCompleteRoutingMap(ranges.Select(r => Tuple.Create(r, (ServiceIdentity)null)), ""); }