public async Task ItemMultiplePartitionQuery() { IList <ToDoActivity> deleteList = await this.CreateRandomItems(3, randomPartitionKey : true); ToDoActivity find = deleteList.First(); CosmosSqlQueryDefinition sql = new CosmosSqlQueryDefinition("select * from toDoActivity t where t.id = '" + find.id + "'"); CosmosQueryRequestOptions requestOptions = new CosmosQueryRequestOptions() { MaxBufferedItemCount = 10, ResponseContinuationTokenLimitInKb = 500 }; CosmosResultSetIterator <ToDoActivity> setIterator = this.Container.Items.CreateItemQuery <ToDoActivity>(sql, maxConcurrency: 1, maxItemCount: 1, requestOptions: requestOptions); while (setIterator.HasMoreResults) { CosmosQueryResponse <ToDoActivity> iter = await setIterator.FetchNextSetAsync(); Assert.AreEqual(1, iter.Count()); ToDoActivity response = iter.First(); Assert.AreEqual(find.id, response.id); } }
/// <summary> /// Initializes a new instance of the CosmosCrossPartitionQueryExecutionContext class. /// </summary> /// <param name="initParams">Constructor parameters for the base class.</param> /// <param name="moveNextComparer">Comparer used to figure out that document producer tree to serve documents from next.</param> /// <param name="fetchPrioirtyFunction">The priority function to determine which partition to fetch documents from next.</param> /// <param name="equalityComparer">Used to determine whether we need to return the continuation token for a partition.</param> protected CosmosCrossPartitionQueryExecutionContext( CosmosQueryContext initParams, IComparer <ItemProducerTree> moveNextComparer, Func <ItemProducerTree, int> fetchPrioirtyFunction, IEqualityComparer <CosmosElement> equalityComparer) { if (moveNextComparer == null) { throw new ArgumentNullException(nameof(moveNextComparer)); } if (fetchPrioirtyFunction == null) { throw new ArgumentNullException(nameof(fetchPrioirtyFunction)); } if (equalityComparer == null) { throw new ArgumentNullException(nameof(equalityComparer)); } this.queryContext = initParams; this.queryRequestOptions = initParams.QueryRequestOptions; this.itemProducerForest = new PriorityQueue <ItemProducerTree>(moveNextComparer, isSynchronized: true); this.fetchPrioirtyFunction = fetchPrioirtyFunction; this.comparableTaskScheduler = new ComparableTaskScheduler(initParams.QueryRequestOptions.MaxConcurrency.GetValueOrDefault(0)); this.equalityComparer = equalityComparer; this.requestChargeTracker = new RequestChargeTracker(); this.partitionedQueryMetrics = new ConcurrentBag <Tuple <string, QueryMetrics> >(); this.actualMaxPageSize = this.queryRequestOptions.MaxItemCount.GetValueOrDefault(ParallelQueryConfig.GetConfig().ClientInternalMaxItemCount); if (this.actualMaxPageSize < 0) { throw new ArgumentOutOfRangeException("actualMaxPageSize should never be less than 0"); } if (this.actualMaxPageSize > int.MaxValue) { throw new ArgumentOutOfRangeException("actualMaxPageSize should never be greater than int.MaxValue"); } if (this.queryRequestOptions.MaxBufferedItemCount.HasValue) { this.actualMaxBufferedItemCount = this.queryRequestOptions.MaxBufferedItemCount.Value; } else { this.actualMaxBufferedItemCount = ParallelQueryConfig.GetConfig().DefaultMaximumBufferSize; } if (this.actualMaxBufferedItemCount < 0) { throw new ArgumentOutOfRangeException("actualMaxBufferedItemCount should never be less than 0"); } if (this.actualMaxBufferedItemCount > int.MaxValue) { throw new ArgumentOutOfRangeException("actualMaxBufferedItemCount should never be greater than int.MaxValue"); } }
private async Task <FeedResponse <CosmosElement> > ExecuteOnceAsync(IDocumentClientRetryPolicy retryPolicyInstance, CancellationToken cancellationToken) { if (this.LogicalPartitionKeyProvided()) { return(await this.queryContext.ExecuteQueryAsync( this.queryContext.SqlQuerySpec, cancellationToken, requestEnricher : (cosmosRequestMessage) => { cosmosRequestMessage.Headers.Add(HttpConstants.HttpHeaders.IsContinuationExpected, bool.FalseString); CosmosQueryRequestOptions.FillContinuationToken(cosmosRequestMessage, this.ContinuationToken); })); } // For non-Windows platforms(like Linux and OSX) in .NET Core SDK, we cannot use ServiceInterop for parsing the query, // so forcing the request through Gateway. We are also now by-passing this for 32-bit host process in NETFX on Windows // as the ServiceInterop dll is only available in 64-bit. return(await this.queryContext.ExecuteQueryAsync( this.queryContext.SqlQuerySpec, cancellationToken, requestEnricher : (cosmosRequestMessage) => { cosmosRequestMessage.UseGatewayMode = true; cosmosRequestMessage.Headers.Add(HttpConstants.HttpHeaders.IsContinuationExpected, this.queryContext.IsContinuationExpected.ToString()); CosmosQueryRequestOptions.FillContinuationToken(cosmosRequestMessage, this.ContinuationToken); })); }
/// <summary> /// Gets the list of partition key ranges. /// 1. Check partition key range id /// 2. Check Partition key /// 3. Check the effective partition key /// 4. Get the range from the PartitionedQueryExecutionInfo /// </summary> internal static async Task <List <PartitionKeyRange> > GetTargetPartitionKeyRanges( CosmosQueryClient queryClient, string resourceLink, PartitionedQueryExecutionInfo partitionedQueryExecutionInfo, CosmosContainerSettings collection, CosmosQueryRequestOptions queryRequestOptions) { List <PartitionKeyRange> targetRanges; if (queryRequestOptions.PartitionKey != null) { targetRanges = await queryClient.GetTargetPartitionKeyRangesByEpkString( resourceLink, collection.ResourceId, new PartitionKey(queryRequestOptions.PartitionKey).InternalKey.GetEffectivePartitionKeyString(collection.PartitionKey)); } else if (TryGetEpkProperty(queryRequestOptions, out string effectivePartitionKeyString)) { targetRanges = await queryClient.GetTargetPartitionKeyRangesByEpkString( resourceLink, collection.ResourceId, effectivePartitionKeyString); } else { targetRanges = await queryClient.GetTargetPartitionKeyRanges( resourceLink, collection.ResourceId, partitionedQueryExecutionInfo.QueryRanges); } return(targetRanges); }
internal ChangeFeedResultSetIterator( int?maxItemCount, string continuationToken, CosmosQueryRequestOptions options, NextResultSetDelegate nextDelegate, object state = null) : base(maxItemCount, continuationToken, options, nextDelegate, state) { }
public async Task ItemMultiplePartitionOrderByQueryStream() { IList <ToDoActivity> deleteList = new List <ToDoActivity>(); try { deleteList = await CreateRandomItems(300, randomPartitionKey : true); CosmosSqlQueryDefinition sql = new CosmosSqlQueryDefinition("SELECT * FROM toDoActivity t ORDER BY t.taskNum "); CosmosQueryRequestOptions requestOptions = new CosmosQueryRequestOptions() { MaxBufferedItemCount = 10, ResponseContinuationTokenLimitInKb = 500 }; List <ToDoActivity> resultList = new List <ToDoActivity>(); double totalRequstCharge = 0; CosmosResultSetIterator setIterator = this.Container.Items.CreateItemQueryAsStream(sql, maxConcurrency: 5, maxItemCount: 1, requestOptions: requestOptions); while (setIterator.HasMoreResults) { using (CosmosQueryResponse iter = await setIterator.FetchNextSetAsync()) { Assert.IsTrue(iter.IsSuccess); Assert.IsNull(iter.ErrorMessage); Assert.IsTrue(iter.Count <= 5); totalRequstCharge += iter.RequestCharge; ToDoActivity response = this.jsonSerializer.FromStream <ToDoActivity[]>(iter.Content).First(); resultList.Add(response); } } Assert.AreEqual(deleteList.Count, resultList.Count); Assert.IsTrue(totalRequstCharge > 0); List <ToDoActivity> verifiedOrderBy = deleteList.OrderBy(x => x.taskNum).ToList(); for (int i = 0; i < verifiedOrderBy.Count(); i++) { Assert.AreEqual(verifiedOrderBy[i].taskNum, resultList[i].taskNum); Assert.AreEqual(verifiedOrderBy[i].id, resultList[i].id); } } finally { foreach (ToDoActivity delete in deleteList) { CosmosResponseMessage deleteResponse = await this.Container.Items.DeleteItemStreamAsync(delete.status, delete.id); deleteResponse.Dispose(); } } }
public CosmosQueryContext( CosmosQueryClient client, ResourceType resourceTypeEnum, OperationType operationType, Type resourceType, SqlQuerySpec sqlQuerySpecFromUser, CosmosQueryRequestOptions queryRequestOptions, Uri resourceLink, bool getLazyFeedResponse, Guid correlatedActivityId, bool isContinuationExpected, bool allowNonValueAggregateQuery, string containerResourceId = null) { if (client == null) { throw new ArgumentNullException(nameof(client)); } if (resourceType == null) { throw new ArgumentNullException(nameof(resourceType)); } if (sqlQuerySpecFromUser == null) { throw new ArgumentNullException(nameof(sqlQuerySpecFromUser)); } if (queryRequestOptions == null) { throw new ArgumentNullException(nameof(queryRequestOptions)); } if (correlatedActivityId == Guid.Empty) { throw new ArgumentException(nameof(correlatedActivityId)); } this.OperationTypeEnum = operationType; this.QueryClient = client; this.ResourceTypeEnum = resourceTypeEnum; this.ResourceType = resourceType; this.SqlQuerySpec = sqlQuerySpecFromUser; this.QueryRequestOptions = queryRequestOptions; this.ResourceLink = resourceLink; this.ContainerResourceId = containerResourceId; this.IsContinuationExpected = isContinuationExpected; this.AllowNonValueAggregateQuery = allowNonValueAggregateQuery; this.CorrelatedActivityId = correlatedActivityId; }
public async Task ItemQueryStreamSerializationSetting() { IList <ToDoActivity> deleteList = await this.CreateRandomItems(101, randomPartitionKey : true); CosmosSqlQueryDefinition sql = new CosmosSqlQueryDefinition("SELECT * FROM toDoActivity t ORDER BY t.taskNum"); CosmosSerializationOptions options = new CosmosSerializationOptions( ContentSerializationFormat.CosmosBinary.ToString(), (content) => JsonNavigator.Create(content), () => JsonWriter.Create(JsonSerializationFormat.Binary)); CosmosQueryRequestOptions requestOptions = new CosmosQueryRequestOptions() { CosmosSerializationOptions = options }; List <ToDoActivity> resultList = new List <ToDoActivity>(); double totalRequstCharge = 0; CosmosResultSetIterator setIterator = this.Container.Items.CreateItemQueryAsStream(sql, maxConcurrency: 5, maxItemCount: 5, requestOptions: requestOptions); while (setIterator.HasMoreResults) { using (CosmosQueryResponse iter = await setIterator.FetchNextSetAsync()) { Assert.IsTrue(iter.IsSuccess); Assert.IsNull(iter.ErrorMessage); Assert.IsTrue(iter.Count <= 5); totalRequstCharge += iter.RequestCharge; IJsonReader reader = JsonReader.Create(iter.Content); IJsonWriter textWriter = JsonWriter.Create(JsonSerializationFormat.Text); textWriter.WriteAll(reader); string json = Encoding.UTF8.GetString(textWriter.GetResult()); Assert.IsNotNull(json); ToDoActivity[] responseActivities = JsonConvert.DeserializeObject <ToDoActivity[]>(json); resultList.AddRange(responseActivities); } } Assert.AreEqual(deleteList.Count, resultList.Count); Assert.IsTrue(totalRequstCharge > 0); List <ToDoActivity> verifiedOrderBy = deleteList.OrderBy(x => x.taskNum).ToList(); for (int i = 0; i < verifiedOrderBy.Count(); i++) { Assert.AreEqual(verifiedOrderBy[i].taskNum, resultList[i].taskNum); Assert.AreEqual(verifiedOrderBy[i].id, resultList[i].id); } }
internal async Task <FeedResponse <CosmosElement> > ExecuteQueryAsync( SqlQuerySpec querySpecForInit, CancellationToken cancellationToken, Action <CosmosRequestMessage> requestEnricher = null) { CosmosQueryRequestOptions requestOptions = this.QueryRequestOptions.Clone(); return(await this.QueryClient.ExecuteItemQueryAsync( this.ResourceLink, this.ResourceTypeEnum, this.OperationTypeEnum, requestOptions, querySpecForInit, requestEnricher, cancellationToken)); }
private static bool TryGetEpkProperty( CosmosQueryRequestOptions queryRequestOptions, out string effectivePartitionKeyString) { if (queryRequestOptions?.Properties != null && queryRequestOptions.Properties.TryGetValue( WFConstants.BackendHeaders.EffectivePartitionKeyString, out object effectivePartitionKeyStringObject)) { effectivePartitionKeyString = effectivePartitionKeyStringObject as string; if (string.IsNullOrEmpty(effectivePartitionKeyString)) { throw new ArgumentOutOfRangeException(nameof(effectivePartitionKeyString)); } return(true); } effectivePartitionKeyString = null; return(false); }
private static async Task QueryPartitionedContainerInParallelAsync(CosmosContainer container) { List <Family> familiesSerial = new List <Family>(); String queryText = "SELECT * FROM Families"; // 0 maximum parallel tasks, effectively serial execution CosmosQueryRequestOptions options = new CosmosQueryRequestOptions() { MaxBufferedItemCount = 100 }; var query = container.Items.CreateItemQuery <Family>( queryText, maxConcurrency: 0, requestOptions: options); while (query.HasMoreResults) { foreach (Family family in await query.FetchNextSetAsync()) { familiesSerial.Add(family); } } Assert("Parallel Query expected two families", familiesSerial.ToList().Count == 2); // 1 maximum parallel tasks, 1 dedicated asynchronous task to continuously make REST calls List <Family> familiesParallel1 = new List <Family>(); query = container.Items.CreateItemQuery <Family>( queryText, maxConcurrency: 1, requestOptions: options); while (query.HasMoreResults) { foreach (Family family in await query.FetchNextSetAsync()) { familiesParallel1.Add(family); } } Assert("Parallel Query expected two families", familiesParallel1.ToList().Count == 2); AssertSequenceEqual("Parallel query returns result out of order compared to serial execution", familiesSerial, familiesParallel1); // 10 maximum parallel tasks, a maximum of 10 dedicated asynchronous tasks to continuously make REST calls List <Family> familiesParallel10 = new List <Family>(); query = container.Items.CreateItemQuery <Family>( queryText, maxConcurrency: 10, requestOptions: options); while (query.HasMoreResults) { foreach (Family family in await query.FetchNextSetAsync()) { familiesParallel10.Add(family); } } Assert("Parallel Query expected two families", familiesParallel10.ToList().Count == 2); AssertSequenceEqual("Parallel query returns result out of order compared to serial execution", familiesSerial, familiesParallel10); }
public CosmosQueryExecutionContextFactory( CosmosQueryClient client, ResourceType resourceTypeEnum, OperationType operationType, Type resourceType, SqlQuerySpec sqlQuerySpec, CosmosQueryRequestOptions queryRequestOptions, Uri resourceLink, bool isContinuationExpected, bool allowNonValueAggregateQuery, Guid correlatedActivityId) { if (client == null) { throw new ArgumentNullException(nameof(client)); } if (sqlQuerySpec == null) { throw new ArgumentNullException(nameof(sqlQuerySpec)); } if (queryRequestOptions == null) { throw new ArgumentNullException(nameof(queryRequestOptions)); } if (resourceLink == null) { throw new ArgumentNullException(nameof(resourceLink)); } // Prevent users from updating the values after creating the execution context. CosmosQueryRequestOptions cloneQueryRequestOptions = queryRequestOptions.Clone(); // Swapping out negative values in feedOptions for int.MaxValue if (cloneQueryRequestOptions.MaxBufferedItemCount.HasValue && cloneQueryRequestOptions.MaxBufferedItemCount < 0) { cloneQueryRequestOptions.MaxBufferedItemCount = int.MaxValue; } if (cloneQueryRequestOptions.MaxConcurrency.HasValue && cloneQueryRequestOptions.MaxConcurrency < 0) { cloneQueryRequestOptions.MaxConcurrency = int.MaxValue; } if (cloneQueryRequestOptions.MaxItemCount.HasValue && cloneQueryRequestOptions.MaxItemCount < 0) { cloneQueryRequestOptions.MaxItemCount = int.MaxValue; } this.cosmosQueryContext = new CosmosQueryContext( client: client, resourceTypeEnum: resourceTypeEnum, operationType: operationType, resourceType: resourceType, sqlQuerySpecFromUser: sqlQuerySpec, queryRequestOptions: cloneQueryRequestOptions, resourceLink: resourceLink, getLazyFeedResponse: isContinuationExpected, isContinuationExpected: isContinuationExpected, allowNonValueAggregateQuery: allowNonValueAggregateQuery, correlatedActivityId: correlatedActivityId); }