public async Task ReadFeedIteratorCore_Trace() { int batchSize = 1000; await this.CreateRandomItems(this.LargerContainer, batchSize, randomPartitionKey : true); ContainerInternal itemsCore = this.LargerContainer; FeedIteratorInternal feedIterator = (FeedIteratorInternal)itemsCore.GetItemQueryStreamIterator( queryDefinition: null, requestOptions: new QueryRequestOptions() { MaxItemCount = int.MaxValue }); ITrace rootTrace; int childCount = 0; using (rootTrace = Trace.GetRootTrace("Cross Partition Read Feed")) { while (feedIterator.HasMoreResults) { using (ResponseMessage responseMessage = await feedIterator.ReadNextAsync(rootTrace, this.cancellationToken)) { responseMessage.EnsureSuccessStatusCode(); childCount++; } } } string trace = TraceWriter.TraceToText(rootTrace); Console.WriteLine(trace); Assert.AreEqual(childCount, rootTrace.Children.Count); }
internal async Task <Response <T> > AggregateResultAsync(CancellationToken cancellationToken = default) { List <T> result = new List <T>(); Headers headers = new Headers(); FeedIterator <T> localFeedIterator = this.CreateFeedIterator(isContinuationExpected: false); FeedIteratorInternal <T> localFeedIteratorInternal = (FeedIteratorInternal <T>)localFeedIterator; ITrace rootTrace; using (rootTrace = Trace.GetRootTrace("Aggregate LINQ Operation")) { while (localFeedIterator.HasMoreResults) { FeedResponse <T> response = await localFeedIteratorInternal.ReadNextAsync(rootTrace, cancellationToken); headers.RequestCharge += response.RequestCharge; result.AddRange(response); } } return(new ItemResponse <T>( System.Net.HttpStatusCode.OK, headers, result.FirstOrDefault(), rootTrace)); }
internal static async Task <List <T> > QueryWithCosmosElementContinuationTokenAsync <T>( Container container, string query, QueryRequestOptions queryRequestOptions = null) { if (queryRequestOptions == null) { queryRequestOptions = new QueryRequestOptions(); } List <T> resultsFromCosmosElementContinuationToken = new List <T>(); CosmosElement continuationToken = null; do { QueryRequestOptions computeRequestOptions = queryRequestOptions.Clone(); computeRequestOptions.ExecutionEnvironment = Cosmos.Query.Core.ExecutionContext.ExecutionEnvironment.Compute; computeRequestOptions.CosmosElementContinuationToken = continuationToken; FeedIteratorInternal <T> itemQuery = (FeedIteratorInternal <T>)container.GetItemQueryIterator <T>( queryText: query, requestOptions: computeRequestOptions); try { FeedResponse <T> cosmosQueryResponse = await itemQuery.ReadNextAsync(); if (queryRequestOptions.MaxItemCount.HasValue) { Assert.IsTrue( cosmosQueryResponse.Count <= queryRequestOptions.MaxItemCount.Value, "Max Item Count is not being honored"); } resultsFromCosmosElementContinuationToken.AddRange(cosmosQueryResponse); // Force a rewrite of the continuation token, so that we test the case where we roundtrip it over the wire. // There was a bug where resuming from double.NaN lead to an exception, // since we parsed the type assuming it was always a double and not a string. CosmosElement originalContinuationToken = itemQuery.GetCosmosElementContinuationToken(); if (originalContinuationToken != null) { continuationToken = CosmosElement.Parse(originalContinuationToken.ToString()); } else { continuationToken = null; } } catch (CosmosException cosmosException) when(cosmosException.StatusCode == (HttpStatusCode)429) { itemQuery = (FeedIteratorInternal <T>)container.GetItemQueryIterator <T>( queryText: query, requestOptions: queryRequestOptions); } } while (continuationToken != null); return(resultsFromCosmosElementContinuationToken); }
private FeedIterator <T> CreateFeedIterator(bool isContinuationExcpected) { SqlQuerySpec querySpec = DocumentQueryEvaluator.Evaluate(this.Expression, this.serializationOptions); FeedIteratorInternal streamIterator = this.CreateStreamIterator(isContinuationExcpected); return(new FeedIteratorInlineCore <T>(new FeedIteratorCore <T>( streamIterator, this.responseFactory.CreateQueryFeedUserTypeResponse <T>))); }
public async Task QueryActivityIdWithContinuationTokenAndTraceTest() { using (ITrace rootTrace = Trace.GetRootTrace("Root Trace")) { CosmosClient client = DirectCosmosClient; Container container = client.GetContainer(DatabaseId, ContainerId); // Create items for (int i = 0; i < 500; i++) { await container.CreateItemAsync <ToDoActivity>(ToDoActivity.CreateRandomToDoActivity()); } QueryRequestOptions queryRequestOptions = new QueryRequestOptions { MaxItemCount = 50 }; FeedIteratorInternal feedIterator = (FeedIteratorInternal)container.GetItemQueryStreamIterator( "select * from c", null, queryRequestOptions); string continuationToken = (await feedIterator.ReadNextAsync(rootTrace, CancellationToken.None)).ContinuationToken; rootTrace.Data.TryGetValue("Query Correlated ActivityId", out object firstCorrelatedActivityId); // use Continuation Token to create new iterator and use same trace FeedIteratorInternal feedIteratorNew = (FeedIteratorInternal)container.GetItemQueryStreamIterator( "select * from c", continuationToken, queryRequestOptions); while (feedIteratorNew.HasMoreResults) { await feedIteratorNew.ReadNextAsync(rootTrace, CancellationToken.None); } // Test trace has 2 correlated ActivityIds rootTrace.Data.TryGetValue("Query Correlated ActivityId", out object correlatedActivityIds); List <string> correlatedIdList = correlatedActivityIds.ToString().Split(',').ToList(); Assert.AreEqual(correlatedIdList.Count, 2); Assert.AreEqual(correlatedIdList[0], firstCorrelatedActivityId.ToString()); } }
internal static async Task <List <T> > QueryWithCosmosElementContinuationTokenAsync <T>( Container container, string query, QueryRequestOptions queryRequestOptions = null) { if (queryRequestOptions == null) { queryRequestOptions = new QueryRequestOptions(); } List <T> resultsFromCosmosElementContinuationToken = new List <T>(); CosmosElement continuationToken = null; do { QueryRequestOptions computeRequestOptions = queryRequestOptions.Clone(); computeRequestOptions.ExecutionEnvironment = Cosmos.Query.Core.ExecutionContext.ExecutionEnvironment.Compute; computeRequestOptions.CosmosElementContinuationToken = continuationToken; FeedIteratorInternal <T> itemQuery = (FeedIteratorInternal <T>)container.GetItemQueryIterator <T>( queryText: query, requestOptions: computeRequestOptions); try { FeedResponse <T> cosmosQueryResponse = await itemQuery.ReadNextAsync(); if (queryRequestOptions.MaxItemCount.HasValue) { Assert.IsTrue( cosmosQueryResponse.Count <= queryRequestOptions.MaxItemCount.Value, "Max Item Count is not being honored"); } resultsFromCosmosElementContinuationToken.AddRange(cosmosQueryResponse); continuationToken = itemQuery.GetCosmosElementContinuationToken(); } catch (CosmosException cosmosException) when(cosmosException.StatusCode == (HttpStatusCode)429) { itemQuery = (FeedIteratorInternal <T>)container.GetItemQueryIterator <T>( queryText: query, requestOptions: queryRequestOptions); } } while (continuationToken != null); return(resultsFromCosmosElementContinuationToken); }
private async Task <(ChangeFeedProcessorState, ResponseMessage)> GetRemainingWorkAsync( DocumentServiceLease existingLease, ITrace trace, CancellationToken cancellationToken) { using FeedIteratorInternal iterator = this.monitoredContainerFeedCreator( existingLease, existingLease.ContinuationToken, string.IsNullOrEmpty(existingLease.ContinuationToken)); try { ResponseMessage response = await iterator.ReadNextAsync(trace, cancellationToken).ConfigureAwait(false); if (response.StatusCode != HttpStatusCode.NotModified) { response.EnsureSuccessStatusCode(); } long parsedLSNFromSessionToken = ChangeFeedEstimatorIterator.TryConvertToNumber(ExtractLsnFromSessionToken(response.Headers.Session)); IEnumerable <JObject> items = ChangeFeedEstimatorIterator.GetItemsFromResponse(response); long lastQueryLSN = items.Any() ? ChangeFeedEstimatorIterator.TryConvertToNumber(ChangeFeedEstimatorIterator.GetFirstItemLSN(items)) - 1 : parsedLSNFromSessionToken; if (lastQueryLSN < 0) { return(new ChangeFeedProcessorState(existingLease.CurrentLeaseToken, 1, existingLease.Owner), response); } long leaseTokenRemainingWork = parsedLSNFromSessionToken - lastQueryLSN; long estimation = leaseTokenRemainingWork < 0 ? 0 : leaseTokenRemainingWork; return(new ChangeFeedProcessorState(existingLease.CurrentLeaseToken, estimation, existingLease.Owner), response); } catch (Exception clientException) { Cosmos.Extensions.TraceException(clientException); DefaultTrace.TraceWarning("GetEstimateWork > exception: lease token '{0}'", existingLease.CurrentLeaseToken); throw; } }
public async Task ParallelizeQueryThroughTokens() { ContainerInternal container = null; try { // Create a container large enough to have at least 2 partitions ContainerResponse containerResponse = await this.database.CreateContainerAsync( id : Guid.NewGuid().ToString(), partitionKeyPath : "/id", throughput : 15000); container = (ContainerInlineCore)containerResponse; List <string> generatedIds = Enumerable.Range(0, 1000).Select(n => $"BasicItem{n}").ToList(); foreach (string id in generatedIds) { string item = $@" {{ ""id"": ""{id}"" }}"; using (ResponseMessage createResponse = await container.CreateItemStreamAsync( QueryFeedRangeTests.GenerateStreamFromString(item), new Cosmos.PartitionKey(id))) { Assert.IsTrue(createResponse.IsSuccessStatusCode); } } IReadOnlyList <FeedRange> feedTokens = await container.GetFeedRangesAsync(); Assert.IsTrue(feedTokens.Count > 1, " RUs of the container needs to be increased to ensure at least 2 partitions."); List <Task <List <string> > > tasks = feedTokens.Select(async feedToken => { List <string> results = new List <string>(); FeedIteratorInternal feedIterator = container.GetItemQueryStreamIterator( queryDefinition: new QueryDefinition("select * from T where STARTSWITH(T.id, \"BasicItem\")"), feedRange: feedToken, continuationToken: null, requestOptions: new QueryRequestOptions() { MaxItemCount = 10 }) as FeedIteratorInternal; string continuation = null; while (feedIterator.HasMoreResults) { using (ResponseMessage responseMessage = await feedIterator.ReadNextAsync(this.cancellationToken)) { if (responseMessage.IsSuccessStatusCode) { using (StreamReader reader = new StreamReader(responseMessage.Content)) { string json = await reader.ReadToEndAsync(); JArray documents = (JArray)JObject.Parse(json).SelectToken("Documents"); foreach (JObject document in documents) { results.Add(document.SelectToken("id").ToString()); } } } continuation = responseMessage.ContinuationToken; break; } } feedIterator = container.GetItemQueryStreamIterator(queryDefinition: new QueryDefinition("select * from T where STARTSWITH(T.id, \"BasicItem\")"), feedRange: feedToken, continuationToken: continuation, requestOptions: new QueryRequestOptions() { MaxItemCount = 10 }) as FeedIteratorInternal; while (feedIterator.HasMoreResults) { using (ResponseMessage responseMessage = await feedIterator.ReadNextAsync(this.cancellationToken)) { if (responseMessage.IsSuccessStatusCode) { using (StreamReader reader = new StreamReader(responseMessage.Content)) { string json = await reader.ReadToEndAsync(); JArray documents = (JArray)JObject.Parse(json).SelectToken("Documents"); foreach (JObject document in documents) { results.Add(document.SelectToken("id").ToString()); } } } } } return(results); }).ToList(); await Task.WhenAll(tasks); CollectionAssert.AreEquivalent(generatedIds, tasks.SelectMany(t => t.Result).ToList()); } finally { await container?.DeleteContainerAsync(); } }