public async Task ReadFeedIteratorCore_Trace()
{
int batchSize = 1000;
await this.CreateRandomItems(this.LargerContainer, batchSize, randomPartitionKey : true);
ContainerInternal itemsCore = this.LargerContainer;
FeedIteratorInternal feedIterator = (FeedIteratorInternal)itemsCore.GetItemQueryStreamIterator(
queryDefinition: null,
requestOptions: new QueryRequestOptions()
{
MaxItemCount = int.MaxValue
});
ITrace rootTrace;
int childCount = 0;
using (rootTrace = Trace.GetRootTrace("Cross Partition Read Feed"))
{
while (feedIterator.HasMoreResults)
{
using (ResponseMessage responseMessage = await feedIterator.ReadNextAsync(rootTrace, this.cancellationToken))
{
responseMessage.EnsureSuccessStatusCode();
childCount++;
}
}
}
string trace = TraceWriter.TraceToText(rootTrace);
Console.WriteLine(trace);
Assert.AreEqual(childCount, rootTrace.Children.Count);
}
internal async Task <Response <T> > AggregateResultAsync(CancellationToken cancellationToken = default)
{
List <T> result = new List <T>();
Headers headers = new Headers();
FeedIterator <T> localFeedIterator = this.CreateFeedIterator(isContinuationExpected: false);
FeedIteratorInternal <T> localFeedIteratorInternal = (FeedIteratorInternal <T>)localFeedIterator;
ITrace rootTrace;
using (rootTrace = Trace.GetRootTrace("Aggregate LINQ Operation"))
{
while (localFeedIterator.HasMoreResults)
{
FeedResponse <T> response = await localFeedIteratorInternal.ReadNextAsync(rootTrace, cancellationToken);
headers.RequestCharge += response.RequestCharge;
result.AddRange(response);
}
}
return(new ItemResponse <T>(
System.Net.HttpStatusCode.OK,
headers,
result.FirstOrDefault(),
rootTrace));
}
internal static async Task <List <T> > QueryWithCosmosElementContinuationTokenAsync <T>(
Container container,
string query,
QueryRequestOptions queryRequestOptions = null)
{
if (queryRequestOptions == null)
{
queryRequestOptions = new QueryRequestOptions();
}
List <T> resultsFromCosmosElementContinuationToken = new List <T>();
CosmosElement continuationToken = null;
do
{
QueryRequestOptions computeRequestOptions = queryRequestOptions.Clone();
computeRequestOptions.ExecutionEnvironment = Cosmos.Query.Core.ExecutionContext.ExecutionEnvironment.Compute;
computeRequestOptions.CosmosElementContinuationToken = continuationToken;
FeedIteratorInternal <T> itemQuery = (FeedIteratorInternal <T>)container.GetItemQueryIterator <T>(
queryText: query,
requestOptions: computeRequestOptions);
try
{
FeedResponse <T> cosmosQueryResponse = await itemQuery.ReadNextAsync();
if (queryRequestOptions.MaxItemCount.HasValue)
{
Assert.IsTrue(
cosmosQueryResponse.Count <= queryRequestOptions.MaxItemCount.Value,
"Max Item Count is not being honored");
}
resultsFromCosmosElementContinuationToken.AddRange(cosmosQueryResponse);
// Force a rewrite of the continuation token, so that we test the case where we roundtrip it over the wire.
// There was a bug where resuming from double.NaN lead to an exception,
// since we parsed the type assuming it was always a double and not a string.
CosmosElement originalContinuationToken = itemQuery.GetCosmosElementContinuationToken();
if (originalContinuationToken != null)
{
continuationToken = CosmosElement.Parse(originalContinuationToken.ToString());
}
else
{
continuationToken = null;
}
}
catch (CosmosException cosmosException) when(cosmosException.StatusCode == (HttpStatusCode)429)
{
itemQuery = (FeedIteratorInternal <T>)container.GetItemQueryIterator <T>(
queryText: query,
requestOptions: queryRequestOptions);
}
} while (continuationToken != null);
return(resultsFromCosmosElementContinuationToken);
}
private FeedIterator <T> CreateFeedIterator(bool isContinuationExcpected)
{
SqlQuerySpec querySpec = DocumentQueryEvaluator.Evaluate(this.Expression, this.serializationOptions);
FeedIteratorInternal streamIterator = this.CreateStreamIterator(isContinuationExcpected);
return(new FeedIteratorInlineCore <T>(new FeedIteratorCore <T>(
streamIterator,
this.responseFactory.CreateQueryFeedUserTypeResponse <T>)));
}
public async Task QueryActivityIdWithContinuationTokenAndTraceTest()
{
using (ITrace rootTrace = Trace.GetRootTrace("Root Trace"))
{
CosmosClient client = DirectCosmosClient;
Container container = client.GetContainer(DatabaseId, ContainerId);
// Create items
for (int i = 0; i < 500; i++)
{
await container.CreateItemAsync <ToDoActivity>(ToDoActivity.CreateRandomToDoActivity());
}
QueryRequestOptions queryRequestOptions = new QueryRequestOptions
{
MaxItemCount = 50
};
FeedIteratorInternal feedIterator =
(FeedIteratorInternal)container.GetItemQueryStreamIterator(
"select * from c",
null,
queryRequestOptions);
string continuationToken = (await feedIterator.ReadNextAsync(rootTrace, CancellationToken.None)).ContinuationToken;
rootTrace.Data.TryGetValue("Query Correlated ActivityId",
out object firstCorrelatedActivityId);
// use Continuation Token to create new iterator and use same trace
FeedIteratorInternal feedIteratorNew =
(FeedIteratorInternal)container.GetItemQueryStreamIterator(
"select * from c",
continuationToken,
queryRequestOptions);
while (feedIteratorNew.HasMoreResults)
{
await feedIteratorNew.ReadNextAsync(rootTrace, CancellationToken.None);
}
// Test trace has 2 correlated ActivityIds
rootTrace.Data.TryGetValue("Query Correlated ActivityId",
out object correlatedActivityIds);
List <string> correlatedIdList = correlatedActivityIds.ToString().Split(',').ToList();
Assert.AreEqual(correlatedIdList.Count, 2);
Assert.AreEqual(correlatedIdList[0], firstCorrelatedActivityId.ToString());
}
}
internal static async Task <List <T> > QueryWithCosmosElementContinuationTokenAsync <T>(
Container container,
string query,
QueryRequestOptions queryRequestOptions = null)
{
if (queryRequestOptions == null)
{
queryRequestOptions = new QueryRequestOptions();
}
List <T> resultsFromCosmosElementContinuationToken = new List <T>();
CosmosElement continuationToken = null;
do
{
QueryRequestOptions computeRequestOptions = queryRequestOptions.Clone();
computeRequestOptions.ExecutionEnvironment = Cosmos.Query.Core.ExecutionContext.ExecutionEnvironment.Compute;
computeRequestOptions.CosmosElementContinuationToken = continuationToken;
FeedIteratorInternal <T> itemQuery = (FeedIteratorInternal <T>)container.GetItemQueryIterator <T>(
queryText: query,
requestOptions: computeRequestOptions);
try
{
FeedResponse <T> cosmosQueryResponse = await itemQuery.ReadNextAsync();
if (queryRequestOptions.MaxItemCount.HasValue)
{
Assert.IsTrue(
cosmosQueryResponse.Count <= queryRequestOptions.MaxItemCount.Value,
"Max Item Count is not being honored");
}
resultsFromCosmosElementContinuationToken.AddRange(cosmosQueryResponse);
continuationToken = itemQuery.GetCosmosElementContinuationToken();
}
catch (CosmosException cosmosException) when(cosmosException.StatusCode == (HttpStatusCode)429)
{
itemQuery = (FeedIteratorInternal <T>)container.GetItemQueryIterator <T>(
queryText: query,
requestOptions: queryRequestOptions);
}
} while (continuationToken != null);
return(resultsFromCosmosElementContinuationToken);
}
private async Task <(ChangeFeedProcessorState, ResponseMessage)> GetRemainingWorkAsync(
DocumentServiceLease existingLease,
ITrace trace,
CancellationToken cancellationToken)
{
using FeedIteratorInternal iterator = this.monitoredContainerFeedCreator(
existingLease,
existingLease.ContinuationToken,
string.IsNullOrEmpty(existingLease.ContinuationToken));
try
{
ResponseMessage response = await iterator.ReadNextAsync(trace, cancellationToken).ConfigureAwait(false);
if (response.StatusCode != HttpStatusCode.NotModified)
{
response.EnsureSuccessStatusCode();
}
long parsedLSNFromSessionToken = ChangeFeedEstimatorIterator.TryConvertToNumber(ExtractLsnFromSessionToken(response.Headers.Session));
IEnumerable <JObject> items = ChangeFeedEstimatorIterator.GetItemsFromResponse(response);
long lastQueryLSN = items.Any()
? ChangeFeedEstimatorIterator.TryConvertToNumber(ChangeFeedEstimatorIterator.GetFirstItemLSN(items)) - 1
: parsedLSNFromSessionToken;
if (lastQueryLSN < 0)
{
return(new ChangeFeedProcessorState(existingLease.CurrentLeaseToken, 1, existingLease.Owner), response);
}
long leaseTokenRemainingWork = parsedLSNFromSessionToken - lastQueryLSN;
long estimation = leaseTokenRemainingWork < 0 ? 0 : leaseTokenRemainingWork;
return(new ChangeFeedProcessorState(existingLease.CurrentLeaseToken, estimation, existingLease.Owner), response);
}
catch (Exception clientException)
{
Cosmos.Extensions.TraceException(clientException);
DefaultTrace.TraceWarning("GetEstimateWork > exception: lease token '{0}'", existingLease.CurrentLeaseToken);
throw;
}
}
public async Task ParallelizeQueryThroughTokens()
{
ContainerInternal container = null;
try
{
// Create a container large enough to have at least 2 partitions
ContainerResponse containerResponse = await this.database.CreateContainerAsync(
id : Guid.NewGuid().ToString(),
partitionKeyPath : "/id",
throughput : 15000);
container = (ContainerInlineCore)containerResponse;
List <string> generatedIds = Enumerable.Range(0, 1000).Select(n => $"BasicItem{n}").ToList();
foreach (string id in generatedIds)
{
string item = $@"
{{
""id"": ""{id}""
}}";
using (ResponseMessage createResponse = await container.CreateItemStreamAsync(
QueryFeedRangeTests.GenerateStreamFromString(item),
new Cosmos.PartitionKey(id)))
{
Assert.IsTrue(createResponse.IsSuccessStatusCode);
}
}
IReadOnlyList <FeedRange> feedTokens = await container.GetFeedRangesAsync();
Assert.IsTrue(feedTokens.Count > 1, " RUs of the container needs to be increased to ensure at least 2 partitions.");
List <Task <List <string> > > tasks = feedTokens.Select(async feedToken =>
{
List <string> results = new List <string>();
FeedIteratorInternal feedIterator = container.GetItemQueryStreamIterator(
queryDefinition: new QueryDefinition("select * from T where STARTSWITH(T.id, \"BasicItem\")"),
feedRange: feedToken,
continuationToken: null,
requestOptions: new QueryRequestOptions()
{
MaxItemCount = 10
}) as FeedIteratorInternal;
string continuation = null;
while (feedIterator.HasMoreResults)
{
using (ResponseMessage responseMessage =
await feedIterator.ReadNextAsync(this.cancellationToken))
{
if (responseMessage.IsSuccessStatusCode)
{
using (StreamReader reader = new StreamReader(responseMessage.Content))
{
string json = await reader.ReadToEndAsync();
JArray documents = (JArray)JObject.Parse(json).SelectToken("Documents");
foreach (JObject document in documents)
{
results.Add(document.SelectToken("id").ToString());
}
}
}
continuation = responseMessage.ContinuationToken;
break;
}
}
feedIterator = container.GetItemQueryStreamIterator(queryDefinition: new QueryDefinition("select * from T where STARTSWITH(T.id, \"BasicItem\")"), feedRange: feedToken, continuationToken: continuation, requestOptions: new QueryRequestOptions()
{
MaxItemCount = 10
}) as FeedIteratorInternal;
while (feedIterator.HasMoreResults)
{
using (ResponseMessage responseMessage =
await feedIterator.ReadNextAsync(this.cancellationToken))
{
if (responseMessage.IsSuccessStatusCode)
{
using (StreamReader reader = new StreamReader(responseMessage.Content))
{
string json = await reader.ReadToEndAsync();
JArray documents = (JArray)JObject.Parse(json).SelectToken("Documents");
foreach (JObject document in documents)
{
results.Add(document.SelectToken("id").ToString());
}
}
}
}
}
return(results);
}).ToList();
await Task.WhenAll(tasks);
CollectionAssert.AreEquivalent(generatedIds, tasks.SelectMany(t => t.Result).ToList());
}
finally
{
await container?.DeleteContainerAsync();
}
}
