internal static async Task <List <T> > QueryWithContinuationTokensAsync <T>(
Container container,
string query,
QueryRequestOptions queryRequestOptions = null)
{
if (queryRequestOptions == null)
{
queryRequestOptions = new QueryRequestOptions();
}
List <T> resultsFromContinuationToken = new List <T>();
string continuationToken = null;
do
{
FeedIterator <T> itemQuery = container.GetItemQueryIterator <T>(
queryText: query,
requestOptions: queryRequestOptions,
continuationToken: continuationToken);
try
{
while (true)
{
try
{
FeedResponse <T> cosmosQueryResponse = await itemQuery.ReadNextAsync();
if (queryRequestOptions.MaxItemCount.HasValue)
{
Assert.IsTrue(
cosmosQueryResponse.Count <= queryRequestOptions.MaxItemCount.Value,
"Max Item Count is not being honored");
}
resultsFromContinuationToken.AddRange(cosmosQueryResponse);
continuationToken = cosmosQueryResponse.ContinuationToken;
break;
}
catch (CosmosException cosmosException) when(cosmosException.StatusCode == (HttpStatusCode)429)
{
itemQuery.Dispose();
itemQuery = container.GetItemQueryIterator <T>(
queryText: query,
requestOptions: queryRequestOptions,
continuationToken: continuationToken);
}
}
}
finally
{
itemQuery.Dispose();
}
} while (continuationToken != null);
return(resultsFromContinuationToken);
}
protected virtual void Dispose(bool disposing)
{
if (disposing)
{
_feedIterator?.Dispose();
}
}
private static async Task <List <Application> > GetMatchesAsync(FeedIterator <Application> query)
{
List <Application> applications = new List <Application>();
while (query.HasMoreResults)
{
FeedResponse <Application> response = await query.ReadNextAsync();
applications.AddRange(response);
}
query.Dispose();
PostProcess(applications);
return(applications);
}
/// <summary>
/// Executes the query and does paging using continuations.
/// </summary>
private async static Task ExecuteQueryWithContinuationAndGatherResultsAsync(CTLConfig config,
CosmosClient cosmosClient,
ILogger logger,
IMetrics metrics,
string loggingContextIdentifier,
CancellationToken cancellationToken,
string queryText,
string queryName,
long expectedResults,
SemaphoreSlim concurrencyControlSemaphore)
{
Stopwatch stopWatch = Stopwatch.StartNew();
GaugeOptions documentGauge = new GaugeOptions
{
Name = $"#{queryName} Query with CT received documents",
MeasurementUnit = Unit.Items,
Context = loggingContextIdentifier
};
Container container = cosmosClient.GetContainer(config.Database, config.Collection);
while (stopWatch.Elapsed <= config.RunningTimeDurationAsTimespan &&
!cancellationToken.IsCancellationRequested)
{
// To really debug what happened on the query, having a list of all continuations would be useful.
List <string> allContinuations = new List <string>();
long documentTotal = 0;
string continuation;
await concurrencyControlSemaphore.WaitAsync();
FeedIterator <Dictionary <string, string> > query = container.GetItemQueryIterator <Dictionary <string, string> >(queryText);
try
{
while (query.HasMoreResults)
{
FeedResponse <Dictionary <string, string> > response = await query.ReadNextAsync();
documentTotal += response.Count;
continuation = response.ContinuationToken;
allContinuations.Add(continuation);
if (continuation != null)
{
// Use continuation to paginate on the query instead of draining just the initial query
// This validates that we can indeed move forward with the continuation
query.Dispose();
query = container.GetItemQueryIterator <Dictionary <string, string> >(queryText, continuation);
}
}
query.Dispose();
metrics.Measure.Gauge.SetValue(documentGauge, documentTotal);
if (expectedResults > 0 && expectedResults != documentTotal)
{
StringBuilder errorDetail = new StringBuilder();
errorDetail.AppendLine($"{queryName} Query with CT expected to read {expectedResults} but got {documentTotal}");
foreach (string c in allContinuations)
{
errorDetail.AppendLine($"Continuation: {c}");
}
logger.LogError(errorDetail.ToString());
}
}
catch (CosmosException ce) when(ce.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
{
//Logging 429s is not relevant
}
catch (Exception ex)
{
metrics.Measure.Gauge.SetValue(documentGauge, documentTotal);
StringBuilder errorDetail = new StringBuilder();
errorDetail.AppendLine($"{queryName} Query with CT failure while looping through query.");
foreach (string c in allContinuations)
{
errorDetail.AppendLine($"Continuation: {c}");
}
logger.LogError(ex, errorDetail.ToString());
}
finally
{
concurrencyControlSemaphore.Release();
}
}
stopWatch.Stop();
}
internal static async Task <List <T> > QueryWithoutContinuationTokensAsync <T>(
Container container,
string query,
QueryRequestOptions queryRequestOptions = null)
{
if (queryRequestOptions == null)
{
queryRequestOptions = new QueryRequestOptions();
}
List <T> results = new List <T>();
FeedIterator <T> itemQuery = container.GetItemQueryIterator <T>(
queryText: query,
requestOptions: queryRequestOptions);
try
{
string continuationTokenForRetries = null;
while (itemQuery.HasMoreResults)
{
try
{
FeedResponse <T> page = await itemQuery.ReadNextAsync();
results.AddRange(page);
if (queryRequestOptions.MaxItemCount.HasValue)
{
Assert.IsTrue(
page.Count <= queryRequestOptions.MaxItemCount.Value,
"Max Item Count is not being honored");
}
try
{
continuationTokenForRetries = page.ContinuationToken;
}
catch (Exception)
{
// Grabbing a continuation token is not supported on all queries.
}
}
catch (CosmosException cosmosException) when(cosmosException.StatusCode == (HttpStatusCode)429)
{
itemQuery.Dispose();
itemQuery = container.GetItemQueryIterator <T>(
queryText: query,
requestOptions: queryRequestOptions,
continuationToken: continuationTokenForRetries);
if (continuationTokenForRetries == null)
{
// The query failed and we don't have a save point, so just restart the whole thing.
results = new List <T>();
}
}
}
}
finally
{
itemQuery.Dispose();
}
return(results);
}