public static Task <PartitionedQueryExecutionInfo> GetQueryPlanThroughGatewayAsync(
CosmosQueryContext queryContext,
SqlQuerySpec sqlQuerySpec,
Uri resourceLink,
PartitionKey?partitionKey,
CancellationToken cancellationToken = default)
{
if (queryContext == null)
{
throw new ArgumentNullException(nameof(queryContext));
}
if (sqlQuerySpec == null)
{
throw new ArgumentNullException(nameof(sqlQuerySpec));
}
if (resourceLink == null)
{
throw new ArgumentNullException(nameof(resourceLink));
}
cancellationToken.ThrowIfCancellationRequested();
return(queryContext.ExecuteQueryPlanRequestAsync(
resourceLink,
ResourceType.Document,
OperationType.QueryPlan,
sqlQuerySpec,
partitionKey,
QueryPlanRetriever.SupportedQueryFeaturesString,
cancellationToken));
}
/// <summary>
/// Callback to create a child document producer tree based on the partition key range.
/// </summary>
/// <param name="queryContext">request context</param>
/// <param name="querySpecForInit">query spec for initialization</param>
/// <param name="produceAsyncCompleteCallback">Callback to invoke once a fetch finishes.</param>
/// <param name="itemProducerTreeComparer">Comparer to determine, which tree to produce from.</param>
/// <param name="equalityComparer">Comparer to see if we need to return the continuation token for a partition.</param>
/// <param name="testFlags">Test flags.</param>
/// <param name="deferFirstPage">Whether or not to defer fetching the first page.</param>
/// <param name="collectionRid">The collection to drain from.</param>
/// <param name="initialPageSize">The initial page size.</param>
/// <returns>A function that given a partition key range and continuation token will create a document producer.</returns>
private static Func <Documents.PartitionKeyRange, string, ItemProducerTree> CreateItemProducerTreeCallback(
CosmosQueryContext queryContext,
SqlQuerySpec querySpecForInit,
ProduceAsyncCompleteDelegate produceAsyncCompleteCallback,
IComparer <ItemProducerTree> itemProducerTreeComparer,
IEqualityComparer <CosmosElement> equalityComparer,
TestInjections testFlags,
bool deferFirstPage,
string collectionRid,
long initialPageSize = 50)
{
return((partitionKeyRange, continuationToken) =>
{
return new ItemProducerTree(
queryContext,
querySpecForInit,
partitionKeyRange,
produceAsyncCompleteCallback,
itemProducerTreeComparer,
equalityComparer,
testFlags,
deferFirstPage,
collectionRid,
initialPageSize,
continuationToken);
});
}
public static CosmosQueryExecutionContext Create(
CosmosQueryContext cosmosQueryContext,
InputParameters inputParameters)
{
if (cosmosQueryContext == null)
{
throw new ArgumentNullException(nameof(cosmosQueryContext));
}
if (inputParameters == null)
{
throw new ArgumentNullException(nameof(inputParameters));
}
CosmosQueryExecutionContextWithNameCacheStaleRetry cosmosQueryExecutionContextWithNameCacheStaleRetry = new CosmosQueryExecutionContextWithNameCacheStaleRetry(
cosmosQueryContext: cosmosQueryContext,
cosmosQueryExecutionContextFactory: () =>
{
// Query Iterator requires that the creation of the query context is deferred until the user calls ReadNextAsync
AsyncLazy <TryCatch <CosmosQueryExecutionContext> > lazyTryCreateCosmosQueryExecutionContext = new AsyncLazy <TryCatch <CosmosQueryExecutionContext> >(valueFactory: (innerCancellationToken) =>
{
innerCancellationToken.ThrowIfCancellationRequested();
return(CosmosQueryExecutionContextFactory.TryCreateCoreContextAsync(
cosmosQueryContext,
inputParameters,
innerCancellationToken));
});
LazyCosmosQueryExecutionContext lazyCosmosQueryExecutionContext = new LazyCosmosQueryExecutionContext(lazyTryCreateCosmosQueryExecutionContext);
return(lazyCosmosQueryExecutionContext);
});
CatchAllCosmosQueryExecutionContext catchAllCosmosQueryExecutionContext = new CatchAllCosmosQueryExecutionContext(cosmosQueryExecutionContextWithNameCacheStaleRetry);
return(catchAllCosmosQueryExecutionContext);
}
private static Task <TryCatch <CosmosQueryExecutionContext> > TryCreatePassthroughQueryExecutionContextAsync(
CosmosQueryContext cosmosQueryContext,
InputParameters inputParameters,
PartitionedQueryExecutionInfo partitionedQueryExecutionInfo,
List <Documents.PartitionKeyRange> targetRanges,
string collectionRid,
CancellationToken cancellationToken)
{
CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams initParams = new CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams(
sqlQuerySpec: inputParameters.SqlQuerySpec,
collectionRid: collectionRid,
partitionedQueryExecutionInfo: partitionedQueryExecutionInfo,
partitionKeyRanges: targetRanges,
initialPageSize: inputParameters.MaxItemCount,
maxConcurrency: inputParameters.MaxConcurrency,
maxItemCount: inputParameters.MaxItemCount,
maxBufferedItemCount: inputParameters.MaxBufferedItemCount,
returnResultsInDeterministicOrder: inputParameters.ReturnResultsInDeterministicOrder,
testSettings: inputParameters.TestInjections);
return(PipelinedDocumentQueryExecutionContext.TryCreatePassthroughAsync(
inputParameters.ExecutionEnvironment,
cosmosQueryContext,
initParams,
inputParameters.InitialUserContinuationToken,
cancellationToken));
}
/// <summary>
/// Initializes a new instance of the ItemProducer class.
/// </summary>
/// <param name="queryContext">request context</param>
/// <param name="querySpecForInit">query spec for initialization</param>
/// <param name="partitionKeyRange">The partition key range.</param>
/// <param name="produceAsyncCompleteCallback">The callback to call once you are done fetching.</param>
/// <param name="equalityComparer">The comparer to use to determine whether the producer has seen a new document.</param>
/// <param name="testFlags">Flags used to help faciliate testing.</param>
/// <param name="initialPageSize">The initial page size.</param>
/// <param name="initialContinuationToken">The initial continuation token.</param>
public ItemProducer(
CosmosQueryContext queryContext,
SqlQuerySpec querySpecForInit,
PartitionKeyRange partitionKeyRange,
ProduceAsyncCompleteDelegate produceAsyncCompleteCallback,
IEqualityComparer <CosmosElement> equalityComparer,
TestInjections testFlags,
long initialPageSize = 50,
string initialContinuationToken = null)
{
this.bufferedPages = new AsyncCollection <QueryResponseCore>();
// We use a binary semaphore to get the behavior of a mutex,
// since fetching documents from the backend using a continuation token is a critical section.
this.fetchSemaphore = new SemaphoreSlim(1, 1);
this.queryContext = queryContext;
this.querySpecForInit = querySpecForInit;
this.PartitionKeyRange = partitionKeyRange ?? throw new ArgumentNullException(nameof(partitionKeyRange));
this.produceAsyncCompleteCallback = produceAsyncCompleteCallback ?? throw new ArgumentNullException(nameof(produceAsyncCompleteCallback));
this.equalityComparer = equalityComparer ?? throw new ArgumentNullException(nameof(equalityComparer));
this.pageSize = initialPageSize;
this.CurrentContinuationToken = initialContinuationToken;
this.BackendContinuationToken = initialContinuationToken;
this.PreviousContinuationToken = initialContinuationToken;
if (!string.IsNullOrEmpty(initialContinuationToken))
{
this.hasStartedFetching = true;
this.IsActive = true;
}
this.testFlags = testFlags;
this.HasMoreResults = true;
}
public static IQueryPipelineStage Create(
DocumentContainer documentContainer,
CosmosQueryContext cosmosQueryContext,
InputParameters inputParameters,
ITrace trace)
{
if (cosmosQueryContext == null)
{
throw new ArgumentNullException(nameof(cosmosQueryContext));
}
if (inputParameters == null)
{
throw new ArgumentNullException(nameof(inputParameters));
}
if (trace == null)
{
throw new ArgumentNullException(nameof(trace));
}
NameCacheStaleRetryQueryPipelineStage nameCacheStaleRetryQueryPipelineStage = new NameCacheStaleRetryQueryPipelineStage(
cosmosQueryContext: cosmosQueryContext,
queryPipelineStageFactory: () =>
{
// Query Iterator requires that the creation of the query context is deferred until the user calls ReadNextAsync
AsyncLazy <TryCatch <IQueryPipelineStage> > lazyTryCreateStage = new AsyncLazy <TryCatch <IQueryPipelineStage> >(
valueFactory: (trace, innerCancellationToken) => CosmosQueryExecutionContextFactory.TryCreateCoreContextAsync(
documentContainer,
cosmosQueryContext,
inputParameters,
trace,
innerCancellationToken));
LazyQueryPipelineStage lazyQueryPipelineStage = new LazyQueryPipelineStage(lazyTryCreateStage: lazyTryCreateStage, cancellationToken: default);
public static async Task <TryCatch <CosmosParallelItemQueryExecutionContext> > TryCreateAsync(
CosmosQueryContext queryContext,
CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams initParams,
string requestContinuationToken,
CancellationToken cancellationToken)
{
Debug.Assert(
!initParams.PartitionedQueryExecutionInfo.QueryInfo.HasOrderBy,
"Parallel~Context must not have order by query info.");
cancellationToken.ThrowIfCancellationRequested();
CosmosParallelItemQueryExecutionContext context = new CosmosParallelItemQueryExecutionContext(
queryContext: queryContext,
maxConcurrency: initParams.MaxConcurrency,
maxItemCount: initParams.MaxItemCount,
maxBufferedItemCount: initParams.MaxBufferedItemCount,
testSettings: initParams.TestSettings);
return(await context.TryInitializeAsync(
sqlQuerySpec : initParams.SqlQuerySpec,
collectionRid : initParams.CollectionRid,
partitionKeyRanges : initParams.PartitionKeyRanges,
initialPageSize : initParams.InitialPageSize,
requestContinuation : requestContinuationToken,
cancellationToken : cancellationToken));
}
public QueryingEnumerable(
CosmosQueryContext cosmosQueryContext,
ISqlExpressionFactory sqlExpressionFactory,
IQuerySqlGeneratorFactory querySqlGeneratorFactory,
SelectExpression selectExpression,
Func <CosmosQueryContext, JObject, T> shaper,
Type contextType,
string partitionKeyFromExtension,
bool standAloneStateManager,
bool threadSafetyChecksEnabled)
{
_cosmosQueryContext = cosmosQueryContext;
_sqlExpressionFactory = sqlExpressionFactory;
_querySqlGeneratorFactory = querySqlGeneratorFactory;
_selectExpression = selectExpression;
_shaper = shaper;
_contextType = contextType;
_queryLogger = cosmosQueryContext.QueryLogger;
_standAloneStateManager = standAloneStateManager;
_threadSafetyChecksEnabled = threadSafetyChecksEnabled;
var partitionKey = selectExpression.GetPartitionKey(cosmosQueryContext.ParameterValues);
if (partitionKey != null && partitionKeyFromExtension != null && partitionKeyFromExtension != partitionKey)
{
throw new InvalidOperationException(CosmosStrings.PartitionKeyMismatch(partitionKeyFromExtension, partitionKey));
}
_partitionKey = partitionKey ?? partitionKeyFromExtension;
}
/// <summary>
/// Initializes a new instance of the CosmosCrossPartitionQueryExecutionContext class.
/// </summary>
/// <param name="queryContext">Constructor parameters for the base class.</param>
/// <param name="maxConcurrency">The max concurrency</param>
/// <param name="maxBufferedItemCount">The max buffered item count</param>
/// <param name="maxItemCount">Max item count</param>
/// <param name="moveNextComparer">Comparer used to figure out that document producer tree to serve documents from next.</param>
/// <param name="fetchPrioirtyFunction">The priority function to determine which partition to fetch documents from next.</param>
/// <param name="equalityComparer">Used to determine whether we need to return the continuation token for a partition.</param>
/// <param name="returnResultsInDeterministicOrder">Whether or not to return results in deterministic order.</param>
/// <param name="testSettings">Test settings.</param>
protected CosmosCrossPartitionQueryExecutionContext(
CosmosQueryContext queryContext,
int?maxConcurrency,
int?maxItemCount,
int?maxBufferedItemCount,
IComparer <ItemProducerTree> moveNextComparer,
Func <ItemProducerTree, int> fetchPrioirtyFunction,
IEqualityComparer <CosmosElement> equalityComparer,
bool returnResultsInDeterministicOrder,
TestInjections testSettings)
{
if (moveNextComparer == null)
{
throw new ArgumentNullException(nameof(moveNextComparer));
}
this.queryContext = queryContext ?? throw new ArgumentNullException(nameof(queryContext));
this.queryClient = queryContext.QueryClient ?? throw new ArgumentNullException(nameof(queryContext.QueryClient));
this.itemProducerForest = new PriorityQueue <ItemProducerTree>(moveNextComparer, isSynchronized: true);
this.fetchPrioirtyFunction = fetchPrioirtyFunction ?? throw new ArgumentNullException(nameof(fetchPrioirtyFunction));
this.comparableTaskScheduler = new ComparableTaskScheduler(maxConcurrency.GetValueOrDefault(0));
this.equalityComparer = equalityComparer ?? throw new ArgumentNullException(nameof(equalityComparer));
this.testSettings = testSettings;
this.requestChargeTracker = new RequestChargeTracker();
this.diagnosticsPages = new ConcurrentBag <QueryPageDiagnostics>();
this.actualMaxPageSize = maxItemCount.GetValueOrDefault(ParallelQueryConfig.GetConfig().ClientInternalMaxItemCount);
if (this.actualMaxPageSize < 0)
{
throw new ArgumentOutOfRangeException("actualMaxPageSize should never be less than 0");
}
if (this.actualMaxPageSize > int.MaxValue)
{
throw new ArgumentOutOfRangeException("actualMaxPageSize should never be greater than int.MaxValue");
}
if (maxBufferedItemCount.HasValue)
{
this.actualMaxBufferedItemCount = maxBufferedItemCount.Value;
}
else
{
this.actualMaxBufferedItemCount = ParallelQueryConfig.GetConfig().DefaultMaximumBufferSize;
}
if (this.actualMaxBufferedItemCount < 0)
{
throw new ArgumentOutOfRangeException("actualMaxBufferedItemCount should never be less than 0");
}
if (this.actualMaxBufferedItemCount > int.MaxValue)
{
throw new ArgumentOutOfRangeException("actualMaxBufferedItemCount should never be greater than int.MaxValue");
}
this.CanPrefetch = maxConcurrency.HasValue && maxConcurrency.Value != 0;
this.returnResultsInDeterministicOrder = returnResultsInDeterministicOrder;
}
public NameCacheStaleRetryQueryPipelineStage(
CosmosQueryContext cosmosQueryContext,
Func <IQueryPipelineStage> queryPipelineStageFactory)
{
this.cosmosQueryContext = cosmosQueryContext ?? throw new ArgumentNullException(nameof(cosmosQueryContext));
this.queryPipelineStageFactory = queryPipelineStageFactory ?? throw new ArgumentNullException(nameof(queryPipelineStageFactory));
this.currentQueryPipelineStage = queryPipelineStageFactory();
}
public NetworkAttachedDocumentContainer(
ContainerCore container,
CosmosQueryContext cosmosQueryContext)
{
this.container = container ?? throw new ArgumentNullException(nameof(container));
this.cosmosQueryContext = cosmosQueryContext ?? throw new ArgumentNullException(nameof(cosmosQueryContext));
this.executeQueryBasedOnFeedRangeVisitor = new ExecuteQueryBasedOnFeedRangeVisitor(this);
}
public CosmosQueryExecutionContextWithNameCacheStaleRetry(
CosmosQueryContext cosmosQueryContext,
Func <CosmosQueryExecutionContext> cosmosQueryExecutionContextFactory)
{
this.cosmosQueryContext = cosmosQueryContext ?? throw new ArgumentNullException(nameof(cosmosQueryContext));
this.cosmosQueryExecutionContextFactory = cosmosQueryExecutionContextFactory ?? throw new ArgumentNullException(nameof(cosmosQueryExecutionContextFactory));
this.currentCosmosQueryExecutionContext = cosmosQueryExecutionContextFactory();
}
private static async Task <TryCatch <CosmosQueryExecutionContext> > TryCreateSpecializedDocumentQueryExecutionContextAsync(
CosmosQueryContext cosmosQueryContext,
InputParameters inputParameters,
PartitionedQueryExecutionInfo partitionedQueryExecutionInfo,
List <Documents.PartitionKeyRange> targetRanges,
string collectionRid,
CancellationToken cancellationToken)
{
QueryInfo queryInfo = partitionedQueryExecutionInfo.QueryInfo;
bool getLazyFeedResponse = queryInfo.HasTop;
// We need to compute the optimal initial page size for order-by queries
long optimalPageSize = inputParameters.MaxItemCount;
if (queryInfo.HasOrderBy)
{
int top;
if (queryInfo.HasTop && (top = partitionedQueryExecutionInfo.QueryInfo.Top.Value) > 0)
{
// All partitions should initially fetch about 1/nth of the top value.
long pageSizeWithTop = (long)Math.Min(
Math.Ceiling(top / (double)targetRanges.Count) * CosmosQueryExecutionContextFactory.PageSizeFactorForTop,
top);
optimalPageSize = Math.Min(pageSizeWithTop, optimalPageSize);
}
else if (cosmosQueryContext.IsContinuationExpected)
{
optimalPageSize = (long)Math.Min(
Math.Ceiling(optimalPageSize / (double)targetRanges.Count) * CosmosQueryExecutionContextFactory.PageSizeFactorForTop,
optimalPageSize);
}
}
Debug.Assert(
(optimalPageSize > 0) && (optimalPageSize <= int.MaxValue),
$"Invalid MaxItemCount {optimalPageSize}");
CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams initParams = new CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams(
sqlQuerySpec: inputParameters.SqlQuerySpec,
collectionRid: collectionRid,
partitionedQueryExecutionInfo: partitionedQueryExecutionInfo,
partitionKeyRanges: targetRanges,
initialPageSize: (int)optimalPageSize,
maxConcurrency: inputParameters.MaxConcurrency,
maxItemCount: inputParameters.MaxItemCount,
maxBufferedItemCount: inputParameters.MaxBufferedItemCount,
returnResultsInDeterministicOrder: inputParameters.ReturnResultsInDeterministicOrder,
testSettings: inputParameters.TestInjections);
return(await PipelinedDocumentQueryExecutionContext.TryCreateAsync(
inputParameters.ExecutionEnvironment,
cosmosQueryContext,
initParams,
inputParameters.InitialUserContinuationToken,
cancellationToken));
}
public static async Task <TryCatch <CosmosQueryExecutionContext> > TryCreatePassthroughAsync(
ExecutionEnvironment executionEnvironment,
CosmosQueryContext queryContext,
CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams initParams,
CosmosElement requestContinuationToken,
CancellationToken cancellationToken)
{
if (queryContext == null)
{
throw new ArgumentNullException(nameof(queryContext));
}
cancellationToken.ThrowIfCancellationRequested();
// Modify query plan
PartitionedQueryExecutionInfo passThroughQueryInfo = new PartitionedQueryExecutionInfo()
{
QueryInfo = new QueryInfo()
{
Aggregates = null,
DistinctType = DistinctQueryType.None,
GroupByAliases = null,
GroupByAliasToAggregateType = null,
GroupByExpressions = null,
HasSelectValue = false,
Limit = null,
Offset = null,
OrderBy = null,
OrderByExpressions = null,
RewrittenQuery = null,
Top = null,
},
QueryRanges = initParams.PartitionedQueryExecutionInfo.QueryRanges,
};
initParams = new CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams(
sqlQuerySpec: initParams.SqlQuerySpec,
collectionRid: initParams.CollectionRid,
partitionedQueryExecutionInfo: passThroughQueryInfo,
partitionKeyRanges: initParams.PartitionKeyRanges,
initialPageSize: initParams.MaxItemCount.GetValueOrDefault(1000),
maxConcurrency: initParams.MaxConcurrency,
maxItemCount: initParams.MaxItemCount,
maxBufferedItemCount: initParams.MaxBufferedItemCount,
returnResultsInDeterministicOrder: initParams.ReturnResultsInDeterministicOrder,
testSettings: initParams.TestSettings);
// Return a parallel context, since we still want to be able to handle splits and concurrency / buffering.
return((await CosmosParallelItemQueryExecutionContext.TryCreateAsync(
queryContext,
initParams,
requestContinuationToken,
cancellationToken))
.Try <CosmosQueryExecutionContext>((source) => new PipelinedDocumentQueryExecutionContext(
source,
initParams.InitialPageSize)));
}
public async Task TestMoveNextWithEmptyPagesAndSplitAsync(string initialContinuationToken)
{
int maxPageSize = 5;
List <MockPartitionResponse[]> mockResponsesScenario = MockQueryFactory.GetSplitScenarios();
foreach (MockPartitionResponse[] mockResponse in mockResponsesScenario)
{
Mock <CosmosQueryClient> mockQueryClient = new Mock <CosmosQueryClient>();
IList <ToDoItem> allItems = MockQueryFactory.GenerateAndMockResponse(
mockQueryClient,
isOrderByQuery: false,
sqlQuerySpec: MockQueryFactory.DefaultQuerySpec,
containerRid: MockQueryFactory.DefaultCollectionRid,
initContinuationToken: initialContinuationToken,
maxPageSize: maxPageSize,
mockResponseForSinglePartition: mockResponse,
cancellationTokenForMocks: this.cancellationToken);
CosmosQueryContext context = MockQueryFactory.CreateContext(
mockQueryClient.Object);
ItemProducerTree itemProducerTree = new ItemProducerTree(
context,
MockQueryFactory.DefaultQuerySpec,
mockResponse[0].PartitionKeyRange,
MockItemProducerFactory.DefaultTreeProduceAsyncCompleteDelegate,
new ParallelItemProducerTreeComparer(),
CosmosElementEqualityComparer.Value,
true,
MockQueryFactory.DefaultCollectionRid,
maxPageSize,
initialContinuationToken: initialContinuationToken);
Assert.IsTrue(itemProducerTree.HasMoreResults);
List <ToDoItem> itemsRead = new List <ToDoItem>();
while ((await itemProducerTree.MoveNextAsync(this.cancellationToken)).successfullyMovedNext)
{
Assert.IsTrue(itemProducerTree.HasMoreResults);
if (itemProducerTree.Current != null)
{
string jsonValue = itemProducerTree.Current.ToString();
ToDoItem item = JsonConvert.DeserializeObject <ToDoItem>(jsonValue);
itemsRead.Add(item);
}
}
Assert.IsFalse(itemProducerTree.HasMoreResults);
Assert.AreEqual(allItems.Count, itemsRead.Count);
List <ToDoItem> exepected = allItems.OrderBy(x => x.id).ToList();
List <ToDoItem> actual = itemsRead.OrderBy(x => x.id).ToList();
CollectionAssert.AreEqual(exepected, actual, new ToDoItemComparer());
}
}
public async Task TestMoveNextWithEmptyPagesAsync(string initialContinuationToken)
{
int maxPageSize = 5;
List <MockPartitionResponse[]> mockResponses = MockQueryFactory.GetAllCombinationWithEmptyPage();
foreach (MockPartitionResponse[] mockResponse in mockResponses)
{
Mock <CosmosQueryClient> mockQueryClient = new Mock <CosmosQueryClient>();
IList <ToDoItem> allItems = MockQueryFactory.GenerateAndMockResponse(
mockQueryClient,
isOrderByQuery: false,
sqlQuerySpec: MockQueryFactory.DefaultQuerySpec,
containerRid: MockQueryFactory.DefaultCollectionRid,
initContinuationToken: initialContinuationToken,
maxPageSize: maxPageSize,
mockResponseForSinglePartition: mockResponse,
cancellationTokenForMocks: this.cancellationToken);
CosmosQueryContext context = MockQueryFactory.CreateContext(
mockQueryClient.Object);
ItemProducerTree itemProducerTree = new ItemProducerTree(
queryContext: context,
querySpecForInit: MockQueryFactory.DefaultQuerySpec,
partitionKeyRange: mockResponse[0].PartitionKeyRange,
produceAsyncCompleteCallback: MockItemProducerFactory.DefaultTreeProduceAsyncCompleteDelegate,
itemProducerTreeComparer: new ParallelItemProducerTreeComparer(),
equalityComparer: CosmosElementEqualityComparer.Value,
testSettings: new TestInjections(simulate429s: false, simulateEmptyPages: false),
deferFirstPage: true,
collectionRid: MockQueryFactory.DefaultCollectionRid,
initialPageSize: maxPageSize,
initialContinuationToken: initialContinuationToken);
Assert.IsTrue(itemProducerTree.HasMoreResults);
List <ToDoItem> itemsRead = new List <ToDoItem>();
while ((await itemProducerTree.TryMoveNextPageAsync(this.cancellationToken)).movedToNextPage)
{
while (itemProducerTree.TryMoveNextDocumentWithinPage())
{
Assert.IsTrue(itemProducerTree.HasMoreResults);
string jsonValue = itemProducerTree.Current.ToString();
ToDoItem item = JsonConvert.DeserializeObject <ToDoItem>(jsonValue);
itemsRead.Add(item);
}
}
Assert.IsFalse(itemProducerTree.HasMoreResults);
Assert.AreEqual(allItems.Count, itemsRead.Count);
CollectionAssert.AreEqual(itemsRead, allItems.ToList(), new ToDoItemComparer());
}
}
internal IDocumentQuery <T> CreateDocumentQuery <T>(
SqlQuerySpec sqlQuerySpec,
FeedOptions feedOptions = null)
{
EnsureArg.IsNotNull(sqlQuerySpec, nameof(sqlQuerySpec));
var context = new CosmosQueryContext(_collectionUri, sqlQuerySpec, feedOptions);
return(_cosmosDocumentQueryFactory.Create <T>(_documentClient.Value, context));
}
public Enumerator(QueryingEnumerable <T> queryingEnumerable)
{
_cosmosQueryContext = queryingEnumerable._cosmosQueryContext;
_shaper = queryingEnumerable._shaper;
_selectExpression = queryingEnumerable._selectExpression;
_sqlExpressionFactory = queryingEnumerable._sqlExpressionFactory;
_querySqlGeneratorFactory = queryingEnumerable._querySqlGeneratorFactory;
_contextType = queryingEnumerable._contextType;
_logger = queryingEnumerable._logger;
}
public AsyncEnumerator(AsyncQueryingEnumerable <T> queryingEnumerable, CancellationToken cancellationToken)
{
_cosmosQueryContext = queryingEnumerable._cosmosQueryContext;
_shaper = queryingEnumerable._shaper;
_selectExpression = queryingEnumerable._selectExpression;
_sqlExpressionFactory = queryingEnumerable._sqlExpressionFactory;
_querySqlGeneratorFactory = queryingEnumerable._querySqlGeneratorFactory;
_contextType = queryingEnumerable._contextType;
_logger = queryingEnumerable._logger;
_cancellationToken = cancellationToken;
}
public static async Task <TryCatch <IDocumentQueryExecutionComponent> > TryCreateAsync(
CosmosQueryContext queryContext,
CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams initParams,
CosmosElement requestContinuationToken,
CancellationToken cancellationToken)
{
Debug.Assert(
initParams.PartitionedQueryExecutionInfo.QueryInfo.HasOrderBy,
"OrderBy~Context must have order by query info.");
if (queryContext == null)
{
throw new ArgumentNullException(nameof(queryContext));
}
cancellationToken.ThrowIfCancellationRequested();
// TODO (brchon): For now we are not honoring non deterministic ORDER BY queries, since there is a bug in the continuation logic.
// We can turn it back on once the bug is fixed.
// This shouldn't hurt any query results.
OrderByItemProducerTreeComparer orderByItemProducerTreeComparer = new OrderByItemProducerTreeComparer(initParams.PartitionedQueryExecutionInfo.QueryInfo.OrderBy.ToArray());
CosmosOrderByItemQueryExecutionContext context = new CosmosOrderByItemQueryExecutionContext(
initPararms: queryContext,
maxConcurrency: initParams.MaxConcurrency,
maxItemCount: initParams.MaxItemCount,
maxBufferedItemCount: initParams.MaxBufferedItemCount,
consumeComparer: orderByItemProducerTreeComparer,
testSettings: initParams.TestSettings);
IReadOnlyList <string> orderByExpressions = initParams.PartitionedQueryExecutionInfo.QueryInfo.OrderByExpressions;
IReadOnlyList <SortOrder> sortOrders = initParams.PartitionedQueryExecutionInfo.QueryInfo.OrderBy;
if (orderByExpressions.Count != sortOrders.Count)
{
throw new ArgumentException("order by expressions count does not match sort order");
}
IReadOnlyList <OrderByColumn> columns = orderByExpressions
.Zip(sortOrders, (expression, order) => new OrderByColumn(expression, order))
.ToList();
return((await context.TryInitializeAsync(
sqlQuerySpec: initParams.SqlQuerySpec,
requestContinuation: requestContinuationToken,
collectionRid: initParams.CollectionRid,
partitionKeyRanges: initParams.PartitionKeyRanges,
initialPageSize: initParams.InitialPageSize,
orderByColumns: columns,
cancellationToken: cancellationToken))
.Try <IDocumentQueryExecutionComponent>(() => context));
}
public Enumerator(QueryingEnumerable <T> queryingEnumerable)
{
_queryingEnumerable = queryingEnumerable;
_cosmosQueryContext = queryingEnumerable._cosmosQueryContext;
_shaper = queryingEnumerable._shaper;
_selectExpression = queryingEnumerable._selectExpression;
_contextType = queryingEnumerable._contextType;
_partitionKey = queryingEnumerable._partitionKey;
_queryLogger = queryingEnumerable._queryLogger;
_standAloneStateManager = queryingEnumerable._standAloneStateManager;
_concurrencyDetector = queryingEnumerable._threadSafetyChecksEnabled
? _cosmosQueryContext.ConcurrencyDetector
: null;
}
public Enumerator(ReadItemQueryingEnumerable <T> readItemEnumerable, CancellationToken cancellationToken = default)
{
_cosmosQueryContext = readItemEnumerable._cosmosQueryContext;
_readItemExpression = readItemEnumerable._readItemExpression;
_shaper = readItemEnumerable._shaper;
_contextType = readItemEnumerable._contextType;
_queryLogger = readItemEnumerable._queryLogger;
_standAloneStateManager = readItemEnumerable._standAloneStateManager;
_readItemEnumerable = readItemEnumerable;
_cancellationToken = cancellationToken;
_concurrencyDetector = readItemEnumerable._threadSafetyChecksEnabled
? _cosmosQueryContext.ConcurrencyDetector
: null;
}
/// <inheritdoc />
public ICosmosQuery <T> Create <T>(Container container, CosmosQueryContext context)
{
EnsureArg.IsNotNull(container, nameof(container));
EnsureArg.IsNotNull(context, nameof(context));
var documentQuery = container
.GetItemQueryIterator <T>(
context.SqlQuerySpec,
continuationToken: context.ContinuationToken,
requestOptions: context.FeedOptions);
return(new CosmosQuery <T>(
context,
documentQuery,
_logger));
}
public ReadItemQueryingEnumerable(
CosmosQueryContext cosmosQueryContext,
ReadItemExpression readItemExpression,
Func <CosmosQueryContext, JObject, T> shaper,
Type contextType,
bool standAloneStateManager,
bool threadSafetyChecksEnabled)
{
_cosmosQueryContext = cosmosQueryContext;
_readItemExpression = readItemExpression;
_shaper = shaper;
_contextType = contextType;
_queryLogger = _cosmosQueryContext.QueryLogger;
_standAloneStateManager = standAloneStateManager;
_threadSafetyChecksEnabled = threadSafetyChecksEnabled;
}
public static async Task <TryCatch <CosmosQueryExecutionContext> > TryCreateFromPartitionedQuerExecutionInfoAsync(
PartitionedQueryExecutionInfo partitionedQueryExecutionInfo,
ContainerQueryProperties containerQueryProperties,
CosmosQueryContext cosmosQueryContext,
InputParameters inputParameters,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
List <Documents.PartitionKeyRange> targetRanges = await CosmosQueryExecutionContextFactory.GetTargetPartitionKeyRangesAsync(
cosmosQueryContext.QueryClient,
cosmosQueryContext.ResourceLink.OriginalString,
partitionedQueryExecutionInfo,
containerQueryProperties,
inputParameters.Properties);
if (!string.IsNullOrEmpty(partitionedQueryExecutionInfo.QueryInfo.RewrittenQuery))
{
// We need pass down the rewritten query.
SqlQuerySpec rewrittenQuerySpec = new SqlQuerySpec()
{
QueryText = partitionedQueryExecutionInfo.QueryInfo.RewrittenQuery,
Parameters = inputParameters.SqlQuerySpec.Parameters
};
inputParameters = new InputParameters(
rewrittenQuerySpec,
inputParameters.InitialUserContinuationToken,
inputParameters.MaxConcurrency,
inputParameters.MaxItemCount,
inputParameters.MaxBufferedItemCount,
inputParameters.PartitionKey,
inputParameters.Properties,
inputParameters.PartitionedQueryExecutionInfo,
inputParameters.ExecutionEnvironment,
inputParameters.ReturnResultsInDeterministicOrder,
inputParameters.TestInjections);
}
return(await CosmosQueryExecutionContextFactory.TryCreateSpecializedDocumentQueryExecutionContextAsync(
cosmosQueryContext,
inputParameters,
partitionedQueryExecutionInfo,
targetRanges,
containerQueryProperties.ResourceId,
cancellationToken));
}
public AsyncQueryingEnumerable(
CosmosQueryContext cosmosQueryContext,
ISqlExpressionFactory sqlExpressionFactory,
IQuerySqlGeneratorFactory querySqlGeneratorFactory,
SelectExpression selectExpression,
Func <QueryContext, JObject, T> shaper,
Type contextType,
IDiagnosticsLogger <DbLoggerCategory.Query> logger)
{
_cosmosQueryContext = cosmosQueryContext;
_sqlExpressionFactory = sqlExpressionFactory;
_querySqlGeneratorFactory = querySqlGeneratorFactory;
_selectExpression = selectExpression;
_shaper = shaper;
_contextType = contextType;
_logger = logger;
}
public AsyncEnumerator(QueryingEnumerable <T> queryingEnumerable, CancellationToken cancellationToken)
{
_queryingEnumerable = queryingEnumerable;
_cosmosQueryContext = queryingEnumerable._cosmosQueryContext;
_shaper = queryingEnumerable._shaper;
_selectExpression = queryingEnumerable._selectExpression;
_contextType = queryingEnumerable._contextType;
_partitionKey = queryingEnumerable._partitionKey;
_queryLogger = queryingEnumerable._queryLogger;
_standAloneStateManager = queryingEnumerable._standAloneStateManager;
_exceptionDetector = _cosmosQueryContext.ExceptionDetector;
_cancellationToken = cancellationToken;
_concurrencyDetector = queryingEnumerable._threadSafetyChecksEnabled
? _cosmosQueryContext.ConcurrencyDetector
: null;
}
/// <summary>
/// Initializes a new instance of the CosmosParallelItemQueryExecutionContext class.
/// </summary>
/// <param name="queryContext">The parameters for constructing the base class.</param>
/// <param name="maxConcurrency">The max concurrency</param>
/// <param name="maxBufferedItemCount">The max buffered item count</param>
/// <param name="maxItemCount">Max item count</param>
/// <param name="testSettings">Test settings.</param>
private CosmosParallelItemQueryExecutionContext(
CosmosQueryContext queryContext,
int?maxConcurrency,
int?maxItemCount,
int?maxBufferedItemCount,
TestInjections testSettings)
: base(
queryContext: queryContext,
maxConcurrency: maxConcurrency,
maxItemCount: maxItemCount,
maxBufferedItemCount: maxBufferedItemCount,
moveNextComparer: CosmosParallelItemQueryExecutionContext.MoveNextComparer,
fetchPrioirtyFunction: CosmosParallelItemQueryExecutionContext.FetchPriorityFunction,
equalityComparer: CosmosParallelItemQueryExecutionContext.EqualityComparer,
testSettings: testSettings)
{
}
public static Task <PartitionedQueryExecutionInfo> GetQueryPlanThroughGatewayAsync(
CosmosQueryContext queryContext,
SqlQuerySpec sqlQuerySpec,
string resourceLink,
PartitionKey?partitionKey,
ITrace trace,
CancellationToken cancellationToken = default)
{
if (queryContext == null)
{
throw new ArgumentNullException(nameof(queryContext));
}
if (sqlQuerySpec == null)
{
throw new ArgumentNullException(nameof(sqlQuerySpec));
}
if (resourceLink == null)
{
throw new ArgumentNullException(nameof(resourceLink));
}
cancellationToken.ThrowIfCancellationRequested();
using (ITrace gatewayQueryPlanTrace = trace.StartChild("Gateway QueryPlan", TraceComponent.Query, TraceLevel.Info))
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) &&
Documents.ServiceInteropWrapper.Is64BitProcess)
{
// It's Windows and x64, should have loaded the DLL
gatewayQueryPlanTrace.AddDatum("ServiceInterop unavailable", true);
}
return(queryContext.ExecuteQueryPlanRequestAsync(
resourceLink,
ResourceType.Document,
OperationType.QueryPlan,
sqlQuerySpec,
partitionKey,
QueryPlanRetriever.SupportedQueryFeaturesString,
trace,
cancellationToken));
}
}
/// <summary>
/// Initializes a new instance of the CosmosOrderByItemQueryExecutionContext class.
/// </summary>
/// <param name="initPararms">The params used to construct the base class.</param>
/// For cross partition order by queries a query like "SELECT c.id, c.field_0 ORDER BY r.field_7 gets rewritten as:
/// <![CDATA[
/// SELECT r._rid, [{"item": r.field_7}] AS orderByItems, {"id": r.id, "field_0": r.field_0} AS payload
/// FROM r
/// WHERE({ document db - formattable order by query - filter})
/// ORDER BY r.field_7]]>
/// This is needed because we need to add additional filters to the query when we resume from a continuation,
/// and it lets us easily parse out the _rid orderByItems, and payload without parsing the entire document (and having to remember the order by field).
/// <param name="maxConcurrency">The max concurrency</param>
/// <param name="maxBufferedItemCount">The max buffered item count</param>
/// <param name="maxItemCount">Max item count</param>
/// <param name="consumeComparer">Comparer used to internally compare documents from different sorted partitions.</param>
/// <param name="testSettings">Test settings.</param>
private CosmosOrderByItemQueryExecutionContext(
CosmosQueryContext initPararms,
int?maxConcurrency,
int?maxItemCount,
int?maxBufferedItemCount,
OrderByItemProducerTreeComparer consumeComparer,
TestInjections testSettings)
: base(
queryContext: initPararms,
maxConcurrency: maxConcurrency,
maxItemCount: maxItemCount,
maxBufferedItemCount: maxBufferedItemCount,
moveNextComparer: consumeComparer,
fetchPrioirtyFunction: CosmosOrderByItemQueryExecutionContext.FetchPriorityFunction,
equalityComparer: new OrderByEqualityComparer(consumeComparer),
returnResultsInDeterministicOrder: true,
testSettings: testSettings)
{
}
