public static async Task <TryCatch <CosmosOrderByItemQueryExecutionContext> > TryCreateAsync(
CosmosQueryContext queryContext,
CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams initParams,
string requestContinuationToken,
CancellationToken cancellationToken)
{
Debug.Assert(
initParams.PartitionedQueryExecutionInfo.QueryInfo.HasOrderBy,
"OrderBy~Context must have order by query info.");
if (queryContext == null)
{
throw new ArgumentNullException(nameof(queryContext));
}
cancellationToken.ThrowIfCancellationRequested();
CosmosOrderByItemQueryExecutionContext context = new CosmosOrderByItemQueryExecutionContext(
initPararms: queryContext,
maxConcurrency: initParams.MaxConcurrency,
maxItemCount: initParams.MaxItemCount,
maxBufferedItemCount: initParams.MaxBufferedItemCount,
consumeComparer: new OrderByConsumeComparer(initParams.PartitionedQueryExecutionInfo.QueryInfo.OrderBy));
return((await context.TryInitializeAsync(
sqlQuerySpec: initParams.SqlQuerySpec,
requestContinuation: requestContinuationToken,
collectionRid: initParams.CollectionRid,
partitionKeyRanges: initParams.PartitionKeyRanges,
initialPageSize: initParams.InitialPageSize,
sortOrders: initParams.PartitionedQueryExecutionInfo.QueryInfo.OrderBy,
orderByExpressions: initParams.PartitionedQueryExecutionInfo.QueryInfo.OrderByExpressions,
cancellationToken: cancellationToken))
.Try <CosmosOrderByItemQueryExecutionContext>((ignore) => context));
}
/// <summary>
/// Callback to create a child document producer tree based on the partition key range.
/// </summary>
/// <param name="queryContext">request context</param>
/// <param name="querySpecForInit">query spec for initialization</param>
/// <param name="produceAsyncCompleteCallback">Callback to invoke once a fetch finishes.</param>
/// <param name="itemProducerTreeComparer">Comparer to determine, which tree to produce from.</param>
/// <param name="equalityComparer">Comparer to see if we need to return the continuation token for a partition.</param>
/// <param name="deferFirstPage">Whether or not to defer fetching the first page.</param>
/// <param name="collectionRid">The collection to drain from.</param>
/// <param name="initialPageSize">The initial page size.</param>
/// <returns>A function that given a partition key range and continuation token will create a document producer.</returns>
private static Func <Documents.PartitionKeyRange, string, ItemProducerTree> CreateItemProducerTreeCallback(
CosmosQueryContext queryContext,
SqlQuerySpec querySpecForInit,
ProduceAsyncCompleteDelegate produceAsyncCompleteCallback,
IComparer <ItemProducerTree> itemProducerTreeComparer,
IEqualityComparer <CosmosElement> equalityComparer,
bool deferFirstPage,
string collectionRid,
long initialPageSize = 50)
{
return((partitionKeyRange, continuationToken) =>
{
return new ItemProducerTree(
queryContext,
querySpecForInit,
partitionKeyRange,
produceAsyncCompleteCallback,
itemProducerTreeComparer,
equalityComparer,
deferFirstPage,
collectionRid,
initialPageSize,
continuationToken);
});
}
/// <summary>
/// Callback to create a child document producer tree based on the partition key range.
/// </summary>
/// <param name="queryContext">request context</param>
/// <param name="querySpecForInit">query spec for initialization</param>
/// <param name="createRetryPolicyFunc">Callback to create a retry policy.</param>
/// <param name="produceAsyncCompleteCallback">Callback to invoke once a fetch finishes.</param>
/// <param name="itemProducerTreeComparer">Comparer to determine, which tree to produce from.</param>
/// <param name="equalityComparer">Comparer to see if we need to return the continuation token for a partition.</param>
/// <param name="deferFirstPage">Whether or not to defer fetching the first page.</param>
/// <param name="collectionRid">The collection to drain from.</param>
/// <param name="initialPageSize">The initial page size.</param>
/// <returns>A function that given a partition key range and continuation token will create a document producer.</returns>
private static Func <PartitionKeyRange, string, ItemProducerTree> CreateItemProducerTreeCallback(
CosmosQueryContext queryContext,
SqlQuerySpec querySpecForInit,
Func <IDocumentClientRetryPolicy> createRetryPolicyFunc,
Action <ItemProducerTree, int, double, QueryMetrics, long, CancellationToken> produceAsyncCompleteCallback,
IComparer <ItemProducerTree> itemProducerTreeComparer,
IEqualityComparer <CosmosElement> equalityComparer,
bool deferFirstPage,
string collectionRid,
long initialPageSize = 50)
{
return((partitionKeyRange, continuationToken) =>
{
return new ItemProducerTree(
queryContext,
querySpecForInit,
partitionKeyRange,
createRetryPolicyFunc,
produceAsyncCompleteCallback,
itemProducerTreeComparer,
equalityComparer,
deferFirstPage,
collectionRid,
initialPageSize,
continuationToken);
});
}
public override async Task <ResponseMessage> ReadNextAsync(CancellationToken cancellationToken = default)
{
CosmosDiagnosticsContext diagnostics = new CosmosDiagnosticsContext();
using (diagnostics.CreateScope("QueryReadNextAsync"))
{
// This catches exception thrown by the pipeline and converts it to QueryResponse
QueryResponseCore responseCore = await this.cosmosQueryExecutionContext.ExecuteNextAsync(cancellationToken);
CosmosQueryContext cosmosQueryContext = this.cosmosQueryContext;
foreach (QueryPageDiagnostics queryPage in responseCore.Diagnostics)
{
diagnostics.Summary.Append(queryPage.DiagnosticsContext.Summary);
diagnostics.AddDiagnosticsInternal(queryPage);
}
QueryResponse queryResponse;
if (responseCore.IsSuccess)
{
queryResponse = QueryResponse.CreateSuccess(
result: responseCore.CosmosElements,
count: responseCore.CosmosElements.Count,
responseLengthBytes: responseCore.ResponseLengthBytes,
diagnostics: diagnostics,
serializationOptions: this.cosmosSerializationFormatOptions,
responseHeaders: new CosmosQueryResponseMessageHeaders(
responseCore.ContinuationToken,
responseCore.DisallowContinuationTokenMessage,
cosmosQueryContext.ResourceTypeEnum,
cosmosQueryContext.ContainerResourceId)
{
RequestCharge = responseCore.RequestCharge,
ActivityId = responseCore.ActivityId,
SubStatusCode = responseCore.SubStatusCode ?? Documents.SubStatusCodes.Unknown
});
}
else
{
queryResponse = QueryResponse.CreateFailure(
statusCode: responseCore.StatusCode,
error: null,
errorMessage: responseCore.ErrorMessage,
requestMessage: null,
diagnostics: diagnostics,
responseHeaders: new CosmosQueryResponseMessageHeaders(
responseCore.ContinuationToken,
responseCore.DisallowContinuationTokenMessage,
cosmosQueryContext.ResourceTypeEnum,
cosmosQueryContext.ContainerResourceId)
{
RequestCharge = responseCore.RequestCharge,
ActivityId = responseCore.ActivityId,
SubStatusCode = responseCore.SubStatusCode ?? Documents.SubStatusCodes.Unknown
});
}
return(queryResponse);
}
}
/// <summary>
/// Initializes a new instance of the CosmosCrossPartitionQueryExecutionContext class.
/// </summary>
/// <param name="initParams">Constructor parameters for the base class.</param>
/// <param name="moveNextComparer">Comparer used to figure out that document producer tree to serve documents from next.</param>
/// <param name="fetchPrioirtyFunction">The priority function to determine which partition to fetch documents from next.</param>
/// <param name="equalityComparer">Used to determine whether we need to return the continuation token for a partition.</param>
protected CosmosCrossPartitionQueryExecutionContext(
CosmosQueryContext initParams,
IComparer <ItemProducerTree> moveNextComparer,
Func <ItemProducerTree, int> fetchPrioirtyFunction,
IEqualityComparer <CosmosElement> equalityComparer)
{
if (moveNextComparer == null)
{
throw new ArgumentNullException(nameof(moveNextComparer));
}
if (fetchPrioirtyFunction == null)
{
throw new ArgumentNullException(nameof(fetchPrioirtyFunction));
}
if (equalityComparer == null)
{
throw new ArgumentNullException(nameof(equalityComparer));
}
this.queryContext = initParams;
this.queryRequestOptions = initParams.QueryRequestOptions;
this.itemProducerForest = new PriorityQueue <ItemProducerTree>(moveNextComparer, isSynchronized: true);
this.fetchPrioirtyFunction = fetchPrioirtyFunction;
this.comparableTaskScheduler = new ComparableTaskScheduler(initParams.QueryRequestOptions.MaxConcurrency.GetValueOrDefault(0));
this.equalityComparer = equalityComparer;
this.requestChargeTracker = new RequestChargeTracker();
this.partitionedQueryMetrics = new ConcurrentBag <Tuple <string, QueryMetrics> >();
this.actualMaxPageSize = this.queryRequestOptions.MaxItemCount.GetValueOrDefault(ParallelQueryConfig.GetConfig().ClientInternalMaxItemCount);
if (this.actualMaxPageSize < 0)
{
throw new ArgumentOutOfRangeException("actualMaxPageSize should never be less than 0");
}
if (this.actualMaxPageSize > int.MaxValue)
{
throw new ArgumentOutOfRangeException("actualMaxPageSize should never be greater than int.MaxValue");
}
if (this.queryRequestOptions.MaxBufferedItemCount.HasValue)
{
this.actualMaxBufferedItemCount = this.queryRequestOptions.MaxBufferedItemCount.Value;
}
else
{
this.actualMaxBufferedItemCount = ParallelQueryConfig.GetConfig().DefaultMaximumBufferSize;
}
if (this.actualMaxBufferedItemCount < 0)
{
throw new ArgumentOutOfRangeException("actualMaxBufferedItemCount should never be less than 0");
}
if (this.actualMaxBufferedItemCount > int.MaxValue)
{
throw new ArgumentOutOfRangeException("actualMaxBufferedItemCount should never be greater than int.MaxValue");
}
}
public CosmosQueryExecutionContextFactory(
CosmosQueryContext cosmosQueryContext,
InputParameters inputParameters)
{
if (cosmosQueryContext == null)
{
throw new ArgumentNullException(nameof(cosmosQueryContext));
}
this.CosmosQueryContext = cosmosQueryContext;
this.inputParameters = inputParameters;
// Swapping out negative values in feedOptions for int.MaxValue
if (this.inputParameters.MaxBufferedItemCount.HasValue && this.inputParameters.MaxBufferedItemCount < 0)
{
this.inputParameters.MaxBufferedItemCount = int.MaxValue;
}
if (this.inputParameters.MaxConcurrency.HasValue && this.inputParameters.MaxConcurrency < 0)
{
this.inputParameters.MaxConcurrency = int.MaxValue;
}
if (this.inputParameters.MaxItemCount.HasValue && this.inputParameters.MaxItemCount < 0)
{
this.inputParameters.MaxItemCount = int.MaxValue;
}
}
public static async Task <TryCatch <CosmosParallelItemQueryExecutionContext> > TryCreateAsync(
CosmosQueryContext queryContext,
CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams initParams,
string requestContinuationToken,
CancellationToken cancellationToken)
{
Debug.Assert(
!initParams.PartitionedQueryExecutionInfo.QueryInfo.HasOrderBy,
"Parallel~Context must not have order by query info.");
cancellationToken.ThrowIfCancellationRequested();
CosmosParallelItemQueryExecutionContext context = new CosmosParallelItemQueryExecutionContext(
queryContext: queryContext,
maxConcurrency: initParams.MaxConcurrency,
maxItemCount: initParams.MaxItemCount,
maxBufferedItemCount: initParams.MaxBufferedItemCount);
return(await context.TryInitializeAsync(
sqlQuerySpec : initParams.SqlQuerySpec,
collectionRid : initParams.CollectionRid,
partitionKeyRanges : initParams.PartitionKeyRanges,
initialPageSize : initParams.InitialPageSize,
requestContinuation : requestContinuationToken,
cancellationToken : cancellationToken));
}
/// <summary>
/// Creates an CosmosOrderByItemQueryExecutionContext
/// </summary>
/// <param name="constructorParams">The parameters for the base class constructor.</param>
/// <param name="initParams">The parameters to initialize the base class.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>A task to await on, which in turn creates an CosmosOrderByItemQueryExecutionContext.</returns>
public static async Task <CosmosOrderByItemQueryExecutionContext> CreateAsync(
CosmosQueryContext constructorParams,
CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams initParams,
CancellationToken token)
{
Debug.Assert(
initParams.PartitionedQueryExecutionInfo.QueryInfo.HasOrderBy,
"OrderBy~Context must have order by query info.");
CosmosOrderByItemQueryExecutionContext context = new CosmosOrderByItemQueryExecutionContext(
constructorParams,
new OrderByConsumeComparer(initParams.PartitionedQueryExecutionInfo.QueryInfo.OrderBy));
await context.InitializeAsync(
constructorParams.SqlQuerySpec,
initParams.RequestContinuation,
initParams.CollectionRid,
initParams.PartitionKeyRanges,
initParams.InitialPageSize,
initParams.PartitionedQueryExecutionInfo.QueryInfo.OrderBy,
initParams.PartitionedQueryExecutionInfo.QueryInfo.OrderByExpressions,
token);
return(context);
}
/// <summary>
/// Creates an CosmosOrderByItemQueryExecutionContext
/// </summary>
/// <param name="queryContext">The parameters for the base class constructor.</param>
/// <param name="initParams">The parameters to initialize the base class.</param>
/// <param name="requestContinuationToken">The request continuation.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>A task to await on, which in turn creates an CosmosOrderByItemQueryExecutionContext.</returns>
public static async Task <CosmosOrderByItemQueryExecutionContext> CreateAsync(
CosmosQueryContext queryContext,
CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams initParams,
string requestContinuationToken,
CancellationToken token)
{
Debug.Assert(
initParams.PartitionedQueryExecutionInfo.QueryInfo.HasOrderBy,
"OrderBy~Context must have order by query info.");
CosmosOrderByItemQueryExecutionContext context = new CosmosOrderByItemQueryExecutionContext(
initPararms: queryContext,
maxConcurrency: initParams.MaxConcurrency,
maxItemCount: initParams.MaxItemCount,
maxBufferedItemCount: initParams.MaxBufferedItemCount,
consumeComparer: new OrderByConsumeComparer(initParams.PartitionedQueryExecutionInfo.QueryInfo.OrderBy));
await context.InitializeAsync(
sqlQuerySpec : initParams.SqlQuerySpec,
requestContinuation : requestContinuationToken,
collectionRid : initParams.CollectionRid,
partitionKeyRanges : initParams.PartitionKeyRanges,
initialPageSize : initParams.InitialPageSize,
sortOrders : initParams.PartitionedQueryExecutionInfo.QueryInfo.OrderBy,
orderByExpressions : initParams.PartitionedQueryExecutionInfo.QueryInfo.OrderByExpressions,
cancellationToken : token);
return(context);
}
/// <summary>
/// Creates a CosmosPipelinedItemQueryExecutionContext.
/// </summary>
/// <param name="queryContext">The parameters for constructing the base class.</param>
/// <param name="initParams">The initial parameters</param>
/// <param name="requestContinuationToken">The request continuation.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>A task to await on, which in turn returns a CosmosPipelinedItemQueryExecutionContext.</returns>
public static async Task <CosmosQueryExecutionContext> CreateAsync(
CosmosQueryContext queryContext,
CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams initParams,
string requestContinuationToken,
CancellationToken cancellationToken)
{
DefaultTrace.TraceInformation(
string.Format(
CultureInfo.InvariantCulture,
"{0}, CorrelatedActivityId: {1} | Pipelined~Context.CreateAsync",
DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture),
queryContext.CorrelatedActivityId));
QueryInfo queryInfo = initParams.PartitionedQueryExecutionInfo.QueryInfo;
int actualPageSize = initParams.InitialPageSize;
int initialPageSize = initParams.InitialPageSize;
CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams parameters = initParams;
if (queryInfo.HasGroupBy)
{
initialPageSize = int.MaxValue;
initParams = new CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams(
sqlQuerySpec: initParams.SqlQuerySpec,
collectionRid: initParams.CollectionRid,
partitionedQueryExecutionInfo: initParams.PartitionedQueryExecutionInfo,
partitionKeyRanges: initParams.PartitionKeyRanges,
initialPageSize: initialPageSize,
maxConcurrency: initParams.MaxConcurrency,
maxItemCount: int.MaxValue,
maxBufferedItemCount: initParams.MaxBufferedItemCount);
}
Func <string, Task <IDocumentQueryExecutionComponent> > createOrderByComponentFunc = async(continuationToken) =>
{
return(await CosmosOrderByItemQueryExecutionContext.CreateAsync(
queryContext,
initParams,
continuationToken,
cancellationToken));
};
Func <string, Task <IDocumentQueryExecutionComponent> > createParallelComponentFunc = async(continuationToken) =>
{
return(await CosmosParallelItemQueryExecutionContext.CreateAsync(
queryContext,
initParams,
continuationToken,
cancellationToken));
};
return((CosmosQueryExecutionContext)await PipelinedDocumentQueryExecutionContext.CreateHelperAsync(
queryContext.QueryClient,
initParams.PartitionedQueryExecutionInfo.QueryInfo,
initialPageSize,
requestContinuationToken,
createOrderByComponentFunc,
createParallelComponentFunc));
}
/// <summary>
/// Initializes a new instance of the ItemProducer class.
/// </summary>
/// <param name="queryContext">request context</param>
/// <param name="querySpecForInit">query spec for initialization</param>
/// <param name="partitionKeyRange">The partition key range.</param>
/// <param name="createRetryPolicyFunc">The callback to create the retry policy.</param>
/// <param name="produceAsyncCompleteCallback">The callback to call once you are done fetching.</param>
/// <param name="equalityComparer">The comparer to use to determine whether the producer has seen a new document.</param>
/// <param name="initialPageSize">The initial page size.</param>
/// <param name="initialContinuationToken">The initial continuation token.</param>
public ItemProducer(
CosmosQueryContext queryContext,
SqlQuerySpec querySpecForInit,
PartitionKeyRange partitionKeyRange,
Func <IDocumentClientRetryPolicy> createRetryPolicyFunc,
ProduceAsyncCompleteDelegate produceAsyncCompleteCallback,
IEqualityComparer <CosmosElement> equalityComparer,
long initialPageSize = 50,
string initialContinuationToken = null)
{
this.bufferedPages = new AsyncCollection <TryMonad <FeedResponse <CosmosElement> > >();
// We use a binary semaphore to get the behavior of a mutex,
// since fetching documents from the backend using a continuation token is a critical section.
this.fetchSemaphore = new SemaphoreSlim(1, 1);
if (partitionKeyRange == null)
{
throw new ArgumentNullException(nameof(partitionKeyRange));
}
if (createRetryPolicyFunc == null)
{
throw new ArgumentNullException(nameof(createRetryPolicyFunc));
}
if (produceAsyncCompleteCallback == null)
{
throw new ArgumentNullException(nameof(produceAsyncCompleteCallback));
}
if (equalityComparer == null)
{
throw new ArgumentNullException(nameof(equalityComparer));
}
this.queryContext = queryContext;
this.querySpecForInit = querySpecForInit;
this.PartitionKeyRange = partitionKeyRange;
this.createRetryPolicyFunc = createRetryPolicyFunc;
this.produceAsyncCompleteCallback = produceAsyncCompleteCallback;
this.equalityComparer = equalityComparer;
this.pageSize = initialPageSize;
this.currentContinuationToken = initialContinuationToken;
this.BackendContinuationToken = initialContinuationToken;
this.PreviousContinuationToken = initialContinuationToken;
if (!string.IsNullOrEmpty(initialContinuationToken))
{
this.hasStartedFetching = true;
this.IsActive = true;
}
this.fetchSchedulingMetrics = new SchedulingStopwatch();
this.fetchSchedulingMetrics.Ready();
this.fetchExecutionRangeAccumulator = new FetchExecutionRangeAccumulator();
this.HasMoreResults = true;
}
/// <summary>
/// Initializes a new instance of the CosmosParallelItemQueryExecutionContext class.
/// </summary>
/// <param name="constructorParams">The parameters for constructing the base class.</param>
private CosmosParallelItemQueryExecutionContext(
CosmosQueryContext constructorParams)
: base(
constructorParams,
CosmosParallelItemQueryExecutionContext.MoveNextComparer,
CosmosParallelItemQueryExecutionContext.FetchPriorityFunction,
CosmosParallelItemQueryExecutionContext.EqualityComparer)
{
}
private QueryIterator(
CosmosQueryContext cosmosQueryContext,
CosmosQueryExecutionContext cosmosQueryExecutionContext,
CosmosSerializationFormatOptions cosmosSerializationFormatOptions)
{
this.cosmosQueryContext = cosmosQueryContext ?? throw new ArgumentNullException(nameof(cosmosQueryContext));
this.cosmosQueryExecutionContext = cosmosQueryExecutionContext ?? throw new ArgumentNullException(nameof(cosmosQueryExecutionContext));
this.cosmosSerializationFormatOptions = cosmosSerializationFormatOptions;
}
/// <summary>
/// Initializes a new instance of the CosmosOrderByItemQueryExecutionContext class.
/// </summary>
/// <param name="initPararms">The params used to construct the base class.</param>
/// For cross partition order by queries a query like "SELECT c.id, c.field_0 ORDER BY r.field_7 gets rewritten as:
/// <![CDATA[
/// SELECT r._rid, [{"item": r.field_7}] AS orderByItems, {"id": r.id, "field_0": r.field_0} AS payload
/// FROM r
/// WHERE({ document db - formattable order by query - filter})
/// ORDER BY r.field_7]]>
/// This is needed because we need to add additional filters to the query when we resume from a continuation,
/// and it lets us easily parse out the _rid orderByItems, and payload without parsing the entire document (and having to remember the order by field).
/// <param name="consumeComparer">Comparer used to internally compare documents from different sorted partitions.</param>
private CosmosOrderByItemQueryExecutionContext(
CosmosQueryContext initPararms,
OrderByConsumeComparer consumeComparer)
: base(
initPararms,
consumeComparer,
CosmosOrderByItemQueryExecutionContext.FetchPriorityFunction,
new OrderByEqualityComparer(consumeComparer))
{
}
/// <summary>
/// Creates a CosmosPipelinedItemQueryExecutionContext.
/// </summary>
/// <param name="executionEnvironment">The environment to execute on.</param>
/// <param name="queryContext">The parameters for constructing the base class.</param>
/// <param name="initParams">The initial parameters</param>
/// <param name="requestContinuationToken">The request continuation.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>A task to await on, which in turn returns a CosmosPipelinedItemQueryExecutionContext.</returns>
public static async Task <CosmosQueryExecutionContext> CreateAsync(
ExecutionEnvironment executionEnvironment,
CosmosQueryContext queryContext,
CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams initParams,
string requestContinuationToken,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
QueryInfo queryInfo = initParams.PartitionedQueryExecutionInfo.QueryInfo;
int initialPageSize = initParams.InitialPageSize;
CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams parameters = initParams;
if (queryInfo.HasGroupBy)
{
// The query will block until all groupings are gathered so we might as well speed up the process.
initParams = new CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams(
sqlQuerySpec: initParams.SqlQuerySpec,
collectionRid: initParams.CollectionRid,
partitionedQueryExecutionInfo: initParams.PartitionedQueryExecutionInfo,
partitionKeyRanges: initParams.PartitionKeyRanges,
initialPageSize: int.MaxValue,
maxConcurrency: initParams.MaxConcurrency,
maxItemCount: int.MaxValue,
maxBufferedItemCount: initParams.MaxBufferedItemCount);
}
Func <string, Task <IDocumentQueryExecutionComponent> > createOrderByComponentFunc = async(continuationToken) =>
{
return(await CosmosOrderByItemQueryExecutionContext.CreateAsync(
queryContext,
initParams,
continuationToken,
cancellationToken));
};
Func <string, Task <IDocumentQueryExecutionComponent> > createParallelComponentFunc = async(continuationToken) =>
{
return(await CosmosParallelItemQueryExecutionContext.CreateAsync(
queryContext,
initParams,
continuationToken,
cancellationToken));
};
return((CosmosQueryExecutionContext)await PipelinedDocumentQueryExecutionContext.CreateHelperAsync(
executionEnvironment,
queryContext.QueryClient,
initParams.PartitionedQueryExecutionInfo.QueryInfo,
initialPageSize,
requestContinuationToken,
createOrderByComponentFunc,
createParallelComponentFunc));
}
/// <summary>
/// Creates a CosmosPipelinedItemQueryExecutionContext.
/// </summary>
/// <param name="constructorParams">The parameters for constructing the base class.</param>
/// <param name="collectionRid">The collection rid.</param>
/// <param name="partitionedQueryExecutionInfo">The partitioned query execution info.</param>
/// <param name="partitionKeyRanges">The partition key ranges.</param>
/// <param name="initialPageSize">The initial page size.</param>
/// <param name="requestContinuation">The request continuation.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>A task to await on, which in turn returns a CosmosPipelinedItemQueryExecutionContext.</returns>
public static async Task <CosmosQueryExecutionContext> CreateAsync(
CosmosQueryContext constructorParams,
string collectionRid,
PartitionedQueryExecutionInfo partitionedQueryExecutionInfo,
List <PartitionKeyRange> partitionKeyRanges,
int initialPageSize,
string requestContinuation,
CancellationToken cancellationToken)
{
DefaultTrace.TraceInformation(
string.Format(
CultureInfo.InvariantCulture,
"{0}, CorrelatedActivityId: {1} | Pipelined~Context.CreateAsync",
DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture),
constructorParams.CorrelatedActivityId));
Func <string, Task <IDocumentQueryExecutionComponent> > createOrderByComponentFunc = async(continuationToken) =>
{
CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams initParams = new CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams(
collectionRid,
partitionedQueryExecutionInfo,
partitionKeyRanges,
initialPageSize,
continuationToken);
return(await CosmosOrderByItemQueryExecutionContext.CreateAsync(
constructorParams,
initParams,
cancellationToken));
};
Func <string, Task <IDocumentQueryExecutionComponent> > createParallelComponentFunc = async(continuationToken) =>
{
CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams initParams = new CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams(
collectionRid,
partitionedQueryExecutionInfo,
partitionKeyRanges,
initialPageSize,
continuationToken);
return(await CosmosParallelItemQueryExecutionContext.CreateAsync(
constructorParams,
initParams,
cancellationToken));
};
return((CosmosQueryExecutionContext)(await PipelinedDocumentQueryExecutionContext.CreateHelperAsync(
partitionedQueryExecutionInfo.QueryInfo,
initialPageSize,
requestContinuation,
createOrderByComponentFunc,
createParallelComponentFunc)));
}
/// <summary>
/// Initializes a new instance of the CosmosParallelItemQueryExecutionContext class.
/// </summary>
/// <param name="queryContext">The parameters for constructing the base class.</param>
/// <param name="maxConcurrency">The max concurrency</param>
/// <param name="maxBufferedItemCount">The max buffered item count</param>
/// <param name="maxItemCount">Max item count</param>
private CosmosParallelItemQueryExecutionContext(
CosmosQueryContext queryContext,
int?maxConcurrency,
int?maxItemCount,
int?maxBufferedItemCount)
: base(
queryContext: queryContext,
maxConcurrency: maxConcurrency,
maxItemCount: maxItemCount,
maxBufferedItemCount: maxBufferedItemCount,
moveNextComparer: CosmosParallelItemQueryExecutionContext.MoveNextComparer,
fetchPrioirtyFunction: CosmosParallelItemQueryExecutionContext.FetchPriorityFunction,
equalityComparer: CosmosParallelItemQueryExecutionContext.EqualityComparer)
{
}
public CosmosGatewayQueryExecutionContext(
CosmosQueryContext cosmosQueryContext)
{
if (cosmosQueryContext == null)
{
throw new ArgumentNullException(nameof(cosmosQueryContext));
}
this.queryContext = cosmosQueryContext;
this.fetchSchedulingMetrics = new SchedulingStopwatch();
this.fetchSchedulingMetrics.Ready();
this.fetchExecutionRangeAccumulator = new FetchExecutionRangeAccumulator();
this.retries = -1;
this.partitionRoutingHelper = new PartitionRoutingHelper();
}
/// <summary>
/// Initializes a new instance of the CosmosOrderByItemQueryExecutionContext class.
/// </summary>
/// <param name="initPararms">The params used to construct the base class.</param>
/// For cross partition order by queries a query like "SELECT c.id, c.field_0 ORDER BY r.field_7 gets rewritten as:
/// <![CDATA[
/// SELECT r._rid, [{"item": r.field_7}] AS orderByItems, {"id": r.id, "field_0": r.field_0} AS payload
/// FROM r
/// WHERE({ document db - formattable order by query - filter})
/// ORDER BY r.field_7]]>
/// This is needed because we need to add additional filters to the query when we resume from a continuation,
/// and it lets us easily parse out the _rid orderByItems, and payload without parsing the entire document (and having to remember the order by field).
/// <param name="maxConcurrency">The max concurrency</param>
/// <param name="maxBufferedItemCount">The max buffered item count</param>
/// <param name="maxItemCount">Max item count</param>
/// <param name="consumeComparer">Comparer used to internally compare documents from different sorted partitions.</param>
private CosmosOrderByItemQueryExecutionContext(
CosmosQueryContext initPararms,
int?maxConcurrency,
int?maxItemCount,
int?maxBufferedItemCount,
OrderByConsumeComparer consumeComparer)
: base(
queryContext: initPararms,
maxConcurrency: maxConcurrency,
maxItemCount: maxItemCount,
maxBufferedItemCount: maxBufferedItemCount,
moveNextComparer: consumeComparer,
fetchPrioirtyFunction: CosmosOrderByItemQueryExecutionContext.FetchPriorityFunction,
equalityComparer: new OrderByEqualityComparer(consumeComparer))
{
}
public CosmosProxyItemQueryExecutionContext(
CosmosQueryContext queryContext,
CosmosContainerSettings containerSettings)
{
if (queryContext == null)
{
throw new ArgumentNullException(nameof(queryContext));
}
if (queryContext == null)
{
throw new ArgumentNullException(nameof(queryContext));
}
this.innerExecutionContext = new CosmosGatewayQueryExecutionContext(queryContext);
this.queryContext = queryContext;
this.containerSettings = containerSettings;
}
/// <summary>
/// Initializes a new instance of the CosmosCrossPartitionQueryExecutionContext class.
/// </summary>
/// <param name="queryContext">Constructor parameters for the base class.</param>
/// <param name="maxConcurrency">The max concurrency</param>
/// <param name="maxBufferedItemCount">The max buffered item count</param>
/// <param name="maxItemCount">Max item count</param>
/// <param name="moveNextComparer">Comparer used to figure out that document producer tree to serve documents from next.</param>
/// <param name="fetchPrioirtyFunction">The priority function to determine which partition to fetch documents from next.</param>
/// <param name="equalityComparer">Used to determine whether we need to return the continuation token for a partition.</param>
protected CosmosCrossPartitionQueryExecutionContext(
CosmosQueryContext queryContext,
int?maxConcurrency,
int?maxItemCount,
int?maxBufferedItemCount,
IComparer <ItemProducerTree> moveNextComparer,
Func <ItemProducerTree, int> fetchPrioirtyFunction,
IEqualityComparer <CosmosElement> equalityComparer)
{
if (moveNextComparer == null)
{
throw new ArgumentNullException(nameof(moveNextComparer));
}
if (fetchPrioirtyFunction == null)
{
throw new ArgumentNullException(nameof(fetchPrioirtyFunction));
}
if (equalityComparer == null)
{
throw new ArgumentNullException(nameof(equalityComparer));
}
this.queryContext = queryContext ?? throw new ArgumentNullException(nameof(queryContext));
this.queryClient = queryContext.QueryClient ?? throw new ArgumentNullException(nameof(queryContext.QueryClient));
this.itemProducerForest = new PriorityQueue <ItemProducerTree>(moveNextComparer, isSynchronized: true);
this.fetchPrioirtyFunction = fetchPrioirtyFunction;
this.comparableTaskScheduler = new ComparableTaskScheduler(maxConcurrency.GetValueOrDefault(0));
this.equalityComparer = equalityComparer;
this.requestChargeTracker = new RequestChargeTracker();
this.diagnosticsPages = new ConcurrentBag <QueryPageDiagnostics>();
this.actualMaxPageSize = maxItemCount.GetValueOrDefault(ParallelQueryConfig.GetConfig().ClientInternalMaxItemCount);
if (this.actualMaxPageSize < 0)
{
throw new ArgumentOutOfRangeException("actualMaxPageSize should never be less than 0");
}
if (this.actualMaxPageSize > int.MaxValue)
{
throw new ArgumentOutOfRangeException("actualMaxPageSize should never be greater than int.MaxValue");
}
if (maxBufferedItemCount.HasValue)
{
this.actualMaxBufferedItemCount = maxBufferedItemCount.Value;
}
else
{
this.actualMaxBufferedItemCount = ParallelQueryConfig.GetConfig().DefaultMaximumBufferSize;
}
if (this.actualMaxBufferedItemCount < 0)
{
throw new ArgumentOutOfRangeException("actualMaxBufferedItemCount should never be less than 0");
}
if (this.actualMaxBufferedItemCount > int.MaxValue)
{
throw new ArgumentOutOfRangeException("actualMaxBufferedItemCount should never be greater than int.MaxValue");
}
this.CanPrefetch = maxConcurrency.HasValue && maxConcurrency.Value != 0;
}
public CosmosQueryExecutionContextFactory(
CosmosQueryClient client,
ResourceType resourceTypeEnum,
OperationType operationType,
Type resourceType,
SqlQuerySpec sqlQuerySpec,
QueryRequestOptions queryRequestOptions,
Uri resourceLink,
bool isContinuationExpected,
bool allowNonValueAggregateQuery,
Guid correlatedActivityId)
{
if (client == null)
{
throw new ArgumentNullException(nameof(client));
}
if (sqlQuerySpec == null)
{
throw new ArgumentNullException(nameof(sqlQuerySpec));
}
if (queryRequestOptions == null)
{
throw new ArgumentNullException(nameof(queryRequestOptions));
}
if (resourceLink == null)
{
throw new ArgumentNullException(nameof(resourceLink));
}
// Prevent users from updating the values after creating the execution context.
QueryRequestOptions cloneQueryRequestOptions = queryRequestOptions.Clone();
// Swapping out negative values in feedOptions for int.MaxValue
if (cloneQueryRequestOptions.MaxBufferedItemCount.HasValue && cloneQueryRequestOptions.MaxBufferedItemCount < 0)
{
cloneQueryRequestOptions.MaxBufferedItemCount = int.MaxValue;
}
if (cloneQueryRequestOptions.MaxConcurrency.HasValue && cloneQueryRequestOptions.MaxConcurrency < 0)
{
cloneQueryRequestOptions.MaxConcurrency = int.MaxValue;
}
if (cloneQueryRequestOptions.MaxItemCount.HasValue && cloneQueryRequestOptions.MaxItemCount < 0)
{
cloneQueryRequestOptions.MaxItemCount = int.MaxValue;
}
this.cosmosQueryContext = new CosmosQueryContext(
client: client,
resourceTypeEnum: resourceTypeEnum,
operationType: operationType,
resourceType: resourceType,
sqlQuerySpecFromUser: sqlQuerySpec,
queryRequestOptions: cloneQueryRequestOptions,
resourceLink: resourceLink,
getLazyFeedResponse: isContinuationExpected,
isContinuationExpected: isContinuationExpected,
allowNonValueAggregateQuery: allowNonValueAggregateQuery,
correlatedActivityId: correlatedActivityId);
}
public static async Task <CosmosQueryExecutionContext> CreateSpecializedDocumentQueryExecutionContextAsync(
CosmosQueryContext cosmosQueryContext,
PartitionedQueryExecutionInfo partitionedQueryExecutionInfo,
List <PartitionKeyRange> targetRanges,
string collectionRid,
CancellationToken cancellationToken)
{
if (!string.IsNullOrEmpty(partitionedQueryExecutionInfo.QueryInfo?.RewrittenQuery))
{
cosmosQueryContext.SqlQuerySpec = new SqlQuerySpec(
partitionedQueryExecutionInfo.QueryInfo.RewrittenQuery,
cosmosQueryContext.SqlQuerySpec.Parameters);
}
// Figure out the optimal page size.
long initialPageSize = cosmosQueryContext.QueryRequestOptions.MaxItemCount.GetValueOrDefault(ParallelQueryConfig.GetConfig().ClientInternalPageSize);
if (initialPageSize < -1 || initialPageSize == 0)
{
throw new BadRequestException(string.Format(CultureInfo.InvariantCulture, "Invalid MaxItemCount {0}", initialPageSize));
}
QueryInfo queryInfo = partitionedQueryExecutionInfo.QueryInfo;
bool getLazyFeedResponse = queryInfo.HasTop;
// We need to compute the optimal initial page size for order-by queries
if (queryInfo.HasOrderBy)
{
int top;
if (queryInfo.HasTop && (top = partitionedQueryExecutionInfo.QueryInfo.Top.Value) > 0)
{
// All partitions should initially fetch about 1/nth of the top value.
long pageSizeWithTop = (long)Math.Min(
Math.Ceiling(top / (double)targetRanges.Count) * PageSizeFactorForTop,
top);
if (initialPageSize > 0)
{
initialPageSize = Math.Min(pageSizeWithTop, initialPageSize);
}
else
{
initialPageSize = pageSizeWithTop;
}
}
else if (cosmosQueryContext.IsContinuationExpected)
{
if (initialPageSize < 0)
{
if (cosmosQueryContext.QueryRequestOptions.MaxBufferedItemCount.HasValue)
{
// Max of what the user is willing to buffer and the default (note this is broken if MaxBufferedItemCount = -1)
initialPageSize = Math.Max(cosmosQueryContext.QueryRequestOptions.MaxBufferedItemCount.Value, ParallelQueryConfig.GetConfig().DefaultMaximumBufferSize);
}
else
{
initialPageSize = ParallelQueryConfig.GetConfig().DefaultMaximumBufferSize;
}
}
initialPageSize = (long)Math.Min(
Math.Ceiling(initialPageSize / (double)targetRanges.Count) * PageSizeFactorForTop,
initialPageSize);
}
}
Debug.Assert(initialPageSize > 0 && initialPageSize <= int.MaxValue,
string.Format(CultureInfo.InvariantCulture, "Invalid MaxItemCount {0}", initialPageSize));
return(await PipelinedDocumentQueryExecutionContext.CreateAsync(
cosmosQueryContext,
collectionRid,
partitionedQueryExecutionInfo,
targetRanges,
(int)initialPageSize,
cosmosQueryContext.QueryRequestOptions.RequestContinuation,
cancellationToken));
}
private async Task <CosmosQueryExecutionContext> CreateItemQueryExecutionContextAsync(
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
CosmosContainerSettings containerSettings = await this.GetContainerSettingsAsync(cancellationToken);
this.cosmosQueryContext.ContainerResourceId = containerSettings.ResourceId;
PartitionedQueryExecutionInfo partitionedQueryExecutionInfo;
if (this.cosmosQueryContext.QueryClient.ByPassQueryParsing() && TestFlag)
{
// For non-Windows platforms(like Linux and OSX) in .NET Core SDK, we cannot use ServiceInterop, so need to bypass in that case.
// We are also now bypassing this for 32 bit host process running even on Windows as there are many 32 bit apps that will not work without this
partitionedQueryExecutionInfo = await QueryPlanRetriever.GetQueryPlanThroughGatewayAsync(
this.cosmosQueryContext.QueryClient,
this.cosmosQueryContext.SqlQuerySpec,
this.cosmosQueryContext.ResourceLink,
cancellationToken);
}
else
{
//todo:elasticcollections this may rely on information from collection cache which is outdated
//if collection is deleted/created with same name.
//need to make it not rely on information from collection cache.
PartitionKeyDefinition partitionKeyDefinition;
object partitionKeyDefinitionObject;
if (this.cosmosQueryContext.QueryRequestOptions?.Properties != null &&
this.cosmosQueryContext.QueryRequestOptions.Properties.TryGetValue(InternalPartitionKeyDefinitionProperty, out partitionKeyDefinitionObject))
{
if (partitionKeyDefinitionObject is PartitionKeyDefinition definition)
{
partitionKeyDefinition = definition;
}
else
{
throw new ArgumentException(
"partitionkeydefinition has invalid type",
nameof(partitionKeyDefinitionObject));
}
}
else
{
partitionKeyDefinition = containerSettings.PartitionKey;
}
partitionedQueryExecutionInfo = await QueryPlanRetriever.GetQueryPlanWithServiceInteropAsync(
this.cosmosQueryContext.QueryClient,
this.cosmosQueryContext.SqlQuerySpec,
partitionKeyDefinition,
cancellationToken);
}
List <PartitionKeyRange> targetRanges = await GetTargetPartitionKeyRangesAsync(
this.cosmosQueryContext.QueryClient,
this.cosmosQueryContext.ResourceLink.OriginalString,
partitionedQueryExecutionInfo,
containerSettings,
this.cosmosQueryContext.QueryRequestOptions);
CosmosQueryContext rewrittenComosQueryContext;
if (!string.IsNullOrEmpty(partitionedQueryExecutionInfo.QueryInfo.RewrittenQuery))
{
// We need pass down the rewritten query.
SqlQuerySpec rewrittenQuerySpec = new SqlQuerySpec()
{
QueryText = partitionedQueryExecutionInfo.QueryInfo.RewrittenQuery,
Parameters = this.cosmosQueryContext.SqlQuerySpec.Parameters
};
rewrittenComosQueryContext = new CosmosQueryContext(
this.cosmosQueryContext.QueryClient,
this.cosmosQueryContext.ResourceTypeEnum,
this.cosmosQueryContext.OperationTypeEnum,
this.cosmosQueryContext.ResourceType,
rewrittenQuerySpec,
this.cosmosQueryContext.QueryRequestOptions,
this.cosmosQueryContext.ResourceLink,
this.cosmosQueryContext.IsContinuationExpected,
this.cosmosQueryContext.CorrelatedActivityId,
this.cosmosQueryContext.IsContinuationExpected,
this.cosmosQueryContext.AllowNonValueAggregateQuery,
this.cosmosQueryContext.ContainerResourceId);
}
else
{
rewrittenComosQueryContext = this.cosmosQueryContext;
}
return(await CreateSpecializedDocumentQueryExecutionContextAsync(
rewrittenComosQueryContext,
partitionedQueryExecutionInfo,
targetRanges,
containerSettings.ResourceId,
cancellationToken));
}
/// <summary>
/// Initializes a new instance of the ItemProducerTree class.
/// </summary>
/// <param name="queryContext">query context.</param>
/// <param name="querySpecForInit">query spec init.</param>
/// <param name="partitionKeyRange">The partition key range.</param>
/// <param name="produceAsyncCompleteCallback">Callback to invoke once a fetch finishes.</param>
/// <param name="itemProducerTreeComparer">Comparer to determine, which tree to produce from.</param>
/// <param name="equalityComparer">Comparer to see if we need to return the continuation token for a partition.</param>
/// <param name="deferFirstPage">Whether or not to defer fetching the first page.</param>
/// <param name="collectionRid">The collection to drain from.</param>
/// <param name="initialPageSize">The initial page size.</param>
/// <param name="initialContinuationToken">The initial continuation token.</param>
public ItemProducerTree(
CosmosQueryContext queryContext,
SqlQuerySpec querySpecForInit,
PartitionKeyRange partitionKeyRange,
Action <ItemProducerTree, int, double, QueryMetrics, long, CancellationToken> produceAsyncCompleteCallback,
IComparer <ItemProducerTree> itemProducerTreeComparer,
IEqualityComparer <CosmosElement> equalityComparer,
bool deferFirstPage,
string collectionRid,
long initialPageSize = 50,
string initialContinuationToken = null)
{
if (queryContext == null)
{
throw new ArgumentNullException($"{nameof(queryContext)}");
}
if (itemProducerTreeComparer == null)
{
throw new ArgumentNullException($"{nameof(itemProducerTreeComparer)}");
}
if (produceAsyncCompleteCallback == null)
{
throw new ArgumentNullException($"{nameof(produceAsyncCompleteCallback)}");
}
if (itemProducerTreeComparer == null)
{
throw new ArgumentNullException($"{nameof(itemProducerTreeComparer)}");
}
if (equalityComparer == null)
{
throw new ArgumentNullException($"{nameof(equalityComparer)}");
}
if (string.IsNullOrEmpty(collectionRid))
{
throw new ArgumentException($"{nameof(collectionRid)} can not be null or empty.");
}
this.Root = new ItemProducer(
queryContext,
querySpecForInit,
partitionKeyRange,
(itemProducer, itemsBuffered, resourceUnitUsage, queryMetrics, requestLength, token) => produceAsyncCompleteCallback(this, itemsBuffered, resourceUnitUsage, queryMetrics, requestLength, token),
equalityComparer,
initialPageSize,
initialContinuationToken);
this.queryClient = queryContext.QueryClient;
this.children = new PriorityQueue <ItemProducerTree>(itemProducerTreeComparer, true);
this.deferFirstPage = deferFirstPage;
this.collectionRid = collectionRid;
this.createItemProducerTreeCallback = ItemProducerTree.CreateItemProducerTreeCallback(
queryContext,
querySpecForInit,
produceAsyncCompleteCallback,
itemProducerTreeComparer,
equalityComparer,
deferFirstPage,
collectionRid,
initialPageSize);
this.executeWithSplitProofingSemaphore = new SemaphoreSlim(1, 1);
}
/// <summary>
/// Creates a CosmosPipelinedItemQueryExecutionContext.
/// </summary>
/// <param name="executionEnvironment">The environment to execute on.</param>
/// <param name="queryContext">The parameters for constructing the base class.</param>
/// <param name="initParams">The initial parameters</param>
/// <param name="requestContinuationToken">The request continuation.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>A task to await on, which in turn returns a CosmosPipelinedItemQueryExecutionContext.</returns>
public static async Task <CosmosQueryExecutionContext> CreateAsync(
ExecutionEnvironment executionEnvironment,
CosmosQueryContext queryContext,
CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams initParams,
string requestContinuationToken,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
if (requestContinuationToken != null)
{
if (!PipelineContinuationToken.TryParse(
requestContinuationToken,
out PipelineContinuationToken pipelineContinuationToken))
{
throw queryContext.QueryClient.CreateBadRequestException(
$"Malformed {nameof(PipelineContinuationToken)}: {requestContinuationToken}.");
}
if (PipelineContinuationToken.IsTokenFromTheFuture(pipelineContinuationToken))
{
throw queryContext.QueryClient.CreateBadRequestException(
$"{nameof(PipelineContinuationToken)} Continuation token is from a newer version of the SDK. Upgrade the SDK to avoid this issue.\n {requestContinuationToken}.");
}
if (!PipelineContinuationToken.TryConvertToLatest(
pipelineContinuationToken,
out PipelineContinuationTokenV1_1 latestVersionPipelineContinuationToken))
{
throw queryContext.QueryClient.CreateBadRequestException(
$"{nameof(PipelineContinuationToken)}: '{requestContinuationToken}' is no longer supported.");
}
requestContinuationToken = latestVersionPipelineContinuationToken.SourceContinuationToken;
}
QueryInfo queryInfo = initParams.PartitionedQueryExecutionInfo.QueryInfo;
int initialPageSize = initParams.InitialPageSize;
CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams parameters = initParams;
if (queryInfo.HasGroupBy)
{
// The query will block until all groupings are gathered so we might as well speed up the process.
initParams = new CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams(
sqlQuerySpec: initParams.SqlQuerySpec,
collectionRid: initParams.CollectionRid,
partitionedQueryExecutionInfo: initParams.PartitionedQueryExecutionInfo,
partitionKeyRanges: initParams.PartitionKeyRanges,
initialPageSize: int.MaxValue,
maxConcurrency: initParams.MaxConcurrency,
maxItemCount: int.MaxValue,
maxBufferedItemCount: initParams.MaxBufferedItemCount);
}
Func <string, Task <IDocumentQueryExecutionComponent> > createOrderByComponentFunc = async(continuationToken) =>
{
return(await CosmosOrderByItemQueryExecutionContext.CreateAsync(
queryContext,
initParams,
continuationToken,
cancellationToken));
};
Func <string, Task <IDocumentQueryExecutionComponent> > createParallelComponentFunc = async(continuationToken) =>
{
return(await CosmosParallelItemQueryExecutionContext.CreateAsync(
queryContext,
initParams,
continuationToken,
cancellationToken));
};
return((CosmosQueryExecutionContext)await PipelinedDocumentQueryExecutionContext.CreateHelperAsync(
executionEnvironment,
queryContext.QueryClient,
initParams.PartitionedQueryExecutionInfo.QueryInfo,
initialPageSize,
requestContinuationToken,
createOrderByComponentFunc,
createParallelComponentFunc));
}
public override async Task <ResponseMessage> ReadNextAsync(CancellationToken cancellationToken = default(CancellationToken))
{
// This catches exception thrown by the pipeline and converts it to QueryResponse
ResponseMessage response;
try
{
QueryResponseCore responseCore = await this.cosmosQueryExecutionContext.ExecuteNextAsync(cancellationToken);
CosmosQueryContext cosmosQueryContext = this.cosmosQueryExecutionContext.CosmosQueryContext;
QueryResponse queryResponse;
if (responseCore.IsSuccess)
{
queryResponse = QueryResponse.CreateSuccess(
result: responseCore.CosmosElements,
count: responseCore.CosmosElements.Count,
responseLengthBytes: responseCore.ResponseLengthBytes,
queryMetrics: responseCore.QueryMetrics,
responseHeaders: new CosmosQueryResponseMessageHeaders(
responseCore.ContinuationToken,
responseCore.DisallowContinuationTokenMessage,
cosmosQueryContext.ResourceTypeEnum,
cosmosQueryContext.ContainerResourceId)
{
RequestCharge = responseCore.RequestCharge,
ActivityId = responseCore.ActivityId
});
}
else
{
queryResponse = QueryResponse.CreateFailure(
statusCode: responseCore.StatusCode,
error: null,
errorMessage: responseCore.ErrorMessage,
requestMessage: null,
responseHeaders: new CosmosQueryResponseMessageHeaders(
responseCore.ContinuationToken,
responseCore.DisallowContinuationTokenMessage,
cosmosQueryContext.ResourceTypeEnum,
cosmosQueryContext.ContainerResourceId)
{
RequestCharge = responseCore.RequestCharge,
ActivityId = responseCore.ActivityId
});
}
if (responseCore.QueryMetrics != null && responseCore.QueryMetrics.Count > 0)
{
queryResponse.Diagnostics = new QueryOperationStatistics(responseCore.QueryMetrics);
}
queryResponse.CosmosSerializationOptions = cosmosQueryContext.QueryRequestOptions.CosmosSerializationFormatOptions;
response = queryResponse;
}
catch (Documents.DocumentClientException exception)
{
response = exception.ToCosmosResponseMessage(request: null);
}
catch (CosmosException exception)
{
response = exception.ToCosmosResponseMessage(request: null);
}
catch (AggregateException ae)
{
response = TransportHandler.AggregateExceptionConverter(ae, null);
if (response == null)
{
throw;
}
}
return(response);
}
/// <summary>
/// Creates a CosmosPipelinedItemQueryExecutionContext.
/// </summary>
/// <param name="constructorParams">The parameters for constructing the base class.</param>
/// <param name="collectionRid">The collection rid.</param>
/// <param name="partitionedQueryExecutionInfo">The partitioned query execution info.</param>
/// <param name="partitionKeyRanges">The partition key ranges.</param>
/// <param name="initialPageSize">The initial page size.</param>
/// <param name="requestContinuation">The request continuation.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>A task to await on, which in turn returns a CosmosPipelinedItemQueryExecutionContext.</returns>
public static async Task <IDocumentQueryExecutionContext> CreateAsync(
CosmosQueryContext constructorParams,
string collectionRid,
PartitionedQueryExecutionInfo partitionedQueryExecutionInfo,
List <PartitionKeyRange> partitionKeyRanges,
int initialPageSize,
string requestContinuation,
CancellationToken cancellationToken)
{
DefaultTrace.TraceInformation(
string.Format(
CultureInfo.InvariantCulture,
"{0}, CorrelatedActivityId: {1} | Pipelined~Context.CreateAsync",
DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture),
constructorParams.CorrelatedActivityId));
Func <string, Task <IDocumentQueryExecutionComponent> > createComponentFunc;
QueryInfo queryInfo = partitionedQueryExecutionInfo.QueryInfo;
if (queryInfo.HasOrderBy)
{
createComponentFunc = async(continuationToken) =>
{
CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams initParams = new CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams(
collectionRid,
partitionedQueryExecutionInfo,
partitionKeyRanges,
initialPageSize,
continuationToken);
return(await CosmosOrderByItemQueryExecutionContext.CreateAsync(
constructorParams,
initParams,
cancellationToken));
};
}
else
{
createComponentFunc = async(continuationToken) =>
{
CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams initParams = new CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams(
collectionRid,
partitionedQueryExecutionInfo,
partitionKeyRanges,
initialPageSize,
continuationToken);
return(await CosmosParallelItemQueryExecutionContext.CreateAsync(
constructorParams,
initParams,
cancellationToken));
};
}
if (queryInfo.HasAggregates)
{
Func <string, Task <IDocumentQueryExecutionComponent> > createSourceCallback = createComponentFunc;
createComponentFunc = async(continuationToken) =>
{
return(await AggregateDocumentQueryExecutionComponent.CreateAsync(
queryInfo.Aggregates,
continuationToken,
createSourceCallback));
};
}
if (queryInfo.HasDistinct)
{
Func <string, Task <IDocumentQueryExecutionComponent> > createSourceCallback = createComponentFunc;
createComponentFunc = async(continuationToken) =>
{
return(await DistinctDocumentQueryExecutionComponent.CreateAsync(
continuationToken,
createSourceCallback,
queryInfo.DistinctType));
};
}
if (queryInfo.HasOffset)
{
if (!constructorParams.QueryRequestOptions.EnableCrossPartitionSkipTake)
{
throw new ArgumentException("Cross Partition OFFSET / LIMIT is not supported.");
}
Func <string, Task <IDocumentQueryExecutionComponent> > createSourceCallback = createComponentFunc;
createComponentFunc = async(continuationToken) =>
{
return(await SkipDocumentQueryExecutionComponent.CreateAsync(
queryInfo.Offset.Value,
continuationToken,
createSourceCallback));
};
}
if (queryInfo.HasLimit)
{
if (!constructorParams.QueryRequestOptions.EnableCrossPartitionSkipTake)
{
throw new ArgumentException("Cross Partition OFFSET / LIMIT is not supported.");
}
Func <string, Task <IDocumentQueryExecutionComponent> > createSourceCallback = createComponentFunc;
createComponentFunc = async(continuationToken) =>
{
return(await TakeDocumentQueryExecutionComponent.CreateLimitDocumentQueryExecutionComponentAsync(
queryInfo.Limit.Value,
continuationToken,
createSourceCallback));
};
}
if (queryInfo.HasTop)
{
Func <string, Task <IDocumentQueryExecutionComponent> > createSourceCallback = createComponentFunc;
createComponentFunc = async(continuationToken) =>
{
return(await TakeDocumentQueryExecutionComponent.CreateTopDocumentQueryExecutionComponentAsync(
queryInfo.Top.Value,
continuationToken,
createSourceCallback));
};
}
return(new CosmosPipelinedItemQueryExecutionContext(
await createComponentFunc(requestContinuation), initialPageSize));
}