/// <summary>
/// Creates an DistinctDocumentQueryExecutionComponent
/// </summary>
/// <param name="queryClient">The query client</param>
/// <param name="requestContinuation">The continuation token.</param>
/// <param name="createSourceCallback">The callback to create the source to drain from.</param>
/// <param name="distinctQueryType">The type of distinct query.</param>
/// <returns>A task to await on and in return </returns>
public static async Task <DistinctDocumentQueryExecutionComponent> CreateAsync(
CosmosQueryClient queryClient,
string requestContinuation,
Func <string, Task <IDocumentQueryExecutionComponent> > createSourceCallback,
DistinctQueryType distinctQueryType)
{
DistinctContinuationToken distinctContinuationToken;
if (requestContinuation != null)
{
if (!DistinctContinuationToken.TryParse(requestContinuation, out distinctContinuationToken))
{
throw queryClient.CreateBadRequestException($"Invalid {nameof(DistinctContinuationToken)}: {requestContinuation}");
}
}
else
{
distinctContinuationToken = new DistinctContinuationToken(sourceToken: null, distinctMapToken: null);
}
DistinctMap distinctMap = DistinctMap.Create(distinctQueryType, distinctContinuationToken.DistinctMapToken);
IDocumentQueryExecutionComponent source = await createSourceCallback(distinctContinuationToken.SourceToken);
return(new ComputeDistinctDocumentQueryExecutionComponent(
distinctQueryType,
distinctMap,
source));
}
/// <summary>
/// Initializes a new instance of the PipelinedDocumentQueryExecutionContext class.
/// </summary>
/// <param name="component">The root level component that all calls will be forwarded to.</param>
/// <param name="actualPageSize">The actual page size to drain.</param>
private PipelinedDocumentQueryExecutionContext(
IDocumentQueryExecutionComponent component,
int actualPageSize)
{
this.component = component ?? throw new ArgumentNullException($"{nameof(component)} can not be null.");
this.actualPageSize = (actualPageSize < 0) ? throw new ArgumentOutOfRangeException($"{nameof(actualPageSize)} can not be negative.") : actualPageSize;
}
protected TakeDocumentQueryExecutionComponent(
IDocumentQueryExecutionComponent source,
int takeCount)
: base(source)
{
this.takeCount = takeCount;
}
protected GroupByDocumentQueryExecutionComponent(
IDocumentQueryExecutionComponent source,
GroupingTable groupingTable)
: base(source)
{
this.groupingTable = groupingTable ?? throw new ArgumentNullException(nameof(groupingTable));
}
/// <summary>
/// Initializes a new instance of the AggregateDocumentQueryExecutionComponent class.
/// </summary>
/// <param name="source">The source component that will supply the local aggregates from multiple continuations and partitions.</param>
/// <param name="aggregateOperators">The aggregate operators for this query.</param>
/// <remarks>This constructor is private since there is some async initialization that needs to happen in CreateAsync().</remarks>
private AggregateDocumentQueryExecutionComponent(IDocumentQueryExecutionComponent source, AggregateOperator[] aggregateOperators)
: base(source)
{
this.aggregators = new IAggregator[aggregateOperators.Length];
for (int i = 0; i < aggregateOperators.Length; ++i)
{
switch (aggregateOperators[i])
{
case AggregateOperator.Average:
this.aggregators[i] = new AverageAggregator();
break;
case AggregateOperator.Count:
this.aggregators[i] = new CountAggregator();
break;
case AggregateOperator.Max:
this.aggregators[i] = new MinMaxAggregator(false);
break;
case AggregateOperator.Min:
this.aggregators[i] = new MinMaxAggregator(true);
break;
case AggregateOperator.Sum:
this.aggregators[i] = new SumAggregator();
break;
default:
string errorMessage = "Unexpected value: " + aggregateOperators[i].ToString();
Debug.Assert(false, errorMessage);
throw new InvalidProgramException(errorMessage);
}
}
}
private GroupByDocumentQueryExecutionComponent(
CosmosQueryClient cosmosQueryClient,
IReadOnlyDictionary <string, AggregateOperator?> groupByAliasToAggregateType,
IReadOnlyList <string> orderedAliases,
bool hasSelectValue,
IDocumentQueryExecutionComponent source)
: base(source)
{
if (cosmosQueryClient == null)
{
throw new ArgumentNullException(nameof(cosmosQueryClient));
}
if (groupByAliasToAggregateType == null)
{
throw new ArgumentNullException(nameof(groupByAliasToAggregateType));
}
if (orderedAliases == null)
{
throw new ArgumentNullException(nameof(orderedAliases));
}
this.cosmosQueryClient = cosmosQueryClient;
this.groupingTable = new Dictionary <UInt192, SingleGroupAggregator>();
this.groupByAliasToAggregateType = groupByAliasToAggregateType;
this.orderedAliases = orderedAliases;
this.hasSelectValue = hasSelectValue;
}
protected DistinctDocumentQueryExecutionComponent(
DistinctMap distinctMap,
IDocumentQueryExecutionComponent source)
: base(source)
{
this.distinctMap = distinctMap ?? throw new ArgumentNullException(nameof(distinctMap));
}
private ComputeDistinctDocumentQueryExecutionComponent(
DistinctQueryType distinctQueryType,
DistinctMap distinctMap,
IDocumentQueryExecutionComponent source)
: base(distinctMap, source)
{
}
private ComputeAggregateDocumentQueryExecutionComponent(
IDocumentQueryExecutionComponent source,
SingleGroupAggregator singleGroupAggregator,
bool isValueAggregateQuery)
: base(source, singleGroupAggregator, isValueAggregateQuery)
{
// all the work is done in the base constructor.
}
private ClientGroupByDocumentQueryExecutionComponent(
IDocumentQueryExecutionComponent source,
GroupingTable groupingTable)
: base(
source,
groupingTable)
{
}
private TakeDocumentQueryExecutionComponent(
IDocumentQueryExecutionComponent source,
int takeCount,
TakeEnum takeEnum)
: base(source)
{
this.takeCount = takeCount;
this.takeEnum = takeEnum;
}
/// <summary>
/// Initializes a new instance of the AggregateDocumentQueryExecutionComponent class.
/// </summary>
/// <param name="source">The source component that will supply the local aggregates from multiple continuations and partitions.</param>
/// <param name="singleGroupAggregator">The single group aggregator that we will feed results into.</param>
/// <param name="isValueAggregateQuery">Whether or not the query has the 'VALUE' keyword.</param>
/// <remarks>This constructor is private since there is some async initialization that needs to happen in CreateAsync().</remarks>
protected AggregateDocumentQueryExecutionComponent(
IDocumentQueryExecutionComponent source,
SingleGroupAggregator singleGroupAggregator,
bool isValueAggregateQuery)
: base(source)
{
this.singleGroupAggregator = singleGroupAggregator ?? throw new ArgumentNullException(nameof(singleGroupAggregator));
this.isValueAggregateQuery = isValueAggregateQuery;
}
/// <summary>
/// Initializes a new instance of the DocumentQueryExecutionComponentBase class.
/// </summary>
/// <param name="source">The source to drain documents from.</param>
protected DocumentQueryExecutionComponentBase(IDocumentQueryExecutionComponent source)
{
if (source == null)
{
throw new ArgumentNullException("source for a component can not be null.");
}
this.Source = source;
}
protected SkipDocumentQueryExecutionComponent(IDocumentQueryExecutionComponent source, long skipCount)
: base(source)
{
if (skipCount > int.MaxValue)
{
throw new ArgumentOutOfRangeException(nameof(skipCount));
}
this.skipCount = (int)skipCount;
}
private TakeDocumentQueryExecutionComponent(IDocumentQueryExecutionComponent source, int takeCount, TakeEnum takeEnum)
: base(source)
{
if (takeCount < 0)
{
throw new ArgumentException($"{nameof(takeCount)} must be a non negative number.");
}
this.takeCount = takeCount;
this.takeEnum = takeEnum;
}
private ClientDistinctDocumentQueryExecutionComponent(
DistinctQueryType distinctQueryType,
DistinctMap distinctMap,
IDocumentQueryExecutionComponent source)
: base(distinctMap, source)
{
if ((distinctQueryType != DistinctQueryType.Unordered) && (distinctQueryType != DistinctQueryType.Ordered))
{
throw new ArgumentException($"Unknown {nameof(DistinctQueryType)}: {distinctQueryType}.");
}
this.distinctQueryType = distinctQueryType;
}
/// <summary>
/// Initializes a new instance of the DistinctDocumentQueryExecutionComponent class.
/// </summary>
/// <param name="distinctQueryType">The type of distinct query.</param>
/// <param name="previousHash">The previous that distinct map saw.</param>
/// <param name="source">The source to drain from.</param>
private DistinctDocumentQueryExecutionComponent(
DistinctQueryType distinctQueryType,
UInt192?previousHash,
IDocumentQueryExecutionComponent source)
: base(source)
{
if (distinctQueryType == DistinctQueryType.None)
{
throw new ArgumentException("It doesn't make sense to create a distinct component of type None.");
}
this.distinctQueryType = distinctQueryType;
this.distinctMap = DistinctMap.Create(distinctQueryType, previousHash);
}
private GroupByDocumentQueryExecutionComponent(
IReadOnlyDictionary <string, AggregateOperator?> groupByAliasToAggregateType,
IDocumentQueryExecutionComponent source)
: base(source)
{
if (groupByAliasToAggregateType == null)
{
throw new ArgumentNullException(nameof(groupByAliasToAggregateType));
}
this.groupingTable = new Dictionary <UInt192, GroupByValues>();
// Using an ordered distinct map to get hashes.
this.distinctMap = DistinctMap.Create(DistinctQueryType.Ordered, null);
this.groupByAliasToAggregateType = groupByAliasToAggregateType;
}
/// <summary>
/// Initializes a new instance of the PipelinedDocumentQueryExecutionContext class.
/// </summary>
/// <param name="component">The root level component that all calls will be forwarded to.</param>
/// <param name="actualPageSize">The actual page size to drain.</param>
private PipelinedDocumentQueryExecutionContext(
IDocumentQueryExecutionComponent component,
int actualPageSize)
{
if (component == null)
{
throw new ArgumentNullException($"{nameof(component)} can not be null.");
}
if (actualPageSize < 0)
{
throw new ArgumentException($"{nameof(actualPageSize)} can not be negative.");
}
this.component = component;
this.actualPageSize = actualPageSize;
}
private PipelinedDocumentQueryExecutionContext(
IDocumentQueryExecutionComponent component,
int actualPageSize,
Guid correlatedActivityId)
{
this.component = component;
this.actualPageSize = actualPageSize;
this.correlatedActivityId = correlatedActivityId;
this.executeNextSchedulingMetrics = new SchedulingStopwatch();
this.executeNextSchedulingMetrics.Ready();
DefaultTrace.TraceVerbose(string.Format(
CultureInfo.InvariantCulture,
"{0} Pipelined~Context, actual page size: {1}",
DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture),
this.actualPageSize));
}
public static async Task <IDocumentQueryExecutionComponent> CreateAsync(
CosmosQueryClient queryClient,
AggregateOperator[] aggregates,
IReadOnlyDictionary <string, AggregateOperator?> aliasToAggregateType,
IReadOnlyList <string> orderedAliases,
bool hasSelectValue,
string requestContinuation,
Func <string, Task <IDocumentQueryExecutionComponent> > createSourceCallback)
{
string sourceContinuationToken;
string singleGroupAggregatorContinuationToken;
if (requestContinuation != null)
{
if (!AggregateContinuationToken.TryParse(requestContinuation, out AggregateContinuationToken aggregateContinuationToken))
{
throw queryClient.CreateBadRequestException($"Malfomed {nameof(AggregateContinuationToken)}: '{requestContinuation}'");
}
sourceContinuationToken = aggregateContinuationToken.SourceContinuationToken;
singleGroupAggregatorContinuationToken = aggregateContinuationToken.SingleGroupAggregatorContinuationToken;
}
else
{
sourceContinuationToken = null;
singleGroupAggregatorContinuationToken = null;
}
IDocumentQueryExecutionComponent source = await createSourceCallback(sourceContinuationToken);
SingleGroupAggregator singleGroupAggregator = SingleGroupAggregator.Create(
queryClient,
aggregates,
aliasToAggregateType,
orderedAliases,
hasSelectValue,
singleGroupAggregatorContinuationToken);
return(new ComputeAggregateDocumentQueryExecutionComponent(
source,
singleGroupAggregator,
hasSelectValue));
}
public static async Task <IDocumentQueryExecutionComponent> CreateAsync(
CosmosQueryClient cosmosQueryClient,
string requestContinuation,
Func <string, Task <IDocumentQueryExecutionComponent> > createSourceCallback,
IReadOnlyDictionary <string, AggregateOperator?> groupByAliasToAggregateType,
IReadOnlyList <string> orderedAliases,
bool hasSelectValue)
{
IDocumentQueryExecutionComponent source = await createSourceCallback(requestContinuation);
GroupingTable groupingTable = GroupingTable.CreateFromContinuationToken(
cosmosQueryClient,
groupByAliasToAggregateType,
orderedAliases,
hasSelectValue,
groupingTableContinuationToken: null);
return(new ClientGroupByDocumentQueryExecutionComponent(
source,
groupingTable));
}
public static async Task <IDocumentQueryExecutionComponent> CreateAsync(
CosmosQueryClient queryClient,
AggregateOperator[] aggregates,
IReadOnlyDictionary <string, AggregateOperator?> aliasToAggregateType,
IReadOnlyList <string> orderedAliases,
bool hasSelectValue,
string requestContinuation,
Func <string, Task <IDocumentQueryExecutionComponent> > createSourceCallback)
{
IDocumentQueryExecutionComponent source = await createSourceCallback(requestContinuation);
SingleGroupAggregator singleGroupAggregator = SingleGroupAggregator.Create(
queryClient,
aggregates,
aliasToAggregateType,
orderedAliases,
hasSelectValue,
continuationToken: null);
return(new ClientAggregateDocumentQueryExecutionComponent(
source,
singleGroupAggregator,
hasSelectValue));
}
private TopDocumentQueryExecutionComponent(IDocumentQueryExecutionComponent source, int topCount)
: base(source)
{
this.topCount = topCount;
}
public async Task TestCosmosOrderByQueryExecutionContextWithFailurePageAsync(bool createInitialContinuationToken)
{
int maxPageSize = 5;
List <MockPartitionResponse[]> mockResponsesScenario = MockQueryFactory.GetFailureScenarios();
Mock <CosmosQueryClient> mockQueryClient = new Mock <CosmosQueryClient>();
foreach (MockPartitionResponse[] mockResponse in mockResponsesScenario)
{
string initialContinuationToken = null;
string fullContinuationToken = null;
if (createInitialContinuationToken)
{
ToDoItem itemToRepresentPreviousQuery = ToDoItem.CreateItems(
1,
"itemToRepresentPreviousQuery",
MockQueryFactory.DefaultCollectionRid).First();
initialContinuationToken = $" - RID:{itemToRepresentPreviousQuery._rid} ==#RT:1#TRC:1";
CompositeContinuationToken compositeContinuation = new CompositeContinuationToken()
{
Range = new Documents.Routing.Range <string>(
min: MockQueryFactory.DefaultPartitionKeyRange.MinInclusive,
max: MockQueryFactory.DefaultPartitionKeyRange.MaxExclusive,
isMaxInclusive: false,
isMinInclusive: true),
Token = initialContinuationToken
};
List <OrderByItem> orderByItems = new List <OrderByItem>()
{
new OrderByItem(CosmosObject.CreateFromBuffer(Encoding.UTF8.GetBytes("{\"item\":\"2c4ce711-13c3-4c93-817c-49287b71b6c3\"}")))
};
OrderByContinuationToken orderByContinuationToken = new OrderByContinuationToken(
compositeContinuationToken: compositeContinuation,
orderByItems: orderByItems,
rid: itemToRepresentPreviousQuery._rid,
skipCount: 0,
filter: null);
fullContinuationToken = CosmosArray.Create(
new List <CosmosElement>()
{
OrderByContinuationToken.ToCosmosElement(orderByContinuationToken)
}).ToString();
}
IList <ToDoItem> allItems = MockQueryFactory.GenerateAndMockResponse(
mockQueryClient,
isOrderByQuery: true,
sqlQuerySpec: MockQueryFactory.DefaultQuerySpec,
containerRid: MockQueryFactory.DefaultCollectionRid,
initContinuationToken: initialContinuationToken,
maxPageSize: maxPageSize,
mockResponseForSinglePartition: mockResponse,
cancellationTokenForMocks: this.cancellationToken);
// Order by drains the partitions until it finds an item
// If there are no items then it's not possible to have a continuation token
if (allItems.Count == 0 && createInitialContinuationToken)
{
continue;
}
CosmosQueryContext context = MockQueryFactory.CreateContext(
mockQueryClient.Object);
QueryInfo queryInfo = new QueryInfo()
{
OrderBy = new SortOrder[] { SortOrder.Ascending },
OrderByExpressions = new string[] { "id" }
};
CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams initParams = new CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams(
sqlQuerySpec: MockQueryFactory.DefaultQuerySpec,
collectionRid: MockQueryFactory.DefaultCollectionRid,
partitionedQueryExecutionInfo: new PartitionedQueryExecutionInfo()
{
QueryInfo = queryInfo
},
partitionKeyRanges: new List <PartitionKeyRange>()
{
MockQueryFactory.DefaultPartitionKeyRange
},
initialPageSize: maxPageSize,
maxConcurrency: null,
maxItemCount: maxPageSize,
maxBufferedItemCount: null,
returnResultsInDeterministicOrder: true,
testSettings: new TestInjections(simulate429s: false, simulateEmptyPages: false));
TryCatch <IDocumentQueryExecutionComponent> tryCreate = await CosmosOrderByItemQueryExecutionContext.TryCreateAsync(
context,
initParams,
fullContinuationToken != null?CosmosElement.Parse(fullContinuationToken) : null,
this.cancellationToken);
if (tryCreate.Succeeded)
{
IDocumentQueryExecutionComponent executionContext = tryCreate.Result;
Assert.IsTrue(!executionContext.IsDone);
// Read all the pages from both splits
List <ToDoItem> itemsRead = new List <ToDoItem>();
QueryResponseCore?failure = null;
while (!executionContext.IsDone)
{
QueryResponseCore queryResponse = await executionContext.DrainAsync(
maxPageSize,
this.cancellationToken);
if (queryResponse.IsSuccess)
{
string responseContinuationToken = queryResponse.ContinuationToken;
foreach (CosmosElement element in queryResponse.CosmosElements)
{
string jsonValue = element.ToString();
ToDoItem item = JsonConvert.DeserializeObject <ToDoItem>(jsonValue);
itemsRead.Add(item);
}
}
else
{
Assert.IsNull(failure, "There should only be one error");
failure = queryResponse;
}
}
Assert.IsNotNull(failure);
Assert.AreEqual((HttpStatusCode)429, failure.Value.StatusCode);
Assert.IsNotNull(failure.Value.CosmosException.ToString());
Assert.AreEqual(0 /*We don't get any items, since we don't buffer the failure anymore*/, itemsRead.Count);
//CollectionAssert.AreEqual(allItems.ToList(), itemsRead, new ToDoItemComparer());
}
else
{
QueryResponseCore queryResponseCore = QueryResponseFactory.CreateFromException(tryCreate.Exception);
Assert.AreEqual((HttpStatusCode)429, queryResponseCore.StatusCode);
}
}
}
protected DocumentQueryExecutionComponentBase(IDocumentQueryExecutionComponent source)
{
this.source = source;
}
private SkipDocumentQueryExecutionComponent(IDocumentQueryExecutionComponent source, int skipCount)
: base(source)
{
this.skipCount = skipCount;
}
private ClientSkipDocumentQueryExecutionComponent(IDocumentQueryExecutionComponent source, long skipCount)
: base(source, skipCount)
{
// Work is done in base constructor.
}
public async Task TestCosmosCrossPartitionQueryExecutionContextWithEmptyPagesAndSplitAsync(bool createInitialContinuationToken)
{
int maxPageSize = 5;
List <MockPartitionResponse[]> mockResponsesScenario = MockQueryFactory.GetSplitScenarios();
foreach (MockPartitionResponse[] mockResponse in mockResponsesScenario)
{
string initialContinuationToken = null;
string fullContinuationToken = null;
if (createInitialContinuationToken)
{
initialContinuationToken = " - RID:02FYAIvUH1kCAAAAAAAAAA ==#RT:1#TRC:1";
CompositeContinuationToken compositeContinuation = new CompositeContinuationToken()
{
Range = new Documents.Routing.Range <string>(
min: MockQueryFactory.DefaultPartitionKeyRange.MinInclusive,
max: MockQueryFactory.DefaultPartitionKeyRange.MaxExclusive,
isMaxInclusive: false,
isMinInclusive: true),
Token = initialContinuationToken
};
fullContinuationToken = CosmosArray.Create(
new List <CosmosElement>()
{
CompositeContinuationToken.ToCosmosElement(compositeContinuation)
}).ToString();
}
Mock <CosmosQueryClient> mockQueryClient = new Mock <CosmosQueryClient>();
IList <ToDoItem> allItems = MockQueryFactory.GenerateAndMockResponse(
mockQueryClient,
isOrderByQuery: false,
sqlQuerySpec: MockQueryFactory.DefaultQuerySpec,
containerRid: MockQueryFactory.DefaultCollectionRid,
initContinuationToken: initialContinuationToken,
maxPageSize: maxPageSize,
mockResponseForSinglePartition: mockResponse,
cancellationTokenForMocks: this.cancellationToken);
CosmosQueryContext context = MockQueryFactory.CreateContext(
mockQueryClient.Object);
CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams initParams = new CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams(
sqlQuerySpec: MockQueryFactory.DefaultQuerySpec,
collectionRid: MockQueryFactory.DefaultCollectionRid,
partitionedQueryExecutionInfo: new PartitionedQueryExecutionInfo()
{
QueryInfo = new QueryInfo()
},
partitionKeyRanges: new List <PartitionKeyRange> {
MockQueryFactory.DefaultPartitionKeyRange
},
initialPageSize: maxPageSize,
maxConcurrency: null,
maxItemCount: maxPageSize,
maxBufferedItemCount: null,
returnResultsInDeterministicOrder: true,
testSettings: new TestInjections(simulate429s: false, simulateEmptyPages: false));
IDocumentQueryExecutionComponent executionContext = (await CosmosParallelItemQueryExecutionContext.TryCreateAsync(
context,
initParams,
fullContinuationToken != null ? CosmosElement.Parse(fullContinuationToken) : null,
this.cancellationToken)).Result;
// Read all the pages from both splits
List <ToDoItem> itemsRead = new List <ToDoItem>();
Assert.IsTrue(!executionContext.IsDone);
while (!executionContext.IsDone)
{
QueryResponseCore queryResponse = await executionContext.DrainAsync(maxPageSize, this.cancellationToken);
string responseContinuationToken = queryResponse.ContinuationToken;
foreach (CosmosElement element in queryResponse.CosmosElements)
{
string jsonValue = element.ToString();
ToDoItem item = JsonConvert.DeserializeObject <ToDoItem>(jsonValue);
itemsRead.Add(item);
}
}
Assert.AreEqual(allItems.Count, itemsRead.Count);
List <ToDoItem> exepected = allItems.OrderBy(x => x.id).ToList();
List <ToDoItem> actual = itemsRead.OrderBy(x => x.id).ToList();
CollectionAssert.AreEqual(exepected, actual, new ToDoItemComparer());
}
}
private ComputeTakeDocumentQueryExecutionComponent(IDocumentQueryExecutionComponent source, int takeCount)
: base(source, takeCount)
{
// Work is done in the base class.
}