public async Task ItemMultiplePartitionQuery()
{
IList <ToDoActivity> deleteList = await this.CreateRandomItems(3, randomPartitionKey : true);
ToDoActivity find = deleteList.First();
CosmosSqlQueryDefinition sql = new CosmosSqlQueryDefinition("select * from toDoActivity t where t.id = '" + find.id + "'");
CosmosQueryRequestOptions requestOptions = new CosmosQueryRequestOptions()
{
MaxBufferedItemCount = 10,
ResponseContinuationTokenLimitInKb = 500
};
CosmosResultSetIterator <ToDoActivity> setIterator =
this.Container.Items.CreateItemQuery <ToDoActivity>(sql, maxConcurrency: 1, maxItemCount: 1, requestOptions: requestOptions);
while (setIterator.HasMoreResults)
{
CosmosQueryResponse <ToDoActivity> iter = await setIterator.FetchNextSetAsync();
Assert.AreEqual(1, iter.Count());
ToDoActivity response = iter.First();
Assert.AreEqual(find.id, response.id);
}
}
/// <summary>
/// Initializes a new instance of the CosmosCrossPartitionQueryExecutionContext class.
/// </summary>
/// <param name="initParams">Constructor parameters for the base class.</param>
/// <param name="moveNextComparer">Comparer used to figure out that document producer tree to serve documents from next.</param>
/// <param name="fetchPrioirtyFunction">The priority function to determine which partition to fetch documents from next.</param>
/// <param name="equalityComparer">Used to determine whether we need to return the continuation token for a partition.</param>
protected CosmosCrossPartitionQueryExecutionContext(
CosmosQueryContext initParams,
IComparer <ItemProducerTree> moveNextComparer,
Func <ItemProducerTree, int> fetchPrioirtyFunction,
IEqualityComparer <CosmosElement> equalityComparer)
{
if (moveNextComparer == null)
{
throw new ArgumentNullException(nameof(moveNextComparer));
}
if (fetchPrioirtyFunction == null)
{
throw new ArgumentNullException(nameof(fetchPrioirtyFunction));
}
if (equalityComparer == null)
{
throw new ArgumentNullException(nameof(equalityComparer));
}
this.queryContext = initParams;
this.queryRequestOptions = initParams.QueryRequestOptions;
this.itemProducerForest = new PriorityQueue <ItemProducerTree>(moveNextComparer, isSynchronized: true);
this.fetchPrioirtyFunction = fetchPrioirtyFunction;
this.comparableTaskScheduler = new ComparableTaskScheduler(initParams.QueryRequestOptions.MaxConcurrency.GetValueOrDefault(0));
this.equalityComparer = equalityComparer;
this.requestChargeTracker = new RequestChargeTracker();
this.partitionedQueryMetrics = new ConcurrentBag <Tuple <string, QueryMetrics> >();
this.actualMaxPageSize = this.queryRequestOptions.MaxItemCount.GetValueOrDefault(ParallelQueryConfig.GetConfig().ClientInternalMaxItemCount);
if (this.actualMaxPageSize < 0)
{
throw new ArgumentOutOfRangeException("actualMaxPageSize should never be less than 0");
}
if (this.actualMaxPageSize > int.MaxValue)
{
throw new ArgumentOutOfRangeException("actualMaxPageSize should never be greater than int.MaxValue");
}
if (this.queryRequestOptions.MaxBufferedItemCount.HasValue)
{
this.actualMaxBufferedItemCount = this.queryRequestOptions.MaxBufferedItemCount.Value;
}
else
{
this.actualMaxBufferedItemCount = ParallelQueryConfig.GetConfig().DefaultMaximumBufferSize;
}
if (this.actualMaxBufferedItemCount < 0)
{
throw new ArgumentOutOfRangeException("actualMaxBufferedItemCount should never be less than 0");
}
if (this.actualMaxBufferedItemCount > int.MaxValue)
{
throw new ArgumentOutOfRangeException("actualMaxBufferedItemCount should never be greater than int.MaxValue");
}
}
private async Task <FeedResponse <CosmosElement> > ExecuteOnceAsync(IDocumentClientRetryPolicy retryPolicyInstance, CancellationToken cancellationToken)
{
if (this.LogicalPartitionKeyProvided())
{
return(await this.queryContext.ExecuteQueryAsync(
this.queryContext.SqlQuerySpec,
cancellationToken,
requestEnricher : (cosmosRequestMessage) =>
{
cosmosRequestMessage.Headers.Add(HttpConstants.HttpHeaders.IsContinuationExpected, bool.FalseString);
CosmosQueryRequestOptions.FillContinuationToken(cosmosRequestMessage, this.ContinuationToken);
}));
}
// For non-Windows platforms(like Linux and OSX) in .NET Core SDK, we cannot use ServiceInterop for parsing the query,
// so forcing the request through Gateway. We are also now by-passing this for 32-bit host process in NETFX on Windows
// as the ServiceInterop dll is only available in 64-bit.
return(await this.queryContext.ExecuteQueryAsync(
this.queryContext.SqlQuerySpec,
cancellationToken,
requestEnricher : (cosmosRequestMessage) =>
{
cosmosRequestMessage.UseGatewayMode = true;
cosmosRequestMessage.Headers.Add(HttpConstants.HttpHeaders.IsContinuationExpected, this.queryContext.IsContinuationExpected.ToString());
CosmosQueryRequestOptions.FillContinuationToken(cosmosRequestMessage, this.ContinuationToken);
}));
}
/// <summary>
/// Gets the list of partition key ranges.
/// 1. Check partition key range id
/// 2. Check Partition key
/// 3. Check the effective partition key
/// 4. Get the range from the PartitionedQueryExecutionInfo
/// </summary>
internal static async Task <List <PartitionKeyRange> > GetTargetPartitionKeyRanges(
CosmosQueryClient queryClient,
string resourceLink,
PartitionedQueryExecutionInfo partitionedQueryExecutionInfo,
CosmosContainerSettings collection,
CosmosQueryRequestOptions queryRequestOptions)
{
List <PartitionKeyRange> targetRanges;
if (queryRequestOptions.PartitionKey != null)
{
targetRanges = await queryClient.GetTargetPartitionKeyRangesByEpkString(
resourceLink,
collection.ResourceId,
new PartitionKey(queryRequestOptions.PartitionKey).InternalKey.GetEffectivePartitionKeyString(collection.PartitionKey));
}
else if (TryGetEpkProperty(queryRequestOptions, out string effectivePartitionKeyString))
{
targetRanges = await queryClient.GetTargetPartitionKeyRangesByEpkString(
resourceLink,
collection.ResourceId,
effectivePartitionKeyString);
}
else
{
targetRanges = await queryClient.GetTargetPartitionKeyRanges(
resourceLink,
collection.ResourceId,
partitionedQueryExecutionInfo.QueryRanges);
}
return(targetRanges);
}
internal ChangeFeedResultSetIterator(
int?maxItemCount,
string continuationToken,
CosmosQueryRequestOptions options,
NextResultSetDelegate nextDelegate,
object state = null) : base(maxItemCount, continuationToken, options, nextDelegate, state)
{
}
public async Task ItemMultiplePartitionOrderByQueryStream()
{
IList <ToDoActivity> deleteList = new List <ToDoActivity>();
try
{
deleteList = await CreateRandomItems(300, randomPartitionKey : true);
CosmosSqlQueryDefinition sql = new CosmosSqlQueryDefinition("SELECT * FROM toDoActivity t ORDER BY t.taskNum ");
CosmosQueryRequestOptions requestOptions = new CosmosQueryRequestOptions()
{
MaxBufferedItemCount = 10,
ResponseContinuationTokenLimitInKb = 500
};
List <ToDoActivity> resultList = new List <ToDoActivity>();
double totalRequstCharge = 0;
CosmosResultSetIterator setIterator =
this.Container.Items.CreateItemQueryAsStream(sql, maxConcurrency: 5, maxItemCount: 1, requestOptions: requestOptions);
while (setIterator.HasMoreResults)
{
using (CosmosQueryResponse iter = await setIterator.FetchNextSetAsync())
{
Assert.IsTrue(iter.IsSuccess);
Assert.IsNull(iter.ErrorMessage);
Assert.IsTrue(iter.Count <= 5);
totalRequstCharge += iter.RequestCharge;
ToDoActivity response = this.jsonSerializer.FromStream <ToDoActivity[]>(iter.Content).First();
resultList.Add(response);
}
}
Assert.AreEqual(deleteList.Count, resultList.Count);
Assert.IsTrue(totalRequstCharge > 0);
List <ToDoActivity> verifiedOrderBy = deleteList.OrderBy(x => x.taskNum).ToList();
for (int i = 0; i < verifiedOrderBy.Count(); i++)
{
Assert.AreEqual(verifiedOrderBy[i].taskNum, resultList[i].taskNum);
Assert.AreEqual(verifiedOrderBy[i].id, resultList[i].id);
}
}
finally
{
foreach (ToDoActivity delete in deleteList)
{
CosmosResponseMessage deleteResponse = await this.Container.Items.DeleteItemStreamAsync(delete.status, delete.id);
deleteResponse.Dispose();
}
}
}
public CosmosQueryContext(
CosmosQueryClient client,
ResourceType resourceTypeEnum,
OperationType operationType,
Type resourceType,
SqlQuerySpec sqlQuerySpecFromUser,
CosmosQueryRequestOptions queryRequestOptions,
Uri resourceLink,
bool getLazyFeedResponse,
Guid correlatedActivityId,
bool isContinuationExpected,
bool allowNonValueAggregateQuery,
string containerResourceId = null)
{
if (client == null)
{
throw new ArgumentNullException(nameof(client));
}
if (resourceType == null)
{
throw new ArgumentNullException(nameof(resourceType));
}
if (sqlQuerySpecFromUser == null)
{
throw new ArgumentNullException(nameof(sqlQuerySpecFromUser));
}
if (queryRequestOptions == null)
{
throw new ArgumentNullException(nameof(queryRequestOptions));
}
if (correlatedActivityId == Guid.Empty)
{
throw new ArgumentException(nameof(correlatedActivityId));
}
this.OperationTypeEnum = operationType;
this.QueryClient = client;
this.ResourceTypeEnum = resourceTypeEnum;
this.ResourceType = resourceType;
this.SqlQuerySpec = sqlQuerySpecFromUser;
this.QueryRequestOptions = queryRequestOptions;
this.ResourceLink = resourceLink;
this.ContainerResourceId = containerResourceId;
this.IsContinuationExpected = isContinuationExpected;
this.AllowNonValueAggregateQuery = allowNonValueAggregateQuery;
this.CorrelatedActivityId = correlatedActivityId;
}
public async Task ItemQueryStreamSerializationSetting()
{
IList <ToDoActivity> deleteList = await this.CreateRandomItems(101, randomPartitionKey : true);
CosmosSqlQueryDefinition sql = new CosmosSqlQueryDefinition("SELECT * FROM toDoActivity t ORDER BY t.taskNum");
CosmosSerializationOptions options = new CosmosSerializationOptions(
ContentSerializationFormat.CosmosBinary.ToString(),
(content) => JsonNavigator.Create(content),
() => JsonWriter.Create(JsonSerializationFormat.Binary));
CosmosQueryRequestOptions requestOptions = new CosmosQueryRequestOptions()
{
CosmosSerializationOptions = options
};
List <ToDoActivity> resultList = new List <ToDoActivity>();
double totalRequstCharge = 0;
CosmosResultSetIterator setIterator =
this.Container.Items.CreateItemQueryAsStream(sql, maxConcurrency: 5, maxItemCount: 5, requestOptions: requestOptions);
while (setIterator.HasMoreResults)
{
using (CosmosQueryResponse iter = await setIterator.FetchNextSetAsync())
{
Assert.IsTrue(iter.IsSuccess);
Assert.IsNull(iter.ErrorMessage);
Assert.IsTrue(iter.Count <= 5);
totalRequstCharge += iter.RequestCharge;
IJsonReader reader = JsonReader.Create(iter.Content);
IJsonWriter textWriter = JsonWriter.Create(JsonSerializationFormat.Text);
textWriter.WriteAll(reader);
string json = Encoding.UTF8.GetString(textWriter.GetResult());
Assert.IsNotNull(json);
ToDoActivity[] responseActivities = JsonConvert.DeserializeObject <ToDoActivity[]>(json);
resultList.AddRange(responseActivities);
}
}
Assert.AreEqual(deleteList.Count, resultList.Count);
Assert.IsTrue(totalRequstCharge > 0);
List <ToDoActivity> verifiedOrderBy = deleteList.OrderBy(x => x.taskNum).ToList();
for (int i = 0; i < verifiedOrderBy.Count(); i++)
{
Assert.AreEqual(verifiedOrderBy[i].taskNum, resultList[i].taskNum);
Assert.AreEqual(verifiedOrderBy[i].id, resultList[i].id);
}
}
internal async Task <FeedResponse <CosmosElement> > ExecuteQueryAsync(
SqlQuerySpec querySpecForInit,
CancellationToken cancellationToken,
Action <CosmosRequestMessage> requestEnricher = null)
{
CosmosQueryRequestOptions requestOptions = this.QueryRequestOptions.Clone();
return(await this.QueryClient.ExecuteItemQueryAsync(
this.ResourceLink,
this.ResourceTypeEnum,
this.OperationTypeEnum,
requestOptions,
querySpecForInit,
requestEnricher,
cancellationToken));
}
private static bool TryGetEpkProperty(
CosmosQueryRequestOptions queryRequestOptions,
out string effectivePartitionKeyString)
{
if (queryRequestOptions?.Properties != null &&
queryRequestOptions.Properties.TryGetValue(
WFConstants.BackendHeaders.EffectivePartitionKeyString,
out object effectivePartitionKeyStringObject))
{
effectivePartitionKeyString = effectivePartitionKeyStringObject as string;
if (string.IsNullOrEmpty(effectivePartitionKeyString))
{
throw new ArgumentOutOfRangeException(nameof(effectivePartitionKeyString));
}
return(true);
}
effectivePartitionKeyString = null;
return(false);
}
private static async Task QueryPartitionedContainerInParallelAsync(CosmosContainer container)
{
List <Family> familiesSerial = new List <Family>();
String queryText = "SELECT * FROM Families";
// 0 maximum parallel tasks, effectively serial execution
CosmosQueryRequestOptions options = new CosmosQueryRequestOptions()
{
MaxBufferedItemCount = 100
};
var query = container.Items.CreateItemQuery <Family>(
queryText,
maxConcurrency: 0,
requestOptions: options);
while (query.HasMoreResults)
{
foreach (Family family in await query.FetchNextSetAsync())
{
familiesSerial.Add(family);
}
}
Assert("Parallel Query expected two families", familiesSerial.ToList().Count == 2);
// 1 maximum parallel tasks, 1 dedicated asynchronous task to continuously make REST calls
List <Family> familiesParallel1 = new List <Family>();
query = container.Items.CreateItemQuery <Family>(
queryText,
maxConcurrency: 1,
requestOptions: options);
while (query.HasMoreResults)
{
foreach (Family family in await query.FetchNextSetAsync())
{
familiesParallel1.Add(family);
}
}
Assert("Parallel Query expected two families", familiesParallel1.ToList().Count == 2);
AssertSequenceEqual("Parallel query returns result out of order compared to serial execution", familiesSerial, familiesParallel1);
// 10 maximum parallel tasks, a maximum of 10 dedicated asynchronous tasks to continuously make REST calls
List <Family> familiesParallel10 = new List <Family>();
query = container.Items.CreateItemQuery <Family>(
queryText,
maxConcurrency: 10,
requestOptions: options);
while (query.HasMoreResults)
{
foreach (Family family in await query.FetchNextSetAsync())
{
familiesParallel10.Add(family);
}
}
Assert("Parallel Query expected two families", familiesParallel10.ToList().Count == 2);
AssertSequenceEqual("Parallel query returns result out of order compared to serial execution", familiesSerial, familiesParallel10);
}
public CosmosQueryExecutionContextFactory(
CosmosQueryClient client,
ResourceType resourceTypeEnum,
OperationType operationType,
Type resourceType,
SqlQuerySpec sqlQuerySpec,
CosmosQueryRequestOptions queryRequestOptions,
Uri resourceLink,
bool isContinuationExpected,
bool allowNonValueAggregateQuery,
Guid correlatedActivityId)
{
if (client == null)
{
throw new ArgumentNullException(nameof(client));
}
if (sqlQuerySpec == null)
{
throw new ArgumentNullException(nameof(sqlQuerySpec));
}
if (queryRequestOptions == null)
{
throw new ArgumentNullException(nameof(queryRequestOptions));
}
if (resourceLink == null)
{
throw new ArgumentNullException(nameof(resourceLink));
}
// Prevent users from updating the values after creating the execution context.
CosmosQueryRequestOptions cloneQueryRequestOptions = queryRequestOptions.Clone();
// Swapping out negative values in feedOptions for int.MaxValue
if (cloneQueryRequestOptions.MaxBufferedItemCount.HasValue && cloneQueryRequestOptions.MaxBufferedItemCount < 0)
{
cloneQueryRequestOptions.MaxBufferedItemCount = int.MaxValue;
}
if (cloneQueryRequestOptions.MaxConcurrency.HasValue && cloneQueryRequestOptions.MaxConcurrency < 0)
{
cloneQueryRequestOptions.MaxConcurrency = int.MaxValue;
}
if (cloneQueryRequestOptions.MaxItemCount.HasValue && cloneQueryRequestOptions.MaxItemCount < 0)
{
cloneQueryRequestOptions.MaxItemCount = int.MaxValue;
}
this.cosmosQueryContext = new CosmosQueryContext(
client: client,
resourceTypeEnum: resourceTypeEnum,
operationType: operationType,
resourceType: resourceType,
sqlQuerySpecFromUser: sqlQuerySpec,
queryRequestOptions: cloneQueryRequestOptions,
resourceLink: resourceLink,
getLazyFeedResponse: isContinuationExpected,
isContinuationExpected: isContinuationExpected,
allowNonValueAggregateQuery: allowNonValueAggregateQuery,
correlatedActivityId: correlatedActivityId);
}
