public static async Task <IEnumerable <T> > WhereAsync <T>(this Container container, QueryDefinition query, QueryRequestOptions options = null)
{
var results = new List <T>();
using var iterator = container.GetItemQueryIterator <T>(query, requestOptions: options);
while (iterator.HasMoreResults)
{
var response = await iterator.ReadNextAsync().ConfigureAwait(false);
results.AddRange(response.Resource);
}
return(results);
}
public static QueryIterator Create(
ContainerCore containerCore,
CosmosQueryClient client,
CosmosClientContext clientContext,
SqlQuerySpec sqlQuerySpec,
string continuationToken,
FeedRangeInternal feedRangeInternal,
QueryRequestOptions queryRequestOptions,
string resourceLink,
bool isContinuationExpected,
bool allowNonValueAggregateQuery,
bool forcePassthrough,
PartitionedQueryExecutionInfo partitionedQueryExecutionInfo)
{
if (queryRequestOptions == null)
{
queryRequestOptions = new QueryRequestOptions();
}
CosmosDiagnosticsContext queryPipelineCreationDiagnostics = CosmosDiagnosticsContext.Create(queryRequestOptions);
CosmosQueryContextCore cosmosQueryContext = new CosmosQueryContextCore(
client: client,
resourceTypeEnum: Documents.ResourceType.Document,
operationType: Documents.OperationType.Query,
resourceType: typeof(QueryResponseCore),
resourceLink: resourceLink,
isContinuationExpected: isContinuationExpected,
allowNonValueAggregateQuery: allowNonValueAggregateQuery,
diagnosticsContext: queryPipelineCreationDiagnostics,
correlatedActivityId: Guid.NewGuid());
NetworkAttachedDocumentContainer networkAttachedDocumentContainer = new NetworkAttachedDocumentContainer(
containerCore,
client,
queryPipelineCreationDiagnostics,
queryRequestOptions);
DocumentContainer documentContainer = new DocumentContainer(networkAttachedDocumentContainer);
CosmosElement requestContinuationToken;
switch (queryRequestOptions.ExecutionEnvironment.GetValueOrDefault(ExecutionEnvironment.Client))
{
case ExecutionEnvironment.Client:
if (continuationToken != null)
{
TryCatch <CosmosElement> tryParse = CosmosElement.Monadic.Parse(continuationToken);
if (tryParse.Failed)
{
return(new QueryIterator(
cosmosQueryContext,
new FaultedQueryPipelineStage(
new MalformedContinuationTokenException(
message: $"Malformed Continuation Token: {continuationToken}",
innerException: tryParse.Exception)),
queryRequestOptions.CosmosSerializationFormatOptions,
queryRequestOptions,
clientContext));
}
requestContinuationToken = tryParse.Result;
}
else
{
requestContinuationToken = null;
}
break;
case ExecutionEnvironment.Compute:
requestContinuationToken = queryRequestOptions.CosmosElementContinuationToken;
break;
default:
throw new ArgumentOutOfRangeException($"Unknown {nameof(ExecutionEnvironment)}: {queryRequestOptions.ExecutionEnvironment.Value}.");
}
CosmosQueryExecutionContextFactory.InputParameters inputParameters = new CosmosQueryExecutionContextFactory.InputParameters(
sqlQuerySpec: sqlQuerySpec,
initialUserContinuationToken: requestContinuationToken,
initialFeedRange: feedRangeInternal,
maxConcurrency: queryRequestOptions.MaxConcurrency,
maxItemCount: queryRequestOptions.MaxItemCount,
maxBufferedItemCount: queryRequestOptions.MaxBufferedItemCount,
partitionKey: queryRequestOptions.PartitionKey,
properties: queryRequestOptions.Properties,
partitionedQueryExecutionInfo: partitionedQueryExecutionInfo,
executionEnvironment: queryRequestOptions.ExecutionEnvironment,
returnResultsInDeterministicOrder: queryRequestOptions.ReturnResultsInDeterministicOrder,
forcePassthrough: forcePassthrough,
testInjections: queryRequestOptions.TestSettings);
return(new QueryIterator(
cosmosQueryContext,
CosmosQueryExecutionContextFactory.Create(documentContainer, cosmosQueryContext, inputParameters),
queryRequestOptions.CosmosSerializationFormatOptions,
queryRequestOptions,
clientContext));
}
public override Task <TryCatch <QueryPage> > ExecuteItemQueryAsync(string resourceUri, ResourceType resourceType, OperationType operationType, Guid clientQueryCorrelationId, Cosmos.FeedRange feedRange, QueryRequestOptions requestOptions, SqlQuerySpec sqlQuerySpec, string continuationToken, bool isContinuationExpected, int pageSize, ITrace trace, CancellationToken cancellationToken)
{
throw new NotImplementedException();
}
public async Task ContainerPartitionResourcePermissionTest(ConnectionMode connectionMode)
{
CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
{
ConnectionMode = connectionMode
};
CosmosClient cosmosClient = TestCommon.CreateCosmosClient(cosmosClientOptions);
Database database = await cosmosClient.CreateDatabaseIfNotExistsAsync("PermissionTest");
//create user
string userId = Guid.NewGuid().ToString();
UserResponse userResponse = await database.CreateUserAsync(userId);
Assert.AreEqual(HttpStatusCode.Created, userResponse.StatusCode);
Assert.AreEqual(userId, userResponse.Resource.Id);
User user = userResponse.User;
//create resource
string containerId = Guid.NewGuid().ToString();
ContainerResponse containerResponse = await database.CreateContainerAsync(
id : containerId,
partitionKeyPath : "/id");
Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode);
Container container = containerResponse.Container;
// Create items to read
ToDoActivity itemAccess = ToDoActivity.CreateRandomToDoActivity();
ToDoActivity itemNoAccess = ToDoActivity.CreateRandomToDoActivity();
await container.CreateItemAsync <ToDoActivity>(
itemAccess,
new PartitionKey(itemAccess.id));
await container.CreateItemAsync <ToDoActivity>(
itemNoAccess,
new PartitionKey(itemNoAccess.id));
//create permission
string permissionId = Guid.NewGuid().ToString();
PartitionKey partitionKey = new PartitionKey(itemAccess.id);
PermissionProperties permissionProperties = new PermissionProperties(
permissionId,
PermissionMode.Read,
container,
partitionKey);
PermissionResponse permissionResponse = await user.CreatePermissionAsync(permissionProperties);
PermissionProperties permission = permissionResponse.Resource;
Assert.AreEqual(HttpStatusCode.Created, userResponse.StatusCode);
Assert.AreEqual(permissionId, permission.Id);
Assert.AreEqual(permissionProperties.PermissionMode, permission.PermissionMode);
using (CosmosClient tokenCosmosClient = TestCommon.CreateCosmosClient(clientOptions: cosmosClientOptions, resourceToken: permission.Token))
{
Container tokenContainer = tokenCosmosClient.GetContainer(database.Id, containerId);
await tokenContainer.ReadItemAsync <ToDoActivity>(itemAccess.id, new PartitionKey(itemAccess.id));
try
{
await tokenContainer.ReadItemAsync <ToDoActivity>(itemNoAccess.id, new PartitionKey(itemNoAccess.id));
Assert.Fail();
}
catch (CosmosException ex)
{
Assert.AreEqual(HttpStatusCode.Forbidden, ex.StatusCode);
}
QueryRequestOptions queryRequestOptions = new QueryRequestOptions()
{
PartitionKey = new PartitionKey(itemAccess.id)
};
FeedIterator <ToDoActivity> feedIterator = tokenContainer.GetItemQueryIterator <ToDoActivity>(
queryText: "select * from T",
requestOptions: queryRequestOptions);
List <ToDoActivity> result = new List <ToDoActivity>();
while (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> toDoActivities = await feedIterator.ReadNextAsync();
result.AddRange(toDoActivities);
}
Assert.AreEqual(1, result.Count);
// Test query with no service interop via gateway query plan to replicate x32 app
ContainerInternal containerCore = (ContainerInlineCore)tokenContainer;
MockCosmosQueryClient mock = new MockCosmosQueryClient(
clientContext: containerCore.ClientContext,
cosmosContainerCore: containerCore,
forceQueryPlanGatewayElseServiceInterop: true);
Container tokenGatewayQueryPlan = new ContainerInlineCore(
containerCore.ClientContext,
(DatabaseInternal)containerCore.Database,
containerCore.Id,
mock);
FeedIterator <ToDoActivity> feedIteratorGateway = tokenGatewayQueryPlan.GetItemQueryIterator <ToDoActivity>(
queryText: "select * from T",
requestOptions: queryRequestOptions);
List <ToDoActivity> resultGateway = new List <ToDoActivity>();
while (feedIteratorGateway.HasMoreResults)
{
FeedResponse <ToDoActivity> toDoActivities = await feedIteratorGateway.ReadNextAsync();
resultGateway.AddRange(toDoActivities);
}
Assert.AreEqual(1, resultGateway.Count);
}
}
public async Task TestDistinct_ExecuteNextAsync()
{
async Task ImplementationAsync(Container container, IReadOnlyList <CosmosObject> documents)
{
#region Queries
// To verify distint queries you can run it once without the distinct clause and run it through a hash set
// then compare to the query with the distinct clause.
List <string> queries = new List <string>()
{
// basic distinct queries
"SELECT {0} VALUE null",
// number value distinct queries
"SELECT {0} VALUE c.income from c",
// string value distinct queries
"SELECT {0} VALUE c.name from c",
// array value distinct queries
"SELECT {0} VALUE c.children from c",
// object value distinct queries
"SELECT {0} VALUE c.pet from c",
// scalar expressions distinct query
"SELECT {0} VALUE c.age % 2 FROM c",
// distinct queries with order by
"SELECT {0} VALUE c.age FROM c ORDER BY c.age",
// distinct queries with top and no matching order by
"SELECT {0} TOP 2147483647 VALUE c.age FROM c",
// distinct queries with top and matching order by
"SELECT {0} TOP 2147483647 VALUE c.age FROM c ORDER BY c.age",
// distinct queries with aggregates
"SELECT {0} VALUE MAX(c.age) FROM c",
// distinct queries with joins
"SELECT {0} VALUE c.age FROM p JOIN c IN p.children",
// distinct queries in subqueries
"SELECT {0} r.age, s FROM r JOIN (SELECT DISTINCT VALUE c FROM (SELECT 1 a) c) s WHERE r.age > 25",
// distinct queries in scalar subqeries
"SELECT {0} p.name, (SELECT DISTINCT VALUE p.age) AS Age FROM p",
// select *
"SELECT {0} * FROM c",
};
#endregion
#region ExecuteNextAsync API
// run the query with distinct and without + MockDistinctMap
// Should receive same results
// PageSize = 1 guarantees that the backend will return some duplicates.
foreach (string query in queries)
{
string queryWithoutDistinct = string.Format(query, "");
QueryRequestOptions requestOptions = new QueryRequestOptions()
{
MaxItemCount = 100, MaxConcurrency = 100
};
FeedIterator <CosmosElement> documentQueryWithoutDistinct = container.GetItemQueryIterator <CosmosElement>(
queryWithoutDistinct,
requestOptions: requestOptions);
MockDistinctMap documentsSeen = new MockDistinctMap();
List <CosmosElement> documentsFromWithoutDistinct = new List <CosmosElement>();
while (documentQueryWithoutDistinct.HasMoreResults)
{
FeedResponse <CosmosElement> cosmosQueryResponse = await documentQueryWithoutDistinct.ReadNextAsync();
foreach (CosmosElement document in cosmosQueryResponse)
{
if (documentsSeen.Add(document, out Cosmos.Query.Core.UInt128 hash))
{
documentsFromWithoutDistinct.Add(document);
}
else
{
// No Op for debugging purposes.
}
}
}
foreach (int pageSize in new int[] { 1, 10, 100 })
{
string queryWithDistinct = string.Format(query, "DISTINCT");
List <CosmosElement> documentsFromWithDistinct = new List <CosmosElement>();
FeedIterator <CosmosElement> documentQueryWithDistinct = container.GetItemQueryIterator <CosmosElement>(
queryWithDistinct,
requestOptions: requestOptions);
while (documentQueryWithDistinct.HasMoreResults)
{
FeedResponse <CosmosElement> cosmosQueryResponse = await documentQueryWithDistinct.ReadNextAsync();
documentsFromWithDistinct.AddRange(cosmosQueryResponse);
}
Assert.AreEqual(documentsFromWithDistinct.Count, documentsFromWithoutDistinct.Count());
for (int i = 0; i < documentsFromWithDistinct.Count; i++)
{
CosmosElement documentFromWithDistinct = documentsFromWithDistinct.ElementAt(i);
CosmosElement documentFromWithoutDistinct = documentsFromWithoutDistinct.ElementAt(i);
Assert.AreEqual(
expected: documentFromWithoutDistinct,
actual: documentFromWithDistinct,
message: $"{documentFromWithDistinct} did not match {documentFromWithoutDistinct} at index {i} for {queryWithDistinct}, with page size: {pageSize} on a container");
}
}
}
#endregion
}
await this.TestQueryDistinctBaseAsync(ImplementationAsync);
}
public async Task TestQueryPlanGatewayAndServiceInteropAsync()
{
int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds;
uint numberOfDocuments = 100;
QueryOracleUtil util = new QueryOracle2(seed);
IEnumerable <string> inputDocuments = util.GetDocuments(numberOfDocuments);
await this.CreateIngestQueryDeleteAsync(
ConnectionModes.Direct,
CollectionTypes.SinglePartition | CollectionTypes.MultiPartition,
inputDocuments,
ImplementationAsync);
async Task ImplementationAsync(Container container, IReadOnlyList <CosmosObject> documents)
{
ContainerCore containerCore = (ContainerInlineCore)container;
foreach (bool isGatewayQueryPlan in new bool[] { true, false })
{
MockCosmosQueryClient cosmosQueryClientCore = new MockCosmosQueryClient(
containerCore.ClientContext,
containerCore,
isGatewayQueryPlan);
ContainerCore containerWithForcedPlan = new ContainerCore(
containerCore.ClientContext,
(DatabaseCore)containerCore.Database,
containerCore.Id,
cosmosQueryClientCore);
int numOfQueries = 0;
foreach (int maxDegreeOfParallelism in new int[] { 1, 100 })
{
foreach (int maxItemCount in new int[] { 10, 100 })
{
numOfQueries++;
QueryRequestOptions feedOptions = new QueryRequestOptions
{
MaxBufferedItemCount = 7000,
MaxConcurrency = maxDegreeOfParallelism,
MaxItemCount = maxItemCount,
};
List <CosmosElement> queryResults = await QueryTestsBase.RunQueryAsync(
containerWithForcedPlan,
"SELECT * FROM c ORDER BY c._ts",
feedOptions);
Assert.AreEqual(documents.Count(), queryResults.Count);
}
}
if (isGatewayQueryPlan)
{
Assert.IsTrue(cosmosQueryClientCore.QueryPlanCalls > numOfQueries);
}
else
{
Assert.AreEqual(0, cosmosQueryClientCore.QueryPlanCalls, "ServiceInterop mode should not be calling gateway plan retriever");
}
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="CosmosDbArgs{T, TModel}"/> class.
/// </summary>
/// <param name="mapper">The <see cref="IEntityMapper{T, TModel}"/>.</param>
/// <param name="containerId">The <see cref="Container"/> identifier.</param>
/// <param name="partitionKey">The <see cref="PartitionKey"/>.</param>
/// <param name="paging">The <see cref="PagingResult"/>.</param>
/// <param name="requestOptions">The optional <see cref="FeedOptions"/>.</param>
public CosmosDbArgs(IEntityMapper <T, TModel> mapper, string containerId, PartitionKey partitionKey, PagingArgs paging, QueryRequestOptions requestOptions = null)
: this(mapper, containerId, partitionKey, new PagingResult(Check.NotNull(paging, (nameof(paging)))), requestOptions)
{
}
/// <summary>
/// This method creates a query for stored procedures under a container using a SQL statement. It returns a FeedIterator.
/// For more information on preparing SQL statements with parameterized values, please see <see cref="QueryDefinition"/> overload.
/// </summary>
/// <param name="queryDefinition">The cosmos SQL query definition.</param>
/// <param name="continuationToken">(Optional) The continuation token in the Azure Cosmos DB service.</param>
/// <param name="requestOptions">(Optional) The options for the item query request <see cref="QueryRequestOptions"/></param>
/// <returns>An iterator to read through the existing stored procedures.</returns>
public abstract FeedIterator <T> GetStoredProcedureQueryIterator <T>(
QueryDefinition queryDefinition,
string continuationToken = null,
QueryRequestOptions requestOptions = null);
//Helper method to run query
static async Task RunQuery(string sqlQueryText, int maxItemCountPerPage = 100, int maxConcurrency = -1, bool useQueryOptions = false)
{
Console.BackgroundColor = ConsoleColor.Blue;
Console.WriteLine($"Running query: \"{sqlQueryText}\" against container {ContainerName}\n");
Console.WriteLine("");
if (useQueryOptions)
{
Console.WriteLine($"Using MaxConcurrency: {maxConcurrency}");
Console.WriteLine($"Using MaxItemCountPerPage: {maxItemCountPerPage}");
}
Console.ResetColor();
double totalRequestCharge = 0;
QueryDefinition queryDefinition = new QueryDefinition(sqlQueryText);
// Run query against Cosmos DB
var container = cosmosClient.GetDatabase(DatabaseName).GetContainer(ContainerName);
QueryRequestOptions requestOptions;
if (useQueryOptions)
{
requestOptions = new QueryRequestOptions()
{
MaxItemCount = maxItemCountPerPage,
MaxConcurrency = maxConcurrency,
};
}
else
{
requestOptions = new QueryRequestOptions(); //use all default query options
}
// Time the query
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
FeedIterator <dynamic> queryResultSetIterator = container.GetItemQueryIterator <dynamic>(queryDefinition, requestOptions: requestOptions);
List <dynamic> reviews = new List <dynamic>();
while (queryResultSetIterator.HasMoreResults)
{
FeedResponse <dynamic> currentResultSet = await queryResultSetIterator.ReadNextAsync();
totalRequestCharge += currentResultSet.RequestCharge;
//Console.WriteLine("another page");
foreach (var item in currentResultSet)
{
reviews.Add(item);
Console.WriteLine(item);
}
if (useQueryOptions)
{
Console.WriteLine($"Result count: {reviews.Count}");
}
}
stopWatch.Stop();
TimeSpan ts = stopWatch.Elapsed;
//Print results
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds,
ts.Milliseconds / 10);
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine($"\tQuery returned {reviews.Count} results");
Console.WriteLine($"\tTotal time: {elapsedTime}");
Console.WriteLine($"\tTotal Request Units consumed: {totalRequestCharge}\n");
Console.WriteLine("\n\n\n");
Console.ResetColor();
}
/// <summary>
/// This method creates a query for user defined functions under a container using a SQL statement. It returns a FeedIterator.
/// For more information on preparing SQL statements with parameterized values, please see <see cref="QueryDefinition"/> overload.
/// </summary>
/// <param name="queryDefinition">The cosmos SQL query definition.</param>
/// <param name="continuationToken">(Optional) The continuation token in the Azure Cosmos DB service.</param>
/// <param name="requestOptions">(Optional) The options for the item query request <see cref="QueryRequestOptions"/></param>
/// <returns>An iterator to read through the existing stored procedures.</returns>
public abstract FeedIterator <T> GetUserDefinedFunctionQueryIterator <T>(
QueryDefinition queryDefinition,
string continuationToken = null,
QueryRequestOptions requestOptions = null);
/// <summary>
/// This method creates a query for user defined functions under a container using a SQL statement. It returns a FeedIterator.
/// For more information on preparing SQL statements with parameterized values, please see <see cref="QueryDefinition"/> overload.
/// </summary>
/// <param name="queryText">The cosmos SQL query text.</param>
/// <param name="continuationToken">(Optional) The continuation token in the Azure Cosmos DB service.</param>
/// <param name="requestOptions">(Optional) The options for the item query request <see cref="QueryRequestOptions"/></param>
/// <returns>An iterator to read through the existing stored procedures.</returns>
public abstract FeedIterator GetUserDefinedFunctionQueryStreamIterator(
string queryText = null,
string continuationToken = null,
QueryRequestOptions requestOptions = null);
/// <summary>
/// This method creates a query for triggers under a container using a SQL statement. It returns a FeedIterator.
/// For more information on preparing SQL statements with parameterized values, please see <see cref="QueryDefinition"/> overload.
/// </summary>
/// <param name="queryText">The cosmos SQL query text.</param>
/// <param name="continuationToken">(Optional) The continuation token in the Azure Cosmos DB service.</param>
/// <param name="requestOptions">(Optional) The options for the item query request <see cref="QueryRequestOptions"/></param>
/// <returns>An iterator to read through the existing stored procedures.</returns>
public abstract FeedIterator GetTriggerQueryStreamIterator(
string queryText = null,
string continuationToken = null,
QueryRequestOptions requestOptions = null);
/// <summary>
/// This method creates a query for triggers under a container using a SQL statement. It returns a FeedIterator.
/// For more information on preparing SQL statements with parameterized values, please see <see cref="QueryDefinition"/> overload.
/// </summary>
/// <param name="queryDefinition">The cosmos SQL query definition.</param>
/// <param name="continuationToken">(Optional) The continuation token in the Azure Cosmos DB service.</param>
/// <param name="requestOptions">(Optional) The options for the item query request <see cref="QueryRequestOptions"/></param>
/// <returns>An iterator to read through the existing stored procedures.</returns>
public abstract FeedIterator GetTriggerQueryStreamIterator(
QueryDefinition queryDefinition,
string continuationToken = null,
QueryRequestOptions requestOptions = null);
/// <summary>
/// This method creates a query for stored procedures under a container using a SQL statement. It returns a FeedIterator.
/// For more information on preparing SQL statements with parameterized values, please see <see cref="QueryDefinition"/> overload.
/// </summary>
/// <param name="queryText">The cosmos SQL query text.</param>
/// <param name="continuationToken">(Optional) The continuation token in the Azure Cosmos DB service.</param>
/// <param name="requestOptions">(Optional) The options for the item query request <see cref="QueryRequestOptions"/></param>
/// <returns>An iterator to read through the existing stored procedures.</returns>
public abstract FeedIterator GetStoredProcedureQueryStreamIterator(
string queryText = null,
string continuationToken = null,
QueryRequestOptions requestOptions = null);
public static QueryIterator Create(
CosmosQueryClient client,
SqlQuerySpec sqlQuerySpec,
string continuationToken,
QueryRequestOptions queryRequestOptions,
Uri resourceLink,
bool isContinuationExpected,
bool allowNonValueAggregateQuery,
PartitionedQueryExecutionInfo partitionedQueryExecutionInfo)
{
if (queryRequestOptions == null)
{
queryRequestOptions = new QueryRequestOptions();
}
CosmosDiagnosticsContext queryPipelineCreationDiagnostics = CosmosDiagnosticsContext.Create(queryRequestOptions);
CosmosQueryContextCore cosmosQueryContext = new CosmosQueryContextCore(
client: client,
queryRequestOptions: queryRequestOptions,
resourceTypeEnum: Documents.ResourceType.Document,
operationType: Documents.OperationType.Query,
resourceType: typeof(QueryResponseCore),
resourceLink: resourceLink,
isContinuationExpected: isContinuationExpected,
allowNonValueAggregateQuery: allowNonValueAggregateQuery,
diagnosticsContext: queryPipelineCreationDiagnostics,
correlatedActivityId: Guid.NewGuid());
CosmosElement requestContinuationToken;
switch (queryRequestOptions.ExecutionEnvironment.GetValueOrDefault(ExecutionEnvironment.Client))
{
case ExecutionEnvironment.Client:
if (continuationToken != null)
{
if (!CosmosElement.TryParse(continuationToken, out requestContinuationToken))
{
return(new QueryIterator(
cosmosQueryContext,
new QueryExecutionContextWithException(
new MalformedContinuationTokenException(
$"Malformed Continuation Token: {continuationToken}")),
queryRequestOptions.CosmosSerializationFormatOptions,
queryRequestOptions));
}
}
else
{
requestContinuationToken = null;
}
break;
case ExecutionEnvironment.Compute:
requestContinuationToken = queryRequestOptions.CosmosElementContinuationToken;
break;
default:
throw new ArgumentOutOfRangeException($"Unknown {nameof(ExecutionEnvironment)}: {queryRequestOptions.ExecutionEnvironment.Value}.");
}
CosmosQueryExecutionContextFactory.InputParameters inputParameters = new CosmosQueryExecutionContextFactory.InputParameters(
sqlQuerySpec: sqlQuerySpec,
initialUserContinuationToken: requestContinuationToken,
maxConcurrency: queryRequestOptions.MaxConcurrency,
maxItemCount: queryRequestOptions.MaxItemCount,
maxBufferedItemCount: queryRequestOptions.MaxBufferedItemCount,
partitionKey: queryRequestOptions.PartitionKey,
properties: queryRequestOptions.Properties,
partitionedQueryExecutionInfo: partitionedQueryExecutionInfo,
executionEnvironment: queryRequestOptions.ExecutionEnvironment,
returnResultsInDeterministicOrder: queryRequestOptions.ReturnResultsInDeterministicOrder,
testInjections: queryRequestOptions.TestSettings);
return(new QueryIterator(
cosmosQueryContext,
CosmosQueryExecutionContextFactory.Create(cosmosQueryContext, inputParameters),
queryRequestOptions.CosmosSerializationFormatOptions,
queryRequestOptions));
}
public async Task TestCosmosQueryPartitionKeyDefinition()
{
PartitionKeyDefinition partitionKeyDefinition = new PartitionKeyDefinition();
QueryRequestOptions queryRequestOptions = new QueryRequestOptions
{
Properties = new Dictionary <string, object>()
{
{ "x-ms-query-partitionkey-definition", partitionKeyDefinition }
}
};
SqlQuerySpec sqlQuerySpec = new SqlQuerySpec(@"select * from t where t.something = 42 ");
bool allowNonValueAggregateQuery = true;
bool isContinuationExpected = true;
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
CancellationToken cancellationtoken = cancellationTokenSource.Token;
Mock <CosmosQueryClient> client = new Mock <CosmosQueryClient>();
string exceptionMessage = "Verified that the PartitionKeyDefinition was correctly set. Cancel the rest of the query";
client
.Setup(x => x.GetCachedContainerQueryPropertiesAsync(It.IsAny <Uri>(), It.IsAny <Cosmos.PartitionKey?>(), cancellationtoken))
.ReturnsAsync(new ContainerQueryProperties("mockContainer", null, partitionKeyDefinition));
client
.Setup(x => x.ByPassQueryParsing())
.Returns(false);
client
.Setup(x => x.TryGetPartitionedQueryExecutionInfoAsync(
It.IsAny <SqlQuerySpec>(),
It.IsAny <PartitionKeyDefinition>(),
It.IsAny <bool>(),
It.IsAny <bool>(),
It.IsAny <bool>(),
It.IsAny <bool>(),
It.IsAny <CancellationToken>()))
.ReturnsAsync(TryCatch <PartitionedQueryExecutionInfo> .FromException(
new InvalidOperationException(
exceptionMessage)));
CosmosQueryExecutionContextFactory.InputParameters inputParameters = new CosmosQueryExecutionContextFactory.InputParameters(
sqlQuerySpec: sqlQuerySpec,
initialUserContinuationToken: null,
maxConcurrency: queryRequestOptions?.MaxConcurrency,
maxItemCount: queryRequestOptions?.MaxItemCount,
maxBufferedItemCount: queryRequestOptions?.MaxBufferedItemCount,
partitionKey: queryRequestOptions?.PartitionKey,
properties: queryRequestOptions?.Properties,
partitionedQueryExecutionInfo: null,
executionEnvironment: queryRequestOptions?.ExecutionEnvironment,
testInjections: queryRequestOptions?.TestSettings);
CosmosQueryContext cosmosQueryContext = new CosmosQueryContextCore(
client: client.Object,
queryRequestOptions: queryRequestOptions,
resourceTypeEnum: ResourceType.Document,
operationType: OperationType.Query,
resourceType: typeof(QueryResponse),
resourceLink: new Uri("dbs/mockdb/colls/mockColl", UriKind.Relative),
isContinuationExpected: isContinuationExpected,
allowNonValueAggregateQuery: allowNonValueAggregateQuery,
correlatedActivityId: new Guid("221FC86C-1825-4284-B10E-A6029652CCA6"));
CosmosQueryExecutionContext context = CosmosQueryExecutionContextFactory.Create(
cosmosQueryContext,
inputParameters);
QueryResponseCore queryResponse = await context.ExecuteNextAsync(cancellationtoken);
Assert.AreEqual(HttpStatusCode.BadRequest, queryResponse.StatusCode);
Assert.IsTrue(queryResponse.ErrorMessage.Contains(exceptionMessage), "response error message did not contain the proper substring.");
}
/// <summary>
/// Executes the cmdlet.
/// </summary>
public override void ExecuteCmdlet()
{
var managementGroups = this.ManagementGroup?.ToList();
if (managementGroups != null && managementGroups.Count > ManagementGroupLimit)
{
managementGroups = managementGroups.Take(ManagementGroupLimit).ToList();
this.WriteWarning("The query included more management groups than allowed. " +
$"Only the first {ManagementGroupLimit} management groups were included for the results. " +
$"To use more than {ManagementGroupLimit} management groups, see the docs for examples: https://aka.ms/arg-error-toomanysubs");
}
IList <string> subscriptions = null;
if (managementGroups == null)
{
subscriptions = this.GetSubscriptions()?.ToList();
if (subscriptions != null && subscriptions.Count > SubscriptionLimit)
{
subscriptions = subscriptions.Take(SubscriptionLimit).ToList();
this.WriteWarning("The query included more subscriptions than allowed. " +
$"Only the first {SubscriptionLimit} subscriptions were included for the results. " +
$"To use more than {SubscriptionLimit} subscriptions, see the docs for examples: https://aka.ms/arg-error-toomanysubs");
}
}
var psResourceGraphResponse = new PSResourceGraphResponse <PSObject>();
QueryResponse response = null;
var resultTruncated = false;
try
{
var skipToken = this.SkipToken;
var isSkipTokenPassed = this.MyInvocation.BoundParameters.ContainsKey("SkipToken");
int?first = null;
if (this.MyInvocation.BoundParameters.ContainsKey("First"))
{
first = Math.Min(First, MaxRowsPerPage);
}
else if (!isSkipTokenPassed)
{
first = 100;
}
int?skip = null;
if (this.MyInvocation.BoundParameters.ContainsKey("Skip"))
{
skip = this.Skip;
}
else if (!isSkipTokenPassed)
{
skip = 0;
}
var allowPartialScopes = AllowPartialScope.IsPresent;
this.WriteVerbose($"Sent top={first} skip={skip} skipToken={skipToken}");
var requestOptions = new QueryRequestOptions(
top: first,
skip: skip,
skipToken: skipToken,
resultFormat: ResultFormat.ObjectArray,
allowPartialScopes: allowPartialScopes);
var request = new QueryRequest(this.Query, subscriptions, managementGroups, options: requestOptions);
response = this.ResourceGraphClient.ResourcesWithHttpMessagesAsync(request)
.Result
.Body;
if (response.ResultTruncated == ResultTruncated.True)
{
resultTruncated = true;
}
var requestResults = response.Data.ToPsObjects();
psResourceGraphResponse.Data = requestResults;
psResourceGraphResponse.SkipToken = response.SkipToken;
this.WriteVerbose($"Received results: {requestResults.Count}");
if (resultTruncated && psResourceGraphResponse.Data.Count < first)
{
this.WriteWarning("Unable to paginate the results of the query. " +
"Some resources may be missing from the results. " +
"To rewrite the query and enable paging, " +
"see the docs for an example: https://aka.ms/arg-results-truncated");
}
}
catch (Exception ex)
{
var aggregateEx = ex as AggregateException;
if (aggregateEx?.InnerException != null && aggregateEx.InnerExceptions.Count == 1)
{
var errorResponseEx = aggregateEx.InnerException as ErrorResponseException;
if (errorResponseEx != null)
{
var errorRecord = new ErrorRecord(
errorResponseEx, errorResponseEx.Body.Error.Code,
ErrorCategory.CloseError, null)
{
ErrorDetails = new System.Management.Automation.ErrorDetails(
errorResponseEx.ToDisplayableJson())
};
this.WriteError(errorRecord);
return;
}
}
this.WriteExceptionError(ex);
return;
}
this.WriteObject(psResourceGraphResponse);
}
public async Task TestPassthroughQueryAsync()
{
string[] inputDocs = new[]
{
@"{""id"":""documentId1"",""key"":""A"",""prop"":3,""shortArray"":[{""a"":5}]}",
@"{""id"":""documentId2"",""key"":""A"",""prop"":2,""shortArray"":[{""a"":6}]}",
@"{""id"":""documentId3"",""key"":""A"",""prop"":1,""shortArray"":[{""a"":7}]}",
@"{""id"":""documentId4"",""key"":5,""prop"":3,""shortArray"":[{""a"":5}]}",
@"{""id"":""documentId5"",""key"":5,""prop"":2,""shortArray"":[{""a"":6}]}",
@"{""id"":""documentId6"",""key"":5,""prop"":1,""shortArray"":[{""a"":7}]}",
@"{""id"":""documentId10"",""prop"":3,""shortArray"":[{""a"":5}]}",
@"{""id"":""documentId11"",""prop"":2,""shortArray"":[{""a"":6}]}",
@"{""id"":""documentId12"",""prop"":1,""shortArray"":[{""a"":7}]}",
};
await this.CreateIngestQueryDeleteAsync(
ConnectionModes.Direct,
CollectionTypes.SinglePartition | CollectionTypes.MultiPartition,
inputDocs,
ImplementationAsync,
"/key");
async Task ImplementationAsync(Container container, IReadOnlyList <CosmosObject> documents)
{
foreach (int maxDegreeOfParallelism in new int[] { 1, 100 })
{
foreach (int maxItemCount in new int[] { 10, 100 })
{
QueryRequestOptions feedOptions = new QueryRequestOptions
{
MaxBufferedItemCount = 7000,
MaxConcurrency = maxDegreeOfParallelism,
MaxItemCount = maxItemCount,
};
foreach (string query in new string[]
{
"SELECT * FROM c WHERE c.key = 5",
"SELECT * FROM c WHERE c.key = 5 ORDER BY c._ts",
})
{
feedOptions.TestSettings = new TestInjections(simulate429s: false, simulateEmptyPages: false, responseStats: new TestInjections.ResponseStats());
List <CosmosElement> queryResults = await QueryTestsBase.RunQueryAsync(
container,
query,
feedOptions);
Assert.IsTrue(feedOptions.TestSettings.Stats.PipelineType.HasValue);
Assert.AreEqual(TestInjections.PipelineType.Passthrough, feedOptions.TestSettings.Stats.PipelineType.Value);
Assert.AreEqual(
3,
queryResults.Count,
$"query: {query} failed with {nameof(maxDegreeOfParallelism)}: {maxDegreeOfParallelism}, {nameof(maxItemCount)}: {maxItemCount}");
}
{
feedOptions.TestSettings = new TestInjections(simulate429s: false, simulateEmptyPages: false, responseStats: new TestInjections.ResponseStats());
string query = "SELECT TOP 2 c.id FROM c WHERE c.key = 5";
List <CosmosElement> queryResults = await QueryTestsBase.RunQueryAsync(
container,
query,
feedOptions);
Assert.IsTrue(feedOptions.TestSettings.Stats.PipelineType.HasValue);
Assert.AreEqual(TestInjections.PipelineType.Passthrough, feedOptions.TestSettings.Stats.PipelineType.Value);
Assert.AreEqual(
2,
queryResults.Count,
$"query: {query} failed with {nameof(maxDegreeOfParallelism)}: {maxDegreeOfParallelism}, {nameof(maxItemCount)}: {maxItemCount}");
}
{
feedOptions.TestSettings = new TestInjections(simulate429s: false, simulateEmptyPages: false, responseStats: new TestInjections.ResponseStats());
string query = "SELECT c.id FROM c WHERE c.key = 5 OFFSET 1 LIMIT 1";
List <CosmosElement> queryResults = await QueryTestsBase.RunQueryAsync(
container,
query,
feedOptions);
Assert.IsTrue(feedOptions.TestSettings.Stats.PipelineType.HasValue);
Assert.AreEqual(TestInjections.PipelineType.Passthrough, feedOptions.TestSettings.Stats.PipelineType.Value);
Assert.AreEqual(
1,
queryResults.Count,
$"query: {query} failed with {nameof(maxDegreeOfParallelism)}: {maxDegreeOfParallelism}, {nameof(maxItemCount)}: {maxItemCount}");
}
{
feedOptions.TestSettings = new TestInjections(simulate429s: false, simulateEmptyPages: false, responseStats: new TestInjections.ResponseStats());
string query = "SELECT VALUE COUNT(1) FROM c WHERE c.key = 5";
List <CosmosElement> queryResults = await QueryTestsBase.RunQueryAsync(
container,
query,
feedOptions);
Assert.IsTrue(feedOptions.TestSettings.Stats.PipelineType.HasValue);
Assert.AreEqual(TestInjections.PipelineType.Specialized, feedOptions.TestSettings.Stats.PipelineType.Value);
Assert.AreEqual(
1,
queryResults.Count,
$"query: {query} failed with {nameof(maxDegreeOfParallelism)}: {maxDegreeOfParallelism}, {nameof(maxItemCount)}: {maxItemCount}");
Assert.AreEqual(
3,
Number64.ToLong((queryResults.First() as CosmosNumber64).GetValue()),
$"query: {query} failed with {nameof(maxDegreeOfParallelism)}: {maxDegreeOfParallelism}, {nameof(maxItemCount)}: {maxItemCount}");
}
{
feedOptions.TestSettings = new TestInjections(simulate429s: false, simulateEmptyPages: false, responseStats: new TestInjections.ResponseStats());
string query = "SELECT VALUE c.key FROM c WHERE c.key = 5 GROUP BY c.key";
List <CosmosElement> queryResults = await QueryTestsBase.RunQueryCombinationsAsync(
container,
query,
feedOptions,
QueryDrainingMode.HoldState | QueryDrainingMode.CosmosElementContinuationToken);
Assert.IsTrue(feedOptions.TestSettings.Stats.PipelineType.HasValue);
Assert.AreEqual(TestInjections.PipelineType.Specialized, feedOptions.TestSettings.Stats.PipelineType.Value);
Assert.AreEqual(
1,
queryResults.Count,
$"query: {query} failed with {nameof(maxDegreeOfParallelism)}: {maxDegreeOfParallelism}, {nameof(maxItemCount)}: {maxItemCount}");
}
{
feedOptions.TestSettings = new TestInjections(simulate429s: false, simulateEmptyPages: false, responseStats: new TestInjections.ResponseStats());
string query = "SELECT DISTINCT VALUE c.key FROM c WHERE c.key = 5";
List <CosmosElement> queryResults = await QueryTestsBase.RunQueryCombinationsAsync(
container,
query,
feedOptions,
QueryDrainingMode.HoldState | QueryDrainingMode.CosmosElementContinuationToken);
Assert.IsTrue(feedOptions.TestSettings.Stats.PipelineType.HasValue);
Assert.AreEqual(TestInjections.PipelineType.Specialized, feedOptions.TestSettings.Stats.PipelineType.Value);
Assert.AreEqual(
1,
queryResults.Count,
$"query: {query} failed with {nameof(maxDegreeOfParallelism)}: {maxDegreeOfParallelism}, {nameof(maxItemCount)}: {maxItemCount}");
}
}
}
}
}
public async Task TestNonDeterministicQueryResultsAsync()
{
int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds;
uint numberOfDocuments = 100;
QueryOracleUtil util = new QueryOracle2(seed);
IEnumerable <string> documents = util.GetDocuments(numberOfDocuments);
await this.CreateIngestQueryDeleteAsync(
ConnectionModes.Direct,
CollectionTypes.MultiPartition,
documents,
ImplementationAsync);
async Task ImplementationAsync(Container container, IReadOnlyList <CosmosObject> inputDocuments)
{
foreach (int maxDegreeOfParallelism in new int[] { 1, 100 })
{
foreach (int maxItemCount in new int[] { 10, 100 })
{
foreach (bool useOrderBy in new bool[] { false, true })
{
string query;
if (useOrderBy)
{
query = "SELECT c._ts, c.id FROM c ORDER BY c._ts";
}
else
{
query = "SELECT c.id FROM c";
}
QueryRequestOptions queryRequestOptions = new QueryRequestOptions
{
MaxBufferedItemCount = 7000,
MaxConcurrency = maxDegreeOfParallelism,
MaxItemCount = maxItemCount,
ReturnResultsInDeterministicOrder = false,
};
async Task ValidateNonDeterministicQuery(
Func <Container, string, QueryRequestOptions, Task <List <CosmosObject> > > queryFunc,
bool hasOrderBy)
{
List <CosmosObject> queryResults = await queryFunc(container, query, queryRequestOptions);
HashSet <string> expectedIds = new HashSet <string>(inputDocuments
.Select(document => ((CosmosString)document["id"]).Value));
HashSet <string> actualIds = new HashSet <string>(queryResults
.Select(queryResult => ((CosmosString)queryResult["id"]).Value));
Assert.IsTrue(
expectedIds.SetEquals(actualIds),
$"query: {query} failed with {nameof(maxDegreeOfParallelism)}: {maxDegreeOfParallelism}, {nameof(maxItemCount)}: {maxItemCount}");
if (hasOrderBy)
{
IEnumerable <long> timestamps = queryResults.Select(token => (long)token["_ts"].ToDouble());
IEnumerable <long> sortedTimestamps = timestamps.OrderBy(x => x);
Assert.IsTrue(timestamps.SequenceEqual(sortedTimestamps), "Items were not sorted.");
}
}
await ValidateNonDeterministicQuery(QueryTestsBase.QueryWithoutContinuationTokensAsync <CosmosObject>, useOrderBy);
await ValidateNonDeterministicQuery(QueryTestsBase.QueryWithContinuationTokensAsync <CosmosObject>, useOrderBy);
await ValidateNonDeterministicQuery(QueryTestsBase.QueryWithCosmosElementContinuationTokenAsync <CosmosObject>, useOrderBy);
}
}
}
}
}
/// <summary>
/// Executes the cmdlet.
/// </summary>
public override void ExecuteCmdlet()
{
var subscriptions = this.GetSubscriptions().ToList();
if (subscriptions == null || subscriptions.Count == 0)
{
var exception = new ArgumentException("No subscriptions were found to run query. " +
"Please try to add them implicitly as param to your request (e.g. Search-AzGraph -Query '' -Subscription '11111111-1111-1111-1111-111111111111')");
var errorRecord = new ErrorRecord(
exception, "400",
ErrorCategory.InvalidArgument, null);
this.WriteError(errorRecord);
return;
}
if (subscriptions.Count > SubscriptionLimit)
{
subscriptions = subscriptions.Take(SubscriptionLimit).ToList();
this.WriteWarning("The query included more subscriptions than allowed. " +
$"Only the first {SubscriptionLimit} subscriptions were included for the results. " +
$"To use more than {SubscriptionLimit} subscriptions, see the docs for examples: https://aka.ms/arg-error-toomanysubs");
}
var first = this.MyInvocation.BoundParameters.ContainsKey("First") ? this.First : 100;
var skip = this.MyInvocation.BoundParameters.ContainsKey("Skip") ? this.Skip : 0;
var results = new List <PSObject>();
QueryResponse response = null;
var resultTruncated = false;
try
{
do
{
var requestTop = Math.Min(first - results.Count, RowsPerPage);
var requestSkip = skip + results.Count;
var requestSkipToken = response?.SkipToken;
this.WriteVerbose($"Sent top={requestTop} skip={requestSkip} skipToken={requestSkipToken}");
var requestOptions = new QueryRequestOptions(
top: requestTop,
skip: requestSkip,
skipToken: requestSkipToken,
resultFormat: ResultFormat.ObjectArray);
var queryExtenstion = (this.Include == IncludeOptionsEnum.DisplayNames && this.QueryExtensionInitizalized()) ?
(queryExtensionToIncludeNames + (this.Query.Length != 0 ? "| " : string.Empty)) :
string.Empty;
var request = new QueryRequest(subscriptions, queryExtenstion + this.Query, options: requestOptions);
response = this.ResourceGraphClient.ResourcesWithHttpMessagesAsync(request)
.Result
.Body;
if (response.ResultTruncated == ResultTruncated.True)
{
resultTruncated = true;
}
var requestResults = response.Data.ToPsObjects();
results.AddRange(requestResults);
this.WriteVerbose($"Received results: {requestResults.Count}");
}while (results.Count < first && response.SkipToken != null);
if (resultTruncated && results.Count < first)
{
this.WriteWarning("Unable to paginate the results of the query. " +
"Some resources may be missing from the results. " +
"To rewrite the query and enable paging, " +
"see the docs for an example: https://aka.ms/arg-results-truncated");
}
}
catch (Exception ex)
{
var aggregateEx = ex as AggregateException;
if (aggregateEx?.InnerException != null && aggregateEx.InnerExceptions.Count == 1)
{
var errorResponseEx = aggregateEx.InnerException as ErrorResponseException;
if (errorResponseEx != null)
{
var errorRecord = new ErrorRecord(
errorResponseEx, errorResponseEx.Body.Error.Code,
ErrorCategory.CloseError, null)
{
ErrorDetails = new System.Management.Automation.ErrorDetails(
errorResponseEx.ToDisplayableJson())
};
this.WriteError(errorRecord);
return;
}
}
this.WriteExceptionError(ex);
return;
}
this.WriteObject(results, true);
}
/// <summary>
/// Initializes a new instance of the <see cref="CosmosDbArgs{T, TModel}"/> class.
/// </summary>
/// <param name="mapper">The <see cref="IEntityMapper{T, TModel}"/>.</param>
/// <param name="containerId">The <see cref="Container"/> identifier.</param>
/// <param name="partitionKey">The <see cref="PartitionKey"/>.</param>
/// <param name="paging">The <see cref="PagingResult"/>.</param>
/// <param name="requestOptions">The optional <see cref="FeedOptions"/>.</param>
public CosmosDbArgs(IEntityMapper <T, TModel> mapper, string containerId, PartitionKey partitionKey, PagingResult paging, QueryRequestOptions requestOptions = null)
{
Mapper = Check.NotNull(mapper, nameof(mapper));
ContainerId = Check.NotEmpty(containerId, nameof(containerId));
PartitionKey = partitionKey;
Paging = Check.NotNull(paging, nameof(paging));
QueryRequestOptions = requestOptions;
}
/// <summary>
/// Executes the cmdlet.
/// </summary>
public override void ExecuteCmdlet()
{
var subscriptions = this.GetSubscriptions().ToList();
var first = this.MyInvocation.BoundParameters.ContainsKey("First") ? this.First : 100;
var skip = this.MyInvocation.BoundParameters.ContainsKey("Skip") ? this.Skip : 0;
var results = new List <PSObject>();
QueryResponse response = null;
try
{
do
{
var requestTop = Math.Min(first - results.Count, RowsPerPage);
var requestSkip = skip + results.Count;
var requestSkipToken = response?.SkipToken;
this.WriteVerbose($"Sent top={requestTop} skip={requestSkip} skipToken={requestSkipToken}");
var requestOptions = new QueryRequestOptions(
top: requestTop,
skip: requestSkip,
skipToken: requestSkipToken);
var request = new QueryRequest(subscriptions, this.Query, requestOptions);
response = this.ResourceGraphClient.ResourcesWithHttpMessagesAsync(request)
.Result
.Body;
var requestResults = response.Data.ToPsObjects().ToList();
results.AddRange(requestResults);
this.WriteVerbose($"Received results: {requestResults.Count}");
}while (results.Count < first && response.SkipToken != null);
}
catch (Exception ex)
{
var aggregateEx = ex as AggregateException;
if (aggregateEx?.InnerException != null && aggregateEx.InnerExceptions.Count == 1)
{
var errorResponseEx = aggregateEx.InnerException as ErrorResponseException;
if (errorResponseEx != null)
{
var errorRecord = new ErrorRecord(
errorResponseEx, errorResponseEx.Body.Error.Code,
ErrorCategory.CloseError, null)
{
ErrorDetails = new System.Management.Automation.ErrorDetails(
errorResponseEx.ToDisplayableJson())
};
this.WriteError(errorRecord);
return;
}
}
this.WriteExceptionError(ex);
return;
}
this.WriteObject(results, true);
}
private async Task <long> ExecuteQueryAndReturnOutputDocumentCount(
string queryText,
int expectedItemCount,
bool disableDiagnostics)
{
QueryDefinition sql = null;
if (queryText != null)
{
sql = new QueryDefinition(queryText);
}
QueryRequestOptions requestOptions = new QueryRequestOptions()
{
MaxItemCount = 1,
MaxConcurrency = 1,
};
if (disableDiagnostics)
{
requestOptions.DiagnosticContextFactory = () => EmptyCosmosDiagnosticsContext.Singleton;
}
;
// Verify the typed query iterator
FeedIterator <ToDoActivity> feedIterator = this.Container.GetItemQueryIterator <ToDoActivity>(
sql,
requestOptions: requestOptions);
List <ToDoActivity> results = new List <ToDoActivity>();
long totalOutDocumentCount = 0;
bool isFirst = true;
while (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> response = await feedIterator.ReadNextAsync();
results.AddRange(response);
if (queryText == null)
{
CosmosDiagnosticsTests.VerifyPointDiagnostics(
response.Diagnostics,
disableDiagnostics);
}
else
{
VerifyQueryDiagnostics(
response.Diagnostics,
isFirst,
disableDiagnostics);
}
isFirst = false;
}
Assert.AreEqual(expectedItemCount, results.Count);
// Verify the stream query iterator
FeedIterator streamIterator = this.Container.GetItemQueryStreamIterator(
sql,
requestOptions: requestOptions);
List <ToDoActivity> streamResults = new List <ToDoActivity>();
long streamTotalOutDocumentCount = 0;
isFirst = true;
while (streamIterator.HasMoreResults)
{
ResponseMessage response = await streamIterator.ReadNextAsync();
Collection <ToDoActivity> result = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(response.Content).Data;
streamResults.AddRange(result);
if (queryText == null)
{
CosmosDiagnosticsTests.VerifyPointDiagnostics(
response.Diagnostics,
disableDiagnostics);
}
else
{
VerifyQueryDiagnostics(
response.Diagnostics,
isFirst,
disableDiagnostics);
}
isFirst = false;
}
Assert.AreEqual(expectedItemCount, streamResults.Count);
Assert.AreEqual(totalOutDocumentCount, streamTotalOutDocumentCount);
return(results.Count);
}
public CosmosQueryExecutionContextFactory(
CosmosQueryClient client,
ResourceType resourceTypeEnum,
OperationType operationType,
Type resourceType,
SqlQuerySpec sqlQuerySpec,
QueryRequestOptions queryRequestOptions,
Uri resourceLink,
bool isContinuationExpected,
bool allowNonValueAggregateQuery,
Guid correlatedActivityId)
{
if (client == null)
{
throw new ArgumentNullException(nameof(client));
}
if (sqlQuerySpec == null)
{
throw new ArgumentNullException(nameof(sqlQuerySpec));
}
if (queryRequestOptions == null)
{
throw new ArgumentNullException(nameof(queryRequestOptions));
}
if (resourceLink == null)
{
throw new ArgumentNullException(nameof(resourceLink));
}
// Prevent users from updating the values after creating the execution context.
QueryRequestOptions cloneQueryRequestOptions = queryRequestOptions.Clone();
// Swapping out negative values in feedOptions for int.MaxValue
if (cloneQueryRequestOptions.MaxBufferedItemCount.HasValue && cloneQueryRequestOptions.MaxBufferedItemCount < 0)
{
cloneQueryRequestOptions.MaxBufferedItemCount = int.MaxValue;
}
if (cloneQueryRequestOptions.MaxConcurrency.HasValue && cloneQueryRequestOptions.MaxConcurrency < 0)
{
cloneQueryRequestOptions.MaxConcurrency = int.MaxValue;
}
if (cloneQueryRequestOptions.MaxItemCount.HasValue && cloneQueryRequestOptions.MaxItemCount < 0)
{
cloneQueryRequestOptions.MaxItemCount = int.MaxValue;
}
this.cosmosQueryContext = new CosmosQueryContext(
client: client,
resourceTypeEnum: resourceTypeEnum,
operationType: operationType,
resourceType: resourceType,
sqlQuerySpecFromUser: sqlQuerySpec,
queryRequestOptions: cloneQueryRequestOptions,
resourceLink: resourceLink,
getLazyFeedResponse: isContinuationExpected,
isContinuationExpected: isContinuationExpected,
allowNonValueAggregateQuery: allowNonValueAggregateQuery,
correlatedActivityId: correlatedActivityId);
}
public async Task <CosmosDbResponse <IEnumerable <T> > > GetByQuery <T>(string partitionKey, string query, IReadOnlyDictionary <string, object> parameters)
{
QueryDefinition queryDef = null;
try
{
queryDef = new QueryDefinition(query);
if (parameters != null)
{
foreach (KeyValuePair <string, object> kvp in parameters)
{
queryDef.WithParameter(kvp.Key, kvp.Value);
}
}
}
catch (ArgumentNullException anx)
{
return(anx.ToCosmosDbQueryResponse <T>());
}
var options = new QueryRequestOptions();
if (!string.IsNullOrWhiteSpace(partitionKey))
{
options.PartitionKey = new PartitionKey(partitionKey);
}
var response = new CosmosDbResponse <IEnumerable <T> >();
var results = new List <T>();
Stopwatch watch = Stopwatch.StartNew();
try
{
FeedIterator <T> resultSetIterator = _container.GetItemQueryIterator <T>(queryDef, requestOptions: options);
while (resultSetIterator.HasMoreResults)
{
FeedResponse <T> feedResponse = await resultSetIterator.ReadNextAsync();
results.AddRange(feedResponse);
response.RequestCharge += feedResponse.RequestCharge;
response.ActivityId = feedResponse.ActivityId;
response.ETag = feedResponse.ETag;
response.Diagnostics = feedResponse.Diagnostics.ToString();
}
watch.Stop();
response.StatusCode = HttpStatusCode.OK;
}
catch (CosmosException cex)
{
watch.Stop();
response.Error = cex;
response.ActivityId = cex.ActivityId;
response.StatusCode = cex.StatusCode;
response.RequestCharge += cex.RequestCharge;
response.Diagnostics = cex.Diagnostics.ToString();
}
finally
{
response.ExecutionTime = watch.Elapsed;
response.Result = results;
}
return(response);
}
public async Task ReadFeedIteratorCore_CrossPartitionBiDirectional(bool useStatelessIteration)
{
ContainerInternal container = null;
try
{
ContainerResponse containerResponse = await this.database.CreateContainerAsync(
new ContainerProperties(id : Guid.NewGuid().ToString(), partitionKeyPath : "/id"),
throughput : 50000,
cancellationToken : this.cancellationToken);
container = (ContainerInlineCore)containerResponse;
//create items
const int total = 30;
QueryRequestOptions requestOptions = new QueryRequestOptions()
{
MaxItemCount = 10
};
List <string> items = new List <string>();
for (int i = 0; i < total; i++)
{
string item = $@"
{{
""id"": ""{i}""
}}";
using (ResponseMessage createResponse = await container.CreateItemStreamAsync(
ReadFeedRangeTests.GenerateStreamFromString(item),
new Cosmos.PartitionKey(i.ToString())))
{
Assert.IsTrue(createResponse.IsSuccessStatusCode);
}
}
string continuation = null;
FeedIterator iter = container.GetItemQueryStreamIterator(
continuationToken: continuation,
requestOptions: requestOptions);
int count = 0;
List <string> forwardOrder = new List <string>();
while (iter.HasMoreResults)
{
if (useStatelessIteration)
{
iter = container.GetItemQueryStreamIterator(
continuationToken: continuation,
requestOptions: requestOptions);
}
using (ResponseMessage response = await iter.ReadNextAsync())
{
Assert.IsNotNull(response);
continuation = response.ContinuationToken;
using (StreamReader reader = new StreamReader(response.Content))
{
string json = await reader.ReadToEndAsync();
JArray documents = (JArray)JObject.Parse(json).SelectToken("Documents");
count += documents.Count;
if (documents.Any())
{
forwardOrder.Add(documents.First().SelectToken("id").ToString());
}
}
}
}
Assert.IsNotNull(forwardOrder);
Assert.AreEqual(total, count);
Assert.IsFalse(forwardOrder.Where(x => string.IsNullOrEmpty(x)).Any());
requestOptions.Properties = requestOptions.Properties = new Dictionary <string, object>();
requestOptions.Properties.Add(Documents.HttpConstants.HttpHeaders.EnumerationDirection, (byte)BinaryScanDirection.Reverse);
count = 0;
List <string> reverseOrder = new List <string>();
continuation = null;
iter = container
.GetItemQueryStreamIterator(queryDefinition: null, continuationToken: continuation, requestOptions: requestOptions);
while (iter.HasMoreResults)
{
if (useStatelessIteration)
{
iter = container
.GetItemQueryStreamIterator(queryDefinition: null, continuationToken: continuation, requestOptions: requestOptions);
}
using (ResponseMessage response = await iter.ReadNextAsync())
{
continuation = response.ContinuationToken;
Assert.IsNotNull(response);
using (StreamReader reader = new StreamReader(response.Content))
{
string json = await reader.ReadToEndAsync();
JArray documents = (JArray)JObject.Parse(json).SelectToken("Documents");
count += documents.Count;
if (documents.Any())
{
reverseOrder.Add(documents.First().SelectToken("id").ToString());
}
}
}
}
Assert.IsNotNull(reverseOrder);
Assert.AreEqual(total, count);
forwardOrder.Reverse();
CollectionAssert.AreEqual(forwardOrder, reverseOrder);
Assert.IsFalse(reverseOrder.Where(x => string.IsNullOrEmpty(x)).Any());
}
finally
{
await container?.DeleteContainerAsync();
}
}
public override PassThroughQueryTestOutput ExecuteTest(PassThroughQueryTestInput input)
{
// gets DocumentContainer
IMonadicDocumentContainer monadicDocumentContainer = new InMemoryContainer(input.PartitionKeyDefinition);
DocumentContainer documentContainer = new DocumentContainer(monadicDocumentContainer);
SqlQuerySpec sqlQuerySpec = new SqlQuerySpec(input.Query);
// gets query context
string databaseId = "db1234";
string resourceLink = string.Format("dbs/{0}/colls", databaseId);
CosmosQueryContextCore cosmosQueryContextCore = new CosmosQueryContextCore(
client: new TestCosmosQueryClient(),
resourceTypeEnum: Documents.ResourceType.Document,
operationType: Documents.OperationType.Query,
resourceType: typeof(QueryResponseCore),
resourceLink: resourceLink,
isContinuationExpected: false,
allowNonValueAggregateQuery: true,
useSystemPrefix: false,
correlatedActivityId: Guid.NewGuid());
// gets input parameters
QueryRequestOptions queryRequestOptions = new QueryRequestOptions();
CosmosSerializerCore serializerCore = new();
using StreamReader streamReader = new(serializerCore.ToStreamSqlQuerySpec(sqlQuerySpec, Documents.ResourceType.Document));
string sqlQuerySpecJsonString = streamReader.ReadToEnd();
PartitionedQueryExecutionInfo partitionedQueryExecutionInfo = GetPartitionedQueryExecutionInfo(sqlQuerySpecJsonString, input.PartitionKeyDefinition);
if (input.PartitionKeyValue == default || input.PartitionKeyValue == Cosmos.PartitionKey.None)
{
input.PartitionKeyValue = Cosmos.PartitionKey.Null;
}
CosmosQueryExecutionContextFactory.InputParameters inputParameters = new CosmosQueryExecutionContextFactory.InputParameters(
sqlQuerySpec: sqlQuerySpec,
initialUserContinuationToken: null,
initialFeedRange: null,
maxConcurrency: queryRequestOptions.MaxConcurrency,
maxItemCount: queryRequestOptions.MaxItemCount,
maxBufferedItemCount: queryRequestOptions.MaxBufferedItemCount,
partitionKey: input.PartitionKeyValue,
properties: queryRequestOptions.Properties,
partitionedQueryExecutionInfo: partitionedQueryExecutionInfo,
executionEnvironment: null,
returnResultsInDeterministicOrder: null,
forcePassthrough: true,
testInjections: null);
IQueryPipelineStage queryPipelineStage = CosmosQueryExecutionContextFactory.Create(
documentContainer,
cosmosQueryContextCore,
inputParameters,
NoOpTrace.Singleton);
bool result = queryPipelineStage.MoveNextAsync(NoOpTrace.Singleton).Result;
Assert.AreEqual(input.ExpectedPassThrough, inputParameters.SqlQuerySpec.PassThrough);
Assert.IsNotNull(queryPipelineStage);
Assert.IsTrue(result);
return(new PassThroughQueryTestOutput(inputParameters.SqlQuerySpec.PassThrough));
}
public ReadFeedIteratorCore(
IDocumentContainer documentContainer,
string continuationToken,
ReadFeedPaginationOptions readFeedPaginationOptions,
QueryRequestOptions queryRequestOptions,
CancellationToken cancellationToken)
{
this.queryRequestOptions = queryRequestOptions;
readFeedPaginationOptions ??= ReadFeedPaginationOptions.Default;
if (!string.IsNullOrEmpty(continuationToken))
{
bool isNewArrayFormat = (continuationToken.Length >= 2) && (continuationToken[0] == '[') && (continuationToken[continuationToken.Length - 1] == ']');
if (!isNewArrayFormat)
{
// One of the two older formats
if (!FeedRangeContinuation.TryParse(continuationToken, out FeedRangeContinuation feedRangeContinuation))
{
// Backward compatible with old format
feedRangeContinuation = new FeedRangeCompositeContinuation(
containerRid: string.Empty,
FeedRangeEpk.FullRange,
new List <Documents.Routing.Range <string> >()
{
new Documents.Routing.Range <string>(
Documents.Routing.PartitionKeyInternal.MinimumInclusiveEffectivePartitionKey,
Documents.Routing.PartitionKeyInternal.MaximumExclusiveEffectivePartitionKey,
isMinInclusive: true,
isMaxInclusive: false)
},
continuationToken);
}
// need to massage it a little
List <CosmosElement> feedRangeStates = new List <CosmosElement>();
string oldContinuationFormat = feedRangeContinuation.ToString();
if (feedRangeContinuation.FeedRange is FeedRangePartitionKey feedRangePartitionKey)
{
CosmosObject cosmosObject = CosmosObject.Parse(oldContinuationFormat);
CosmosArray continuations = (CosmosArray)cosmosObject["Continuation"];
if (continuations.Count != 1)
{
throw new InvalidOperationException("Expected only one continuation for partition key queries");
}
CosmosElement continuation = continuations[0];
CosmosObject continuationObject = (CosmosObject)continuation;
CosmosElement token = continuationObject["token"];
ReadFeedState state;
if (token is CosmosNull)
{
state = ReadFeedState.Beginning();
}
else
{
CosmosString tokenAsString = (CosmosString)token;
state = ReadFeedState.Continuation(CosmosElement.Parse(tokenAsString.Value));
}
FeedRangeState <ReadFeedState> feedRangeState = new FeedRangeState <ReadFeedState>(feedRangePartitionKey, state);
feedRangeStates.Add(ReadFeedFeedRangeStateSerializer.ToCosmosElement(feedRangeState));
}
else
{
CosmosObject cosmosObject = CosmosObject.Parse(oldContinuationFormat);
CosmosArray continuations = (CosmosArray)cosmosObject["Continuation"];
foreach (CosmosElement continuation in continuations)
{
CosmosObject continuationObject = (CosmosObject)continuation;
CosmosObject rangeObject = (CosmosObject)continuationObject["range"];
string min = ((CosmosString)rangeObject["min"]).Value;
string max = ((CosmosString)rangeObject["max"]).Value;
CosmosElement token = continuationObject["token"];
FeedRangeInternal feedRange = new FeedRangeEpk(new Documents.Routing.Range <string>(min, max, isMinInclusive: true, isMaxInclusive: false));
ReadFeedState state;
if (token is CosmosNull)
{
state = ReadFeedState.Beginning();
}
else
{
CosmosString tokenAsString = (CosmosString)token;
state = ReadFeedState.Continuation(CosmosElement.Parse(tokenAsString.Value));
}
FeedRangeState <ReadFeedState> feedRangeState = new FeedRangeState <ReadFeedState>(feedRange, state);
feedRangeStates.Add(ReadFeedFeedRangeStateSerializer.ToCosmosElement(feedRangeState));
}
}
CosmosArray cosmosArrayContinuationTokens = CosmosArray.Create(feedRangeStates);
continuationToken = cosmosArrayContinuationTokens.ToString();
}
}
TryCatch <ReadFeedCrossFeedRangeState> monadicReadFeedState;
if (continuationToken == null)
{
FeedRange feedRange;
if ((this.queryRequestOptions != null) && this.queryRequestOptions.PartitionKey.HasValue)
{
feedRange = new FeedRangePartitionKey(this.queryRequestOptions.PartitionKey.Value);
}
else if ((this.queryRequestOptions != null) && (this.queryRequestOptions.FeedRange != null))
{
feedRange = this.queryRequestOptions.FeedRange;
}
else
{
feedRange = FeedRangeEpk.FullRange;
}
monadicReadFeedState = TryCatch <ReadFeedCrossFeedRangeState> .FromResult(ReadFeedCrossFeedRangeState.CreateFromBeginning(feedRange));
}
else
{
monadicReadFeedState = ReadFeedCrossFeedRangeState.Monadic.Parse(continuationToken);
}
if (monadicReadFeedState.Failed)
{
this.monadicEnumerator = TryCatch <CrossPartitionReadFeedAsyncEnumerator> .FromException(monadicReadFeedState.Exception);
}
else
{
this.monadicEnumerator = TryCatch <CrossPartitionReadFeedAsyncEnumerator> .FromResult(
CrossPartitionReadFeedAsyncEnumerator.Create(
documentContainer,
new CrossFeedRangeState <ReadFeedState>(monadicReadFeedState.Result.FeedRangeStates),
readFeedPaginationOptions,
cancellationToken));
}
this.hasMoreResults = true;
}
private async Task <List <T> > ToListAsync <T>(
QueryStream createStreamQuery,
Query <T> createQuery,
string queryText,
QueryRequestOptions requestOptions)
{
HttpStatusCode expectedStatus = HttpStatusCode.OK;
FeedIterator feedStreamIterator = createStreamQuery(queryText, null, requestOptions);
List <T> streamResults = new List <T>();
while (feedStreamIterator.HasMoreResults)
{
ResponseMessage response = await feedStreamIterator.ReadNextAsync();
response.EnsureSuccessStatusCode();
Assert.AreEqual(expectedStatus, response.StatusCode);
StreamReader sr = new StreamReader(response.Content);
string result = await sr.ReadToEndAsync();
ICollection <T> responseResults;
responseResults = JsonConvert.DeserializeObject <CosmosFeedResponseUtil <T> >(result).Data;
Assert.IsTrue(responseResults.Count <= 1);
streamResults.AddRange(responseResults);
}
string continuationToken = null;
List <T> pagedStreamResults = new List <T>();
do
{
FeedIterator pagedFeedIterator = createStreamQuery(queryText, continuationToken, requestOptions);
ResponseMessage response = await pagedFeedIterator.ReadNextAsync();
response.EnsureSuccessStatusCode();
Assert.AreEqual(expectedStatus, response.StatusCode);
IEnumerable <T> responseResults = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <T> >(response.Content).Data;
Assert.IsTrue(responseResults.Count() <= 1);
pagedStreamResults.AddRange(responseResults);
continuationToken = response.Headers.ContinuationToken;
Assert.AreEqual(response.ContinuationToken, response.Headers.ContinuationToken);
} while (continuationToken != null);
Assert.AreEqual(pagedStreamResults.Count, streamResults.Count);
// Both lists should be the same if not PermssionsProperties. PermissionProperties will have a different ResouceToken in the payload when read.
string streamResultString = JsonConvert.SerializeObject(streamResults);
string streamPagedResultString = JsonConvert.SerializeObject(pagedStreamResults);
if (typeof(T) != typeof(PermissionProperties))
{
Assert.AreEqual(streamPagedResultString, streamResultString);
}
FeedIterator <T> feedIterator = createQuery(queryText, null, requestOptions);
List <T> results = new List <T>();
while (feedIterator.HasMoreResults)
{
FeedResponse <T> response = await feedIterator.ReadNextAsync();
Assert.AreEqual(expectedStatus, response.StatusCode);
Assert.IsTrue(response.Count <= 1);
Assert.IsTrue(response.Resource.Count() <= 1);
results.AddRange(response);
}
continuationToken = null;
List <T> pagedResults = new List <T>();
do
{
FeedIterator <T> pagedFeedIterator = createQuery(queryText, continuationToken, requestOptions);
FeedResponse <T> response = await pagedFeedIterator.ReadNextAsync();
Assert.AreEqual(expectedStatus, response.StatusCode);
Assert.IsTrue(response.Count <= 1);
Assert.IsTrue(response.Resource.Count() <= 1);
pagedResults.AddRange(response);
continuationToken = response.ContinuationToken;
} while (continuationToken != null);
Assert.AreEqual(pagedResults.Count, results.Count);
// Both lists should be the same
string resultString = JsonConvert.SerializeObject(results);
string pagedResultString = JsonConvert.SerializeObject(pagedResults);
if (typeof(T) != typeof(PermissionProperties))
{
Assert.AreEqual(pagedResultString, resultString);
Assert.AreEqual(streamPagedResultString, resultString);
}
return(results);
}
public async Task <CosmosListPage <T> > GetItemsAsync(IQueryable <T> queryable, QueryRequestOptions requestOptions, string continuationToken = null)
{
QueryDefinition queryDefinition = queryable.ToQueryDefinition();
FeedIterator <T> queryResultSetIterator = _container.GetItemQueryIterator <T>(queryDefinition, continuationToken, requestOptions);
List <T> results = new List <T>();
FeedResponse <T> currentResultSet = await queryResultSetIterator.ReadNextAsync();
foreach (T document in currentResultSet)
{
results.Add(document);
}
Response <int> count = await queryable.CountAsync();
return(new CosmosListPage <T>()
{
Meta = new CosmosMeta()
{
PageSize = (int)requestOptions.MaxItemCount, Total = count.Resource, ContinuationToken = currentResultSet.ContinuationToken
},
Items = results
});
}
