public async Task MemoryLeak()
{
int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds;
uint numberOfDocuments = 100;
QueryOracleUtil util = new QueryOracle2(seed);
IEnumerable <string> inputDocuments = util.GetDocuments(numberOfDocuments);
await this.CreateIngestQueryDeleteAsync(
ConnectionModes.Direct,
CollectionTypes.MultiPartition,
inputDocuments,
ImplementationAsync);
async Task ImplementationAsync(Container container, IReadOnlyList <CosmosObject> documents)
{
List <WeakReference> weakReferences = await CreateWeakReferenceToFeedIterator(container);
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
foreach (WeakReference weakReference in weakReferences)
{
Assert.IsFalse(weakReference.IsAlive);
}
}
}
public async Task Sanity()
{
int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds;
uint numberOfDocuments = 100;
QueryOracleUtil util = new QueryOracle2(seed);
IEnumerable <string> inputDocuments = util.GetDocuments(numberOfDocuments);
await this.CreateIngestQueryDeleteAsync(
ConnectionModes.Direct,
CollectionTypes.SinglePartition | CollectionTypes.MultiPartition,
inputDocuments,
ImplementationAsync);
public async Task TestBasicCrossPartitionQueryAsync()
{
int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds;
uint numberOfDocuments = 100;
QueryOracleUtil util = new QueryOracle2(seed);
IEnumerable <string> inputDocuments = util.GetDocuments(numberOfDocuments);
await this.CreateIngestQueryDeleteAsync(
ConnectionModes.Direct,
CollectionTypes.SinglePartition | CollectionTypes.MultiPartition,
inputDocuments,
ImplementationAsync);
async Task ImplementationAsync(Container container, IReadOnlyList <CosmosObject> documents)
{
foreach (int maxDegreeOfParallelism in new int[] { 1, 100 })
{
foreach (int maxItemCount in new int[] { 10, 100 })
{
foreach (string query in new string[] { "SELECT c.id FROM c", "SELECT c._ts, c.id FROM c ORDER BY c._ts" })
{
QueryRequestOptions feedOptions = new QueryRequestOptions
{
MaxBufferedItemCount = 7000,
MaxConcurrency = maxDegreeOfParallelism,
MaxItemCount = maxItemCount,
ReturnResultsInDeterministicOrder = true,
};
List <CosmosElement> queryResults = await QueryTestsBase.RunQueryAsync(
container,
query,
feedOptions);
Assert.AreEqual(
documents.Count(),
queryResults.Count,
$"query: {query} failed with {nameof(maxDegreeOfParallelism)}: {maxDegreeOfParallelism}, {nameof(maxItemCount)}: {maxItemCount}");
}
}
}
}
}
public async Task TestExceptionlessFailuresAsync()
{
int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds;
uint numberOfDocuments = 100;
QueryOracleUtil util = new QueryOracle2(seed);
IEnumerable <string> inputDocuments = util.GetDocuments(numberOfDocuments);
await this.CreateIngestQueryDeleteAsync(
ConnectionModes.Direct,
CollectionTypes.SinglePartition | CollectionTypes.MultiPartition,
inputDocuments,
ImplementationAsync);
async Task ImplementationAsync(Container container, IReadOnlyList <CosmosObject> documents)
{
foreach (int maxItemCount in new int[] { 10, 100 })
{
foreach (string query in new string[] { "SELECT c.id FROM c", "SELECT c._ts, c.id FROM c ORDER BY c._ts" })
{
QueryRequestOptions feedOptions = new QueryRequestOptions
{
MaxBufferedItemCount = 7000,
MaxConcurrency = 2,
MaxItemCount = maxItemCount,
TestSettings = new TestInjections(simulate429s: true, simulateEmptyPages: false)
};
List <CosmosElement> queryResults = await QueryTestsBase.RunQueryAsync(
container,
query,
feedOptions);
Assert.AreEqual(
documents.Count(),
queryResults.Count,
$"query: {query} failed with {nameof(maxItemCount)}: {maxItemCount}");
}
}
}
}
public async Task TestQueryCrossPartitionTopAsync()
{
int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds;
uint numDocuments = 100;
string partitionKey = "field_0";
QueryOracleUtil util = new QueryOracle2(seed);
IEnumerable <string> documentsToInsert = util.GetDocuments(numDocuments);
await this.CreateIngestQueryDeleteAsync(
ConnectionModes.Direct,
CollectionTypes.SinglePartition | CollectionTypes.MultiPartition,
documentsToInsert,
ImplementationAsync,
"/" + partitionKey);
async Task ImplementationAsync(Container container, IReadOnlyList <CosmosObject> documents)
{
List <string> queryFormats = new List <string>()
{
"SELECT {0} TOP {1} * FROM c",
// Can't do order by since order by needs to look at all partitions before returning a single document =>
// thus we can't tell how many documents the SDK needs to recieve.
//"SELECT {0} TOP {1} * FROM c ORDER BY c._ts",
// Can't do aggregates since that also retrieves more documents than the user sees
//"SELECT {0} TOP {1} VALUE AVG(c._ts) FROM c",
};
foreach (string queryFormat in queryFormats)
{
foreach (bool useDistinct in new bool[] { true, false })
{
foreach (int topCount in new int[] { 0, 1, 10 })
{
foreach (int pageSize in new int[] { 1, 10 })
{
// Run the query and use the query metrics to make sure the query didn't grab more documents
// than needed.
string query = string.Format(queryFormat, useDistinct ? "DISTINCT" : string.Empty, topCount);
FeedOptions feedOptions = new FeedOptions
{
MaxBufferedItemCount = 1000,
};
// Max DOP needs to be 0 since the query needs to run in serial =>
// otherwise the parallel code will prefetch from other partitions,
// since the first N-1 partitions might be empty.
using (FeedIterator <dynamic> documentQuery = container.GetItemQueryIterator <dynamic>(
query,
requestOptions: new QueryRequestOptions()
{
MaxConcurrency = 0, MaxItemCount = pageSize
}))
{
//QueryMetrics aggregatedQueryMetrics = QueryMetrics.Zero;
int numberOfDocuments = 0;
while (documentQuery.HasMoreResults)
{
FeedResponse <dynamic> cosmosQueryResponse = await documentQuery.ReadNextAsync();
numberOfDocuments += cosmosQueryResponse.Count();
//foreach (QueryMetrics queryMetrics in cosmosQueryResponse.QueryMetrics.Values)
//{
// aggregatedQueryMetrics += queryMetrics;
//}
}
Assert.IsTrue(
numberOfDocuments <= topCount,
$"Received {numberOfDocuments} documents with query: {query} and pageSize: {pageSize}");
}
//if (!useDistinct)
//{
// Assert.IsTrue(
// aggregatedQueryMetrics.OutputDocumentCount <= topCount,
// $"Received {aggregatedQueryMetrics.OutputDocumentCount} documents query: {query} and pageSize: {pageSize}");
//}
}
}
}
}
}
}
public async Task TestNonDeterministicQueryResultsAsync()
{
int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds;
uint numberOfDocuments = 100;
QueryOracleUtil util = new QueryOracle2(seed);
IEnumerable <string> documents = util.GetDocuments(numberOfDocuments);
await this.CreateIngestQueryDeleteAsync(
ConnectionModes.Direct,
CollectionTypes.MultiPartition,
documents,
ImplementationAsync);
async Task ImplementationAsync(Container container, IReadOnlyList <CosmosObject> inputDocuments)
{
foreach (int maxDegreeOfParallelism in new int[] { 1, 100 })
{
foreach (int maxItemCount in new int[] { 10, 100 })
{
foreach (bool useOrderBy in new bool[] { false, true })
{
string query;
if (useOrderBy)
{
query = "SELECT c._ts, c.id FROM c ORDER BY c._ts";
}
else
{
query = "SELECT c.id FROM c";
}
QueryRequestOptions queryRequestOptions = new QueryRequestOptions
{
MaxBufferedItemCount = 7000,
MaxConcurrency = maxDegreeOfParallelism,
MaxItemCount = maxItemCount,
ReturnResultsInDeterministicOrder = false,
};
async Task ValidateNonDeterministicQuery(
Func <Container, string, QueryRequestOptions, Task <List <CosmosObject> > > queryFunc,
bool hasOrderBy)
{
List <CosmosObject> queryResults = await queryFunc(container, query, queryRequestOptions);
HashSet <string> expectedIds = new HashSet <string>(inputDocuments
.Select(document => ((CosmosString)document["id"]).Value));
HashSet <string> actualIds = new HashSet <string>(queryResults
.Select(queryResult => ((CosmosString)queryResult["id"]).Value));
Assert.IsTrue(
expectedIds.SetEquals(actualIds),
$"query: {query} failed with {nameof(maxDegreeOfParallelism)}: {maxDegreeOfParallelism}, {nameof(maxItemCount)}: {maxItemCount}");
if (hasOrderBy)
{
IEnumerable <long> timestamps = queryResults.Select(token => (long)token["_ts"].ToDouble());
IEnumerable <long> sortedTimestamps = timestamps.OrderBy(x => x);
Assert.IsTrue(timestamps.SequenceEqual(sortedTimestamps), "Items were not sorted.");
}
}
await ValidateNonDeterministicQuery(QueryTestsBase.QueryWithoutContinuationTokensAsync <CosmosObject>, useOrderBy);
await ValidateNonDeterministicQuery(QueryTestsBase.QueryWithContinuationTokensAsync <CosmosObject>, useOrderBy);
await ValidateNonDeterministicQuery(QueryTestsBase.QueryWithCosmosElementContinuationTokenAsync <CosmosObject>, useOrderBy);
}
}
}
}
}
public async Task TestQueryPlanGatewayAndServiceInteropAsync()
{
int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds;
uint numberOfDocuments = 100;
QueryOracleUtil util = new QueryOracle2(seed);
IEnumerable <string> inputDocuments = util.GetDocuments(numberOfDocuments);
await this.CreateIngestQueryDeleteAsync(
ConnectionModes.Direct,
CollectionTypes.SinglePartition | CollectionTypes.MultiPartition,
inputDocuments,
ImplementationAsync);
async Task ImplementationAsync(Container container, IReadOnlyList <CosmosObject> documents)
{
ContainerCore containerCore = (ContainerInlineCore)container;
foreach (bool isGatewayQueryPlan in new bool[] { true, false })
{
MockCosmosQueryClient cosmosQueryClientCore = new MockCosmosQueryClient(
containerCore.ClientContext,
containerCore,
isGatewayQueryPlan);
ContainerCore containerWithForcedPlan = new ContainerCore(
containerCore.ClientContext,
(DatabaseCore)containerCore.Database,
containerCore.Id,
cosmosQueryClientCore);
int numOfQueries = 0;
foreach (int maxDegreeOfParallelism in new int[] { 1, 100 })
{
foreach (int maxItemCount in new int[] { 10, 100 })
{
numOfQueries++;
QueryRequestOptions feedOptions = new QueryRequestOptions
{
MaxBufferedItemCount = 7000,
MaxConcurrency = maxDegreeOfParallelism,
MaxItemCount = maxItemCount,
};
List <CosmosElement> queryResults = await QueryTestsBase.RunQueryAsync(
containerWithForcedPlan,
"SELECT * FROM c ORDER BY c._ts",
feedOptions);
Assert.AreEqual(documents.Count(), queryResults.Count);
}
}
if (isGatewayQueryPlan)
{
Assert.IsTrue(cosmosQueryClientCore.QueryPlanCalls > numOfQueries);
}
else
{
Assert.AreEqual(0, cosmosQueryClientCore.QueryPlanCalls, "ServiceInterop mode should not be calling gateway plan retriever");
}
}
}
}
