public async Task UndefinedSinglePageAsync()
{
IReadOnlyList <IReadOnlyList <CosmosElement> > pages = new List <List <CosmosElement> >()
{
new List <CosmosElement>()
{
CosmosElement.Parse("{\"payload\": {\"$1\": {}}}")
}
};
List <CosmosElement> elements = await AggregateQueryPipelineStageTests.CreateAndDrain(
pages : pages,
executionEnvironment : ExecutionEnvironment.Compute,
aggregates : new List <AggregateOperator>()
{
AggregateOperator.Sum
},
aliasToAggregateType : new Dictionary <string, AggregateOperator?>()
{
{ "$1", AggregateOperator.Sum }
},
orderedAliases : new List <string>()
{
"$1"
},
hasSelectValue : false,
continuationToken : null);
Assert.AreEqual(1, elements.Count);
Assert.AreEqual(0, (elements[0] as CosmosObject).Keys.Count());
}
public override async ValueTask <bool> MoveNextAsync(ITrace trace)
{
this.cancellationToken.ThrowIfCancellationRequested();
if (trace == null)
{
throw new ArgumentNullException(nameof(trace));
}
if (!await this.inputStage.MoveNextAsync(trace))
{
this.Current = default;
return(false);
}
TryCatch <QueryPage> tryGetSourcePage = this.inputStage.Current;
if (tryGetSourcePage.Failed)
{
this.Current = tryGetSourcePage;
return(true);
}
QueryPage sourcePage = tryGetSourcePage.Result;
// Skip the documents but keep all the other headers
IReadOnlyList <CosmosElement> documentsAfterSkip = sourcePage.Documents.Skip(this.skipCount).ToList();
int numberOfDocumentsSkipped = sourcePage.Documents.Count - documentsAfterSkip.Count;
this.skipCount -= numberOfDocumentsSkipped;
QueryState state;
if ((sourcePage.State != null) && (sourcePage.DisallowContinuationTokenMessage == null))
{
string token = new OffsetContinuationToken(
offset: this.skipCount,
sourceToken: sourcePage.State?.Value.ToString()).ToString();
state = new QueryState(CosmosElement.Parse(token));
}
else
{
state = null;
}
QueryPage queryPage = new QueryPage(
documents: documentsAfterSkip,
requestCharge: sourcePage.RequestCharge,
activityId: sourcePage.ActivityId,
responseLengthInBytes: sourcePage.ResponseLengthInBytes,
cosmosQueryExecutionInfo: sourcePage.CosmosQueryExecutionInfo,
disallowContinuationTokenMessage: sourcePage.DisallowContinuationTokenMessage,
additionalHeaders: sourcePage.AdditionalHeaders,
state: state);
this.Current = TryCatch <QueryPage> .FromResult(queryPage);
return(true);
}
public async Task SinglePageAsync()
{
IReadOnlyList <IReadOnlyList <CosmosElement> > pages = new List <List <CosmosElement> >()
{
new List <CosmosElement>()
{
CosmosElement.Parse("{\"groupByItems\": [{\"item\" : \"John\"}], \"payload\" : {\"name\": \"John\", \"count\": {\"item\": 42}}}")
}
};
List <CosmosElement> elements = await GroupByQueryPipelineStageTests.CreateAndDrainAsync(
pages : pages,
executionEnvironment : ExecutionEnvironment.Compute,
continuationToken : null,
groupByAliasToAggregateType : new Dictionary <string, AggregateOperator?>()
{
{ "name", null }, { "count", AggregateOperator.Sum }
},
orderedAliases : new List <string>()
{
"name", "count"
},
hasSelectValue : false);
Assert.AreEqual(1, elements.Count);
Assert.AreEqual(42, Number64.ToLong(((elements[0] as CosmosObject)["count"] as CosmosNumber).Value));
Assert.AreEqual("John", ((elements[0] as CosmosObject)["name"] as CosmosString).Value.ToString());
}
public void SanityTest()
{
List <CosmosElement> allElements = GenerateInputsFromClass(typeof(Elements))
.Concat(GenerateInputsFromClass(typeof(ExtendedTypes)))
.Concat(GenerateInputsFromClass(typeof(Numbers)))
.ToList();
foreach (CosmosElement first in allElements)
{
foreach (CosmosElement second in allElements)
{
if (object.ReferenceEquals(first, second))
{
// They should be equals semantically too
Assert.AreEqual(first, second);
// Try with a deep copy to make sure it's not just asserting reference eqaulity.
Assert.AreEqual(first, CosmosElement.Parse(second.ToString()));
// If two objects are equal, then so are their hashes (consistent hashing).
Assert.AreEqual(first.GetHashCode(), second.GetHashCode());
}
else
{
Assert.AreNotEqual(first, second);
}
}
}
}
public async Task MultiplePagesAsync()
{
long[] values = new long[] { 42, 1337 };
IReadOnlyList <IReadOnlyList <CosmosElement> > pages = values
.Select(value => new List <CosmosElement>()
{
CosmosElement.Parse($"{{\"payload\": {{\"$1\": {{\"item\": {value}}}}}}}")
})
.ToList();
List <CosmosElement> elements = await AggregateQueryPipelineStageTests.CreateAndDrain(
pages : pages,
executionEnvironment : ExecutionEnvironment.Compute,
aggregates : new List <AggregateOperator>()
{
AggregateOperator.Sum
},
aliasToAggregateType : new Dictionary <string, AggregateOperator?>()
{
{ "$1", AggregateOperator.Sum }
},
orderedAliases : new List <string>()
{
"$1"
},
hasSelectValue : false,
continuationToken : null);
Assert.AreEqual(1, elements.Count);
Assert.AreEqual(values.Sum(), Number64.ToLong(((elements[0] as CosmosObject)["$1"] as CosmosNumber).Value));
}
public override async ValueTask <bool> MoveNextAsync()
{
this.cancellationToken.ThrowIfCancellationRequested();
if (!await this.inputStage.MoveNextAsync())
{
this.Current = default;
return(false);
}
TryCatch <QueryPage> tryGetSourcePage = this.inputStage.Current;
if (tryGetSourcePage.Failed)
{
this.Current = tryGetSourcePage;
return(true);
}
QueryPage sourcePage = tryGetSourcePage.Result;
List <CosmosElement> takedDocuments = sourcePage.Documents.Take(this.takeCount).ToList();
this.takeCount -= takedDocuments.Count;
QueryState state;
if ((sourcePage.State != null) && (sourcePage.DisallowContinuationTokenMessage == null))
{
string updatedContinuationToken = this.takeEnum switch
{
TakeEnum.Limit => new LimitContinuationToken(
limit: this.takeCount,
sourceToken: sourcePage.State?.Value.ToString()).ToString(),
TakeEnum.Top => new TopContinuationToken(
top: this.takeCount,
sourceToken: sourcePage.State?.Value.ToString()).ToString(),
_ => throw new ArgumentOutOfRangeException($"Unknown {nameof(TakeEnum)}: {this.takeEnum}."),
};
state = new QueryState(CosmosElement.Parse(updatedContinuationToken));
}
else
{
state = null;
}
QueryPage queryPage = new QueryPage(
documents: takedDocuments,
requestCharge: sourcePage.RequestCharge,
activityId: sourcePage.ActivityId,
responseLengthInBytes: sourcePage.ResponseLengthInBytes,
cosmosQueryExecutionInfo: sourcePage.CosmosQueryExecutionInfo,
disallowContinuationTokenMessage: sourcePage.DisallowContinuationTokenMessage,
state: state);
this.Current = TryCatch <QueryPage> .FromResult(queryPage);
return(true);
}
internal static async Task <List <T> > QueryWithCosmosElementContinuationTokenAsync <T>(
Container container,
string query,
QueryRequestOptions queryRequestOptions = null)
{
if (queryRequestOptions == null)
{
queryRequestOptions = new QueryRequestOptions();
}
List <T> resultsFromCosmosElementContinuationToken = new List <T>();
CosmosElement continuationToken = null;
do
{
QueryRequestOptions computeRequestOptions = queryRequestOptions.Clone();
computeRequestOptions.ExecutionEnvironment = Cosmos.Query.Core.ExecutionContext.ExecutionEnvironment.Compute;
computeRequestOptions.CosmosElementContinuationToken = continuationToken;
FeedIteratorInternal <T> itemQuery = (FeedIteratorInternal <T>)container.GetItemQueryIterator <T>(
queryText: query,
requestOptions: computeRequestOptions);
try
{
FeedResponse <T> cosmosQueryResponse = await itemQuery.ReadNextAsync();
if (queryRequestOptions.MaxItemCount.HasValue)
{
Assert.IsTrue(
cosmosQueryResponse.Count <= queryRequestOptions.MaxItemCount.Value,
"Max Item Count is not being honored");
}
resultsFromCosmosElementContinuationToken.AddRange(cosmosQueryResponse);
// Force a rewrite of the continuation token, so that we test the case where we roundtrip it over the wire.
// There was a bug where resuming from double.NaN lead to an exception,
// since we parsed the type assuming it was always a double and not a string.
CosmosElement originalContinuationToken = itemQuery.GetCosmosElementContinuationToken();
if (originalContinuationToken != null)
{
continuationToken = CosmosElement.Parse(originalContinuationToken.ToString());
}
else
{
continuationToken = null;
}
}
catch (CosmosException cosmosException) when(cosmosException.StatusCode == (HttpStatusCode)429)
{
itemQuery = (FeedIteratorInternal <T>)container.GetItemQueryIterator <T>(
queryText: query,
requestOptions: queryRequestOptions);
}
} while (continuationToken != null);
return(resultsFromCosmosElementContinuationToken);
}
static IReadOnlyList <(PageList, int, DistinctQueryType[])> TestCases()
{
List <(PageList, int, DistinctQueryType[])> result = new List <(PageList, int, DistinctQueryType[])>();
long[] values = new long[] { 42, 1337, 1337, 42 };
PageList pages = values
.Select(value => new List <CosmosElement>()
{
CosmosElement.Parse($"{{\"item\": {value}}}")
})
.ToList();
result.Add((pages, values.Distinct().Count(), new[] { DistinctQueryType.Unordered }));
values = new long[] { 0, 1, 1, 2, 3, 5, 8, 13 };
pages = values
.Select(value => new List <CosmosElement>()
{
CosmosElement.Parse($"{{\"item\": {value}}}")
})
.ToList();
result.Add((pages, values.Distinct().Count(), new[] { DistinctQueryType.Unordered, DistinctQueryType.Ordered }));
int expectedCount = 50;
pages = Enumerable
.Range(0, 10)
.Select(_ => Enumerable
.Range(0, expectedCount)
.Select(j => CosmosElement.Parse($"{{ \"val\": {j} }}"))
.ToList())
.ToList();
result.Add((pages, expectedCount, new[] { DistinctQueryType.Unordered }));
pages = Enumerable
.Range(0, expectedCount)
.Select(i => Enumerable
.Repeat(i, 50)
.Select(j => CosmosElement.Parse($"{{ \"val\": {j} }}"))
.ToList())
.ToList();
result.Add((pages, expectedCount, new[] { DistinctQueryType.Unordered, DistinctQueryType.Ordered }));
return(result);
}
public async Task SanityTests()
{
long[] values = new long[] { 42, 1337, 1337, 42 };
IReadOnlyList <IReadOnlyList <CosmosElement> > pages = values
.Select(value => new List <CosmosElement>()
{
CosmosElement.Parse($"{{\"item\": {value}}}")
})
.ToList();
List <CosmosElement> elements = await DistinctQueryPipelineStageTests.CreateAndDrainAsync(
pages : pages,
executionEnvironment : ExecutionEnvironment.Compute,
continuationToken : null,
distinctQueryType : DistinctQueryType.Unordered);
Assert.AreEqual(values.Distinct().Count(), elements.Count);
}
public async Task TestNegativeDistinctComponentCreation()
{
TryCatch <IDocumentQueryExecutionComponent> tryCreateWhenSourceFails = await DistinctDocumentQueryExecutionComponent.TryCreateAsync(
ExecutionEnvironment.Client,
null,
FailToCreateSource,
DistinctQueryType.Ordered);
Assert.IsFalse(tryCreateWhenSourceFails.Succeeded);
TryCatch <IDocumentQueryExecutionComponent> tryCreateWhenInvalidContinuationToken = await DistinctDocumentQueryExecutionComponent.TryCreateAsync(
ExecutionEnvironment.Client,
CosmosElement.Parse("\"This is not a valid continuation token\""),
CreateSource,
DistinctQueryType.Unordered);
Assert.IsFalse(tryCreateWhenInvalidContinuationToken.Succeeded);
}
/// <summary>
/// Gets the next page of results from this context.
/// </summary>
/// <param name="token">The cancellation token.</param>
/// <returns>A task to await on that in turn returns a DoucmentFeedResponse of results.</returns>
public override async Task <QueryResponseCore> ExecuteNextAsync(CancellationToken token)
{
try
{
QueryResponseCore queryResponse = await this.component.DrainAsync(this.actualPageSize, token);
if (!queryResponse.IsSuccess)
{
this.component.Stop();
return(queryResponse);
}
string updatedContinuationToken;
if (queryResponse.DisallowContinuationTokenMessage == null)
{
if (queryResponse.ContinuationToken != null)
{
updatedContinuationToken = new PipelineContinuationTokenV0(CosmosElement.Parse(queryResponse.ContinuationToken)).ToString();
}
else
{
updatedContinuationToken = null;
}
}
else
{
updatedContinuationToken = null;
}
return(QueryResponseCore.CreateSuccess(
result: queryResponse.CosmosElements,
continuationToken: updatedContinuationToken,
disallowContinuationTokenMessage: queryResponse.DisallowContinuationTokenMessage,
activityId: queryResponse.ActivityId,
requestCharge: queryResponse.RequestCharge,
diagnostics: queryResponse.Diagnostics,
responseLengthBytes: queryResponse.ResponseLengthBytes));
}
catch (Exception)
{
this.component.Stop();
throw;
}
}
public static Payload Create(string name, string jsonText)
{
if (jsonText == null)
{
throw new ArgumentNullException(nameof(jsonText));
}
CosmosElement element = CosmosElement.Parse(jsonText);
IJsonWriter jsonBinaryWriter = JsonWriter.Create(JsonSerializationFormat.Binary);
IJsonWriter jsonTextWriter = JsonWriter.Create(JsonSerializationFormat.Text);
element.WriteTo(jsonTextWriter);
element.WriteTo(jsonBinaryWriter);
ReadOnlyMemory <byte> text = jsonTextWriter.GetResult();
ReadOnlyMemory <byte> binary = jsonBinaryWriter.GetResult();
return(new Payload(name, text, binary));
}
public async Task SanityTests()
{
long[] values = new long[] { 42, 1337 };
PageList pages = values
.Select(value => new List <CosmosElement>()
{
CosmosElement.Parse($"{{\"item\": {value}}}")
})
.ToList();
foreach (int takeCount in new int[] { 0, 1, values.Length, 2 * values.Length })
{
(List <CosmosElement> elements, _) = await TakeQueryPipelineStageTests.CreateAndDrainAsync(
pages : pages,
executionEnvironment : ExecutionEnvironment.Compute,
takeCount : takeCount,
continuationToken : null);
Assert.AreEqual(Math.Min(takeCount, values.Length), elements.Count);
}
}
public async Task SanityTests()
{
long[] values = new long[] { 42, 1337 };
IReadOnlyList <IReadOnlyList <CosmosElement> > pages = values
.Select(value => new List <CosmosElement>()
{
CosmosElement.Parse($"{{\"item\": {value}}}")
})
.ToList();
foreach (int offsetCount in new int[] { 0, 1, values.Length, 2 * values.Length })
{
List <CosmosElement> elements = await SkipQueryPipelineStageTests.CreateAndDrainAsync(
pages : pages,
executionEnvironment : ExecutionEnvironment.Compute,
offsetCount : offsetCount,
continuationToken : null);
Assert.AreEqual(Math.Max(values.Length - offsetCount, 0), elements.Count);
}
}
public DeserializerBenchmark()
{
List <Person> people = new List <Person>();
for (int i = 0; i < 1000; i++)
{
people.Add(Person.GetRandomPerson());
}
this.json = JsonConvert.SerializeObject(people);
IJsonWriter jsonTextWriter = Microsoft.Azure.Cosmos.Json.JsonWriter.Create(JsonSerializationFormat.Text);
CosmosElement.Parse(this.json).WriteTo(jsonTextWriter);
this.textBuffer = jsonTextWriter.GetResult();
IJsonWriter jsonBinaryWriter = Microsoft.Azure.Cosmos.Json.JsonWriter.Create(JsonSerializationFormat.Binary);
CosmosElement.Parse(this.json).WriteTo(jsonBinaryWriter);
this.binaryBuffer = jsonBinaryWriter.GetResult();
}
static IReadOnlyList <(PageList, int, long)> TestCases()
{
// List<(pages, takeCount, pageIndex)>
List <(PageList, int, long)> result = new List <(PageList, int, long)>();
long[] values = new long[] { 0, 1, 13, 42, 1337, 1337, 42, 1, 2, 3 };
PageList pages = Enumerable
.Range(0, 10)
.Select(_ =>
values
.Select(value => CosmosElement.Parse($"{{\"item\": {value}}}"))
.ToList())
.ToList();
foreach (int takeCount in new[] { 1, 3, 6, 10, 15, 20, 23, 32, 41, 56, 81, 97, 100 })
{
long pageIndex = takeCount < values.Length ? 1 : Convert.ToInt64(Math.Ceiling((decimal)takeCount / values.Length));
result.Add((pages, takeCount, pageIndex));
}
return(result);
}
public void ArrayIteratorCollectionExpressionTest()
{
// FROM p in f.parents
SqlArrayIteratorCollectionExpression fromPInFDotParents = SqlArrayIteratorCollectionExpression.Create(
SqlIdentifier.Create("p"),
SqlInputPathCollection.Create(
SqlIdentifier.Create("f"),
SqlStringPathExpression.Create(
null,
SqlStringLiteral.Create("parents"))));
CosmosElement andersenParent1 = CosmosElement.Parse(@"{ ""p"" : { ""firstName"": ""Thomas"" }, ""_rid"": ""0fomAIxnukU1AQAAAAAAAA==""}");
CosmosElement andersenParent2 = CosmosElement.Parse(@"{ ""p"" : { ""firstName"": ""Mary Kay""}, ""_rid"": ""0fomAIxnukU1AQAAAAAAAA=="" }");
CosmosElement wakeFieldParent1 = CosmosElement.Parse(@"{ ""p"" : { ""familyName"": ""Wakefield"", ""givenName"": ""Robin"" }, ""_rid"": ""0fomAIxnukU1AQAAAAAAAB=="" }");
CosmosElement wakeFieldParent2 = CosmosElement.Parse(@"{ ""p"" : { ""familyName"": ""Miller"", ""givenName"": ""Ben"" }, ""_rid"": ""0fomAIxnukU1AQAAAAAAAB=="" }");
AssertEvaluation(new CosmosElement[]
{
andersenParent1,
andersenParent2,
wakeFieldParent1,
wakeFieldParent2,
}, fromPInFDotParents, DataSource);
}
public async Task TestCosmosCrossPartitionQueryExecutionContextWithEmptyPagesAndSplitAsync(bool createInitialContinuationToken)
{
int maxPageSize = 5;
List <MockPartitionResponse[]> mockResponsesScenario = MockQueryFactory.GetSplitScenarios();
foreach (MockPartitionResponse[] mockResponse in mockResponsesScenario)
{
string initialContinuationToken = null;
string fullContinuationToken = null;
if (createInitialContinuationToken)
{
initialContinuationToken = " - RID:02FYAIvUH1kCAAAAAAAAAA ==#RT:1#TRC:1";
CompositeContinuationToken compositeContinuation = new CompositeContinuationToken()
{
Range = new Documents.Routing.Range <string>(
min: MockQueryFactory.DefaultPartitionKeyRange.MinInclusive,
max: MockQueryFactory.DefaultPartitionKeyRange.MaxExclusive,
isMaxInclusive: false,
isMinInclusive: true),
Token = initialContinuationToken
};
fullContinuationToken = CosmosArray.Create(
new List <CosmosElement>()
{
CompositeContinuationToken.ToCosmosElement(compositeContinuation)
}).ToString();
}
Mock <CosmosQueryClient> mockQueryClient = new Mock <CosmosQueryClient>();
IList <ToDoItem> allItems = MockQueryFactory.GenerateAndMockResponse(
mockQueryClient,
isOrderByQuery: false,
sqlQuerySpec: MockQueryFactory.DefaultQuerySpec,
containerRid: MockQueryFactory.DefaultCollectionRid,
initContinuationToken: initialContinuationToken,
maxPageSize: maxPageSize,
mockResponseForSinglePartition: mockResponse,
cancellationTokenForMocks: this.cancellationToken);
CosmosQueryContext context = MockQueryFactory.CreateContext(
mockQueryClient.Object);
CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams initParams = new CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams(
sqlQuerySpec: MockQueryFactory.DefaultQuerySpec,
collectionRid: MockQueryFactory.DefaultCollectionRid,
partitionedQueryExecutionInfo: new PartitionedQueryExecutionInfo()
{
QueryInfo = new QueryInfo()
},
partitionKeyRanges: new List <PartitionKeyRange> {
MockQueryFactory.DefaultPartitionKeyRange
},
initialPageSize: maxPageSize,
maxConcurrency: null,
maxItemCount: maxPageSize,
maxBufferedItemCount: null,
returnResultsInDeterministicOrder: true,
testSettings: new TestInjections(simulate429s: false, simulateEmptyPages: false));
IDocumentQueryExecutionComponent executionContext = (await CosmosParallelItemQueryExecutionContext.TryCreateAsync(
context,
initParams,
fullContinuationToken != null ? CosmosElement.Parse(fullContinuationToken) : null,
this.cancellationToken)).Result;
// Read all the pages from both splits
List <ToDoItem> itemsRead = new List <ToDoItem>();
Assert.IsTrue(!executionContext.IsDone);
while (!executionContext.IsDone)
{
QueryResponseCore queryResponse = await executionContext.DrainAsync(maxPageSize, this.cancellationToken);
string responseContinuationToken = queryResponse.ContinuationToken;
foreach (CosmosElement element in queryResponse.CosmosElements)
{
string jsonValue = element.ToString();
ToDoItem item = JsonConvert.DeserializeObject <ToDoItem>(jsonValue);
itemsRead.Add(item);
}
}
Assert.AreEqual(allItems.Count, itemsRead.Count);
List <ToDoItem> exepected = allItems.OrderBy(x => x.id).ToList();
List <ToDoItem> actual = itemsRead.OrderBy(x => x.id).ToList();
CollectionAssert.AreEqual(exepected, actual, new ToDoItemComparer());
}
}
public async Task TestCosmosOrderByQueryExecutionContextWithFailurePageAsync(bool createInitialContinuationToken)
{
int maxPageSize = 5;
List <MockPartitionResponse[]> mockResponsesScenario = MockQueryFactory.GetFailureScenarios();
Mock <CosmosQueryClient> mockQueryClient = new Mock <CosmosQueryClient>();
foreach (MockPartitionResponse[] mockResponse in mockResponsesScenario)
{
string initialContinuationToken = null;
string fullContinuationToken = null;
if (createInitialContinuationToken)
{
ToDoItem itemToRepresentPreviousQuery = ToDoItem.CreateItems(
1,
"itemToRepresentPreviousQuery",
MockQueryFactory.DefaultCollectionRid).First();
initialContinuationToken = $" - RID:{itemToRepresentPreviousQuery._rid} ==#RT:1#TRC:1";
CompositeContinuationToken compositeContinuation = new CompositeContinuationToken()
{
Range = new Documents.Routing.Range <string>(
min: MockQueryFactory.DefaultPartitionKeyRange.MinInclusive,
max: MockQueryFactory.DefaultPartitionKeyRange.MaxExclusive,
isMaxInclusive: false,
isMinInclusive: true),
Token = initialContinuationToken
};
List <OrderByItem> orderByItems = new List <OrderByItem>()
{
new OrderByItem(CosmosObject.CreateFromBuffer(Encoding.UTF8.GetBytes("{\"item\":\"2c4ce711-13c3-4c93-817c-49287b71b6c3\"}")))
};
OrderByContinuationToken orderByContinuationToken = new OrderByContinuationToken(
compositeContinuationToken: compositeContinuation,
orderByItems: orderByItems,
rid: itemToRepresentPreviousQuery._rid,
skipCount: 0,
filter: null);
fullContinuationToken = CosmosArray.Create(
new List <CosmosElement>()
{
OrderByContinuationToken.ToCosmosElement(orderByContinuationToken)
}).ToString();
}
IList <ToDoItem> allItems = MockQueryFactory.GenerateAndMockResponse(
mockQueryClient,
isOrderByQuery: true,
sqlQuerySpec: MockQueryFactory.DefaultQuerySpec,
containerRid: MockQueryFactory.DefaultCollectionRid,
initContinuationToken: initialContinuationToken,
maxPageSize: maxPageSize,
mockResponseForSinglePartition: mockResponse,
cancellationTokenForMocks: this.cancellationToken);
// Order by drains the partitions until it finds an item
// If there are no items then it's not possible to have a continuation token
if (allItems.Count == 0 && createInitialContinuationToken)
{
continue;
}
CosmosQueryContext context = MockQueryFactory.CreateContext(
mockQueryClient.Object);
QueryInfo queryInfo = new QueryInfo()
{
OrderBy = new SortOrder[] { SortOrder.Ascending },
OrderByExpressions = new string[] { "id" }
};
CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams initParams = new CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams(
sqlQuerySpec: MockQueryFactory.DefaultQuerySpec,
collectionRid: MockQueryFactory.DefaultCollectionRid,
partitionedQueryExecutionInfo: new PartitionedQueryExecutionInfo()
{
QueryInfo = queryInfo
},
partitionKeyRanges: new List <PartitionKeyRange>()
{
MockQueryFactory.DefaultPartitionKeyRange
},
initialPageSize: maxPageSize,
maxConcurrency: null,
maxItemCount: maxPageSize,
maxBufferedItemCount: null,
returnResultsInDeterministicOrder: true,
testSettings: new TestInjections(simulate429s: false, simulateEmptyPages: false));
TryCatch <IDocumentQueryExecutionComponent> tryCreate = await CosmosOrderByItemQueryExecutionContext.TryCreateAsync(
context,
initParams,
fullContinuationToken != null?CosmosElement.Parse(fullContinuationToken) : null,
this.cancellationToken);
if (tryCreate.Succeeded)
{
IDocumentQueryExecutionComponent executionContext = tryCreate.Result;
Assert.IsTrue(!executionContext.IsDone);
// Read all the pages from both splits
List <ToDoItem> itemsRead = new List <ToDoItem>();
QueryResponseCore?failure = null;
while (!executionContext.IsDone)
{
QueryResponseCore queryResponse = await executionContext.DrainAsync(
maxPageSize,
this.cancellationToken);
if (queryResponse.IsSuccess)
{
string responseContinuationToken = queryResponse.ContinuationToken;
foreach (CosmosElement element in queryResponse.CosmosElements)
{
string jsonValue = element.ToString();
ToDoItem item = JsonConvert.DeserializeObject <ToDoItem>(jsonValue);
itemsRead.Add(item);
}
}
else
{
Assert.IsNull(failure, "There should only be one error");
failure = queryResponse;
}
}
Assert.IsNotNull(failure);
Assert.AreEqual((HttpStatusCode)429, failure.Value.StatusCode);
Assert.IsNotNull(failure.Value.CosmosException.ToString());
Assert.AreEqual(0 /*We don't get any items, since we don't buffer the failure anymore*/, itemsRead.Count);
//CollectionAssert.AreEqual(allItems.ToList(), itemsRead, new ToDoItemComparer());
}
else
{
QueryResponseCore queryResponseCore = QueryResponseFactory.CreateFromException(tryCreate.Exception);
Assert.AreEqual((HttpStatusCode)429, queryResponseCore.StatusCode);
}
}
}
public ReadFeedIteratorCore(
IDocumentContainer documentContainer,
QueryRequestOptions queryRequestOptions,
string continuationToken,
int pageSize,
CancellationToken cancellationToken)
{
if (!string.IsNullOrEmpty(continuationToken))
{
bool isNewArrayFormat = (continuationToken.Length >= 2) && (continuationToken[0] == '[') && (continuationToken[continuationToken.Length - 1] == ']');
if (!isNewArrayFormat)
{
// One of the two older formats
if (!FeedRangeContinuation.TryParse(continuationToken, out FeedRangeContinuation feedRangeContinuation))
{
// Backward compatible with old format
feedRangeContinuation = new FeedRangeCompositeContinuation(
containerRid: string.Empty,
FeedRangeEpk.FullRange,
new List <Documents.Routing.Range <string> >()
{
new Documents.Routing.Range <string>(
Documents.Routing.PartitionKeyInternal.MinimumInclusiveEffectivePartitionKey,
Documents.Routing.PartitionKeyInternal.MaximumExclusiveEffectivePartitionKey,
isMinInclusive: true,
isMaxInclusive: false)
},
continuationToken);
}
// need to massage it a little
string oldContinuationFormat = feedRangeContinuation.ToString();
CosmosObject cosmosObject = CosmosObject.Parse(oldContinuationFormat);
CosmosArray continuations = (CosmosArray)cosmosObject["Continuation"];
List <CosmosElement> readFeedContinuationTokens = new List <CosmosElement>();
foreach (CosmosElement continuation in continuations)
{
CosmosObject continuationObject = (CosmosObject)continuation;
CosmosObject rangeObject = (CosmosObject)continuationObject["range"];
string min = ((CosmosString)rangeObject["min"]).Value;
string max = ((CosmosString)rangeObject["max"]).Value;
CosmosElement token = CosmosElement.Parse(((CosmosString)continuationObject["token"]).Value);
FeedRangeInternal feedRange = new FeedRangeEpk(new Documents.Routing.Range <string>(min, max, isMinInclusive: true, isMaxInclusive: false));
ReadFeedState state = new ReadFeedState(token);
ReadFeedContinuationToken readFeedContinuationToken = new ReadFeedContinuationToken(feedRange, state);
readFeedContinuationTokens.Add(ReadFeedContinuationToken.ToCosmosElement(readFeedContinuationToken));
}
CosmosArray cosmosArrayContinuationTokens = CosmosArray.Create(readFeedContinuationTokens);
continuationToken = cosmosArrayContinuationTokens.ToString();
}
}
this.monadicEnumerator = CrossPartitionReadFeedAsyncEnumerator.MonadicCreate(
documentContainer,
queryRequestOptions,
continuationToken: continuationToken,
pageSize,
cancellationToken);
this.hasMoreResults = true;
}
public override async ValueTask <bool> MoveNextAsync()
{
this.cancellationToken.ThrowIfCancellationRequested();
if (!await this.inputStage.MoveNextAsync())
{
this.Current = default;
return(false);
}
TryCatch <QueryPage> tryGetSourcePage = this.inputStage.Current;
if (tryGetSourcePage.Failed)
{
this.Current = tryGetSourcePage;
return(true);
}
QueryPage sourcePage = tryGetSourcePage.Result;
List <CosmosElement> distinctResults = new List <CosmosElement>();
foreach (CosmosElement document in sourcePage.Documents)
{
this.cancellationToken.ThrowIfCancellationRequested();
if (this.distinctMap.Add(document, out UInt128 _))
{
distinctResults.Add(document);
}
}
// For clients we write out the continuation token if it's a streaming query.
QueryPage queryPage;
if (this.distinctQueryType == DistinctQueryType.Ordered)
{
QueryState state;
if (sourcePage.State != null)
{
string updatedContinuationToken = new DistinctContinuationToken(
sourceToken: sourcePage.State.Value.ToString(),
distinctMapToken: this.distinctMap.GetContinuationToken()).ToString();
state = new QueryState(CosmosElement.Parse(updatedContinuationToken));
}
else
{
state = null;
}
queryPage = new QueryPage(
documents: distinctResults,
requestCharge: sourcePage.RequestCharge,
activityId: sourcePage.ActivityId,
responseLengthInBytes: sourcePage.ResponseLengthInBytes,
cosmosQueryExecutionInfo: sourcePage.CosmosQueryExecutionInfo,
disallowContinuationTokenMessage: sourcePage.DisallowContinuationTokenMessage,
state: state);
}
else
{
queryPage = new QueryPage(
documents: distinctResults,
requestCharge: sourcePage.RequestCharge,
activityId: sourcePage.ActivityId,
responseLengthInBytes: sourcePage.ResponseLengthInBytes,
cosmosQueryExecutionInfo: sourcePage.CosmosQueryExecutionInfo,
disallowContinuationTokenMessage: ClientDistinctQueryPipelineStage.DisallowContinuationTokenMessage,
state: null);
}
this.Current = TryCatch <QueryPage> .FromResult(queryPage);
return(true);
}
public void JoinCollectionExpressionTest()
{
// FROM f
// JOIN c IN f.children
// JOIN p in c.pets
SqlAliasedCollectionExpression fromf = SqlAliasedCollectionExpression.Create(
SqlInputPathCollection.Create(
SqlIdentifier.Create("f"),
relativePath: null),
alias: null);
SqlArrayIteratorCollectionExpression cInFDotChildren = SqlArrayIteratorCollectionExpression.Create(
SqlIdentifier.Create("c"),
SqlInputPathCollection.Create(
SqlIdentifier.Create("f"),
SqlStringPathExpression.Create(
null,
SqlStringLiteral.Create("children"))));
SqlJoinCollectionExpression joinFC = SqlJoinCollectionExpression.Create(
fromf,
cInFDotChildren);
CosmosElement andersenChild1 = CosmosElement.Parse(@"
{
""f"": {
""id"": ""AndersenFamily"",
""lastName"": ""Andersen"",
""parents"": [
{ ""firstName"": ""Thomas"" },
{ ""firstName"": ""Mary Kay""}
],
""children"": [
{
""firstName"": ""Henriette Thaulow"",
""gender"": ""female"",
""grade"": 5,
""pets"": [{ ""givenName"": ""Fluffy"" }]
}
],
""address"": { ""state"": ""WA"", ""county"": ""King"", ""city"": ""seattle"" },
""creationDate"": 1431620472,
""isRegistered"": true,
""_rid"": ""0fomAIxnukU1AQAAAAAAAA==""
},
""c"": {
""firstName"": ""Henriette Thaulow"",
""gender"": ""female"",
""grade"": 5,
""pets"": [{ ""givenName"": ""Fluffy"" }]
}, ""_rid"": ""0fomAIxnukU1AQAAAAAAAA==""
}");
CosmosElement wakeFieldChild1 = CosmosElement.Parse(@"
{
""f"": {
""id"": ""WakefieldFamily"",
""parents"": [
{ ""familyName"": ""Wakefield"", ""givenName"": ""Robin"" },
{ ""familyName"": ""Miller"", ""givenName"": ""Ben"" }
],
""children"": [
{
""familyName"": ""Merriam"",
""givenName"": ""Jesse"",
""gender"": ""female"", ""grade"": 1,
""pets"": [
{ ""givenName"": ""Goofy"" },
{ ""givenName"": ""Shadow"" }
]
},
{
""familyName"": ""Miller"",
""givenName"": ""Lisa"",
""gender"": ""female"",
""grade"": 8 }
],
""address"": { ""state"": ""NY"", ""county"": ""Manhattan"", ""city"": ""NY"" },
""creationDate"": 1431620462,
""isRegistered"": false,
""_rid"": ""0fomAIxnukU1AQAAAAAAAB==""
},
""c"": {
""familyName"": ""Merriam"",
""givenName"": ""Jesse"",
""gender"": ""female"", ""grade"": 1,
""pets"": [
{ ""givenName"": ""Goofy"" },
{ ""givenName"": ""Shadow"" }
]
}, ""_rid"": ""0fomAIxnukU1AQAAAAAAAB==""
}");
CosmosElement wakeFieldChild2 = CosmosElement.Parse(@"
{
""f"": {
""id"": ""WakefieldFamily"",
""parents"": [
{ ""familyName"": ""Wakefield"", ""givenName"": ""Robin"" },
{ ""familyName"": ""Miller"", ""givenName"": ""Ben"" }
],
""children"": [
{
""familyName"": ""Merriam"",
""givenName"": ""Jesse"",
""gender"": ""female"", ""grade"": 1,
""pets"": [
{ ""givenName"": ""Goofy"" },
{ ""givenName"": ""Shadow"" }
]
},
{
""familyName"": ""Miller"",
""givenName"": ""Lisa"",
""gender"": ""female"",
""grade"": 8 }
],
""address"": { ""state"": ""NY"", ""county"": ""Manhattan"", ""city"": ""NY"" },
""creationDate"": 1431620462,
""isRegistered"": false,
""_rid"": ""0fomAIxnukU1AQAAAAAAAB==""
},
""c"": {
""familyName"": ""Miller"",
""givenName"": ""Lisa"",
""gender"": ""female"",
""grade"": 8 }, ""_rid"": ""0fomAIxnukU1AQAAAAAAAB==""
}");
AssertEvaluation(new CosmosElement[] { andersenChild1, wakeFieldChild1, wakeFieldChild2 }, joinFC, DataSource);
SqlArrayIteratorCollectionExpression pInCDotPets = SqlArrayIteratorCollectionExpression.Create(
SqlIdentifier.Create("p"),
SqlInputPathCollection.Create(
SqlIdentifier.Create("c"),
SqlStringPathExpression.Create(
null,
SqlStringLiteral.Create("pets"))));
SqlJoinCollectionExpression joinFCP = SqlJoinCollectionExpression.Create(
SqlJoinCollectionExpression.Create(fromf, cInFDotChildren),
pInCDotPets);
CosmosElement andersenChild1Pet1 = CosmosElement.Parse(@"
{
""f"": {
""id"": ""AndersenFamily"",
""lastName"": ""Andersen"",
""parents"": [
{ ""firstName"": ""Thomas"" },
{ ""firstName"": ""Mary Kay""}
],
""children"": [
{
""firstName"": ""Henriette Thaulow"",
""gender"": ""female"",
""grade"": 5,
""pets"": [{ ""givenName"": ""Fluffy"" }]
}
],
""address"": { ""state"": ""WA"", ""county"": ""King"", ""city"": ""seattle"" },
""creationDate"": 1431620472,
""isRegistered"": true,
""_rid"": ""0fomAIxnukU1AQAAAAAAAA==""
},
""c"": {
""firstName"": ""Henriette Thaulow"",
""gender"": ""female"",
""grade"": 5,
""pets"": [{ ""givenName"": ""Fluffy"" }]
},
""p"": { ""givenName"": ""Fluffy"" },
""_rid"": ""0fomAIxnukU1AQAAAAAAAA==""
}");
CosmosElement wakeFieldChild1Pet1 = CosmosElement.Parse(@"
{
""f"": {
""id"": ""WakefieldFamily"",
""parents"": [
{ ""familyName"": ""Wakefield"", ""givenName"": ""Robin"" },
{ ""familyName"": ""Miller"", ""givenName"": ""Ben"" }
],
""children"": [
{
""familyName"": ""Merriam"",
""givenName"": ""Jesse"",
""gender"": ""female"", ""grade"": 1,
""pets"": [
{ ""givenName"": ""Goofy"" },
{ ""givenName"": ""Shadow"" }
]
},
{
""familyName"": ""Miller"",
""givenName"": ""Lisa"",
""gender"": ""female"",
""grade"": 8 }
],
""address"": { ""state"": ""NY"", ""county"": ""Manhattan"", ""city"": ""NY"" },
""creationDate"": 1431620462,
""isRegistered"": false,
""_rid"": ""0fomAIxnukU1AQAAAAAAAB==""
},
""c"": {
""familyName"": ""Merriam"",
""givenName"": ""Jesse"",
""gender"": ""female"", ""grade"": 1,
""pets"": [
{ ""givenName"": ""Goofy"" },
{ ""givenName"": ""Shadow"" }
]
},
""p"": { ""givenName"": ""Goofy"" },
""_rid"": ""0fomAIxnukU1AQAAAAAAAB==""
}");
CosmosElement wakeFieldChild1Pet2 = CosmosElement.Parse(@"
{
""f"": {
""id"": ""WakefieldFamily"",
""parents"": [
{ ""familyName"": ""Wakefield"", ""givenName"": ""Robin"" },
{ ""familyName"": ""Miller"", ""givenName"": ""Ben"" }
],
""children"": [
{
""familyName"": ""Merriam"",
""givenName"": ""Jesse"",
""gender"": ""female"", ""grade"": 1,
""pets"": [
{ ""givenName"": ""Goofy"" },
{ ""givenName"": ""Shadow"" }
]
},
{
""familyName"": ""Miller"",
""givenName"": ""Lisa"",
""gender"": ""female"",
""grade"": 8 }
],
""address"": { ""state"": ""NY"", ""county"": ""Manhattan"", ""city"": ""NY"" },
""creationDate"": 1431620462,
""isRegistered"": false,
""_rid"": ""0fomAIxnukU1AQAAAAAAAB==""
},
""c"": {
""familyName"": ""Merriam"",
""givenName"": ""Jesse"",
""gender"": ""female"", ""grade"": 1,
""pets"": [
{ ""givenName"": ""Goofy"" },
{ ""givenName"": ""Shadow"" }
]
},
""p"": { ""givenName"": ""Shadow"" },
""_rid"": ""0fomAIxnukU1AQAAAAAAAB==""
}");
AssertEvaluation(new CosmosElement[] { andersenChild1Pet1, wakeFieldChild1Pet1, wakeFieldChild1Pet2 }, joinFCP, DataSource);
// SELECT c.id
// FROM c
// JOIN c.nonExistent
SqlAliasedCollectionExpression c = SqlAliasedCollectionExpression.Create(
SqlInputPathCollection.Create(
SqlIdentifier.Create("c"),
relativePath: null),
alias: null);
SqlAliasedCollectionExpression cDotNonExistent = SqlAliasedCollectionExpression.Create(
SqlInputPathCollection.Create(
SqlIdentifier.Create("c"),
SqlStringPathExpression.Create(
null,
SqlStringLiteral.Create("nonExistent"))),
alias: null);
SqlJoinCollectionExpression joinCAndCDotNonExistent = SqlJoinCollectionExpression.Create(
c,
cDotNonExistent);
AssertEvaluation(new CosmosElement[] { }, joinCAndCDotNonExistent, DataSource);
}
public override CosmosElement GetCosmosElementContinuationToken()
{
return(CosmosElement.Parse(this.FeedRangeContinuation.ToString()));
}
public ReadFeedIteratorCore(
IDocumentContainer documentContainer,
string continuationToken,
ReadFeedPaginationOptions readFeedPaginationOptions,
QueryRequestOptions queryRequestOptions,
CancellationToken cancellationToken)
{
this.queryRequestOptions = queryRequestOptions;
readFeedPaginationOptions ??= ReadFeedPaginationOptions.Default;
if (!string.IsNullOrEmpty(continuationToken))
{
bool isNewArrayFormat = (continuationToken.Length >= 2) && (continuationToken[0] == '[') && (continuationToken[continuationToken.Length - 1] == ']');
if (!isNewArrayFormat)
{
// One of the two older formats
if (!FeedRangeContinuation.TryParse(continuationToken, out FeedRangeContinuation feedRangeContinuation))
{
// Backward compatible with old format
feedRangeContinuation = new FeedRangeCompositeContinuation(
containerRid: string.Empty,
FeedRangeEpk.FullRange,
new List <Documents.Routing.Range <string> >()
{
new Documents.Routing.Range <string>(
Documents.Routing.PartitionKeyInternal.MinimumInclusiveEffectivePartitionKey,
Documents.Routing.PartitionKeyInternal.MaximumExclusiveEffectivePartitionKey,
isMinInclusive: true,
isMaxInclusive: false)
},
continuationToken);
}
// need to massage it a little
List <CosmosElement> feedRangeStates = new List <CosmosElement>();
string oldContinuationFormat = feedRangeContinuation.ToString();
if (feedRangeContinuation.FeedRange is FeedRangePartitionKey feedRangePartitionKey)
{
CosmosObject cosmosObject = CosmosObject.Parse(oldContinuationFormat);
CosmosArray continuations = (CosmosArray)cosmosObject["Continuation"];
if (continuations.Count != 1)
{
throw new InvalidOperationException("Expected only one continuation for partition key queries");
}
CosmosElement continuation = continuations[0];
CosmosObject continuationObject = (CosmosObject)continuation;
CosmosElement token = continuationObject["token"];
ReadFeedState state;
if (token is CosmosNull)
{
state = ReadFeedState.Beginning();
}
else
{
CosmosString tokenAsString = (CosmosString)token;
state = ReadFeedState.Continuation(CosmosElement.Parse(tokenAsString.Value));
}
FeedRangeState <ReadFeedState> feedRangeState = new FeedRangeState <ReadFeedState>(feedRangePartitionKey, state);
feedRangeStates.Add(ReadFeedFeedRangeStateSerializer.ToCosmosElement(feedRangeState));
}
else
{
CosmosObject cosmosObject = CosmosObject.Parse(oldContinuationFormat);
CosmosArray continuations = (CosmosArray)cosmosObject["Continuation"];
foreach (CosmosElement continuation in continuations)
{
CosmosObject continuationObject = (CosmosObject)continuation;
CosmosObject rangeObject = (CosmosObject)continuationObject["range"];
string min = ((CosmosString)rangeObject["min"]).Value;
string max = ((CosmosString)rangeObject["max"]).Value;
CosmosElement token = continuationObject["token"];
FeedRangeInternal feedRange = new FeedRangeEpk(new Documents.Routing.Range <string>(min, max, isMinInclusive: true, isMaxInclusive: false));
ReadFeedState state;
if (token is CosmosNull)
{
state = ReadFeedState.Beginning();
}
else
{
CosmosString tokenAsString = (CosmosString)token;
state = ReadFeedState.Continuation(CosmosElement.Parse(tokenAsString.Value));
}
FeedRangeState <ReadFeedState> feedRangeState = new FeedRangeState <ReadFeedState>(feedRange, state);
feedRangeStates.Add(ReadFeedFeedRangeStateSerializer.ToCosmosElement(feedRangeState));
}
}
CosmosArray cosmosArrayContinuationTokens = CosmosArray.Create(feedRangeStates);
continuationToken = cosmosArrayContinuationTokens.ToString();
}
}
TryCatch <ReadFeedCrossFeedRangeState> monadicReadFeedState;
if (continuationToken == null)
{
FeedRange feedRange;
if ((this.queryRequestOptions != null) && this.queryRequestOptions.PartitionKey.HasValue)
{
feedRange = new FeedRangePartitionKey(this.queryRequestOptions.PartitionKey.Value);
}
else if ((this.queryRequestOptions != null) && (this.queryRequestOptions.FeedRange != null))
{
feedRange = this.queryRequestOptions.FeedRange;
}
else
{
feedRange = FeedRangeEpk.FullRange;
}
monadicReadFeedState = TryCatch <ReadFeedCrossFeedRangeState> .FromResult(ReadFeedCrossFeedRangeState.CreateFromBeginning(feedRange));
}
else
{
monadicReadFeedState = ReadFeedCrossFeedRangeState.Monadic.Parse(continuationToken);
}
if (monadicReadFeedState.Failed)
{
this.monadicEnumerator = TryCatch <CrossPartitionReadFeedAsyncEnumerator> .FromException(monadicReadFeedState.Exception);
}
else
{
this.monadicEnumerator = TryCatch <CrossPartitionReadFeedAsyncEnumerator> .FromResult(
CrossPartitionReadFeedAsyncEnumerator.Create(
documentContainer,
new CrossFeedRangeState <ReadFeedState>(monadicReadFeedState.Result.FeedRangeStates),
readFeedPaginationOptions,
cancellationToken));
}
this.hasMoreResults = true;
}