public async Task Tracing()
{
List <CosmosObject> documents = new List <CosmosObject>();
for (int i = 0; i < 250; i++)
{
documents.Add(CosmosObject.Parse($"{{\"pk\" : {i} }}"));
}
IDocumentContainer documentContainer = await CreateDocumentContainerAsync(documents);
IQueryPipelineStage pipelineStage = CreatePipeline(documentContainer, "SELECT * FROM c", pageSize: 10);
Trace rootTrace;
int numTraces = 1;
using (rootTrace = Trace.GetRootTrace("Cross Partition Query"))
{
while (await pipelineStage.MoveNextAsync(rootTrace))
{
TryCatch <QueryPage> tryGetQueryPage = pipelineStage.Current;
tryGetQueryPage.ThrowIfFailed();
numTraces++;
}
}
string traceString = TraceWriter.TraceToText(rootTrace);
Console.WriteLine(traceString);
Assert.AreEqual(numTraces, rootTrace.Children.Count);
}
public async Task Tracing()
{
List <CosmosObject> documents = new List <CosmosObject>();
for (int i = 0; i < 250; i++)
{
documents.Add(CosmosObject.Parse($"{{\"pk\" : {i} }}"));
}
IDocumentContainer documentContainer = await CreateDocumentContainerAsync(documents);
IQueryPipelineStage pipelineStage = await CreatePipelineAsync(documentContainer, "SELECT * FROM c", pageSize : 10);
Trace rootTrace;
int numTraces = (await documentContainer.GetFeedRangesAsync(NoOpTrace.Singleton, default)).Count;
using (rootTrace = Trace.GetRootTrace("Cross Partition Query"))
{
while (await pipelineStage.MoveNextAsync(rootTrace))
{
TryCatch <QueryPage> tryGetQueryPage = pipelineStage.Current;
tryGetQueryPage.ThrowIfFailed();
numTraces++;
}
}
Assert.AreEqual(numTraces, rootTrace.Children.Count);
}
public async Task StartFromTimeAsync(bool useContinuations)
{
int numItems = 100;
IDocumentContainer documentContainer = await CreateDocumentContainerAsync(numItems);
CrossPartitionChangeFeedAsyncEnumerator enumerator = CrossPartitionChangeFeedAsyncEnumerator.Create(
documentContainer,
new CrossFeedRangeState<ChangeFeedState>(
new FeedRangeState<ChangeFeedState>[]
{
new FeedRangeState<ChangeFeedState>(FeedRangeEpk.FullRange, ChangeFeedState.Time(DateTime.UtcNow))
}),
ChangeFeedPaginationOptions.Default,
cancellationToken: default);
for (int i = 0; i < numItems; i++)
{
// Insert an item
CosmosObject item = CosmosObject.Parse($"{{\"pk\" : {i} }}");
while (true)
{
TryCatch<Record> monadicCreateRecord = await documentContainer.MonadicCreateItemAsync(item, cancellationToken: default);
if (monadicCreateRecord.Succeeded)
{
break;
}
}
}
(int globalCount, double _) = await (useContinuations
? DrainWithUntilNotModifiedWithContinuationTokens(documentContainer, enumerator)
: DrainUntilNotModifedAsync(enumerator));
Assert.AreEqual(numItems, globalCount);
}
public async Task StartFromTimeAsync(bool useContinuations)
{
int numItems = 100;
IDocumentContainer documentContainer = await CreateDocumentContainerAsync(numItems);
TryCatch <CrossPartitionChangeFeedAsyncEnumerator> monadicEnumerator = CrossPartitionChangeFeedAsyncEnumerator.MonadicCreate(
documentContainer,
new ChangeFeedRequestOptions(),
ChangeFeedStartFrom.Time(DateTime.UtcNow),
cancellationToken: default);
for (int i = 0; i < numItems; i++)
{
// Insert an item
CosmosObject item = CosmosObject.Parse($"{{\"pk\" : {i} }}");
while (true)
{
TryCatch <Record> monadicCreateRecord = await documentContainer.MonadicCreateItemAsync(item, cancellationToken : default);
if (monadicCreateRecord.Succeeded)
{
break;
}
}
}
Assert.IsTrue(monadicEnumerator.Succeeded);
CrossPartitionChangeFeedAsyncEnumerator enumerator = monadicEnumerator.Result;
int globalCount = await(useContinuations
? DrainWithUntilNotModifiedWithContinuationTokens(documentContainer, enumerator)
: DrainUntilNotModifedAsync(enumerator));
Assert.AreEqual(numItems, globalCount);
}
private static int ParseQueryState(QueryState state)
{
if (state == default)
{
return(0);
}
CosmosObject parsedContinuationToken = CosmosObject.Parse(((CosmosString)state.Value).Value);
int continuationCount = (int)Number64.ToLong(((CosmosNumber64)parsedContinuationToken["continuationCount"]).Value);
return(continuationCount);
}
public async Task OrderBy()
{
List <CosmosObject> documents = new List <CosmosObject>();
for (int i = 0; i < 250; i++)
{
documents.Add(CosmosObject.Parse($"{{\"pk\" : {i} }}"));
}
List <CosmosElement> documentsQueried = await ExecuteQueryAsync(
query : "SELECT * FROM c ORDER BY c._ts",
documents : documents);
Assert.AreEqual(expected: documents.Count, actual: documentsQueried.Count);
}
// Need to implement group by continuation token on the in memory collection.
public async Task GroupBy()
{
List <CosmosObject> documents = new List <CosmosObject>();
for (int i = 0; i < 250; i++)
{
documents.Add(CosmosObject.Parse($"{{\"pk\" : {i} }}"));
}
List <CosmosElement> documentsQueried = await ExecuteQueryAsync(
query : "SELECT VALUE COUNT(1) FROM c GROUP BY c.pk",
documents : documents);
Assert.AreEqual(expected: documents.Count, actual: documentsQueried.Count);
}
public async Task OffsetLimit()
{
List <CosmosObject> documents = new List <CosmosObject>();
for (int i = 0; i < 250; i++)
{
documents.Add(CosmosObject.Parse($"{{\"pk\" : {i} }}"));
}
List <CosmosElement> documentsQueried = await ExecuteQueryAsync(
query : "SELECT * FROM c OFFSET 10 LIMIT 103",
documents : documents);
Assert.AreEqual(expected: 103, actual: documentsQueried.Count);
}
[Ignore] // Continuation token for in memory container needs to be updated to suppport this query
public async Task OrderByWithJoins()
{
List <CosmosObject> documents = new List <CosmosObject>()
{
CosmosObject.Parse($"{{\"pk\" : {1}, \"children\" : [\"Alice\", \"Bob\", \"Charlie\"]}}"),
CosmosObject.Parse($"{{\"pk\" : {2}, \"children\" : [\"Dave\", \"Eve\", \"Fancy\"]}}"),
CosmosObject.Parse($"{{\"pk\" : {3}, \"children\" : [\"George\", \"Henry\", \"Igor\"]}}"),
CosmosObject.Parse($"{{\"pk\" : {4}, \"children\" : [\"Jack\", \"Kim\", \"Levin\"]}}"),
};
List <CosmosElement> documentsQueried = await ExecuteQueryAsync(
query : "SELECT d FROM c JOIN d in c.children ORDER BY c.pk",
documents : documents,
pageSize : 2);
Assert.AreEqual(expected: documents.Count * 3, actual: documentsQueried.Count);
}
public async Task ChangeFeedIteratorCore_BreathFirst()
{
int expected = 500;
CosmosObject previousToken = null;
ContainerInternal itemsCore = await this.InitializeLargeContainerAsync();
await this.CreateRandomItems(itemsCore, expected, randomPartitionKey : true);
ChangeFeedIteratorCore feedIterator = itemsCore.GetChangeFeedStreamIterator(
ChangeFeedStartFrom.Beginning(),
ChangeFeedMode.Incremental,
new ChangeFeedRequestOptions()
{
PageSizeHint = 1,
}) as ChangeFeedIteratorCore;
while (true)
{
using (ResponseMessage responseMessage = await feedIterator.ReadNextAsync(this.cancellationToken))
{
CosmosObject cosmosObject = CosmosObject.Parse(responseMessage.ContinuationToken);
if (!cosmosObject.TryGetValue("Continuation", out CosmosArray cosmosArray))
{
Assert.Fail();
throw new Exception();
}
CosmosObject currentToken = (CosmosObject)cosmosArray[0];
if (previousToken != null)
{
// Verify that the token, even though it yielded results, it moved to a new range
Assert.AreNotEqual(previousToken, currentToken);
break;
}
previousToken = currentToken;
if (responseMessage.StatusCode == HttpStatusCode.NotModified)
{
break;
}
}
}
}
private static IReadOnlyList <(string serializedToken, CosmosElement element)> TokenTestData()
{
Guid guid = Guid.Parse("69D5AB17-C94A-4173-A278-B59D0D9C7C37");
byte[] randomBytes = guid.ToByteArray();
string hexString = PartitionKeyInternal.HexConvert.ToHex(randomBytes, 0, randomBytes.Length);
return(new List <(string, CosmosElement)>
{
("[42, 37]", CosmosArray.Parse("[42, 37]")),
($@"{{C_Binary(""0x{hexString}"")}}", CosmosBinary.Create(new ReadOnlyMemory <byte>(randomBytes))),
("false", CosmosBoolean.Create(false)),
($@"{{C_Guid(""{guid}"")}}", CosmosGuid.Create(guid)),
("null", CosmosNull.Create()),
("1", CosmosInt64.Create(1)),
("{\"foo\": false}", CosmosObject.Parse("{\"foo\": false}")),
("asdf", CosmosString.Create("asdf"))
});
public async Task DCount()
{
List <CosmosObject> documents = new List <CosmosObject>();
for (int i = 0; i < 250; i++)
{
documents.Add(CosmosObject.Parse($"{{\"pk\" : {i}, \"val\": {i % 50} }}"));
}
List <CosmosElement> documentsQueried = await ExecuteQueryAsync(
query : "SELECT VALUE COUNT(1) FROM (SELECT DISTINCT VALUE c.val FROM c)",
documents : documents);
Assert.AreEqual(expected: 1, actual: documentsQueried.Count);
Assert.IsTrue(documentsQueried[0] is CosmosNumber);
CosmosNumber result = documentsQueried[0] as CosmosNumber;
Assert.AreEqual(expected: 50, actual: result);
}
public async Task TestMerge()
{
List <CosmosObject> documents = Enumerable
.Range(0, 100)
.Select(x => CosmosObject.Parse($"{{\"pk\" : {x} }}"))
.ToList();
MergeTestUtil mergeTest = new MergeTestUtil();
mergeTest.DocumentContainer = await CreateDocumentContainerAsync(
documents : documents,
numPartitions : 2,
failureConfigs : new FlakyDocumentContainer.FailureConfigs(
inject429s: false,
injectEmptyPages: false,
shouldReturnFailure: mergeTest.ShouldReturnFailure));
string query = "SELECT * FROM c ORDER BY c._ts";
int pageSize = 10;
IQueryPipelineStage pipelineStage = CreatePipeline(mergeTest.DocumentContainer, query, pageSize);
List <CosmosElement> elements = new List <CosmosElement>();
int iteration = 0;
while (await pipelineStage.MoveNextAsync())
{
TryCatch <QueryPage> tryGetQueryPage = pipelineStage.Current;
tryGetQueryPage.ThrowIfFailed();
elements.AddRange(tryGetQueryPage.Result.Documents);
++iteration;
if (iteration == 1)
{
mergeTest.ShouldMerge = MergeTestUtil.TriState.Ready;
}
}
Assert.AreEqual(expected: documents.Count, actual: elements.Count);
}
private ResponseMessage CreateQueryResponse()
{
List <CosmosElement> cosmosElements = new List <CosmosElement>();
string serializedItem = this.GetSerializedToDoActivity();
CosmosObject cosmosObject = CosmosObject.Parse(serializedItem);
cosmosElements.Add(cosmosObject);
ResponseMessage cosmosResponse = QueryResponse.CreateSuccess(
cosmosElements,
1,
Encoding.UTF8.GetByteCount(serializedItem),
new CosmosQueryResponseMessageHeaders(
continauationToken: null,
disallowContinuationTokenMessage: null,
resourceType: Documents.ResourceType.Document,
"+o4fAPfXPzw="),
new CosmosDiagnosticsContextCore(),
null);
return(cosmosResponse);
}
internal override IDocumentContainer CreateDocumentContainer(
PartitionKeyDefinition partitionKeyDefinition,
int numItems = 0,
FlakyDocumentContainer.FailureConfigs failureConfigs = null)
{
IMonadicDocumentContainer monadicDocumentContainer = new InMemoryContainer(partitionKeyDefinition);
if (failureConfigs != null)
{
monadicDocumentContainer = new FlakyDocumentContainer(monadicDocumentContainer, failureConfigs);
}
DocumentContainer documentContainer = new DocumentContainer(monadicDocumentContainer);
for (int i = 0; i < numItems; i++)
{
// Insert an item
CosmosObject item = CosmosObject.Parse($"{{\"pk\" : {i} }}");
documentContainer.CreateItemAsync(item, cancellationToken: default).Wait();
}
return(documentContainer);
}
public async Task Aggregates()
{
const int DocumentCount = 250;
List <CosmosObject> documents = new List <CosmosObject>();
for (int i = 0; i < DocumentCount; i++)
{
documents.Add(CosmosObject.Parse($"{{\"pk\" : {i} }}"));
}
List <CosmosElement> documentsQueried = await ExecuteQueryAsync(
query : "SELECT VALUE COUNT(1) FROM c",
documents : documents);
Assert.AreEqual(expected: 1, actual: documentsQueried.Count);
if (documentsQueried[0] is CosmosNumber number)
{
Assert.AreEqual(expected: DocumentCount, actual: Number64.ToLong(number.Value));
}
else
{
Assert.Fail();
}
}
public ReadFeedIteratorCore(
IDocumentContainer documentContainer,
string continuationToken,
ReadFeedPaginationOptions readFeedPaginationOptions,
QueryRequestOptions queryRequestOptions,
CancellationToken cancellationToken)
{
this.queryRequestOptions = queryRequestOptions;
readFeedPaginationOptions ??= ReadFeedPaginationOptions.Default;
if (!string.IsNullOrEmpty(continuationToken))
{
bool isNewArrayFormat = (continuationToken.Length >= 2) && (continuationToken[0] == '[') && (continuationToken[continuationToken.Length - 1] == ']');
if (!isNewArrayFormat)
{
// One of the two older formats
if (!FeedRangeContinuation.TryParse(continuationToken, out FeedRangeContinuation feedRangeContinuation))
{
// Backward compatible with old format
feedRangeContinuation = new FeedRangeCompositeContinuation(
containerRid: string.Empty,
FeedRangeEpk.FullRange,
new List <Documents.Routing.Range <string> >()
{
new Documents.Routing.Range <string>(
Documents.Routing.PartitionKeyInternal.MinimumInclusiveEffectivePartitionKey,
Documents.Routing.PartitionKeyInternal.MaximumExclusiveEffectivePartitionKey,
isMinInclusive: true,
isMaxInclusive: false)
},
continuationToken);
}
// need to massage it a little
List <CosmosElement> feedRangeStates = new List <CosmosElement>();
string oldContinuationFormat = feedRangeContinuation.ToString();
if (feedRangeContinuation.FeedRange is FeedRangePartitionKey feedRangePartitionKey)
{
CosmosObject cosmosObject = CosmosObject.Parse(oldContinuationFormat);
CosmosArray continuations = (CosmosArray)cosmosObject["Continuation"];
if (continuations.Count != 1)
{
throw new InvalidOperationException("Expected only one continuation for partition key queries");
}
CosmosElement continuation = continuations[0];
CosmosObject continuationObject = (CosmosObject)continuation;
CosmosElement token = continuationObject["token"];
ReadFeedState state;
if (token is CosmosNull)
{
state = ReadFeedState.Beginning();
}
else
{
CosmosString tokenAsString = (CosmosString)token;
state = ReadFeedState.Continuation(CosmosElement.Parse(tokenAsString.Value));
}
FeedRangeState <ReadFeedState> feedRangeState = new FeedRangeState <ReadFeedState>(feedRangePartitionKey, state);
feedRangeStates.Add(ReadFeedFeedRangeStateSerializer.ToCosmosElement(feedRangeState));
}
else
{
CosmosObject cosmosObject = CosmosObject.Parse(oldContinuationFormat);
CosmosArray continuations = (CosmosArray)cosmosObject["Continuation"];
foreach (CosmosElement continuation in continuations)
{
CosmosObject continuationObject = (CosmosObject)continuation;
CosmosObject rangeObject = (CosmosObject)continuationObject["range"];
string min = ((CosmosString)rangeObject["min"]).Value;
string max = ((CosmosString)rangeObject["max"]).Value;
CosmosElement token = continuationObject["token"];
FeedRangeInternal feedRange = new FeedRangeEpk(new Documents.Routing.Range <string>(min, max, isMinInclusive: true, isMaxInclusive: false));
ReadFeedState state;
if (token is CosmosNull)
{
state = ReadFeedState.Beginning();
}
else
{
CosmosString tokenAsString = (CosmosString)token;
state = ReadFeedState.Continuation(CosmosElement.Parse(tokenAsString.Value));
}
FeedRangeState <ReadFeedState> feedRangeState = new FeedRangeState <ReadFeedState>(feedRange, state);
feedRangeStates.Add(ReadFeedFeedRangeStateSerializer.ToCosmosElement(feedRangeState));
}
}
CosmosArray cosmosArrayContinuationTokens = CosmosArray.Create(feedRangeStates);
continuationToken = cosmosArrayContinuationTokens.ToString();
}
}
TryCatch <ReadFeedCrossFeedRangeState> monadicReadFeedState;
if (continuationToken == null)
{
FeedRange feedRange;
if ((this.queryRequestOptions != null) && this.queryRequestOptions.PartitionKey.HasValue)
{
feedRange = new FeedRangePartitionKey(this.queryRequestOptions.PartitionKey.Value);
}
else if ((this.queryRequestOptions != null) && (this.queryRequestOptions.FeedRange != null))
{
feedRange = this.queryRequestOptions.FeedRange;
}
else
{
feedRange = FeedRangeEpk.FullRange;
}
monadicReadFeedState = TryCatch <ReadFeedCrossFeedRangeState> .FromResult(ReadFeedCrossFeedRangeState.CreateFromBeginning(feedRange));
}
else
{
monadicReadFeedState = ReadFeedCrossFeedRangeState.Monadic.Parse(continuationToken);
}
if (monadicReadFeedState.Failed)
{
this.monadicEnumerator = TryCatch <CrossPartitionReadFeedAsyncEnumerator> .FromException(monadicReadFeedState.Exception);
}
else
{
this.monadicEnumerator = TryCatch <CrossPartitionReadFeedAsyncEnumerator> .FromResult(
CrossPartitionReadFeedAsyncEnumerator.Create(
documentContainer,
new CrossFeedRangeState <ReadFeedState>(monadicReadFeedState.Result.FeedRangeStates),
readFeedPaginationOptions,
cancellationToken));
}
this.hasMoreResults = true;
}
private async Task <(Container, IReadOnlyList <CosmosObject>)> CreateContainerAndIngestDocumentsAsync(
CollectionTypes collectionType,
IEnumerable <string> documents,
string partitionKey = "/id",
Cosmos.IndexingPolicy indexingPolicy = null)
{
Container container = collectionType switch
{
CollectionTypes.NonPartitioned => await this.CreateNonPartitionedContainerAsync(indexingPolicy),
CollectionTypes.SinglePartition => await this.CreateSinglePartitionContainer(partitionKey, indexingPolicy),
CollectionTypes.MultiPartition => await this.CreateMultiPartitionContainer(partitionKey, indexingPolicy),
_ => throw new ArgumentException($"Unknown {nameof(CollectionTypes)} : {collectionType}"),
};
List <CosmosObject> insertedDocuments = new List <CosmosObject>();
foreach (string document in documents)
{
JObject documentObject = JsonConvert.DeserializeObject <JObject>(document);
// Add an id
if (documentObject["id"] == null)
{
documentObject["id"] = Guid.NewGuid().ToString();
}
// Get partition key value.
Cosmos.PartitionKey pkValue;
if (partitionKey != null)
{
string jObjectPartitionKey = partitionKey.Remove(0, 1);
JValue pkToken = (JValue)documentObject[jObjectPartitionKey];
if (pkToken == null)
{
pkValue = Cosmos.PartitionKey.None;
}
else
{
switch (pkToken.Type)
{
case JTokenType.Integer:
case JTokenType.Float:
pkValue = new Cosmos.PartitionKey(pkToken.Value <double>());
break;
case JTokenType.String:
pkValue = new Cosmos.PartitionKey(pkToken.Value <string>());
break;
case JTokenType.Boolean:
pkValue = new Cosmos.PartitionKey(pkToken.Value <bool>());
break;
case JTokenType.Null:
pkValue = Cosmos.PartitionKey.Null;
break;
default:
throw new ArgumentException("Unknown partition key type");
}
}
}
else
{
pkValue = Cosmos.PartitionKey.None;
}
JObject createdDocument = await container.CreateItemAsync(documentObject, pkValue);
CosmosObject insertedDocument = CosmosObject.Parse <CosmosObject>(createdDocument.ToString());
insertedDocuments.Add(insertedDocument);
}
return(container, insertedDocuments);
}
public ReadFeedIteratorCore(
IDocumentContainer documentContainer,
QueryRequestOptions queryRequestOptions,
string continuationToken,
int pageSize,
CancellationToken cancellationToken)
{
if (!string.IsNullOrEmpty(continuationToken))
{
bool isNewArrayFormat = (continuationToken.Length >= 2) && (continuationToken[0] == '[') && (continuationToken[continuationToken.Length - 1] == ']');
if (!isNewArrayFormat)
{
// One of the two older formats
if (!FeedRangeContinuation.TryParse(continuationToken, out FeedRangeContinuation feedRangeContinuation))
{
// Backward compatible with old format
feedRangeContinuation = new FeedRangeCompositeContinuation(
containerRid: string.Empty,
FeedRangeEpk.FullRange,
new List <Documents.Routing.Range <string> >()
{
new Documents.Routing.Range <string>(
Documents.Routing.PartitionKeyInternal.MinimumInclusiveEffectivePartitionKey,
Documents.Routing.PartitionKeyInternal.MaximumExclusiveEffectivePartitionKey,
isMinInclusive: true,
isMaxInclusive: false)
},
continuationToken);
}
// need to massage it a little
string oldContinuationFormat = feedRangeContinuation.ToString();
CosmosObject cosmosObject = CosmosObject.Parse(oldContinuationFormat);
CosmosArray continuations = (CosmosArray)cosmosObject["Continuation"];
List <CosmosElement> readFeedContinuationTokens = new List <CosmosElement>();
foreach (CosmosElement continuation in continuations)
{
CosmosObject continuationObject = (CosmosObject)continuation;
CosmosObject rangeObject = (CosmosObject)continuationObject["range"];
string min = ((CosmosString)rangeObject["min"]).Value;
string max = ((CosmosString)rangeObject["max"]).Value;
CosmosElement token = CosmosElement.Parse(((CosmosString)continuationObject["token"]).Value);
FeedRangeInternal feedRange = new FeedRangeEpk(new Documents.Routing.Range <string>(min, max, isMinInclusive: true, isMaxInclusive: false));
ReadFeedState state = new ReadFeedState(token);
ReadFeedContinuationToken readFeedContinuationToken = new ReadFeedContinuationToken(feedRange, state);
readFeedContinuationTokens.Add(ReadFeedContinuationToken.ToCosmosElement(readFeedContinuationToken));
}
CosmosArray cosmosArrayContinuationTokens = CosmosArray.Create(readFeedContinuationTokens);
continuationToken = cosmosArrayContinuationTokens.ToString();
}
}
this.monadicEnumerator = CrossPartitionReadFeedAsyncEnumerator.MonadicCreate(
documentContainer,
queryRequestOptions,
continuationToken: continuationToken,
pageSize,
cancellationToken);
this.hasMoreResults = true;
}