public async Task GetFeedRangesAsync_MatchesPkRanges()
{
int pkRangesCount = (await this.LargerContainer.ClientContext.DocumentClient.ReadPartitionKeyRangeFeedAsync(this.LargerContainer.LinkUri)).Count;
ContainerInternal itemsCore = this.LargerContainer;
IEnumerable <FeedRange> tokens = await itemsCore.GetFeedRangesAsync();
Assert.AreEqual(pkRangesCount, tokens.Count());
}
public async Task GetFeedRangesAsync_AllowsParallelProcessing()
{
ContainerInternal itemsCore = await this.InitializeLargeContainerAsync();
int pkRangesCount = (await itemsCore.ClientContext.DocumentClient.ReadPartitionKeyRangeFeedAsync(itemsCore.LinkUri)).Count;
IEnumerable <FeedRange> tokens = await itemsCore.GetFeedRangesAsync();
Assert.IsTrue(pkRangesCount > 1, "Should have created a multi partition container.");
Assert.AreEqual(pkRangesCount, tokens.Count());
int totalDocuments = 200;
await this.CreateRandomItems(itemsCore, totalDocuments, randomPartitionKey : true);
List <Task <int> > tasks = tokens.Select(token => Task.Run(async() =>
{
int count = 0;
ChangeFeedIteratorCore iteratorForToken =
itemsCore.GetChangeFeedStreamIterator(
ChangeFeedStartFrom.Beginning(token),
ChangeFeedMode.Incremental) as ChangeFeedIteratorCore;
while (true)
{
using (ResponseMessage responseMessage =
await iteratorForToken.ReadNextAsync(this.cancellationToken))
{
if (!responseMessage.IsSuccessStatusCode)
{
break;
}
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
count += response.Count;
}
}
return(count);
})).ToList();
await Task.WhenAll(tasks);
int documentsRead = 0;
foreach (Task <int> task in tasks)
{
documentsRead += task.Result;
}
Assert.AreEqual(totalDocuments, documentsRead);
}
public async Task GetTargetPartitionKeyRangesAsyncWithFeedRange()
{
ContainerInternal container = null;
try
{
// Create a container large enough to have at least 2 partitions
ContainerResponse containerResponse = await this.database.CreateContainerAsync(
id : Guid.NewGuid().ToString(),
partitionKeyPath : "/pk",
throughput : 15000);
container = (ContainerInlineCore)containerResponse;
// Get all the partition key ranges to verify there is more than one partition
IRoutingMapProvider routingMapProvider = await this.cosmosClient.DocumentClient.GetPartitionKeyRangeCacheAsync(NoOpTrace.Singleton);
ContainerQueryProperties containerQueryProperties = new ContainerQueryProperties(
containerResponse.Resource.ResourceId,
null,
containerResponse.Resource.PartitionKey);
IReadOnlyList <FeedRange> feedTokens = await container.GetFeedRangesAsync();
Assert.IsTrue(feedTokens.Count > 1, " RUs of the container needs to be increased to ensure at least 2 partitions.");
// There should only be one range since we get 1 FeedRange per range
foreach (FeedRange feedToken in feedTokens)
{
List <PartitionKeyRange> partitionKeyRanges = await CosmosQueryExecutionContextFactory.GetTargetPartitionKeyRangesAsync(
queryClient : new CosmosQueryClientCore(container.ClientContext, container),
resourceLink : container.LinkUri,
partitionedQueryExecutionInfo : null,
containerQueryProperties : containerQueryProperties,
properties : null,
feedRangeInternal : feedToken as FeedRangeInternal,
NoOpTrace.Singleton);
Assert.IsTrue(partitionKeyRanges.Count == 1, "Only 1 partition key range should be selected since the FeedRange represents a single range.");
}
}
finally
{
await container?.DeleteContainerAsync();
}
}
public async Task ReadFeedIteratorCore_AllowsParallelProcessing()
{
int batchSize = 1000;
await this.CreateRandomItems(this.LargerContainer, batchSize, randomPartitionKey : true);
ContainerInternal itemsCore = this.LargerContainer;
IReadOnlyList <FeedRange> tokens = await itemsCore.GetFeedRangesAsync();
List <Task <int> > tasks = tokens.Select(token => Task.Run(async() =>
{
int count = 0;
FeedIterator feedIterator = itemsCore.GetItemQueryStreamIterator(
queryDefinition: null,
feedRange: token,
continuationToken: null,
requestOptions: null);
while (feedIterator.HasMoreResults)
{
using (ResponseMessage responseMessage =
await feedIterator.ReadNextAsync(this.cancellationToken))
{
if (responseMessage.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
count += response.Count;
}
}
}
return(count);
})).ToList();
await Task.WhenAll(tasks);
int documentsRead = 0;
foreach (Task <int> task in tasks)
{
documentsRead += task.Result;
}
Assert.AreEqual(batchSize, documentsRead);
}
public async Task ReadFeedIteratorCore_OfT_WithFeedRange_ReadAll_StopResume()
{
int batchSize = 1000;
await this.CreateRandomItems(this.LargerContainer, batchSize, randomPartitionKey : true);
ContainerInternal itemsCore = this.LargerContainer;
IReadOnlyList <FeedRange> tokens = await itemsCore.GetFeedRangesAsync();
List <Task <int> > tasks = tokens.Select(token => Task.Run(async() =>
{
int count = 0;
FeedIterator <ToDoActivity> feedIterator = itemsCore.GetItemQueryIterator <ToDoActivity>(queryDefinition: null, feedRange: token);
string continuation = null;
while (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> response = await feedIterator.ReadNextAsync(this.cancellationToken);
count += response.Count;
continuation = response.ContinuationToken;
break;
}
feedIterator = itemsCore.GetItemQueryIterator <ToDoActivity>(queryDefinition: null, continuationToken: continuation);
while (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> response = await feedIterator.ReadNextAsync(this.cancellationToken);
count += response.Count;
}
return(count);
})).ToList();
await Task.WhenAll(tasks);
int documentsRead = 0;
foreach (Task <int> task in tasks)
{
documentsRead += task.Result;
}
Assert.AreEqual(batchSize, documentsRead);
}
public async Task ChangeFeedIteratorCore_CannotMixTokensFromOtherContainers()
{
ContainerInternal oneContainer = await this.InitializeContainerAsync();
ContainerInternal otherContainer = await this.InitializeContainerAsync();
IReadOnlyList <FeedRange> tokens = await oneContainer.GetFeedRangesAsync();
FeedIterator iterator = oneContainer.GetChangeFeedStreamIterator(
ChangeFeedStartFrom.Beginning(tokens[0]),
ChangeFeedMode.Incremental);
ResponseMessage responseMessage = await iterator.ReadNextAsync();
iterator = otherContainer.GetChangeFeedStreamIterator(
ChangeFeedStartFrom.ContinuationToken(responseMessage.ContinuationToken),
ChangeFeedMode.Incremental);
responseMessage = await iterator.ReadNextAsync();
Assert.IsNotNull(responseMessage.CosmosException);
Assert.AreEqual(HttpStatusCode.BadRequest, responseMessage.StatusCode);
}
public async Task ParallelizeQueryThroughTokens_OfT()
{
ContainerInternal container = null;
try
{
// Create a container large enough to have at least 2 partitions
ContainerResponse containerResponse = await this.database.CreateContainerAsync(
id : Guid.NewGuid().ToString(),
partitionKeyPath : "/id",
throughput : 15000);
container = (ContainerInlineCore)containerResponse;
List <string> generatedIds = Enumerable.Range(0, 1000).Select(n => $"BasicItem{n}").ToList();
foreach (string id in generatedIds)
{
string item = $@"
{{
""id"": ""{id}""
}}";
using (ResponseMessage createResponse = await container.CreateItemStreamAsync(
QueryFeedRangeTests.GenerateStreamFromString(item),
new Cosmos.PartitionKey(id)))
{
Assert.IsTrue(createResponse.IsSuccessStatusCode);
}
}
IReadOnlyList <FeedRange> feedTokens = await container.GetFeedRangesAsync();
Assert.IsTrue(feedTokens.Count > 1, " RUs of the container needs to be increased to ensure at least 2 partitions.");
List <Task <List <string> > > tasks = feedTokens.Select(async feedToken =>
{
List <string> results = new List <string>();
FeedIterator <ToDoActivity> feedIterator = container.GetItemQueryIterator <ToDoActivity>(queryDefinition: new QueryDefinition("select * from T where STARTSWITH(T.id, \"BasicItem\")"), feedRange: feedToken, requestOptions: new QueryRequestOptions()
{
MaxItemCount = 10
});
string continuation = null;
while (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> response = await feedIterator.ReadNextAsync();
foreach (ToDoActivity toDoActivity in response)
{
results.Add(toDoActivity.id);
}
continuation = response.ContinuationToken;
break;
}
feedIterator = container.GetItemQueryIterator <ToDoActivity>(queryDefinition: new QueryDefinition("select * from T where STARTSWITH(T.id, \"BasicItem\")"), feedRange: feedToken, continuationToken: continuation, requestOptions: new QueryRequestOptions()
{
MaxItemCount = 10
});
while (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> response = await feedIterator.ReadNextAsync();
foreach (ToDoActivity toDoActivity in response)
{
results.Add(toDoActivity.id);
}
}
return(results);
}).ToList();
await Task.WhenAll(tasks);
CollectionAssert.AreEquivalent(generatedIds, tasks.SelectMany(t => t.Result).ToList());
}
finally
{
await container?.DeleteContainerAsync();
}
}
public async Task ChangeFeed_FeedRange_FromV0Token()
{
ContainerResponse largerContainer = await this.database.CreateContainerAsync(
new ContainerProperties(id : Guid.NewGuid().ToString(), partitionKeyPath : "/pk"),
throughput : 20000,
cancellationToken : this.cancellationToken);
ContainerInternal container = (ContainerInlineCore)largerContainer;
int expected = 100;
int count = 0;
await this.CreateRandomItems((ContainerCore)container, expected, randomPartitionKey : true);
IReadOnlyList <FeedRange> feedRanges = await container.GetFeedRangesAsync();
List <string> continuations = new List <string>();
// First do one request to construct the old model information based on Etag
foreach (FeedRange feedRange in feedRanges)
{
IEnumerable <string> pkRangeIds = await container.GetPartitionKeyRangesAsync(feedRange);
ChangeFeedRequestOptions requestOptions = new ChangeFeedRequestOptions()
{
PageSizeHint = 1
};
ChangeFeedIteratorCore feedIterator = container.GetChangeFeedStreamIterator(
changeFeedStartFrom: ChangeFeedStartFrom.Beginning(feedRange),
changeFeedMode: ChangeFeedMode.Incremental,
changeFeedRequestOptions: requestOptions) as ChangeFeedIteratorCore;
ResponseMessage firstResponse = await feedIterator.ReadNextAsync();
// Construct the continuation's range, using PKRangeId + ETag
List <dynamic> ct = new List <dynamic>()
{
new
{
FeedRange = new
{
type = "Physical Partition Key Range Id",
value = pkRangeIds.First()
},
State = new
{
type = "continuation",
value = JObject.Parse(firstResponse.ContinuationToken)["Continuation"][0]["State"]["value"].ToString()
}
}
};
if (firstResponse.Content != null)
{
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(firstResponse.Content).Data;
count += response.Count;
}
// Extract Etag and manually construct the continuation
dynamic oldContinuation = new
{
V = 2,
Rid = await container.GetCachedRIDAsync(cancellationToken : this.cancellationToken),
Continuation = ct
};
continuations.Add(JsonConvert.SerializeObject(oldContinuation));
}
// Now start the new iterators with the constructed continuations from migration
foreach (string continuation in continuations)
{
ChangeFeedRequestOptions requestOptions = new ChangeFeedRequestOptions()
{
PageSizeHint = 100,
};
ChangeFeedIteratorCore feedIterator = container.GetChangeFeedStreamIterator(
changeFeedStartFrom: ChangeFeedStartFrom.ContinuationToken(continuation),
changeFeedMode: ChangeFeedMode.Incremental,
changeFeedRequestOptions: requestOptions) as ChangeFeedIteratorCore;
ResponseMessage firstResponse = await feedIterator.ReadNextAsync();
if (firstResponse.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(firstResponse.Content).Data;
count += response.Count;
string migratedContinuation = firstResponse.ContinuationToken;
TryCatch <CosmosElement> monadicParsedToken = CosmosElement.Monadic.Parse(migratedContinuation);
Assert.IsFalse(monadicParsedToken.Failed);
TryCatch <VersionedAndRidCheckedCompositeToken> monadicVersionedToken = VersionedAndRidCheckedCompositeToken
.MonadicCreateFromCosmosElement(monadicParsedToken.Result);
Assert.IsFalse(monadicVersionedToken.Failed);
VersionedAndRidCheckedCompositeToken versionedAndRidCheckedCompositeToken = monadicVersionedToken.Result;
Assert.AreEqual(VersionedAndRidCheckedCompositeToken.Version.V2, versionedAndRidCheckedCompositeToken.VersionNumber);
}
}
Assert.AreEqual(expected, count);
}
public async Task ChangeFeed_FeedRange_FromV0Token()
{
ContainerResponse largerContainer = await this.database.CreateContainerAsync(
new ContainerProperties(id : Guid.NewGuid().ToString(), partitionKeyPath : "/status"),
throughput : 20000,
cancellationToken : this.cancellationToken);
ContainerInternal container = (ContainerInlineCore)largerContainer;
int expected = 100;
int count = 0;
await this.CreateRandomItems((ContainerCore)container, expected, randomPartitionKey : true);
IReadOnlyList <FeedRange> feedRanges = await container.GetFeedRangesAsync();
List <string> continuations = new List <string>();
// First do one request to construct the old model information based on Etag
foreach (FeedRange feedRange in feedRanges)
{
IEnumerable <string> pkRangeIds = await container.GetPartitionKeyRangesAsync(feedRange);
ChangeFeedRequestOptions requestOptions = new ChangeFeedRequestOptions()
{
PageSizeHint = 1
};
ChangeFeedIteratorCore feedIterator = container.GetChangeFeedStreamIterator(
changeFeedStartFrom: ChangeFeedStartFrom.Beginning(feedRange),
changeFeedRequestOptions: requestOptions) as ChangeFeedIteratorCore;
ResponseMessage firstResponse = await feedIterator.ReadNextAsync();
FeedRangeEpk FeedRangeEpk = feedRange as FeedRangeEpk;
// Construct the continuation's range, using PKRangeId + ETag
List <dynamic> ct = new List <dynamic>()
{
new
{
min = FeedRangeEpk.Range.Min,
max = FeedRangeEpk.Range.Max,
token = (string)null
}
};
// Extract Etag and manually construct the continuation
dynamic oldContinuation = new
{
V = 0,
Rid = await container.GetCachedRIDAsync(cancellationToken : this.cancellationToken),
Continuation = ct
};
continuations.Add(JsonConvert.SerializeObject(oldContinuation));
}
// Now start the new iterators with the constructed continuations from migration
foreach (string continuation in continuations)
{
ChangeFeedRequestOptions requestOptions = new ChangeFeedRequestOptions()
{
PageSizeHint = 100,
EmitOldContinuationToken = true,
};
ChangeFeedIteratorCore feedIterator = container.GetChangeFeedStreamIterator(
changeFeedStartFrom: ChangeFeedStartFrom.ContinuationToken(continuation),
changeFeedRequestOptions: requestOptions) as ChangeFeedIteratorCore;
ResponseMessage firstResponse = await feedIterator.ReadNextAsync();
if (firstResponse.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(firstResponse.Content).Data;
count += response.Count;
string migratedContinuation = firstResponse.ContinuationToken;
Assert.IsTrue(FeedRangeContinuation.TryParse(migratedContinuation, out FeedRangeContinuation feedRangeContinuation));
Assert.IsTrue(feedRangeContinuation.FeedRange is FeedRangeEpk);
}
}
Assert.AreEqual(expected, count);
}