public async Task GetFeedRangesAsync_MatchesPkRanges()
{
int pkRangesCount = (await this.LargerContainer.ClientContext.DocumentClient.ReadPartitionKeyRangeFeedAsync(this.LargerContainer.LinkUri)).Count;
ContainerCore itemsCore = this.LargerContainer;
IEnumerable <FeedRange> tokens = await itemsCore.GetFeedRangesAsync();
Assert.AreEqual(pkRangesCount, tokens.Count());
}
public async Task GetFeedRangesAsync_AllowsParallelProcessing()
{
int pkRangesCount = (await this.LargerContainer.ClientContext.DocumentClient.ReadPartitionKeyRangeFeedAsync(this.LargerContainer.LinkUri)).Count;
ContainerCore itemsCore = this.LargerContainer;
IEnumerable <FeedRange> tokens = await itemsCore.GetFeedRangesAsync();
Assert.IsTrue(pkRangesCount > 1, "Should have created a multi partition container.");
Assert.AreEqual(pkRangesCount, tokens.Count());
int totalDocuments = 200;
await this.CreateRandomItems(this.LargerContainer, totalDocuments, randomPartitionKey : true);
List <Task <int> > tasks = tokens.Select(token => Task.Run(async() =>
{
int count = 0;
ChangeFeedIteratorCore iteratorForToken =
itemsCore.GetChangeFeedStreamIterator(token, changeFeedRequestOptions: new ChangeFeedRequestOptions()
{
StartTime = DateTime.MinValue
}) as ChangeFeedIteratorCore;
while (true)
{
using (ResponseMessage responseMessage =
await iteratorForToken.ReadNextAsync(this.cancellationToken))
{
if (!responseMessage.IsSuccessStatusCode)
{
break;
}
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
count += response.Count;
}
}
return(count);
})).ToList();
await Task.WhenAll(tasks);
int documentsRead = 0;
foreach (Task <int> task in tasks)
{
documentsRead += task.Result;
}
Assert.AreEqual(totalDocuments, documentsRead);
}
public async Task GetTargetPartitionKeyRangesAsyncWithFeedRange()
{
ContainerCore container = null;
try
{
// Create a container large enough to have at least 2 partitions
ContainerResponse containerResponse = await this.database.CreateContainerAsync(
id : Guid.NewGuid().ToString(),
partitionKeyPath : "/pk",
throughput : 15000);
container = (ContainerInlineCore)containerResponse;
// Get all the partition key ranges to verify there is more than one partition
IRoutingMapProvider routingMapProvider = await this.cosmosClient.DocumentClient.GetPartitionKeyRangeCacheAsync();
ContainerQueryProperties containerQueryProperties = new ContainerQueryProperties(
containerResponse.Resource.ResourceId,
null,
containerResponse.Resource.PartitionKey);
IReadOnlyList <FeedRange> feedTokens = await container.GetFeedRangesAsync();
Assert.IsTrue(feedTokens.Count > 1, " RUs of the container needs to be increased to ensure at least 2 partitions.");
// There should only be one range since we get 1 FeedRange per range
foreach (FeedRange feedToken in feedTokens)
{
List <PartitionKeyRange> partitionKeyRanges = await CosmosQueryExecutionContextFactory.GetTargetPartitionKeyRangesAsync(
queryClient : new CosmosQueryClientCore(container.ClientContext, container),
resourceLink : container.LinkUri.OriginalString,
partitionedQueryExecutionInfo : null,
containerQueryProperties : containerQueryProperties,
properties : null,
feedRangeInternal : feedToken as FeedRangeInternal);
Assert.IsTrue(partitionKeyRanges.Count == 1, "Only 1 partition key range should be selected since the FeedRange represents a single range.");
}
}
finally
{
await container?.DeleteContainerAsync();
}
}
public async Task ReadFeedIteratorCore_OfT_WithFeedRange_ReadAll_StopResume()
{
int batchSize = 1000;
await this.CreateRandomItems(this.LargerContainer, batchSize, randomPartitionKey : true);
ContainerCore itemsCore = this.LargerContainer;
IReadOnlyList <FeedRange> tokens = await itemsCore.GetFeedRangesAsync();
List <Task <int> > tasks = tokens.Select(token => Task.Run(async() =>
{
int count = 0;
FeedIterator <ToDoActivity> feedIterator = itemsCore.GetItemQueryIterator <ToDoActivity>(queryDefinition: null, feedRange: token);
string continuation = null;
while (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> response = await feedIterator.ReadNextAsync(this.cancellationToken);
count += response.Count;
continuation = response.ContinuationToken;
break;
}
feedIterator = itemsCore.GetItemQueryIterator <ToDoActivity>(queryDefinition: null, continuationToken: continuation);
while (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> response = await feedIterator.ReadNextAsync(this.cancellationToken);
count += response.Count;
}
return(count);
})).ToList();
await Task.WhenAll(tasks);
int documentsRead = 0;
foreach (Task <int> task in tasks)
{
documentsRead += task.Result;
}
Assert.AreEqual(batchSize, documentsRead);
}
public async Task ReadFeedIteratorCore_AllowsParallelProcessing()
{
int batchSize = 1000;
await this.CreateRandomItems(this.LargerContainer, batchSize, randomPartitionKey : true);
ContainerCore itemsCore = this.LargerContainer;
IReadOnlyList <FeedRange> tokens = await itemsCore.GetFeedRangesAsync();
List <Task <int> > tasks = tokens.Select(token => Task.Run(async() =>
{
int count = 0;
FeedIterator feedIterator = itemsCore.GetItemQueryStreamIterator(queryDefinition: null, feedRange: token);
while (feedIterator.HasMoreResults)
{
using (ResponseMessage responseMessage =
await feedIterator.ReadNextAsync(this.cancellationToken))
{
if (responseMessage.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
count += response.Count;
}
}
}
return(count);
})).ToList();
await Task.WhenAll(tasks);
int documentsRead = 0;
foreach (Task <int> task in tasks)
{
documentsRead += task.Result;
}
Assert.AreEqual(batchSize, documentsRead);
}
public async Task ChangeFeed_FeedRange_FromV2SDK()
{
ContainerResponse largerContainer = await this.database.CreateContainerAsync(
new ContainerProperties(id : Guid.NewGuid().ToString(), partitionKeyPath : "/status"),
throughput : 20000,
cancellationToken : this.cancellationToken);
ContainerCore container = (ContainerInlineCore)largerContainer;
int expected = 100;
int count = 0;
await this.CreateRandomItems(container, expected, randomPartitionKey : true);
IReadOnlyList <FeedRange> feedRanges = await container.GetFeedRangesAsync();
List <string> continuations = new List <string>();
// First do one request to construct the old model information based on Etag
foreach (FeedRange feedRange in feedRanges)
{
IEnumerable <string> pkRangeIds = await container.GetPartitionKeyRangesAsync(feedRange);
ChangeFeedRequestOptions requestOptions = new ChangeFeedRequestOptions()
{
StartTime = DateTime.MinValue.ToUniversalTime(), MaxItemCount = 1
};
ChangeFeedIteratorCore feedIterator = container.GetChangeFeedStreamIterator(feedRange: feedRange, changeFeedRequestOptions: requestOptions) as ChangeFeedIteratorCore;
ResponseMessage firstResponse = await feedIterator.ReadNextAsync();
if (firstResponse.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(firstResponse.Content).Data;
count += response.Count;
}
// Construct the continuation's range, using PKRangeId + ETag
List <dynamic> ct = new List <dynamic>()
{
new { min = string.Empty, max = string.Empty, token = firstResponse.Headers.ETag }
};
// Extract Etag and manually construct the continuation
dynamic oldContinuation = new { V = 0, PKRangeId = pkRangeIds.First(), Continuation = ct };
continuations.Add(JsonConvert.SerializeObject(oldContinuation));
}
// Now start the new iterators with the constructed continuations from migration
foreach (string continuation in continuations)
{
ChangeFeedRequestOptions requestOptions = new ChangeFeedRequestOptions()
{
MaxItemCount = 100
};
ChangeFeedIteratorCore feedIterator = container.GetChangeFeedStreamIterator(continuationToken: continuation, changeFeedRequestOptions: requestOptions) as ChangeFeedIteratorCore;
ResponseMessage firstResponse = await feedIterator.ReadNextAsync();
if (firstResponse.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(firstResponse.Content).Data;
count += response.Count;
}
}
Assert.AreEqual(expected, count);
}
public async Task ParallelizeQueryThroughTokens()
{
ContainerCore container = null;
try
{
// Create a container large enough to have at least 2 partitions
ContainerResponse containerResponse = await this.database.CreateContainerAsync(
id : Guid.NewGuid().ToString(),
partitionKeyPath : "/id",
throughput : 15000);
container = (ContainerInlineCore)containerResponse;
List <string> generatedIds = Enumerable.Range(0, 1000).Select(n => $"BasicItem{n}").ToList();
foreach (string id in generatedIds)
{
string item = $@"
{{
""id"": ""{id}""
}}";
using (ResponseMessage createResponse = await container.CreateItemStreamAsync(
QueryFeedRangeTests.GenerateStreamFromString(item),
new Cosmos.PartitionKey(id)))
{
Assert.IsTrue(createResponse.IsSuccessStatusCode);
}
}
IReadOnlyList <FeedRange> feedTokens = await container.GetFeedRangesAsync();
Assert.IsTrue(feedTokens.Count > 1, " RUs of the container needs to be increased to ensure at least 2 partitions.");
List <Task <List <string> > > tasks = feedTokens.Select(async feedToken =>
{
List <string> results = new List <string>();
FeedIteratorInternal feedIterator = container.GetItemQueryStreamIterator(queryDefinition: new QueryDefinition("select * from T where STARTSWITH(T.id, \"BasicItem\")"), feedRange: feedToken, requestOptions: new QueryRequestOptions()
{
MaxItemCount = 10
}) as FeedIteratorInternal;
string continuation = null;
while (feedIterator.HasMoreResults)
{
using (ResponseMessage responseMessage =
await feedIterator.ReadNextAsync(this.cancellationToken))
{
if (responseMessage.IsSuccessStatusCode)
{
using (StreamReader reader = new StreamReader(responseMessage.Content))
{
string json = await reader.ReadToEndAsync();
JArray documents = (JArray)JObject.Parse(json).SelectToken("Documents");
foreach (JObject document in documents)
{
results.Add(document.SelectToken("id").ToString());
}
}
}
continuation = responseMessage.ContinuationToken;
break;
}
}
feedIterator = container.GetItemQueryStreamIterator(queryDefinition: new QueryDefinition("select * from T where STARTSWITH(T.id, \"BasicItem\")"), feedRange: feedToken, continuationToken: continuation, requestOptions: new QueryRequestOptions()
{
MaxItemCount = 10
}) as FeedIteratorInternal;
while (feedIterator.HasMoreResults)
{
using (ResponseMessage responseMessage =
await feedIterator.ReadNextAsync(this.cancellationToken))
{
if (responseMessage.IsSuccessStatusCode)
{
using (StreamReader reader = new StreamReader(responseMessage.Content))
{
string json = await reader.ReadToEndAsync();
JArray documents = (JArray)JObject.Parse(json).SelectToken("Documents");
foreach (JObject document in documents)
{
results.Add(document.SelectToken("id").ToString());
}
}
}
}
}
return(results);
}).ToList();
await Task.WhenAll(tasks);
CollectionAssert.AreEquivalent(generatedIds, tasks.SelectMany(t => t.Result).ToList());
}
finally
{
await container?.DeleteContainerAsync();
}
}
public async Task ChangeFeed_FeedRange_FromV0Token()
{
ContainerResponse largerContainer = await this.database.CreateContainerAsync(
new ContainerProperties(id : Guid.NewGuid().ToString(), partitionKeyPath : "/status"),
throughput : 20000,
cancellationToken : this.cancellationToken);
ContainerCore container = (ContainerInlineCore)largerContainer;
int expected = 100;
int count = 0;
await this.CreateRandomItems(container, expected, randomPartitionKey : true);
IReadOnlyList <FeedRange> feedRanges = await container.GetFeedRangesAsync();
List <string> continuations = new List <string>();
// First do one request to construct the old model information based on Etag
foreach (FeedRange feedRange in feedRanges)
{
IEnumerable <string> pkRangeIds = await container.GetPartitionKeyRangesAsync(feedRange);
ChangeFeedRequestOptions requestOptions = new ChangeFeedRequestOptions()
{
PageSizeHint = 1
};
ChangeFeedIteratorCore feedIterator = container.GetChangeFeedStreamIterator(
changeFeedStartFrom: ChangeFeedStartFrom.Beginning(feedRange),
changeFeedRequestOptions: requestOptions) as ChangeFeedIteratorCore;
ResponseMessage firstResponse = await feedIterator.ReadNextAsync();
FeedRangeEpk FeedRangeEpk = feedRange as FeedRangeEpk;
// Construct the continuation's range, using PKRangeId + ETag
List <dynamic> ct = new List <dynamic>()
{
new
{
min = FeedRangeEpk.Range.Min,
max = FeedRangeEpk.Range.Max,
token = (string)null
}
};
// Extract Etag and manually construct the continuation
dynamic oldContinuation = new
{
V = 0,
Rid = await container.GetRIDAsync(this.cancellationToken),
Continuation = ct
};
continuations.Add(JsonConvert.SerializeObject(oldContinuation));
}
// Now start the new iterators with the constructed continuations from migration
foreach (string continuation in continuations)
{
ChangeFeedRequestOptions requestOptions = new ChangeFeedRequestOptions()
{
PageSizeHint = 100,
EmitOldContinuationToken = true,
};
ChangeFeedIteratorCore feedIterator = container.GetChangeFeedStreamIterator(
changeFeedStartFrom: ChangeFeedStartFrom.ContinuationToken(continuation),
changeFeedRequestOptions: requestOptions) as ChangeFeedIteratorCore;
ResponseMessage firstResponse = await feedIterator.ReadNextAsync();
if (firstResponse.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(firstResponse.Content).Data;
count += response.Count;
string migratedContinuation = firstResponse.ContinuationToken;
Assert.IsTrue(FeedRangeContinuation.TryParse(migratedContinuation, out FeedRangeContinuation feedRangeContinuation));
Assert.IsTrue(feedRangeContinuation.FeedRange is FeedRangeEpk);
}
}
Assert.AreEqual(expected, count);
}