public async Task ReadFeedIteratorCore_MigrateFromOlder()
{
int totalCount = 0;
int batchSize = 1000;
await this.CreateRandomItems(this.LargerContainer, batchSize, randomPartitionKey : true);
ContainerInternal itemsCore = this.LargerContainer;
FeedIterator feedIterator = itemsCore.GetItemQueryStreamIterator(queryDefinition: null, requestOptions: new QueryRequestOptions()
{
MaxItemCount = 1
});
ResponseMessage firstResponse = await feedIterator.ReadNextAsync(this.cancellationToken);
feedIterator = itemsCore.GetItemQueryStreamIterator(queryDefinition: null, continuationToken: firstResponse.Headers.ContinuationToken, requestOptions: new QueryRequestOptions()
{
MaxItemCount = 1
});
while (feedIterator.HasMoreResults)
{
using (ResponseMessage responseMessage =
await feedIterator.ReadNextAsync(this.cancellationToken))
{
if (responseMessage.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
totalCount += response.Count;
}
}
}
Assert.AreEqual(batchSize - 1 /* from first request */, totalCount);
}
public async Task ReadFeedIteratorCore_ReadAll_MixContinuationToken()
{
int totalCount = 0;
int batchSize = 1000;
await this.CreateRandomItems(this.LargerContainer, batchSize, randomPartitionKey : true);
ContainerInternal itemsCore = this.LargerContainer;
// Do a read without FeedRange and get the older CT from Header
string olderContinuationToken = null;
FeedIterator feedIterator = itemsCore.GetItemQueryStreamIterator(queryDefinition: null, requestOptions: new QueryRequestOptions()
{
MaxItemCount = 1
});
while (feedIterator.HasMoreResults)
{
using (ResponseMessage responseMessage =
await feedIterator.ReadNextAsync(this.cancellationToken))
{
olderContinuationToken = responseMessage.Headers.ContinuationToken;
if (responseMessage.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
totalCount += response.Count;
}
break;
}
}
// start a new iterator using the older CT and expect it to continue
feedIterator = itemsCore.GetItemQueryStreamIterator(queryDefinition: null, continuationToken: olderContinuationToken, requestOptions: new QueryRequestOptions()
{
MaxItemCount = 1
});
while (feedIterator.HasMoreResults)
{
using (ResponseMessage responseMessage =
await feedIterator.ReadNextAsync(this.cancellationToken))
{
if (responseMessage.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
totalCount += response.Count;
}
}
}
Assert.AreEqual(batchSize, totalCount);
}
public async Task ReadFeedIteratorCore_Trace()
{
int batchSize = 1000;
await this.CreateRandomItems(this.LargerContainer, batchSize, randomPartitionKey : true);
ContainerInternal itemsCore = this.LargerContainer;
FeedIteratorInternal feedIterator = (FeedIteratorInternal)itemsCore.GetItemQueryStreamIterator(
queryDefinition: null,
requestOptions: new QueryRequestOptions()
{
MaxItemCount = int.MaxValue
});
ITrace rootTrace;
int childCount = 0;
using (rootTrace = Trace.GetRootTrace("Cross Partition Read Feed"))
{
while (feedIterator.HasMoreResults)
{
using (ResponseMessage responseMessage = await feedIterator.ReadNextAsync(rootTrace, this.cancellationToken))
{
responseMessage.EnsureSuccessStatusCode();
childCount++;
}
}
}
string trace = TraceWriter.TraceToText(rootTrace);
Console.WriteLine(trace);
Assert.AreEqual(childCount, rootTrace.Children.Count);
}
private static async Task ValidateQueryResponseAsync(ContainerInternal containerCore, string query)
{
FeedIterator feedIterator = containerCore.GetItemQueryStreamIterator(query);
while (feedIterator.HasMoreResults)
{
ResponseMessage response = await feedIterator.ReadNextAsync();
Assert.IsTrue(response.IsSuccessStatusCode);
Assert.IsNull(response.ErrorMessage);
}
}
public async Task ReadFeedIteratorCore_AllowsParallelProcessing()
{
int batchSize = 1000;
await this.CreateRandomItems(this.LargerContainer, batchSize, randomPartitionKey : true);
ContainerInternal itemsCore = this.LargerContainer;
IReadOnlyList <FeedRange> tokens = await itemsCore.GetFeedRangesAsync();
List <Task <int> > tasks = tokens.Select(token => Task.Run(async() =>
{
int count = 0;
FeedIterator feedIterator = itemsCore.GetItemQueryStreamIterator(
queryDefinition: null,
feedRange: token,
continuationToken: null,
requestOptions: null);
while (feedIterator.HasMoreResults)
{
using (ResponseMessage responseMessage =
await feedIterator.ReadNextAsync(this.cancellationToken))
{
if (responseMessage.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
count += response.Count;
}
}
}
return(count);
})).ToList();
await Task.WhenAll(tasks);
int documentsRead = 0;
foreach (Task <int> task in tasks)
{
documentsRead += task.Result;
}
Assert.AreEqual(batchSize, documentsRead);
}
public async Task ReadFeedIteratorCore_ReadOnlyPartitionKey()
{
int totalCount = 0;
int firstRunTotal = 25;
int batchSize = 25;
string pkToRead = "pkToRead";
string otherPK = "otherPK";
for (int i = 0; i < batchSize; i++)
{
await this.Container.CreateItemAsync(this.CreateRandomToDoActivity(pkToRead));
}
for (int i = 0; i < batchSize; i++)
{
await this.Container.CreateItemAsync(this.CreateRandomToDoActivity(otherPK));
}
ContainerInternal itemsCore = this.Container;
FeedIterator feedIterator = itemsCore.GetItemQueryStreamIterator(requestOptions: new QueryRequestOptions()
{
PartitionKey = new PartitionKey(pkToRead)
});
while (feedIterator.HasMoreResults)
{
using (ResponseMessage responseMessage =
await feedIterator.ReadNextAsync(this.cancellationToken))
{
if (responseMessage.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
totalCount += response.Count;
foreach (ToDoActivity toDoActivity in response)
{
Assert.AreEqual(pkToRead, toDoActivity.status);
}
}
}
}
Assert.AreEqual(firstRunTotal, totalCount);
}
public async Task ParallelizeQueryThroughTokens()
{
ContainerInternal container = null;
try
{
// Create a container large enough to have at least 2 partitions
ContainerResponse containerResponse = await this.database.CreateContainerAsync(
id : Guid.NewGuid().ToString(),
partitionKeyPath : "/id",
throughput : 15000);
container = (ContainerInlineCore)containerResponse;
List <string> generatedIds = Enumerable.Range(0, 1000).Select(n => $"BasicItem{n}").ToList();
foreach (string id in generatedIds)
{
string item = $@"
{{
""id"": ""{id}""
}}";
using (ResponseMessage createResponse = await container.CreateItemStreamAsync(
QueryFeedRangeTests.GenerateStreamFromString(item),
new Cosmos.PartitionKey(id)))
{
Assert.IsTrue(createResponse.IsSuccessStatusCode);
}
}
IReadOnlyList <FeedRange> feedTokens = await container.GetFeedRangesAsync();
Assert.IsTrue(feedTokens.Count > 1, " RUs of the container needs to be increased to ensure at least 2 partitions.");
List <Task <List <string> > > tasks = feedTokens.Select(async feedToken =>
{
List <string> results = new List <string>();
FeedIteratorInternal feedIterator = container.GetItemQueryStreamIterator(
queryDefinition: new QueryDefinition("select * from T where STARTSWITH(T.id, \"BasicItem\")"),
feedRange: feedToken,
continuationToken: null,
requestOptions: new QueryRequestOptions()
{
MaxItemCount = 10
}) as FeedIteratorInternal;
string continuation = null;
while (feedIterator.HasMoreResults)
{
using (ResponseMessage responseMessage =
await feedIterator.ReadNextAsync(this.cancellationToken))
{
if (responseMessage.IsSuccessStatusCode)
{
using (StreamReader reader = new StreamReader(responseMessage.Content))
{
string json = await reader.ReadToEndAsync();
JArray documents = (JArray)JObject.Parse(json).SelectToken("Documents");
foreach (JObject document in documents)
{
results.Add(document.SelectToken("id").ToString());
}
}
}
continuation = responseMessage.ContinuationToken;
break;
}
}
feedIterator = container.GetItemQueryStreamIterator(queryDefinition: new QueryDefinition("select * from T where STARTSWITH(T.id, \"BasicItem\")"), feedRange: feedToken, continuationToken: continuation, requestOptions: new QueryRequestOptions()
{
MaxItemCount = 10
}) as FeedIteratorInternal;
while (feedIterator.HasMoreResults)
{
using (ResponseMessage responseMessage =
await feedIterator.ReadNextAsync(this.cancellationToken))
{
if (responseMessage.IsSuccessStatusCode)
{
using (StreamReader reader = new StreamReader(responseMessage.Content))
{
string json = await reader.ReadToEndAsync();
JArray documents = (JArray)JObject.Parse(json).SelectToken("Documents");
foreach (JObject document in documents)
{
results.Add(document.SelectToken("id").ToString());
}
}
}
}
}
return(results);
}).ToList();
await Task.WhenAll(tasks);
CollectionAssert.AreEquivalent(generatedIds, tasks.SelectMany(t => t.Result).ToList());
}
finally
{
await container?.DeleteContainerAsync();
}
}
public async Task ReadFeedIteratorCore_CrossPartitionBiDirectional(bool useStatelessIteration)
{
ContainerInternal container = null;
try
{
ContainerResponse containerResponse = await this.database.CreateContainerAsync(
new ContainerProperties(id : Guid.NewGuid().ToString(), partitionKeyPath : "/id"),
throughput : 50000,
cancellationToken : this.cancellationToken);
container = (ContainerInlineCore)containerResponse;
// Create Items
const int total = 30;
QueryRequestOptions requestOptions = new QueryRequestOptions()
{
MaxItemCount = 10
};
List <string> items = new List <string>();
for (int i = 0; i < total; i++)
{
string item = $@"
{{
""id"": ""{i}""
}}";
using (ResponseMessage createResponse = await container.CreateItemStreamAsync(
ReadFeedRangeTests.GenerateStreamFromString(item),
new Cosmos.PartitionKey(i.ToString())))
{
Assert.IsTrue(createResponse.IsSuccessStatusCode);
}
}
string continuation = null;
FeedIterator iter = container.GetItemQueryStreamIterator(
continuationToken: continuation,
requestOptions: requestOptions);
int count = 0;
List <string> forwardOrder = new List <string>();
while (iter.HasMoreResults)
{
if (useStatelessIteration)
{
iter = container.GetItemQueryStreamIterator(
continuationToken: continuation,
requestOptions: requestOptions);
}
using (ResponseMessage response = await iter.ReadNextAsync())
{
Assert.IsNotNull(response);
continuation = response.ContinuationToken;
using (StreamReader reader = new StreamReader(response.Content))
{
string json = await reader.ReadToEndAsync();
JArray documents = (JArray)JObject.Parse(json).SelectToken("Documents");
count += documents.Count;
if (documents.Any())
{
forwardOrder.Add(documents.First().SelectToken("id").ToString());
}
}
}
}
Assert.IsNotNull(forwardOrder);
Assert.AreEqual(total, count);
Assert.IsFalse(forwardOrder.Where(x => string.IsNullOrEmpty(x)).Any());
requestOptions.Properties = requestOptions.Properties = new Dictionary <string, object>()
{
{ Documents.HttpConstants.HttpHeaders.EnumerationDirection, (byte)BinaryScanDirection.Reverse },
};
count = 0;
List <string> reverseOrder = new List <string>();
continuation = null;
iter = container
.GetItemQueryStreamIterator(queryDefinition: null, continuationToken: continuation, requestOptions: requestOptions);
while (iter.HasMoreResults)
{
if (useStatelessIteration)
{
iter = container
.GetItemQueryStreamIterator(queryDefinition: null, continuationToken: continuation, requestOptions: requestOptions);
}
using (ResponseMessage response = await iter.ReadNextAsync())
{
continuation = response.ContinuationToken;
Assert.IsNotNull(response);
using (StreamReader reader = new StreamReader(response.Content))
{
string json = await reader.ReadToEndAsync();
JArray documents = (JArray)JObject.Parse(json).SelectToken("Documents");
count += documents.Count;
if (documents.Any())
{
reverseOrder.Add(documents.First().SelectToken("id").ToString());
}
}
}
}
Assert.IsNotNull(reverseOrder);
Assert.AreEqual(total, count);
forwardOrder.Reverse();
CollectionAssert.AreEqual(forwardOrder, reverseOrder);
Assert.IsFalse(reverseOrder.Where(x => string.IsNullOrEmpty(x)).Any());
}
finally
{
await container?.DeleteContainerAsync();
}
}