public async Task StandByFeedIterator_WithMaxItemCount()
{
await this.CreateRandomItems(2, randomPartitionKey : true);
CosmosItemsCore itemsCore = (CosmosItemsCore)this.Container.Items;
CosmosFeedResultSetIterator setIterator = itemsCore.GetStandByFeedIterator(maxItemCount: 1, requestOptions: new CosmosChangeFeedRequestOptions()
{
StartTime = DateTime.MinValue
});
while (setIterator.HasMoreResults)
{
using (CosmosResponseMessage responseMessage =
await setIterator.FetchNextSetAsync(this.cancellationToken))
{
if (responseMessage.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = new CosmosDefaultJsonSerializer().FromStream <CosmosFeedResponse <ToDoActivity> >(responseMessage.Content).Data;
if (response.Count > 0)
{
Assert.AreEqual(1, response.Count);
return;
}
}
}
}
Assert.Fail("Found no batch with size 1");
}
private static async Task ItemStreamFeed(CosmosContainer container)
{
int totalCount = 0;
// SQL
CosmosFeedResultSetIterator setIterator = container.Items.GetItemStreamIterator();
while (setIterator.HasMoreResults)
{
int count = 0;
using (CosmosResponseMessage response = await setIterator.FetchNextSetAsync())
{
response.EnsureSuccessStatusCode();
count++;
using (StreamReader sr = new StreamReader(response.Content))
using (JsonTextReader jtr = new JsonTextReader(sr))
{
JsonSerializer jsonSerializer = new JsonSerializer();
dynamic array = jsonSerializer.Deserialize <dynamic>(jtr);
totalCount += array.Documents.Count;
}
}
}
Assert("Expected two families", totalCount == 2);
}
public async Task StandByFeedIterator_NoFetchNext()
{
var pkRanges = await this.Container.ClientContext.DocumentClient.ReadPartitionKeyRangeFeedAsync(this.Container.LinkUri);
int expected = 25;
int iterations = 0;
await this.CreateRandomItems(expected, randomPartitionKey : true);
CosmosItemsCore itemsCore = (CosmosItemsCore)this.Container.Items;
string continuationToken = null;
int count = 0;
while (true)
{
CosmosChangeFeedRequestOptions requestOptions = new CosmosChangeFeedRequestOptions()
{
StartTime = DateTime.MinValue
};
CosmosFeedResultSetIterator setIterator = itemsCore.GetStandByFeedIterator(continuationToken, requestOptions: requestOptions);
using (CosmosResponseMessage responseMessage =
await setIterator.FetchNextSetAsync(this.cancellationToken))
{
continuationToken = responseMessage.Headers.Continuation;
if (responseMessage.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = new CosmosDefaultJsonSerializer().FromStream <CosmosFeedResponse <ToDoActivity> >(responseMessage.Content).Data;
count += response.Count;
}
}
if (count > expected)
{
Assert.Fail($"{count} does not equal {expected}");
}
if (count.Equals(expected))
{
break;
}
if (iterations++ > pkRanges.Count)
{
Assert.Fail("Feed does not contain all elements even after looping through PK ranges. Either the continuation is not moving forward or there is some state problem.");
}
}
}
public async Task StreamIteratorTest()
{
string containerName = Guid.NewGuid().ToString();
string partitionKeyPath = "/users";
CosmosContainerResponse containerResponse = await this.cosmosDatabase.Containers.CreateContainerAsync(containerName, partitionKeyPath);
Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode);
Assert.AreEqual(containerName, containerResponse.Resource.Id);
Assert.AreEqual(partitionKeyPath, containerResponse.Resource.PartitionKey.Paths.First());
containerName = Guid.NewGuid().ToString();
containerResponse = await this.cosmosDatabase.Containers.CreateContainerAsync(containerName, partitionKeyPath);
Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode);
Assert.AreEqual(containerName, containerResponse.Resource.Id);
Assert.AreEqual(partitionKeyPath, containerResponse.Resource.PartitionKey.Paths.First());
HashSet <string> containerIds = new HashSet <string>();
CosmosFeedResultSetIterator resultSet = this.cosmosDatabase.Containers.GetContainerStreamIterator(
maxItemCount: 1,
requestOptions: new CosmosQueryRequestOptions());
while (resultSet.HasMoreResults)
{
using (CosmosResponseMessage message = await resultSet.FetchNextSetAsync())
{
Assert.AreEqual(HttpStatusCode.OK, message.StatusCode);
CosmosDefaultJsonSerializer defaultJsonSerializer = new CosmosDefaultJsonSerializer();
dynamic containers = defaultJsonSerializer.FromStream <dynamic>(message.Content).DocumentCollections;
foreach (dynamic container in containers)
{
string id = container.id.ToString();
containerIds.Add(id);
}
}
}
Assert.IsTrue(containerIds.Count > 0, "The iterator did not find any containers.");
Assert.IsTrue(containerIds.Contains(containerName), "The iterator did not find the created container");
containerResponse = await containerResponse.Container.DeleteAsync();
Assert.AreEqual(HttpStatusCode.NoContent, containerResponse.StatusCode);
}
public async Task ItemStreamIterator()
{
IList <ToDoActivity> deleteList = null;
HashSet <string> itemIds = null;
try
{
deleteList = await CreateRandomItems(3, randomPartitionKey : true);
itemIds = deleteList.Select(x => x.id).ToHashSet <string>();
CosmosFeedResultSetIterator setIterator =
this.Container.Items.GetItemStreamIterator();
while (setIterator.HasMoreResults)
{
using (CosmosResponseMessage iterator =
await setIterator.FetchNextSetAsync(this.cancellationToken))
{
Collection <ToDoActivity> response = new CosmosDefaultJsonSerializer().FromStream <CosmosFeedResponse <ToDoActivity> >(iterator.Content).Data;
foreach (ToDoActivity toDoActivity in response)
{
if (itemIds.Contains(toDoActivity.id))
{
itemIds.Remove(toDoActivity.id);
}
}
}
}
}
finally
{
foreach (ToDoActivity delete in deleteList)
{
CosmosResponseMessage deleteResponse = await this.Container.Items.DeleteItemStreamAsync(delete.status, delete.id);
deleteResponse.Dispose();
}
}
Assert.AreEqual(itemIds.Count, 0);
}
public async Task StandByFeedIterator_WithInexistentRange()
{
// Add some random range, this will force the failure
List <CompositeContinuationToken> corruptedTokens = new List <CompositeContinuationToken>();
corruptedTokens.Add(new CompositeContinuationToken()
{
Range = new Documents.Routing.Range <string>("whatever", "random", true, false),
Token = "oops"
});
string corruptedTokenSerialized = JsonConvert.SerializeObject(corruptedTokens);
CosmosItemsCore itemsCore = (CosmosItemsCore)this.Container.Items;
CosmosFeedResultSetIterator setIteratorNew =
itemsCore.GetStandByFeedIterator(corruptedTokenSerialized);
CosmosResponseMessage responseMessage =
await setIteratorNew.FetchNextSetAsync(this.cancellationToken);
Assert.Fail("Should have thrown.");
}
public async Task ItemStreamIterator(bool useStatelessIterator)
{
IList <ToDoActivity> deleteList = await this.CreateRandomItems(3, randomPartitionKey : true);
HashSet <string> itemIds = deleteList.Select(x => x.id).ToHashSet <string>();
string lastContinuationToken = null;
int pageSize = 1;
CosmosItemRequestOptions requestOptions = new CosmosItemRequestOptions();
CosmosFeedResultSetIterator setIterator =
this.Container.Items.GetItemStreamIterator(maxItemCount: pageSize, continuationToken: lastContinuationToken, requestOptions: requestOptions);
while (setIterator.HasMoreResults)
{
if (useStatelessIterator)
{
setIterator = this.Container.Items.GetItemStreamIterator(maxItemCount: pageSize, continuationToken: lastContinuationToken, requestOptions: requestOptions);
}
using (CosmosResponseMessage responseMessage =
await setIterator.FetchNextSetAsync(this.cancellationToken))
{
lastContinuationToken = responseMessage.Headers.Continuation;
Collection <ToDoActivity> response = new CosmosDefaultJsonSerializer().FromStream <CosmosFeedResponse <ToDoActivity> >(responseMessage.Content).Data;
foreach (ToDoActivity toDoActivity in response)
{
if (itemIds.Contains(toDoActivity.id))
{
itemIds.Remove(toDoActivity.id);
}
}
}
}
Assert.IsNull(lastContinuationToken);
Assert.AreEqual(itemIds.Count, 0);
}
public async Task CrossPartitionBiDirectionalItemReadFeedTest(bool useStatelessIteration)
{
//create items
const int total = 30;
const int maxItemCount = 10;
List <string> items = new List <string>();
for (int i = 0; i < total; i++)
{
string item = $@"
{{
""id"": ""{i}""
}}";
using (CosmosResponseMessage createResponse = await this.Container.Items.CreateItemStreamAsync(
i.ToString(),
CosmosReadFeedTests.GenerateStreamFromString(item),
requestOptions: new CosmosItemRequestOptions()))
{
Assert.IsTrue(createResponse.IsSuccessStatusCode);
}
}
string lastKnownContinuationToken = null;
CosmosFeedResultSetIterator iter = this.Container.Database.Containers[this.Container.Id].Items
.GetItemStreamIterator(maxItemCount, continuationToken: lastKnownContinuationToken);
int count = 0;
List <string> forwardOrder = new List <string>();
while (iter.HasMoreResults)
{
if (useStatelessIteration)
{
iter = this.Container.Database.Containers[this.Container.Id].Items
.GetItemStreamIterator(maxItemCount, continuationToken: lastKnownContinuationToken);
}
using (CosmosResponseMessage response = await iter.FetchNextSetAsync())
{
lastKnownContinuationToken = response.Headers.Continuation;
Assert.IsNotNull(response);
using (StreamReader reader = new StreamReader(response.Content))
{
string json = await reader.ReadToEndAsync();
JArray documents = (JArray)JObject.Parse(json).SelectToken("Documents");
count += documents.Count;
if (documents.Any())
{
forwardOrder.Add(documents.First().SelectToken("id").ToString());
}
}
}
}
Assert.IsNull(lastKnownContinuationToken);
Assert.IsNotNull(forwardOrder);
Assert.AreEqual(total, count);
Assert.IsFalse(forwardOrder.Where(x => string.IsNullOrEmpty(x)).Any());
CosmosItemRequestOptions requestOptions = new CosmosItemRequestOptions();
requestOptions.Properties = requestOptions.Properties = new Dictionary <string, object>();
requestOptions.Properties.Add(HttpConstants.HttpHeaders.EnumerationDirection, (byte)BinaryScanDirection.Reverse);
count = 0;
List <string> reverseOrder = new List <string>();
lastKnownContinuationToken = null;
iter = this.Container.Database.Containers[this.Container.Id].Items
.GetItemStreamIterator(maxItemCount, continuationToken: lastKnownContinuationToken, requestOptions: requestOptions);
while (iter.HasMoreResults)
{
if (useStatelessIteration)
{
iter = this.Container.Database.Containers[this.Container.Id].Items
.GetItemStreamIterator(maxItemCount, continuationToken: lastKnownContinuationToken, requestOptions: requestOptions);
}
using (CosmosResponseMessage response = await iter.FetchNextSetAsync())
{
lastKnownContinuationToken = response.Headers.Continuation;
Assert.IsNotNull(response);
using (StreamReader reader = new StreamReader(response.Content))
{
string json = await reader.ReadToEndAsync();
JArray documents = (JArray)JObject.Parse(json).SelectToken("Documents");
count += documents.Count;
if (documents.Any())
{
reverseOrder.Add(documents.First().SelectToken("id").ToString());
}
}
}
}
Assert.IsNull(lastKnownContinuationToken);
Assert.IsNotNull(reverseOrder);
Assert.AreEqual(total, count);
forwardOrder.Reverse();
CollectionAssert.AreEqual(forwardOrder, reverseOrder);
Assert.IsFalse(reverseOrder.Where(x => string.IsNullOrEmpty(x)).Any());
}
public async Task StandByFeedIterator()
{
int totalCount = 0;
string lastcontinuation = string.Empty;
int firstRunTotal = 25;
int batchSize = 25;
Documents.Routing.Range <string> previousRange = null;
Documents.Routing.Range <string> currentRange = null;
int pkRangesCount = (await this.Container.ClientContext.DocumentClient.ReadPartitionKeyRangeFeedAsync(this.Container.LinkUri)).Count;
int visitedPkRanges = 0;
await this.CreateRandomItems(batchSize, randomPartitionKey : true);
CosmosItemsCore itemsCore = (CosmosItemsCore)this.Container.Items;
CosmosFeedResultSetIterator setIterator = itemsCore.GetStandByFeedIterator(requestOptions: new CosmosChangeFeedRequestOptions()
{
StartTime = DateTime.MinValue
});
while (setIterator.HasMoreResults)
{
using (CosmosResponseMessage responseMessage =
await setIterator.FetchNextSetAsync(this.cancellationToken))
{
lastcontinuation = responseMessage.Headers.Continuation;
List <CompositeContinuationToken> deserializedToken = JsonConvert.DeserializeObject <List <CompositeContinuationToken> >(lastcontinuation);
currentRange = deserializedToken[0].Range;
Assert.AreEqual(pkRangesCount, deserializedToken.Count);
if (responseMessage.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = new CosmosDefaultJsonSerializer().FromStream <CosmosFeedResponse <ToDoActivity> >(responseMessage.Content).Data;
totalCount += response.Count;
}
if (!currentRange.Equals(previousRange))
{
visitedPkRanges++;
}
if (visitedPkRanges == pkRangesCount && responseMessage.StatusCode == System.Net.HttpStatusCode.NotModified)
{
break;
}
previousRange = currentRange;
}
}
Assert.AreEqual(firstRunTotal, totalCount);
int expectedFinalCount = 50;
previousRange = null;
currentRange = null;
visitedPkRanges = 0;
// Insert another batch of 25 and use the last continuation token from the first cycle
await this.CreateRandomItems(batchSize, randomPartitionKey : true);
CosmosFeedResultSetIterator setIteratorNew =
itemsCore.GetStandByFeedIterator(lastcontinuation);
while (setIteratorNew.HasMoreResults)
{
using (CosmosResponseMessage responseMessage =
await setIteratorNew.FetchNextSetAsync(this.cancellationToken))
{
lastcontinuation = responseMessage.Headers.Continuation;
currentRange = JsonConvert.DeserializeObject <List <CompositeContinuationToken> >(lastcontinuation)[0].Range;
if (responseMessage.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = new CosmosDefaultJsonSerializer().FromStream <CosmosFeedResponse <ToDoActivity> >(responseMessage.Content).Data;
totalCount += response.Count;
}
if (!currentRange.Equals(previousRange))
{
visitedPkRanges++;
}
if (visitedPkRanges == pkRangesCount && responseMessage.StatusCode == System.Net.HttpStatusCode.NotModified)
{
break;
}
previousRange = currentRange;
}
}
Assert.AreEqual(expectedFinalCount, totalCount);
}
