public async Task BufferMore(int maxPageSize, int[] pageSizes)
{
(ItemProducer itemProducer, ReadOnlyCollection <ToDoItem> allItems)itemFactory = MockItemProducerFactory.Create(
responseMessagesPageSize: pageSizes,
maxPageSize: maxPageSize,
cancellationToken: this.cancellationToken);
ItemProducer itemProducer = itemFactory.itemProducer;
int currentItemCount = pageSizes[0];
// Single thread
for (int iterations = 0; iterations < 10; iterations++)
{
// Only first buffer more should go through
await itemProducer.BufferMoreIfEmptyAsync(this.cancellationToken);
Assert.AreEqual(currentItemCount, itemProducer.BufferedItemCount);
}
for (int roundTrip = 1; roundTrip < pageSizes.Length; roundTrip++)
{
await itemProducer.BufferMoreDocumentsAsync(this.cancellationToken);
currentItemCount += pageSizes[roundTrip];
Assert.AreEqual(currentItemCount, itemProducer.BufferedItemCount);
}
Assert.AreEqual(itemFactory.allItems.Count, itemProducer.BufferedItemCount);
// Verify that Buffer More does nothing after all pages are already loaded
for (int roundTrip = 0; roundTrip < pageSizes.Length; roundTrip++)
{
await itemProducer.BufferMoreDocumentsAsync(this.cancellationToken);
Assert.AreEqual(itemFactory.allItems.Count, itemProducer.BufferedItemCount);
}
// Parallel
itemFactory = MockItemProducerFactory.Create(
responseMessagesPageSize: pageSizes,
maxPageSize: 10,
cancellationToken: this.cancellationToken);
itemProducer = itemFactory.itemProducer;
List <Task> tasks = new List <Task>();
for (int i = 0; i < pageSizes.Length; i++)
{
tasks.Add(Task.Run(async() => { await itemProducer.BufferMoreDocumentsAsync(this.cancellationToken); }));
}
await Task.WhenAll(tasks);
Assert.AreEqual(currentItemCount, itemProducer.BufferedItemCount);
}
public async Task TestMoveNextAsync(int pageSize, int maxPageSize)
{
List <int[]> combinations = new List <int[]>()
{
// Create all combination with empty pages
new int[] { pageSize },
new int[] { pageSize, 0 },
new int[] { 0, pageSize },
new int[] { pageSize, pageSize },
new int[] { pageSize, 0, 0 },
new int[] { 0, pageSize, 0 },
new int[] { 0, 0, pageSize },
new int[] { pageSize, 0, pageSize },
new int[] { pageSize, pageSize, pageSize },
};
foreach (int[] combination in combinations)
{
(ItemProducer itemProducer, ReadOnlyCollection <ToDoItem> allItems)itemFactory = MockItemProducerFactory.Create(
responseMessagesPageSize: combination,
maxPageSize: maxPageSize,
cancellationToken: this.cancellationToken);
ItemProducer itemProducer = itemFactory.itemProducer;
List <ToDoItem> itemsRead = new List <ToDoItem>();
Assert.IsTrue(itemProducer.HasMoreResults);
while ((await itemProducer.MoveNextAsync(this.cancellationToken)).successfullyMovedNext)
{
Assert.IsTrue(itemProducer.HasMoreResults);
string jsonValue = itemProducer.Current.ToString();
ToDoItem item = JsonConvert.DeserializeObject <ToDoItem>(jsonValue);
itemsRead.Add(item);
}
Assert.IsFalse(itemProducer.HasMoreResults);
Assert.AreEqual(itemFactory.allItems.Count, itemsRead.Count);
CollectionAssert.AreEqual(itemsRead, itemFactory.allItems, new ToDoItemComparer());
}
}
private static IList <ToDoItem> GenerateAndMockResponseHelper(
Mock <CosmosQueryClient> mockQueryClient,
Mock <IRoutingMapProvider> mockRoutingMap,
IList <ToDoItem> allItemsOrdered,
bool isOrderByQuery,
SqlQuerySpec sqlQuerySpec,
string containerRid,
string initContinuationToken,
int maxPageSize,
MockPartitionResponse[] mockResponseForSinglePartition,
CancellationToken cancellationTokenForMocks)
{
if (mockResponseForSinglePartition == null)
{
throw new ArgumentNullException(nameof(mockResponseForSinglePartition));
}
// Loop through all the partitions
foreach (MockPartitionResponse partitionAndMessages in mockResponseForSinglePartition)
{
PartitionKeyRange partitionKeyRange = partitionAndMessages.PartitionKeyRange;
string previousContinuationToken = initContinuationToken;
// Loop through each message inside the partition
List <int[]> messages = partitionAndMessages.MessagesWithItemIndex;
int messagesCount = messages == null ? 0 : messages.Count;
int lastMessageIndex = messagesCount - 1;
for (int i = 0; i < messagesCount; i++)
{
int[] message = partitionAndMessages.MessagesWithItemIndex[i];
string newContinuationToken = null;
List <ToDoItem> currentPageItems = new List <ToDoItem>();
// Null represents an empty page
if (message != null)
{
foreach (int itemPosition in message)
{
currentPageItems.Add(allItemsOrdered[itemPosition]);
}
}
// Last message should have null continuation token
// Split means it's not the last message for this PK range
if (i != lastMessageIndex || partitionAndMessages.HasSplit)
{
newContinuationToken = Guid.NewGuid().ToString();
}
QueryResponse queryResponse = QueryResponseMessageFactory.CreateQueryResponse(
currentPageItems,
isOrderByQuery,
newContinuationToken,
containerRid);
mockQueryClient.Setup(x =>
x.ExecuteItemQueryAsync(
It.IsAny <Uri>(),
ResourceType.Document,
OperationType.Query,
containerRid,
It.IsAny <QueryRequestOptions>(),
It.Is <SqlQuerySpec>(specInput => MockItemProducerFactory.IsSqlQuerySpecEqual(sqlQuerySpec, specInput)),
previousContinuationToken,
It.Is <PartitionKeyRangeIdentity>(rangeId => string.Equals(rangeId.PartitionKeyRangeId, partitionKeyRange.Id) && string.Equals(rangeId.CollectionRid, containerRid)),
It.IsAny <bool>(),
maxPageSize,
cancellationTokenForMocks))
.Returns(Task.FromResult(queryResponse));
previousContinuationToken = newContinuationToken;
}
if (partitionAndMessages.HasSplit)
{
QueryResponse querySplitResponse = QueryResponseMessageFactory.CreateSplitResponse(containerRid);
mockRoutingMap.Setup(x =>
x.TryGetOverlappingRangesAsync(
containerRid,
It.Is <Documents.Routing.Range <string> >(inputRange => inputRange.Equals(partitionKeyRange.ToRange())),
true)).Returns(Task.FromResult(partitionAndMessages.GetPartitionKeyRangeOfSplit()));
mockQueryClient.Setup(x =>
x.ExecuteItemQueryAsync(
It.IsAny <Uri>(),
ResourceType.Document,
OperationType.Query,
containerRid,
It.IsAny <QueryRequestOptions>(),
It.Is <SqlQuerySpec>(specInput => MockItemProducerFactory.IsSqlQuerySpecEqual(sqlQuerySpec, specInput)),
previousContinuationToken,
It.Is <PartitionKeyRangeIdentity>(rangeId => string.Equals(rangeId.PartitionKeyRangeId, partitionKeyRange.Id) && string.Equals(rangeId.CollectionRid, containerRid)),
It.IsAny <bool>(),
maxPageSize,
cancellationTokenForMocks))
.Returns(Task.FromResult(querySplitResponse));
GenerateAndMockResponseHelper(
mockQueryClient: mockQueryClient,
mockRoutingMap: mockRoutingMap,
allItemsOrdered: allItemsOrdered,
isOrderByQuery: isOrderByQuery,
sqlQuerySpec: sqlQuerySpec,
containerRid: containerRid,
initContinuationToken: previousContinuationToken,
maxPageSize: maxPageSize,
mockResponseForSinglePartition: partitionAndMessages.Split,
cancellationTokenForMocks: cancellationTokenForMocks);
}
}
return(allItemsOrdered);
}
public async Task ConcurrentMoveNextAndBufferMore()
{
int[] pageSizes = new int[] { 2, 3, 1, 4 };
(ItemProducer itemProducer, ReadOnlyCollection <ToDoItem> allItems)itemFactory = MockItemProducerFactory.Create(
responseMessagesPageSize: pageSizes,
maxPageSize: 10,
responseDelay: TimeSpan.FromSeconds(1),
cancellationToken: this.cancellationToken);
ItemProducer itemProducer = itemFactory.itemProducer;
// BufferMore
// Fire and Forget this task.
#pragma warning disable 4014
Task.Run(() => itemProducer.BufferMoreDocumentsAsync(this.cancellationToken));
#pragma warning restore 4014
List <ToDoItem> itemsRead = new List <ToDoItem>();
Assert.AreEqual(0, itemProducer.BufferedItemCount, "Mocked response should be delayed until after move next is called.");
int itemsToRead = pageSizes[0] + pageSizes[1];
// Call move next while buffer more is waiting for response.
// Move next should wait for buffer more to complete then use the results of the buffer more.
List <ToDoItem> currentPage = await this.ReadItemProducer(itemProducer, itemsToRead);
itemsRead.AddRange(currentPage);
Assert.AreEqual(itemsToRead, itemsRead.Count, "All of the first and 2nd page should be read.");
Assert.AreEqual(1, itemProducer.BufferedItemCount, "The last element should still be buffered. Moving next will cause another buffer.");
itemsToRead = pageSizes[2] + pageSizes[3];
#pragma warning disable 4014
Task.Run(() => itemProducer.MoveNextAsync(this.cancellationToken));
#pragma warning restore 4014
await itemProducer.BufferMoreDocumentsAsync(this.cancellationToken);
Assert.AreEqual(itemsToRead, itemProducer.BufferedItemCount, "2nd Page should be loaded.");
}
public async Task ConcurrentMoveNextAndBufferMore()
{
bool blockExecute = true;
int callBackCount = 0;
Action callbackBlock = () =>
{
int callBackWaitCount = 0;
callBackCount++;
while (blockExecute)
{
if (callBackWaitCount++ > 200)
{
Assert.Fail("The task never started to buffer the items. The callback was never called");
}
Thread.Sleep(TimeSpan.FromSeconds(.1));
}
// Reset the block for the next call
blockExecute = true;
};
int[] pageSizes = new int[] { 2, 3, 1, 4 };
(ItemProducer itemProducer, ReadOnlyCollection <ToDoItem> allItems)itemFactory = MockItemProducerFactory.Create(
responseMessagesPageSize: pageSizes,
maxPageSize: 10,
executeCallback: callbackBlock,
cancellationToken: this.cancellationToken);
ItemProducer itemProducer = itemFactory.itemProducer;
// BufferMore
// Fire and Forget this task.
#pragma warning disable 4014
Task bufferTask = Task.Run(() => itemProducer.BufferMoreDocumentsAsync(this.cancellationToken));
// Verify the task started
int waitCount = 0;
while (callBackCount == 0)
{
if (waitCount++ > 100)
{
Assert.Fail("The task never started to buffer the items. The callback was never called");
}
Thread.Sleep(TimeSpan.FromSeconds(.1));
}
List <ToDoItem> itemsRead = new List <ToDoItem>();
Assert.AreEqual(0, itemProducer.BufferedItemCount, "Mocked response should be delayed until after move next is called.");
int itemsToRead = pageSizes[0] + pageSizes[1];
// Call move next while buffer more is waiting for response.
// Move next should wait for buffer more to complete then use the results of the buffer more.
#pragma warning disable 4014
bool readTaskRunning = false;
Task readTask = Task.Run(async() =>
{
readTaskRunning = true;
List <ToDoItem> currentPage = await this.ReadItemProducer(itemProducer, itemsToRead);
itemsRead.AddRange(currentPage);
});
#pragma warning restore 4014
// Verify the task started
while (readTaskRunning == false)
{
Thread.Sleep(TimeSpan.FromSeconds(.1));
}
Assert.AreEqual(0, itemProducer.BufferedItemCount, "The call back block will prevent any item from being buffered");
Assert.AreEqual(1, callBackCount, "Buffer more should have a lock which prevents multiple executes.");
// Unblock the buffer task
blockExecute = false;
await bufferTask;
Assert.AreEqual(2, itemProducer.BufferedItemCount, "Buffer should be completed and have 2 items from first page");
// Unblock the read task
blockExecute = false;
await readTask;
Assert.AreEqual(itemsToRead, itemsRead.Count, "All of the first and 2nd page should be read.");
Assert.AreEqual(1, itemProducer.BufferedItemCount, "The last element should still be buffered. Moving next will cause another buffer.");
itemsToRead = pageSizes[2] + pageSizes[3];
#pragma warning disable 4014
Task moveNext = Task.Run(() => itemProducer.MoveNextAsync(this.cancellationToken));
#pragma warning restore 4014
while (callBackCount == 2)
{
if (waitCount++ > 100)
{
Assert.Fail("The task never started to buffer the items. The callback was never called");
}
Thread.Sleep(TimeSpan.FromSeconds(.1));
}
bufferTask = Task.Run(() => itemProducer.BufferMoreDocumentsAsync(this.cancellationToken));
Assert.AreEqual(3, callBackCount, "Buffer more should have a lock which prevents multiple executes.");
blockExecute = false;
await moveNext;
blockExecute = false;
await bufferTask;
Assert.AreEqual(itemsToRead, itemProducer.BufferedItemCount, "2nd Page should be loaded.");
}
