/// <summary>
/// Create a item producer with a list of responses mocked
/// </summary>
/// <param name="responseMessagesPageSize">Each entry represents a response message and the number of items in the response.</param>
/// <param name="sqlQuerySpec">The query spec the backend should be expecting</param>
/// <param name="partitionKeyRange">The partition key range</param>
/// <param name="continuationToken">The initial continuation token.</param>
/// <param name="maxPageSize">The max page size</param>
/// <param name="completeDelegate">A delegate that is called when a page is loaded</param>
/// <param name="responseDelay">Delays the ExecuteQueryAsync response. This allows testing race conditions.</param>
/// <param name="cancellationToken">The expected continuation token</param>
public static (ItemProducer itemProducer, ReadOnlyCollection <ToDoItem> allItems) Create(
int[] responseMessagesPageSize = null,
SqlQuerySpec sqlQuerySpec = null,
PartitionKeyRange partitionKeyRange = null,
string continuationToken = null,
int maxPageSize = 50,
ItemProducer.ProduceAsyncCompleteDelegate completeDelegate = null,
Action executeCallback = null,
CancellationToken cancellationToken = default(CancellationToken))
{
if (responseMessagesPageSize == null)
{
responseMessagesPageSize = DefaultResponseSizes;
}
if (sqlQuerySpec == null)
{
sqlQuerySpec = DefaultQuerySpec;
}
if (partitionKeyRange == null)
{
partitionKeyRange = DefaultPartitionKeyRange;
}
if (completeDelegate == null)
{
completeDelegate = DefaultProduceAsyncCompleteDelegate;
}
Mock <CosmosQueryContext> mockQueryContext = new Mock <CosmosQueryContext>();
mockQueryContext.Setup(x => x.ContainerResourceId).Returns(DefaultCollectionRid);
// Setup a list of query responses. It generates a new continuation token for each response. This allows the mock to return the messages in the correct order.
List <ToDoItem> allItems = MockSinglePartitionKeyRangeContext(
mockQueryContext,
responseMessagesPageSize,
sqlQuerySpec,
partitionKeyRange,
continuationToken,
maxPageSize,
DefaultCollectionRid,
executeCallback,
cancellationToken);
ItemProducer itemProducer = new ItemProducer(
mockQueryContext.Object,
sqlQuerySpec,
partitionKeyRange,
completeDelegate,
CosmosElementEqualityComparer.Value,
new TestInjections(simulate429s: false, simulateEmptyPages: false),
maxPageSize,
initialContinuationToken: continuationToken);
return(itemProducer, allItems.AsReadOnly());
}
/// <summary>
/// Gets whether or not we should increment the skip count based on the rid of the document.
/// </summary>
/// <param name="currentItemProducer">The current document producer.</param>
/// <returns>Whether or not we should increment the skip count.</returns>
private bool ShouldIncrementSkipCount(ItemProducer currentItemProducer)
{
// If we are not at the beginning of the page and we saw the same rid again.
return(!currentItemProducer.IsAtBeginningOfPage &&
string.Equals(
this.previousRid,
new OrderByQueryResult(currentItemProducer.Current).Rid,
StringComparison.Ordinal));
}
public async Task BufferMore(int maxPageSize, int[] pageSizes)
{
(ItemProducer itemProducer, ReadOnlyCollection <ToDoItem> allItems)itemFactory = MockItemProducerFactory.Create(
responseMessagesPageSize: pageSizes,
maxPageSize: maxPageSize,
cancellationToken: this.cancellationToken);
ItemProducer itemProducer = itemFactory.itemProducer;
int currentItemCount = pageSizes[0];
// Single thread
for (int iterations = 0; iterations < 10; iterations++)
{
// Only first buffer more should go through
await itemProducer.BufferMoreIfEmptyAsync(this.cancellationToken);
Assert.AreEqual(currentItemCount, itemProducer.BufferedItemCount);
}
for (int roundTrip = 1; roundTrip < pageSizes.Length; roundTrip++)
{
await itemProducer.BufferMoreDocumentsAsync(this.cancellationToken);
currentItemCount += pageSizes[roundTrip];
Assert.AreEqual(currentItemCount, itemProducer.BufferedItemCount);
}
Assert.AreEqual(itemFactory.allItems.Count, itemProducer.BufferedItemCount);
// Verify that Buffer More does nothing after all pages are already loaded
for (int roundTrip = 0; roundTrip < pageSizes.Length; roundTrip++)
{
await itemProducer.BufferMoreDocumentsAsync(this.cancellationToken);
Assert.AreEqual(itemFactory.allItems.Count, itemProducer.BufferedItemCount);
}
// Parallel
itemFactory = MockItemProducerFactory.Create(
responseMessagesPageSize: pageSizes,
maxPageSize: 10,
cancellationToken: this.cancellationToken);
itemProducer = itemFactory.itemProducer;
List <Task> tasks = new List <Task>();
for (int i = 0; i < pageSizes.Length; i++)
{
tasks.Add(Task.Run(async() => { await itemProducer.BufferMoreDocumentsAsync(this.cancellationToken); }));
}
await Task.WhenAll(tasks);
Assert.AreEqual(currentItemCount, itemProducer.BufferedItemCount);
}
/// <summary>
/// Initializes a new instance of the <see cref="ItemProducerCollection"/> class.
/// </summary>
/// <param name="parent">The parent worker instance.</param>
/// <param name="entries">All directory entries that should be backuped.</param>
public ItemProducerCollection(BackupWorker parent, Dom.DirectoryEntryReadonly[] entries)
{
// TODO find out, which entries belong to which hard disk
// Entries with the same hard disk should be grouped in one CollectorItem, since they should be evaluated sequential
// Entries with different hard disk can be put in different CollectorItems.
_collectorItems = new ItemProducer[1];
var item = new ItemProducer(parent, entries);
item.OutputAvailable += item => ItemAvailable?.Invoke(item);
_collectorItems[0] = item;
}
private async Task <List <ToDoItem> > ReadItemProducer(ItemProducer itemProducer, int numItems)
{
List <ToDoItem> itemsRead = new List <ToDoItem>();
for (int i = 0; i < numItems; i++)
{
(bool successfullyMovedNext, QueryResponse failureResponse)movedNext = await itemProducer.MoveNextAsync(this.cancellationToken);
Assert.IsTrue(movedNext.successfullyMovedNext);
Assert.IsTrue(itemProducer.HasMoreResults);
itemsRead.Add(this.ConvertCosmosElement(itemProducer.Current));
}
return(itemsRead);
}
public async Task TestMoveNextAsync(int pageSize, int maxPageSize)
{
List <int[]> combinations = new List <int[]>()
{
// Create all combination with empty pages
new int[] { pageSize },
new int[] { pageSize, 0 },
new int[] { 0, pageSize },
new int[] { pageSize, pageSize },
new int[] { pageSize, 0, 0 },
new int[] { 0, pageSize, 0 },
new int[] { 0, 0, pageSize },
new int[] { pageSize, 0, pageSize },
new int[] { pageSize, pageSize, pageSize },
};
foreach (int[] combination in combinations)
{
(ItemProducer itemProducer, ReadOnlyCollection <ToDoItem> allItems)itemFactory = MockItemProducerFactory.Create(
responseMessagesPageSize: combination,
maxPageSize: maxPageSize,
cancellationToken: this.cancellationToken);
ItemProducer itemProducer = itemFactory.itemProducer;
List <ToDoItem> itemsRead = new List <ToDoItem>();
Assert.IsTrue(itemProducer.HasMoreResults);
while ((await itemProducer.TryMoveNextPageAsync(this.cancellationToken)).movedToNextPage)
{
while (itemProducer.TryMoveNextDocumentWithinPage())
{
Assert.IsTrue(itemProducer.HasMoreResults);
string jsonValue = itemProducer.Current.ToString();
ToDoItem item = JsonConvert.DeserializeObject <ToDoItem>(jsonValue);
itemsRead.Add(item);
}
}
Assert.IsFalse(itemProducer.HasMoreResults);
Assert.AreEqual(itemFactory.allItems.Count, itemsRead.Count);
CollectionAssert.AreEqual(itemsRead, itemFactory.allItems, new ToDoItemComparer());
}
}
private async Task <List <ToDoItem> > ReadItemProducer(ItemProducer itemProducer, int numItems)
{
List <ToDoItem> itemsRead = new List <ToDoItem>();
for (int i = 0; i < numItems; i++)
{
if (!itemProducer.TryMoveNextDocumentWithinPage())
{
Assert.IsTrue((await itemProducer.TryMoveNextPageAsync(this.cancellationToken)).movedToNextPage);
Assert.IsTrue(itemProducer.TryMoveNextDocumentWithinPage());
}
Assert.IsTrue(itemProducer.HasMoreResults);
itemsRead.Add(this.ConvertCosmosElement(itemProducer.Current));
}
return(itemsRead);
}
public async Task ConcurrentMoveNextAndBufferMore()
{
int[] pageSizes = new int[] { 2, 3, 1, 4 };
(ItemProducer itemProducer, ReadOnlyCollection <ToDoItem> allItems)itemFactory = MockItemProducerFactory.Create(
responseMessagesPageSize: pageSizes,
maxPageSize: 10,
responseDelay: TimeSpan.FromSeconds(1),
cancellationToken: this.cancellationToken);
ItemProducer itemProducer = itemFactory.itemProducer;
// BufferMore
// Fire and Forget this task.
#pragma warning disable 4014
Task.Run(() => itemProducer.BufferMoreDocumentsAsync(this.cancellationToken));
#pragma warning restore 4014
List <ToDoItem> itemsRead = new List <ToDoItem>();
Assert.AreEqual(0, itemProducer.BufferedItemCount, "Mocked response should be delayed until after move next is called.");
int itemsToRead = pageSizes[0] + pageSizes[1];
// Call move next while buffer more is waiting for response.
// Move next should wait for buffer more to complete then use the results of the buffer more.
List <ToDoItem> currentPage = await this.ReadItemProducer(itemProducer, itemsToRead);
itemsRead.AddRange(currentPage);
Assert.AreEqual(itemsToRead, itemsRead.Count, "All of the first and 2nd page should be read.");
Assert.AreEqual(1, itemProducer.BufferedItemCount, "The last element should still be buffered. Moving next will cause another buffer.");
itemsToRead = pageSizes[2] + pageSizes[3];
#pragma warning disable 4014
Task.Run(() => itemProducer.MoveNextAsync(this.cancellationToken));
#pragma warning restore 4014
await itemProducer.BufferMoreDocumentsAsync(this.cancellationToken);
Assert.AreEqual(itemsToRead, itemProducer.BufferedItemCount, "2nd Page should be loaded.");
}
public async Task ConcurrentMoveNextAndBufferMore()
{
bool blockExecute = true;
int callBackCount = 0;
Action callbackBlock = () =>
{
int callBackWaitCount = 0;
callBackCount++;
while (blockExecute)
{
if (callBackWaitCount++ > 200)
{
Assert.Fail("The task never started to buffer the items. The callback was never called");
}
Thread.Sleep(TimeSpan.FromSeconds(.1));
}
// Reset the block for the next call
blockExecute = true;
};
int[] pageSizes = new int[] { 2, 3, 1, 4 };
(ItemProducer itemProducer, ReadOnlyCollection <ToDoItem> allItems)itemFactory = MockItemProducerFactory.Create(
responseMessagesPageSize: pageSizes,
maxPageSize: 10,
executeCallback: callbackBlock,
cancellationToken: this.cancellationToken);
ItemProducer itemProducer = itemFactory.itemProducer;
// BufferMore
// Fire and Forget this task.
#pragma warning disable 4014
Task bufferTask = Task.Run(() => itemProducer.BufferMoreDocumentsAsync(this.cancellationToken));
// Verify the task started
int waitCount = 0;
while (callBackCount == 0)
{
if (waitCount++ > 100)
{
Assert.Fail("The task never started to buffer the items. The callback was never called");
}
Thread.Sleep(TimeSpan.FromSeconds(.1));
}
List <ToDoItem> itemsRead = new List <ToDoItem>();
Assert.AreEqual(0, itemProducer.BufferedItemCount, "Mocked response should be delayed until after move next is called.");
int itemsToRead = pageSizes[0] + pageSizes[1];
// Call move next while buffer more is waiting for response.
// Move next should wait for buffer more to complete then use the results of the buffer more.
#pragma warning disable 4014
bool readTaskRunning = false;
Task readTask = Task.Run(async() =>
{
readTaskRunning = true;
List <ToDoItem> currentPage = await this.ReadItemProducer(itemProducer, itemsToRead);
itemsRead.AddRange(currentPage);
});
#pragma warning restore 4014
// Verify the task started
while (readTaskRunning == false)
{
Thread.Sleep(TimeSpan.FromSeconds(.1));
}
Assert.AreEqual(0, itemProducer.BufferedItemCount, "The call back block will prevent any item from being buffered");
Assert.AreEqual(1, callBackCount, "Buffer more should have a lock which prevents multiple executes.");
// Unblock the buffer task
blockExecute = false;
await bufferTask;
Assert.AreEqual(2, itemProducer.BufferedItemCount, "Buffer should be completed and have 2 items from first page");
// Unblock the read task
blockExecute = false;
await readTask;
Assert.AreEqual(itemsToRead, itemsRead.Count, "All of the first and 2nd page should be read.");
Assert.AreEqual(1, itemProducer.BufferedItemCount, "The last element should still be buffered. Moving next will cause another buffer.");
itemsToRead = pageSizes[2] + pageSizes[3];
#pragma warning disable 4014
Task moveNext = Task.Run(() => itemProducer.MoveNextAsync(this.cancellationToken));
#pragma warning restore 4014
while (callBackCount == 2)
{
if (waitCount++ > 100)
{
Assert.Fail("The task never started to buffer the items. The callback was never called");
}
Thread.Sleep(TimeSpan.FromSeconds(.1));
}
bufferTask = Task.Run(() => itemProducer.BufferMoreDocumentsAsync(this.cancellationToken));
Assert.AreEqual(3, callBackCount, "Buffer more should have a lock which prevents multiple executes.");
blockExecute = false;
await moveNext;
blockExecute = false;
await bufferTask;
Assert.AreEqual(itemsToRead, itemProducer.BufferedItemCount, "2nd Page should be loaded.");
}