/// <summary>
        /// Create a item producer with a list of responses mocked
        /// </summary>
        /// <param name="responseMessagesPageSize">Each entry represents a response message and the number of items in the response.</param>
        /// <param name="sqlQuerySpec">The query spec the backend should be expecting</param>
        /// <param name="partitionKeyRange">The partition key range</param>
        /// <param name="continuationToken">The initial continuation token.</param>
        /// <param name="maxPageSize">The max page size</param>
        /// <param name="completeDelegate">A delegate that is called when a page is loaded</param>
        /// <param name="responseDelay">Delays the ExecuteQueryAsync response. This allows testing race conditions.</param>
        /// <param name="cancellationToken">The expected continuation token</param>
        public static (ItemProducer itemProducer, ReadOnlyCollection <ToDoItem> allItems) Create(
            int[] responseMessagesPageSize      = null,
            SqlQuerySpec sqlQuerySpec           = null,
            PartitionKeyRange partitionKeyRange = null,
            string continuationToken            = null,
            int maxPageSize = 50,
            ItemProducer.ProduceAsyncCompleteDelegate completeDelegate = null,
            Action executeCallback = null,
            CancellationToken cancellationToken = default(CancellationToken))
        {
            if (responseMessagesPageSize == null)
            {
                responseMessagesPageSize = DefaultResponseSizes;
            }

            if (sqlQuerySpec == null)
            {
                sqlQuerySpec = DefaultQuerySpec;
            }

            if (partitionKeyRange == null)
            {
                partitionKeyRange = DefaultPartitionKeyRange;
            }

            if (completeDelegate == null)
            {
                completeDelegate = DefaultProduceAsyncCompleteDelegate;
            }

            Mock <CosmosQueryContext> mockQueryContext = new Mock <CosmosQueryContext>();

            mockQueryContext.Setup(x => x.ContainerResourceId).Returns(DefaultCollectionRid);

            // Setup a list of query responses. It generates a new continuation token for each response. This allows the mock to return the messages in the correct order.
            List <ToDoItem> allItems = MockSinglePartitionKeyRangeContext(
                mockQueryContext,
                responseMessagesPageSize,
                sqlQuerySpec,
                partitionKeyRange,
                continuationToken,
                maxPageSize,
                DefaultCollectionRid,
                executeCallback,
                cancellationToken);

            ItemProducer itemProducer = new ItemProducer(
                mockQueryContext.Object,
                sqlQuerySpec,
                partitionKeyRange,
                completeDelegate,
                CosmosElementEqualityComparer.Value,
                new TestInjections(simulate429s: false, simulateEmptyPages: false),
                maxPageSize,
                initialContinuationToken: continuationToken);

            return(itemProducer, allItems.AsReadOnly());
        }
 /// <summary>
 /// Gets whether or not we should increment the skip count based on the rid of the document.
 /// </summary>
 /// <param name="currentItemProducer">The current document producer.</param>
 /// <returns>Whether or not we should increment the skip count.</returns>
 private bool ShouldIncrementSkipCount(ItemProducer currentItemProducer)
 {
     // If we are not at the beginning of the page and we saw the same rid again.
     return(!currentItemProducer.IsAtBeginningOfPage &&
            string.Equals(
                this.previousRid,
                new OrderByQueryResult(currentItemProducer.Current).Rid,
                StringComparison.Ordinal));
 }
        public async Task BufferMore(int maxPageSize, int[] pageSizes)
        {
            (ItemProducer itemProducer, ReadOnlyCollection <ToDoItem> allItems)itemFactory = MockItemProducerFactory.Create(
                responseMessagesPageSize: pageSizes,
                maxPageSize: maxPageSize,
                cancellationToken: this.cancellationToken);

            ItemProducer itemProducer     = itemFactory.itemProducer;
            int          currentItemCount = pageSizes[0];

            // Single thread
            for (int iterations = 0; iterations < 10; iterations++)
            {
                // Only first buffer more should go through
                await itemProducer.BufferMoreIfEmptyAsync(this.cancellationToken);

                Assert.AreEqual(currentItemCount, itemProducer.BufferedItemCount);
            }

            for (int roundTrip = 1; roundTrip < pageSizes.Length; roundTrip++)
            {
                await itemProducer.BufferMoreDocumentsAsync(this.cancellationToken);

                currentItemCount += pageSizes[roundTrip];
                Assert.AreEqual(currentItemCount, itemProducer.BufferedItemCount);
            }

            Assert.AreEqual(itemFactory.allItems.Count, itemProducer.BufferedItemCount);

            // Verify that Buffer More does nothing after all pages are already loaded
            for (int roundTrip = 0; roundTrip < pageSizes.Length; roundTrip++)
            {
                await itemProducer.BufferMoreDocumentsAsync(this.cancellationToken);

                Assert.AreEqual(itemFactory.allItems.Count, itemProducer.BufferedItemCount);
            }

            // Parallel
            itemFactory = MockItemProducerFactory.Create(
                responseMessagesPageSize: pageSizes,
                maxPageSize: 10,
                cancellationToken: this.cancellationToken);
            itemProducer = itemFactory.itemProducer;

            List <Task> tasks = new List <Task>();

            for (int i = 0; i < pageSizes.Length; i++)
            {
                tasks.Add(Task.Run(async() => { await itemProducer.BufferMoreDocumentsAsync(this.cancellationToken); }));
            }

            await Task.WhenAll(tasks);

            Assert.AreEqual(currentItemCount, itemProducer.BufferedItemCount);
        }
            /// <summary>
            /// Initializes a new instance of the <see cref="ItemProducerCollection"/> class.
            /// </summary>
            /// <param name="parent">The parent worker instance.</param>
            /// <param name="entries">All directory entries that should be backuped.</param>
            public ItemProducerCollection(BackupWorker parent, Dom.DirectoryEntryReadonly[] entries)
            {
                // TODO find out, which entries belong to which hard disk
                // Entries with the same hard disk should be grouped in one CollectorItem, since they should be evaluated sequential
                // Entries with different hard disk can be put in different CollectorItems.

                _collectorItems = new ItemProducer[1];
                var item = new ItemProducer(parent, entries);

                item.OutputAvailable += item => ItemAvailable?.Invoke(item);
                _collectorItems[0]    = item;
            }
        private async Task <List <ToDoItem> > ReadItemProducer(ItemProducer itemProducer, int numItems)
        {
            List <ToDoItem> itemsRead = new List <ToDoItem>();

            for (int i = 0; i < numItems; i++)
            {
                (bool successfullyMovedNext, QueryResponse failureResponse)movedNext = await itemProducer.MoveNextAsync(this.cancellationToken);

                Assert.IsTrue(movedNext.successfullyMovedNext);
                Assert.IsTrue(itemProducer.HasMoreResults);
                itemsRead.Add(this.ConvertCosmosElement(itemProducer.Current));
            }

            return(itemsRead);
        }
Example #6
0
        public async Task TestMoveNextAsync(int pageSize, int maxPageSize)
        {
            List <int[]> combinations = new List <int[]>()
            {
                // Create all combination with empty pages
                new int[] { pageSize },
                new int[] { pageSize, 0 },
                new int[] { 0, pageSize },
                new int[] { pageSize, pageSize },
                new int[] { pageSize, 0, 0 },
                new int[] { 0, pageSize, 0 },
                new int[] { 0, 0, pageSize },
                new int[] { pageSize, 0, pageSize },
                new int[] { pageSize, pageSize, pageSize },
            };

            foreach (int[] combination in combinations)
            {
                (ItemProducer itemProducer, ReadOnlyCollection <ToDoItem> allItems)itemFactory = MockItemProducerFactory.Create(
                    responseMessagesPageSize: combination,
                    maxPageSize: maxPageSize,
                    cancellationToken: this.cancellationToken);

                ItemProducer itemProducer = itemFactory.itemProducer;

                List <ToDoItem> itemsRead = new List <ToDoItem>();

                Assert.IsTrue(itemProducer.HasMoreResults);

                while ((await itemProducer.TryMoveNextPageAsync(this.cancellationToken)).movedToNextPage)
                {
                    while (itemProducer.TryMoveNextDocumentWithinPage())
                    {
                        Assert.IsTrue(itemProducer.HasMoreResults);
                        string   jsonValue = itemProducer.Current.ToString();
                        ToDoItem item      = JsonConvert.DeserializeObject <ToDoItem>(jsonValue);
                        itemsRead.Add(item);
                    }
                }

                Assert.IsFalse(itemProducer.HasMoreResults);

                Assert.AreEqual(itemFactory.allItems.Count, itemsRead.Count);

                CollectionAssert.AreEqual(itemsRead, itemFactory.allItems, new ToDoItemComparer());
            }
        }
Example #7
0
        private async Task <List <ToDoItem> > ReadItemProducer(ItemProducer itemProducer, int numItems)
        {
            List <ToDoItem> itemsRead = new List <ToDoItem>();

            for (int i = 0; i < numItems; i++)
            {
                if (!itemProducer.TryMoveNextDocumentWithinPage())
                {
                    Assert.IsTrue((await itemProducer.TryMoveNextPageAsync(this.cancellationToken)).movedToNextPage);
                    Assert.IsTrue(itemProducer.TryMoveNextDocumentWithinPage());
                }

                Assert.IsTrue(itemProducer.HasMoreResults);
                itemsRead.Add(this.ConvertCosmosElement(itemProducer.Current));
            }

            return(itemsRead);
        }
        public async Task ConcurrentMoveNextAndBufferMore()
        {
            int[] pageSizes = new int[] { 2, 3, 1, 4 };
            (ItemProducer itemProducer, ReadOnlyCollection <ToDoItem> allItems)itemFactory = MockItemProducerFactory.Create(
                responseMessagesPageSize: pageSizes,
                maxPageSize: 10,
                responseDelay: TimeSpan.FromSeconds(1),
                cancellationToken: this.cancellationToken);

            ItemProducer itemProducer = itemFactory.itemProducer;

            // BufferMore
            // Fire and Forget this task.
#pragma warning disable 4014
            Task.Run(() => itemProducer.BufferMoreDocumentsAsync(this.cancellationToken));
#pragma warning restore 4014

            List <ToDoItem> itemsRead = new List <ToDoItem>();
            Assert.AreEqual(0, itemProducer.BufferedItemCount, "Mocked response should be delayed until after move next is called.");

            int itemsToRead = pageSizes[0] + pageSizes[1];
            // Call move next while buffer more is waiting for response.
            // Move next should wait for buffer more to complete then use the results of the buffer more.
            List <ToDoItem> currentPage = await this.ReadItemProducer(itemProducer, itemsToRead);

            itemsRead.AddRange(currentPage);

            Assert.AreEqual(itemsToRead, itemsRead.Count, "All of the first and 2nd page should be read.");
            Assert.AreEqual(1, itemProducer.BufferedItemCount, "The last element should still be buffered. Moving next will cause another buffer.");

            itemsToRead = pageSizes[2] + pageSizes[3];
#pragma warning disable 4014
            Task.Run(() => itemProducer.MoveNextAsync(this.cancellationToken));
#pragma warning restore 4014

            await itemProducer.BufferMoreDocumentsAsync(this.cancellationToken);

            Assert.AreEqual(itemsToRead, itemProducer.BufferedItemCount, "2nd Page should be loaded.");
        }
        public async Task ConcurrentMoveNextAndBufferMore()
        {
            bool   blockExecute  = true;
            int    callBackCount = 0;
            Action callbackBlock = () =>
            {
                int callBackWaitCount = 0;
                callBackCount++;
                while (blockExecute)
                {
                    if (callBackWaitCount++ > 200)
                    {
                        Assert.Fail("The task never started to buffer the items. The callback was never called");
                    }

                    Thread.Sleep(TimeSpan.FromSeconds(.1));
                }

                // Reset the block for the  next call
                blockExecute = true;
            };

            int[] pageSizes = new int[] { 2, 3, 1, 4 };
            (ItemProducer itemProducer, ReadOnlyCollection <ToDoItem> allItems)itemFactory = MockItemProducerFactory.Create(
                responseMessagesPageSize: pageSizes,
                maxPageSize: 10,
                executeCallback: callbackBlock,
                cancellationToken: this.cancellationToken);

            ItemProducer itemProducer = itemFactory.itemProducer;

            // BufferMore
            // Fire and Forget this task.
#pragma warning disable 4014
            Task bufferTask = Task.Run(() => itemProducer.BufferMoreDocumentsAsync(this.cancellationToken));

            // Verify the task started
            int waitCount = 0;
            while (callBackCount == 0)
            {
                if (waitCount++ > 100)
                {
                    Assert.Fail("The task never started to buffer the items. The callback was never called");
                }

                Thread.Sleep(TimeSpan.FromSeconds(.1));
            }

            List <ToDoItem> itemsRead = new List <ToDoItem>();
            Assert.AreEqual(0, itemProducer.BufferedItemCount, "Mocked response should be delayed until after move next is called.");

            int itemsToRead = pageSizes[0] + pageSizes[1];
            // Call move next while buffer more is waiting for response.
            // Move next should wait for buffer more to complete then use the results of the buffer more.
#pragma warning disable 4014
            bool readTaskRunning = false;
            Task readTask        = Task.Run(async() =>
            {
                readTaskRunning             = true;
                List <ToDoItem> currentPage = await this.ReadItemProducer(itemProducer, itemsToRead);
                itemsRead.AddRange(currentPage);
            });
#pragma warning restore 4014

            // Verify the task started
            while (readTaskRunning == false)
            {
                Thread.Sleep(TimeSpan.FromSeconds(.1));
            }

            Assert.AreEqual(0, itemProducer.BufferedItemCount, "The call back block will prevent any item from being buffered");
            Assert.AreEqual(1, callBackCount, "Buffer more should have a lock which prevents multiple executes.");

            // Unblock the buffer task
            blockExecute = false;
            await bufferTask;
            Assert.AreEqual(2, itemProducer.BufferedItemCount, "Buffer should be completed and have 2 items from first page");

            // Unblock the read task
            blockExecute = false;
            await readTask;
            Assert.AreEqual(itemsToRead, itemsRead.Count, "All of the first and 2nd page should be read.");
            Assert.AreEqual(1, itemProducer.BufferedItemCount, "The last element should still be buffered. Moving next will cause another buffer.");

            itemsToRead = pageSizes[2] + pageSizes[3];
#pragma warning disable 4014
            Task moveNext = Task.Run(() => itemProducer.MoveNextAsync(this.cancellationToken));
#pragma warning restore 4014
            while (callBackCount == 2)
            {
                if (waitCount++ > 100)
                {
                    Assert.Fail("The task never started to buffer the items. The callback was never called");
                }

                Thread.Sleep(TimeSpan.FromSeconds(.1));
            }

            bufferTask = Task.Run(() => itemProducer.BufferMoreDocumentsAsync(this.cancellationToken));

            Assert.AreEqual(3, callBackCount, "Buffer more should have a lock which prevents multiple executes.");

            blockExecute = false;
            await moveNext;

            blockExecute = false;
            await bufferTask;
            Assert.AreEqual(itemsToRead, itemProducer.BufferedItemCount, "2nd Page should be loaded.");
        }