/// <summary> /// Clears all of the items from a queue partition. /// </summary> private async Task ClearQueueAsync() { HttpQueueClient <Item> qc = new HttpQueueClient <Item>(new Uri(serviceUri), c_ListenerName); Assert.IsNotNull(qc); long queueCount = await qc.CountAsync().ConfigureAwait(false); long itemCount = await qc.CountAsync(QueueType.ItemQueue); // Visit each partition looking for items. for (long partition = 0; partition < qc.ServicePartitionCount; partition++) { // Get the items in this partition. IEnumerable <QueueItem <Item> > results = await qc.GetItemsAsync(partition).ConfigureAwait(false); foreach (QueueItem <Item> qi in results) { var deleteResults = await qc.DeleteItemAsync(qi.Key, CancellationToken.None).ConfigureAwait(false); } } int attempt = qc.ServicePartitionCount; // Get the count, there may be items hanging out in the queues, which won't be removed until a dequeue operation for that partition is attempted // and the corresponding item is not found in the list of items. while ((await qc.CountAsync().ConfigureAwait(false) > 0) && (attempt-- > 0)) { IEnumerable <QueueItem <Item> > results = await qc.DequeueAsync().ConfigureAwait(false); } }
public async Task GetCount_Test() { long queueLength = -1; // Get the queue length for the cluster using the load APIs. FabricClient sfc = new FabricClient(); var li = await sfc.QueryManager.GetClusterLoadInformationAsync(); foreach (var lmi in li.LoadMetricInformationList) { if ("QueueLength" == lmi.Name) { queueLength = lmi.ClusterLoad; break; } } Assert.IsTrue(queueLength >= 0); // Get the cluster count based on the count of each queue partition. HttpQueueClient <string> qc = new HttpQueueClient <string>(new Uri(serviceUri), c_ListenerName); Assert.IsNotNull(qc); long count = await qc.CountAsync().ConfigureAwait(false); Assert.AreEqual(queueLength, count); }
/// <summary> /// Gets the number of items in the cache. /// </summary> private static async Task GetItemCountAsync() { HttpQueueClient <Item> qc = new HttpQueueClient <Item>(_serviceUri, c_ListenerName); long priorityCount = await qc.PriorityCountAsync().ConfigureAwait(false); Stopwatch sw = Stopwatch.StartNew(); Task[] tasks = new Task[priorityCount + 4]; long itemCount = 0, allQueueCount = 0, leasedCount = 0, expiredCount = 0; long itemTimes = 0, allQueueTimes = 0, leasedTimes = 0, expiredTimes = 0; tasks[0] = qc.CountAsync(QueueType.ItemQueue).ContinueWith((t) => { itemCount = t.Result; itemTimes = sw.ElapsedMilliseconds; }); tasks[1] = qc.CountAsync(QueueType.AllQueues).ContinueWith((t) => { allQueueCount = t.Result; allQueueTimes = sw.ElapsedMilliseconds; }); tasks[2] = qc.CountAsync(QueueType.LeaseQueue).ContinueWith((t) => { leasedCount = t.Result; leasedTimes = sw.ElapsedMilliseconds; }); tasks[3] = qc.CountAsync(QueueType.ExpiredQueue).ContinueWith((t) => { expiredCount = t.Result; expiredTimes = sw.ElapsedMilliseconds; }); long[] queues = new long[priorityCount]; for (int i = 0; i < priorityCount; i++) { var index = i; tasks[i + 4] = qc.CountAsync(i).ContinueWith((t) => { queues[index] = t.Result; }); } // Wait for all queue calls to complete. Task.WaitAll(tasks); Console.WriteLine($" Item count: {itemCount:N0} in {itemTimes:N0}ms"); Console.WriteLine($" Leased item count: {leasedCount:N0} in {allQueueTimes:N0}ms"); Console.WriteLine($"Expired item count: {expiredCount:N0} in {leasedTimes:N0}ms"); Console.WriteLine($" All queues count: {allQueueCount:N0} in {expiredTimes:N0}ms"); for (int i = 0; i < queues.Length; i++) { Console.WriteLine($" Queue {i,2:N0}: {queues[i]:N0}"); } }
/// <summary> /// Gets the number of priorities. /// </summary> private static void GetPriorityCount() { HttpQueueClient <Item> qc = new HttpQueueClient <Item>(_serviceUri, c_ListenerName); long totalCount = qc.PriorityCountAsync().GetAwaiter().GetResult(); Console.WriteLine($"There are {totalCount} priorities."); }
public async Task DeleteItems_Test() { HttpQueueClient <Item> qc = new HttpQueueClient <Item>(new Uri(serviceUri), c_ListenerName); Assert.IsNotNull(qc); // Check that the count is zero. long beginQueueCount = await qc.CountAsync().ConfigureAwait(false); long beginItemCount = await qc.CountAsync(QueueType.ItemQueue).ConfigureAwait(false); // Add three items to the queue. Item[] items = new Item[] { i1, i2, i3 }; var enqueueResult = await qc.EnqueueAsync(items).ConfigureAwait(false); Assert.IsNotNull(enqueueResult); foreach (QueueItem <Item> item in enqueueResult) { var deletedItem = await qc.DeleteItemAsync(item.Key, CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(item.Key, deletedItem.Key); Assert.AreEqual(item.DequeueCount, deletedItem.DequeueCount); Assert.AreEqual(item.EnqueueTime, deletedItem.EnqueueTime); Assert.AreEqual(item.ExpirationTime, deletedItem.ExpirationTime); Assert.IsTrue(item.Item.Equals(deletedItem.Item)); Assert.AreEqual(item.LeasedUntil, deletedItem.LeasedUntil); Assert.AreEqual(item.LeaseDuration, deletedItem.LeaseDuration); Assert.AreEqual(item.Queue, deletedItem.Queue); } Assert.AreEqual(beginQueueCount + items.Count(), await qc.CountAsync().ConfigureAwait(false)); Assert.AreEqual(beginItemCount, await qc.CountAsync(QueueType.ItemQueue).ConfigureAwait(false)); }
/// <summary> /// Read test thread procedure. /// </summary> /// <param name="data">AutoResetEvent instance to signal when complete.</param> internal void ReadThreadProc(object data) { AutoResetEvent evt = (AutoResetEvent)data; // Create a queue client. HttpQueueClient <T> qc = new HttpQueueClient <T>(new Uri(serviceUri), c_ListenerName); // Check that cancellation isn't requested and we are not at the requested count. while ((_requestCount < _maxRequests) && (false == _cancellationToken.IsCancellationRequested)) { try { Stopwatch swCall = Stopwatch.StartNew(); IEnumerable <QueueItem <T> > items = qc.DequeueAsync(_batchSize, cancellationToken: _cancellationToken).GetAwaiter().GetResult(); swCall.Stop(); // Track the call time. TrackCallData(swCall.ElapsedMilliseconds); // Build the list of leases to release. List <PopReceipt> keys = new List <PopReceipt>(_batchSize); foreach (QueueItem <T> item in items) { keys.Add(item.Key); } // Were any items returned? if (0 == keys.Count) { Interlocked.Increment(ref _notFoundOrConflict); } else // Release the held leases for each item. { IEnumerable <bool> results = qc.ReleaseLeaseAsync(keys, _cancellationToken).GetAwaiter().GetResult(); foreach (bool bResult in results) { if (bResult) { Interlocked.Increment(ref _successfulRequests); } } } } catch (TimeoutException) { Interlocked.Increment(ref _timedoutRequests); } catch (Exception ex) { Console.WriteLine($"ReadTest.ReadThreadProc exception {ex.GetType().Name}: {ex.Message}, {ex.StackTrace}"); } // Increment the request count. Interlocked.Increment(ref _requestCount); if (0 == _requestCount % 100) { Console.Write("\rTotal Read: {0,-15:N0}", _requestCount); } } // Signal this thread has completed. evt.Set(); }
public async Task GetPriorityCount_Test() { HttpQueueClient <string> qc = new HttpQueueClient <string>(new Uri(serviceUri), c_ListenerName); Assert.IsNotNull(qc); int count = await qc.PriorityCountAsync().ConfigureAwait(false); Assert.AreEqual(5, count); }
/// <summary> /// Releases the lease for each of the items in the list. /// </summary> /// <param name="client">QueueClient instance.</param> /// <param name="items">Items to release leases.</param> private async Task ReleaseLeasesAsync(HttpQueueClient <Item> client, IEnumerable <QueueItem <Item> > items) { // Release the lease. var results = await client.ReleaseLeaseAsync(KeyArrayFromItems(items), CancellationToken.None).ConfigureAwait(false); foreach (bool b in results) { Assert.IsTrue(b); } }
/// <summary> /// ReadTest Constructor. /// </summary> /// <param name="batchSize">Size of the batch to process.</param> /// <param name="threads">Number of threads to run concurrently.</param> /// <param name="requests">Number of requests to process.</param> /// <param name="outputFileName">Name of the file to output the results to.</param> /// <param name="cancellationToken">CanellationToken instance.</param> public WriteTest(int batchSize, int threads, long requests, string outputFileName, Func <T> create, CancellationToken cancellationToken) : base(threads, requests, outputFileName) { Guard.ArgumentNotNull(create, nameof(create)); _batchSize = batchSize; _createInstance = create; // Create a queue client and retrieve the number of priorities. HttpQueueClient <T> qc = new HttpQueueClient <T>(new Uri(serviceUri), c_ListenerName); _priorityCount = qc.PriorityCountAsync().GetAwaiter().GetResult(); }
/// <summary> /// Write test thread procedure. /// </summary> /// <param name="data">AutoResetEvent instance to signal when complete.</param> internal void WriteThreadProc(object data) { AutoResetEvent evt = (AutoResetEvent)data; HttpQueueClient <T> qc = new HttpQueueClient <T>(new Uri(serviceUri), c_ListenerName); // Check that cancellation isn't requested and we are not at the requested count. while ((_requestCount < _maxRequests) && (false == _cancellationToken.IsCancellationRequested)) { try { // Create a list of items to add. List <T> addItems = new List <T>(); for (int i = 0; i < _batchSize; i++) { addItems.Add(_createInstance()); } int queue = RandomThreadSafe.Instance.Next(0, _priorityCount); //Console.WriteLine($"WriteTest.WriteThreadProc Thread:{System.Threading.Thread.CurrentThread.ManagedThreadId}, Queue{queue}"); Stopwatch swCall = Stopwatch.StartNew(); //TODO: change to specify queue. Note can also specify lease time and expiration here as well! //IEnumerable<QueueItem<T>> items = qc.EnqueueAsync(addItems, cancellationToken: _cancellationToken).GetAwaiter().GetResult(); IEnumerable <QueueItem <T> > items = qc.EnqueueAsync(addItems, queue, cancellationToken: _cancellationToken).GetAwaiter().GetResult(); swCall.Stop(); // Add the number of items added successfully. Interlocked.Add(ref _successfulRequests, items.Count()); // Track the call time. TrackCallData(swCall.ElapsedMilliseconds); } catch (TimeoutException) { Interlocked.Increment(ref _timedoutRequests); } catch (HttpResponseException ex) { Console.WriteLine($"WriteTest.WriteThreadProc exception {ex.GetType().Name}: {ex.Message}, {ex.Response.StatusCode}, {ex.Response.ReasonPhrase}, {ex.Response.RequestMessage.RequestUri.AbsoluteUri}, {ex.StackTrace}"); } catch (Exception ex) { Console.WriteLine($"WriteTest.WriteThreadProc exception {ex.GetType().Name}: {ex.Message}, {ex.StackTrace}"); } // Increment the request count. Interlocked.Increment(ref _requestCount); if (0 == _requestCount % 100) { Console.Write("\rTotal written: {0,-15:N0}", _requestCount); } } // Signal this thread has completed. evt.Set(); }
/// <summary> /// Gets the next item that would be returned from the queue. /// </summary> private static void Peek() { HttpQueueClient <Item> qc = new HttpQueueClient <Item>(_serviceUri, c_ListenerName); var item = qc.PeekItemAsync().GetAwaiter().GetResult(); if (QueueItem <Item> .Default == item) { Console.WriteLine("No items to retrieve."); } else { Console.WriteLine($"{item.ToString()}"); } }
/// <summary> /// ThreadProc for testing multiple thread construction. /// </summary> /// <param name="data">Thread offset count.</param> private void ThreadProc(object data) { int offset = (int)data; Interlocked.Increment(ref threadStartCount); HttpQueueClient <Item> qc = new HttpQueueClient <Item>(new Uri(serviceUri), c_ListenerName); if (null != qc) { Console.WriteLine($"Thread {offset}[{Thread.CurrentThread.ManagedThreadId}] completed."); } Interlocked.Increment(ref threadCompleteCount); }
public async Task Dequeue_Test() { HttpQueueClient <Item> qc = new HttpQueueClient <Item>(new Uri(serviceUri), c_ListenerName); Assert.IsNotNull(qc); Item[] items = new Item[] { i1, i2, i3 }; // Clear the existing queue. await ClearQueueAsync(); // Check that the count is zero. long count = await qc.CountAsync().ConfigureAwait(false); Assert.AreEqual(0, count); // Enqueue the items. var response = await qc.EnqueueAsync(items).ConfigureAwait(false); Assert.IsNotNull(response); // Check that all items were queued. count = await qc.CountAsync().ConfigureAwait(false); Assert.AreEqual(items.Length, count); // Dequeue items. response = await qc.DequeueAsync().ConfigureAwait(false); Assert.IsNotNull(response); QueueItem <Item> item = response.First(); Assert.IsNotNull(item); Assert.IsTrue(i1.Equals(item.Item)); Assert.AreEqual(0, item.Queue); Assert.AreEqual(1, item.DequeueCount); Assert.AreEqual(TimeSpan.FromMinutes(10), item.LeaseDuration); Assert.AreEqual(DateTimeOffset.MaxValue, item.ExpirationTime); // Get the queue count at the end. The queue must be started empty for this test to pass correctly. Assert.AreEqual(3, await qc.CountAsync(QueueType.ItemQueue).ConfigureAwait(false)); Assert.AreEqual(1, await qc.CountAsync(QueueType.LeaseQueue).ConfigureAwait(false)); Assert.AreEqual(2, await qc.CountAsync(QueueType.AllQueues).ConfigureAwait(false)); }
public void Constructor_Test() { HttpQueueClient <string> qc = new HttpQueueClient <string>(new Uri(serviceUri), c_ListenerName); Assert.IsNotNull(qc); threadStartCount = 0; threadCompleteCount = 0; // Check it with multiple threads doing the initialization. Thread[] threads = new Thread[numThreads]; for (int i = 0; i < threads.Length; i++) { threads[i] = new Thread(ThreadProc); threads[i].Start(i); } Thread.Sleep(5000); Assert.AreEqual(threadStartCount, threadCompleteCount); }
/// <summary> /// Gets a set of items. /// </summary> private static void GetItems() { int temp; Console.WriteLine($"Enter the partition number to retrieve items from. Current value is {_itemPartition}: "); _itemPartition = int.TryParse(Console.ReadLine(), out temp) ? temp : _itemPartition; Console.WriteLine($"Enter the number of items to retrieve. Current value is {_itemCount}: "); _itemCount = int.TryParse(Console.ReadLine(), out temp) ? temp : _itemCount; Console.WriteLine($"Enter the number of items to skip. Current value is {_itemSkip}: "); _itemSkip = int.TryParse(Console.ReadLine(), out temp) ? temp : _itemSkip; HttpQueueClient <Item> qc = new HttpQueueClient <Item>(_serviceUri, c_ListenerName); var items = qc.GetItemsAsync(_itemPartition, _itemCount, _itemSkip).GetAwaiter().GetResult(); if (null != items) { foreach (var item in items) { Console.WriteLine(item.ToString()); } } }
public async Task ClearQueue_Test() { HttpQueueClient <Item> qc = new HttpQueueClient <Item>(new Uri(serviceUri), c_ListenerName); long queueCount = await qc.CountAsync().ConfigureAwait(false); long itemCount = await qc.CountAsync(QueueType.ItemQueue).ConfigureAwait(false); // If there are no items, add some so there is something to clear. if (0 == (queueCount + itemCount)) { Item[] items = new Item[] { i1, i2, i3 }; var response = await qc.EnqueueAsync(items).ConfigureAwait(false); Assert.IsNotNull(response); queueCount = items.Length; } await ClearQueueAsync(); Assert.AreEqual(0, await qc.CountAsync().ConfigureAwait(false)); Assert.AreEqual(0, await qc.CountAsync(QueueType.ItemQueue).ConfigureAwait(false)); }
public async Task Enqueue_Test() { HttpQueueClient <Item> qc = new HttpQueueClient <Item>(new Uri(serviceUri), c_ListenerName); Assert.IsNotNull(qc); Item[] items = new Item[] { i1, i2, i3 }; // Clear the existing queue. await ClearQueueAsync(); // Get the queue count at the beginning. long beginCount = await qc.CountAsync().ConfigureAwait(false); // Enqueue a single item. var response = await qc.EnqueueAsync(items).ConfigureAwait(false); Assert.IsNotNull(response); List <QueueItem <Item> > responseItems = new List <QueueItem <Item> >(response); Assert.AreEqual(items.Length, responseItems.Count); Assert.AreEqual(0, responseItems[0].Queue); Assert.AreEqual(0, responseItems[0].DequeueCount); Assert.AreEqual(TimeSpan.FromMinutes(10), responseItems[0].LeaseDuration); Assert.AreEqual(DateTimeOffset.MaxValue, responseItems[0].ExpirationTime); for (int i = 0; i < items.Length; i++) { Assert.IsTrue(items[i].Equals(responseItems[i].Item)); } // Get the queue count at the end. long endCount = await qc.CountAsync(); Assert.AreEqual(beginCount + items.Length, endCount); }
/// <summary> /// Empties a partition of all items. /// </summary> private static async Task EmptyAsync() { int partition = 0, temp; Console.WriteLine($"Enter the partition number to retrieve items from. Current value is {partition}: "); partition = int.TryParse(Console.ReadLine(), out temp) ? temp : partition; HttpQueueClient <Item> qc = new HttpQueueClient <Item>(_serviceUri, c_ListenerName); int count = 0; do { var items = await qc.GetItemsAsync(_itemPartition, _itemCount, _itemSkip); foreach (var item in items) { count++; var i = await qc.DeleteItemAsync(item.Key, CancellationToken.None); } } while (count > 0); Console.WriteLine("Items removed from partition."); }
public async Task Component_Test() { Item i1 = CreateRandomInstance(); Item i2 = CreateRandomInstance(); Item i3 = CreateRandomInstance(); HttpQueueClient <Item> qc = new HttpQueueClient <Item>(new Uri(serviceUri), c_ListenerName); Assert.IsNotNull(qc); long allBase = await qc.CountAsync(QueueType.AllQueues).ConfigureAwait(false); long leaseBase = await qc.CountAsync(QueueType.LeaseQueue).ConfigureAwait(false); long expiredBase = await qc.CountAsync(QueueType.ExpiredQueue).ConfigureAwait(false); long itemsBase = await qc.CountAsync(QueueType.ItemQueue).ConfigureAwait(false); // Add an single item to the queue and validate the count has changed. await qc.EnqueueAsync(new[] { i1 }, QueueType.FirstQueue, TimeSpan.FromSeconds(30), TimeSpan.FromMinutes(5), CancellationToken.None).ConfigureAwait(false); long count = await qc.CountAsync(QueueType.AllQueues).ConfigureAwait(false); Assert.AreEqual(1 + allBase, count); count = await qc.CountAsync(QueueType.ItemQueue).ConfigureAwait(false); Assert.AreEqual(1 + itemsBase, count); // Add a batch of items to the queue and validate the count has changed. await qc.EnqueueAsync(new[] { i1, i2, i3 }, QueueType.FirstQueue, TimeSpan.FromSeconds(30), TimeSpan.FromMinutes(5), CancellationToken.None).ConfigureAwait(false); count = await qc.CountAsync(QueueType.AllQueues, CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(4 + allBase, count); count = await qc.CountAsync(QueueType.ItemQueue, CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(4 + itemsBase, count); // Enqueue a single item to the second queue and check the counts. await qc.EnqueueAsync(new[] { i1 }, 1, TimeSpan.FromSeconds(30), TimeSpan.FromMinutes(5), CancellationToken.None).ConfigureAwait(false); count = await qc.CountAsync(QueueType.AllQueues).ConfigureAwait(false); Assert.AreEqual(5 + allBase, count); count = await qc.CountAsync(QueueType.ItemQueue, CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(5 + itemsBase, count); // Dequeue a single item from the first queue and check the counts. IEnumerable <QueueItem <Item> > items = await qc.DequeueAsync(1, QueueType.FirstQueue, QueueType.LastQueue, CancellationToken.None).ConfigureAwait(false); count = await qc.CountAsync(QueueType.AllQueues, CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(4 + allBase, count); count = await qc.CountAsync(QueueType.LeaseQueue, CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(1 + leaseBase, count); count = await qc.CountAsync(QueueType.ItemQueue, CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(5 + itemsBase, count); // Peek the next item. QueueItem <Item> item = await qc.PeekItemAsync(QueueType.FirstQueue, QueueType.LastQueue, CancellationToken.None).ConfigureAwait(false); Assert.IsNotNull(item); Console.WriteLine(item.ToString()); // Get a list of the items. items = await qc.GetItemsAsync(0, 1000, 0).ConfigureAwait(false); foreach (var qi in items) { Console.WriteLine(qi.Item.Name); } // Wait for the lease to expire and check the counts. Thread.Sleep(TimeSpan.FromSeconds(125)); count = await qc.CountAsync(QueueType.AllQueues, CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(4 + allBase, count); count = await qc.CountAsync(QueueType.LeaseQueue, CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(0 + leaseBase, count); count = await qc.CountAsync(QueueType.ItemQueue, CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(5 + itemsBase, count); // Dequeue again, release the lease and check the counts. items = await qc.DequeueAsync(1, QueueType.FirstQueue, QueueType.LastQueue, CancellationToken.None).ConfigureAwait(false); await ReleaseLeasesAsync(qc, items); count = await qc.CountAsync(QueueType.AllQueues, CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(4 + allBase, count); count = await qc.CountAsync(QueueType.LeaseQueue, CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(0 + leaseBase, count); count = await qc.CountAsync(QueueType.ItemQueue, CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(4 + itemsBase, count); // Dequeue two, release the lease and check the counts. items = await qc.DequeueAsync(2, QueueType.FirstQueue, QueueType.LastQueue, CancellationToken.None).ConfigureAwait(false); await ReleaseLeasesAsync(qc, items); count = await qc.CountAsync(QueueType.AllQueues, CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(2 + allBase, count); count = await qc.CountAsync(QueueType.LeaseQueue, CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(0 + leaseBase, count); count = await qc.CountAsync(QueueType.ItemQueue, CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(2 + itemsBase, count); // Dequeue two, release the lease and check the counts. items = await qc.DequeueAsync(2, QueueType.FirstQueue, QueueType.LastQueue, CancellationToken.None).ConfigureAwait(false); await ReleaseLeasesAsync(qc, items); count = await qc.CountAsync(QueueType.AllQueues, CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(0 + allBase, count); count = await qc.CountAsync(QueueType.LeaseQueue, CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(0 + leaseBase, count); count = await qc.CountAsync(QueueType.ItemQueue, CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(0 + itemsBase, count); }
public ProductController(HttpQueueClient httpQueueClient) { _httpQueueClient = httpQueueClient; }