protected void DoWork(QueueEntry <SimpleWorkItem> w, CountdownEvent latch, WorkInfo info) { Trace.WriteLine(String.Format("Starting: {0}", w.Value.Id)); Assert.Equal("Hello", w.Value.Data); try { // randomly complete, abandon or blowup. if (RandomData.GetBool()) { Trace.WriteLine(String.Format("Completing: {0}", w.Value.Id)); w.Complete(); info.IncrementCompletedCount(); } else if (RandomData.GetBool()) { Trace.WriteLine(String.Format("Abandoning: {0}", w.Value.Id)); w.Abandon(); info.IncrementAbandonCount(); } else { Trace.WriteLine(String.Format("Erroring: {0}", w.Value.Id)); info.IncrementErrorCount(); throw new ApplicationException(); } } finally { latch.Signal(); } }
public virtual async Task CanHaveMultipleQueueInstancesAsync() { var queue = GetQueue(retries: 0, retryDelay: TimeSpan.Zero); if (queue == null) { return; } try { await queue.DeleteQueueAsync(); await AssertEmptyQueueAsync(queue); const int workItemCount = 500; const int workerCount = 3; var countdown = new AsyncCountdownEvent(workItemCount); var info = new WorkInfo(); var workers = new List <IQueue <SimpleWorkItem> > { queue }; try { for (int i = 0; i < workerCount; i++) { var q = GetQueue(retries: 0, retryDelay: TimeSpan.Zero); _logger.Trace("Queue Id: {0}, I: {1}", q.QueueId, i); await q.StartWorkingAsync(w => DoWorkAsync(w, countdown, info)); workers.Add(q); } await Run.InParallelAsync(workItemCount, async i => { string id = await queue.EnqueueAsync(new SimpleWorkItem { Data = "Hello", Id = i }); _logger.Trace("Enqueued Index: {0} Id: {1}", i, id); }); await countdown.WaitAsync(); await SystemClock.SleepAsync(50); _logger.Trace("Completed: {0} Abandoned: {1} Error: {2}", info.CompletedCount, info.AbandonCount, info.ErrorCount); _logger.Info("Work Info Stats: Completed: {completed} Abandoned: {abandoned} Error: {errors}", info.CompletedCount, info.AbandonCount, info.ErrorCount); Assert.Equal(workItemCount, info.CompletedCount + info.AbandonCount + info.ErrorCount); // In memory queue doesn't share state. if (queue.GetType() == typeof(InMemoryQueue <SimpleWorkItem>)) { var stats = await queue.GetQueueStatsAsync(); Assert.Equal(0, stats.Working); Assert.Equal(0, stats.Timeouts); Assert.Equal(workItemCount, stats.Enqueued); Assert.Equal(workItemCount, stats.Dequeued); Assert.Equal(info.CompletedCount, stats.Completed); Assert.Equal(info.ErrorCount, stats.Errors); Assert.Equal(info.AbandonCount, stats.Abandoned - info.ErrorCount); Assert.Equal(info.AbandonCount + stats.Errors, stats.Deadletter); } else { var workerStats = new List <QueueStats>(); for (int i = 0; i < workers.Count; i++) { var stats = await workers[i].GetQueueStatsAsync(); _logger.Info("Worker#{i} Working: {working} Completed: {completed} Abandoned: {abandoned} Error: {errors} Deadletter: {deadletter}", i, stats.Working, stats.Completed, stats.Abandoned, stats.Errors, stats.Deadletter); workerStats.Add(stats); } Assert.Equal(info.CompletedCount, workerStats.Sum(s => s.Completed)); Assert.Equal(info.ErrorCount, workerStats.Sum(s => s.Errors)); Assert.Equal(info.AbandonCount, workerStats.Sum(s => s.Abandoned) - info.ErrorCount); Assert.Equal(info.AbandonCount + workerStats.Sum(s => s.Errors), (workerStats.LastOrDefault()?.Deadletter ?? 0)); } } finally { foreach (var q in workers) { await CleanupQueueAsync(q); } } } finally { await CleanupQueueAsync(queue); } }
protected async Task DoWorkAsync(IQueueEntry <SimpleWorkItem> w, AsyncCountdownEvent countdown, WorkInfo info) { _logger.Trace($"Starting: {w.Value.Id}"); Assert.Equal("Hello", w.Value.Data); try { // randomly complete, abandon or blowup. if (RandomData.GetBool()) { _logger.Trace($"Completing: {w.Value.Id}"); await w.CompleteAsync(); info.IncrementCompletedCount(); } else if (RandomData.GetBool()) { _logger.Trace($"Abandoning: {w.Value.Id}"); await w.AbandonAsync(); info.IncrementAbandonCount(); } else { _logger.Trace($"Erroring: {w.Value.Id}"); info.IncrementErrorCount(); throw new Exception(); } } finally { _logger.Trace($"Signal {countdown.CurrentCount}"); countdown.Signal(); } }
protected async Task CanHaveMultipleQueueInstancesWithLockingImplAsync(CacheLockProvider distributedLock) { var queue = GetQueue(retries: 0, retryDelay: TimeSpan.Zero); if (queue == null) { return; } try { await queue.DeleteQueueAsync(); await AssertEmptyQueueAsync(queue); const int workItemCount = 16; const int workerCount = 4; var countdown = new AsyncCountdownEvent(workItemCount); var info = new WorkInfo(); var workers = new List <IQueue <SimpleWorkItem> > { queue }; try { for (int i = 0; i < workerCount; i++) { var q = GetQueue(retries: 0, retryDelay: TimeSpan.Zero); int instanceCount = i; await q.StartWorkingAsync(async w => { _logger.Info("[{0}] Acquiring distributed lock in work item: {1}", instanceCount, w.Id); var l = await distributedLock.AcquireAsync("test"); Assert.NotNull(l); _logger.Info("[{0}] Acquired distributed lock: {1}", instanceCount, w.Id); SystemClock.Sleep(TimeSpan.FromMilliseconds(50)); await l.ReleaseAsync(); _logger.Info("[{0}] Released distributed lock: {1}", instanceCount, w.Id); await w.CompleteAsync(); info.IncrementCompletedCount(); countdown.Signal(); _logger.Info("[{0}] Signaled countdown: {1}", instanceCount, w.Id); }); workers.Add(q); } await Run.InParallelAsync(workItemCount, async i => { string id = await queue.EnqueueAsync(new SimpleWorkItem { Data = "Hello", Id = i }); _logger.Trace("Enqueued Index: {0} Id: {1}", i, id); }); await countdown.WaitAsync(TimeSpan.FromSeconds(5).ToCancellationToken()); await SystemClock.SleepAsync(50); _logger.Trace("Completed: {0} Abandoned: {1} Error: {2}", info.CompletedCount, info.AbandonCount, info.ErrorCount); _logger.Info("Work Info Stats: Completed: {completed} Abandoned: {abandoned} Error: {errors}", info.CompletedCount, info.AbandonCount, info.ErrorCount); Assert.Equal(workItemCount, info.CompletedCount + info.AbandonCount + info.ErrorCount); // In memory queue doesn't share state. if (queue.GetType() == typeof(InMemoryQueue <SimpleWorkItem>)) { var stats = await queue.GetQueueStatsAsync(); Assert.Equal(info.CompletedCount, stats.Completed); } else { var workerStats = new List <QueueStats>(); for (int i = 0; i < workers.Count; i++) { var stats = await workers[i].GetQueueStatsAsync(); _logger.Info("Worker#{i} Working: {working} Completed: {completed} Abandoned: {abandoned} Error: {errors} Deadletter: {deadletter}", i, stats.Working, stats.Completed, stats.Abandoned, stats.Errors, stats.Deadletter); workerStats.Add(stats); } Assert.Equal(info.CompletedCount, workerStats.Sum(s => s.Completed)); } } finally { foreach (var q in workers) { await CleanupQueueAsync(q); } } } finally { await CleanupQueueAsync(queue); } }
public virtual async Task CanHaveMultipleQueueInstances() { var queue = GetQueue(retries: 0, retryDelay: TimeSpan.Zero); if (queue == null) { return; } using (queue) { Logger.Trace().Message("Queue Id: {0}", queue.QueueId).Write(); await queue.DeleteQueueAsync(); const int workItemCount = 50; const int workerCount = 3; var countdown = new AsyncCountdownEvent(workItemCount); var info = new WorkInfo(); var workers = new List <IQueue <SimpleWorkItem> > { queue }; for (int i = 0; i < workerCount; i++) { var q = GetQueue(retries: 0, retryDelay: TimeSpan.Zero); Logger.Trace().Message("Queue Id: {0}, I: {1}", q.QueueId, i).Write(); q.StartWorking(async w => await DoWorkAsync(w, countdown, info)); workers.Add(q); } await Run.InParallel(workItemCount, async i => { var id = await queue.EnqueueAsync(new SimpleWorkItem { Data = "Hello", Id = i }); Logger.Trace().Message("Enqueued Index: {0} Id: {1}", i, id).Write(); }); await countdown.WaitAsync(); await Task.Delay(50); Logger.Trace().Message("Completed: {0} Abandoned: {1} Error: {2}", info.CompletedCount, info.AbandonCount, info.ErrorCount).Write(); Logger.Info().Message($"Work Info Stats: Completed: {info.CompletedCount} Abandoned: {info.AbandonCount} Error: {info.ErrorCount}").Write(); Assert.Equal(workItemCount, info.CompletedCount + info.AbandonCount + info.ErrorCount); // In memory queue doesn't share state. if (queue.GetType() == typeof(InMemoryQueue <SimpleWorkItem>)) { var stats = await queue.GetQueueStatsAsync(); Assert.Equal(0, stats.Working); Assert.Equal(0, stats.Timeouts); Assert.Equal(workItemCount, stats.Enqueued); Assert.Equal(workItemCount, stats.Dequeued); Assert.Equal(info.CompletedCount, stats.Completed); Assert.Equal(info.ErrorCount, stats.Errors); Assert.Equal(info.AbandonCount, stats.Abandoned - info.ErrorCount); Assert.Equal(info.AbandonCount + stats.Errors, stats.Deadletter); } else { var workerStats = new List <QueueStats>(); for (int i = 0; i < workers.Count; i++) { var stats = await workers[i].GetQueueStatsAsync(); Logger.Info().Message($"Worker#{i} Working: {stats.Working} Completed: {stats.Completed} Abandoned: {stats.Abandoned} Error: {stats.Errors} Deadletter: {stats.Deadletter}").Write(); workerStats.Add(stats); } Assert.Equal(info.CompletedCount, workerStats.Sum(s => s.Completed)); Assert.Equal(info.ErrorCount, workerStats.Sum(s => s.Errors)); Assert.Equal(info.AbandonCount, workerStats.Sum(s => s.Abandoned) - info.ErrorCount); Assert.Equal(info.AbandonCount + workerStats.Sum(s => s.Errors), (workerStats.LastOrDefault()?.Deadletter ?? 0)); } workers.ForEach(w => w.Dispose()); } }
protected void DoWork(QueueEntry<SimpleWorkItem> w, CountdownEvent latch, WorkInfo info) { Trace.WriteLine($"Starting: {w.Value.Id}"); Assert.Equal("Hello", w.Value.Data); try { // randomly complete, abandon or blowup. if (RandomData.GetBool()) { Trace.WriteLine($"Completing: {w.Value.Id}"); w.Complete(); info.IncrementCompletedCount(); } else if (RandomData.GetBool()) { Trace.WriteLine($"Abandoning: {w.Value.Id}"); w.Abandon(); info.IncrementAbandonCount(); } else { Trace.WriteLine($"Erroring: {w.Value.Id}"); info.IncrementErrorCount(); throw new ApplicationException(); } } finally { Trace.WriteLine($"Signal {latch.CurrentCount}"); latch.Signal(); } }
public virtual void CanHaveMultipleQueueInstances() { var queue = GetQueue(retries: 0, retryDelay: TimeSpan.Zero); if (queue == null) return; using (queue) { Logger.Trace().Message("Queue Id: {0}", queue.QueueId).Write(); queue.DeleteQueue(); const int workItemCount = 10; const int workerCount = 3; var latch = new CountdownEvent(workItemCount); var info = new WorkInfo(); var workers = new List<IQueue<SimpleWorkItem>> {queue}; for (int i = 0; i < workerCount; i++) { var q = GetQueue(retries: 0, retryDelay: TimeSpan.Zero); Logger.Trace().Message("Queue Id: {0}, I: {1}", q.QueueId, i).Write(); q.StartWorking(w => DoWork(w, latch, info)); workers.Add(q); } Parallel.For(0, workItemCount, i => { var id = queue.Enqueue(new SimpleWorkItem { Data = "Hello", Id = i }); Logger.Trace().Message("Enqueued Index: {0} Id: {1}", i, id).Write(); }); Assert.True(latch.Wait(TimeSpan.FromSeconds(20))); Thread.Sleep(100); // needed to make sure the worker error handler has time to finish Logger.Trace().Message("Completed: {0} Abandoned: {1} Error: {2}", info.CompletedCount, info.AbandonCount, info.ErrorCount).Write(); for (int i = 0; i < workers.Count; i++) { var workerStats = workers[i].GetQueueStats(); Trace.WriteLine($"Worker#{i} Completed: {workerStats.Completed} Abandoned: {workerStats.Abandoned} Error: {workerStats.Errors}"); } Assert.Equal(workItemCount, info.CompletedCount + info.AbandonCount + info.ErrorCount); var stats = queue.GetQueueStats(); // In memory queue doesn't share state. if (queue.GetType() == typeof (InMemoryQueue<SimpleWorkItem>)) { Assert.Equal(info.CompletedCount, stats.Completed); Assert.Equal(info.AbandonCount, stats.Abandoned - stats.Errors); Assert.Equal(info.ErrorCount, stats.Errors); } else { Assert.Equal(info.CompletedCount, workers.Sum(q => q.GetQueueStats().Completed)); Assert.Equal(info.AbandonCount, workers.Sum(q => q.GetQueueStats().Abandoned) - workers.Sum(q => q.GetQueueStats().Errors)); Assert.Equal(info.ErrorCount, workers.Sum(q => q.GetQueueStats().Errors)); } workers.ForEach(w => w.Dispose()); } }
protected async Task DoWorkAsync(QueueEntry<SimpleWorkItem> w, AsyncCountdownEvent countdown, WorkInfo info) { Trace.WriteLine($"Starting: {w.Value.Id}"); Assert.Equal("Hello", w.Value.Data); try { // randomly complete, abandon or blowup. if (RandomData.GetBool()) { Trace.WriteLine($"Completing: {w.Value.Id}"); await w.CompleteAsync(); info.IncrementCompletedCount(); } else if (RandomData.GetBool()) { Trace.WriteLine($"Abandoning: {w.Value.Id}"); await w.AbandonAsync(); info.IncrementAbandonCount(); } else { Trace.WriteLine($"Erroring: {w.Value.Id}"); info.IncrementErrorCount(); throw new ApplicationException(); } } finally { Trace.WriteLine($"Signal {countdown.CurrentCount}"); countdown.Signal(); } }
public virtual async Task CanHaveMultipleQueueInstances() { var queue = GetQueue(retries: 0, retryDelay: TimeSpan.Zero); if (queue == null) return; using (queue) { Logger.Trace().Message("Queue Id: {0}", queue.QueueId).Write(); await queue.DeleteQueueAsync(); const int workItemCount = 50; const int workerCount = 3; var countdown = new AsyncCountdownEvent(workItemCount); var info = new WorkInfo(); var workers = new List<IQueue<SimpleWorkItem>> {queue}; for (int i = 0; i < workerCount; i++) { var q = GetQueue(retries: 0, retryDelay: TimeSpan.Zero); Logger.Trace().Message("Queue Id: {0}, I: {1}", q.QueueId, i).Write(); q.StartWorking(async w => await DoWorkAsync(w, countdown, info)); workers.Add(q); } await Run.InParallel(workItemCount, async i => { var id = await queue.EnqueueAsync(new SimpleWorkItem { Data = "Hello", Id = i }); Logger.Trace().Message("Enqueued Index: {0} Id: {1}", i, id).Write(); }); await countdown.WaitAsync(); await Task.Delay(50); Logger.Trace().Message("Completed: {0} Abandoned: {1} Error: {2}", info.CompletedCount, info.AbandonCount, info.ErrorCount).Write(); Logger.Info().Message($"Work Info Stats: Completed: {info.CompletedCount} Abandoned: {info.AbandonCount} Error: {info.ErrorCount}").Write(); Assert.Equal(workItemCount, info.CompletedCount + info.AbandonCount + info.ErrorCount); // In memory queue doesn't share state. if (queue.GetType() == typeof (InMemoryQueue<SimpleWorkItem>)) { var stats = await queue.GetQueueStatsAsync(); Assert.Equal(0, stats.Working); Assert.Equal(0, stats.Timeouts); Assert.Equal(workItemCount, stats.Enqueued); Assert.Equal(workItemCount, stats.Dequeued); Assert.Equal(info.CompletedCount, stats.Completed); Assert.Equal(info.ErrorCount, stats.Errors); Assert.Equal(info.AbandonCount, stats.Abandoned - info.ErrorCount); Assert.Equal(info.AbandonCount + stats.Errors, stats.Deadletter); } else { var workerStats = new List<QueueStats>(); for (int i = 0; i < workers.Count; i++) { var stats = await workers[i].GetQueueStatsAsync(); Logger.Info().Message($"Worker#{i} Working: {stats.Working} Completed: {stats.Completed} Abandoned: {stats.Abandoned} Error: {stats.Errors} Deadletter: {stats.Deadletter}").Write(); workerStats.Add(stats); } Assert.Equal(info.CompletedCount, workerStats.Sum(s => s.Completed)); Assert.Equal(info.ErrorCount, workerStats.Sum(s => s.Errors)); Assert.Equal(info.AbandonCount, workerStats.Sum(s => s.Abandoned) - info.ErrorCount); Assert.Equal(info.AbandonCount + workerStats.Sum(s => s.Errors), (workerStats.LastOrDefault()?.Deadletter ?? 0)); } workers.ForEach(w => w.Dispose()); } }
public virtual void CanHaveMultipleQueueInstances() { for (int x = 0; x < 5; x++) { var queue = GetQueue(retries: 0, retryDelay: TimeSpan.Zero); if (queue == null) { return; } using (queue) { Trace.WriteLine(String.Format("Queue Id: {0}", queue.QueueId)); queue.DeleteQueue(); const int workItemCount = 10; const int workerCount = 3; var latch = new CountdownEvent(workItemCount); var info = new WorkInfo(); var workers = new List <IQueue <SimpleWorkItem> > { queue }; for (int i = 0; i < workerCount; i++) { var q = GetQueue(retries: 0, retryDelay: TimeSpan.Zero); Trace.WriteLine(String.Format("Queue Id: {0}", q.QueueId)); q.StartWorking(w => DoWork(w, latch, info)); workers.Add(q); } Parallel.For(0, workItemCount, i => { var id = queue.Enqueue(new SimpleWorkItem { Data = "Hello", Id = i }); Trace.WriteLine(String.Format("Enqueued Index: {0} Id: {1}", i, id)); }); Assert.True(latch.Wait(TimeSpan.FromSeconds(5))); Thread.Sleep(100); // needed to make sure the worker error handler has time to finish Trace.WriteLine(String.Format("Completed: {0} Abandoned: {1} Error: {2}", info.CompletedCount, info.AbandonCount, info.ErrorCount)); for (int i = 0; i < workers.Count; i++) { Trace.WriteLine(String.Format("Worker#{0} Completed: {1} Abandoned: {2} Error: {3}", i, workers[i].CompletedCount, workers[i].AbandonedCount, workers[i].WorkerErrorCount)); } Assert.Equal(workItemCount, info.CompletedCount + info.AbandonCount + info.ErrorCount); // In memory queue doesn't share state. if (queue.GetType() == typeof(InMemoryQueue <SimpleWorkItem>)) { Assert.Equal(info.CompletedCount, queue.CompletedCount); Assert.Equal(info.AbandonCount, queue.AbandonedCount - queue.WorkerErrorCount); Assert.Equal(info.ErrorCount, queue.WorkerErrorCount); } else { Assert.Equal(info.CompletedCount, workers.Sum(q => q.CompletedCount)); Assert.Equal(info.AbandonCount, workers.Sum(q => q.AbandonedCount) - workers.Sum(q => q.WorkerErrorCount)); Assert.Equal(info.ErrorCount, workers.Sum(q => q.WorkerErrorCount)); } workers.ForEach(w => w.Dispose()); } } }