protected void DoWork(QueueEntry <SimpleWorkItem> w, CountdownEvent latch, WorkInfo info)
{
Trace.WriteLine(String.Format("Starting: {0}", w.Value.Id));
Assert.Equal("Hello", w.Value.Data);
try {
// randomly complete, abandon or blowup.
if (RandomData.GetBool())
{
Trace.WriteLine(String.Format("Completing: {0}", w.Value.Id));
w.Complete();
info.IncrementCompletedCount();
}
else if (RandomData.GetBool())
{
Trace.WriteLine(String.Format("Abandoning: {0}", w.Value.Id));
w.Abandon();
info.IncrementAbandonCount();
}
else
{
Trace.WriteLine(String.Format("Erroring: {0}", w.Value.Id));
info.IncrementErrorCount();
throw new ApplicationException();
}
} finally {
latch.Signal();
}
}
public virtual async Task CanHaveMultipleQueueInstancesAsync()
{
var queue = GetQueue(retries: 0, retryDelay: TimeSpan.Zero);
if (queue == null)
{
return;
}
try {
await queue.DeleteQueueAsync();
await AssertEmptyQueueAsync(queue);
const int workItemCount = 500;
const int workerCount = 3;
var countdown = new AsyncCountdownEvent(workItemCount);
var info = new WorkInfo();
var workers = new List <IQueue <SimpleWorkItem> > {
queue
};
try {
for (int i = 0; i < workerCount; i++)
{
var q = GetQueue(retries: 0, retryDelay: TimeSpan.Zero);
_logger.Trace("Queue Id: {0}, I: {1}", q.QueueId, i);
await q.StartWorkingAsync(w => DoWorkAsync(w, countdown, info));
workers.Add(q);
}
await Run.InParallelAsync(workItemCount, async i => {
string id = await queue.EnqueueAsync(new SimpleWorkItem {
Data = "Hello",
Id = i
});
_logger.Trace("Enqueued Index: {0} Id: {1}", i, id);
});
await countdown.WaitAsync();
await SystemClock.SleepAsync(50);
_logger.Trace("Completed: {0} Abandoned: {1} Error: {2}",
info.CompletedCount,
info.AbandonCount,
info.ErrorCount);
_logger.Info("Work Info Stats: Completed: {completed} Abandoned: {abandoned} Error: {errors}", info.CompletedCount, info.AbandonCount, info.ErrorCount);
Assert.Equal(workItemCount, info.CompletedCount + info.AbandonCount + info.ErrorCount);
// In memory queue doesn't share state.
if (queue.GetType() == typeof(InMemoryQueue <SimpleWorkItem>))
{
var stats = await queue.GetQueueStatsAsync();
Assert.Equal(0, stats.Working);
Assert.Equal(0, stats.Timeouts);
Assert.Equal(workItemCount, stats.Enqueued);
Assert.Equal(workItemCount, stats.Dequeued);
Assert.Equal(info.CompletedCount, stats.Completed);
Assert.Equal(info.ErrorCount, stats.Errors);
Assert.Equal(info.AbandonCount, stats.Abandoned - info.ErrorCount);
Assert.Equal(info.AbandonCount + stats.Errors, stats.Deadletter);
}
else
{
var workerStats = new List <QueueStats>();
for (int i = 0; i < workers.Count; i++)
{
var stats = await workers[i].GetQueueStatsAsync();
_logger.Info("Worker#{i} Working: {working} Completed: {completed} Abandoned: {abandoned} Error: {errors} Deadletter: {deadletter}", i, stats.Working, stats.Completed, stats.Abandoned, stats.Errors, stats.Deadletter);
workerStats.Add(stats);
}
Assert.Equal(info.CompletedCount, workerStats.Sum(s => s.Completed));
Assert.Equal(info.ErrorCount, workerStats.Sum(s => s.Errors));
Assert.Equal(info.AbandonCount, workerStats.Sum(s => s.Abandoned) - info.ErrorCount);
Assert.Equal(info.AbandonCount + workerStats.Sum(s => s.Errors), (workerStats.LastOrDefault()?.Deadletter ?? 0));
}
} finally {
foreach (var q in workers)
{
await CleanupQueueAsync(q);
}
}
} finally {
await CleanupQueueAsync(queue);
}
}
protected async Task DoWorkAsync(IQueueEntry <SimpleWorkItem> w, AsyncCountdownEvent countdown, WorkInfo info)
{
_logger.Trace($"Starting: {w.Value.Id}");
Assert.Equal("Hello", w.Value.Data);
try {
// randomly complete, abandon or blowup.
if (RandomData.GetBool())
{
_logger.Trace($"Completing: {w.Value.Id}");
await w.CompleteAsync();
info.IncrementCompletedCount();
}
else if (RandomData.GetBool())
{
_logger.Trace($"Abandoning: {w.Value.Id}");
await w.AbandonAsync();
info.IncrementAbandonCount();
}
else
{
_logger.Trace($"Erroring: {w.Value.Id}");
info.IncrementErrorCount();
throw new Exception();
}
} finally {
_logger.Trace($"Signal {countdown.CurrentCount}");
countdown.Signal();
}
}
protected async Task CanHaveMultipleQueueInstancesWithLockingImplAsync(CacheLockProvider distributedLock)
{
var queue = GetQueue(retries: 0, retryDelay: TimeSpan.Zero);
if (queue == null)
{
return;
}
try {
await queue.DeleteQueueAsync();
await AssertEmptyQueueAsync(queue);
const int workItemCount = 16;
const int workerCount = 4;
var countdown = new AsyncCountdownEvent(workItemCount);
var info = new WorkInfo();
var workers = new List <IQueue <SimpleWorkItem> > {
queue
};
try {
for (int i = 0; i < workerCount; i++)
{
var q = GetQueue(retries: 0, retryDelay: TimeSpan.Zero);
int instanceCount = i;
await q.StartWorkingAsync(async w => {
_logger.Info("[{0}] Acquiring distributed lock in work item: {1}", instanceCount, w.Id);
var l = await distributedLock.AcquireAsync("test");
Assert.NotNull(l);
_logger.Info("[{0}] Acquired distributed lock: {1}", instanceCount, w.Id);
SystemClock.Sleep(TimeSpan.FromMilliseconds(50));
await l.ReleaseAsync();
_logger.Info("[{0}] Released distributed lock: {1}", instanceCount, w.Id);
await w.CompleteAsync();
info.IncrementCompletedCount();
countdown.Signal();
_logger.Info("[{0}] Signaled countdown: {1}", instanceCount, w.Id);
});
workers.Add(q);
}
await Run.InParallelAsync(workItemCount, async i => {
string id = await queue.EnqueueAsync(new SimpleWorkItem {
Data = "Hello",
Id = i
});
_logger.Trace("Enqueued Index: {0} Id: {1}", i, id);
});
await countdown.WaitAsync(TimeSpan.FromSeconds(5).ToCancellationToken());
await SystemClock.SleepAsync(50);
_logger.Trace("Completed: {0} Abandoned: {1} Error: {2}", info.CompletedCount, info.AbandonCount, info.ErrorCount);
_logger.Info("Work Info Stats: Completed: {completed} Abandoned: {abandoned} Error: {errors}", info.CompletedCount, info.AbandonCount, info.ErrorCount);
Assert.Equal(workItemCount, info.CompletedCount + info.AbandonCount + info.ErrorCount);
// In memory queue doesn't share state.
if (queue.GetType() == typeof(InMemoryQueue <SimpleWorkItem>))
{
var stats = await queue.GetQueueStatsAsync();
Assert.Equal(info.CompletedCount, stats.Completed);
}
else
{
var workerStats = new List <QueueStats>();
for (int i = 0; i < workers.Count; i++)
{
var stats = await workers[i].GetQueueStatsAsync();
_logger.Info("Worker#{i} Working: {working} Completed: {completed} Abandoned: {abandoned} Error: {errors} Deadletter: {deadletter}", i, stats.Working, stats.Completed, stats.Abandoned, stats.Errors, stats.Deadletter);
workerStats.Add(stats);
}
Assert.Equal(info.CompletedCount, workerStats.Sum(s => s.Completed));
}
} finally {
foreach (var q in workers)
{
await CleanupQueueAsync(q);
}
}
} finally {
await CleanupQueueAsync(queue);
}
}
public virtual async Task CanHaveMultipleQueueInstances()
{
var queue = GetQueue(retries: 0, retryDelay: TimeSpan.Zero);
if (queue == null)
{
return;
}
using (queue)
{
Logger.Trace().Message("Queue Id: {0}", queue.QueueId).Write();
await queue.DeleteQueueAsync();
const int workItemCount = 50;
const int workerCount = 3;
var countdown = new AsyncCountdownEvent(workItemCount);
var info = new WorkInfo();
var workers = new List <IQueue <SimpleWorkItem> > {
queue
};
for (int i = 0; i < workerCount; i++)
{
var q = GetQueue(retries: 0, retryDelay: TimeSpan.Zero);
Logger.Trace().Message("Queue Id: {0}, I: {1}", q.QueueId, i).Write();
q.StartWorking(async w => await DoWorkAsync(w, countdown, info));
workers.Add(q);
}
await Run.InParallel(workItemCount, async i => {
var id = await queue.EnqueueAsync(new SimpleWorkItem
{
Data = "Hello",
Id = i
});
Logger.Trace().Message("Enqueued Index: {0} Id: {1}", i, id).Write();
});
await countdown.WaitAsync();
await Task.Delay(50);
Logger.Trace().Message("Completed: {0} Abandoned: {1} Error: {2}",
info.CompletedCount,
info.AbandonCount,
info.ErrorCount).Write();
Logger.Info().Message($"Work Info Stats: Completed: {info.CompletedCount} Abandoned: {info.AbandonCount} Error: {info.ErrorCount}").Write();
Assert.Equal(workItemCount, info.CompletedCount + info.AbandonCount + info.ErrorCount);
// In memory queue doesn't share state.
if (queue.GetType() == typeof(InMemoryQueue <SimpleWorkItem>))
{
var stats = await queue.GetQueueStatsAsync();
Assert.Equal(0, stats.Working);
Assert.Equal(0, stats.Timeouts);
Assert.Equal(workItemCount, stats.Enqueued);
Assert.Equal(workItemCount, stats.Dequeued);
Assert.Equal(info.CompletedCount, stats.Completed);
Assert.Equal(info.ErrorCount, stats.Errors);
Assert.Equal(info.AbandonCount, stats.Abandoned - info.ErrorCount);
Assert.Equal(info.AbandonCount + stats.Errors, stats.Deadletter);
}
else
{
var workerStats = new List <QueueStats>();
for (int i = 0; i < workers.Count; i++)
{
var stats = await workers[i].GetQueueStatsAsync();
Logger.Info().Message($"Worker#{i} Working: {stats.Working} Completed: {stats.Completed} Abandoned: {stats.Abandoned} Error: {stats.Errors} Deadletter: {stats.Deadletter}").Write();
workerStats.Add(stats);
}
Assert.Equal(info.CompletedCount, workerStats.Sum(s => s.Completed));
Assert.Equal(info.ErrorCount, workerStats.Sum(s => s.Errors));
Assert.Equal(info.AbandonCount, workerStats.Sum(s => s.Abandoned) - info.ErrorCount);
Assert.Equal(info.AbandonCount + workerStats.Sum(s => s.Errors), (workerStats.LastOrDefault()?.Deadletter ?? 0));
}
workers.ForEach(w => w.Dispose());
}
}
protected void DoWork(QueueEntry<SimpleWorkItem> w, CountdownEvent latch, WorkInfo info)
{
Trace.WriteLine($"Starting: {w.Value.Id}");
Assert.Equal("Hello", w.Value.Data);
try {
// randomly complete, abandon or blowup.
if (RandomData.GetBool()) {
Trace.WriteLine($"Completing: {w.Value.Id}");
w.Complete();
info.IncrementCompletedCount();
} else if (RandomData.GetBool()) {
Trace.WriteLine($"Abandoning: {w.Value.Id}");
w.Abandon();
info.IncrementAbandonCount();
} else {
Trace.WriteLine($"Erroring: {w.Value.Id}");
info.IncrementErrorCount();
throw new ApplicationException();
}
} finally {
Trace.WriteLine($"Signal {latch.CurrentCount}");
latch.Signal();
}
}
public virtual void CanHaveMultipleQueueInstances()
{
var queue = GetQueue(retries: 0, retryDelay: TimeSpan.Zero);
if (queue == null)
return;
using (queue) {
Logger.Trace().Message("Queue Id: {0}", queue.QueueId).Write();
queue.DeleteQueue();
const int workItemCount = 10;
const int workerCount = 3;
var latch = new CountdownEvent(workItemCount);
var info = new WorkInfo();
var workers = new List<IQueue<SimpleWorkItem>> {queue};
for (int i = 0; i < workerCount; i++) {
var q = GetQueue(retries: 0, retryDelay: TimeSpan.Zero);
Logger.Trace().Message("Queue Id: {0}, I: {1}", q.QueueId, i).Write();
q.StartWorking(w => DoWork(w, latch, info));
workers.Add(q);
}
Parallel.For(0, workItemCount, i => {
var id = queue.Enqueue(new SimpleWorkItem {
Data = "Hello",
Id = i
});
Logger.Trace().Message("Enqueued Index: {0} Id: {1}", i, id).Write();
});
Assert.True(latch.Wait(TimeSpan.FromSeconds(20)));
Thread.Sleep(100); // needed to make sure the worker error handler has time to finish
Logger.Trace().Message("Completed: {0} Abandoned: {1} Error: {2}",
info.CompletedCount,
info.AbandonCount,
info.ErrorCount).Write();
for (int i = 0; i < workers.Count; i++)
{
var workerStats = workers[i].GetQueueStats();
Trace.WriteLine($"Worker#{i} Completed: {workerStats.Completed} Abandoned: {workerStats.Abandoned} Error: {workerStats.Errors}");
}
Assert.Equal(workItemCount, info.CompletedCount + info.AbandonCount + info.ErrorCount);
var stats = queue.GetQueueStats();
// In memory queue doesn't share state.
if (queue.GetType() == typeof (InMemoryQueue<SimpleWorkItem>)) {
Assert.Equal(info.CompletedCount, stats.Completed);
Assert.Equal(info.AbandonCount, stats.Abandoned - stats.Errors);
Assert.Equal(info.ErrorCount, stats.Errors);
} else {
Assert.Equal(info.CompletedCount, workers.Sum(q => q.GetQueueStats().Completed));
Assert.Equal(info.AbandonCount, workers.Sum(q => q.GetQueueStats().Abandoned) - workers.Sum(q => q.GetQueueStats().Errors));
Assert.Equal(info.ErrorCount, workers.Sum(q => q.GetQueueStats().Errors));
}
workers.ForEach(w => w.Dispose());
}
}
protected async Task DoWorkAsync(QueueEntry<SimpleWorkItem> w, AsyncCountdownEvent countdown, WorkInfo info) {
Trace.WriteLine($"Starting: {w.Value.Id}");
Assert.Equal("Hello", w.Value.Data);
try {
// randomly complete, abandon or blowup.
if (RandomData.GetBool()) {
Trace.WriteLine($"Completing: {w.Value.Id}");
await w.CompleteAsync();
info.IncrementCompletedCount();
} else if (RandomData.GetBool()) {
Trace.WriteLine($"Abandoning: {w.Value.Id}");
await w.AbandonAsync();
info.IncrementAbandonCount();
} else {
Trace.WriteLine($"Erroring: {w.Value.Id}");
info.IncrementErrorCount();
throw new ApplicationException();
}
} finally {
Trace.WriteLine($"Signal {countdown.CurrentCount}");
countdown.Signal();
}
}
public virtual async Task CanHaveMultipleQueueInstances() {
var queue = GetQueue(retries: 0, retryDelay: TimeSpan.Zero);
if (queue == null)
return;
using (queue) {
Logger.Trace().Message("Queue Id: {0}", queue.QueueId).Write();
await queue.DeleteQueueAsync();
const int workItemCount = 50;
const int workerCount = 3;
var countdown = new AsyncCountdownEvent(workItemCount);
var info = new WorkInfo();
var workers = new List<IQueue<SimpleWorkItem>> {queue};
for (int i = 0; i < workerCount; i++) {
var q = GetQueue(retries: 0, retryDelay: TimeSpan.Zero);
Logger.Trace().Message("Queue Id: {0}, I: {1}", q.QueueId, i).Write();
q.StartWorking(async w => await DoWorkAsync(w, countdown, info));
workers.Add(q);
}
await Run.InParallel(workItemCount, async i => {
var id = await queue.EnqueueAsync(new SimpleWorkItem {
Data = "Hello",
Id = i
});
Logger.Trace().Message("Enqueued Index: {0} Id: {1}", i, id).Write();
});
await countdown.WaitAsync();
await Task.Delay(50);
Logger.Trace().Message("Completed: {0} Abandoned: {1} Error: {2}",
info.CompletedCount,
info.AbandonCount,
info.ErrorCount).Write();
Logger.Info().Message($"Work Info Stats: Completed: {info.CompletedCount} Abandoned: {info.AbandonCount} Error: {info.ErrorCount}").Write();
Assert.Equal(workItemCount, info.CompletedCount + info.AbandonCount + info.ErrorCount);
// In memory queue doesn't share state.
if (queue.GetType() == typeof (InMemoryQueue<SimpleWorkItem>)) {
var stats = await queue.GetQueueStatsAsync();
Assert.Equal(0, stats.Working);
Assert.Equal(0, stats.Timeouts);
Assert.Equal(workItemCount, stats.Enqueued);
Assert.Equal(workItemCount, stats.Dequeued);
Assert.Equal(info.CompletedCount, stats.Completed);
Assert.Equal(info.ErrorCount, stats.Errors);
Assert.Equal(info.AbandonCount, stats.Abandoned - info.ErrorCount);
Assert.Equal(info.AbandonCount + stats.Errors, stats.Deadletter);
} else {
var workerStats = new List<QueueStats>();
for (int i = 0; i < workers.Count; i++) {
var stats = await workers[i].GetQueueStatsAsync();
Logger.Info().Message($"Worker#{i} Working: {stats.Working} Completed: {stats.Completed} Abandoned: {stats.Abandoned} Error: {stats.Errors} Deadletter: {stats.Deadletter}").Write();
workerStats.Add(stats);
}
Assert.Equal(info.CompletedCount, workerStats.Sum(s => s.Completed));
Assert.Equal(info.ErrorCount, workerStats.Sum(s => s.Errors));
Assert.Equal(info.AbandonCount, workerStats.Sum(s => s.Abandoned) - info.ErrorCount);
Assert.Equal(info.AbandonCount + workerStats.Sum(s => s.Errors), (workerStats.LastOrDefault()?.Deadletter ?? 0));
}
workers.ForEach(w => w.Dispose());
}
}
public virtual void CanHaveMultipleQueueInstances()
{
for (int x = 0; x < 5; x++)
{
var queue = GetQueue(retries: 0, retryDelay: TimeSpan.Zero);
if (queue == null)
{
return;
}
using (queue) {
Trace.WriteLine(String.Format("Queue Id: {0}", queue.QueueId));
queue.DeleteQueue();
const int workItemCount = 10;
const int workerCount = 3;
var latch = new CountdownEvent(workItemCount);
var info = new WorkInfo();
var workers = new List <IQueue <SimpleWorkItem> > {
queue
};
for (int i = 0; i < workerCount; i++)
{
var q = GetQueue(retries: 0, retryDelay: TimeSpan.Zero);
Trace.WriteLine(String.Format("Queue Id: {0}", q.QueueId));
q.StartWorking(w => DoWork(w, latch, info));
workers.Add(q);
}
Parallel.For(0, workItemCount, i => {
var id = queue.Enqueue(new SimpleWorkItem {
Data = "Hello",
Id = i
});
Trace.WriteLine(String.Format("Enqueued Index: {0} Id: {1}", i, id));
});
Assert.True(latch.Wait(TimeSpan.FromSeconds(5)));
Thread.Sleep(100); // needed to make sure the worker error handler has time to finish
Trace.WriteLine(String.Format("Completed: {0} Abandoned: {1} Error: {2}",
info.CompletedCount,
info.AbandonCount,
info.ErrorCount));
for (int i = 0; i < workers.Count; i++)
{
Trace.WriteLine(String.Format("Worker#{0} Completed: {1} Abandoned: {2} Error: {3}", i,
workers[i].CompletedCount, workers[i].AbandonedCount, workers[i].WorkerErrorCount));
}
Assert.Equal(workItemCount, info.CompletedCount + info.AbandonCount + info.ErrorCount);
// In memory queue doesn't share state.
if (queue.GetType() == typeof(InMemoryQueue <SimpleWorkItem>))
{
Assert.Equal(info.CompletedCount, queue.CompletedCount);
Assert.Equal(info.AbandonCount, queue.AbandonedCount - queue.WorkerErrorCount);
Assert.Equal(info.ErrorCount, queue.WorkerErrorCount);
}
else
{
Assert.Equal(info.CompletedCount, workers.Sum(q => q.CompletedCount));
Assert.Equal(info.AbandonCount, workers.Sum(q => q.AbandonedCount) - workers.Sum(q => q.WorkerErrorCount));
Assert.Equal(info.ErrorCount, workers.Sum(q => q.WorkerErrorCount));
}
workers.ForEach(w => w.Dispose());
}
}
}
