public void CanRunMultipleQueueJobs()
{
const int jobCount = 5;
const int workItemCount = 1000;
var metrics = new InMemoryMetricsClient();
metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(100), _writer);
var queues = new List<RedisQueue<SampleQueueWorkItem>>();
for (int i = 0; i < jobCount; i++)
{
var q = new RedisQueue<SampleQueueWorkItem>(SharedConnection.GetMuxer(), retries: 3, retryDelay: TimeSpan.FromSeconds(1));
q.AttachBehavior(new MetricsQueueBehavior<SampleQueueWorkItem>(metrics, "test"));
queues.Add(q);
}
Task.Run(() =>
{
Parallel.For(0, workItemCount, i => {
var queue = queues[RandomData.GetInt(0, 4)];
queue.Enqueue(new SampleQueueWorkItem { Created = DateTime.Now, Path = RandomData.GetString() });
});
});
Parallel.For(0, jobCount, index =>
{
var queue = queues[index];
var job = new SampleQueueJob(queue, metrics);
job.RunUntilEmpty();
});
metrics.DisplayStats(_writer);
}
public virtual async Task CanRunQueueJob() {
const int workItemCount = 100;
var metrics = new InMemoryMetricsClient();
var queue = GetSampleWorkItemQueue(retries: 0, retryDelay: TimeSpan.Zero);
await queue.DeleteQueueAsync();
queue.AttachBehavior(new MetricsQueueBehavior<SampleQueueWorkItem>(metrics, "test"));
metrics.StartDisplayingStats(TimeSpan.FromSeconds(1), _writer);
var enqueueTask = Run.InParallel(workItemCount, async index => {
await queue.EnqueueAsync(new SampleQueueWorkItem {
Created = DateTime.Now,
Path = "somepath" + index
});
});
var job = new SampleQueueJob(queue, metrics);
await Task.Delay(10);
await Task.WhenAll(job.RunUntilEmptyAsync(), enqueueTask);
metrics.DisplayStats(_writer);
var stats = await queue.GetQueueStatsAsync();
Assert.Equal(0, stats.Queued);
Assert.Equal(workItemCount, stats.Enqueued);
Assert.Equal(workItemCount, stats.Dequeued);
}
public virtual async Task CanRunMultipleQueueJobs() {
const int jobCount = 5;
const int workItemCount = 100;
var metrics = new InMemoryMetricsClient();
metrics.StartDisplayingStats(TimeSpan.FromSeconds(1), _writer);
var queues = new List<IQueue<SampleQueueWorkItem>>();
for (int i = 0; i < jobCount; i++) {
var q = GetSampleWorkItemQueue(retries: 3, retryDelay: TimeSpan.FromSeconds(1));
await q.DeleteQueueAsync();
q.AttachBehavior(new MetricsQueueBehavior<SampleQueueWorkItem>(metrics, "test"));
queues.Add(q);
}
var enqueueTask = Run.InParallel(workItemCount, async index => {
var queue = queues[RandomData.GetInt(0, jobCount - 1)];
await queue.EnqueueAsync(new SampleQueueWorkItem {
Created = DateTime.Now,
Path = RandomData.GetString()
});
});
var cancellationTokenSource = new CancellationTokenSource();
await Run.InParallel(jobCount, async index => {
var queue = queues[index - 1];
var job = new SampleQueueJob(queue, metrics);
await job.RunUntilEmptyAsync(cancellationTokenSource.Token);
cancellationTokenSource.Cancel();
});
await enqueueTask;
var queueStats = new List<QueueStats>();
for (int i = 0; i < queues.Count; i++) {
var stats = await queues[i].GetQueueStatsAsync();
Logger.Info().Message($"Queue#{i}: Working: {stats.Working} Completed: {stats.Completed} Abandoned: {stats.Abandoned} Error: {stats.Errors} Deadletter: {stats.Deadletter}").Write();
queueStats.Add(stats);
}
metrics.DisplayStats(_writer);
Assert.Equal(metrics.GetCount("completed"), queueStats.Sum(s => s.Completed));
Assert.InRange(queueStats.Sum(s => s.Completed), 0, workItemCount);
}
public void CanRunQueueJob()
{
const int workItemCount = 10000;
var metrics = new InMemoryMetricsClient();
var queue = new RedisQueue<SampleQueueWorkItem>(SharedConnection.GetMuxer(), null, null, 0, TimeSpan.Zero);
queue.AttachBehavior(new MetricsQueueBehavior<SampleQueueWorkItem>(metrics, "test"));
metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(100), _writer);
Task.Factory.StartNew(() => {
Parallel.For(0, workItemCount, i => {
queue.Enqueue(new SampleQueueWorkItem { Created = DateTime.Now, Path = "somepath" + i });
});
});
var job = new SampleQueueJob(queue, metrics);
job.RunUntilEmpty();
metrics.DisplayStats(_writer);
Assert.Equal(0, queue.GetQueueStats().Queued);
}
public void CanRunQueueJob()
{
const int workItemCount = 10000;
var metrics = new InMemoryMetricsClient();
var queue = new RedisQueue <SampleQueueWorkItem>(SharedConnection.GetMuxer(), null, null, 0, TimeSpan.Zero, metrics: metrics);
Task.Factory.StartNew(() => {
Parallel.For(0, workItemCount, i => {
queue.Enqueue(new SampleQueueWorkItem {
Created = DateTime.Now, Path = "somepath" + i
});
});
});
var job = new SampleQueueJob(queue, metrics);
job.RunUntilEmpty();
metrics.DisplayStats();
Assert.Equal(0, queue.GetQueueCount());
}
public virtual async Task CanRunQueueJob() {
const int workItemCount = 100;
using (var queue = GetSampleWorkItemQueue(retries: 0, retryDelay: TimeSpan.Zero)) {
await queue.DeleteQueueAsync();
var enqueueTask = Run.InParallel(workItemCount, async index => {
await queue.EnqueueAsync(new SampleQueueWorkItem {
Created = SystemClock.UtcNow,
Path = "somepath" + index
});
});
var job = new SampleQueueJob(queue, null, Log);
await SystemClock.SleepAsync(10);
await Task.WhenAll(job.RunUntilEmptyAsync(), enqueueTask);
var stats = await queue.GetQueueStatsAsync();
Assert.Equal(0, stats.Queued);
Assert.Equal(workItemCount, stats.Enqueued);
Assert.Equal(workItemCount, stats.Dequeued);
}
}
public void CanRunQueueJob() {
const int workItemCount = 10000;
var queue = new InMemoryQueue<SampleQueueWorkItem>(0, TimeSpan.Zero);
for (int i = 0; i < workItemCount; i++)
queue.Enqueue(new SampleQueueWorkItem { Created = DateTime.Now, Path = "somepath" + i });
var metrics = new InMemoryMetricsClient();
var job = new SampleQueueJob(queue, metrics);
job.RunUntilEmpty(new CancellationTokenSource(30000).Token);
metrics.DisplayStats(_writer);
Assert.Equal(0, queue.GetQueueStats().Queued);
}
public virtual async Task CanRunMultipleQueueJobs() {
const int jobCount = 5;
const int workItemCount = 100;
Log.SetLogLevel<SampleQueueJob>(LogLevel.Information);
Log.SetLogLevel<InMemoryMetricsClient>(LogLevel.None);
using (var metrics = new InMemoryMetricsClient(true, loggerFactory: Log)) {
var queues = new List<IQueue<SampleQueueWorkItem>>();
try {
for (int i = 0; i < jobCount; i++) {
var q = GetSampleWorkItemQueue(retries: 1, retryDelay: TimeSpan.Zero);
await q.DeleteQueueAsync();
q.AttachBehavior(new MetricsQueueBehavior<SampleQueueWorkItem>(metrics, "test", Log));
queues.Add(q);
}
_logger.Info("Done setting up queues");
var enqueueTask = Run.InParallel(workItemCount, async index => {
var queue = queues[RandomData.GetInt(0, jobCount - 1)];
await queue.EnqueueAsync(new SampleQueueWorkItem {
Created = SystemClock.UtcNow,
Path = RandomData.GetString()
});
});
_logger.Info("Done enqueueing");
var cancellationTokenSource = new CancellationTokenSource();
await Run.InParallel(jobCount, async index => {
var queue = queues[index - 1];
var job = new SampleQueueJob(queue, metrics, Log);
await job.RunUntilEmptyAsync(cancellationTokenSource.Token);
cancellationTokenSource.Cancel();
});
_logger.Info("Done running jobs until empty");
await enqueueTask;
var queueStats = new List<QueueStats>();
for (int i = 0; i < queues.Count; i++) {
var stats = await queues[i].GetQueueStatsAsync();
_logger.Info("Queue#{i}: Working: {working} Completed: {completed} Abandoned: {abandoned} Error: {errors} Deadletter: {deadletter}", i, stats.Working, stats.Completed, stats.Abandoned, stats.Errors, stats.Deadletter);
queueStats.Add(stats);
}
_logger.Info("Done getting queue stats");
await metrics.FlushAsync();
_logger.Info("Done flushing metrics");
var queueSummary = await metrics.GetQueueStatsAsync("test.samplequeueworkitem");
Assert.Equal(queueStats.Sum(s => s.Completed), queueSummary.Completed.Count);
Assert.InRange(queueStats.Sum(s => s.Completed), 0, workItemCount);
} finally {
foreach (var q in queues) {
await q.DeleteQueueAsync();
q.Dispose();
}
}
}
}
