public virtual async Task CanRunQueueJob()
{
const int workItemCount = 100;
var metrics = new InMemoryMetricsClient();
var queue = GetSampleWorkItemQueue(retries: 0, retryDelay: TimeSpan.Zero);
await queue.DeleteQueueAsync();
queue.AttachBehavior(new MetricsQueueBehavior <SampleQueueWorkItem>(metrics, "test"));
metrics.StartDisplayingStats(TimeSpan.FromSeconds(1), _writer);
var enqueueTask = Run.InParallel(workItemCount, async index => {
await queue.EnqueueAsync(new SampleQueueWorkItem {
Created = DateTime.Now,
Path = "somepath" + index
});
});
var job = new SampleQueueJob(queue, metrics);
await Task.Delay(10);
await Task.WhenAll(job.RunUntilEmptyAsync(), enqueueTask);
metrics.DisplayStats(_writer);
var stats = await queue.GetQueueStatsAsync();
Assert.Equal(0, stats.Queued);
Assert.Equal(workItemCount, stats.Enqueued);
Assert.Equal(workItemCount, stats.Dequeued);
}
public virtual async Task CanRunQueueJob()
{
const int workItemCount = 100;
using (var queue = GetSampleWorkItemQueue(retries: 0, retryDelay: TimeSpan.Zero)) {
await queue.DeleteQueueAsync();
var enqueueTask = Run.InParallel(workItemCount, async index => {
await queue.EnqueueAsync(new SampleQueueWorkItem {
Created = SystemClock.UtcNow,
Path = "somepath" + index
});
});
var job = new SampleQueueJob(queue, null, Log);
await SystemClock.SleepAsync(10);
await Task.WhenAll(job.RunUntilEmptyAsync(), enqueueTask);
var stats = await queue.GetQueueStatsAsync();
Assert.Equal(0, stats.Queued);
Assert.Equal(workItemCount, stats.Enqueued);
Assert.Equal(workItemCount, stats.Dequeued);
}
}
public virtual async Task CanRunMultipleQueueJobs()
{
const int jobCount = 5;
const int workItemCount = 100;
Log.SetLogLevel <SampleQueueJob>(LogLevel.Error);
Log.SetLogLevel <InMemoryMetricsClient>(LogLevel.Trace);
var metrics = new InMemoryMetricsClient(true, loggerFactory: Log);
var queues = new List <IQueue <SampleQueueWorkItem> >();
for (int i = 0; i < jobCount; i++)
{
var q = GetSampleWorkItemQueue(retries: 3, retryDelay: TimeSpan.FromSeconds(1));
await q.DeleteQueueAsync();
q.AttachBehavior(new MetricsQueueBehavior <SampleQueueWorkItem>(metrics, "test", Log));
queues.Add(q);
}
var enqueueTask = Run.InParallel(workItemCount, async index => {
var queue = queues[RandomData.GetInt(0, jobCount - 1)];
await queue.EnqueueAsync(new SampleQueueWorkItem {
Created = DateTime.Now,
Path = RandomData.GetString()
});
});
var cancellationTokenSource = new CancellationTokenSource();
await Run.InParallel(jobCount, async index => {
var queue = queues[index - 1];
var job = new SampleQueueJob(queue, metrics, Log);
await job.RunUntilEmptyAsync(cancellationTokenSource.Token);
cancellationTokenSource.Cancel();
});
await enqueueTask;
var queueStats = new List <QueueStats>();
for (int i = 0; i < queues.Count; i++)
{
var stats = await queues[i].GetQueueStatsAsync();
_logger.Info("Queue#{i}: Working: {working} Completed: {completed} Abandoned: {abandoned} Error: {errors} Deadletter: {deadletter}", i, stats.Working, stats.Completed, stats.Abandoned, stats.Errors, stats.Deadletter);
queueStats.Add(stats);
}
await metrics.FlushAsync();
var counter = await metrics.GetCounterStatsAsync("completed");
Assert.Equal(queueStats.Sum(s => s.Completed), counter.Count);
Assert.InRange(queueStats.Sum(s => s.Completed), 0, workItemCount);
}
public virtual async Task CanRunMultipleQueueJobs()
{
const int jobCount = 5;
const int workItemCount = 100;
var metrics = new InMemoryMetricsClient();
metrics.StartDisplayingStats(TimeSpan.FromSeconds(1), _writer);
var queues = new List <IQueue <SampleQueueWorkItem> >();
for (int i = 0; i < jobCount; i++)
{
var q = GetSampleWorkItemQueue(retries: 3, retryDelay: TimeSpan.FromSeconds(1));
await q.DeleteQueueAsync();
q.AttachBehavior(new MetricsQueueBehavior <SampleQueueWorkItem>(metrics, "test"));
queues.Add(q);
}
var enqueueTask = Run.InParallel(workItemCount, async index => {
var queue = queues[RandomData.GetInt(0, jobCount - 1)];
await queue.EnqueueAsync(new SampleQueueWorkItem {
Created = DateTime.Now,
Path = RandomData.GetString()
});
});
var cancellationTokenSource = new CancellationTokenSource();
await Run.InParallel(jobCount, async index => {
var queue = queues[index - 1];
var job = new SampleQueueJob(queue, metrics);
await job.RunUntilEmptyAsync(cancellationTokenSource.Token);
cancellationTokenSource.Cancel();
});
await enqueueTask;
var queueStats = new List <QueueStats>();
for (int i = 0; i < queues.Count; i++)
{
var stats = await queues[i].GetQueueStatsAsync();
Logger.Info().Message($"Queue#{i}: Working: {stats.Working} Completed: {stats.Completed} Abandoned: {stats.Abandoned} Error: {stats.Errors} Deadletter: {stats.Deadletter}").Write();
queueStats.Add(stats);
}
metrics.DisplayStats(_writer);
Assert.Equal(metrics.GetCount("completed"), queueStats.Sum(s => s.Completed));
Assert.InRange(queueStats.Sum(s => s.Completed), 0, workItemCount);
}
public virtual async Task ActivityWillFlowThroughQueueJobAsync()
{
using (var queue = GetSampleWorkItemQueue(retries: 0, retryDelay: TimeSpan.Zero)) {
await queue.DeleteQueueAsync();
Activity parentActivity = null;
using var listener = new ActivityListener {
ShouldListenTo = s => s.Name == nameof(JobQueueTestsBase) || s.Name == "Foundatio",
Sample = (ref ActivityCreationOptions <ActivityContext> _) => ActivitySamplingResult.AllDataAndRecorded,
ActivityStarted = a => {
if (a.OperationName != "ProcessQueueEntry")
{
return;
}
Assert.Equal(parentActivity.RootId, a.RootId);
Assert.Equal(parentActivity.SpanId, a.ParentSpanId);
},
ActivityStopped = a => {}
};
ActivitySource.AddActivityListener(listener);
parentActivity = _activitySource.StartActivity("Parent");
Assert.NotNull(parentActivity);
var enqueueTask = await queue.EnqueueAsync(new SampleQueueWorkItem {
Created = SystemClock.UtcNow,
Path = "somepath"
});
// clear activity and then verify that
Activity.Current = null;
var job = new SampleQueueJob(queue, null, Log);
await job.RunAsync();
var stats = await queue.GetQueueStatsAsync();
Assert.Equal(0, stats.Queued);
Assert.Equal(1, stats.Enqueued);
Assert.Equal(1, stats.Dequeued);
}
}
public void CanRunQueueJob()
{
const int workItemCount = 1000;
var metrics = new InMemoryMetricsClient();
var queue = new InMemoryQueue <SampleQueueWorkItem>(0, TimeSpan.Zero, metrics: metrics);
for (int i = 0; i < workItemCount; i++)
{
queue.Enqueue(new SampleQueueWorkItem {
Created = DateTime.Now, Path = "somepath" + i
});
}
var job = new SampleQueueJob(queue, metrics);
job.RunUntilEmpty();
metrics.DisplayStats();
Assert.Equal(0, queue.GetQueueCount());
}
public virtual async Task CanRunMultipleQueueJobsAsync()
{
const int jobCount = 5;
const int workItemCount = 100;
Log.SetLogLevel <SampleQueueJob>(LogLevel.Information);
Log.SetLogLevel <InMemoryMetricsClient>(LogLevel.None);
using (var metrics = new InMemoryMetricsClient(new InMemoryMetricsClientOptions {
LoggerFactory = Log, Buffered = true
})) {
var queues = new List <IQueue <SampleQueueWorkItem> >();
try {
for (int i = 0; i < jobCount; i++)
{
var q = GetSampleWorkItemQueue(retries: 1, retryDelay: TimeSpan.Zero);
await q.DeleteQueueAsync();
q.AttachBehavior(new MetricsQueueBehavior <SampleQueueWorkItem>(metrics, "test", loggerFactory: Log));
queues.Add(q);
}
_logger.LogInformation("Done setting up queues");
var enqueueTask = Run.InParallelAsync(workItemCount, index => {
var queue = queues[RandomData.GetInt(0, jobCount - 1)];
return(queue.EnqueueAsync(new SampleQueueWorkItem {
Created = SystemClock.UtcNow,
Path = RandomData.GetString()
}));
});
_logger.LogInformation("Done enqueueing");
var cancellationTokenSource = new CancellationTokenSource();
await Run.InParallelAsync(jobCount, async index => {
var queue = queues[index - 1];
var job = new SampleQueueJob(queue, metrics, Log);
await job.RunUntilEmptyAsync(cancellationTokenSource.Token);
cancellationTokenSource.Cancel();
});
_logger.LogInformation("Done running jobs until empty");
await enqueueTask;
var queueStats = new List <QueueStats>();
for (int i = 0; i < queues.Count; i++)
{
var stats = await queues[i].GetQueueStatsAsync();
if (_logger.IsEnabled(LogLevel.Information))
{
_logger.LogInformation("Queue#{Id}: Working: {Working} Completed: {Completed} Abandoned: {Abandoned} Error: {Errors} Deadletter: {Deadletter}", i, stats.Working, stats.Completed, stats.Abandoned, stats.Errors, stats.Deadletter);
}
queueStats.Add(stats);
}
_logger.LogInformation("Done getting queue stats");
await metrics.FlushAsync();
_logger.LogInformation("Done flushing metrics");
var queueSummary = await metrics.GetQueueStatsAsync("test.samplequeueworkitem");
Assert.Equal(queueStats.Sum(s => s.Completed), queueSummary.Completed.Count);
Assert.InRange(queueStats.Sum(s => s.Completed), 0, workItemCount);
} finally {
foreach (var q in queues)
{
await q.DeleteQueueAsync();
q.Dispose();
}
}
}
}
