public virtual async Task CanRunQueueJob() {
const int workItemCount = 100;
var metrics = new InMemoryMetricsClient();
var queue = GetSampleWorkItemQueue(retries: 0, retryDelay: TimeSpan.Zero);
await queue.DeleteQueueAsync();
queue.AttachBehavior(new MetricsQueueBehavior<SampleQueueWorkItem>(metrics, "test"));
metrics.StartDisplayingStats(TimeSpan.FromSeconds(1), _writer);
var enqueueTask = Run.InParallel(workItemCount, async index => {
await queue.EnqueueAsync(new SampleQueueWorkItem {
Created = DateTime.Now,
Path = "somepath" + index
});
});
var job = new SampleQueueJob(queue, metrics);
await Task.Delay(10);
await Task.WhenAll(job.RunUntilEmptyAsync(), enqueueTask);
metrics.DisplayStats(_writer);
var stats = await queue.GetQueueStatsAsync();
Assert.Equal(0, stats.Queued);
Assert.Equal(workItemCount, stats.Enqueued);
Assert.Equal(workItemCount, stats.Dequeued);
}
public void CanRunMultipleQueueJobs()
{
const int jobCount = 5;
const int workItemCount = 1000;
var metrics = new InMemoryMetricsClient();
metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(100), _writer);
var queues = new List<RedisQueue<SampleQueueWorkItem>>();
for (int i = 0; i < jobCount; i++)
{
var q = new RedisQueue<SampleQueueWorkItem>(SharedConnection.GetMuxer(), retries: 3, retryDelay: TimeSpan.FromSeconds(1));
q.AttachBehavior(new MetricsQueueBehavior<SampleQueueWorkItem>(metrics, "test"));
queues.Add(q);
}
Task.Run(() =>
{
Parallel.For(0, workItemCount, i => {
var queue = queues[RandomData.GetInt(0, 4)];
queue.Enqueue(new SampleQueueWorkItem { Created = DateTime.Now, Path = RandomData.GetString() });
});
});
Parallel.For(0, jobCount, index =>
{
var queue = queues[index];
var job = new SampleQueueJob(queue, metrics);
job.RunUntilEmpty();
});
metrics.DisplayStats(_writer);
}
public async Task CanWaitForCounter() {
var metrics = new InMemoryMetricsClient();
metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(50), _writer);
Task.Run(async () => {
await Task.Delay(50);
await metrics.CounterAsync("Test");
await metrics.CounterAsync("Test");
});
var success = await metrics.WaitForCounterAsync("Test", 2, TimeSpan.FromMilliseconds(500));
Assert.True(success);
Task.Run(async () => {
await Task.Delay(50);
await metrics.CounterAsync("Test");
});
success = await metrics.WaitForCounterAsync("Test", timeout: TimeSpan.FromMilliseconds(500));
Assert.True(success);
success = await metrics.WaitForCounterAsync("Test", timeout: TimeSpan.FromMilliseconds(100));
Assert.False(success);
Task.Run(async () => {
await Task.Delay(50);
await metrics.CounterAsync("Test", 2);
});
success = await metrics.WaitForCounterAsync("Test", 2, TimeSpan.FromMilliseconds(500));
Assert.True(success);
success = await metrics.WaitForCounterAsync("Test", async () => await metrics.CounterAsync("Test"), cancellationToken: TimeSpan.FromMilliseconds(500).ToCancellationToken());
Assert.True(success);
Task.Run(async () => {
await Task.Delay(50);
await metrics.CounterAsync("Test");
});
success = await metrics.WaitForCounterAsync("Test", timeout: TimeSpan.FromMilliseconds(500));
Assert.True(success);
metrics.DisplayStats(_writer);
}
public virtual async Task CanRunMultipleQueueJobs() {
const int jobCount = 5;
const int workItemCount = 100;
var metrics = new InMemoryMetricsClient();
metrics.StartDisplayingStats(TimeSpan.FromSeconds(1), _writer);
var queues = new List<IQueue<SampleQueueWorkItem>>();
for (int i = 0; i < jobCount; i++) {
var q = GetSampleWorkItemQueue(retries: 3, retryDelay: TimeSpan.FromSeconds(1));
await q.DeleteQueueAsync();
q.AttachBehavior(new MetricsQueueBehavior<SampleQueueWorkItem>(metrics, "test"));
queues.Add(q);
}
var enqueueTask = Run.InParallel(workItemCount, async index => {
var queue = queues[RandomData.GetInt(0, jobCount - 1)];
await queue.EnqueueAsync(new SampleQueueWorkItem {
Created = DateTime.Now,
Path = RandomData.GetString()
});
});
var cancellationTokenSource = new CancellationTokenSource();
await Run.InParallel(jobCount, async index => {
var queue = queues[index - 1];
var job = new SampleQueueJob(queue, metrics);
await job.RunUntilEmptyAsync(cancellationTokenSource.Token);
cancellationTokenSource.Cancel();
});
await enqueueTask;
var queueStats = new List<QueueStats>();
for (int i = 0; i < queues.Count; i++) {
var stats = await queues[i].GetQueueStatsAsync();
Logger.Info().Message($"Queue#{i}: Working: {stats.Working} Completed: {stats.Completed} Abandoned: {stats.Abandoned} Error: {stats.Errors} Deadletter: {stats.Deadletter}").Write();
queueStats.Add(stats);
}
metrics.DisplayStats(_writer);
Assert.Equal(metrics.GetCount("completed"), queueStats.Sum(s => s.Completed));
Assert.InRange(queueStats.Sum(s => s.Completed), 0, workItemCount);
}
public void CanRunQueueJob()
{
const int workItemCount = 10000;
var metrics = new InMemoryMetricsClient();
var queue = new RedisQueue<SampleQueueWorkItem>(SharedConnection.GetMuxer(), null, null, 0, TimeSpan.Zero);
queue.AttachBehavior(new MetricsQueueBehavior<SampleQueueWorkItem>(metrics, "test"));
metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(100), _writer);
Task.Factory.StartNew(() => {
Parallel.For(0, workItemCount, i => {
queue.Enqueue(new SampleQueueWorkItem { Created = DateTime.Now, Path = "somepath" + i });
});
});
var job = new SampleQueueJob(queue, metrics);
job.RunUntilEmpty();
metrics.DisplayStats(_writer);
Assert.Equal(0, queue.GetQueueStats().Queued);
}
public async Task CanIncrementCounter() {
var metrics = new InMemoryMetricsClient();
await metrics.CounterAsync("c1");
Assert.Equal(1, metrics.GetCount("c1"));
await metrics.CounterAsync("c1", 5);
Assert.Equal(6, metrics.GetCount("c1"));
var counter = metrics.Counters["c1"];
Assert.True(counter.Rate > 400);
await metrics.GaugeAsync("g1", 2.534);
Assert.Equal(2.534, metrics.GetGaugeValue("g1"));
await metrics.TimerAsync("t1", 50788);
var stats = metrics.GetMetricStats();
Assert.Equal(1, stats.Timings.Count);
metrics.DisplayStats(_writer);
}
public async void CanWaitForCounter() {
var metrics = new InMemoryMetricsClient();
metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(50), _writer);
Task.Run(() => {
Thread.Sleep(50);
metrics.Counter("Test");
metrics.Counter("Test");
});
var success = await metrics.WaitForCounterAsync("Test", TimeSpan.FromMilliseconds(500), 2);
Assert.True(success);
Task.Run(() => {
Thread.Sleep(50);
metrics.Counter("Test");
});
success = await metrics.WaitForCounterAsync("Test", TimeSpan.FromMilliseconds(500));
Assert.True(success);
success = await metrics.WaitForCounterAsync("Test", TimeSpan.FromMilliseconds(100));
Assert.False(success);
Task.Run(() => {
Thread.Sleep(50);
metrics.Counter("Test", 2);
});
success = await metrics.WaitForCounterAsync("Test", TimeSpan.FromMilliseconds(500), 2);
Assert.True(success);
success = await metrics.WaitForCounterAsync("Test", () => metrics.CounterAsync("Test"), TimeSpan.FromMilliseconds(500));
Assert.True(success);
Task.Run(() => {
Thread.Sleep(50);
metrics.Counter("Test");
});
success = metrics.WaitForCounter("Test", TimeSpan.FromMilliseconds(500));
Assert.True(success);
metrics.DisplayStats(_writer);
}
public virtual void MeasureThroughput()
{
var cacheClient = GetCacheClient();
if (cacheClient == null)
return;
cacheClient.FlushAll();
const int itemCount = 10000;
var metrics = new InMemoryMetricsClient();
for (int i = 0; i < itemCount; i++)
{
cacheClient.Set("test", 13422);
cacheClient.Set("flag", true);
Assert.Equal(13422, cacheClient.Get<int>("test"));
Assert.Null(cacheClient.Get<int?>("test2"));
Assert.True(cacheClient.Get<bool>("flag"));
metrics.Counter("work");
}
metrics.DisplayStats(_writer);
}
public virtual void MeasureSerializerSimpleThroughput()
{
var cacheClient = GetCacheClient();
if (cacheClient == null)
return;
cacheClient.FlushAll();
const int itemCount = 10000;
var metrics = new InMemoryMetricsClient();
for (int i = 0; i < itemCount; i++)
{
cacheClient.Set("test", new SimpleModel
{
Data1 = "Hello",
Data2 = 12
});
var model = cacheClient.Get<SimpleModel>("test");
Assert.NotNull(model);
Assert.Equal("Hello", model.Data1);
Assert.Equal(12, model.Data2);
metrics.Counter("work");
}
metrics.DisplayStats(_writer);
}
public virtual void MeasureSerializerComplexThroughput()
{
var cacheClient = GetCacheClient();
if (cacheClient == null)
return;
cacheClient.FlushAll();
const int itemCount = 10000;
var metrics = new InMemoryMetricsClient();
for (int i = 0; i < itemCount; i++)
{
cacheClient.Set("test", new ComplexModel
{
Data1 = "Hello",
Data2 = 12,
Data3 = true,
Simple = new SimpleModel { Data1 = "hi", Data2 = 13 },
Simples = new List<SimpleModel> { new SimpleModel { Data1 = "hey", Data2 = 45 }, new SimpleModel { Data1 = "next", Data2 = 3423 } },
DictionarySimples = new Dictionary<string, SimpleModel> { { "sdf", new SimpleModel { Data1 = "Sachin" } } }
});
var model = cacheClient.Get<ComplexModel>("test");
Assert.NotNull(model);
Assert.Equal("Hello", model.Data1);
Assert.Equal(12, model.Data2);
metrics.Counter("work");
}
metrics.DisplayStats(_writer);
}
public virtual void CanHandleErrorInWorker()
{
var queue = GetQueue(retries: 0);
if (queue == null)
return;
var metrics = new InMemoryMetricsClient();
queue.AttachBehavior(new MetricsQueueBehavior<SimpleWorkItem>(metrics));
using (queue) {
queue.DeleteQueue();
queue.StartWorking(w => {
Debug.WriteLine("WorkAction");
Assert.Equal("Hello", w.Value.Data);
throw new ApplicationException();
});
metrics.DisplayStats(_writer);
var success = metrics.WaitForCounter("simpleworkitem.hello.abandoned", () => queue.Enqueue(new SimpleWorkItem
{
Data = "Hello"
}), TimeSpan.FromSeconds(1));
metrics.DisplayStats(_writer);
Assert.True(success);
Assert.Equal(0, queue.GetQueueStats().Completed);
Assert.Equal(1, queue.GetQueueStats().Errors);
Assert.Equal(1, queue.GetQueueStats().Deadletter);
}
}
public virtual void CanRunWorkItemWithMetrics()
{
var eventRaised = new ManualResetEvent(false);
var metricsClient = new InMemoryMetricsClient();
var behavior = new MetricsQueueBehavior<WorkItemData>(metricsClient, "metric");
var queue = new InMemoryQueue<WorkItemData>(behaviours: new[] { behavior });
queue.Completed += (sender, e) => { eventRaised.Set(); };
var work = new SimpleWorkItem { Id = 1, Data = "Testing" };
queue.Enqueue(work);
var item = queue.Dequeue();
item.Complete();
metricsClient.DisplayStats(_writer);
Assert.True(eventRaised.WaitOne(TimeSpan.FromMinutes(1)));
Assert.Equal(6, metricsClient.Counters.Count);
Assert.Equal(4, metricsClient.Timings.Count);
Assert.Equal(1, metricsClient.Counters["metric.workitemdata.simple.enqueued"]?.CurrentValue);
Assert.Equal(1, metricsClient.Counters["metric.workitemdata.simple.dequeued"]?.CurrentValue);
Assert.Equal(1, metricsClient.Counters["metric.workitemdata.simple.completed"]?.CurrentValue);
Assert.True(0 < metricsClient.Timings["metric.workitemdata.simple.queuetime"]?.Count);
Assert.True(0 < metricsClient.Timings["metric.workitemdata.simple.processtime"]?.Count);
}
public void JobLoopPerf()
{
const int iterations = 10000;
var metrics = new InMemoryMetricsClient();
var job = new SampleJob(metrics);
job.RunContinuous(null, iterations);
metrics.DisplayStats(_writer);
}
public void MeasureThroughputWithRandomFailures()
{
var queue = GetQueue(retries: 3, workItemTimeout: TimeSpan.FromSeconds(2), retryDelay: TimeSpan.Zero);
if (queue == null)
return;
FlushAll();
using (queue) {
queue.DeleteQueue();
const int workItemCount = 1000;
for (int i = 0; i < workItemCount; i++) {
queue.Enqueue(new SimpleWorkItem {
Data = "Hello"
});
}
Assert.Equal(workItemCount, queue.GetQueueStats().Queued);
var metrics = new InMemoryMetricsClient();
var workItem = queue.Dequeue(TimeSpan.Zero);
while (workItem != null) {
Assert.Equal("Hello", workItem.Value.Data);
if (RandomData.GetBool(10))
workItem.Abandon();
else
workItem.Complete();
metrics.Counter("work");
workItem = queue.Dequeue(TimeSpan.FromMilliseconds(100));
}
metrics.DisplayStats(_output);
var stats = queue.GetQueueStats();
Assert.True(stats.Dequeued >= workItemCount);
Assert.Equal(workItemCount, stats.Completed + stats.Deadletter);
Assert.Equal(0, stats.Queued);
Trace.WriteLine(CountAllKeys());
}
}
public async Task CanSendMultiple() {
const int iterations = 100000;
await StartListeningAsync(iterations);
var metrics = new InMemoryMetricsClient();
var sw = Stopwatch.StartNew();
for (int index = 0; index < iterations; index++) {
if (index % (iterations / 10) == 0)
StopListening();
await _client.CounterAsync("counter");
await metrics.CounterAsync("counter");
if (index % (iterations / 10) == 0)
await StartListeningAsync(iterations - index);
if (index % (iterations / 20) == 0)
metrics.DisplayStats(_writer);
}
sw.Stop();
metrics.DisplayStats(_writer);
// Require at least 10,000 operations/s
Assert.InRange(sw.ElapsedMilliseconds, 0, (iterations / 10000.0) * 1000);
await Task.Delay(250);
var messages = GetMessages();
int expected = iterations - (iterations / (iterations / 10));
Assert.InRange(messages.Count, expected - 10, expected + 10);
foreach (string message in messages)
Assert.Equal("test.counter:1|c", message);
}
public async Task MeasureWorkerThroughput() {
var queue = GetQueue(retries: 3, workItemTimeout: TimeSpan.FromSeconds(2), retryDelay: TimeSpan.FromSeconds(1));
if (queue == null)
return;
FlushAll();
using (queue) {
await queue.DeleteQueueAsync();
const int workItemCount = 1;
for (int i = 0; i < workItemCount; i++) {
await queue.EnqueueAsync(new SimpleWorkItem {
Data = "Hello"
});
}
Assert.Equal(workItemCount, (await queue.GetQueueStatsAsync()).Queued);
var countdown = new AsyncCountdownEvent(workItemCount);
var metrics = new InMemoryMetricsClient();
queue.StartWorking(async workItem => {
Assert.Equal("Hello", workItem.Value.Data);
await workItem.CompleteAsync();
await metrics.CounterAsync("work");
countdown.Signal();
});
await countdown.WaitAsync(TimeSpan.FromMinutes(1));
metrics.DisplayStats(_writer);
var stats = await queue.GetQueueStatsAsync();
Assert.Equal(workItemCount, stats.Dequeued);
Assert.Equal(workItemCount, stats.Completed);
Assert.Equal(0, stats.Queued);
Trace.WriteLine(CountAllKeys());
}
}
public virtual async Task CanDequeueEfficiently() {
const int iterations = 100;
var queue = GetQueue(runQueueMaintenance: false);
if (queue == null)
return;
var metrics = new InMemoryMetricsClient();
queue.AttachBehavior(new MetricsQueueBehavior<SimpleWorkItem>(metrics));
using (queue) {
await queue.DeleteQueueAsync();
Task.Run(async () => {
for (int index = 0; index < iterations; index++) {
await Task.Delay(RandomData.GetInt(100, 300));
await queue.EnqueueAsync(new SimpleWorkItem {
Data = "Hello"
});
}
Logger.Trace().Message("Done enqueuing.").Write();
});
Logger.Trace().Message("Starting dequeue loop.").Write();
var sw = Stopwatch.StartNew();
for (int index = 0; index < iterations; index++) {
var item = await queue.DequeueAsync(TimeSpan.FromSeconds(5));
Assert.NotNull(item);
await item.CompleteAsync();
}
sw.Stop();
metrics.DisplayStats(_writer);
Assert.InRange(sw.ElapsedMilliseconds, iterations * 100, iterations * 325);
Assert.InRange(metrics.Timings["simpleworkitem.queuetime"].Average, 0, 25);
}
}
public virtual async Task CanHandleErrorInWorker() {
var queue = GetQueue(retries: 0);
if (queue == null)
return;
var metrics = new InMemoryMetricsClient();
queue.AttachBehavior(new MetricsQueueBehavior<SimpleWorkItem>(metrics));
using (queue) {
await queue.DeleteQueueAsync();
queue.StartWorking(w => {
Debug.WriteLine("WorkAction");
Assert.Equal("Hello", w.Value.Data);
throw new ApplicationException();
});
metrics.DisplayStats(_writer);
var success = await metrics.WaitForCounterAsync("simpleworkitem.hello.abandoned", async () => await queue.EnqueueAsync(new SimpleWorkItem {
Data = "Hello"
}), cancellationToken: TimeSpan.FromSeconds(1).ToCancellationToken());
await Task.Delay(10);
metrics.DisplayStats(_writer);
Assert.True(success);
var stats = await queue.GetQueueStatsAsync();
Assert.Equal(0, stats.Completed);
Assert.Equal(1, stats.Errors);
Assert.Equal(1, stats.Deadletter);
}
}
public void MeasureWorkerThroughput()
{
var queue = GetQueue(retries: 3, workItemTimeout: TimeSpan.FromSeconds(2), retryDelay: TimeSpan.FromSeconds(1));
if (queue == null)
return;
FlushAll();
using (queue) {
queue.DeleteQueue();
const int workItemCount = 1000;
for (int i = 0; i < workItemCount; i++) {
queue.Enqueue(new SimpleWorkItem {
Data = "Hello"
});
}
Assert.Equal(workItemCount, queue.GetQueueStats().Queued);
var countdown = new CountDownLatch(workItemCount);
var metrics = new InMemoryMetricsClient();
queue.StartWorking(workItem => {
Assert.Equal("Hello", workItem.Value.Data);
workItem.Complete();
metrics.Counter("work");
countdown.Signal();
});
countdown.Wait(60 * 1000);
metrics.DisplayStats(_output);
var stats = queue.GetQueueStats();
Assert.Equal(workItemCount, stats.Dequeued);
Assert.Equal(workItemCount, stats.Completed);
Assert.Equal(0, stats.Queued);
Trace.WriteLine(CountAllKeys());
}
}
public void CanRunQueueJob() {
const int workItemCount = 10000;
var queue = new InMemoryQueue<SampleQueueWorkItem>(0, TimeSpan.Zero);
for (int i = 0; i < workItemCount; i++)
queue.Enqueue(new SampleQueueWorkItem { Created = DateTime.Now, Path = "somepath" + i });
var metrics = new InMemoryMetricsClient();
var job = new SampleQueueJob(queue, metrics);
job.RunUntilEmpty(new CancellationTokenSource(30000).Token);
metrics.DisplayStats(_writer);
Assert.Equal(0, queue.GetQueueStats().Queued);
}
public async Task MeasureThroughput() {
var queue = GetQueue(retries: 3, workItemTimeout: TimeSpan.FromSeconds(2), retryDelay: TimeSpan.FromSeconds(1));
if (queue == null)
return;
FlushAll();
using (queue) {
await queue.DeleteQueueAsync();
const int workItemCount = 1000;
for (int i = 0; i < workItemCount; i++) {
await queue.EnqueueAsync(new SimpleWorkItem {
Data = "Hello"
});
}
Assert.Equal(workItemCount, (await queue.GetQueueStatsAsync()).Queued);
var metrics = new InMemoryMetricsClient();
var workItem = await queue.DequeueAsync(TimeSpan.Zero);
while (workItem != null) {
Assert.Equal("Hello", workItem.Value.Data);
await workItem.CompleteAsync();
await metrics.CounterAsync("work");
workItem = await queue.DequeueAsync(TimeSpan.Zero);
}
metrics.DisplayStats(_writer);
var stats = await queue.GetQueueStatsAsync();
Assert.Equal(workItemCount, stats.Dequeued);
Assert.Equal(workItemCount, stats.Completed);
Assert.Equal(0, stats.Queued);
Trace.WriteLine(CountAllKeys());
}
}
public async Task CanHandleMultipleWorkItemInstances() {
const int workItemCount = 1000;
var metrics = new InMemoryMetricsClient();
var queue = new InMemoryQueue<WorkItemData>(retries: 0, retryDelay: TimeSpan.Zero);
queue.AttachBehavior(new MetricsQueueBehavior<WorkItemData>(metrics));
var messageBus = new InMemoryMessageBus();
var handlerRegistry = new WorkItemHandlers();
var j1 = new WorkItemJob(queue, messageBus, handlerRegistry);
var j2 = new WorkItemJob(queue, messageBus, handlerRegistry);
var j3 = new WorkItemJob(queue, messageBus, handlerRegistry);
int errors = 0;
var jobIds = new ConcurrentDictionary<string, int>();
handlerRegistry.Register<MyWorkItem>(async ctx => {
var jobData = ctx.GetData<MyWorkItem>();
Assert.Equal("Test", jobData.SomeData);
var jobWorkTotal = jobIds.AddOrUpdate(ctx.JobId, 1, (key, value) => value + 1);
Logger.Trace().Message($"Job {ctx.JobId} processing work item #: {jobWorkTotal}").Write();
for (int i = 0; i < 10; i++)
await ctx.ReportProgressAsync(10 * i);
if (RandomData.GetBool(1)) {
Interlocked.Increment(ref errors);
throw new ApplicationException("Boom!");
}
});
for (int i = 0; i < workItemCount; i++)
await queue.EnqueueAsync(new MyWorkItem {
SomeData = "Test",
Index = i
}, true);
var completedItems = new List<string>();
object completedItemsLock = new object();
messageBus.Subscribe<WorkItemStatus>(status => {
Logger.Trace().Message($"Progress: {status.Progress}").Write();
if (status.Progress < 100)
return;
lock (completedItemsLock)
completedItems.Add(status.WorkItemId);
});
var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10));
var tasks = new List<Task> {
Task.Run(async () => {
await j1.RunUntilEmptyAsync(cancellationTokenSource.Token);
cancellationTokenSource.Cancel();
}, cancellationTokenSource.Token),
Task.Run(async () => {
await j2.RunUntilEmptyAsync(cancellationTokenSource.Token);
cancellationTokenSource.Cancel();
}, cancellationTokenSource.Token),
Task.Run(async () => {
await j3.RunUntilEmptyAsync(cancellationTokenSource.Token);
cancellationTokenSource.Cancel();
}, cancellationTokenSource.Token)
};
try {
await Task.WhenAll(tasks);
await Task.Delay(100);
} catch (TaskCanceledException) {}
Logger.Info().Message($"Completed: {completedItems.Count} Errors: {errors}").Write();
metrics.DisplayStats(_writer);
Assert.Equal(workItemCount, completedItems.Count + errors);
Assert.Equal(3, jobIds.Count);
Assert.Equal(workItemCount, jobIds.Sum(kvp => kvp.Value));
}
