public async Task CanSendMultiple() {
const int iterations = 100000;
await StartListeningAsync(iterations);
var metrics = new InMemoryMetricsClient();
var sw = Stopwatch.StartNew();
for (int index = 0; index < iterations; index++) {
if (index % (iterations / 10) == 0)
StopListening();
await _client.CounterAsync("counter");
await metrics.CounterAsync("counter");
if (index % (iterations / 10) == 0)
await StartListeningAsync(iterations - index);
if (index % (iterations / 20) == 0)
metrics.DisplayStats(_writer);
}
sw.Stop();
metrics.DisplayStats(_writer);
// Require at least 10,000 operations/s
Assert.InRange(sw.ElapsedMilliseconds, 0, (iterations / 10000.0) * 1000);
await Task.Delay(250);
var messages = GetMessages();
int expected = iterations - (iterations / (iterations / 10));
Assert.InRange(messages.Count, expected - 10, expected + 10);
foreach (string message in messages)
Assert.Equal("test.counter:1|c", message);
}
public virtual async Task CanRunMultipleQueueJobs() {
const int jobCount = 5;
const int workItemCount = 100;
Log.SetLogLevel<SampleQueueJob>(LogLevel.Information);
Log.SetLogLevel<InMemoryMetricsClient>(LogLevel.None);
using (var metrics = new InMemoryMetricsClient(true, loggerFactory: Log)) {
var queues = new List<IQueue<SampleQueueWorkItem>>();
try {
for (int i = 0; i < jobCount; i++) {
var q = GetSampleWorkItemQueue(retries: 1, retryDelay: TimeSpan.Zero);
await q.DeleteQueueAsync();
q.AttachBehavior(new MetricsQueueBehavior<SampleQueueWorkItem>(metrics, "test", Log));
queues.Add(q);
}
_logger.Info("Done setting up queues");
var enqueueTask = Run.InParallel(workItemCount, async index => {
var queue = queues[RandomData.GetInt(0, jobCount - 1)];
await queue.EnqueueAsync(new SampleQueueWorkItem {
Created = SystemClock.UtcNow,
Path = RandomData.GetString()
});
});
_logger.Info("Done enqueueing");
var cancellationTokenSource = new CancellationTokenSource();
await Run.InParallel(jobCount, async index => {
var queue = queues[index - 1];
var job = new SampleQueueJob(queue, metrics, Log);
await job.RunUntilEmptyAsync(cancellationTokenSource.Token);
cancellationTokenSource.Cancel();
});
_logger.Info("Done running jobs until empty");
await enqueueTask;
var queueStats = new List<QueueStats>();
for (int i = 0; i < queues.Count; i++) {
var stats = await queues[i].GetQueueStatsAsync();
_logger.Info("Queue#{i}: Working: {working} Completed: {completed} Abandoned: {abandoned} Error: {errors} Deadletter: {deadletter}", i, stats.Working, stats.Completed, stats.Abandoned, stats.Errors, stats.Deadletter);
queueStats.Add(stats);
}
_logger.Info("Done getting queue stats");
await metrics.FlushAsync();
_logger.Info("Done flushing metrics");
var queueSummary = await metrics.GetQueueStatsAsync("test.samplequeueworkitem");
Assert.Equal(queueStats.Sum(s => s.Completed), queueSummary.Completed.Count);
Assert.InRange(queueStats.Sum(s => s.Completed), 0, workItemCount);
} finally {
foreach (var q in queues) {
await q.DeleteQueueAsync();
q.Dispose();
}
}
}
}
public async Task CanHandleMultipleWorkItemInstances()
{
var metrics = new InMemoryMetricsClient();
var queue = new InMemoryQueue<WorkItemData>(retryDelay: TimeSpan.Zero, retries: 0);
queue.AttachBehavior(new MetricsQueueBehavior<WorkItemData>(metrics));
var messageBus = new InMemoryMessageBus();
var handlerRegistry = new WorkItemHandlers();
var j1 = new WorkItemJob(queue, messageBus, handlerRegistry);
var j2 = new WorkItemJob(queue, messageBus, handlerRegistry);
var j3 = new WorkItemJob(queue, messageBus, handlerRegistry);
int errors = 0;
var jobIds = new ConcurrentDictionary<string, int>();
handlerRegistry.Register<MyWorkItem>(ctx => {
var jobData = ctx.GetData<MyWorkItem>();
Assert.Equal("Test", jobData.SomeData);
jobIds.AddOrUpdate(ctx.JobId, 1, (key, value) => value + 1);
for (int i = 0; i < 10; i++)
ctx.ReportProgress(10 * i);
if (RandomData.GetBool(1))
{
Interlocked.Increment(ref errors);
throw new ApplicationException("Boom!");
}
return TaskHelper.Completed();
});
for (int i = 0; i < 100; i++)
queue.Enqueue(new MyWorkItem { SomeData = "Test", Index = i }, true);
var completedItems = new List<string>();
object completedItemsLock = new object();
messageBus.Subscribe<WorkItemStatus>(status =>
{
if (status.Progress < 100)
return;
lock (completedItemsLock)
completedItems.Add(status.WorkItemId);
});
var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10));
var token = cancellationTokenSource.Token;
var tasks = new List<Task>();
tasks.AddRange(new[] {
Task.Run(async () => await j1.RunUntilEmptyAsync(token), token),
Task.Run(async () => await j2.RunUntilEmptyAsync(token), token),
Task.Run(async () => await j3.RunUntilEmptyAsync(token), token),
});
await Task.WhenAll(tasks);
Thread.Sleep(10);
Assert.Equal(100, completedItems.Count + errors);
Assert.Equal(3, jobIds.Count);
Assert.Equal(100, jobIds.Sum(kvp => kvp.Value));
}
public virtual async Task CanRunQueueJob() {
const int workItemCount = 100;
var metrics = new InMemoryMetricsClient();
var queue = GetSampleWorkItemQueue(retries: 0, retryDelay: TimeSpan.Zero);
await queue.DeleteQueueAsync();
queue.AttachBehavior(new MetricsQueueBehavior<SampleQueueWorkItem>(metrics, "test"));
metrics.StartDisplayingStats(TimeSpan.FromSeconds(1), _writer);
var enqueueTask = Run.InParallel(workItemCount, async index => {
await queue.EnqueueAsync(new SampleQueueWorkItem {
Created = DateTime.Now,
Path = "somepath" + index
});
});
var job = new SampleQueueJob(queue, metrics);
await Task.Delay(10);
await Task.WhenAll(job.RunUntilEmptyAsync(), enqueueTask);
metrics.DisplayStats(_writer);
var stats = await queue.GetQueueStatsAsync();
Assert.Equal(0, stats.Queued);
Assert.Equal(workItemCount, stats.Enqueued);
Assert.Equal(workItemCount, stats.Dequeued);
}
public static void RegisterServices(Container container, ILoggerFactory loggerFactory) {
container.RegisterSingleton<ILoggerFactory>(loggerFactory);
container.RegisterSingleton(typeof(ILogger<>), typeof(Logger<>));
ServiceProvider.Current = container;
container.RegisterSingleton<ISerializer>(() => new JsonNetSerializer());
var metricsClient = new InMemoryMetricsClient();
container.RegisterSingleton<IMetricsClient>(metricsClient);
container.RegisterSingleton<ICacheClient, InMemoryCacheClient>();
container.RegisterCollection(typeof(IQueueBehavior<ValuesPost>), new[] {
Lifestyle.Singleton.CreateRegistration(
() => new MetricsQueueBehavior<ValuesPost>(metricsClient), container)
});
container.RegisterSingleton<IQueue<ValuesPost>>(() => new InMemoryQueue<ValuesPost>(behaviors: container.GetAllInstances<IQueueBehavior<ValuesPost>>()));
container.RegisterCollection(typeof(IQueueBehavior<WorkItemData>), new[] {
Lifestyle.Singleton.CreateRegistration(
() => new MetricsQueueBehavior<WorkItemData>(metricsClient), container)
});
var handlers = new WorkItemHandlers();
handlers.Register<DeleteValueWorkItem, DeleteValueWorkItemHandler>();
container.RegisterSingleton(handlers);
container.RegisterSingleton<IQueue<WorkItemData>>(() => new InMemoryQueue<WorkItemData>(behaviors: container.GetAllInstances<IQueueBehavior<WorkItemData>>(), workItemTimeout: TimeSpan.FromHours(1)));
container.RegisterSingleton<IMessageBus, InMemoryMessageBus>();
container.RegisterSingleton<IMessagePublisher>(container.GetInstance<IMessageBus>);
container.RegisterSingleton<IMessageSubscriber>(container.GetInstance<IMessageBus>);
container.RegisterSingleton<ILockProvider, CacheLockProvider>();
container.RegisterSingleton<IFileStorage>(new InMemoryFileStorage());
}
public void CanRunMultipleQueueJobs()
{
const int jobCount = 5;
const int workItemCount = 1000;
var metrics = new InMemoryMetricsClient();
metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(100), _writer);
var queues = new List<RedisQueue<SampleQueueWorkItem>>();
for (int i = 0; i < jobCount; i++)
{
var q = new RedisQueue<SampleQueueWorkItem>(SharedConnection.GetMuxer(), retries: 3, retryDelay: TimeSpan.FromSeconds(1));
q.AttachBehavior(new MetricsQueueBehavior<SampleQueueWorkItem>(metrics, "test"));
queues.Add(q);
}
Task.Run(() =>
{
Parallel.For(0, workItemCount, i => {
var queue = queues[RandomData.GetInt(0, 4)];
queue.Enqueue(new SampleQueueWorkItem { Created = DateTime.Now, Path = RandomData.GetString() });
});
});
Parallel.For(0, jobCount, index =>
{
var queue = queues[index];
var job = new SampleQueueJob(queue, metrics);
job.RunUntilEmpty();
});
metrics.DisplayStats(_writer);
}
public void CanDisplayStatsMultithreaded() {
var metrics = new InMemoryMetricsClient();
metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(10), _writer);
Parallel.For(0, 100, i => {
metrics.Counter("Test");
Thread.Sleep(50);
});
}
public async Task CanWaitForCounter() {
var metrics = new InMemoryMetricsClient();
metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(50), _writer);
Task.Run(async () => {
await Task.Delay(50);
await metrics.CounterAsync("Test");
await metrics.CounterAsync("Test");
});
var success = await metrics.WaitForCounterAsync("Test", 2, TimeSpan.FromMilliseconds(500));
Assert.True(success);
Task.Run(async () => {
await Task.Delay(50);
await metrics.CounterAsync("Test");
});
success = await metrics.WaitForCounterAsync("Test", timeout: TimeSpan.FromMilliseconds(500));
Assert.True(success);
success = await metrics.WaitForCounterAsync("Test", timeout: TimeSpan.FromMilliseconds(100));
Assert.False(success);
Task.Run(async () => {
await Task.Delay(50);
await metrics.CounterAsync("Test", 2);
});
success = await metrics.WaitForCounterAsync("Test", 2, TimeSpan.FromMilliseconds(500));
Assert.True(success);
success = await metrics.WaitForCounterAsync("Test", async () => await metrics.CounterAsync("Test"), cancellationToken: TimeSpan.FromMilliseconds(500).ToCancellationToken());
Assert.True(success);
Task.Run(async () => {
await Task.Delay(50);
await metrics.CounterAsync("Test");
});
success = await metrics.WaitForCounterAsync("Test", timeout: TimeSpan.FromMilliseconds(500));
Assert.True(success);
metrics.DisplayStats(_writer);
}
public virtual async Task CanRunMultipleQueueJobs() {
const int jobCount = 5;
const int workItemCount = 100;
var metrics = new InMemoryMetricsClient();
metrics.StartDisplayingStats(TimeSpan.FromSeconds(1), _writer);
var queues = new List<IQueue<SampleQueueWorkItem>>();
for (int i = 0; i < jobCount; i++) {
var q = GetSampleWorkItemQueue(retries: 3, retryDelay: TimeSpan.FromSeconds(1));
await q.DeleteQueueAsync();
q.AttachBehavior(new MetricsQueueBehavior<SampleQueueWorkItem>(metrics, "test"));
queues.Add(q);
}
var enqueueTask = Run.InParallel(workItemCount, async index => {
var queue = queues[RandomData.GetInt(0, jobCount - 1)];
await queue.EnqueueAsync(new SampleQueueWorkItem {
Created = DateTime.Now,
Path = RandomData.GetString()
});
});
var cancellationTokenSource = new CancellationTokenSource();
await Run.InParallel(jobCount, async index => {
var queue = queues[index - 1];
var job = new SampleQueueJob(queue, metrics);
await job.RunUntilEmptyAsync(cancellationTokenSource.Token);
cancellationTokenSource.Cancel();
});
await enqueueTask;
var queueStats = new List<QueueStats>();
for (int i = 0; i < queues.Count; i++) {
var stats = await queues[i].GetQueueStatsAsync();
Logger.Info().Message($"Queue#{i}: Working: {stats.Working} Completed: {stats.Completed} Abandoned: {stats.Abandoned} Error: {stats.Errors} Deadletter: {stats.Deadletter}").Write();
queueStats.Add(stats);
}
metrics.DisplayStats(_writer);
Assert.Equal(metrics.GetCount("completed"), queueStats.Sum(s => s.Completed));
Assert.InRange(queueStats.Sum(s => s.Completed), 0, workItemCount);
}
public void CanRunQueueJob()
{
const int workItemCount = 10000;
var metrics = new InMemoryMetricsClient();
var queue = new RedisQueue<SampleQueueWorkItem>(SharedConnection.GetMuxer(), null, null, 0, TimeSpan.Zero);
queue.AttachBehavior(new MetricsQueueBehavior<SampleQueueWorkItem>(metrics, "test"));
metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(100), _writer);
Task.Factory.StartNew(() => {
Parallel.For(0, workItemCount, i => {
queue.Enqueue(new SampleQueueWorkItem { Created = DateTime.Now, Path = "somepath" + i });
});
});
var job = new SampleQueueJob(queue, metrics);
job.RunUntilEmpty();
metrics.DisplayStats(_writer);
Assert.Equal(0, queue.GetQueueStats().Queued);
}
public async Task CanIncrementCounter() {
var metrics = new InMemoryMetricsClient();
await metrics.CounterAsync("c1");
Assert.Equal(1, metrics.GetCount("c1"));
await metrics.CounterAsync("c1", 5);
Assert.Equal(6, metrics.GetCount("c1"));
var counter = metrics.Counters["c1"];
Assert.True(counter.Rate > 400);
await metrics.GaugeAsync("g1", 2.534);
Assert.Equal(2.534, metrics.GetGaugeValue("g1"));
await metrics.TimerAsync("t1", 50788);
var stats = metrics.GetMetricStats();
Assert.Equal(1, stats.Timings.Count);
metrics.DisplayStats(_writer);
}
public async void CanWaitForCounter() {
var metrics = new InMemoryMetricsClient();
metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(50), _writer);
Task.Run(() => {
Thread.Sleep(50);
metrics.Counter("Test");
metrics.Counter("Test");
});
var success = await metrics.WaitForCounterAsync("Test", TimeSpan.FromMilliseconds(500), 2);
Assert.True(success);
Task.Run(() => {
Thread.Sleep(50);
metrics.Counter("Test");
});
success = await metrics.WaitForCounterAsync("Test", TimeSpan.FromMilliseconds(500));
Assert.True(success);
success = await metrics.WaitForCounterAsync("Test", TimeSpan.FromMilliseconds(100));
Assert.False(success);
Task.Run(() => {
Thread.Sleep(50);
metrics.Counter("Test", 2);
});
success = await metrics.WaitForCounterAsync("Test", TimeSpan.FromMilliseconds(500), 2);
Assert.True(success);
success = await metrics.WaitForCounterAsync("Test", () => metrics.CounterAsync("Test"), TimeSpan.FromMilliseconds(500));
Assert.True(success);
Task.Run(() => {
Thread.Sleep(50);
metrics.Counter("Test");
});
success = metrics.WaitForCounter("Test", TimeSpan.FromMilliseconds(500));
Assert.True(success);
metrics.DisplayStats(_writer);
}
public async Task MeasureWorkerThroughput() {
var queue = GetQueue(retries: 3, workItemTimeout: TimeSpan.FromSeconds(2), retryDelay: TimeSpan.FromSeconds(1));
if (queue == null)
return;
using (queue) {
await queue.DeleteQueueAsync();
const int workItemCount = 1;
for (int i = 0; i < workItemCount; i++) {
await queue.EnqueueAsync(new SimpleWorkItem {
Data = "Hello"
});
}
Assert.Equal(workItemCount, (await queue.GetQueueStatsAsync()).Queued);
var countdown = new AsyncCountdownEvent(workItemCount);
var metrics = new InMemoryMetricsClient();
await queue.StartWorkingAsync(async workItem => {
Assert.Equal("Hello", workItem.Value.Data);
await workItem.CompleteAsync();
await metrics.CounterAsync("work");
countdown.Signal();
});
await countdown.WaitAsync(TimeSpan.FromMinutes(1));
Assert.Equal(0, countdown.CurrentCount);
_logger.Trace((await metrics.GetCounterStatsAsync("work")).ToString());
var stats = await queue.GetQueueStatsAsync();
Assert.Equal(workItemCount, stats.Dequeued);
Assert.Equal(workItemCount, stats.Completed);
Assert.Equal(0, stats.Queued);
_logger.Trace("# Keys: {0}", CountAllKeysAsync());
}
}
public async Task MeasureThroughput() {
var queue = GetQueue(retries: 3, workItemTimeout: TimeSpan.FromSeconds(2), retryDelay: TimeSpan.FromSeconds(1));
if (queue == null)
return;
using (queue) {
await queue.DeleteQueueAsync();
const int workItemCount = 1000;
for (int i = 0; i < workItemCount; i++) {
await queue.EnqueueAsync(new SimpleWorkItem {
Data = "Hello"
});
}
Assert.Equal(workItemCount, (await queue.GetQueueStatsAsync()).Queued);
var metrics = new InMemoryMetricsClient();
var workItem = await queue.DequeueAsync(TimeSpan.Zero);
while (workItem != null) {
Assert.Equal("Hello", workItem.Value.Data);
await workItem.CompleteAsync();
await metrics.CounterAsync("work");
workItem = await queue.DequeueAsync(TimeSpan.Zero);
}
_logger.Trace((await metrics.GetCounterStatsAsync("work")).ToString());
var stats = await queue.GetQueueStatsAsync();
Assert.Equal(workItemCount, stats.Dequeued);
Assert.Equal(workItemCount, stats.Completed);
Assert.Equal(0, stats.Queued);
_logger.Trace("# Keys: {0}", CountAllKeysAsync());
}
}
public virtual void MeasureThroughput()
{
var cacheClient = GetCacheClient();
if (cacheClient == null)
return;
cacheClient.FlushAll();
const int itemCount = 10000;
var metrics = new InMemoryMetricsClient();
for (int i = 0; i < itemCount; i++)
{
cacheClient.Set("test", 13422);
cacheClient.Set("flag", true);
Assert.Equal(13422, cacheClient.Get<int>("test"));
Assert.Null(cacheClient.Get<int?>("test2"));
Assert.True(cacheClient.Get<bool>("flag"));
metrics.Counter("work");
}
metrics.DisplayStats(_writer);
}
public virtual void MeasureSerializerSimpleThroughput()
{
var cacheClient = GetCacheClient();
if (cacheClient == null)
return;
cacheClient.FlushAll();
const int itemCount = 10000;
var metrics = new InMemoryMetricsClient();
for (int i = 0; i < itemCount; i++)
{
cacheClient.Set("test", new SimpleModel
{
Data1 = "Hello",
Data2 = 12
});
var model = cacheClient.Get<SimpleModel>("test");
Assert.NotNull(model);
Assert.Equal("Hello", model.Data1);
Assert.Equal(12, model.Data2);
metrics.Counter("work");
}
metrics.DisplayStats(_writer);
}
public virtual void MeasureSerializerComplexThroughput()
{
var cacheClient = GetCacheClient();
if (cacheClient == null)
return;
cacheClient.FlushAll();
const int itemCount = 10000;
var metrics = new InMemoryMetricsClient();
for (int i = 0; i < itemCount; i++)
{
cacheClient.Set("test", new ComplexModel
{
Data1 = "Hello",
Data2 = 12,
Data3 = true,
Simple = new SimpleModel { Data1 = "hi", Data2 = 13 },
Simples = new List<SimpleModel> { new SimpleModel { Data1 = "hey", Data2 = 45 }, new SimpleModel { Data1 = "next", Data2 = 3423 } },
DictionarySimples = new Dictionary<string, SimpleModel> { { "sdf", new SimpleModel { Data1 = "Sachin" } } }
});
var model = cacheClient.Get<ComplexModel>("test");
Assert.NotNull(model);
Assert.Equal("Hello", model.Data1);
Assert.Equal(12, model.Data2);
metrics.Counter("work");
}
metrics.DisplayStats(_writer);
}
public virtual void CanHandleErrorInWorker()
{
var queue = GetQueue(retries: 0);
if (queue == null)
return;
var metrics = new InMemoryMetricsClient();
queue.AttachBehavior(new MetricsQueueBehavior<SimpleWorkItem>(metrics));
using (queue) {
queue.DeleteQueue();
queue.StartWorking(w => {
Debug.WriteLine("WorkAction");
Assert.Equal("Hello", w.Value.Data);
throw new ApplicationException();
});
metrics.DisplayStats(_writer);
var success = metrics.WaitForCounter("simpleworkitem.hello.abandoned", () => queue.Enqueue(new SimpleWorkItem
{
Data = "Hello"
}), TimeSpan.FromSeconds(1));
metrics.DisplayStats(_writer);
Assert.True(success);
Assert.Equal(0, queue.GetQueueStats().Completed);
Assert.Equal(1, queue.GetQueueStats().Errors);
Assert.Equal(1, queue.GetQueueStats().Deadletter);
}
}
public void MeasureWorkerThroughput()
{
var queue = GetQueue(retries: 3, workItemTimeout: TimeSpan.FromSeconds(2), retryDelay: TimeSpan.FromSeconds(1));
if (queue == null)
return;
FlushAll();
using (queue) {
queue.DeleteQueue();
const int workItemCount = 1000;
for (int i = 0; i < workItemCount; i++) {
queue.Enqueue(new SimpleWorkItem {
Data = "Hello"
});
}
Assert.Equal(workItemCount, queue.GetQueueStats().Queued);
var countdown = new CountDownLatch(workItemCount);
var metrics = new InMemoryMetricsClient();
queue.StartWorking(workItem => {
Assert.Equal("Hello", workItem.Value.Data);
workItem.Complete();
metrics.Counter("work");
countdown.Signal();
});
countdown.Wait(60 * 1000);
metrics.DisplayStats(_output);
var stats = queue.GetQueueStats();
Assert.Equal(workItemCount, stats.Dequeued);
Assert.Equal(workItemCount, stats.Completed);
Assert.Equal(0, stats.Queued);
Trace.WriteLine(CountAllKeys());
}
}
public void JobLoopPerf()
{
const int iterations = 10000;
var metrics = new InMemoryMetricsClient();
var job = new SampleJob(metrics);
job.RunContinuous(null, iterations);
metrics.DisplayStats(_writer);
}
public void CanRunQueueJob() {
const int workItemCount = 10000;
var queue = new InMemoryQueue<SampleQueueWorkItem>(0, TimeSpan.Zero);
for (int i = 0; i < workItemCount; i++)
queue.Enqueue(new SampleQueueWorkItem { Created = DateTime.Now, Path = "somepath" + i });
var metrics = new InMemoryMetricsClient();
var job = new SampleQueueJob(queue, metrics);
job.RunUntilEmpty(new CancellationTokenSource(30000).Token);
metrics.DisplayStats(_writer);
Assert.Equal(0, queue.GetQueueStats().Queued);
}
public async Task CanHandleMultipleWorkItemInstances() {
const int workItemCount = 1000;
using (var metrics = new InMemoryMetricsClient(loggerFactory: Log)) {
using (var queue = new InMemoryQueue<WorkItemData>(retries: 0, retryDelay: TimeSpan.Zero, loggerFactory: Log)) {
queue.AttachBehavior(new MetricsQueueBehavior<WorkItemData>(metrics, loggerFactory: Log));
using (var messageBus = new InMemoryMessageBus(Log)) {
var handlerRegistry = new WorkItemHandlers();
var j1 = new WorkItemJob(queue, messageBus, handlerRegistry, Log);
var j2 = new WorkItemJob(queue, messageBus, handlerRegistry, Log);
var j3 = new WorkItemJob(queue, messageBus, handlerRegistry, Log);
int errors = 0;
var jobIds = new ConcurrentDictionary<string, int>();
handlerRegistry.Register<MyWorkItem>(async ctx => {
var jobData = ctx.GetData<MyWorkItem>();
Assert.Equal("Test", jobData.SomeData);
var jobWorkTotal = jobIds.AddOrUpdate(ctx.JobId, 1, (key, value) => value + 1);
if (jobData.Index % 100 == 0)
_logger.Trace("Job {jobId} processing work item #: {jobWorkTotal}", ctx.JobId, jobWorkTotal);
for (int i = 0; i < 10; i++)
await ctx.ReportProgressAsync(10 * i);
if (RandomData.GetBool(1)) {
Interlocked.Increment(ref errors);
throw new Exception("Boom!");
}
});
for (int i = 0; i < workItemCount; i++)
await queue.EnqueueAsync(new MyWorkItem {
SomeData = "Test",
Index = i
}, true);
var completedItems = new List<string>();
object completedItemsLock = new object();
messageBus.Subscribe<WorkItemStatus>(status => {
if (status.Progress == 100)
_logger.Trace("Progress: {progress}", status.Progress);
if (status.Progress < 100)
return;
lock (completedItemsLock)
completedItems.Add(status.WorkItemId);
});
var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10));
var tasks = new List<Task> {
Task.Run(async () => {
await j1.RunUntilEmptyAsync(cancellationTokenSource.Token);
cancellationTokenSource.Cancel();
}, cancellationTokenSource.Token),
Task.Run(async () => {
await j2.RunUntilEmptyAsync(cancellationTokenSource.Token);
cancellationTokenSource.Cancel();
}, cancellationTokenSource.Token),
Task.Run(async () => {
await j3.RunUntilEmptyAsync(cancellationTokenSource.Token);
cancellationTokenSource.Cancel();
}, cancellationTokenSource.Token)
};
try {
await Task.WhenAll(tasks);
await SystemClock.SleepAsync(100);
} catch (TaskCanceledException) {}
_logger.Info("Completed: {completedItems} Errors: {errors}", completedItems.Count, errors);
Assert.Equal(workItemCount, completedItems.Count + errors);
Assert.Equal(3, jobIds.Count);
Assert.Equal(workItemCount, jobIds.Sum(kvp => kvp.Value));
}
}
}
}
public async Task JobLoopPerf() {
const int iterations = 10000;
var metrics = new InMemoryMetricsClient();
var job = new SampleJob(metrics, Log);
var sw = Stopwatch.StartNew();
await job.RunContinuousAsync(null, iterations);
sw.Stop();
await metrics.FlushAsync();
_logger.Trace((await metrics.GetCounterStatsAsync("runs")).ToString());
_logger.Trace((await metrics.GetCounterStatsAsync("errors")).ToString());
_logger.Trace((await metrics.GetCounterStatsAsync("failed")).ToString());
_logger.Trace((await metrics.GetCounterStatsAsync("completed")).ToString());
}
public virtual async Task CanDequeueEfficiently() {
const int iterations = 100;
var queue = GetQueue(runQueueMaintenance: false);
if (queue == null)
return;
var metrics = new InMemoryMetricsClient();
queue.AttachBehavior(new MetricsQueueBehavior<SimpleWorkItem>(metrics));
using (queue) {
await queue.DeleteQueueAsync();
Task.Run(async () => {
for (int index = 0; index < iterations; index++) {
await Task.Delay(RandomData.GetInt(100, 300));
await queue.EnqueueAsync(new SimpleWorkItem {
Data = "Hello"
});
}
Logger.Trace().Message("Done enqueuing.").Write();
});
Logger.Trace().Message("Starting dequeue loop.").Write();
var sw = Stopwatch.StartNew();
for (int index = 0; index < iterations; index++) {
var item = await queue.DequeueAsync(TimeSpan.FromSeconds(5));
Assert.NotNull(item);
await item.CompleteAsync();
}
sw.Stop();
metrics.DisplayStats(_writer);
Assert.InRange(sw.ElapsedMilliseconds, iterations * 100, iterations * 325);
Assert.InRange(metrics.Timings["simpleworkitem.queuetime"].Average, 0, 25);
}
}
public virtual async Task CanHandleErrorInWorker() {
var queue = GetQueue(retries: 0);
if (queue == null)
return;
var metrics = new InMemoryMetricsClient();
queue.AttachBehavior(new MetricsQueueBehavior<SimpleWorkItem>(metrics));
using (queue) {
await queue.DeleteQueueAsync();
queue.StartWorking(w => {
Debug.WriteLine("WorkAction");
Assert.Equal("Hello", w.Value.Data);
throw new ApplicationException();
});
metrics.DisplayStats(_writer);
var success = await metrics.WaitForCounterAsync("simpleworkitem.hello.abandoned", async () => await queue.EnqueueAsync(new SimpleWorkItem {
Data = "Hello"
}), cancellationToken: TimeSpan.FromSeconds(1).ToCancellationToken());
await Task.Delay(10);
metrics.DisplayStats(_writer);
Assert.True(success);
var stats = await queue.GetQueueStatsAsync();
Assert.Equal(0, stats.Completed);
Assert.Equal(1, stats.Errors);
Assert.Equal(1, stats.Deadletter);
}
}
public void RegisterServices(Container container) {
// Foundation service provider
ServiceProvider.Current = container;
container.RegisterSingle<IDependencyResolver>(() => new SimpleInjectorCoreDependencyResolver(container));
JsonConvert.DefaultSettings = () => new JsonSerializerSettings {
DateParseHandling = DateParseHandling.DateTimeOffset
};
var contractResolver = new ExceptionlessContractResolver();
contractResolver.UseDefaultResolverFor(typeof(DataDictionary), typeof(SettingsDictionary), typeof(VersionOne.VersionOneWebHookStack), typeof(VersionOne.VersionOneWebHookEvent));
var settings = new JsonSerializerSettings {
MissingMemberHandling = MissingMemberHandling.Ignore,
DateParseHandling = DateParseHandling.DateTimeOffset,
ContractResolver = contractResolver
};
settings.AddModelConverters();
container.RegisterSingle<IContractResolver>(() => contractResolver);
container.RegisterSingle<JsonSerializerSettings>(settings);
container.RegisterSingle<JsonSerializer>(JsonSerializer.Create(settings));
container.RegisterSingle<ISerializer>(() => new JsonNetSerializer(settings));
var metricsClient = new InMemoryMetricsClient();
metricsClient.StartDisplayingStats();
container.RegisterSingle<IMetricsClient>(metricsClient);
container.RegisterSingle<IElasticClient>(() => container.GetInstance<ElasticSearchConfiguration>().GetClient(Settings.Current.ElasticSearchConnectionString.Split(',').Select(url => new Uri(url))));
container.RegisterSingle<ICacheClient, InMemoryCacheClient>();
container.RegisterSingle<IQueue<EventPost>>(() => new InMemoryQueue<EventPost>(statName: MetricNames.PostsQueueSize, metrics: container.GetInstance<IMetricsClient>()));
container.RegisterSingle<IQueue<EventUserDescription>>(() => new InMemoryQueue<EventUserDescription>(statName: MetricNames.EventsUserDescriptionQueueSize, metrics: container.GetInstance<IMetricsClient>()));
container.RegisterSingle<IQueue<EventNotificationWorkItem>>(() => new InMemoryQueue<EventNotificationWorkItem>(statName: MetricNames.EventNotificationQueueSize, metrics: container.GetInstance<IMetricsClient>()));
container.RegisterSingle<IQueue<WebHookNotification>>(() => new InMemoryQueue<WebHookNotification>(statName: MetricNames.WebHookQueueSize, metrics: container.GetInstance<IMetricsClient>()));
container.RegisterSingle<IQueue<MailMessage>>(() => new InMemoryQueue<MailMessage>(statName: MetricNames.EmailsQueueSize, metrics: container.GetInstance<IMetricsClient>()));
container.RegisterSingle<IQueue<StatusMessage>>(() => new InMemoryQueue<StatusMessage>());
var workItemHandlers = new WorkItemHandlers();
workItemHandlers.Register<ReindexWorkItem, ReindexWorkItemHandler>();
container.RegisterSingle<WorkItemHandlers>(workItemHandlers);
container.RegisterSingle<IQueue<WorkItemData>>(() => new InMemoryQueue<WorkItemData>(statName: MetricNames.WorkItemQueueSize, metrics: container.GetInstance<IMetricsClient>(), workItemTimeout: TimeSpan.FromHours(1)));
container.RegisterSingle<IMessageBus, InMemoryMessageBus>();
container.RegisterSingle<IMessagePublisher>(container.GetInstance<IMessageBus>);
container.RegisterSingle<IMessageSubscriber>(container.GetInstance<IMessageBus>);
if (!String.IsNullOrEmpty(Settings.Current.StorageFolder))
container.RegisterSingle<IFileStorage>(new FolderFileStorage(Settings.Current.StorageFolder));
else
container.RegisterSingle<IFileStorage>(new InMemoryFileStorage());
container.RegisterSingle<IStackRepository, StackRepository>();
container.RegisterSingle<IEventRepository, EventRepository>();
container.RegisterSingle<IOrganizationRepository, OrganizationRepository>();
container.RegisterSingle<IProjectRepository, ProjectRepository>();
container.RegisterSingle<IUserRepository, UserRepository>();
container.RegisterSingle<IWebHookRepository, WebHookRepository>();
container.RegisterSingle<ITokenRepository, TokenRepository>();
container.RegisterSingle<IApplicationRepository, ApplicationRepository>();
container.RegisterSingle<IGeoIPResolver, MindMaxGeoIPResolver>();
container.RegisterSingle<IValidator<Application>, ApplicationValidator>();
container.RegisterSingle<IValidator<Organization>, OrganizationValidator>();
container.RegisterSingle<IValidator<PersistentEvent>, PersistentEventValidator>();
container.RegisterSingle<IValidator<Project>, ProjectValidator>();
container.RegisterSingle<IValidator<Stack>, StackValidator>();
container.RegisterSingle<IValidator<Models.Token>, TokenValidator>();
container.RegisterSingle<IValidator<UserDescription>, UserDescriptionValidator>();
container.RegisterSingle<IValidator<User>, UserValidator>();
container.RegisterSingle<IValidator<WebHook>, WebHookValidator>();
container.RegisterSingle<IEmailGenerator>(() => new RazorEmailGenerator(@"Mail\Templates"));
container.RegisterSingle<IMailer, Mailer>();
if (Settings.Current.WebsiteMode != WebsiteMode.Dev)
container.RegisterSingle<IMailSender, SmtpMailSender>();
else
container.RegisterSingle<IMailSender>(() => new InMemoryMailSender());
container.Register<ILockProvider, CacheLockProvider>();
container.Register<StripeEventHandler>();
container.RegisterSingle<BillingManager>();
container.RegisterSingle<DataHelper>();
container.RegisterSingle<EventStats>();
container.RegisterSingle<EventPipeline>();
container.RegisterSingle<EventPluginManager>();
container.RegisterSingle<FormattingPluginManager>();
container.RegisterSingle<ICoreLastReferenceIdManager, NullCoreLastReferenceIdManager>();
}
public virtual void CanRunWorkItemWithMetrics()
{
var eventRaised = new ManualResetEvent(false);
var metricsClient = new InMemoryMetricsClient();
var behavior = new MetricsQueueBehavior<WorkItemData>(metricsClient, "metric");
var queue = new InMemoryQueue<WorkItemData>(behaviours: new[] { behavior });
queue.Completed += (sender, e) => { eventRaised.Set(); };
var work = new SimpleWorkItem { Id = 1, Data = "Testing" };
queue.Enqueue(work);
var item = queue.Dequeue();
item.Complete();
metricsClient.DisplayStats(_writer);
Assert.True(eventRaised.WaitOne(TimeSpan.FromMinutes(1)));
Assert.Equal(6, metricsClient.Counters.Count);
Assert.Equal(4, metricsClient.Timings.Count);
Assert.Equal(1, metricsClient.Counters["metric.workitemdata.simple.enqueued"]?.CurrentValue);
Assert.Equal(1, metricsClient.Counters["metric.workitemdata.simple.dequeued"]?.CurrentValue);
Assert.Equal(1, metricsClient.Counters["metric.workitemdata.simple.completed"]?.CurrentValue);
Assert.True(0 < metricsClient.Timings["metric.workitemdata.simple.queuetime"]?.Count);
Assert.True(0 < metricsClient.Timings["metric.workitemdata.simple.processtime"]?.Count);
}
public void MeasureThroughputWithRandomFailures()
{
var queue = GetQueue(retries: 3, workItemTimeout: TimeSpan.FromSeconds(2), retryDelay: TimeSpan.Zero);
if (queue == null)
return;
FlushAll();
using (queue) {
queue.DeleteQueue();
const int workItemCount = 1000;
for (int i = 0; i < workItemCount; i++) {
queue.Enqueue(new SimpleWorkItem {
Data = "Hello"
});
}
Assert.Equal(workItemCount, queue.GetQueueStats().Queued);
var metrics = new InMemoryMetricsClient();
var workItem = queue.Dequeue(TimeSpan.Zero);
while (workItem != null) {
Assert.Equal("Hello", workItem.Value.Data);
if (RandomData.GetBool(10))
workItem.Abandon();
else
workItem.Complete();
metrics.Counter("work");
workItem = queue.Dequeue(TimeSpan.FromMilliseconds(100));
}
metrics.DisplayStats(_output);
var stats = queue.GetQueueStats();
Assert.True(stats.Dequeued >= workItemCount);
Assert.Equal(workItemCount, stats.Completed + stats.Deadletter);
Assert.Equal(0, stats.Queued);
Trace.WriteLine(CountAllKeys());
}
}
public async Task CanDisplayStatsMultithreaded() {
var metrics = new InMemoryMetricsClient();
metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(10), _writer);
await Run.InParallel(100, async i => {
await metrics.CounterAsync("Test");
await Task.Delay(50);
});
}
public virtual async Task CanDequeueEfficiently() {
const int iterations = 100;
var queue = GetQueue(runQueueMaintenance: false);
if (queue == null)
return;
using (queue) {
await queue.DeleteQueueAsync();
await AssertEmptyQueueAsync(queue);
using (var metrics = new InMemoryMetricsClient()) {
queue.AttachBehavior(new MetricsQueueBehavior<SimpleWorkItem>(metrics));
Task.Run(async () => {
for (int index = 0; index < iterations; index++) {
await SystemClock.SleepAsync(RandomData.GetInt(10, 30));
await queue.EnqueueAsync(new SimpleWorkItem { Data = "Hello" });
}
_logger.Trace("Done enqueuing.");
});
_logger.Trace("Starting dequeue loop.");
var sw = Stopwatch.StartNew();
for (int index = 0; index < iterations; index++) {
var item = await queue.DequeueAsync(TimeSpan.FromSeconds(3));
Assert.NotNull(item);
await item.CompleteAsync();
}
sw.Stop();
Assert.InRange(sw.ElapsedMilliseconds, iterations * 10, iterations * 50);
var timing = await metrics.GetTimerStatsAsync("simpleworkitem.queuetime");
Assert.InRange(timing.AverageDuration, 0, 25);
}
}
}
