public async Task CanHandleMultipleWorkItemInstances() { var metrics = new InMemoryMetricsClient(); var queue = new InMemoryQueue<WorkItemData>(retryDelay: TimeSpan.Zero, retries: 0); queue.AttachBehavior(new MetricsQueueBehavior<WorkItemData>(metrics)); var messageBus = new InMemoryMessageBus(); var handlerRegistry = new WorkItemHandlers(); var j1 = new WorkItemJob(queue, messageBus, handlerRegistry); var j2 = new WorkItemJob(queue, messageBus, handlerRegistry); var j3 = new WorkItemJob(queue, messageBus, handlerRegistry); int errors = 0; var jobIds = new ConcurrentDictionary<string, int>(); handlerRegistry.Register<MyWorkItem>(ctx => { var jobData = ctx.GetData<MyWorkItem>(); Assert.Equal("Test", jobData.SomeData); jobIds.AddOrUpdate(ctx.JobId, 1, (key, value) => value + 1); for (int i = 0; i < 10; i++) ctx.ReportProgress(10 * i); if (RandomData.GetBool(1)) { Interlocked.Increment(ref errors); throw new ApplicationException("Boom!"); } return TaskHelper.Completed(); }); for (int i = 0; i < 100; i++) queue.Enqueue(new MyWorkItem { SomeData = "Test", Index = i }, true); var completedItems = new List<string>(); object completedItemsLock = new object(); messageBus.Subscribe<WorkItemStatus>(status => { if (status.Progress < 100) return; lock (completedItemsLock) completedItems.Add(status.WorkItemId); }); var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10)); var token = cancellationTokenSource.Token; var tasks = new List<Task>(); tasks.AddRange(new[] { Task.Run(async () => await j1.RunUntilEmptyAsync(token), token), Task.Run(async () => await j2.RunUntilEmptyAsync(token), token), Task.Run(async () => await j3.RunUntilEmptyAsync(token), token), }); await Task.WhenAll(tasks); Thread.Sleep(10); Assert.Equal(100, completedItems.Count + errors); Assert.Equal(3, jobIds.Count); Assert.Equal(100, jobIds.Sum(kvp => kvp.Value)); }
public async Task CanHandleMultipleWorkItemInstances() { const int workItemCount = 1000; using (var metrics = new InMemoryMetricsClient(new InMemoryMetricsClientOptions { LoggerFactory = Log })) { var options = new InMemoryQueueOptions <WorkItemData> { Retries = 0, RetryDelay = TimeSpan.Zero, LoggerFactory = Log }; using (var queue = new InMemoryQueue <WorkItemData>(options)) { queue.AttachBehavior(new MetricsQueueBehavior <WorkItemData>(metrics, loggerFactory: Log)); using (var messageBus = new InMemoryMessageBus(new InMemoryMessageBusOptions { LoggerFactory = Log })) { var handlerRegistry = new WorkItemHandlers(); var j1 = new WorkItemJob(queue, messageBus, handlerRegistry, Log); var j2 = new WorkItemJob(queue, messageBus, handlerRegistry, Log); var j3 = new WorkItemJob(queue, messageBus, handlerRegistry, Log); int errors = 0; var jobIds = new ConcurrentDictionary <string, int>(); handlerRegistry.Register <MyWorkItem>(async ctx => { var jobData = ctx.GetData <MyWorkItem>(); Assert.Equal("Test", jobData.SomeData); int jobWorkTotal = jobIds.AddOrUpdate(ctx.JobId, 1, (key, value) => value + 1); if (jobData.Index % 100 == 0) { _logger.Trace("Job {jobId} processing work item #: {jobWorkTotal}", ctx.JobId, jobWorkTotal); } for (int i = 0; i < 10; i++) { await ctx.ReportProgressAsync(10 * i); } if (RandomData.GetBool(1)) { Interlocked.Increment(ref errors); throw new Exception("Boom!"); } }); for (int i = 0; i < workItemCount; i++) { await queue.EnqueueAsync(new MyWorkItem { SomeData = "Test", Index = i }, true); } var completedItems = new List <string>(); object completedItemsLock = new object(); await messageBus.SubscribeAsync <WorkItemStatus>(status => { if (status.Progress == 100) { _logger.Trace("Progress: {progress}", status.Progress); } if (status.Progress < 100) { return; } lock (completedItemsLock) completedItems.Add(status.WorkItemId); }); var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10)); var tasks = new List <Task> { Task.Run(async() => { await j1.RunUntilEmptyAsync(cancellationTokenSource.Token); cancellationTokenSource.Cancel(); }, cancellationTokenSource.Token), Task.Run(async() => { await j2.RunUntilEmptyAsync(cancellationTokenSource.Token); cancellationTokenSource.Cancel(); }, cancellationTokenSource.Token), Task.Run(async() => { await j3.RunUntilEmptyAsync(cancellationTokenSource.Token); cancellationTokenSource.Cancel(); }, cancellationTokenSource.Token) }; try { await Task.WhenAll(tasks); } catch (OperationCanceledException ex) { _logger.Error(ex, $"One or more tasks were cancelled: {ex.Message}"); } await SystemClock.SleepAsync(100); _logger.Info("Completed: {completedItems} Errors: {errors}", completedItems.Count, errors); Assert.Equal(workItemCount, completedItems.Count + errors); Assert.Equal(3, jobIds.Count); Assert.Equal(workItemCount, jobIds.Sum(kvp => kvp.Value)); } } } }
public async Task CanHandleMultipleWorkItemInstances() { const int workItemCount = 1000; using (var metrics = new InMemoryMetricsClient(loggerFactory: Log)) { using (var queue = new InMemoryQueue<WorkItemData>(retries: 0, retryDelay: TimeSpan.Zero, loggerFactory: Log)) { queue.AttachBehavior(new MetricsQueueBehavior<WorkItemData>(metrics, loggerFactory: Log)); using (var messageBus = new InMemoryMessageBus(Log)) { var handlerRegistry = new WorkItemHandlers(); var j1 = new WorkItemJob(queue, messageBus, handlerRegistry, Log); var j2 = new WorkItemJob(queue, messageBus, handlerRegistry, Log); var j3 = new WorkItemJob(queue, messageBus, handlerRegistry, Log); int errors = 0; var jobIds = new ConcurrentDictionary<string, int>(); handlerRegistry.Register<MyWorkItem>(async ctx => { var jobData = ctx.GetData<MyWorkItem>(); Assert.Equal("Test", jobData.SomeData); var jobWorkTotal = jobIds.AddOrUpdate(ctx.JobId, 1, (key, value) => value + 1); if (jobData.Index % 100 == 0) _logger.Trace("Job {jobId} processing work item #: {jobWorkTotal}", ctx.JobId, jobWorkTotal); for (int i = 0; i < 10; i++) await ctx.ReportProgressAsync(10 * i); if (RandomData.GetBool(1)) { Interlocked.Increment(ref errors); throw new Exception("Boom!"); } }); for (int i = 0; i < workItemCount; i++) await queue.EnqueueAsync(new MyWorkItem { SomeData = "Test", Index = i }, true); var completedItems = new List<string>(); object completedItemsLock = new object(); messageBus.Subscribe<WorkItemStatus>(status => { if (status.Progress == 100) _logger.Trace("Progress: {progress}", status.Progress); if (status.Progress < 100) return; lock (completedItemsLock) completedItems.Add(status.WorkItemId); }); var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10)); var tasks = new List<Task> { Task.Run(async () => { await j1.RunUntilEmptyAsync(cancellationTokenSource.Token); cancellationTokenSource.Cancel(); }, cancellationTokenSource.Token), Task.Run(async () => { await j2.RunUntilEmptyAsync(cancellationTokenSource.Token); cancellationTokenSource.Cancel(); }, cancellationTokenSource.Token), Task.Run(async () => { await j3.RunUntilEmptyAsync(cancellationTokenSource.Token); cancellationTokenSource.Cancel(); }, cancellationTokenSource.Token) }; try { await Task.WhenAll(tasks); await SystemClock.SleepAsync(100); } catch (TaskCanceledException) {} _logger.Info("Completed: {completedItems} Errors: {errors}", completedItems.Count, errors); Assert.Equal(workItemCount, completedItems.Count + errors); Assert.Equal(3, jobIds.Count); Assert.Equal(workItemCount, jobIds.Sum(kvp => kvp.Value)); } } } }
public async Task CanHandleMultipleWorkItemInstances() { const int workItemCount = 1000; var metrics = new InMemoryMetricsClient(); var queue = new InMemoryQueue <WorkItemData>(retries: 0, retryDelay: TimeSpan.Zero); queue.AttachBehavior(new MetricsQueueBehavior <WorkItemData>(metrics)); var messageBus = new InMemoryMessageBus(); var handlerRegistry = new WorkItemHandlers(); var j1 = new WorkItemJob(queue, messageBus, handlerRegistry); var j2 = new WorkItemJob(queue, messageBus, handlerRegistry); var j3 = new WorkItemJob(queue, messageBus, handlerRegistry); int errors = 0; var jobIds = new ConcurrentDictionary <string, int>(); handlerRegistry.Register <MyWorkItem>(async ctx => { var jobData = ctx.GetData <MyWorkItem>(); Assert.Equal("Test", jobData.SomeData); var jobWorkTotal = jobIds.AddOrUpdate(ctx.JobId, 1, (key, value) => value + 1); Logger.Trace().Message($"Job {ctx.JobId} processing work item #: {jobWorkTotal}").Write(); for (int i = 0; i < 10; i++) { await ctx.ReportProgressAsync(10 * i); } if (RandomData.GetBool(1)) { Interlocked.Increment(ref errors); throw new ApplicationException("Boom!"); } }); for (int i = 0; i < workItemCount; i++) { await queue.EnqueueAsync(new MyWorkItem { SomeData = "Test", Index = i }, true); } var completedItems = new List <string>(); object completedItemsLock = new object(); messageBus.Subscribe <WorkItemStatus>(status => { Logger.Trace().Message($"Progress: {status.Progress}").Write(); if (status.Progress < 100) { return; } lock (completedItemsLock) completedItems.Add(status.WorkItemId); }); var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10)); var tasks = new List <Task> { Task.Run(async() => { await j1.RunUntilEmptyAsync(cancellationTokenSource.Token); cancellationTokenSource.Cancel(); }, cancellationTokenSource.Token), Task.Run(async() => { await j2.RunUntilEmptyAsync(cancellationTokenSource.Token); cancellationTokenSource.Cancel(); }, cancellationTokenSource.Token), Task.Run(async() => { await j3.RunUntilEmptyAsync(cancellationTokenSource.Token); cancellationTokenSource.Cancel(); }, cancellationTokenSource.Token) }; try { await Task.WhenAll(tasks); await Task.Delay(100); } catch (TaskCanceledException) {} Logger.Info().Message($"Completed: {completedItems.Count} Errors: {errors}").Write(); metrics.DisplayStats(_writer); Assert.Equal(workItemCount, completedItems.Count + errors); Assert.Equal(3, jobIds.Count); Assert.Equal(workItemCount, jobIds.Sum(kvp => kvp.Value)); }