public async Task Test_CanDequeue_WithTaskTypes_MultipleTypesPerDequeueCall() { DateTimeOffset refNow = mDataSource .LastPostedAt .AddMilliseconds(1); int expectedDequeueCount = mDataSource .NumTasksInQueue; string[] taskTypes = mDataSource.InQueueTaskTypes .Select(t => t.FullName) .ToArray(); using (PostgreSqlTaskQueueConsumer taskQueue = CreateTaskQueue(() => refNow)) using (ConsumedQueuedTaskTokenChecker checker = new ConsumedQueuedTaskTokenChecker(mDataSource)) { for (int i = 0; i < expectedDequeueCount; i++) { await Run_ConsumeTestAsync(taskQueue, checker, refNow, taskTypes); } } }
protected void Dispose(bool disposing) { if (!mIsDisposed) { if (disposing) { StopAync().Wait(); mTaskPoller.Dispose(); mTaskBuffer.Dispose(); if (mTaskResultQueue is IDisposable) { (( IDisposable )mTaskResultQueue).Dispose(); } mTaskPoller = null; mTaskBuffer = null; mTaskResultQueue = null; mTaskQueueConsumer = null; mExecutorRegistry = null; } mIsDisposed = true; } }
public async Task Test_PeekMatchesDequeuedItem_SingleConsumer() { IQueuedTask peekTask = null; IQueuedTaskToken dequeuedTaskToken = null; PostgreSqlTaskQueueInfo taskQueueInfo = CreateTaskQueueInfo(() => mDataSource.LastPostedAt); using (PostgreSqlTaskQueueConsumer taskQueue = CreateTaskQueueConsumer(() => mDataSource.LastPostedAt)) { int expectedDequeueCount = mDataSource .NumTasksInQueue; for (int i = 0; i < expectedDequeueCount; i++) { peekTask = await taskQueueInfo.PeekAsync(); Assert.NotNull(peekTask); dequeuedTaskToken = await taskQueue.DequeueAsync(); Assert.NotNull(dequeuedTaskToken); Assert.AreEqual(peekTask.Id, dequeuedTaskToken .DequeuedTask .Id); } } }
public async Task Test_StatsAreCorrectlyUpdated_AfterDequeue_NoTaskType() { PostgreSqlTaskQueueInfo taskQueueInfo = CreateTaskQueueInfo(() => mDataSource.LastPostedAt); using (PostgreSqlTaskQueueConsumer taskQueueConsumer = CreateTaskQueueConsumer(() => mDataSource.LastPostedAt)) using (TaskQueueMetricsDiffChecker diff = new TaskQueueMetricsDiffChecker(async() => await taskQueueInfo.ComputeMetricsAsync())) { await diff.CaptureInitialMetricsAsync(); IQueuedTaskToken dequeuedToken = await taskQueueConsumer .DequeueAsync(); QueuedTaskStatus origStatus = mDataSource .GetOriginalTokenData(dequeuedToken.DequeuedTask.Id) .LastQueuedTaskResult .Status; await diff.CaptureNewMetricsAndAssertCorrectDiff(delta : new TaskQueueMetrics( totalUnprocessed: origStatus == QueuedTaskStatus.Unprocessed ? -1 : 0, totalProcessing: 1, totalErrored: origStatus == QueuedTaskStatus.Error ? -1 : 0, totalFaulted: origStatus == QueuedTaskStatus.Faulted ? -1 : 0, totalFataled: origStatus == QueuedTaskStatus.Fatal ? -1 : 0, totalProcessed: origStatus == QueuedTaskStatus.Processed ? -1 : 0)); } }
public async Task Test_DequeueChangesPeekResult_SingleConsumer() { IQueuedTask peekTask = null, rePeekTask = null; IQueuedTaskToken dequeuedTaskToken = null; PostgreSqlTaskQueueInfo taskQueueInfo = CreateTaskQueueInfo(() => mDataSource.LastPostedAt); using (PostgreSqlTaskQueueConsumer taskQueueConsumer = CreateTaskQueueConsumer(() => mDataSource.LastPostedAt)) { peekTask = await taskQueueInfo.PeekAsync(); Assert.NotNull(peekTask); dequeuedTaskToken = await taskQueueConsumer.DequeueAsync(); Assert.NotNull(dequeuedTaskToken); rePeekTask = await taskQueueInfo.PeekAsync(); Assert.NotNull(rePeekTask); //Removing a new element from the queue // occurs at the beginning of the queue, // so peeking must yield a different result // than before dequeue-ing Assert.AreNotEqual(rePeekTask.Id, peekTask.Id); } }
public async Task Test_CanEnqueue_NewTask_Serial() { Faker faker = new Faker(); ManualResetEvent notificationWaitHandle = new ManualResetEvent(false); DateTimeOffset postedAt = mDataSource.LastPostedAt .AddTicks(1); PostgreSqlTaskQueueProducer taskQueueProducer = CreateTaskQueueProducer(() => postedAt); EventHandler <ClearForDequeueEventArgs> handleClearForDequeue = (s, e) => { if (e.Reason == ClearForDequeReason.NewTaskPostedNotificationReceived) { notificationWaitHandle.Set(); } }; using (PostgreSqlTaskQueueConsumer taskQueueConsumer = CreateTaskQueueConsumer(() => postedAt)) { taskQueueConsumer.ClearForDequeue += handleClearForDequeue; await taskQueueConsumer .StartReceivingNewTaskUpdatesAsync(); Assert.IsTrue(taskQueueConsumer .IsReceivingNewTaskUpdates); //Enqueue task and check result IQueuedTask queuedTask = await taskQueueProducer .EnqueueAsync(payload : new SampleTaskPayload(100), source : nameof(Test_CanEnqueue_NewTask_Serial), priority : faker.Random.Int(1, 100)); Assert.NotNull(queuedTask); await Assert_ResultAddedOrUpdatedCorrectly(queuedTask); notificationWaitHandle.WaitOne(); await taskQueueConsumer .StopReceivingNewTaskUpdatesAsync(); Assert.IsFalse(taskQueueConsumer .IsReceivingNewTaskUpdates); taskQueueConsumer.ClearForDequeue -= handleClearForDequeue; } }
public async Task Test_CanDequeue_WithTaskTypes_MultipleTypesPerDequeueCall_ParallelConsumers(int nConsumers) { int expectedDequeueCount = mDataSource .NumTasksInQueue; DateTimeOffset refNow = mDataSource .LastPostedAt .AddMilliseconds(1); string[] taskTypes = mDataSource.InQueueTaskTypes .Select(t => t.FullName) .ToArray(); Task[] consumers = new Task[nConsumers]; int[] loopPartitions = expectedDequeueCount .PartitionValue(nConsumers); Assert.AreEqual(expectedDequeueCount, loopPartitions.Sum()); for (int iConsumer = 0; iConsumer < nConsumers; iConsumer++) { int loopCount = loopPartitions[iConsumer]; consumers[iConsumer] = Task.Run(async() => { using (PostgreSqlTaskQueueConsumer taskQueue = CreateTaskQueue(() => refNow)) using (ConsumedQueuedTaskTokenChecker checker = new ConsumedQueuedTaskTokenChecker(mDataSource)) { for (int iTest = 0; iTest < loopCount; iTest++) { await Run_ConsumeTestAsync(taskQueue, checker, refNow, taskTypes); } } }); } await Task.WhenAll(consumers); }
public StandardTaskEngine(TaskEngineOptions engineOptions, TaskQueueOptions producerAndResultOptions, TaskQueueConsumerOptions consumerOptions, ITaskExecutorRegistry executorRegistry, IExecutionPerformanceMonitorWriter executionPerfMonWriter, ITimestampProvider timestampProvider) { if (engineOptions == null) { throw new ArgumentNullException(nameof(engineOptions)); } if (consumerOptions == null) { throw new ArgumentNullException(nameof(consumerOptions)); } if (producerAndResultOptions == null) { throw new ArgumentNullException(nameof(producerAndResultOptions)); } mExecutorRegistry = executorRegistry ?? throw new ArgumentNullException(nameof(executorRegistry)); mExecutionPerfMonWriter = executionPerfMonWriter ?? throw new ArgumentNullException(nameof(executionPerfMonWriter)); mExecutionPerfMon = new StandardExecutionPerformanceMonitor(); mTaskQueueConsumer = new PostgreSqlTaskQueueConsumer(consumerOptions, timestampProvider); mTaskQueueProducer = new PostgreSqlTaskQueueProducer(producerAndResultOptions, timestampProvider); mTaskResultQueue = new PostgreSqlTaskResultQueue(producerAndResultOptions); mTaskBuffer = new StandardTaskBuffer(engineOptions.WorkerCount); mTaskPoller = new StandardTaskPoller(engineOptions.TaskProcessingOptions, mTaskQueueConsumer, mTaskBuffer); mOptions = engineOptions; }
public async Task Test_CanDequeue_WithoutTaskTypes() { DateTimeOffset refNow = mDataSource .LastPostedAt .AddMilliseconds(1); int expectedDequeueCount = mDataSource .NumTasksInQueue; using (PostgreSqlTaskQueueConsumer taskQueue = CreateTaskQueue(() => refNow)) using (ConsumedQueuedTaskTokenChecker checker = new ConsumedQueuedTaskTokenChecker(mDataSource)) { for (int i = 0; i < expectedDequeueCount; i++) { await Run_ConsumeTestAsync(taskQueue, checker, refNow); } } }
public async Task Test_StatsAreUpdatedCorrectly_AfterEnqueue_NewTask() { Faker faker = new Faker(); DateTimeOffset postedAt = mDataSource.LastPostedAt .AddTicks(1); PostgreSqlTaskQueueProducer taskQueueProducer = CreateTaskQueueProducer(() => postedAt); PostgreSqlTaskQueueInfo taskQueueInfo = CreateTaskQueueInfo(() => postedAt); using (PostgreSqlTaskQueueConsumer taskQueueConsumer = CreateTaskQueueConsumer(() => postedAt)) using (TaskQueueMetricsDiffChecker diff = new TaskQueueMetricsDiffChecker(async() => await taskQueueInfo.ComputeMetricsAsync())) { await taskQueueConsumer.StartReceivingNewTaskUpdatesAsync(); Assert.IsTrue(taskQueueConsumer.IsReceivingNewTaskUpdates); //Capture previous metrics await diff.CaptureInitialMetricsAsync(); await taskQueueProducer.EnqueueAsync(payload : new SampleTaskPayload(100), source : nameof(Test_StatsAreUpdatedCorrectly_AfterEnqueue_NewTask), priority : faker.Random.Int(1, 100)); //Check that new metrics differ from the previous ones as expected await diff.CaptureNewMetricsAndAssertCorrectDiff(delta : new TaskQueueMetrics( totalUnprocessed: 1, totalProcessing: 0, totalErrored: 0, totalFaulted: 0, totalFataled: 0, totalProcessed: 0)); } }
private async Task Run_ConsumeTestAsync(PostgreSqlTaskQueueConsumer taskQueue, ConsumedQueuedTaskTokenChecker checker, DateTimeOffset refNow, params string[] payloadTypes) { IQueuedTaskToken newTaskToken; if (payloadTypes != null && payloadTypes.Length > 0) { newTaskToken = await taskQueue.DequeueAsync(payloadTypes); } else { newTaskToken = await taskQueue.DequeueAsync(); } checker.AssertConsumedTokenValid(newTaskToken, refNow); await checker.AssertTaskNotInDbAnymoreAsync(newTaskToken); await checker.AssertTaskResultInDbAndCorrectAsync(newTaskToken); }
public async Task Test_CanStartStopReceivingNewTaskNotificationUpdates() { ManualResetEvent notificationWaitHandle = new ManualResetEvent(false); using (PostgreSqlTaskQueueConsumer taskQueue = CreateTaskQueue(() => mDataSource.LastPostedAt)) { taskQueue.ClearForDequeue += (s, e) => notificationWaitHandle.Set(); await taskQueue.StartReceivingNewTaskUpdatesAsync(); Assert.IsTrue(taskQueue.IsReceivingNewTaskUpdates); await SendNewTaskNotificationAsync(); notificationWaitHandle.WaitOne(); await taskQueue.StopReceivingNewTaskUpdatesAsync(); Assert.IsFalse(taskQueue.IsReceivingNewTaskUpdates); } }
public async Task Test_CanDequeue_WithTaskTypes_OneTypePerDequeueCall() { DateTimeOffset refNow = mDataSource .LastPostedAt .AddMilliseconds(1); using (PostgreSqlTaskQueueConsumer taskQueue = CreateTaskQueue(() => refNow)) using (ConsumedQueuedTaskTokenChecker checker = new ConsumedQueuedTaskTokenChecker(mDataSource)) { foreach (Type taskType in mDataSource.InQueueTaskTypes) { int expectedDequeueCount = mDataSource.CountTasksOfTypeInQueue(taskType); for (int i = 0; i < expectedDequeueCount; i++) { await Run_ConsumeTestAsync(taskQueue, checker, refNow, taskType.FullName); } } } }
public async Task Test_CanEnqueue_NewTask_ParallelProducers(int nProducers) { Faker faker = new Faker(); CountdownEvent notificationWaitHandle = new CountdownEvent(nProducers); DateTimeOffset postedAt = mDataSource.LastPostedAt .AddTicks(1); PostgreSqlTaskQueueProducer taskQueueProducer = CreateTaskQueueProducer(() => postedAt); EventHandler <ClearForDequeueEventArgs> handleClearForDequeue = (s, e) => { if (e.Reason == ClearForDequeReason.NewTaskPostedNotificationReceived) { notificationWaitHandle.Signal(); } }; Task[] producers = new Task[nProducers]; using (PostgreSqlTaskQueueConsumer taskQueueConsumer = CreateTaskQueueConsumer(() => postedAt)) { taskQueueConsumer.ClearForDequeue += handleClearForDequeue; await taskQueueConsumer .StartReceivingNewTaskUpdatesAsync(); Assert.IsTrue(taskQueueConsumer .IsReceivingNewTaskUpdates); for (int i = 0; i < nProducers; i++) { producers[i] = Task.Run(async() => { //Enqueue task and check result IQueuedTask queuedTask = await taskQueueProducer .EnqueueAsync(payload: new SampleTaskPayload(100), source: nameof(Test_CanEnqueue_NewTask_ParallelProducers), priority: faker.Random.Int(1, 100)); Assert.NotNull(queuedTask); await Assert_ResultAddedOrUpdatedCorrectly(queuedTask); }); } notificationWaitHandle.Wait(); await taskQueueConsumer .StopReceivingNewTaskUpdatesAsync(); Assert.IsFalse(taskQueueConsumer .IsReceivingNewTaskUpdates); taskQueueConsumer.ClearForDequeue -= handleClearForDequeue; } }