public async Task Test_CanDequeue_WithTaskTypes_MultipleTypesPerDequeueCall()
        {
            DateTimeOffset refNow = mDataSource
                                    .LastPostedAt
                                    .AddMilliseconds(1);

            int expectedDequeueCount = mDataSource
                                       .NumTasksInQueue;

            string[] taskTypes = mDataSource.InQueueTaskTypes
                                 .Select(t => t.FullName)
                                 .ToArray();

            using (PostgreSqlTaskQueueConsumer taskQueue =
                       CreateTaskQueue(() => refNow))
                using (ConsumedQueuedTaskTokenChecker checker =
                           new ConsumedQueuedTaskTokenChecker(mDataSource))
                {
                    for (int i = 0; i < expectedDequeueCount; i++)
                    {
                        await Run_ConsumeTestAsync(taskQueue,
                                                   checker,
                                                   refNow,
                                                   taskTypes);
                    }
                }
        }
Exemplo n.º 2
0
        protected void Dispose(bool disposing)
        {
            if (!mIsDisposed)
            {
                if (disposing)
                {
                    StopAync().Wait();

                    mTaskPoller.Dispose();
                    mTaskBuffer.Dispose();

                    if (mTaskResultQueue is IDisposable)
                    {
                        (( IDisposable )mTaskResultQueue).Dispose();
                    }

                    mTaskPoller      = null;
                    mTaskBuffer      = null;
                    mTaskResultQueue = null;

                    mTaskQueueConsumer = null;
                    mExecutorRegistry  = null;
                }

                mIsDisposed = true;
            }
        }
        public async Task Test_PeekMatchesDequeuedItem_SingleConsumer()
        {
            IQueuedTask      peekTask          = null;
            IQueuedTaskToken dequeuedTaskToken = null;

            PostgreSqlTaskQueueInfo taskQueueInfo =
                CreateTaskQueueInfo(() => mDataSource.LastPostedAt);

            using (PostgreSqlTaskQueueConsumer taskQueue =
                       CreateTaskQueueConsumer(() => mDataSource.LastPostedAt))
            {
                int expectedDequeueCount = mDataSource
                                           .NumTasksInQueue;

                for (int i = 0; i < expectedDequeueCount; i++)
                {
                    peekTask = await taskQueueInfo.PeekAsync();

                    Assert.NotNull(peekTask);

                    dequeuedTaskToken = await taskQueue.DequeueAsync();

                    Assert.NotNull(dequeuedTaskToken);

                    Assert.AreEqual(peekTask.Id, dequeuedTaskToken
                                    .DequeuedTask
                                    .Id);
                }
            }
        }
        public async Task Test_StatsAreCorrectlyUpdated_AfterDequeue_NoTaskType()
        {
            PostgreSqlTaskQueueInfo taskQueueInfo =
                CreateTaskQueueInfo(() => mDataSource.LastPostedAt);

            using (PostgreSqlTaskQueueConsumer taskQueueConsumer =
                       CreateTaskQueueConsumer(() => mDataSource.LastPostedAt))
                using (TaskQueueMetricsDiffChecker diff = new TaskQueueMetricsDiffChecker(async()
                                                                                          => await taskQueueInfo.ComputeMetricsAsync()))
                {
                    await diff.CaptureInitialMetricsAsync();

                    IQueuedTaskToken dequeuedToken = await taskQueueConsumer
                                                     .DequeueAsync();

                    QueuedTaskStatus origStatus = mDataSource
                                                  .GetOriginalTokenData(dequeuedToken.DequeuedTask.Id)
                                                  .LastQueuedTaskResult
                                                  .Status;

                    await diff.CaptureNewMetricsAndAssertCorrectDiff(delta : new TaskQueueMetrics(
                                                                         totalUnprocessed: origStatus == QueuedTaskStatus.Unprocessed ? -1 : 0,
                                                                         totalProcessing: 1,
                                                                         totalErrored: origStatus == QueuedTaskStatus.Error ? -1 : 0,
                                                                         totalFaulted: origStatus == QueuedTaskStatus.Faulted ? -1 : 0,
                                                                         totalFataled: origStatus == QueuedTaskStatus.Fatal ? -1 : 0,
                                                                         totalProcessed: origStatus == QueuedTaskStatus.Processed ? -1 : 0));
                }
        }
        public async Task Test_DequeueChangesPeekResult_SingleConsumer()
        {
            IQueuedTask peekTask   = null,
                        rePeekTask = null;

            IQueuedTaskToken dequeuedTaskToken = null;

            PostgreSqlTaskQueueInfo taskQueueInfo =
                CreateTaskQueueInfo(() => mDataSource.LastPostedAt);

            using (PostgreSqlTaskQueueConsumer taskQueueConsumer =
                       CreateTaskQueueConsumer(() => mDataSource.LastPostedAt))
            {
                peekTask = await taskQueueInfo.PeekAsync();

                Assert.NotNull(peekTask);

                dequeuedTaskToken = await taskQueueConsumer.DequeueAsync();

                Assert.NotNull(dequeuedTaskToken);

                rePeekTask = await taskQueueInfo.PeekAsync();

                Assert.NotNull(rePeekTask);

                //Removing a new element from the queue
                //  occurs at the beginning of the queue,
                //  so peeking must yield a different result
                //  than before dequeue-ing
                Assert.AreNotEqual(rePeekTask.Id,
                                   peekTask.Id);
            }
        }
Exemplo n.º 6
0
        public async Task Test_CanEnqueue_NewTask_Serial()
        {
            Faker faker =
                new Faker();

            ManualResetEvent notificationWaitHandle =
                new ManualResetEvent(false);

            DateTimeOffset postedAt = mDataSource.LastPostedAt
                                      .AddTicks(1);

            PostgreSqlTaskQueueProducer taskQueueProducer =
                CreateTaskQueueProducer(() => postedAt);

            EventHandler <ClearForDequeueEventArgs> handleClearForDequeue = (s, e) =>
            {
                if (e.Reason == ClearForDequeReason.NewTaskPostedNotificationReceived)
                {
                    notificationWaitHandle.Set();
                }
            };

            using (PostgreSqlTaskQueueConsumer taskQueueConsumer =
                       CreateTaskQueueConsumer(() => postedAt))
            {
                taskQueueConsumer.ClearForDequeue +=
                    handleClearForDequeue;

                await taskQueueConsumer
                .StartReceivingNewTaskUpdatesAsync();

                Assert.IsTrue(taskQueueConsumer
                              .IsReceivingNewTaskUpdates);

                //Enqueue task and check result
                IQueuedTask queuedTask = await taskQueueProducer
                                         .EnqueueAsync(payload : new SampleTaskPayload(100),
                                                       source : nameof(Test_CanEnqueue_NewTask_Serial),
                                                       priority : faker.Random.Int(1, 100));

                Assert.NotNull(queuedTask);
                await Assert_ResultAddedOrUpdatedCorrectly(queuedTask);

                notificationWaitHandle.WaitOne();

                await taskQueueConsumer
                .StopReceivingNewTaskUpdatesAsync();

                Assert.IsFalse(taskQueueConsumer
                               .IsReceivingNewTaskUpdates);

                taskQueueConsumer.ClearForDequeue -=
                    handleClearForDequeue;
            }
        }
        public async Task Test_CanDequeue_WithTaskTypes_MultipleTypesPerDequeueCall_ParallelConsumers(int nConsumers)
        {
            int expectedDequeueCount = mDataSource
                                       .NumTasksInQueue;

            DateTimeOffset refNow = mDataSource
                                    .LastPostedAt
                                    .AddMilliseconds(1);

            string[] taskTypes = mDataSource.InQueueTaskTypes
                                 .Select(t => t.FullName)
                                 .ToArray();

            Task[] consumers = new Task[nConsumers];

            int[] loopPartitions = expectedDequeueCount
                                   .PartitionValue(nConsumers);

            Assert.AreEqual(expectedDequeueCount,
                            loopPartitions.Sum());

            for (int iConsumer = 0; iConsumer < nConsumers; iConsumer++)
            {
                int loopCount = loopPartitions[iConsumer];
                consumers[iConsumer] = Task.Run(async() =>
                {
                    using (PostgreSqlTaskQueueConsumer taskQueue =
                               CreateTaskQueue(() => refNow))
                        using (ConsumedQueuedTaskTokenChecker checker =
                                   new ConsumedQueuedTaskTokenChecker(mDataSource))
                        {
                            for (int iTest = 0; iTest < loopCount; iTest++)
                            {
                                await Run_ConsumeTestAsync(taskQueue,
                                                           checker,
                                                           refNow,
                                                           taskTypes);
                            }
                        }
                });
            }

            await Task.WhenAll(consumers);
        }
Exemplo n.º 8
0
        public StandardTaskEngine(TaskEngineOptions engineOptions,
                                  TaskQueueOptions producerAndResultOptions,
                                  TaskQueueConsumerOptions consumerOptions,
                                  ITaskExecutorRegistry executorRegistry,
                                  IExecutionPerformanceMonitorWriter executionPerfMonWriter,
                                  ITimestampProvider timestampProvider)
        {
            if (engineOptions == null)
            {
                throw new ArgumentNullException(nameof(engineOptions));
            }

            if (consumerOptions == null)
            {
                throw new ArgumentNullException(nameof(consumerOptions));
            }

            if (producerAndResultOptions == null)
            {
                throw new ArgumentNullException(nameof(producerAndResultOptions));
            }

            mExecutorRegistry = executorRegistry
                                ?? throw new ArgumentNullException(nameof(executorRegistry));

            mExecutionPerfMonWriter = executionPerfMonWriter
                                      ?? throw new ArgumentNullException(nameof(executionPerfMonWriter));

            mExecutionPerfMon  = new StandardExecutionPerformanceMonitor();
            mTaskQueueConsumer = new PostgreSqlTaskQueueConsumer(consumerOptions,
                                                                 timestampProvider);
            mTaskQueueProducer = new PostgreSqlTaskQueueProducer(producerAndResultOptions,
                                                                 timestampProvider);

            mTaskResultQueue = new PostgreSqlTaskResultQueue(producerAndResultOptions);

            mTaskBuffer = new StandardTaskBuffer(engineOptions.WorkerCount);
            mTaskPoller = new StandardTaskPoller(engineOptions.TaskProcessingOptions,
                                                 mTaskQueueConsumer,
                                                 mTaskBuffer);

            mOptions = engineOptions;
        }
        public async Task Test_CanDequeue_WithoutTaskTypes()
        {
            DateTimeOffset refNow = mDataSource
                                    .LastPostedAt
                                    .AddMilliseconds(1);

            int expectedDequeueCount = mDataSource
                                       .NumTasksInQueue;

            using (PostgreSqlTaskQueueConsumer taskQueue =
                       CreateTaskQueue(() => refNow))
                using (ConsumedQueuedTaskTokenChecker checker =
                           new ConsumedQueuedTaskTokenChecker(mDataSource))
                {
                    for (int i = 0; i < expectedDequeueCount; i++)
                    {
                        await Run_ConsumeTestAsync(taskQueue, checker, refNow);
                    }
                }
        }
        public async Task Test_StatsAreUpdatedCorrectly_AfterEnqueue_NewTask()
        {
            Faker faker =
                new Faker();

            DateTimeOffset postedAt = mDataSource.LastPostedAt
                                      .AddTicks(1);

            PostgreSqlTaskQueueProducer taskQueueProducer =
                CreateTaskQueueProducer(() => postedAt);

            PostgreSqlTaskQueueInfo taskQueueInfo =
                CreateTaskQueueInfo(() => postedAt);

            using (PostgreSqlTaskQueueConsumer taskQueueConsumer =
                       CreateTaskQueueConsumer(() => postedAt))
                using (TaskQueueMetricsDiffChecker diff = new TaskQueueMetricsDiffChecker(async()
                                                                                          => await taskQueueInfo.ComputeMetricsAsync()))
                {
                    await taskQueueConsumer.StartReceivingNewTaskUpdatesAsync();

                    Assert.IsTrue(taskQueueConsumer.IsReceivingNewTaskUpdates);

                    //Capture previous metrics
                    await diff.CaptureInitialMetricsAsync();

                    await taskQueueProducer.EnqueueAsync(payload : new SampleTaskPayload(100),
                                                         source : nameof(Test_StatsAreUpdatedCorrectly_AfterEnqueue_NewTask),
                                                         priority : faker.Random.Int(1, 100));

                    //Check that new metrics differ from the previous ones as expected
                    await diff.CaptureNewMetricsAndAssertCorrectDiff(delta : new TaskQueueMetrics(
                                                                         totalUnprocessed: 1,
                                                                         totalProcessing: 0,
                                                                         totalErrored: 0,
                                                                         totalFaulted: 0,
                                                                         totalFataled: 0,
                                                                         totalProcessed: 0));
                }
        }
        private async Task Run_ConsumeTestAsync(PostgreSqlTaskQueueConsumer taskQueue,
                                                ConsumedQueuedTaskTokenChecker checker,
                                                DateTimeOffset refNow,
                                                params string[] payloadTypes)
        {
            IQueuedTaskToken newTaskToken;

            if (payloadTypes != null && payloadTypes.Length > 0)
            {
                newTaskToken = await taskQueue.DequeueAsync(payloadTypes);
            }
            else
            {
                newTaskToken = await taskQueue.DequeueAsync();
            }

            checker.AssertConsumedTokenValid(newTaskToken, refNow);

            await checker.AssertTaskNotInDbAnymoreAsync(newTaskToken);

            await checker.AssertTaskResultInDbAndCorrectAsync(newTaskToken);
        }
        public async Task Test_CanStartStopReceivingNewTaskNotificationUpdates()
        {
            ManualResetEvent notificationWaitHandle = new
                                                      ManualResetEvent(false);

            using (PostgreSqlTaskQueueConsumer taskQueue = CreateTaskQueue(() => mDataSource.LastPostedAt))
            {
                taskQueue.ClearForDequeue += (s, e) =>
                                             notificationWaitHandle.Set();

                await taskQueue.StartReceivingNewTaskUpdatesAsync();

                Assert.IsTrue(taskQueue.IsReceivingNewTaskUpdates);

                await SendNewTaskNotificationAsync();

                notificationWaitHandle.WaitOne();

                await taskQueue.StopReceivingNewTaskUpdatesAsync();

                Assert.IsFalse(taskQueue.IsReceivingNewTaskUpdates);
            }
        }
        public async Task Test_CanDequeue_WithTaskTypes_OneTypePerDequeueCall()
        {
            DateTimeOffset refNow = mDataSource
                                    .LastPostedAt
                                    .AddMilliseconds(1);

            using (PostgreSqlTaskQueueConsumer taskQueue =
                       CreateTaskQueue(() => refNow))
                using (ConsumedQueuedTaskTokenChecker checker =
                           new ConsumedQueuedTaskTokenChecker(mDataSource))
                {
                    foreach (Type taskType in mDataSource.InQueueTaskTypes)
                    {
                        int expectedDequeueCount = mDataSource.CountTasksOfTypeInQueue(taskType);
                        for (int i = 0; i < expectedDequeueCount; i++)
                        {
                            await Run_ConsumeTestAsync(taskQueue,
                                                       checker,
                                                       refNow,
                                                       taskType.FullName);
                        }
                    }
                }
        }
Exemplo n.º 14
0
        public async Task Test_CanEnqueue_NewTask_ParallelProducers(int nProducers)
        {
            Faker faker =
                new Faker();

            CountdownEvent notificationWaitHandle =
                new CountdownEvent(nProducers);

            DateTimeOffset postedAt = mDataSource.LastPostedAt
                                      .AddTicks(1);

            PostgreSqlTaskQueueProducer taskQueueProducer =
                CreateTaskQueueProducer(() => postedAt);

            EventHandler <ClearForDequeueEventArgs> handleClearForDequeue = (s, e) =>
            {
                if (e.Reason == ClearForDequeReason.NewTaskPostedNotificationReceived)
                {
                    notificationWaitHandle.Signal();
                }
            };

            Task[] producers = new Task[nProducers];

            using (PostgreSqlTaskQueueConsumer taskQueueConsumer =
                       CreateTaskQueueConsumer(() => postedAt))
            {
                taskQueueConsumer.ClearForDequeue +=
                    handleClearForDequeue;

                await taskQueueConsumer
                .StartReceivingNewTaskUpdatesAsync();

                Assert.IsTrue(taskQueueConsumer
                              .IsReceivingNewTaskUpdates);

                for (int i = 0; i < nProducers; i++)
                {
                    producers[i] = Task.Run(async() =>
                    {
                        //Enqueue task and check result
                        IQueuedTask queuedTask = await taskQueueProducer
                                                 .EnqueueAsync(payload: new SampleTaskPayload(100),
                                                               source: nameof(Test_CanEnqueue_NewTask_ParallelProducers),
                                                               priority: faker.Random.Int(1, 100));

                        Assert.NotNull(queuedTask);
                        await Assert_ResultAddedOrUpdatedCorrectly(queuedTask);
                    });
                }

                notificationWaitHandle.Wait();

                await taskQueueConsumer
                .StopReceivingNewTaskUpdatesAsync();

                Assert.IsFalse(taskQueueConsumer
                               .IsReceivingNewTaskUpdates);

                taskQueueConsumer.ClearForDequeue -=
                    handleClearForDequeue;
            }
        }