public async Task <IQueuedTask> EnqueueAsync(QueuedTaskInfo queuedTaskInfo) { if (queuedTaskInfo == null) { throw new ArgumentNullException(nameof(queuedTaskInfo)); } if (queuedTaskInfo.Priority < 0) { throw new ArgumentOutOfRangeException(nameof(queuedTaskInfo), "Priority must be greater than or equal to 0"); } QueuedTask queuedTask = CreateNewTaskFromInfo(queuedTaskInfo); using (NpgsqlConnection conn = await TryOpenConnectionAsync()) using (NpgsqlTransaction tx = conn.BeginTransaction()) { queuedTask = await TryPostTaskAsync(queuedTask, conn, tx); await TryInitOrUpdateResultAsync(queuedTask, conn, tx); await NotifyNewTaskPostedAsync(queuedTask, conn, tx); tx.Commit(); } return(queuedTask); }
public void Test_CanUpdateFromExecutionResult_Successful() { Faker faker = new Faker(); DateTimeOffset now = DateTimeOffset .UtcNow; Faker <QueuedTask> taskFaker = GetQueuedTaskFaker(); QueuedTask task = taskFaker .Generate(); QueuedTaskResult result = new QueuedTaskResult(task); TaskExecutionResult successful = new TaskExecutionResult(TaskExecutionResultInfo.Successful(), duration: faker.Date.Timespan(), retryAt: faker.Date.FutureOffset(), faultErrorThresholdCount: faker.Random.Int(1, 5)); QueuedTaskInfo repostWithInfo = result.UdpateFromExecutionResult(successful); Assert.Null(repostWithInfo); Assert.IsNull(result.LastError); Assert.AreEqual(QueuedTaskStatus.Processed, result.Status); Assert.AreEqual(successful.ProcessingTimeMilliseconds, result.ProcessingTimeMilliseconds); Assert.GreaterOrEqual(result.ProcessingFinalizedAtTs, now); Assert.GreaterOrEqual(result.FirstProcessingAttemptedAtTs, now); Assert.GreaterOrEqual(result.LastProcessingAttemptedAtTs, now); }
private async Task ProcessResultAsync ( IQueuedTaskToken queuedTaskToken, TaskExecutionResult result ) { try { //Update worker execution stats UpdateTaskProcessingStats( result ); //There is no result - most likely, no executor found; // nothing to process, just stop and return if ( !result.HasResult ) { mLogger.Debug( "No result info returned. Task will be discarded." ); return; } //Execution has been cancelled, usually as a response // to a cancellation request; // nothing to process, just stop and return if ( result.ExecutionCancelled ) { mLogger.Debug( "Task execution cancelled. Task will be discarded." ); return; } //Update execution result and see whether // we need to repost the task to retry its execution QueuedTaskInfo repostWithInfo = queuedTaskToken .UdpateFromExecutionResult( result ); //Post result mLogger.Debug( "Will post task execution result." ); await mTaskResultQueue.PostResultAsync( queuedTaskToken ); mLogger.Debug( "Successfully posted task execution result." ); //If the task needs to be reposted, do so if ( repostWithInfo != null ) { mLogger.Debug( "Will repost task for execution." ); await mTaskQueueProducer.EnqueueAsync( repostWithInfo ); mLogger.Debug( "Sucessfully reposted task for execution." ); } else mLogger.Debug( "Will not repost task for execution." ); //Finally, report execution time await mPerformanceMonitor.ReportExecutionTimeAsync( queuedTaskToken, result ); } catch ( Exception exc ) { mLogger.Error( "Failed to set queued task result. Task will be discarded.", exc ); } }
public async Task Test_StatsAreUpdatedCorrectly_AfterEnqueue_RepostExistingTask() { Faker faker = new Faker(); DateTimeOffset postedAt = mDataSource.LastPostedAt .AddTicks(1); PostgreSqlTaskQueueProducer taskQueueProducer = CreateTaskQueueProducer(() => postedAt); PostgreSqlTaskQueueInfo taskQueueInfo = CreateTaskQueueInfo(() => postedAt); foreach (IQueuedTaskToken token in mDataSource.CanBeRepostedSeededTaskTokens) { using (TaskQueueMetricsDiffChecker diff = new TaskQueueMetricsDiffChecker(async() => await taskQueueInfo.ComputeMetricsAsync())) { QueuedTaskStatus prevStatus = token .LastQueuedTaskResult .Status; await diff.CaptureInitialMetricsAsync(); QueuedTaskInfo repostTaskInfo = new QueuedTaskInfo() { Id = token.DequeuedTask.Id, Priority = faker.Random.Int(1, 100), Payload = token.DequeuedTask.Payload, Source = nameof(Test_StatsAreUpdatedCorrectly_AfterEnqueue_RepostExistingTask), Type = token.DequeuedTask.Type, LockedUntilTs = postedAt.AddMinutes(faker.Random.Long(1000, 10000)) }; //Remove task record from DB - only dequeued tasks get reposted await mDataSource.RemoveQueuedTaskFromDbByIdAsync(token .DequeuedTask .Id); await taskQueueProducer.EnqueueAsync(repostTaskInfo); await diff.CaptureNewMetricsAndAssertCorrectDiff(delta : new TaskQueueMetrics( totalUnprocessed: prevStatus != QueuedTaskStatus.Unprocessed ? 1 : 0, totalProcessing: prevStatus == QueuedTaskStatus.Processing ? -1 : 0, totalErrored: prevStatus == QueuedTaskStatus.Error ? -1 : 0, totalFaulted: prevStatus == QueuedTaskStatus.Faulted ? -1 : 0, totalFataled: prevStatus == QueuedTaskStatus.Fatal ? -1 : 0, totalProcessed: prevStatus == QueuedTaskStatus.Processed ? -1 : 0)); } } }
private QueuedTask CreateNewTaskFromInfo(QueuedTaskInfo queuedTaskInfo) { QueuedTask queuedTask = new QueuedTask(); queuedTask.Id = queuedTaskInfo.HasId ? queuedTaskInfo.Id : Guid.NewGuid(); queuedTask.Payload = queuedTaskInfo.Payload; queuedTask.Type = queuedTaskInfo.Type; queuedTask.Source = queuedTaskInfo.Source; queuedTask.Priority = queuedTaskInfo.Priority; queuedTask.PostedAtTs = mTimestampProvider.GetNow(); queuedTask.LockedUntilTs = queuedTaskInfo.LockedUntilTs; return(queuedTask); }
public async Task Test_CanEnqueue_RepostExistingTask_Serial() { Faker faker = new Faker(); DateTimeOffset postedAt = mDataSource.LastPostedAt .AddSeconds(1); PostgreSqlTaskQueueProducer taskQueueProducer = CreateTaskQueueProducer(() => postedAt); foreach (IQueuedTaskToken token in mDataSource.CanBeRepostedSeededTaskTokens) { QueuedTaskInfo repostTaskInfo = new QueuedTaskInfo() { Id = token.DequeuedTask.Id, Priority = faker.Random.Int(1, 100), Payload = token.DequeuedTask.Payload, Source = nameof(Test_CanEnqueue_RepostExistingTask_Serial), Type = token.DequeuedTask.Type, LockedUntilTs = postedAt.AddMilliseconds(faker.Random.Long(1000, 10000)) }; //Remove task record from DB - only dequeued tasks get reposted await mDataSource.RemoveQueuedTaskFromDbByIdAsync(token .DequeuedTask .Id); //Enqueue task and check result IQueuedTask requeuedTask = await taskQueueProducer .EnqueueAsync(repostTaskInfo); Assert.NotNull(requeuedTask); await Assert_ResultAddedOrUpdatedCorrectly(requeuedTask); } }
public void Test_CanUpdateFromExecutionResult_WithError_Recoverable(int faultErrorThresholdCount) { Faker faker = new Faker(); QueuedTaskInfo repostWithInfo = null; DateTimeOffset now = DateTimeOffset .UtcNow; Faker <QueuedTask> taskFaker = GetQueuedTaskFaker(); QueuedTask task = taskFaker .Generate(); QueuedTaskResult result = new QueuedTaskResult(task); TaskExecutionResultInfo failedWithErrorInfo = TaskExecutionResultInfo .ExecutedWithError(new QueuedTaskError(faker.System.Exception()), isRecoverable: true); TaskExecutionResult failedWithError = new TaskExecutionResult(failedWithErrorInfo, duration: faker.Date.Timespan(), retryAt: faker.Date.FutureOffset(), faultErrorThresholdCount: faultErrorThresholdCount); //1 to faultErrorThresholdCount -> Error status for (int i = 1; i <= faultErrorThresholdCount; i++) { repostWithInfo = result.UdpateFromExecutionResult(failedWithError); Assert.NotNull(repostWithInfo); Assert.AreEqual(QueuedTaskStatus.Error, result.Status); Assert.AreEqual(0, result.ProcessingTimeMilliseconds); Assert.GreaterOrEqual(result.FirstProcessingAttemptedAtTs, now); Assert.GreaterOrEqual(result.LastProcessingAttemptedAtTs, now); Assert.AreEqual(failedWithErrorInfo.Error, result.LastError); Assert.AreEqual(i, result.ErrorCount); } //Antoher failure -> Faulted repostWithInfo = result.UdpateFromExecutionResult(failedWithError); Assert.NotNull(repostWithInfo); Assert.AreEqual(QueuedTaskStatus.Faulted, result.Status); Assert.AreEqual(0, result.ProcessingTimeMilliseconds); Assert.GreaterOrEqual(result.FirstProcessingAttemptedAtTs, now); Assert.GreaterOrEqual(result.LastProcessingAttemptedAtTs, now); Assert.AreEqual(failedWithErrorInfo.Error, result.LastError); Assert.AreEqual(faultErrorThresholdCount + 1, result.ErrorCount); //Antoher failure after that -> Fataled repostWithInfo = result.UdpateFromExecutionResult(failedWithError); Assert.Null(repostWithInfo); Assert.AreEqual(QueuedTaskStatus.Fatal, result.Status); Assert.AreEqual(0, result.ProcessingTimeMilliseconds); Assert.GreaterOrEqual(result.FirstProcessingAttemptedAtTs, now); Assert.GreaterOrEqual(result.LastProcessingAttemptedAtTs, now); Assert.AreEqual(failedWithErrorInfo.Error, result.LastError); Assert.AreEqual(faultErrorThresholdCount + 2, result.ErrorCount); }