public async Task <IQueuedTask> EnqueueAsync(QueuedTaskInfo queuedTaskInfo)
{
if (queuedTaskInfo == null)
{
throw new ArgumentNullException(nameof(queuedTaskInfo));
}
if (queuedTaskInfo.Priority < 0)
{
throw new ArgumentOutOfRangeException(nameof(queuedTaskInfo), "Priority must be greater than or equal to 0");
}
QueuedTask queuedTask = CreateNewTaskFromInfo(queuedTaskInfo);
using (NpgsqlConnection conn = await TryOpenConnectionAsync())
using (NpgsqlTransaction tx = conn.BeginTransaction())
{
queuedTask = await TryPostTaskAsync(queuedTask, conn, tx);
await TryInitOrUpdateResultAsync(queuedTask, conn, tx);
await NotifyNewTaskPostedAsync(queuedTask, conn, tx);
tx.Commit();
}
return(queuedTask);
}
public void Test_CanUpdateFromExecutionResult_Successful()
{
Faker faker = new Faker();
DateTimeOffset now = DateTimeOffset
.UtcNow;
Faker <QueuedTask> taskFaker =
GetQueuedTaskFaker();
QueuedTask task = taskFaker
.Generate();
QueuedTaskResult result = new QueuedTaskResult(task);
TaskExecutionResult successful = new TaskExecutionResult(TaskExecutionResultInfo.Successful(),
duration: faker.Date.Timespan(),
retryAt: faker.Date.FutureOffset(),
faultErrorThresholdCount: faker.Random.Int(1, 5));
QueuedTaskInfo repostWithInfo = result.UdpateFromExecutionResult(successful);
Assert.Null(repostWithInfo);
Assert.IsNull(result.LastError);
Assert.AreEqual(QueuedTaskStatus.Processed, result.Status);
Assert.AreEqual(successful.ProcessingTimeMilliseconds, result.ProcessingTimeMilliseconds);
Assert.GreaterOrEqual(result.ProcessingFinalizedAtTs, now);
Assert.GreaterOrEqual(result.FirstProcessingAttemptedAtTs, now);
Assert.GreaterOrEqual(result.LastProcessingAttemptedAtTs, now);
}
private async Task ProcessResultAsync ( IQueuedTaskToken queuedTaskToken,
TaskExecutionResult result )
{
try
{
//Update worker execution stats
UpdateTaskProcessingStats( result );
//There is no result - most likely, no executor found;
// nothing to process, just stop and return
if ( !result.HasResult )
{
mLogger.Debug( "No result info returned. Task will be discarded." );
return;
}
//Execution has been cancelled, usually as a response
// to a cancellation request;
// nothing to process, just stop and return
if ( result.ExecutionCancelled )
{
mLogger.Debug( "Task execution cancelled. Task will be discarded." );
return;
}
//Update execution result and see whether
// we need to repost the task to retry its execution
QueuedTaskInfo repostWithInfo = queuedTaskToken
.UdpateFromExecutionResult( result );
//Post result
mLogger.Debug( "Will post task execution result." );
await mTaskResultQueue.PostResultAsync( queuedTaskToken );
mLogger.Debug( "Successfully posted task execution result." );
//If the task needs to be reposted, do so
if ( repostWithInfo != null )
{
mLogger.Debug( "Will repost task for execution." );
await mTaskQueueProducer.EnqueueAsync( repostWithInfo );
mLogger.Debug( "Sucessfully reposted task for execution." );
}
else
mLogger.Debug( "Will not repost task for execution." );
//Finally, report execution time
await mPerformanceMonitor.ReportExecutionTimeAsync( queuedTaskToken,
result );
}
catch ( Exception exc )
{
mLogger.Error( "Failed to set queued task result. Task will be discarded.",
exc );
}
}
public async Task Test_StatsAreUpdatedCorrectly_AfterEnqueue_RepostExistingTask()
{
Faker faker =
new Faker();
DateTimeOffset postedAt = mDataSource.LastPostedAt
.AddTicks(1);
PostgreSqlTaskQueueProducer taskQueueProducer =
CreateTaskQueueProducer(() => postedAt);
PostgreSqlTaskQueueInfo taskQueueInfo =
CreateTaskQueueInfo(() => postedAt);
foreach (IQueuedTaskToken token in mDataSource.CanBeRepostedSeededTaskTokens)
{
using (TaskQueueMetricsDiffChecker diff = new TaskQueueMetricsDiffChecker(async()
=> await taskQueueInfo.ComputeMetricsAsync()))
{
QueuedTaskStatus prevStatus = token
.LastQueuedTaskResult
.Status;
await diff.CaptureInitialMetricsAsync();
QueuedTaskInfo repostTaskInfo = new QueuedTaskInfo()
{
Id = token.DequeuedTask.Id,
Priority = faker.Random.Int(1, 100),
Payload = token.DequeuedTask.Payload,
Source = nameof(Test_StatsAreUpdatedCorrectly_AfterEnqueue_RepostExistingTask),
Type = token.DequeuedTask.Type,
LockedUntilTs = postedAt.AddMinutes(faker.Random.Long(1000, 10000))
};
//Remove task record from DB - only dequeued tasks get reposted
await mDataSource.RemoveQueuedTaskFromDbByIdAsync(token
.DequeuedTask
.Id);
await taskQueueProducer.EnqueueAsync(repostTaskInfo);
await diff.CaptureNewMetricsAndAssertCorrectDiff(delta : new TaskQueueMetrics(
totalUnprocessed: prevStatus != QueuedTaskStatus.Unprocessed ? 1 : 0,
totalProcessing: prevStatus == QueuedTaskStatus.Processing ? -1 : 0,
totalErrored: prevStatus == QueuedTaskStatus.Error ? -1 : 0,
totalFaulted: prevStatus == QueuedTaskStatus.Faulted ? -1 : 0,
totalFataled: prevStatus == QueuedTaskStatus.Fatal ? -1 : 0,
totalProcessed: prevStatus == QueuedTaskStatus.Processed ? -1 : 0));
}
}
}
private QueuedTask CreateNewTaskFromInfo(QueuedTaskInfo queuedTaskInfo)
{
QueuedTask queuedTask =
new QueuedTask();
queuedTask.Id = queuedTaskInfo.HasId
? queuedTaskInfo.Id
: Guid.NewGuid();
queuedTask.Payload = queuedTaskInfo.Payload;
queuedTask.Type = queuedTaskInfo.Type;
queuedTask.Source = queuedTaskInfo.Source;
queuedTask.Priority = queuedTaskInfo.Priority;
queuedTask.PostedAtTs = mTimestampProvider.GetNow();
queuedTask.LockedUntilTs = queuedTaskInfo.LockedUntilTs;
return(queuedTask);
}
public async Task Test_CanEnqueue_RepostExistingTask_Serial()
{
Faker faker =
new Faker();
DateTimeOffset postedAt = mDataSource.LastPostedAt
.AddSeconds(1);
PostgreSqlTaskQueueProducer taskQueueProducer =
CreateTaskQueueProducer(() => postedAt);
foreach (IQueuedTaskToken token in mDataSource.CanBeRepostedSeededTaskTokens)
{
QueuedTaskInfo repostTaskInfo = new QueuedTaskInfo()
{
Id = token.DequeuedTask.Id,
Priority = faker.Random.Int(1, 100),
Payload = token.DequeuedTask.Payload,
Source = nameof(Test_CanEnqueue_RepostExistingTask_Serial),
Type = token.DequeuedTask.Type,
LockedUntilTs = postedAt.AddMilliseconds(faker.Random.Long(1000, 10000))
};
//Remove task record from DB - only dequeued tasks get reposted
await mDataSource.RemoveQueuedTaskFromDbByIdAsync(token
.DequeuedTask
.Id);
//Enqueue task and check result
IQueuedTask requeuedTask = await taskQueueProducer
.EnqueueAsync(repostTaskInfo);
Assert.NotNull(requeuedTask);
await Assert_ResultAddedOrUpdatedCorrectly(requeuedTask);
}
}
public void Test_CanUpdateFromExecutionResult_WithError_Recoverable(int faultErrorThresholdCount)
{
Faker faker = new Faker();
QueuedTaskInfo repostWithInfo = null;
DateTimeOffset now = DateTimeOffset
.UtcNow;
Faker <QueuedTask> taskFaker =
GetQueuedTaskFaker();
QueuedTask task = taskFaker
.Generate();
QueuedTaskResult result = new QueuedTaskResult(task);
TaskExecutionResultInfo failedWithErrorInfo = TaskExecutionResultInfo
.ExecutedWithError(new QueuedTaskError(faker.System.Exception()),
isRecoverable: true);
TaskExecutionResult failedWithError = new TaskExecutionResult(failedWithErrorInfo,
duration: faker.Date.Timespan(),
retryAt: faker.Date.FutureOffset(),
faultErrorThresholdCount: faultErrorThresholdCount);
//1 to faultErrorThresholdCount -> Error status
for (int i = 1; i <= faultErrorThresholdCount; i++)
{
repostWithInfo = result.UdpateFromExecutionResult(failedWithError);
Assert.NotNull(repostWithInfo);
Assert.AreEqual(QueuedTaskStatus.Error, result.Status);
Assert.AreEqual(0, result.ProcessingTimeMilliseconds);
Assert.GreaterOrEqual(result.FirstProcessingAttemptedAtTs, now);
Assert.GreaterOrEqual(result.LastProcessingAttemptedAtTs, now);
Assert.AreEqual(failedWithErrorInfo.Error, result.LastError);
Assert.AreEqual(i, result.ErrorCount);
}
//Antoher failure -> Faulted
repostWithInfo = result.UdpateFromExecutionResult(failedWithError);
Assert.NotNull(repostWithInfo);
Assert.AreEqual(QueuedTaskStatus.Faulted, result.Status);
Assert.AreEqual(0, result.ProcessingTimeMilliseconds);
Assert.GreaterOrEqual(result.FirstProcessingAttemptedAtTs, now);
Assert.GreaterOrEqual(result.LastProcessingAttemptedAtTs, now);
Assert.AreEqual(failedWithErrorInfo.Error, result.LastError);
Assert.AreEqual(faultErrorThresholdCount + 1, result.ErrorCount);
//Antoher failure after that -> Fataled
repostWithInfo = result.UdpateFromExecutionResult(failedWithError);
Assert.Null(repostWithInfo);
Assert.AreEqual(QueuedTaskStatus.Fatal, result.Status);
Assert.AreEqual(0, result.ProcessingTimeMilliseconds);
Assert.GreaterOrEqual(result.FirstProcessingAttemptedAtTs, now);
Assert.GreaterOrEqual(result.LastProcessingAttemptedAtTs, now);
Assert.AreEqual(failedWithErrorInfo.Error, result.LastError);
Assert.AreEqual(faultErrorThresholdCount + 2, result.ErrorCount);
}
