public void Test_CanUpdateFromExecutionResult_Successful()
{
Faker faker = new Faker();
DateTimeOffset now = DateTimeOffset
.UtcNow;
Faker <QueuedTask> taskFaker =
GetQueuedTaskFaker();
QueuedTask task = taskFaker
.Generate();
QueuedTaskResult result = new QueuedTaskResult(task);
TaskExecutionResult successful = new TaskExecutionResult(TaskExecutionResultInfo.Successful(),
duration: faker.Date.Timespan(),
retryAt: faker.Date.FutureOffset(),
faultErrorThresholdCount: faker.Random.Int(1, 5));
QueuedTaskInfo repostWithInfo = result.UdpateFromExecutionResult(successful);
Assert.Null(repostWithInfo);
Assert.IsNull(result.LastError);
Assert.AreEqual(QueuedTaskStatus.Processed, result.Status);
Assert.AreEqual(successful.ProcessingTimeMilliseconds, result.ProcessingTimeMilliseconds);
Assert.GreaterOrEqual(result.ProcessingFinalizedAtTs, now);
Assert.GreaterOrEqual(result.FirstProcessingAttemptedAtTs, now);
Assert.GreaterOrEqual(result.LastProcessingAttemptedAtTs, now);
}
public async Task Test_CanReadQueuedTaskResult()
{
Faker <QueuedTaskResult> qFaker =
GetQueuedTaskResultFaker();
QueuedTaskResult expectedTaskResult = qFaker
.Generate();
await mOperations
.AddQueuedTaskResultAsync(expectedTaskResult);
using (NpgsqlConnection conn = await OpenDbConnectionAsync(ConnectionString))
using (NpgsqlCommand cmd = new NpgsqlCommand($"SELECT * FROM {mMapping.ResultsQueueTableName} WHERE task_id = @t_id", conn))
{
cmd.Parameters.AddWithValue("t_id",
NpgsqlDbType.Uuid,
expectedTaskResult.Id);
await cmd.PrepareAsync();
using (NpgsqlDataReader rdr = await cmd.ExecuteReaderAsync())
{
if (await rdr.ReadAsync())
{
QueuedTaskResult actualTaskResult = await rdr.ReadQueuedTaskResultAsync();
Assert.NotNull(actualTaskResult);
Assert.NotNull(actualTaskResult.Payload);
Assert.IsInstanceOf <SampleTaskPayload>(actualTaskResult.Payload);
Assert.AreEqual((( SampleTaskPayload )expectedTaskResult.Payload).Counter,
(( SampleTaskPayload )actualTaskResult.Payload).Counter);
Assert.AreEqual(expectedTaskResult.Id, actualTaskResult.Id);
Assert.AreEqual(expectedTaskResult.Type, actualTaskResult.Type);
Assert.AreEqual(expectedTaskResult.Source, actualTaskResult.Source);
Assert.AreEqual(expectedTaskResult.Priority, actualTaskResult.Priority);
Assert.AreEqual(expectedTaskResult.ErrorCount, actualTaskResult.ErrorCount);
Assert.AreEqual(expectedTaskResult.LastErrorIsRecoverable, actualTaskResult.LastErrorIsRecoverable);
Assert.AreEqual(expectedTaskResult.LastError, actualTaskResult.LastError);
Assert.AreEqual(expectedTaskResult.Status, actualTaskResult.Status);
Assert.IsTrue(expectedTaskResult.PostedAtTs
.EqualsAproximately(actualTaskResult.PostedAtTs));
Assert.IsTrue(expectedTaskResult.ProcessingFinalizedAtTs
.EqualsAproximately(actualTaskResult.ProcessingFinalizedAtTs));
Assert.IsTrue(expectedTaskResult.FirstProcessingAttemptedAtTs
.EqualsAproximately(actualTaskResult.FirstProcessingAttemptedAtTs));
Assert.IsTrue(expectedTaskResult.LastProcessingAttemptedAtTs
.EqualsAproximately(actualTaskResult.LastProcessingAttemptedAtTs));
}
}
await conn.CloseAsync();
}
}
public async Task AddQueuedTaskResultAsync(QueuedTaskResult resultData)
{
using (NpgsqlConnection conn = await OpenDbConnectionAsync())
{
await AddQueuedTaskResultAsync(resultData, conn,
tx : null);
}
}
public async Task AssertTaskResultInDbAndCorrectAsync(IQueuedTaskToken newTaskToken)
{
QueuedTaskResult dbResult = await mDataSource.GetQueuedTaskResultFromDbByIdAsync(newTaskToken
.DequeuedTask
.Id);
dbResult.AssertMatchesResult(newTaskToken
.LastQueuedTaskResult);
}
public PostgreSqlQueuedTaskToken(QueuedTask dequeuedTask,
QueuedTaskResult lastQueuedTaskResult,
DateTimeOffset dequeuedAt)
{
DequeuedTask = dequeuedTask
?? throw new ArgumentNullException(nameof(dequeuedTask));
LastQueuedTaskResult = lastQueuedTaskResult
?? throw new ArgumentNullException(nameof(lastQueuedTaskResult));
DequeuedAt = dequeuedAt;
}
public Task ProduceTasksAsync(int numberOfTasks)
{
ManualResetEvent bufferSpaceAvailableWaitHandle =
new ManualResetEvent(false);
Queue <Type> taskPayloadTypes =
new Queue <Type>(mPayloadTypes);
return(Task.Run(() =>
{
Type currentPayloadType;
IQueuedTaskToken newTaskToken;
QueuedTask newTask;
QueuedTaskResult newLastTaskResult;
EventHandler handleBufferElementRemoved
= (s, e) => bufferSpaceAvailableWaitHandle.Set();
mTaskBuffer.QueuedTaskRetrieved
+= handleBufferElementRemoved;
while (taskPayloadTypes.TryDequeue(out currentPayloadType))
{
for (int i = 0; i < numberOfTasks; i++)
{
newTask = new QueuedTask(Guid.NewGuid())
{
Payload = Activator.CreateInstance(currentPayloadType),
Type = currentPayloadType.FullName
};
newLastTaskResult = new QueuedTaskResult(newTask)
{
Status = QueuedTaskStatus.Unprocessed
};
newTaskToken = new MockQueuedTaskToken(newTask, newLastTaskResult);
mProducedTasks.Add(newTaskToken);
while (!mTaskBuffer.TryAddNewTask(newTaskToken))
{
bufferSpaceAvailableWaitHandle.WaitOne();
bufferSpaceAvailableWaitHandle.Reset();
}
}
}
mTaskBuffer.CompleteAdding();
mTaskBuffer.QueuedTaskRetrieved
-= handleBufferElementRemoved;
}));
}
public void Test_CanCreateFromQueuedTask()
{
Faker <QueuedTask> taskFaker =
GetQueuedTaskFaker();
QueuedTask task = taskFaker
.Generate();
QueuedTaskResult result = new QueuedTaskResult(task);
Assert.AreEqual(task.Id, result.Id);
Assert.AreEqual(task.Type, result.Type);
Assert.AreSame(task.Payload, result.Payload);
Assert.AreEqual(task.Source, result.Source);
Assert.AreEqual(task.PostedAtTs, result.PostedAtTs);
Assert.AreEqual(0, result.ProcessingTimeMilliseconds);
Assert.AreEqual(QueuedTaskStatus.Unprocessed, result.Status);
}
public void Test_CanUpdateFromExecutionResult_WithError_NotRecoverable(int faultErrorThresholdCount)
{
Faker faker = new Faker();
DateTimeOffset now = DateTimeOffset
.UtcNow;
Faker <QueuedTask> taskFaker =
GetQueuedTaskFaker();
QueuedTask task = taskFaker
.Generate();
QueuedTaskResult result = new QueuedTaskResult(task);
TaskExecutionResultInfo failedWithErrorInfo = TaskExecutionResultInfo
.ExecutedWithError(new QueuedTaskError(faker.System.Exception()),
isRecoverable: false);
TaskExecutionResult failedWithError = new TaskExecutionResult(failedWithErrorInfo,
duration: faker.Date.Timespan(),
retryAt: faker.Date.FutureOffset(),
faultErrorThresholdCount: faultErrorThresholdCount);
for (int i = 1; i <= faultErrorThresholdCount + 2; i++)
{
if (i > 1)
{
Assert.Throws <InvalidOperationException>(() => result.UdpateFromExecutionResult(failedWithError));
}
else
{
Assert.IsNull(result.UdpateFromExecutionResult(failedWithError));
}
Assert.AreEqual(1, result.ErrorCount);
Assert.AreEqual(failedWithErrorInfo.Error, result.LastError);
Assert.IsFalse(result.LastErrorIsRecoverable);
Assert.AreEqual(QueuedTaskStatus.Fatal, result.Status);
Assert.AreEqual(0, result.ProcessingTimeMilliseconds);
Assert.GreaterOrEqual(result.FirstProcessingAttemptedAtTs, now);
Assert.GreaterOrEqual(result.LastProcessingAttemptedAtTs, now);
}
}
private async Task AddQueuedTaskResultAsync(QueuedTaskResult resultData,
NpgsqlConnection conn,
NpgsqlTransaction tx)
{
Dictionary <string, object> insertDataTaskResult = new Dictionary <string, object>()
{
{ "task_id", resultData.Id },
{ "task_payload", resultData.Payload.ToJson(includeTypeInformation: true) },
{ "task_type", resultData.Type },
{ "task_source", resultData.Source },
{ "task_priority", resultData.Priority },
{ "task_posted_at_ts", resultData.PostedAtTs },
{ "task_status", resultData.Status },
{ "task_processing_time_milliseconds", resultData.ProcessingTimeMilliseconds },
{ "task_error_count", resultData.ErrorCount },
{ "task_last_error", resultData.LastError.ToJson() },
{ "task_last_error_is_recoverable", resultData.LastErrorIsRecoverable },
{ "task_first_processing_attempted_at_ts", resultData.FirstProcessingAttemptedAtTs },
{ "task_last_processing_attempted_at_ts", resultData.LastProcessingAttemptedAtTs },
{ "task_processing_finalized_at_ts", resultData.ProcessingFinalizedAtTs }
};
if (tx != null)
{
await new QueryFactory(conn, new PostgresCompiler())
.Query(mMapping.ResultsQueueTableName)
.InsertAsync(insertDataTaskResult, tx);
}
else
{
await new QueryFactory(conn, new PostgresCompiler())
.Query(mMapping.ResultsQueueTableName)
.InsertAsync(insertDataTaskResult);
}
}
private async Task Run_PostResultTests(Func <TaskExecutionResult> rsFactory)
{
using (PostgreSqlTaskResultQueue rq = CreateResultQueue())
{
await rq.StartAsync();
foreach (IQueuedTaskToken token in mDataSource.SeededTaskTokens)
{
token.UdpateFromExecutionResult(rsFactory.Invoke());
int affectedRows = await rq.PostResultAsync(token);
Assert.AreEqual(1, affectedRows);
QueuedTaskResult dbResult = await mDataSource
.GetQueuedTaskResultFromDbByIdAsync(token.DequeuedTask.Id);
dbResult.AssertMatchesResult(token
.LastQueuedTaskResult);
}
await rq.StopAsync();
}
}
private async Task <QueuedTaskResult> TryUpdateTaskResultAsync(QueuedTask dequeuedTask,
NpgsqlConnection conn,
NpgsqlTransaction tx)
{
QueuedTaskResult dequeuedTaskResult = null;
using (NpgsqlCommand addOrUpdateResultCmd = new NpgsqlCommand(mTaskResultUpdateSql, conn, tx))
{
addOrUpdateResultCmd.Parameters.AddWithValue("t_id",
NpgsqlDbType.Uuid,
dequeuedTask.Id);
addOrUpdateResultCmd.Parameters.AddWithValue("t_status",
NpgsqlDbType.Integer,
( int )QueuedTaskStatus.Processing);
await addOrUpdateResultCmd.PrepareAsync();
using (NpgsqlDataReader resultRdr = await addOrUpdateResultCmd.ExecuteReaderAsync())
{
if (await resultRdr.ReadAsync())
{
dequeuedTaskResult = await resultRdr.ReadQueuedTaskResultAsync();
}
if (dequeuedTaskResult != null)
{
mLogger.Debug("Successfully dequeued, acquired and initialized/updated task result.");
}
else
{
mLogger.Debug("Failed to initialize or update task result. Will release lock...");
}
}
}
return(dequeuedTaskResult);
}
public async Task <IQueuedTaskToken> DequeueAsync(params string[] selectTaskTypes)
{
NpgsqlConnection conn = null;
QueuedTask dequeuedTask = null;
QueuedTaskResult dequeuedTaskResult = null;
PostgreSqlQueuedTaskToken dequeuedTaskToken = null;
MonotonicTimestamp startDequeue;
DateTimeOffset refNow = mTimestampProvider.GetNow();
CheckNotDisposedOrThrow();
try
{
mLogger.DebugFormat("Begin dequeue task. Looking for types: {0}.",
string.Join <string>(",", selectTaskTypes));
startDequeue = MonotonicTimestamp
.Now();
conn = await OpenQueueConnectionAsync();
if (conn == null)
{
return(null);
}
using (NpgsqlTransaction tx = conn.BeginTransaction(IsolationLevel.ReadCommitted))
{
//1. Dequeue means that we acquire lock on a task in the queue
// with the guarantee that nobody else did, and respecting
// the priority and static locks (basically the task_locked_until which says
// that it should not be pulled out of the queue until the
// current abstract time reaches that tick value)
dequeuedTask = await TryDequeueTaskAsync(selectTaskTypes, refNow, conn, tx);
if (dequeuedTask != null)
{
//2. Mark the task as being "Processing" and pull result info
// The result is stored separately and it's what allows us to remove
// the task from the queue at step #2,
// whils also tracking it's processing status and previous results
dequeuedTaskResult = await TryUpdateTaskResultAsync(dequeuedTask, conn, tx);
if (dequeuedTaskResult != null)
{
await tx.CommitAsync();
dequeuedTaskToken = new PostgreSqlQueuedTaskToken(dequeuedTask,
dequeuedTaskResult,
refNow);
}
}
if (dequeuedTaskToken != null)
{
IncrementDequeueCount(MonotonicTimestamp.Since(startDequeue));
}
else
{
await tx.RollbackAsync();
}
}
}
finally
{
if (conn != null)
{
await conn.CloseAsync();
conn.Dispose();
}
}
return(dequeuedTaskToken);
}
public void Test_CanUpdateFromExecutionResult_WithError_Recoverable(int faultErrorThresholdCount)
{
Faker faker = new Faker();
QueuedTaskInfo repostWithInfo = null;
DateTimeOffset now = DateTimeOffset
.UtcNow;
Faker <QueuedTask> taskFaker =
GetQueuedTaskFaker();
QueuedTask task = taskFaker
.Generate();
QueuedTaskResult result = new QueuedTaskResult(task);
TaskExecutionResultInfo failedWithErrorInfo = TaskExecutionResultInfo
.ExecutedWithError(new QueuedTaskError(faker.System.Exception()),
isRecoverable: true);
TaskExecutionResult failedWithError = new TaskExecutionResult(failedWithErrorInfo,
duration: faker.Date.Timespan(),
retryAt: faker.Date.FutureOffset(),
faultErrorThresholdCount: faultErrorThresholdCount);
//1 to faultErrorThresholdCount -> Error status
for (int i = 1; i <= faultErrorThresholdCount; i++)
{
repostWithInfo = result.UdpateFromExecutionResult(failedWithError);
Assert.NotNull(repostWithInfo);
Assert.AreEqual(QueuedTaskStatus.Error, result.Status);
Assert.AreEqual(0, result.ProcessingTimeMilliseconds);
Assert.GreaterOrEqual(result.FirstProcessingAttemptedAtTs, now);
Assert.GreaterOrEqual(result.LastProcessingAttemptedAtTs, now);
Assert.AreEqual(failedWithErrorInfo.Error, result.LastError);
Assert.AreEqual(i, result.ErrorCount);
}
//Antoher failure -> Faulted
repostWithInfo = result.UdpateFromExecutionResult(failedWithError);
Assert.NotNull(repostWithInfo);
Assert.AreEqual(QueuedTaskStatus.Faulted, result.Status);
Assert.AreEqual(0, result.ProcessingTimeMilliseconds);
Assert.GreaterOrEqual(result.FirstProcessingAttemptedAtTs, now);
Assert.GreaterOrEqual(result.LastProcessingAttemptedAtTs, now);
Assert.AreEqual(failedWithErrorInfo.Error, result.LastError);
Assert.AreEqual(faultErrorThresholdCount + 1, result.ErrorCount);
//Antoher failure after that -> Fataled
repostWithInfo = result.UdpateFromExecutionResult(failedWithError);
Assert.Null(repostWithInfo);
Assert.AreEqual(QueuedTaskStatus.Fatal, result.Status);
Assert.AreEqual(0, result.ProcessingTimeMilliseconds);
Assert.GreaterOrEqual(result.FirstProcessingAttemptedAtTs, now);
Assert.GreaterOrEqual(result.LastProcessingAttemptedAtTs, now);
Assert.AreEqual(failedWithErrorInfo.Error, result.LastError);
Assert.AreEqual(faultErrorThresholdCount + 2, result.ErrorCount);
}
public static async Task <QueuedTaskResult> ReadQueuedTaskResultAsync(this NpgsqlDataReader reader)
{
if (reader == null)
{
throw new ArgumentNullException(nameof(reader));
}
string payloadString,
taskErrorString;
QueuedTaskResult result =
new QueuedTaskResult();
result.Id = await reader.GetFieldValueAsync <Guid>("task_id",
defaultValue : Guid.Empty);
result.Type = await reader.GetFieldValueAsync <string>("task_type",
defaultValue : string.Empty);
result.Source = await reader.GetFieldValueAsync <string>("task_source",
defaultValue : string.Empty);
result.Status = ( QueuedTaskStatus )(await reader.GetFieldValueAsync <int>("task_status",
defaultValue: 0));
result.Priority = await reader.GetFieldValueAsync <int>("task_priority",
defaultValue : 0);
result.LastErrorIsRecoverable = await reader.GetFieldValueAsync <bool>("task_last_error_is_recoverable",
defaultValue : false);
result.ErrorCount = await reader.GetFieldValueAsync <int>("task_error_count",
defaultValue : 0);
//Get payoad
payloadString = await reader.GetFieldValueAsync <string>("task_payload",
defaultValue : string.Empty);
result.Payload = payloadString
.AsObjectFromJson();
//Get last task error
taskErrorString = await reader.GetFieldValueAsync <string>("task_last_error",
defaultValue : string.Empty);
result.LastError = taskErrorString
.AsObjectFromJson <QueuedTaskError>();
result.PostedAtTs = await reader.GetFieldValueAsync <DateTimeOffset>("task_posted_at_ts",
defaultValue : DateTimeOffset.MinValue);
result.ProcessingTimeMilliseconds = await reader.GetFieldValueAsync <long>("task_processing_time_milliseconds",
defaultValue : 0);
result.FirstProcessingAttemptedAtTs = await reader.GetNullableFieldValueAsync <DateTimeOffset>("task_first_processing_attempted_at_ts",
defaultValue : null);
result.LastProcessingAttemptedAtTs = await reader.GetNullableFieldValueAsync <DateTimeOffset>("task_last_processing_attempted_at_ts",
defaultValue : null);
result.ProcessingFinalizedAtTs = await reader.GetNullableFieldValueAsync <DateTimeOffset>("task_processing_finalized_at_ts",
defaultValue : null);
return(result);
}
public MockQueuedTaskToken(Guid queuedTaskId)
{
mQueuedTask = new QueuedTask(queuedTaskId);
mLastQueuedTaskResult = new QueuedTaskResult(mQueuedTask);
}
public MockQueuedTaskToken(QueuedTask queuedTask, QueuedTaskResult lastQueuedTaskResult)
{
mQueuedTask = queuedTask;
mLastQueuedTaskResult = lastQueuedTaskResult;
}
