private async Task ExecuteAndProcessResultAsync ( IQueuedTaskToken queuedTaskToken,
CancellationToken stopToken )
{
mLogger.DebugFormat( "New task to execute retrieved from buffer: task id = {0}.",
queuedTaskToken.DequeuedTask.Id );
//Prepare execution context
TaskExecutionContext executionContext =
new TaskExecutionContext( queuedTaskToken,
stopToken );
//Execute task
TaskExecutionResult executionResult =
await ExecuteTaskAsync( executionContext );
//We will not observe cancellation token
// during result processing:
// if task executed till the end, we must at least
// attempt to set the result
if (executionResult != null)
await ProcessResultAsync( queuedTaskToken,
executionResult );
mLogger.DebugFormat( "Done executing task with id = {0}.",
queuedTaskToken.DequeuedTask.Id );
}
public async Task Test_StatsAreCorrectlyUpdated_AfterDequeue_NoTaskType()
{
PostgreSqlTaskQueueInfo taskQueueInfo =
CreateTaskQueueInfo(() => mDataSource.LastPostedAt);
using (PostgreSqlTaskQueueConsumer taskQueueConsumer =
CreateTaskQueueConsumer(() => mDataSource.LastPostedAt))
using (TaskQueueMetricsDiffChecker diff = new TaskQueueMetricsDiffChecker(async()
=> await taskQueueInfo.ComputeMetricsAsync()))
{
await diff.CaptureInitialMetricsAsync();
IQueuedTaskToken dequeuedToken = await taskQueueConsumer
.DequeueAsync();
QueuedTaskStatus origStatus = mDataSource
.GetOriginalTokenData(dequeuedToken.DequeuedTask.Id)
.LastQueuedTaskResult
.Status;
await diff.CaptureNewMetricsAndAssertCorrectDiff(delta : new TaskQueueMetrics(
totalUnprocessed: origStatus == QueuedTaskStatus.Unprocessed ? -1 : 0,
totalProcessing: 1,
totalErrored: origStatus == QueuedTaskStatus.Error ? -1 : 0,
totalFaulted: origStatus == QueuedTaskStatus.Faulted ? -1 : 0,
totalFataled: origStatus == QueuedTaskStatus.Fatal ? -1 : 0,
totalProcessed: origStatus == QueuedTaskStatus.Processed ? -1 : 0));
}
}
public Task <int> PostResultAsync(IQueuedTaskToken token, int timeoutMilliseconds)
{
CheckNotDisposedOrThrow();
CheckRunningOrThrow();
if (token == null)
{
throw new ArgumentNullException(nameof(token));
}
long requestId = Interlocked.Increment(ref mLastRequestId);
PostgreSqlTaskResultQueueProcessRequest processRequest =
new PostgreSqlTaskResultQueueProcessRequest(requestId,
token.LastQueuedTaskResult,
timeoutMilliseconds: timeoutMilliseconds,
maxFailCount: 3);
mResultProcessingQueue.Add(processRequest);
IncrementPostResultCount();
return(processRequest.Task.WithCleanup((prev) =>
{
if (processRequest.IsTimedOut)
{
IncrementResultWriteRequestTimeoutCount();
}
processRequest.Dispose();
}));
}
public async Task Test_PeekMatchesDequeuedItem_SingleConsumer()
{
IQueuedTask peekTask = null;
IQueuedTaskToken dequeuedTaskToken = null;
PostgreSqlTaskQueueInfo taskQueueInfo =
CreateTaskQueueInfo(() => mDataSource.LastPostedAt);
using (PostgreSqlTaskQueueConsumer taskQueue =
CreateTaskQueueConsumer(() => mDataSource.LastPostedAt))
{
int expectedDequeueCount = mDataSource
.NumTasksInQueue;
for (int i = 0; i < expectedDequeueCount; i++)
{
peekTask = await taskQueueInfo.PeekAsync();
Assert.NotNull(peekTask);
dequeuedTaskToken = await taskQueue.DequeueAsync();
Assert.NotNull(dequeuedTaskToken);
Assert.AreEqual(peekTask.Id, dequeuedTaskToken
.DequeuedTask
.Id);
}
}
}
public bool TryAddNewTask(IQueuedTaskToken task)
{
CheckDisposedOrThrow();
if (task == null)
{
throw new ArgumentNullException(nameof(task));
}
if (mInnerBuffer.IsAddingCompleted)
{
return(false);
}
int oldCount = mInnerBuffer.Count;
bool wasFull = (oldCount == mCapacity);
bool isAdded = mInnerBuffer.TryAdd(task);
if (isAdded)
{
NotifyQueuedTaskAdded();
UpdateOnBufferItemAdd(Math.Min(mCapacity, oldCount + 1), wasFull);
}
return(isAdded);
}
public async Task Test_DequeueChangesPeekResult_SingleConsumer()
{
IQueuedTask peekTask = null,
rePeekTask = null;
IQueuedTaskToken dequeuedTaskToken = null;
PostgreSqlTaskQueueInfo taskQueueInfo =
CreateTaskQueueInfo(() => mDataSource.LastPostedAt);
using (PostgreSqlTaskQueueConsumer taskQueueConsumer =
CreateTaskQueueConsumer(() => mDataSource.LastPostedAt))
{
peekTask = await taskQueueInfo.PeekAsync();
Assert.NotNull(peekTask);
dequeuedTaskToken = await taskQueueConsumer.DequeueAsync();
Assert.NotNull(dequeuedTaskToken);
rePeekTask = await taskQueueInfo.PeekAsync();
Assert.NotNull(rePeekTask);
//Removing a new element from the queue
// occurs at the beginning of the queue,
// so peeking must yield a different result
// than before dequeue-ing
Assert.AreNotEqual(rePeekTask.Id,
peekTask.Id);
}
}
public void Test_CanGet_FromNonEmptyBuffer(int capacity)
{
using (StandardTaskBuffer buffer = new StandardTaskBuffer(capacity))
{
//Add some items
int actualItemNumber = ProduceItemNumber(capacity);
List <IQueuedTaskToken> addedTasks = new List <IQueuedTaskToken>();
for (int i = 0; i < actualItemNumber; i++)
{
addedTasks.Add(new MockQueuedTaskToken(Guid.NewGuid()));
buffer.TryAddNewTask(addedTasks[i]);
}
for (int i = 0; i < actualItemNumber; i++)
{
IQueuedTaskToken queuedTaskToken = buffer.TryGetNextTask();
Assert.NotNull(queuedTaskToken);
Assert.IsTrue(addedTasks.Any(t => t.DequeuedTask.Id == queuedTaskToken.DequeuedTask.Id));
}
Assert.IsFalse(buffer.IsFull);
Assert.IsFalse(buffer.IsCompleted);
}
}
private bool CanTaskBeReposted(IQueuedTaskToken token)
{
return(token.LastQueuedTaskResult.Status != QueuedTaskStatus.Fatal &&
token.LastQueuedTaskResult.Status != QueuedTaskStatus.Faulted &&
token.LastQueuedTaskResult.Status != QueuedTaskStatus.Cancelled &&
token.LastQueuedTaskResult.Status != QueuedTaskStatus.Processed);
}
public async Task AssertTaskResultInDbAndCorrectAsync(IQueuedTaskToken newTaskToken)
{
QueuedTaskResult dbResult = await mDataSource.GetQueuedTaskResultFromDbByIdAsync(newTaskToken
.DequeuedTask
.Id);
dbResult.AssertMatchesResult(newTaskToken
.LastQueuedTaskResult);
}
private async Task ProcessResultAsync ( IQueuedTaskToken queuedTaskToken,
TaskExecutionResult result )
{
try
{
//Update worker execution stats
UpdateTaskProcessingStats( result );
//There is no result - most likely, no executor found;
// nothing to process, just stop and return
if ( !result.HasResult )
{
mLogger.Debug( "No result info returned. Task will be discarded." );
return;
}
//Execution has been cancelled, usually as a response
// to a cancellation request;
// nothing to process, just stop and return
if ( result.ExecutionCancelled )
{
mLogger.Debug( "Task execution cancelled. Task will be discarded." );
return;
}
//Update execution result and see whether
// we need to repost the task to retry its execution
QueuedTaskInfo repostWithInfo = queuedTaskToken
.UdpateFromExecutionResult( result );
//Post result
mLogger.Debug( "Will post task execution result." );
await mTaskResultQueue.PostResultAsync( queuedTaskToken );
mLogger.Debug( "Successfully posted task execution result." );
//If the task needs to be reposted, do so
if ( repostWithInfo != null )
{
mLogger.Debug( "Will repost task for execution." );
await mTaskQueueProducer.EnqueueAsync( repostWithInfo );
mLogger.Debug( "Sucessfully reposted task for execution." );
}
else
mLogger.Debug( "Will not repost task for execution." );
//Finally, report execution time
await mPerformanceMonitor.ReportExecutionTimeAsync( queuedTaskToken,
result );
}
catch ( Exception exc )
{
mLogger.Error( "Failed to set queued task result. Task will be discarded.",
exc );
}
}
private async Task RunWorkerAsync ()
{
//Pull the cancellation token
CancellationToken stopToken = mStopTokenSource
.Token;
//Check for cancellation before we start
// the processing loop
if ( stopToken.IsCancellationRequested )
return;
while ( true )
{
try
{
//Check for cancellation at the beginning
// of processing each loop
stopToken.ThrowIfCancellationRequested();
//Check if buffer can deliver new tasks to us.
// If not (and it's permanent), break worker processing loop.
if ( !await PerformBufferCheckAsync() )
break;
//It may be that the wait handle was signaled
// as part of the Stop operation,
// so we need to check for that as well.
stopToken.ThrowIfCancellationRequested();
//Finally, dequeue and execute the task
// and forward the result to the result queue
IQueuedTaskToken queuedTaskToken = mTaskBuffer.TryGetNextTask();
if ( queuedTaskToken != null )
await ExecuteAndProcessResultAsync( queuedTaskToken, stopToken );
else
mLogger.Debug( "Nothing to execute: no task was retrieved." );
//At the end of the loop, reset the handle
mWaitForClearToFetchTask.Reset();
}
catch ( OperationCanceledException )
{
mLogger.Debug( "Worker stop requested. Breaking processing loop..." );
break;
}
}
}
public void ConsumeBuffer()
{
mConsumeBufferTask = Task.Run(() =>
{
while (!mTaskBuffer.IsCompleted)
{
IQueuedTaskToken queuedTaskToken = mTaskBuffer.TryGetNextTask();
if (queuedTaskToken != null)
{
mConsumedTasks.Add(queuedTaskToken);
}
else
{
Task.Delay(10).Wait();
}
}
});
}
private void AssertCorrectDefaultCalculateDelayMillisecondsTaskAfterFailureFn(StakhanoviseSetupDefaults setupDefaults)
{
Assert.NotNull(setupDefaults.CalculateDelayMillisecondsTaskAfterFailure);
for (int iErrorCount = 0; iErrorCount < TestCalculateDelayMillisecondsErrorCountMax; iErrorCount++)
{
long expectedDelayMilliseconds =
( long )Math.Pow(10, iErrorCount + 1);
IQueuedTaskToken mockTokenWithErrorCount =
MockQueuedTaskTokenWithErrorCount(iErrorCount);
long actualDelayMilliseconds = setupDefaults
.CalculateDelayMillisecondsTaskAfterFailure(mockTokenWithErrorCount);
Assert.AreEqual(expectedDelayMilliseconds,
actualDelayMilliseconds);
}
}
private DateTimeOffset ComputeRetryAt ( IQueuedTaskToken queuedTaskToken )
{
long delayMilliseconds = 0;
try
{
//Compute the absolute time, in ticks,
// until the task execution is delayed.
delayMilliseconds = mOptions.CalculateRetryMillisecondsDelay( queuedTaskToken );
}
catch ( Exception exc )
{
mLogger.Error( "Failed to compute delay. Using default value of 0.",
exc );
}
return DateTimeOffset.UtcNow.AddMilliseconds( delayMilliseconds );
}
public void AssertConsumedTokenValid(IQueuedTaskToken newTaskToken, DateTimeOffset now)
{
Assert.NotNull(newTaskToken);
Assert.NotNull(newTaskToken.DequeuedAt);
Assert.NotNull(newTaskToken.DequeuedTask);
Assert.NotNull(newTaskToken.LastQueuedTaskResult);
//Assert.AreEqual( now, newTaskToken.DequeuedAt );
Assert.IsFalse(mDequeuedTokens.Any(t => t.DequeuedTask.Id == newTaskToken.DequeuedTask.Id));
if (mPreviousTaskToken != null)
{
Assert.GreaterOrEqual(newTaskToken.DequeuedTask.PostedAtTs, mPreviousTaskToken.DequeuedTask.PostedAtTs);
}
mPreviousTaskToken = newTaskToken;
mDequeuedTokens.Add(newTaskToken);
}
public static async Task ReportExecutionTimeAsync(this IExecutionPerformanceMonitor executionPerformanceMonitor,
IQueuedTaskToken queuedTaskToken,
TaskExecutionResult result)
{
if (executionPerformanceMonitor == null)
{
throw new ArgumentNullException(nameof(executionPerformanceMonitor));
}
if (queuedTaskToken == null)
{
throw new ArgumentNullException(nameof(queuedTaskToken));
}
if (result == null)
{
throw new ArgumentNullException(nameof(result));
}
await executionPerformanceMonitor.ReportExecutionTimeAsync(
queuedTaskToken.DequeuedTask.Type,
result.ProcessingTimeMilliseconds,
timeoutMilliseconds : 0);
}
public void Dispose()
{
mPreviousTaskToken = null;
mDequeuedTokens.Clear();
}
public void Test_ConsumerProducerScenario(int nProducers, int nConsumers)
{
Task coordinator;
Task[] allProducers = new Task[nProducers];
Task[] allConsumers = new Task[nConsumers];
int expectedTotal = 0;
ConcurrentBag <IQueuedTaskToken> processedTasks = new ConcurrentBag <IQueuedTaskToken>();
using (StandardTaskBuffer buffer = new StandardTaskBuffer(10))
{
for (int iProducer = 0; iProducer < nProducers; iProducer++)
{
allProducers[iProducer] = Task.Run(() =>
{
//Generate a number of items to produce
// and add that to the expected total
int nItems = new Random().Next(1, 100);
Interlocked.Add(ref expectedTotal, nItems);
while (nItems > 0)
{
bool isAdded = buffer.TryAddNewTask(new MockQueuedTaskToken(Guid.NewGuid()));
if (isAdded)
{
nItems--;
}
else
{
Task.Delay(10).Wait();
}
}
});
}
for (int iConsumer = 0; iConsumer < nConsumers; iConsumer++)
{
allConsumers[iConsumer] = Task.Run(() =>
{
//Consumers run until the buffer is completed:
// - marked as completed with respect to additons
// AND
// - has no more items
while (!buffer.IsCompleted)
{
IQueuedTaskToken queuedTaskToken = buffer.TryGetNextTask();
if (queuedTaskToken != null)
{
processedTasks.Add(queuedTaskToken);
}
else
{
Task.Delay(10).Wait();
}
}
});
}
coordinator = Task.Run(() =>
{
//The coordinator waits for all producers
// to finish and then marks buffer
// addition operations as being completed
Task.WaitAll(allProducers);
buffer.CompleteAdding();
});
//Wait for all threads to stop
Task.WaitAll(coordinator);
Task.WaitAll(allConsumers);
//Check that:
// a) we have the exact number of items we added
// b) there are no items that have been processed two times
Assert.AreEqual(expectedTotal,
processedTasks.Count);
foreach (IQueuedTaskToken queuedTaskToken in processedTasks)
{
Assert.AreEqual(1, processedTasks.Count(t => t.DequeuedTask.Id == queuedTaskToken.DequeuedTask.Id));
}
}
}
public async Task AssertTaskNotInDbAnymoreAsync(IQueuedTaskToken newTaskToken)
{
Assert.IsNull(await mDataSource.GetQueuedTaskFromDbByIdAsync(newTaskToken
.DequeuedTask
.Id));
}
private async Task PollForQueuedTasksAsync()
{
CancellationToken stopToken = mStopTokenSource
.Token;
//Check cancellation token before starting
// the polling loop
if (stopToken.IsCancellationRequested)
{
return;
}
while (true)
{
try
{
//Check for token cancellation at the beginning of the loop
stopToken.ThrowIfCancellationRequested();
//If the buffer is full, we wait for some space to become available,
// since, even if we can dequeue an task,
// we won't have anywhere to place it yet and we
// may be needlessly helding a lock to that task
if (mTaskBuffer.IsFull)
{
mLogger.Debug("Task buffer is full. Waiting for available space...");
mMetrics.UpdateMetric(AppMetricId.PollerWaitForBufferSpaceCount, m => m.Increment());
await mWaitForClearToAddToBuffer.ToTask();
}
//It may be that the wait handle was signaled
// as part of the Stop operation,
// so we need to check for that as well.
stopToken.ThrowIfCancellationRequested();
//Attempt to dequeue and then check cancellation
IQueuedTaskToken queuedTaskToken = await mTaskQueueConsumer
.DequeueAsync(mRequiredPayloadTypes);
//Before posting the token to the buffer,
// check if cancellation was requested
stopToken.ThrowIfCancellationRequested();
if (queuedTaskToken != null)
{
//If we have found a token, attempt to set it as started
// and only then add it to buffer for processing.
//If not, dispose and discard the token
mLogger.DebugFormat("Task found with id = {0}, type = {1}. Acquiring reservation...",
queuedTaskToken.DequeuedTask.Id,
queuedTaskToken.DequeuedTask.Type);
mMetrics.UpdateMetric(AppMetricId.PollerDequeueCount, m => m.Increment());
mTaskBuffer.TryAddNewTask(queuedTaskToken);
}
else
{
//If there is no task available in the queue, begin waiting for
// a notification of new added tasks
mLogger.Debug("No task dequeued when polled. Waiting for available task...");
mMetrics.UpdateMetric(AppMetricId.PollerWaitForDequeueCount, m => m.Increment());
await mWaitForClearToDequeue.ToTask();
}
//It may be that the wait handle was signaled
// as part of the Stop operation,
// so we need to check for that as well.
stopToken.ThrowIfCancellationRequested();
//Finally, reset all the handles, at the end of the loop
mWaitForClearToAddToBuffer.Reset();
mWaitForClearToDequeue.Reset();
}
catch (OperationCanceledException)
{
mLogger.Debug("Stop requested. Breaking polling loop...");
break;
}
}
mTaskBuffer.CompleteAdding();
}
public TaskExecutionContext(IQueuedTaskToken taskToken, CancellationToken stopToken)
{
mTaskToken = taskToken ?? throw new ArgumentNullException(nameof(taskToken));
mCancellationToken = stopToken;
}
public Task <int> PostResultAsync(IQueuedTaskToken token)
{
return(PostResultAsync(token,
timeoutMilliseconds: 0));
}
