private async Task ExecuteAndProcessResultAsync ( IQueuedTaskToken queuedTaskToken, CancellationToken stopToken ) { mLogger.DebugFormat( "New task to execute retrieved from buffer: task id = {0}.", queuedTaskToken.DequeuedTask.Id ); //Prepare execution context TaskExecutionContext executionContext = new TaskExecutionContext( queuedTaskToken, stopToken ); //Execute task TaskExecutionResult executionResult = await ExecuteTaskAsync( executionContext ); //We will not observe cancellation token // during result processing: // if task executed till the end, we must at least // attempt to set the result if (executionResult != null) await ProcessResultAsync( queuedTaskToken, executionResult ); mLogger.DebugFormat( "Done executing task with id = {0}.", queuedTaskToken.DequeuedTask.Id ); }
public async Task Test_StatsAreCorrectlyUpdated_AfterDequeue_NoTaskType() { PostgreSqlTaskQueueInfo taskQueueInfo = CreateTaskQueueInfo(() => mDataSource.LastPostedAt); using (PostgreSqlTaskQueueConsumer taskQueueConsumer = CreateTaskQueueConsumer(() => mDataSource.LastPostedAt)) using (TaskQueueMetricsDiffChecker diff = new TaskQueueMetricsDiffChecker(async() => await taskQueueInfo.ComputeMetricsAsync())) { await diff.CaptureInitialMetricsAsync(); IQueuedTaskToken dequeuedToken = await taskQueueConsumer .DequeueAsync(); QueuedTaskStatus origStatus = mDataSource .GetOriginalTokenData(dequeuedToken.DequeuedTask.Id) .LastQueuedTaskResult .Status; await diff.CaptureNewMetricsAndAssertCorrectDiff(delta : new TaskQueueMetrics( totalUnprocessed: origStatus == QueuedTaskStatus.Unprocessed ? -1 : 0, totalProcessing: 1, totalErrored: origStatus == QueuedTaskStatus.Error ? -1 : 0, totalFaulted: origStatus == QueuedTaskStatus.Faulted ? -1 : 0, totalFataled: origStatus == QueuedTaskStatus.Fatal ? -1 : 0, totalProcessed: origStatus == QueuedTaskStatus.Processed ? -1 : 0)); } }
public Task <int> PostResultAsync(IQueuedTaskToken token, int timeoutMilliseconds) { CheckNotDisposedOrThrow(); CheckRunningOrThrow(); if (token == null) { throw new ArgumentNullException(nameof(token)); } long requestId = Interlocked.Increment(ref mLastRequestId); PostgreSqlTaskResultQueueProcessRequest processRequest = new PostgreSqlTaskResultQueueProcessRequest(requestId, token.LastQueuedTaskResult, timeoutMilliseconds: timeoutMilliseconds, maxFailCount: 3); mResultProcessingQueue.Add(processRequest); IncrementPostResultCount(); return(processRequest.Task.WithCleanup((prev) => { if (processRequest.IsTimedOut) { IncrementResultWriteRequestTimeoutCount(); } processRequest.Dispose(); })); }
public async Task Test_PeekMatchesDequeuedItem_SingleConsumer() { IQueuedTask peekTask = null; IQueuedTaskToken dequeuedTaskToken = null; PostgreSqlTaskQueueInfo taskQueueInfo = CreateTaskQueueInfo(() => mDataSource.LastPostedAt); using (PostgreSqlTaskQueueConsumer taskQueue = CreateTaskQueueConsumer(() => mDataSource.LastPostedAt)) { int expectedDequeueCount = mDataSource .NumTasksInQueue; for (int i = 0; i < expectedDequeueCount; i++) { peekTask = await taskQueueInfo.PeekAsync(); Assert.NotNull(peekTask); dequeuedTaskToken = await taskQueue.DequeueAsync(); Assert.NotNull(dequeuedTaskToken); Assert.AreEqual(peekTask.Id, dequeuedTaskToken .DequeuedTask .Id); } } }
public bool TryAddNewTask(IQueuedTaskToken task) { CheckDisposedOrThrow(); if (task == null) { throw new ArgumentNullException(nameof(task)); } if (mInnerBuffer.IsAddingCompleted) { return(false); } int oldCount = mInnerBuffer.Count; bool wasFull = (oldCount == mCapacity); bool isAdded = mInnerBuffer.TryAdd(task); if (isAdded) { NotifyQueuedTaskAdded(); UpdateOnBufferItemAdd(Math.Min(mCapacity, oldCount + 1), wasFull); } return(isAdded); }
public async Task Test_DequeueChangesPeekResult_SingleConsumer() { IQueuedTask peekTask = null, rePeekTask = null; IQueuedTaskToken dequeuedTaskToken = null; PostgreSqlTaskQueueInfo taskQueueInfo = CreateTaskQueueInfo(() => mDataSource.LastPostedAt); using (PostgreSqlTaskQueueConsumer taskQueueConsumer = CreateTaskQueueConsumer(() => mDataSource.LastPostedAt)) { peekTask = await taskQueueInfo.PeekAsync(); Assert.NotNull(peekTask); dequeuedTaskToken = await taskQueueConsumer.DequeueAsync(); Assert.NotNull(dequeuedTaskToken); rePeekTask = await taskQueueInfo.PeekAsync(); Assert.NotNull(rePeekTask); //Removing a new element from the queue // occurs at the beginning of the queue, // so peeking must yield a different result // than before dequeue-ing Assert.AreNotEqual(rePeekTask.Id, peekTask.Id); } }
public void Test_CanGet_FromNonEmptyBuffer(int capacity) { using (StandardTaskBuffer buffer = new StandardTaskBuffer(capacity)) { //Add some items int actualItemNumber = ProduceItemNumber(capacity); List <IQueuedTaskToken> addedTasks = new List <IQueuedTaskToken>(); for (int i = 0; i < actualItemNumber; i++) { addedTasks.Add(new MockQueuedTaskToken(Guid.NewGuid())); buffer.TryAddNewTask(addedTasks[i]); } for (int i = 0; i < actualItemNumber; i++) { IQueuedTaskToken queuedTaskToken = buffer.TryGetNextTask(); Assert.NotNull(queuedTaskToken); Assert.IsTrue(addedTasks.Any(t => t.DequeuedTask.Id == queuedTaskToken.DequeuedTask.Id)); } Assert.IsFalse(buffer.IsFull); Assert.IsFalse(buffer.IsCompleted); } }
private bool CanTaskBeReposted(IQueuedTaskToken token) { return(token.LastQueuedTaskResult.Status != QueuedTaskStatus.Fatal && token.LastQueuedTaskResult.Status != QueuedTaskStatus.Faulted && token.LastQueuedTaskResult.Status != QueuedTaskStatus.Cancelled && token.LastQueuedTaskResult.Status != QueuedTaskStatus.Processed); }
public async Task AssertTaskResultInDbAndCorrectAsync(IQueuedTaskToken newTaskToken) { QueuedTaskResult dbResult = await mDataSource.GetQueuedTaskResultFromDbByIdAsync(newTaskToken .DequeuedTask .Id); dbResult.AssertMatchesResult(newTaskToken .LastQueuedTaskResult); }
private async Task ProcessResultAsync ( IQueuedTaskToken queuedTaskToken, TaskExecutionResult result ) { try { //Update worker execution stats UpdateTaskProcessingStats( result ); //There is no result - most likely, no executor found; // nothing to process, just stop and return if ( !result.HasResult ) { mLogger.Debug( "No result info returned. Task will be discarded." ); return; } //Execution has been cancelled, usually as a response // to a cancellation request; // nothing to process, just stop and return if ( result.ExecutionCancelled ) { mLogger.Debug( "Task execution cancelled. Task will be discarded." ); return; } //Update execution result and see whether // we need to repost the task to retry its execution QueuedTaskInfo repostWithInfo = queuedTaskToken .UdpateFromExecutionResult( result ); //Post result mLogger.Debug( "Will post task execution result." ); await mTaskResultQueue.PostResultAsync( queuedTaskToken ); mLogger.Debug( "Successfully posted task execution result." ); //If the task needs to be reposted, do so if ( repostWithInfo != null ) { mLogger.Debug( "Will repost task for execution." ); await mTaskQueueProducer.EnqueueAsync( repostWithInfo ); mLogger.Debug( "Sucessfully reposted task for execution." ); } else mLogger.Debug( "Will not repost task for execution." ); //Finally, report execution time await mPerformanceMonitor.ReportExecutionTimeAsync( queuedTaskToken, result ); } catch ( Exception exc ) { mLogger.Error( "Failed to set queued task result. Task will be discarded.", exc ); } }
private async Task RunWorkerAsync () { //Pull the cancellation token CancellationToken stopToken = mStopTokenSource .Token; //Check for cancellation before we start // the processing loop if ( stopToken.IsCancellationRequested ) return; while ( true ) { try { //Check for cancellation at the beginning // of processing each loop stopToken.ThrowIfCancellationRequested(); //Check if buffer can deliver new tasks to us. // If not (and it's permanent), break worker processing loop. if ( !await PerformBufferCheckAsync() ) break; //It may be that the wait handle was signaled // as part of the Stop operation, // so we need to check for that as well. stopToken.ThrowIfCancellationRequested(); //Finally, dequeue and execute the task // and forward the result to the result queue IQueuedTaskToken queuedTaskToken = mTaskBuffer.TryGetNextTask(); if ( queuedTaskToken != null ) await ExecuteAndProcessResultAsync( queuedTaskToken, stopToken ); else mLogger.Debug( "Nothing to execute: no task was retrieved." ); //At the end of the loop, reset the handle mWaitForClearToFetchTask.Reset(); } catch ( OperationCanceledException ) { mLogger.Debug( "Worker stop requested. Breaking processing loop..." ); break; } } }
public void ConsumeBuffer() { mConsumeBufferTask = Task.Run(() => { while (!mTaskBuffer.IsCompleted) { IQueuedTaskToken queuedTaskToken = mTaskBuffer.TryGetNextTask(); if (queuedTaskToken != null) { mConsumedTasks.Add(queuedTaskToken); } else { Task.Delay(10).Wait(); } } }); }
private void AssertCorrectDefaultCalculateDelayMillisecondsTaskAfterFailureFn(StakhanoviseSetupDefaults setupDefaults) { Assert.NotNull(setupDefaults.CalculateDelayMillisecondsTaskAfterFailure); for (int iErrorCount = 0; iErrorCount < TestCalculateDelayMillisecondsErrorCountMax; iErrorCount++) { long expectedDelayMilliseconds = ( long )Math.Pow(10, iErrorCount + 1); IQueuedTaskToken mockTokenWithErrorCount = MockQueuedTaskTokenWithErrorCount(iErrorCount); long actualDelayMilliseconds = setupDefaults .CalculateDelayMillisecondsTaskAfterFailure(mockTokenWithErrorCount); Assert.AreEqual(expectedDelayMilliseconds, actualDelayMilliseconds); } }
private DateTimeOffset ComputeRetryAt ( IQueuedTaskToken queuedTaskToken ) { long delayMilliseconds = 0; try { //Compute the absolute time, in ticks, // until the task execution is delayed. delayMilliseconds = mOptions.CalculateRetryMillisecondsDelay( queuedTaskToken ); } catch ( Exception exc ) { mLogger.Error( "Failed to compute delay. Using default value of 0.", exc ); } return DateTimeOffset.UtcNow.AddMilliseconds( delayMilliseconds ); }
public void AssertConsumedTokenValid(IQueuedTaskToken newTaskToken, DateTimeOffset now) { Assert.NotNull(newTaskToken); Assert.NotNull(newTaskToken.DequeuedAt); Assert.NotNull(newTaskToken.DequeuedTask); Assert.NotNull(newTaskToken.LastQueuedTaskResult); //Assert.AreEqual( now, newTaskToken.DequeuedAt ); Assert.IsFalse(mDequeuedTokens.Any(t => t.DequeuedTask.Id == newTaskToken.DequeuedTask.Id)); if (mPreviousTaskToken != null) { Assert.GreaterOrEqual(newTaskToken.DequeuedTask.PostedAtTs, mPreviousTaskToken.DequeuedTask.PostedAtTs); } mPreviousTaskToken = newTaskToken; mDequeuedTokens.Add(newTaskToken); }
public static async Task ReportExecutionTimeAsync(this IExecutionPerformanceMonitor executionPerformanceMonitor, IQueuedTaskToken queuedTaskToken, TaskExecutionResult result) { if (executionPerformanceMonitor == null) { throw new ArgumentNullException(nameof(executionPerformanceMonitor)); } if (queuedTaskToken == null) { throw new ArgumentNullException(nameof(queuedTaskToken)); } if (result == null) { throw new ArgumentNullException(nameof(result)); } await executionPerformanceMonitor.ReportExecutionTimeAsync( queuedTaskToken.DequeuedTask.Type, result.ProcessingTimeMilliseconds, timeoutMilliseconds : 0); }
public void Dispose() { mPreviousTaskToken = null; mDequeuedTokens.Clear(); }
public void Test_ConsumerProducerScenario(int nProducers, int nConsumers) { Task coordinator; Task[] allProducers = new Task[nProducers]; Task[] allConsumers = new Task[nConsumers]; int expectedTotal = 0; ConcurrentBag <IQueuedTaskToken> processedTasks = new ConcurrentBag <IQueuedTaskToken>(); using (StandardTaskBuffer buffer = new StandardTaskBuffer(10)) { for (int iProducer = 0; iProducer < nProducers; iProducer++) { allProducers[iProducer] = Task.Run(() => { //Generate a number of items to produce // and add that to the expected total int nItems = new Random().Next(1, 100); Interlocked.Add(ref expectedTotal, nItems); while (nItems > 0) { bool isAdded = buffer.TryAddNewTask(new MockQueuedTaskToken(Guid.NewGuid())); if (isAdded) { nItems--; } else { Task.Delay(10).Wait(); } } }); } for (int iConsumer = 0; iConsumer < nConsumers; iConsumer++) { allConsumers[iConsumer] = Task.Run(() => { //Consumers run until the buffer is completed: // - marked as completed with respect to additons // AND // - has no more items while (!buffer.IsCompleted) { IQueuedTaskToken queuedTaskToken = buffer.TryGetNextTask(); if (queuedTaskToken != null) { processedTasks.Add(queuedTaskToken); } else { Task.Delay(10).Wait(); } } }); } coordinator = Task.Run(() => { //The coordinator waits for all producers // to finish and then marks buffer // addition operations as being completed Task.WaitAll(allProducers); buffer.CompleteAdding(); }); //Wait for all threads to stop Task.WaitAll(coordinator); Task.WaitAll(allConsumers); //Check that: // a) we have the exact number of items we added // b) there are no items that have been processed two times Assert.AreEqual(expectedTotal, processedTasks.Count); foreach (IQueuedTaskToken queuedTaskToken in processedTasks) { Assert.AreEqual(1, processedTasks.Count(t => t.DequeuedTask.Id == queuedTaskToken.DequeuedTask.Id)); } } }
public async Task AssertTaskNotInDbAnymoreAsync(IQueuedTaskToken newTaskToken) { Assert.IsNull(await mDataSource.GetQueuedTaskFromDbByIdAsync(newTaskToken .DequeuedTask .Id)); }
private async Task PollForQueuedTasksAsync() { CancellationToken stopToken = mStopTokenSource .Token; //Check cancellation token before starting // the polling loop if (stopToken.IsCancellationRequested) { return; } while (true) { try { //Check for token cancellation at the beginning of the loop stopToken.ThrowIfCancellationRequested(); //If the buffer is full, we wait for some space to become available, // since, even if we can dequeue an task, // we won't have anywhere to place it yet and we // may be needlessly helding a lock to that task if (mTaskBuffer.IsFull) { mLogger.Debug("Task buffer is full. Waiting for available space..."); mMetrics.UpdateMetric(AppMetricId.PollerWaitForBufferSpaceCount, m => m.Increment()); await mWaitForClearToAddToBuffer.ToTask(); } //It may be that the wait handle was signaled // as part of the Stop operation, // so we need to check for that as well. stopToken.ThrowIfCancellationRequested(); //Attempt to dequeue and then check cancellation IQueuedTaskToken queuedTaskToken = await mTaskQueueConsumer .DequeueAsync(mRequiredPayloadTypes); //Before posting the token to the buffer, // check if cancellation was requested stopToken.ThrowIfCancellationRequested(); if (queuedTaskToken != null) { //If we have found a token, attempt to set it as started // and only then add it to buffer for processing. //If not, dispose and discard the token mLogger.DebugFormat("Task found with id = {0}, type = {1}. Acquiring reservation...", queuedTaskToken.DequeuedTask.Id, queuedTaskToken.DequeuedTask.Type); mMetrics.UpdateMetric(AppMetricId.PollerDequeueCount, m => m.Increment()); mTaskBuffer.TryAddNewTask(queuedTaskToken); } else { //If there is no task available in the queue, begin waiting for // a notification of new added tasks mLogger.Debug("No task dequeued when polled. Waiting for available task..."); mMetrics.UpdateMetric(AppMetricId.PollerWaitForDequeueCount, m => m.Increment()); await mWaitForClearToDequeue.ToTask(); } //It may be that the wait handle was signaled // as part of the Stop operation, // so we need to check for that as well. stopToken.ThrowIfCancellationRequested(); //Finally, reset all the handles, at the end of the loop mWaitForClearToAddToBuffer.Reset(); mWaitForClearToDequeue.Reset(); } catch (OperationCanceledException) { mLogger.Debug("Stop requested. Breaking polling loop..."); break; } } mTaskBuffer.CompleteAdding(); }
public TaskExecutionContext(IQueuedTaskToken taskToken, CancellationToken stopToken) { mTaskToken = taskToken ?? throw new ArgumentNullException(nameof(taskToken)); mCancellationToken = stopToken; }
public Task <int> PostResultAsync(IQueuedTaskToken token) { return(PostResultAsync(token, timeoutMilliseconds: 0)); }