private async Task ProcessStatsBatchAsync(IEnumerable <StandardExecutionPerformanceMonitorWriteRequest> currentBatch)
{
MonotonicTimestamp startWrite = MonotonicTimestamp
.Now();
List <TaskPerformanceStats> executionTimeInfoBatch =
new List <TaskPerformanceStats>();
try
{
foreach (StandardExecutionPerformanceMonitorWriteRequest rq in currentBatch)
{
executionTimeInfoBatch.Add(new TaskPerformanceStats(rq.PayloadType, rq.DurationMilliseconds));
}
await mStatsWriter.WriteAsync(executionTimeInfoBatch);
foreach (StandardExecutionPerformanceMonitorWriteRequest rq in currentBatch)
{
rq.SetCompleted(1);
}
IncrementPerfMonWriteCount(MonotonicTimestamp
.Since(startWrite));
}
catch (Exception exc)
{
foreach (StandardExecutionPerformanceMonitorWriteRequest rq in currentBatch)
{
rq.SetFailed(exc);
if (rq.CanBeRetried)
{
mStatsProcessingQueue.Add(rq);
}
}
mLogger.Error("Error processing performance stats batch", exc);
}
}
private void WaitForNotifications(NpgsqlConnection signalingConn, CancellationToken cancellationToken)
{
//At this point, check if cancellation was requested
// and exit if so
cancellationToken.ThrowIfCancellationRequested();
while (true)
{
mLogger.DebugFormat("Waiting for notifications on channel {0}...",
mNewTaskNotificationChannelName);
//Pass the cancellation token to the WaitAsync
// to be able to stop the listener when requested
try
{
//TODO: timeout should be configurable
bool hadNotification = signalingConn.Wait(250);
if (!hadNotification)
{
ProcessListenerTimedOutWhileWaiting();
mLogger.Debug("Listener timed out while waiting. Checking stop token and restarting wait...");
}
else
{
mLogger.Debug("Task Notification received.");
}
}
catch (NullReferenceException exc)
{
mLogger.Error("Possible connection failure while waiting", exc);
break;
}
//At this point a notification has been received:
// before the next go-around of WaitAsync,
// check if cancellation was requested
cancellationToken.ThrowIfCancellationRequested();
}
}
private async Task<TaskExecutionResult> ExecuteTaskAsync ( TaskExecutionContext executionContext )
{
ITaskExecutor taskExecutor = null;
DateTimeOffset retryAt = DateTimeOffset.UtcNow;
IQueuedTask dequeuedTask = executionContext
.TaskToken
.DequeuedTask;
try
{
//Check for cancellation before we start execution
executionContext.StartTimingExecution();
executionContext.ThrowIfCancellationRequested();
//Attempt to resolve and run task executor
if ( ( taskExecutor = ResolveTaskExecutor( dequeuedTask ) ) != null )
{
mLogger.DebugFormat( "Beginning task execution. Task id = {0}.",
dequeuedTask.Id );
//Execute task
await taskExecutor.ExecuteAsync( dequeuedTask.Payload,
executionContext );
mLogger.DebugFormat( "Task execution completed. Task id = {0}.",
dequeuedTask.Id );
//Ensure we have a result - since no exception was thrown
// and no result explicitly set, assume success.
if ( !executionContext.HasResult )
executionContext.NotifyTaskCompleted();
}
}
catch ( OperationCanceledException )
{
//User code has observed cancellation request
executionContext?.NotifyCancellationObserved();
}
catch ( Exception exc )
{
mLogger.Error( "Error executing queued task",
exception: exc );
bool isRecoverable = mOptions.IsTaskErrorRecoverable( dequeuedTask,
exc );
executionContext?.NotifyTaskErrored( new QueuedTaskError( exc ),
isRecoverable: isRecoverable );
}
finally
{
executionContext.StopTimingExecution();
}
//Compute the amount of time to delay task execution
// if execution failed
if ( executionContext.HasResult
&& executionContext.ExecutionFailed )
retryAt = ComputeRetryAt( executionContext.TaskToken );
return taskExecutor != null
? new TaskExecutionResult( executionContext.ResultInfo,
duration: executionContext.Duration,
retryAt: retryAt,
faultErrorThresholdCount: mOptions.FaultErrorThresholdCount )
: null;
}
private async Task ProcessResultBatchAsync(Queue <PostgreSqlTaskResultQueueProcessRequest> currentBatch)
{
MonotonicTimestamp startWrite = MonotonicTimestamp
.Now();
//An explicit choice has been made not to use transactions
// since failing to update a result MUST NOT
// cause the other successful updates to be rolled back.
using (NpgsqlConnection conn = await OpenConnectionAsync(CancellationToken.None))
using (NpgsqlCommand updateCmd = new NpgsqlCommand(mUpdateSql, conn))
{
NpgsqlParameter pStatus = updateCmd.Parameters
.Add("t_status", NpgsqlDbType.Integer);
NpgsqlParameter pLastError = updateCmd.Parameters
.Add("t_last_error", NpgsqlDbType.Text);
NpgsqlParameter pErrorCount = updateCmd.Parameters
.Add("t_error_count", NpgsqlDbType.Integer);
NpgsqlParameter pLastErrorIsRecoverable = updateCmd.Parameters
.Add("t_last_error_recoverable", NpgsqlDbType.Boolean);
NpgsqlParameter pProcessingTime = updateCmd.Parameters
.Add("t_processing_time_milliseconds", NpgsqlDbType.Bigint);
NpgsqlParameter pFinalizedAt = updateCmd.Parameters
.Add("t_processing_finalized_at_ts", NpgsqlDbType.TimestampTz);
NpgsqlParameter pId = updateCmd.Parameters
.Add("t_id", NpgsqlDbType.Uuid);
await updateCmd.PrepareAsync();
while (currentBatch.Count > 0)
{
PostgreSqlTaskResultQueueProcessRequest processRq =
currentBatch.Dequeue();
try
{
pStatus.Value = ( int )processRq.ResultToUpdate.Status;
string strLastError = processRq.ResultToUpdate.LastError.ToJson();
if (strLastError != null)
{
pLastError.Value = strLastError;
}
else
{
pLastError.Value = DBNull.Value;
}
pErrorCount.Value = processRq.ResultToUpdate.ErrorCount;
pLastErrorIsRecoverable.Value = processRq.ResultToUpdate.LastErrorIsRecoverable;
pProcessingTime.Value = processRq.ResultToUpdate.ProcessingTimeMilliseconds;
if (processRq.ResultToUpdate.ProcessingFinalizedAtTs.HasValue)
{
pFinalizedAt.Value = processRq.ResultToUpdate.ProcessingFinalizedAtTs;
}
else
{
pFinalizedAt.Value = DBNull.Value;
}
pId.Value = processRq.ResultToUpdate.Id;
int affectedRows = await updateCmd.ExecuteNonQueryAsync();
processRq.SetCompleted(affectedRows);
IncrementResultWriteCount(MonotonicTimestamp
.Since(startWrite));
}
catch (OperationCanceledException)
{
processRq.SetCancelled();
throw;
}
catch (Exception exc)
{
processRq.SetFailed(exc);
if (processRq.CanBeRetried)
{
mResultProcessingQueue.Add(processRq);
}
mLogger.Error("Error processing result", exc);
}
}
await conn.CloseAsync();
}
}