public async Task <ActionResult> Sweep()
{
if (!_taskScheduler.VerifyAuthToken(Request.Headers["X-AUTH-TOKEN"]))
{
return(new HttpUnauthorizedResult());
}
var pendingTasks = await _scheduleTaskService.GetPendingTasksAsync();
var count = 0;
var taskParameters = QueryString.Current.ToDictionary();
if (pendingTasks.Count > 0)
{
Virtualize(taskParameters);
}
for (var i = 0; i < pendingTasks.Count; i++)
{
var task = pendingTasks[i];
if (i > 0 /*&& (DateTime.UtcNow - _sweepStart).TotalMinutes > _taskScheduler.SweepIntervalMinutes*/)
{
// Maybe a subsequent Sweep call or another machine in a webfarm executed
// successive tasks already.
// To be able to determine this, we need to reload the entity from the database.
// The TaskExecutor will exit when the task should be in running state then.
_services.DbContext.ReloadEntity(task);
task.LastHistoryEntry = _scheduleTaskService.GetLastHistoryEntryByTaskId(task.Id);
}
if (task.IsPending)
{
await _taskExecutor.ExecuteAsync(task, taskParameters);
count++;
}
}
return(Content("{0} of {1} pending tasks executed".FormatInvariant(count, pendingTasks.Count)));
}
private async Task Run(int taskId, HttpContext context, ITaskStore taskStore, ITaskExecutor executor, IDictionary <string, string> taskParameters)
{
var task = await taskStore.GetTaskByIdAsync(taskId);
if (task == null)
{
context.Response.StatusCode = StatusCodes.Status404NotFound;
return;
}
await Virtualize(context, taskParameters);
await executor.ExecuteAsync(task, context, taskParameters);
context.Response.StatusCode = StatusCodes.Status200OK;
await context.Response.WriteAsync($"Task '{task.Name}' executed.");
}
private async Task Poll(HttpContext context, ITaskStore taskStore, ITaskExecutor executor, IDictionary <string, string> taskParameters)
{
var pendingTasks = await taskStore.GetPendingTasksAsync();
var numTasks = pendingTasks.Count;
var numExecuted = 0;
if (pendingTasks.Any())
{
await Virtualize(context, taskParameters);
}
for (var i = 0; i < numTasks; i++)
{
var task = pendingTasks[i];
if (i > 0 /*&& (DateTime.UtcNow - _sweepStart).TotalMinutes > _taskScheduler.SweepIntervalMinutes*/)
{
// Maybe a subsequent Sweep call or another machine in a webfarm executed
// successive tasks already.
// To be able to determine this, we need to reload the entity from the database.
// The TaskExecutor will exit when the task should be in running state then.
await taskStore.ReloadTaskAsync(task);
task.LastExecution = await taskStore.GetLastExecutionInfoByTaskIdAsync(task.Id);
}
if (task.IsPending)
{
await executor.ExecuteAsync(task, context, taskParameters);
numExecuted++;
}
}
context.Response.StatusCode = StatusCodes.Status200OK;
await context.Response.WriteAsync($"{numExecuted} of {numTasks} pending tasks executed.");
}
private async Task<TaskExecutionResult> ExecuteTaskAsync ( TaskExecutionContext executionContext )
{
ITaskExecutor taskExecutor = null;
DateTimeOffset retryAt = DateTimeOffset.UtcNow;
IQueuedTask dequeuedTask = executionContext
.TaskToken
.DequeuedTask;
try
{
//Check for cancellation before we start execution
executionContext.StartTimingExecution();
executionContext.ThrowIfCancellationRequested();
//Attempt to resolve and run task executor
if ( ( taskExecutor = ResolveTaskExecutor( dequeuedTask ) ) != null )
{
mLogger.DebugFormat( "Beginning task execution. Task id = {0}.",
dequeuedTask.Id );
//Execute task
await taskExecutor.ExecuteAsync( dequeuedTask.Payload,
executionContext );
mLogger.DebugFormat( "Task execution completed. Task id = {0}.",
dequeuedTask.Id );
//Ensure we have a result - since no exception was thrown
// and no result explicitly set, assume success.
if ( !executionContext.HasResult )
executionContext.NotifyTaskCompleted();
}
}
catch ( OperationCanceledException )
{
//User code has observed cancellation request
executionContext?.NotifyCancellationObserved();
}
catch ( Exception exc )
{
mLogger.Error( "Error executing queued task",
exception: exc );
bool isRecoverable = mOptions.IsTaskErrorRecoverable( dequeuedTask,
exc );
executionContext?.NotifyTaskErrored( new QueuedTaskError( exc ),
isRecoverable: isRecoverable );
}
finally
{
executionContext.StopTimingExecution();
}
//Compute the amount of time to delay task execution
// if execution failed
if ( executionContext.HasResult
&& executionContext.ExecutionFailed )
retryAt = ComputeRetryAt( executionContext.TaskToken );
return taskExecutor != null
? new TaskExecutionResult( executionContext.ResultInfo,
duration: executionContext.Duration,
retryAt: retryAt,
faultErrorThresholdCount: mOptions.FaultErrorThresholdCount )
: null;
}
public async Task ExecuteAsync(Job job, CancellationToken cancellationToken)
{
_logger.LogInformation("Start executing job '{jobId}'.", job.Id);
// extract patient ids from group if they aren't extracted before for group
if (job.FilterInfo.FilterScope == FilterScope.Group && !job.Patients.Any())
{
_logger.LogInformation("Start extracting patients from group '{groupId}'.", job.FilterInfo.GroupId);
// For now, the queryParameters is always null.
// This parameter will be used when we enable filter groups in the future.
var patientIds = await _groupMemberExtractor.GetGroupPatientsAsync(
job.FilterInfo.GroupId,
null,
job.DataPeriod.End,
cancellationToken);
// set the version id for processed patient
// the processed patients is empty at the beginning , and will be updated when completing a successful job.
job.Patients = patientIds.Select(patientId =>
new PatientWrapper(
patientId,
job.FilterInfo.ProcessedPatients.ContainsKey(patientId) ? job.FilterInfo.ProcessedPatients[patientId] : 0));
_logger.LogInformation(
"Extract {patientCount} patients from group '{groupId}', including {newPatientCount} new patients.",
patientIds.Count,
job.FilterInfo.GroupId,
job.Patients.Where(p => p.VersionId == 0).ToList().Count);
}
var jobProgressUpdater = _jobProgressUpdaterFactory.Create(job);
var progressReportTask = Task.Run(() => jobProgressUpdater.Consume(cancellationToken), cancellationToken);
var tasks = new List <Task <TaskResult> >();
try
{
// add running task to task list for resume job
tasks.AddRange(job.RunningTasks.Values.Select(taskContext => Task.Run(async() =>
await _taskExecutor.ExecuteAsync(taskContext, jobProgressUpdater, cancellationToken))).ToList());
var filters = job.FilterInfo.TypeFilters.ToList();
while (true)
{
if (tasks.Count >= _schedulerConfiguration.MaxConcurrencyCount)
{
var finishedTask = await Task.WhenAny(tasks);
tasks.Remove(finishedTask);
if (finishedTask.IsFaulted)
{
_logger.LogError(finishedTask.Exception, "Process task failed.");
// if there is a task failed, we need to wait all the task to be completed
// otherwise, the data in other tasks may be missing.
await Task.WhenAll(tasks);
throw new ExecuteTaskFailedException("Task execution failed", finishedTask.Exception);
}
}
TaskContext taskContext = null;
switch (job.FilterInfo.FilterScope)
{
case FilterScope.System:
if (job.NextTaskIndex < job.FilterInfo.TypeFilters.Count())
{
var typeFilters = new List <TypeFilter>
{
filters[job.NextTaskIndex]
};
taskContext = TaskContext.CreateFromJob(job, typeFilters);
}
break;
case FilterScope.Group:
if (job.NextTaskIndex * JobConfigurationConstants.NumberOfPatientsPerTask < job.Patients.ToList().Count)
{
var selectedPatients = job.Patients.Skip(job.NextTaskIndex * JobConfigurationConstants.NumberOfPatientsPerTask)
.Take(JobConfigurationConstants.NumberOfPatientsPerTask).ToList();
taskContext = TaskContext.CreateFromJob(job, filters, selectedPatients);
}
break;
default:
// this case should not happen
throw new ArgumentOutOfRangeException($"The filterScope {job.FilterInfo.FilterScope} isn't supported now.");
}
// if there is no new task context created, then all the tasks are generated, break the while loop;
if (taskContext == null)
{
break;
}
job.RunningTasks[taskContext.Id] = taskContext;
tasks.Add(Task.Run(async() => await _taskExecutor.ExecuteAsync(taskContext, jobProgressUpdater, cancellationToken)));
_logger.LogInformation("Start processing task '{taskIndex}'", job.NextTaskIndex);
job.NextTaskIndex++;
}
var taskResults = await Task.WhenAll(tasks);
}
catch (Exception ex)
{
_logger.LogError(ex, "Process task failed.");
throw new ExecuteTaskFailedException("Task execution failed", ex);
}
finally
{
// if there is a task failed, we need to consume all the task contexts before exit jobExecutor
// otherwise, the task contexts may be consumed and update the job to active after the job is completed in JobManager
jobProgressUpdater.Complete();
await progressReportTask;
}
_logger.LogInformation("Finish scheduling job '{jobId}'", job.Id);
}