public async Task <IActionResult> UpdateJobQueue(int projectId, int queueId, UpdateJobDto job)
{
_logger.LogInformation("Update job queue {queueId} in project {projectId}", queueId, projectId);
if (queueId != job.Id)
{
return(BadRequest("Queue Id doesn't match."));
}
var entity = _mapper.Map <JobQueue>(job);
entity.ProjectId = projectId;
await _jobQueueService.UpdateJobQueue(entity);
return(Ok());
}
public override string Execute()
{
Console.WriteLine($"Trying to cancel queue \"{Number}\" in project {Project}...");
string message;
var code = "";
if (!int.TryParse(Number, out var id))
{
code = Number;
}
var project = _projectService.GetProjectByName(Project).Result;
if (project != null)
{
var queue = !string.IsNullOrEmpty(code) ? _jobQueueService.GetJobQueue(project.Id, code).Result : _jobQueueService.GetJobQueue(project.Id, id).Result;
if (queue != null)
{
if (!CancellablleJobStatus.Contains(queue.Status))
{
message = $"Cannot cancel queue {Number} with status {queue.Status}";
return(message);
}
if (queue.Status == JobStatus.Processing && !string.IsNullOrEmpty(queue.CatapultEngineId))
{
if (!Console.GetYesNo($"The job queue {Number} was being processed by engine {queue.CatapultEngineId}. Make sure the engine process has stopped. Continue the job cancellation?", false))
{
return(string.Empty);
}
}
_jobQueueService.UpdateJobQueue(project.Id, queue.Id, new UpdateJobDto
{
Id = queue.Id,
Status = JobStatus.Cancelled,
CatapultEngineId = queue.CatapultEngineId,
CatapultEngineIPAddress = queue.CatapultEngineIPAddress,
CatapultEngineMachineName = queue.CatapultEngineMachineName,
CatapultEngineVersion = queue.CatapultEngineVersion,
JobTasksStatus = queue.JobTasksStatus,
JobType = queue.JobType,
OutputValues = queue.OutputValues,
Remarks = Reason ?? queue.Remarks
}).Wait();
message = $"Queue {Number} has been cancelled successfully";
Logger.LogInformation(message);
return(message);
}
}
message = $"Failed to cancel queue {Number}. Make sure the project name and queue number are correct.";
return(message);
}
public async Task ExecuteJob(JobDto jobQueue)
{
using (_logger.BeginScope(new JobScope(jobQueue.ProjectId, jobQueue.Id)))
{
try
{
_logger.LogInformation($"Executing job queue {jobQueue.Code}.");
var jobTasks = await _jobDefinitionService.GetJobTaskDefinitions(jobQueue.ProjectId, jobQueue.JobDefinitionId ?? 0);
var workingLocation = Path.Combine(_engineConfig.WorkingLocation, jobQueue.Code);
var result = await _taskRunner.Run(jobQueue.ProjectId, jobQueue, jobTasks, _engineConfig.TaskProvidersLocation, workingLocation);
if (result.Values.Any(t => t.IsSuccess && t.StopTheProcess))
{
jobQueue.Status = JobStatus.Pending;
}
else if (result.Values.Any(t => !t.IsSuccess))
{
jobQueue.Status = JobStatus.Error;
}
else
{
jobQueue.Status = JobStatus.Completed;
}
}
catch (Exception ex)
{
_logger.LogError(ex, ex.Message);
jobQueue.Status = JobStatus.Error;
}
await _jobQueueService.UpdateJobQueue(jobQueue.Id, new UpdateJobDto
{
Id = jobQueue.Id,
CatapultEngineId = jobQueue.CatapultEngineId,
CatapultEngineIPAddress = jobQueue.CatapultEngineIPAddress,
CatapultEngineMachineName = jobQueue.CatapultEngineMachineName,
CatapultEngineVersion = jobQueue.CatapultEngineVersion,
JobType = jobQueue.JobType,
Status = jobQueue.Status,
JobTasksStatus = jobQueue.JobTasksStatus,
OutputValues = jobQueue.OutputValues
});
await _jobLogWriter.EndJobLog(jobQueue.Id);
await _jobQueueService.SendNotification(jobQueue.Id);
if (jobQueue.Status == JobStatus.Completed &&
jobQueue.IsDeletion &&
jobQueue.ProjectStatus == ProjectStatusFilterType.Deleting)
{
_logger.LogInformation($"Deleting project {jobQueue.ProjectId}");
await _projectService.DeleteProjectByEngine(jobQueue.ProjectId);
}
}
}
public override string Execute()
{
Console.WriteLine($"Trying to restart queue \"{Number}\" in project {Project}...");
string message;
var code = "";
if (!int.TryParse(Number, out var id))
{
code = Number;
}
var project = _projectService.GetProjectByName(Project).Result;
if (project != null)
{
var queue = !string.IsNullOrEmpty(code) ? _jobQueueService.GetJobQueue(project.Id, code).Result : _jobQueueService.GetJobQueue(project.Id, id).Result;
if (queue != null)
{
_jobQueueService.UpdateJobQueue(queue.Id, new UpdateJobDto
{
Id = queue.Id,
Status = JobStatus.Queued,
CatapultEngineId = queue.CatapultEngineId,
CatapultEngineIPAddress = queue.CatapultEngineIPAddress,
CatapultEngineMachineName = queue.CatapultEngineMachineName,
CatapultEngineVersion = queue.CatapultEngineVersion,
JobTasksStatus = queue.JobTasksStatus,
JobType = queue.JobType,
OutputValues = queue.OutputValues,
Remarks = null
}).Wait();
message = $"Queue {Number} has been restarted successfully";
Logger.LogInformation(message);
return(message);
}
}
message = $"Failed to restart queue {Number}. Make sure the project name and queue number are correct.";
return(message);
}
public async Task <IActionResult> UpdateJobQueue(int queueId, UpdateJobDto job)
{
_logger.LogInformation("Updating job queue. Request body: {@job}", job);
try
{
if (queueId != job.Id)
{
return(BadRequest("Queue Id doesn't match."));
}
var entity = _mapper.Map <JobQueue>(job);
await _jobQueueService.UpdateJobQueue(entity);
return(Ok());
}
catch (JobProcessedByOtherEngineException ex)
{
_logger.LogWarning(ex, "Job is processed by other engine");
return(BadRequest(ex.Message));
}
}
public async Task <Dictionary <int, TaskRunnerResult> > Run(int projectId, JobDto job, List <JobTaskDefinitionDto> jobTasks, string pluginsLocation, string workingLocation)
{
_pluginManager.RefreshPlugins();
// update the status to processing so the clients can start listening for logs
job.Status = JobStatus.Processing;
var orderedJobTasks = jobTasks.OrderBy(t => t.Sequence).ToList();
_logger.LogInformation("[Queue {Code}] Attempting to run {Count} job tasks", job.Code, orderedJobTasks.Count);
var results = orderedJobTasks.ToDictionary(t => t.Id, t => new TaskRunnerResult());
var outputValues = job.OutputValues ?? new Dictionary <string, string>();
job.JobTasksStatus = job.JobTasksStatus?.Count > 0 ? job.JobTasksStatus :
orderedJobTasks.Select((t, idx) => new JobTaskStatusDto
{
TaskName = t.Name,
Sequence = idx + 1,
Status = JobTaskStatusType.NotExecuted
}).ToList();
foreach (var jobTask in orderedJobTasks)
{
using (_logger.BeginScope(new TaskScope(jobTask.Name)))
{
// skip previously success task
var jobTaskStatus = job.JobTasksStatus.First(t => t.TaskName == jobTask.Name);
if (jobTaskStatus.Status == JobTaskStatusType.Success)
{
results[jobTask.Id] = new TaskRunnerResult(true, "");
continue;
}
jobTaskStatus.Status = JobTaskStatusType.Executing;
await _jobQueueService.UpdateJobQueue(job.Id, new UpdateJobDto
{
Id = job.Id,
CatapultEngineId = job.CatapultEngineId,
CatapultEngineIPAddress = job.CatapultEngineIPAddress,
CatapultEngineMachineName = job.CatapultEngineMachineName,
CatapultEngineVersion = job.CatapultEngineVersion,
JobTasksStatus = job.JobTasksStatus,
JobType = job.JobType,
Status = job.Status,
OutputValues = outputValues
});
var taskObj = GetJobTaskInstance(projectId, job.Code, jobTask, workingLocation);
TaskRunnerResult result;
if (job.IsDeletion)
{
_logger.LogInformation("[Queue {Code}] Running {jobTask.Type} task", job.Code, jobTask.Type);
System.Console.WriteLine($"Invoking \"{jobTask.Type}\" task.");
result = await taskObj.RunMainTask(outputValues);
results[jobTask.Id] = result;
if (!result.IsSuccess && result.StopTheProcess)
{
_logger.LogError("[Queue {Code}] Execution of {Type} task was failed, stopping the next task execution. Error: {ErrorMessage}", job.Code, jobTask.Type, result.ErrorMessage);
jobTaskStatus.Status = JobTaskStatusType.Failed;
jobTaskStatus.Remarks = result.ErrorMessage;
break;
}
}
else
{
// pre-processing
_logger.LogInformation("[Queue {Code}] Running {Type} pre-processing task", job.Code, jobTask.Type);
var preResult = await taskObj.RunPreprocessingTask();
if (!preResult.IsSuccess && preResult.StopTheProcess)
{
_logger.LogError("[Queue {Code}] Execution of {Type} pre-processing task was failed, stopping the next task execution. Error: {ErrorMessage}", job.Code, jobTask.Type, preResult.ErrorMessage);
jobTaskStatus.Status = JobTaskStatusType.Failed;
jobTaskStatus.Remarks = preResult.ErrorMessage;
break;
}
// main process
_logger.LogInformation("[Queue {Code}] Running {jobTask.Type} task", job.Code, jobTask.Type);
System.Console.WriteLine($"Invoking \"{jobTask.Type}\" task.");
result = await taskObj.RunMainTask(outputValues);
results[jobTask.Id] = result;
if (!result.IsSuccess && result.StopTheProcess)
{
_logger.LogError("[Queue {Code}] Execution of {Type} task was failed, stopping the next task execution. Error: {ErrorMessage}", job.Code, jobTask.Type, result.ErrorMessage);
jobTaskStatus.Status = JobTaskStatusType.Failed;
jobTaskStatus.Remarks = result.ErrorMessage;
break;
}
// save output values to be used as the input for the next tasks
if (result.OutputValues != null)
{
foreach (var key in result.OutputValues.Keys)
{
if (outputValues.ContainsKey(key))
{
outputValues[key] = result.OutputValues[key];
}
else
{
outputValues.Add(key, result.OutputValues[key]);
}
}
}
// post-processing
_logger.LogInformation("[Queue {Code}] Running {Type} post-processing task", job.Code, jobTask.Type);
var postResult = await taskObj.RunPostprocessingTask();
if (!postResult.IsSuccess && postResult.StopTheProcess)
{
_logger.LogError("[Queue {Code}] Execution of {Type} post-processing task was failed, stopping the next task execution. Error: {ErrorMessage}", job.Code, jobTask.Type, postResult.ErrorMessage);
jobTaskStatus.Status = JobTaskStatusType.Failed;
jobTaskStatus.Remarks = postResult.ErrorMessage;
break;
}
// check if there's a need to postpone the next task
if (result.IsSuccess && result.StopTheProcess)
{
_logger.LogInformation("[Queue {Code}] Execution of {Type} require the job to be halted. Run the restart command in order to restart the job", job.Code, jobTask.Type);
jobTaskStatus.Status = JobTaskStatusType.Success;
jobTaskStatus.Remarks = result.TaskRemarks;
int currentTaskStatusIdx = job.JobTasksStatus.IndexOf(jobTaskStatus);
var nextTaskStatus = currentTaskStatusIdx == job.JobTasksStatus.Count - 1 ? null : job.JobTasksStatus[currentTaskStatusIdx + 1];
nextTaskStatus.Status = JobTaskStatusType.Pending;
nextTaskStatus.Remarks = result.StopRemarks;
break;
}
}
jobTaskStatus.Status = JobTaskStatusType.Success;
jobTaskStatus.Remarks = result.TaskRemarks;
await _jobQueueService.UpdateJobQueue(job.Id, new UpdateJobDto
{
Id = job.Id,
CatapultEngineId = job.CatapultEngineId,
CatapultEngineIPAddress = job.CatapultEngineIPAddress,
CatapultEngineMachineName = job.CatapultEngineMachineName,
CatapultEngineVersion = job.CatapultEngineVersion,
JobTasksStatus = job.JobTasksStatus,
JobType = job.JobType,
Status = job.Status,
OutputValues = outputValues
});
}
}
job.OutputValues = outputValues;
var serializedResult = JsonConvert.SerializeObject(results);
_logger.LogInformation("[Queue {Code}] Job tasks execution complete with the following result: {serializedResult}", job.Code, serializedResult);
return(results);
}
