public async Task CheckAndProcessTimedOutJobs_GivenNonCompletedJobsAndHasTimeOutButFailedToUpdate_LogsError()
{
//Arrange
Job job = new Job {
Id = "job-id-1", JobDefinitionId = "job-def-2", Created = DateTimeOffset.Now.AddHours(-13)
};
IEnumerable <Job> nonCompletedJobs = new[]
{
job
};
IJobRepository jobRepository = CreateJobRepository();
jobRepository
.GetNonCompletedJobs()
.Returns(nonCompletedJobs);
jobRepository
.GetLatestJobBySpecificationIdAndDefinitionId(Arg.Any <string>(), Arg.Any <string>())
.Returns(job);
jobRepository
.UpdateJob(Arg.Any <Job>())
.Returns(HttpStatusCode.BadRequest);
IEnumerable <JobDefinition> jobDefinitions = new[]
{
new JobDefinition
{
Id = "job-def-1",
Timeout = TimeSpan.FromHours(12)
},
new JobDefinition
{
Id = "job-def-2",
Timeout = TimeSpan.FromHours(12)
}
};
IJobDefinitionsService jobDefinitionsService = CreateJobDefinitionsService();
jobDefinitionsService
.GetAllJobDefinitions()
.Returns(jobDefinitions);
ILogger logger = CreateLogger();
JobManagementService jobManagementService = CreateJobManagementService(jobRepository, logger: logger, jobDefinitionsService: jobDefinitionsService);
//Act
await jobManagementService.CheckAndProcessTimedOutJobs();
//Assert
logger
.Received(1)
.Error(Arg.Is("Failed to update timeout job, Id: 'job-id-1' with status code 400"));
}
public void CheckAndProcessTimedOutJobs_GivenNonCompletedJobsFoundButNoDefinitionsReturned_LogsAndThrowsException()
{
//Arrange
IEnumerable <Job> nonCompletedJobs = new[]
{
new Job {
Id = "job-id-1"
},
new Job {
Id = "job-id-2"
}
};
IJobRepository jobRepository = CreateJobRepository();
jobRepository
.GetNonCompletedJobs()
.Returns(nonCompletedJobs);
IJobDefinitionsService jobDefinitionsService = CreateJobDefinitionsService();
jobDefinitionsService
.GetAllJobDefinitions()
.Returns(Enumerable.Empty <JobDefinition>());
ILogger logger = CreateLogger();
JobManagementService jobManagementService = CreateJobManagementService(jobRepository, logger: logger, jobDefinitionsService: jobDefinitionsService);
//Act
Func <Task> test = async() => await jobManagementService.CheckAndProcessTimedOutJobs();
//Assert
test
.Should()
.ThrowExactly <Exception>()
.Which
.Message
.Should()
.Be("Failed to retrieve job definitions when processing timed out jobs");
logger
.Received(1)
.Error(Arg.Is("Failed to retrieve job definitions when processing timed out jobs"));
logger
.Received(1)
.Information($"{nonCompletedJobs.Count()} non completed jobs to process");
}
public async Task CheckAndProcessTimedOutJobs_GivenNonCompletedJobsFoundButHasNotTimedOut_DoesNotUpdateJob()
{
//Arrange
IEnumerable <Job> nonCompletedJobs = new[]
{
new Job {
Id = "job-id-1", JobDefinitionId = "job-def-2", Created = DateTimeOffset.Now.AddHours(-6)
},
};
IJobRepository jobRepository = CreateJobRepository();
jobRepository
.GetNonCompletedJobs()
.Returns(nonCompletedJobs);
IEnumerable <JobDefinition> jobDefinitions = new[]
{
new JobDefinition
{
Id = "job-def-1",
Timeout = TimeSpan.FromHours(12)
},
new JobDefinition
{
Id = "job-def-2",
Timeout = TimeSpan.FromHours(12)
}
};
IJobDefinitionsService jobDefinitionsService = CreateJobDefinitionsService();
jobDefinitionsService
.GetAllJobDefinitions()
.Returns(jobDefinitions);
ILogger logger = CreateLogger();
JobManagementService jobManagementService = CreateJobManagementService(jobRepository, logger: logger, jobDefinitionsService: jobDefinitionsService);
//Act
await jobManagementService.CheckAndProcessTimedOutJobs();
//Assert
await
jobRepository
.DidNotReceive()
.UpdateJob(Arg.Any <Job>());
}
public async Task CheckAndProcessTimedOutJobs_GivenNonCompletedJobsFoundButCouldNotFindJobDefinition_LogsError()
{
//Arrange
IEnumerable <Job> nonCompletedJobs = new[]
{
new Job {
Id = "job-id-1", JobDefinitionId = "job-def-3"
},
};
IJobRepository jobRepository = CreateJobRepository();
jobRepository
.GetNonCompletedJobs()
.Returns(nonCompletedJobs);
IEnumerable <JobDefinition> jobDefinitions = new[]
{
new JobDefinition
{
Id = "job-def-1",
Timeout = TimeSpan.FromHours(12)
},
new JobDefinition
{
Id = "job-def-2",
Timeout = TimeSpan.FromHours(12)
}
};
IJobDefinitionsService jobDefinitionsService = CreateJobDefinitionsService();
jobDefinitionsService
.GetAllJobDefinitions()
.Returns(jobDefinitions);
ILogger logger = CreateLogger();
JobManagementService jobManagementService = CreateJobManagementService(jobRepository, logger: logger, jobDefinitionsService: jobDefinitionsService);
//Act
await jobManagementService.CheckAndProcessTimedOutJobs();
//Assert
logger
.Received(1)
.Error($"Failed to find job definition : 'job-def-3' for job id: 'job-id-1'");
}
private async Task <(bool valid, IEnumerable <JobDefinition> jobDefinitions, IActionResult failureResponse)> ValidateCreateJobsRequests(IEnumerable <JobCreateModel> jobs)
{
Guard.ArgumentNotNull(jobs, nameof(jobs));
if (!jobs.Any())
{
string message = "Empty collection of job create models was provided";
_logger.Warning(message);
return(false, ArraySegment <JobDefinition> .Empty, new BadRequestObjectResult(message));
}
IEnumerable <JobDefinition> jobDefinitions = await _jobDefinitionsService.GetAllJobDefinitions();
if (jobDefinitions.IsNullOrEmpty())
{
string message = "Failed to retrieve job definitions";
_logger.Error(message);
return(false, ArraySegment <JobDefinition> .Empty, new InternalServerErrorResult(message));
}
IList <ValidationResult> validationResults = new List <ValidationResult>();
//ensure all jobs in batch have the correct job definition
foreach (JobCreateModel jobCreateModel in jobs)
{
Guard.IsNullOrWhiteSpace(jobCreateModel.JobDefinitionId, nameof(jobCreateModel.JobDefinitionId));
JobDefinition jobDefinition = jobDefinitions?.FirstOrDefault(m => m.Id == jobCreateModel.JobDefinitionId);
if (jobDefinition == null)
{
string message = $"A job definition could not be found for id: {jobCreateModel.JobDefinitionId}";
_logger.Warning(message);
return(false, ArraySegment <JobDefinition> .Empty, new PreconditionFailedResult(message));
}
CreateJobValidationModel createJobValidationModel = new CreateJobValidationModel
{
JobCreateModel = jobCreateModel,
JobDefinition = jobDefinition
};
ValidationResult validationResult = _createJobValidator.Validate(createJobValidationModel);
if (validationResult != null && !validationResult.IsValid)
{
validationResults.Add(validationResult);
}
}
if (validationResults.Any())
{
return(false, ArraySegment <JobDefinition> .Empty, new BadRequestObjectResult(validationResults));
}
return(true, jobDefinitions, null);
}
public async Task ProcessJobCompletion_JobHasParentWithMultipleCompletedChildrenWithAllSucceededAndPreCompletionJobsNotRunning_ThenPreCompletionJobQueued()
{
// Arrange
string parentJobId = "parent123";
string jobId = "child123";
string preCompletionJobDefinition = "PreCompletionJobDefinitionId";
Job job = new Job {
Id = jobId, ParentJobId = parentJobId, CompletionStatus = CompletionStatus.Succeeded, RunningStatus = RunningStatus.Completed
};
Job job2 = new Job {
Id = "child456", ParentJobId = parentJobId, RunningStatus = RunningStatus.Completed, CompletionStatus = CompletionStatus.Succeeded
};
Job job3 = new Job {
Id = "child789", ParentJobId = parentJobId, RunningStatus = RunningStatus.Completed, CompletionStatus = CompletionStatus.Succeeded
};
Job preCompletionJob = new Job
{
Id = "preCompletionJob",
ParentJobId = parentJobId,
JobDefinitionId = preCompletionJobDefinition,
RunningStatus = RunningStatus.InProgress
};
Job parentJob = new Job
{
Id = parentJobId,
RunningStatus = RunningStatus.InProgress,
JobDefinitionId = jobDefinitionId,
Trigger = new Trigger {
}
};
ILogger logger = CreateLogger();
IJobRepository jobRepository = CreateJobRepository();
jobRepository
.GetJobById(Arg.Is(jobId))
.Returns(job);
jobRepository
.GetJobById(Arg.Is(parentJobId))
.Returns(parentJob);
jobRepository
.GetChildJobsForParent(Arg.Is(parentJobId))
.Returns(new List <Job> {
job, job2
});
jobRepository
.CreateJob(Arg.Any <Job>())
.Returns(preCompletionJob);
IJobDefinitionsService jobDefinitionsService = CreateJobDefinitionsService();
jobDefinitionsService
.GetAllJobDefinitions()
.Returns(new[] { new JobDefinition {
Id = jobDefinitionId, PreCompletionJobs = new[] { preCompletionJobDefinition }
},
new JobDefinition {
Id = preCompletionJobDefinition
} });
JobManagementService jobManagementService = CreateJobManagementService(jobRepository,
logger: logger,
jobDefinitionsService: jobDefinitionsService);
JobSummary JobSummary = new JobSummary {
JobId = jobId, RunningStatus = RunningStatus.Completed
};
string json = JsonConvert.SerializeObject(JobSummary);
Message message = new Message(Encoding.UTF8.GetBytes(json));
message.UserProperties["jobId"] = jobId;
// Act
await jobManagementService.Process(message);
// Assert
await jobRepository
.Received(1)
.UpdateJob(Arg.Is <Job>(j => j.Id == parentJobId && j.RunningStatus == RunningStatus.Completing));
}
public async Task CheckAndProcessTimedOutJobs_GivenTwoNonCompletedJobsAndHasTimeOutButOnlyoneUpdatesSuccessfully_SendsOneNotification()
{
//Arrange
string outcome = "outcome-1";
List <Outcome> outcomes = new List <Outcome>
{
new Outcome
{
Description = outcome
}
};
IEnumerable <Job> nonCompletedJobs = new[]
{
new Job
{
Id = "job-id-1",
JobDefinitionId = "job-def-2",
Created = DateTimeOffset.Now.AddHours(-13),
Outcomes = outcomes,
Outcome = outcome
},
new Job
{
Id = "job-id-2",
JobDefinitionId = "job-def-1",
Created = DateTimeOffset.Now.AddHours(-13)
},
};
IJobRepository jobRepository = CreateJobRepository();
jobRepository
.GetNonCompletedJobs()
.Returns(nonCompletedJobs);
jobRepository
.GetLatestJobBySpecificationIdAndDefinitionId(Arg.Any <string>(), Arg.Any <string>())
.Returns(nonCompletedJobs.First());
jobRepository
.UpdateJob(Arg.Any <Job>())
.Returns(HttpStatusCode.OK, HttpStatusCode.BadRequest);
IEnumerable <JobDefinition> jobDefinitions = new[]
{
new JobDefinition
{
Id = "job-def-1",
Timeout = TimeSpan.FromHours(12)
},
new JobDefinition
{
Id = "job-def-2",
Timeout = TimeSpan.FromHours(12)
}
};
IJobDefinitionsService jobDefinitionsService = CreateJobDefinitionsService();
jobDefinitionsService
.GetAllJobDefinitions()
.Returns(jobDefinitions);
INotificationService notificationService = CreateNotificationsService();
ILogger logger = CreateLogger();
JobManagementService jobManagementService = CreateJobManagementService(jobRepository, logger: logger,
jobDefinitionsService: jobDefinitionsService, notificationService: notificationService);
//Act
await jobManagementService.CheckAndProcessTimedOutJobs();
//Assert
await
jobRepository
.Received(1)
.UpdateJob(Arg.Is <Job>(
m => m.Id == "job-id-1" &&
m.CompletionStatus == CompletionStatus.TimedOut &&
m.RunningStatus == RunningStatus.Completed &&
m.Outcomes.SequenceEqual(outcomes) &&
m.Outcome == outcome &&
m.OutcomeType == OutcomeType.Inconclusive
));
await
notificationService
.Received(1)
.SendNotification(Arg.Is <JobSummary>(
m => m.JobId == "job-id-1" &&
m.CompletionStatus == CompletionStatus.TimedOut &&
m.RunningStatus == RunningStatus.Completed
));
logger
.Received(1)
.Error(Arg.Is("Failed to update timeout job, Id: 'job-id-2' with status code 400"));
}
public async Task CheckAndProcessTimedOutJobs_GivenNonCompletedJobsAndHasTimeOutAndUpdatesSuccessfully_SendsNotification()
{
//Arrange
IEnumerable <Job> nonCompletedJobs = new[]
{
new Job {
Id = "job-id-1", JobDefinitionId = "job-def-2", Created = DateTimeOffset.Now.AddHours(-13)
},
};
IJobRepository jobRepository = CreateJobRepository();
jobRepository
.GetNonCompletedJobs()
.Returns(nonCompletedJobs);
jobRepository
.UpdateJob(Arg.Any <Job>())
.Returns(HttpStatusCode.OK);
IEnumerable <JobDefinition> jobDefinitions = new[]
{
new JobDefinition
{
Id = "job-def-1",
Timeout = TimeSpan.FromHours(12)
},
new JobDefinition
{
Id = "job-def-2",
Timeout = TimeSpan.FromHours(12)
}
};
IJobDefinitionsService jobDefinitionsService = CreateJobDefinitionsService();
jobDefinitionsService
.GetAllJobDefinitions()
.Returns(jobDefinitions);
INotificationService notificationService = CreateNotificationsService();
ILogger logger = CreateLogger();
JobManagementService jobManagementService = CreateJobManagementService(jobRepository, logger: logger,
jobDefinitionsService: jobDefinitionsService, notificationService: notificationService);
//Act
await jobManagementService.CheckAndProcessTimedOutJobs();
//Assert
await
jobRepository
.Received(1)
.UpdateJob(Arg.Is <Job>(
m => m.Id == "job-id-1" &&
m.CompletionStatus == CompletionStatus.TimedOut &&
m.RunningStatus == RunningStatus.Completed &&
m.OutcomeType == OutcomeType.Inconclusive
));
await
notificationService
.Received(1)
.SendNotification(Arg.Is <JobSummary>(
m => m.JobId == "job-id-1" &&
m.CompletionStatus == CompletionStatus.TimedOut &&
m.RunningStatus == RunningStatus.Completed
));
}
public async Task CheckAndProcessTimedOutJobs_GivenTwoNonCompletedJobsAndHasTimeOutButOnlyoneUpdatesSuccessfully_SendsOneNotification()
{
//Arrange
IEnumerable <Job> nonCompletedJobs = new[]
{
new Job {
Id = "job-id-1", JobDefinitionId = "job-def-2", Created = DateTimeOffset.Now.AddHours(-13)
},
new Job {
Id = "job-id-2", JobDefinitionId = "job-def-1", Created = DateTimeOffset.Now.AddHours(-13)
},
};
IJobRepository jobRepository = CreateJobRepository();
jobRepository
.GetNonCompletedJobs()
.Returns(nonCompletedJobs);
jobRepository
.UpdateJob(Arg.Any <Job>())
.Returns(HttpStatusCode.OK, HttpStatusCode.BadRequest);
IEnumerable <JobDefinition> jobDefinitions = new[]
{
new JobDefinition
{
Id = "job-def-1",
Timeout = TimeSpan.FromHours(12)
},
new JobDefinition
{
Id = "job-def-2",
Timeout = TimeSpan.FromHours(12)
}
};
IJobDefinitionsService jobDefinitionsService = CreateJobDefinitionsService();
jobDefinitionsService
.GetAllJobDefinitions()
.Returns(jobDefinitions);
INotificationService notificationService = CreateNotificationsService();
ILogger logger = CreateLogger();
JobManagementService jobManagementService = CreateJobManagementService(jobRepository, logger: logger,
jobDefinitionsService: jobDefinitionsService, notificationService: notificationService);
//Act
await jobManagementService.CheckAndProcessTimedOutJobs();
//Assert
await
jobRepository
.Received(1)
.UpdateJob(Arg.Is <Job>(
m => m.Id == "job-id-1" &&
m.Completed.Value.Date == DateTimeOffset.Now.Date &&
m.CompletionStatus == CompletionStatus.TimedOut &&
m.RunningStatus == RunningStatus.Completed
));
await
notificationService
.Received(1)
.SendNotification(Arg.Is <JobNotification>(
m => m.JobId == "job-id-1" &&
m.CompletionStatus == CompletionStatus.TimedOut &&
m.RunningStatus == RunningStatus.Completed &&
m.StatusDateTime.Date == DateTimeOffset.Now.Date
));
logger
.Received(1)
.Error(Arg.Is("Failed to update timeout job, Id: 'job-id-2' with status code 400"));
}
public async Task <IActionResult> CreateJobs(IEnumerable <JobCreateModel> jobs, HttpRequest request)
{
Guard.ArgumentNotNull(jobs, nameof(jobs));
Guard.ArgumentNotNull(request, nameof(request));
if (!jobs.Any())
{
string message = "Empty collection of job create models was provided";
_logger.Warning(message);
return(new BadRequestObjectResult(message));
}
IEnumerable <JobDefinition> jobDefinitions = await _jobDefinitionsService.GetAllJobDefinitions();
if (jobDefinitions.IsNullOrEmpty())
{
string message = "Failed to retrieve job definitions";
_logger.Error(message);
return(new InternalServerErrorResult(message));
}
IList <ValidationResult> validationResults = new List <ValidationResult>();
Reference user = request?.GetUser();
//ensure all jobs in batch have the correct job definition
foreach (JobCreateModel jobCreateModel in jobs)
{
Guard.IsNullOrWhiteSpace(jobCreateModel.JobDefinitionId, nameof(jobCreateModel.JobDefinitionId));
JobDefinition jobDefinition = jobDefinitions?.FirstOrDefault(m => m.Id == jobCreateModel.JobDefinitionId);
if (jobDefinition == null)
{
string message = $"A job definition could not be found for id: {jobCreateModel.JobDefinitionId}";
_logger.Warning(message);
return(new PreconditionFailedResult(message));
}
if (!jobCreateModel.Properties.ContainsKey("sfa-correlationId"))
{
jobCreateModel.Properties.Add("sfa-correlationId", request?.GetCorrelationId());
}
CreateJobValidationModel createJobValidationModel = new CreateJobValidationModel
{
JobCreateModel = jobCreateModel,
JobDefinition = jobDefinition
};
ValidationResult validationResult = _createJobValidator.Validate(createJobValidationModel);
if (validationResult != null && !validationResult.IsValid)
{
validationResults.Add(validationResult);
}
}
if (validationResults.Any())
{
return(new BadRequestObjectResult(validationResults));
}
IList <Job> createdJobs = new List <Job>();
foreach (JobCreateModel job in jobs)
{
Job newJobResult = await JobFromJobCreateModel(job, jobDefinitions, user);
if (newJobResult == null)
{
string message = $"Failed to create a job for job definition id: {job.JobDefinitionId}";
_logger.Error(message);
return(new InternalServerErrorResult(message));
}
createdJobs.Add(newJobResult);
}
IEnumerable <JobDefinition> jobDefinitionsToSupersede = await SupersedeJobs(createdJobs, jobDefinitions);
await QueueNotifications(createdJobs, jobDefinitionsToSupersede);
return(new OkObjectResult(createdJobs));
}