public JobsController(IJobService jobService,
IJobManagementService jobManagementService, IJobDefinitionsService jobDefinitionsService)
{
_jobService = jobService;
_jobManagementService = jobManagementService;
_jobDefinitionsService = jobDefinitionsService;
}
public JobManagementService(
IJobRepository jobRepository,
INotificationService notificationService,
IJobDefinitionsService jobDefinitionsService,
IJobsResiliencePolicies resiliencePolicies,
ILogger logger,
IValidator <CreateJobValidationModel> createJobValidator,
IMessengerService messengerService,
ICacheProvider cacheProvider)
{
Guard.ArgumentNotNull(jobRepository, nameof(jobRepository));
Guard.ArgumentNotNull(notificationService, nameof(notificationService));
Guard.ArgumentNotNull(jobDefinitionsService, nameof(jobDefinitionsService));
Guard.ArgumentNotNull(logger, nameof(logger));
Guard.ArgumentNotNull(createJobValidator, nameof(createJobValidator));
Guard.ArgumentNotNull(messengerService, nameof(messengerService));
Guard.ArgumentNotNull(resiliencePolicies?.JobRepository, nameof(resiliencePolicies.JobRepository));
Guard.ArgumentNotNull(resiliencePolicies?.JobDefinitionsRepository, nameof(resiliencePolicies.JobDefinitionsRepository));
Guard.ArgumentNotNull(resiliencePolicies?.CacheProviderPolicy, nameof(resiliencePolicies.CacheProviderPolicy));
Guard.ArgumentNotNull(cacheProvider, nameof(cacheProvider));
_jobRepository = jobRepository;
_notificationService = notificationService;
_jobDefinitionsService = jobDefinitionsService;
_jobsRepositoryPolicy = resiliencePolicies.JobRepository;
_logger = logger;
_createJobValidator = createJobValidator;
_messengerService = messengerService;
_cacheProvider = cacheProvider;
_cacheProviderPolicy = resiliencePolicies.CacheProviderPolicy;
}
public async Task CheckAndProcessTimedOutJobs_GivenNonCompletedJobsAndHasTimeOutButFailedToUpdate_LogsError()
{
//Arrange
Job job = new Job {
Id = "job-id-1", JobDefinitionId = "job-def-2", Created = DateTimeOffset.Now.AddHours(-13)
};
IEnumerable <Job> nonCompletedJobs = new[]
{
job
};
IJobRepository jobRepository = CreateJobRepository();
jobRepository
.GetNonCompletedJobs()
.Returns(nonCompletedJobs);
jobRepository
.GetLatestJobBySpecificationIdAndDefinitionId(Arg.Any <string>(), Arg.Any <string>())
.Returns(job);
jobRepository
.UpdateJob(Arg.Any <Job>())
.Returns(HttpStatusCode.BadRequest);
IEnumerable <JobDefinition> jobDefinitions = new[]
{
new JobDefinition
{
Id = "job-def-1",
Timeout = TimeSpan.FromHours(12)
},
new JobDefinition
{
Id = "job-def-2",
Timeout = TimeSpan.FromHours(12)
}
};
IJobDefinitionsService jobDefinitionsService = CreateJobDefinitionsService();
jobDefinitionsService
.GetAllJobDefinitions()
.Returns(jobDefinitions);
ILogger logger = CreateLogger();
JobManagementService jobManagementService = CreateJobManagementService(jobRepository, logger: logger, jobDefinitionsService: jobDefinitionsService);
//Act
await jobManagementService.CheckAndProcessTimedOutJobs();
//Assert
logger
.Received(1)
.Error(Arg.Is("Failed to update timeout job, Id: 'job-id-1' with status code 400"));
}
public void CheckAndProcessTimedOutJobs_GivenNonCompletedJobsFoundButNoDefinitionsReturned_LogsAndThrowsException()
{
//Arrange
IEnumerable <Job> nonCompletedJobs = new[]
{
new Job {
Id = "job-id-1"
},
new Job {
Id = "job-id-2"
}
};
IJobRepository jobRepository = CreateJobRepository();
jobRepository
.GetNonCompletedJobs()
.Returns(nonCompletedJobs);
IJobDefinitionsService jobDefinitionsService = CreateJobDefinitionsService();
jobDefinitionsService
.GetAllJobDefinitions()
.Returns(Enumerable.Empty <JobDefinition>());
ILogger logger = CreateLogger();
JobManagementService jobManagementService = CreateJobManagementService(jobRepository, logger: logger, jobDefinitionsService: jobDefinitionsService);
//Act
Func <Task> test = async() => await jobManagementService.CheckAndProcessTimedOutJobs();
//Assert
test
.Should()
.ThrowExactly <Exception>()
.Which
.Message
.Should()
.Be("Failed to retrieve job definitions when processing timed out jobs");
logger
.Received(1)
.Error(Arg.Is("Failed to retrieve job definitions when processing timed out jobs"));
logger
.Received(1)
.Information($"{nonCompletedJobs.Count()} non completed jobs to process");
}
public async Task CheckAndProcessTimedOutJobs_GivenNonCompletedJobsFoundButHasNotTimedOut_DoesNotUpdateJob()
{
//Arrange
IEnumerable <Job> nonCompletedJobs = new[]
{
new Job {
Id = "job-id-1", JobDefinitionId = "job-def-2", Created = DateTimeOffset.Now.AddHours(-6)
},
};
IJobRepository jobRepository = CreateJobRepository();
jobRepository
.GetNonCompletedJobs()
.Returns(nonCompletedJobs);
IEnumerable <JobDefinition> jobDefinitions = new[]
{
new JobDefinition
{
Id = "job-def-1",
Timeout = TimeSpan.FromHours(12)
},
new JobDefinition
{
Id = "job-def-2",
Timeout = TimeSpan.FromHours(12)
}
};
IJobDefinitionsService jobDefinitionsService = CreateJobDefinitionsService();
jobDefinitionsService
.GetAllJobDefinitions()
.Returns(jobDefinitions);
ILogger logger = CreateLogger();
JobManagementService jobManagementService = CreateJobManagementService(jobRepository, logger: logger, jobDefinitionsService: jobDefinitionsService);
//Act
await jobManagementService.CheckAndProcessTimedOutJobs();
//Assert
await
jobRepository
.DidNotReceive()
.UpdateJob(Arg.Any <Job>());
}
public async Task CheckAndProcessTimedOutJobs_GivenNonCompletedJobsFoundButCouldNotFindJobDefinition_LogsError()
{
//Arrange
IEnumerable <Job> nonCompletedJobs = new[]
{
new Job {
Id = "job-id-1", JobDefinitionId = "job-def-3"
},
};
IJobRepository jobRepository = CreateJobRepository();
jobRepository
.GetNonCompletedJobs()
.Returns(nonCompletedJobs);
IEnumerable <JobDefinition> jobDefinitions = new[]
{
new JobDefinition
{
Id = "job-def-1",
Timeout = TimeSpan.FromHours(12)
},
new JobDefinition
{
Id = "job-def-2",
Timeout = TimeSpan.FromHours(12)
}
};
IJobDefinitionsService jobDefinitionsService = CreateJobDefinitionsService();
jobDefinitionsService
.GetAllJobDefinitions()
.Returns(jobDefinitions);
ILogger logger = CreateLogger();
JobManagementService jobManagementService = CreateJobManagementService(jobRepository, logger: logger, jobDefinitionsService: jobDefinitionsService);
//Act
await jobManagementService.CheckAndProcessTimedOutJobs();
//Assert
logger
.Received(1)
.Error($"Failed to find job definition : 'job-def-3' for job id: 'job-id-1'");
}
public JobManagementService CreateJobManagementService(
IJobRepository jobRepository = null,
INotificationService notificationService = null,
IJobDefinitionsService jobDefinitionsService = null,
IJobsResiliencePolicies resiliencePolicies = null,
ILogger logger = null,
IValidator <CreateJobValidationModel> createJobValidator = null,
IMessengerService messengerService = null)
{
return(new JobManagementService(
jobRepository ?? CreateJobRepository(),
notificationService ?? CreateNotificationsService(),
jobDefinitionsService ?? CreateJobDefinitionsService(),
resiliencePolicies ?? CreateResiliencePolicies(),
logger ?? CreateLogger(),
createJobValidator ?? CreateNewCreateJobValidator(),
messengerService ?? CreateMessengerService()
));
}
public async Task ProcessJobCompletion_JobHasParentWithMultipleCompletedChildrenWithAllSucceededAndPreCompletionJobsNotRunning_ThenPreCompletionJobQueued()
{
// Arrange
string parentJobId = "parent123";
string jobId = "child123";
string preCompletionJobDefinition = "PreCompletionJobDefinitionId";
Job job = new Job {
Id = jobId, ParentJobId = parentJobId, CompletionStatus = CompletionStatus.Succeeded, RunningStatus = RunningStatus.Completed
};
Job job2 = new Job {
Id = "child456", ParentJobId = parentJobId, RunningStatus = RunningStatus.Completed, CompletionStatus = CompletionStatus.Succeeded
};
Job job3 = new Job {
Id = "child789", ParentJobId = parentJobId, RunningStatus = RunningStatus.Completed, CompletionStatus = CompletionStatus.Succeeded
};
Job preCompletionJob = new Job
{
Id = "preCompletionJob",
ParentJobId = parentJobId,
JobDefinitionId = preCompletionJobDefinition,
RunningStatus = RunningStatus.InProgress
};
Job parentJob = new Job
{
Id = parentJobId,
RunningStatus = RunningStatus.InProgress,
JobDefinitionId = jobDefinitionId,
Trigger = new Trigger {
}
};
ILogger logger = CreateLogger();
IJobRepository jobRepository = CreateJobRepository();
jobRepository
.GetJobById(Arg.Is(jobId))
.Returns(job);
jobRepository
.GetJobById(Arg.Is(parentJobId))
.Returns(parentJob);
jobRepository
.GetChildJobsForParent(Arg.Is(parentJobId))
.Returns(new List <Job> {
job, job2
});
jobRepository
.CreateJob(Arg.Any <Job>())
.Returns(preCompletionJob);
IJobDefinitionsService jobDefinitionsService = CreateJobDefinitionsService();
jobDefinitionsService
.GetAllJobDefinitions()
.Returns(new[] { new JobDefinition {
Id = jobDefinitionId, PreCompletionJobs = new[] { preCompletionJobDefinition }
},
new JobDefinition {
Id = preCompletionJobDefinition
} });
JobManagementService jobManagementService = CreateJobManagementService(jobRepository,
logger: logger,
jobDefinitionsService: jobDefinitionsService);
JobSummary JobSummary = new JobSummary {
JobId = jobId, RunningStatus = RunningStatus.Completed
};
string json = JsonConvert.SerializeObject(JobSummary);
Message message = new Message(Encoding.UTF8.GetBytes(json));
message.UserProperties["jobId"] = jobId;
// Act
await jobManagementService.Process(message);
// Assert
await jobRepository
.Received(1)
.UpdateJob(Arg.Is <Job>(j => j.Id == parentJobId && j.RunningStatus == RunningStatus.Completing));
}
public async Task CheckAndProcessTimedOutJobs_GivenTwoNonCompletedJobsAndHasTimeOutButOnlyoneUpdatesSuccessfully_SendsOneNotification()
{
//Arrange
string outcome = "outcome-1";
List <Outcome> outcomes = new List <Outcome>
{
new Outcome
{
Description = outcome
}
};
IEnumerable <Job> nonCompletedJobs = new[]
{
new Job
{
Id = "job-id-1",
JobDefinitionId = "job-def-2",
Created = DateTimeOffset.Now.AddHours(-13),
Outcomes = outcomes,
Outcome = outcome
},
new Job
{
Id = "job-id-2",
JobDefinitionId = "job-def-1",
Created = DateTimeOffset.Now.AddHours(-13)
},
};
IJobRepository jobRepository = CreateJobRepository();
jobRepository
.GetNonCompletedJobs()
.Returns(nonCompletedJobs);
jobRepository
.GetLatestJobBySpecificationIdAndDefinitionId(Arg.Any <string>(), Arg.Any <string>())
.Returns(nonCompletedJobs.First());
jobRepository
.UpdateJob(Arg.Any <Job>())
.Returns(HttpStatusCode.OK, HttpStatusCode.BadRequest);
IEnumerable <JobDefinition> jobDefinitions = new[]
{
new JobDefinition
{
Id = "job-def-1",
Timeout = TimeSpan.FromHours(12)
},
new JobDefinition
{
Id = "job-def-2",
Timeout = TimeSpan.FromHours(12)
}
};
IJobDefinitionsService jobDefinitionsService = CreateJobDefinitionsService();
jobDefinitionsService
.GetAllJobDefinitions()
.Returns(jobDefinitions);
INotificationService notificationService = CreateNotificationsService();
ILogger logger = CreateLogger();
JobManagementService jobManagementService = CreateJobManagementService(jobRepository, logger: logger,
jobDefinitionsService: jobDefinitionsService, notificationService: notificationService);
//Act
await jobManagementService.CheckAndProcessTimedOutJobs();
//Assert
await
jobRepository
.Received(1)
.UpdateJob(Arg.Is <Job>(
m => m.Id == "job-id-1" &&
m.CompletionStatus == CompletionStatus.TimedOut &&
m.RunningStatus == RunningStatus.Completed &&
m.Outcomes.SequenceEqual(outcomes) &&
m.Outcome == outcome &&
m.OutcomeType == OutcomeType.Inconclusive
));
await
notificationService
.Received(1)
.SendNotification(Arg.Is <JobSummary>(
m => m.JobId == "job-id-1" &&
m.CompletionStatus == CompletionStatus.TimedOut &&
m.RunningStatus == RunningStatus.Completed
));
logger
.Received(1)
.Error(Arg.Is("Failed to update timeout job, Id: 'job-id-2' with status code 400"));
}
public async Task CheckAndProcessTimedOutJobs_GivenNonCompletedJobsAndHasTimeOutAndUpdatesSuccessfully_SendsNotification()
{
//Arrange
IEnumerable <Job> nonCompletedJobs = new[]
{
new Job {
Id = "job-id-1", JobDefinitionId = "job-def-2", Created = DateTimeOffset.Now.AddHours(-13)
},
};
IJobRepository jobRepository = CreateJobRepository();
jobRepository
.GetNonCompletedJobs()
.Returns(nonCompletedJobs);
jobRepository
.UpdateJob(Arg.Any <Job>())
.Returns(HttpStatusCode.OK);
IEnumerable <JobDefinition> jobDefinitions = new[]
{
new JobDefinition
{
Id = "job-def-1",
Timeout = TimeSpan.FromHours(12)
},
new JobDefinition
{
Id = "job-def-2",
Timeout = TimeSpan.FromHours(12)
}
};
IJobDefinitionsService jobDefinitionsService = CreateJobDefinitionsService();
jobDefinitionsService
.GetAllJobDefinitions()
.Returns(jobDefinitions);
INotificationService notificationService = CreateNotificationsService();
ILogger logger = CreateLogger();
JobManagementService jobManagementService = CreateJobManagementService(jobRepository, logger: logger,
jobDefinitionsService: jobDefinitionsService, notificationService: notificationService);
//Act
await jobManagementService.CheckAndProcessTimedOutJobs();
//Assert
await
jobRepository
.Received(1)
.UpdateJob(Arg.Is <Job>(
m => m.Id == "job-id-1" &&
m.CompletionStatus == CompletionStatus.TimedOut &&
m.RunningStatus == RunningStatus.Completed &&
m.OutcomeType == OutcomeType.Inconclusive
));
await
notificationService
.Received(1)
.SendNotification(Arg.Is <JobSummary>(
m => m.JobId == "job-id-1" &&
m.CompletionStatus == CompletionStatus.TimedOut &&
m.RunningStatus == RunningStatus.Completed
));
}
public async Task CheckAndProcessTimedOutJobs_GivenTwoNonCompletedJobsAndHasTimeOutButOnlyoneUpdatesSuccessfully_SendsOneNotification()
{
//Arrange
IEnumerable <Job> nonCompletedJobs = new[]
{
new Job {
Id = "job-id-1", JobDefinitionId = "job-def-2", Created = DateTimeOffset.Now.AddHours(-13)
},
new Job {
Id = "job-id-2", JobDefinitionId = "job-def-1", Created = DateTimeOffset.Now.AddHours(-13)
},
};
IJobRepository jobRepository = CreateJobRepository();
jobRepository
.GetNonCompletedJobs()
.Returns(nonCompletedJobs);
jobRepository
.UpdateJob(Arg.Any <Job>())
.Returns(HttpStatusCode.OK, HttpStatusCode.BadRequest);
IEnumerable <JobDefinition> jobDefinitions = new[]
{
new JobDefinition
{
Id = "job-def-1",
Timeout = TimeSpan.FromHours(12)
},
new JobDefinition
{
Id = "job-def-2",
Timeout = TimeSpan.FromHours(12)
}
};
IJobDefinitionsService jobDefinitionsService = CreateJobDefinitionsService();
jobDefinitionsService
.GetAllJobDefinitions()
.Returns(jobDefinitions);
INotificationService notificationService = CreateNotificationsService();
ILogger logger = CreateLogger();
JobManagementService jobManagementService = CreateJobManagementService(jobRepository, logger: logger,
jobDefinitionsService: jobDefinitionsService, notificationService: notificationService);
//Act
await jobManagementService.CheckAndProcessTimedOutJobs();
//Assert
await
jobRepository
.Received(1)
.UpdateJob(Arg.Is <Job>(
m => m.Id == "job-id-1" &&
m.Completed.Value.Date == DateTimeOffset.Now.Date &&
m.CompletionStatus == CompletionStatus.TimedOut &&
m.RunningStatus == RunningStatus.Completed
));
await
notificationService
.Received(1)
.SendNotification(Arg.Is <JobNotification>(
m => m.JobId == "job-id-1" &&
m.CompletionStatus == CompletionStatus.TimedOut &&
m.RunningStatus == RunningStatus.Completed &&
m.StatusDateTime.Date == DateTimeOffset.Now.Date
));
logger
.Received(1)
.Error(Arg.Is("Failed to update timeout job, Id: 'job-id-2' with status code 400"));
}
