public async Task TransitionState_Fail_ShallPutJobInFaultedState()
{
var job = new InferenceJob();
job.JobId = Guid.NewGuid().ToString();
job.PayloadId = Guid.NewGuid().ToString();
job.SetStoragePath("/path/to/job");
job.State = InferenceJobState.Creating;
job.TryCount = 3;
var cancellationSource = new CancellationTokenSource();
_inferenceJobRepository.SetupSequence(p => p.AsQueryable())
.Returns((new List <InferenceJob>()
{
job
}).AsQueryable());
_inferenceJobRepository.Setup(p => p.SaveChangesAsync(It.IsAny <CancellationToken>()));
var jobStore = new ClaraJobRepository(
_logger.Object,
_configuration,
_fileSystem,
_inferenceJobRepository.Object);
var result = await jobStore.TransitionState(job, InferenceJobStatus.Fail, cancellationSource.Token);
Assert.Equal(job, result);
Assert.Equal(InferenceJobState.Faulted, result.State);
Assert.Equal(4, result.TryCount);
_logger.VerifyLoggingMessageBeginsWith($"Job {job.JobId} exceeded maximum number of retries.", LogLevel.Warning, Times.Once());
_inferenceJobRepository.Verify(p => p.SaveChangesAsync(cancellationSource.Token), Times.Once());
}
public async Task TransitionState_Fail_ShallTransitionJob(InferenceJobState initalState, InferenceJobState endingState)
{
var job = new InferenceJob();
job.JobId = Guid.NewGuid().ToString();
job.PayloadId = Guid.NewGuid().ToString();
job.SetStoragePath("/path/to/job");
job.State = initalState;
job.TryCount = 1;
var cancellationSource = new CancellationTokenSource();
_inferenceJobRepository.SetupSequence(p => p.AsQueryable())
.Returns((new List <InferenceJob>()
{
job
}).AsQueryable());
_inferenceJobRepository.Setup(p => p.SaveChangesAsync(It.IsAny <CancellationToken>()));
var jobStore = new ClaraJobRepository(
_logger.Object,
_configuration,
_fileSystem,
_inferenceJobRepository.Object);
var result = await jobStore.TransitionState(job, InferenceJobStatus.Fail, cancellationSource.Token);
Assert.Equal(job, result);
Assert.Equal(endingState, endingState);
Assert.Equal(2, result.TryCount);
_logger.VerifyLoggingMessageBeginsWith($"Putting inference job {job.JobId} back to {endingState} state for retry.", LogLevel.Information, Times.Once());
_inferenceJobRepository.Verify(p => p.SaveChangesAsync(cancellationSource.Token), Times.Once());
}
public async Task Take_ShallReturnAJob(InferenceJobState initalState, InferenceJobState endingState)
{
var job = new InferenceJob();
job.JobId = Guid.NewGuid().ToString();
job.PayloadId = Guid.NewGuid().ToString();
job.SetStoragePath("/path/to/job");
job.State = initalState;
var cancellationSource = new CancellationTokenSource();
_inferenceJobRepository.SetupSequence(p => p.AsQueryable())
.Returns((new List <InferenceJob>()
{
job
}).AsQueryable());
var jobStore = new ClaraJobRepository(
_logger.Object,
_configuration,
_fileSystem,
_inferenceJobRepository.Object);
var result = await jobStore.Take(cancellationSource.Token);
Assert.Equal(job, result);
Assert.Equal(endingState, job.State);
_logger.VerifyLoggingMessageBeginsWith($"Updating inference job {job.JobId} from {initalState } to {endingState}.", LogLevel.Information, Times.Once());
}
public async Task Take_ShallThrowWhenCancelled()
{
var cancellationSource = new CancellationTokenSource();
_inferenceJobRepository.Setup(p => p.AsQueryable())
.Returns((new List <InferenceJob>()).AsQueryable());
var jobStore = new ClaraJobRepository(
_logger.Object,
_configuration,
_fileSystem,
_inferenceJobRepository.Object);
cancellationSource.CancelAfter(100);
await Assert.ThrowsAsync <OperationCanceledException>(async() => await jobStore.Take(cancellationSource.Token));
}
public async Task Add_ShallAddItem()
{
var job = new InferenceJob();
job.JobId = Guid.NewGuid().ToString();
job.PayloadId = Guid.NewGuid().ToString();
job.Instances.Add(InstanceGenerator.GenerateInstance("./aet", "aet", fileSystem: _fileSystem));
var jobStore = new ClaraJobRepository(
_logger.Object,
_configuration,
_fileSystem,
_inferenceJobRepository.Object);
await jobStore.Add(job);
_inferenceJobRepository.Verify(p => p.AddAsync(It.IsAny <InferenceJob>(), It.IsAny <CancellationToken>()), Times.Once());
_inferenceJobRepository.Verify(p => p.SaveChangesAsync(It.IsAny <CancellationToken>()), Times.Once());
}
public async Task Add_ShallRetryOnFailure()
{
var job = new InferenceJob();
job.JobId = Guid.NewGuid().ToString();
job.PayloadId = Guid.NewGuid().ToString();
job.Instances.Add(InstanceGenerator.GenerateInstance("./aet", "aet", fileSystem: _fileSystem));
var jobStore = new ClaraJobRepository(
_logger.Object,
_configuration,
_fileSystem,
_inferenceJobRepository.Object);
_inferenceJobRepository.Setup(p => p.AddAsync(It.IsAny <InferenceJob>(), It.IsAny <CancellationToken>())).Throws(new Exception("error"));
await Assert.ThrowsAsync <Exception>(async() => await jobStore.Add(job));
_logger.VerifyLoggingMessageBeginsWith($"Error saving inference job.", LogLevel.Error, Times.Exactly(3));
}
public async Task Add_ShallRetryCopyThenThrow()
{
var fileSystem = new Mock <IFileSystem>();
fileSystem.Setup(p => p.Directory).Returns(_fileSystem.Directory);
fileSystem.Setup(p => p.Path).Returns(_fileSystem.Path);
fileSystem.Setup(p => p.File.Create(It.IsAny <string>()))
.Returns((string path) => _fileSystem.File.Create(path));
fileSystem.Setup(p => p.File.Copy(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <bool>()))
.Throws(new IOException("error", ClaraJobRepository.ERROR_DISK_FULL));
var job = new InferenceJob();
job.JobId = Guid.NewGuid().ToString();
job.PayloadId = Guid.NewGuid().ToString();
job.SetStoragePath("/path/to/job");
job.Instances.Add(InstanceGenerator.GenerateInstance("./aet", "aet", fileSystem: fileSystem.Object));
_configuration.Value.Storage.Temporary = "./aet";
var cancellationSource = new CancellationTokenSource();
_inferenceJobRepository.SetupSequence(p => p.AsQueryable())
.Returns((new List <InferenceJob>()
{
job
}).AsQueryable());
var jobStore = new ClaraJobRepository(
_logger.Object,
_configuration,
fileSystem.Object,
_inferenceJobRepository.Object);
await Assert.ThrowsAsync <IOException>(async() => await jobStore.Add(job));
_logger.VerifyLoggingMessageBeginsWith($"Error copying file to {job.JobPayloadsStoragePath}; destination may be out of disk space, will retry in {1000}ms.", LogLevel.Error, Times.Exactly(3));
_logger.VerifyLoggingMessageBeginsWith($"Error copying file to {job.JobPayloadsStoragePath}; destination may be out of disk space. Exceeded maximum retries.", LogLevel.Error, Times.Once());
}
public async Task Add_ThrowsWhenFailToCopy()
{
var fileSystem = new Mock <IFileSystem>();
fileSystem.Setup(p => p.Directory).Returns(_fileSystem.Directory);
fileSystem.Setup(p => p.Path).Returns(_fileSystem.Path);
fileSystem.Setup(p => p.File.Create(It.IsAny <string>()))
.Returns((string path) => _fileSystem.File.Create(path));
fileSystem.Setup(p => p.File.Copy(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <bool>())).Throws(new Exception("error"));
var job = new InferenceJob();
job.JobId = Guid.NewGuid().ToString();
job.PayloadId = Guid.NewGuid().ToString();
job.SetStoragePath("/path/to/job");
job.Instances.Add(InstanceGenerator.GenerateInstance("./aet", "aet", fileSystem: fileSystem.Object));
_configuration.Value.Storage.Temporary = "./aet";
var cancellationSource = new CancellationTokenSource();
_inferenceJobRepository.SetupSequence(p => p.AsQueryable())
.Returns((new List <InferenceJob>()
{
job
}).AsQueryable());
var jobStore = new ClaraJobRepository(
_logger.Object,
_configuration,
fileSystem.Object,
_inferenceJobRepository.Object);
await Assert.ThrowsAsync <Exception>(async() => await jobStore.Add(job));
_logger.VerifyLoggingMessageBeginsWith($"Failed to copy file {job.JobPayloadsStoragePath}.", LogLevel.Error, Times.Once());
}
