public async Task TransitionState_Fail_ShallPutJobInFaultedState() { var job = new InferenceJob(); job.JobId = Guid.NewGuid().ToString(); job.PayloadId = Guid.NewGuid().ToString(); job.SetStoragePath("/path/to/job"); job.State = InferenceJobState.Creating; job.TryCount = 3; var cancellationSource = new CancellationTokenSource(); _inferenceJobRepository.SetupSequence(p => p.AsQueryable()) .Returns((new List <InferenceJob>() { job }).AsQueryable()); _inferenceJobRepository.Setup(p => p.SaveChangesAsync(It.IsAny <CancellationToken>())); var jobStore = new ClaraJobRepository( _logger.Object, _configuration, _fileSystem, _inferenceJobRepository.Object); var result = await jobStore.TransitionState(job, InferenceJobStatus.Fail, cancellationSource.Token); Assert.Equal(job, result); Assert.Equal(InferenceJobState.Faulted, result.State); Assert.Equal(4, result.TryCount); _logger.VerifyLoggingMessageBeginsWith($"Job {job.JobId} exceeded maximum number of retries.", LogLevel.Warning, Times.Once()); _inferenceJobRepository.Verify(p => p.SaveChangesAsync(cancellationSource.Token), Times.Once()); }
public async Task TransitionState_Fail_ShallTransitionJob(InferenceJobState initalState, InferenceJobState endingState) { var job = new InferenceJob(); job.JobId = Guid.NewGuid().ToString(); job.PayloadId = Guid.NewGuid().ToString(); job.SetStoragePath("/path/to/job"); job.State = initalState; job.TryCount = 1; var cancellationSource = new CancellationTokenSource(); _inferenceJobRepository.SetupSequence(p => p.AsQueryable()) .Returns((new List <InferenceJob>() { job }).AsQueryable()); _inferenceJobRepository.Setup(p => p.SaveChangesAsync(It.IsAny <CancellationToken>())); var jobStore = new ClaraJobRepository( _logger.Object, _configuration, _fileSystem, _inferenceJobRepository.Object); var result = await jobStore.TransitionState(job, InferenceJobStatus.Fail, cancellationSource.Token); Assert.Equal(job, result); Assert.Equal(endingState, endingState); Assert.Equal(2, result.TryCount); _logger.VerifyLoggingMessageBeginsWith($"Putting inference job {job.JobId} back to {endingState} state for retry.", LogLevel.Information, Times.Once()); _inferenceJobRepository.Verify(p => p.SaveChangesAsync(cancellationSource.Token), Times.Once()); }
public async Task Take_ShallReturnAJob(InferenceJobState initalState, InferenceJobState endingState) { var job = new InferenceJob(); job.JobId = Guid.NewGuid().ToString(); job.PayloadId = Guid.NewGuid().ToString(); job.SetStoragePath("/path/to/job"); job.State = initalState; var cancellationSource = new CancellationTokenSource(); _inferenceJobRepository.SetupSequence(p => p.AsQueryable()) .Returns((new List <InferenceJob>() { job }).AsQueryable()); var jobStore = new ClaraJobRepository( _logger.Object, _configuration, _fileSystem, _inferenceJobRepository.Object); var result = await jobStore.Take(cancellationSource.Token); Assert.Equal(job, result); Assert.Equal(endingState, job.State); _logger.VerifyLoggingMessageBeginsWith($"Updating inference job {job.JobId} from {initalState } to {endingState}.", LogLevel.Information, Times.Once()); }
public async Task Take_ShallThrowWhenCancelled() { var cancellationSource = new CancellationTokenSource(); _inferenceJobRepository.Setup(p => p.AsQueryable()) .Returns((new List <InferenceJob>()).AsQueryable()); var jobStore = new ClaraJobRepository( _logger.Object, _configuration, _fileSystem, _inferenceJobRepository.Object); cancellationSource.CancelAfter(100); await Assert.ThrowsAsync <OperationCanceledException>(async() => await jobStore.Take(cancellationSource.Token)); }
public async Task Add_ShallAddItem() { var job = new InferenceJob(); job.JobId = Guid.NewGuid().ToString(); job.PayloadId = Guid.NewGuid().ToString(); job.Instances.Add(InstanceGenerator.GenerateInstance("./aet", "aet", fileSystem: _fileSystem)); var jobStore = new ClaraJobRepository( _logger.Object, _configuration, _fileSystem, _inferenceJobRepository.Object); await jobStore.Add(job); _inferenceJobRepository.Verify(p => p.AddAsync(It.IsAny <InferenceJob>(), It.IsAny <CancellationToken>()), Times.Once()); _inferenceJobRepository.Verify(p => p.SaveChangesAsync(It.IsAny <CancellationToken>()), Times.Once()); }
public async Task Add_ShallRetryOnFailure() { var job = new InferenceJob(); job.JobId = Guid.NewGuid().ToString(); job.PayloadId = Guid.NewGuid().ToString(); job.Instances.Add(InstanceGenerator.GenerateInstance("./aet", "aet", fileSystem: _fileSystem)); var jobStore = new ClaraJobRepository( _logger.Object, _configuration, _fileSystem, _inferenceJobRepository.Object); _inferenceJobRepository.Setup(p => p.AddAsync(It.IsAny <InferenceJob>(), It.IsAny <CancellationToken>())).Throws(new Exception("error")); await Assert.ThrowsAsync <Exception>(async() => await jobStore.Add(job)); _logger.VerifyLoggingMessageBeginsWith($"Error saving inference job.", LogLevel.Error, Times.Exactly(3)); }
public async Task Add_ShallRetryCopyThenThrow() { var fileSystem = new Mock <IFileSystem>(); fileSystem.Setup(p => p.Directory).Returns(_fileSystem.Directory); fileSystem.Setup(p => p.Path).Returns(_fileSystem.Path); fileSystem.Setup(p => p.File.Create(It.IsAny <string>())) .Returns((string path) => _fileSystem.File.Create(path)); fileSystem.Setup(p => p.File.Copy(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <bool>())) .Throws(new IOException("error", ClaraJobRepository.ERROR_DISK_FULL)); var job = new InferenceJob(); job.JobId = Guid.NewGuid().ToString(); job.PayloadId = Guid.NewGuid().ToString(); job.SetStoragePath("/path/to/job"); job.Instances.Add(InstanceGenerator.GenerateInstance("./aet", "aet", fileSystem: fileSystem.Object)); _configuration.Value.Storage.Temporary = "./aet"; var cancellationSource = new CancellationTokenSource(); _inferenceJobRepository.SetupSequence(p => p.AsQueryable()) .Returns((new List <InferenceJob>() { job }).AsQueryable()); var jobStore = new ClaraJobRepository( _logger.Object, _configuration, fileSystem.Object, _inferenceJobRepository.Object); await Assert.ThrowsAsync <IOException>(async() => await jobStore.Add(job)); _logger.VerifyLoggingMessageBeginsWith($"Error copying file to {job.JobPayloadsStoragePath}; destination may be out of disk space, will retry in {1000}ms.", LogLevel.Error, Times.Exactly(3)); _logger.VerifyLoggingMessageBeginsWith($"Error copying file to {job.JobPayloadsStoragePath}; destination may be out of disk space. Exceeded maximum retries.", LogLevel.Error, Times.Once()); }
public async Task Add_ThrowsWhenFailToCopy() { var fileSystem = new Mock <IFileSystem>(); fileSystem.Setup(p => p.Directory).Returns(_fileSystem.Directory); fileSystem.Setup(p => p.Path).Returns(_fileSystem.Path); fileSystem.Setup(p => p.File.Create(It.IsAny <string>())) .Returns((string path) => _fileSystem.File.Create(path)); fileSystem.Setup(p => p.File.Copy(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <bool>())).Throws(new Exception("error")); var job = new InferenceJob(); job.JobId = Guid.NewGuid().ToString(); job.PayloadId = Guid.NewGuid().ToString(); job.SetStoragePath("/path/to/job"); job.Instances.Add(InstanceGenerator.GenerateInstance("./aet", "aet", fileSystem: fileSystem.Object)); _configuration.Value.Storage.Temporary = "./aet"; var cancellationSource = new CancellationTokenSource(); _inferenceJobRepository.SetupSequence(p => p.AsQueryable()) .Returns((new List <InferenceJob>() { job }).AsQueryable()); var jobStore = new ClaraJobRepository( _logger.Object, _configuration, fileSystem.Object, _inferenceJobRepository.Object); await Assert.ThrowsAsync <Exception>(async() => await jobStore.Add(job)); _logger.VerifyLoggingMessageBeginsWith($"Failed to copy file {job.JobPayloadsStoragePath}.", LogLevel.Error, Times.Once()); }