public async Task Add_ShallRetryOnFailure()
{
_kubernetesClient
.Setup(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>()))
.Throws(new HttpOperationException("error message")
{
Response = new HttpResponseMessageWrapper(new HttpResponseMessage(HttpStatusCode.Conflict), "error content")
});
var job = new Job();
job.JobId = Guid.NewGuid().ToString();
job.PayloadId = Guid.NewGuid().ToString();
var jobStore = new JobStore(
_loggerFactory.Object,
_configuration,
_kubernetesClient.Object,
_fileSystem);
var instance = InstanceGenerator.GenerateInstance("./aet", "aet", fileSystem: _fileSystem);
await Assert.ThrowsAsync <HttpOperationException>(async() => await jobStore.Add(job, "job-name", new List <InstanceStorageInfo> {
instance
}));
_logger.VerifyLoggingMessageBeginsWith($"Failed to add new job {job.JobId} in CRD", LogLevel.Warning, Times.Exactly(3));
_kubernetesClient.Verify(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>()), Times.Exactly(4));
}
public async Task Add_ShallAddItemToCrd()
{
_kubernetesClient
.Setup(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>()))
.Returns(Task.FromResult(new HttpOperationResponse <object>
{
Response = new HttpResponseMessage()
}));
var job = new Job();
job.JobId = Guid.NewGuid().ToString();
job.PayloadId = Guid.NewGuid().ToString();
var jobStore = new JobStore(
_loggerFactory.Object,
_configuration,
_kubernetesClient.Object,
_fileSystem);
var instance = InstanceGenerator.GenerateInstance("./aet", "aet", fileSystem: _fileSystem);
await jobStore.Add(job, "job-name", new List <InstanceStorageInfo> {
instance
});
_logger.VerifyLoggingMessageBeginsWith($"Failed to add save new job {job.JobId} in CRD", LogLevel.Warning, Times.Never());
_kubernetesClient.Verify(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>()), Times.Once());
}
public async Task Add_ShallAddItem()
{
var job = new InferenceJob();
job.JobId = Guid.NewGuid().ToString();
job.PayloadId = Guid.NewGuid().ToString();
job.Instances.Add(InstanceGenerator.GenerateInstance("./aet", "aet", fileSystem: _fileSystem));
var jobStore = new ClaraJobRepository(
_logger.Object,
_configuration,
_fileSystem,
_inferenceJobRepository.Object);
await jobStore.Add(job);
_inferenceJobRepository.Verify(p => p.AddAsync(It.IsAny <InferenceJob>(), It.IsAny <CancellationToken>()), Times.Once());
_inferenceJobRepository.Verify(p => p.SaveChangesAsync(It.IsAny <CancellationToken>()), Times.Once());
}
public async Task Add_ShallRetryOnFailure()
{
var job = new InferenceJob();
job.JobId = Guid.NewGuid().ToString();
job.PayloadId = Guid.NewGuid().ToString();
job.Instances.Add(InstanceGenerator.GenerateInstance("./aet", "aet", fileSystem: _fileSystem));
var jobStore = new ClaraJobRepository(
_logger.Object,
_configuration,
_fileSystem,
_inferenceJobRepository.Object);
_inferenceJobRepository.Setup(p => p.AddAsync(It.IsAny <InferenceJob>(), It.IsAny <CancellationToken>())).Throws(new Exception("error"));
await Assert.ThrowsAsync <Exception>(async() => await jobStore.Add(job));
_logger.VerifyLoggingMessageBeginsWith($"Error saving inference job.", LogLevel.Error, Times.Exactly(3));
}
public void WorkflowTest()
{
var service = new InstanceStoredNotificationService(_logger.Object, _cleanupQueue.Object);
var observer = new Mock <IObserver <InstanceStorageInfo> >();
observer.Setup(p => p.OnNext(It.IsAny <InstanceStorageInfo>()));
var cancel = service.Subscribe(observer.Object);
var instance = InstanceGenerator.GenerateInstance("/storage", "AET");
service.NewInstanceStored(instance);
service.NewInstanceStored(instance);
service.NewInstanceStored(instance);
observer.Verify(p => p.OnNext(It.IsAny <InstanceStorageInfo>()), Times.Exactly(3));
cancel.Dispose();
observer.Reset();
service.NewInstanceStored(instance);
observer.Verify(p => p.OnNext(It.IsAny <InstanceStorageInfo>()), Times.Never());
}
public async Task Add_ShallRetryCopyThenThrow()
{
var fileSystem = new Mock <IFileSystem>();
fileSystem.Setup(p => p.Directory).Returns(_fileSystem.Directory);
fileSystem.Setup(p => p.Path).Returns(_fileSystem.Path);
fileSystem.Setup(p => p.File.Create(It.IsAny <string>()))
.Returns((string path) => _fileSystem.File.Create(path));
fileSystem.Setup(p => p.File.Copy(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <bool>()))
.Throws(new IOException("error", ClaraJobRepository.ERROR_DISK_FULL));
var job = new InferenceJob();
job.JobId = Guid.NewGuid().ToString();
job.PayloadId = Guid.NewGuid().ToString();
job.SetStoragePath("/path/to/job");
job.Instances.Add(InstanceGenerator.GenerateInstance("./aet", "aet", fileSystem: fileSystem.Object));
_configuration.Value.Storage.Temporary = "./aet";
var cancellationSource = new CancellationTokenSource();
_inferenceJobRepository.SetupSequence(p => p.AsQueryable())
.Returns((new List <InferenceJob>()
{
job
}).AsQueryable());
var jobStore = new ClaraJobRepository(
_logger.Object,
_configuration,
fileSystem.Object,
_inferenceJobRepository.Object);
await Assert.ThrowsAsync <IOException>(async() => await jobStore.Add(job));
_logger.VerifyLoggingMessageBeginsWith($"Error copying file to {job.JobPayloadsStoragePath}; destination may be out of disk space, will retry in {1000}ms.", LogLevel.Error, Times.Exactly(3));
_logger.VerifyLoggingMessageBeginsWith($"Error copying file to {job.JobPayloadsStoragePath}; destination may be out of disk space. Exceeded maximum retries.", LogLevel.Error, Times.Once());
}
public void NewInstanceStored_NoSupportedObservers()
{
var instance = InstanceGenerator.GenerateInstance("/storage", "AET");
_cleanupQueue.Setup(p => p.QueueInstance(It.IsAny <string>()));
var service = new InstanceStoredNotificationService(_logger.Object, _cleanupQueue.Object);
var observer = new Mock <IObserver <InstanceStorageInfo> >();
observer.Setup(p => p.OnNext(It.IsAny <InstanceStorageInfo>())).Throws(new InstanceNotSupportedException(instance));
var cancel = service.Subscribe(observer.Object);
service.NewInstanceStored(instance);
observer.Verify(p => p.OnNext(It.IsAny <InstanceStorageInfo>()), Times.Once());
_cleanupQueue.Verify(p => p.QueueInstance(instance.InstanceStorageFullPath), Times.Once());
cancel.Dispose();
observer.Reset();
service.NewInstanceStored(instance);
observer.Verify(p => p.OnNext(It.IsAny <InstanceStorageInfo>()), Times.Never());
}
public async Task Add_ThrowsWhenFailToCopy()
{
var fileSystem = new Mock <IFileSystem>();
fileSystem.Setup(p => p.Directory).Returns(_fileSystem.Directory);
fileSystem.Setup(p => p.Path).Returns(_fileSystem.Path);
fileSystem.Setup(p => p.File.Create(It.IsAny <string>()))
.Returns((string path) => _fileSystem.File.Create(path));
fileSystem.Setup(p => p.File.Copy(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <bool>())).Throws(new Exception("error"));
var job = new InferenceJob();
job.JobId = Guid.NewGuid().ToString();
job.PayloadId = Guid.NewGuid().ToString();
job.SetStoragePath("/path/to/job");
job.Instances.Add(InstanceGenerator.GenerateInstance("./aet", "aet", fileSystem: fileSystem.Object));
_configuration.Value.Storage.Temporary = "./aet";
var cancellationSource = new CancellationTokenSource();
_inferenceJobRepository.SetupSequence(p => p.AsQueryable())
.Returns((new List <InferenceJob>()
{
job
}).AsQueryable());
var jobStore = new ClaraJobRepository(
_logger.Object,
_configuration,
fileSystem.Object,
_inferenceJobRepository.Object);
await Assert.ThrowsAsync <Exception>(async() => await jobStore.Add(job));
_logger.VerifyLoggingMessageBeginsWith($"Failed to copy file {job.JobPayloadsStoragePath}.", LogLevel.Error, Times.Once());
}