public async Task ShallLogErrorOnJobTransitionError()
{
var request = new InferenceJob
{
JobId = "1",
PayloadId = "1",
State = InferenceJobState.Created,
Source = "Source"
};
request.SetStoragePath("/job");
_jobStore.SetupSequence(p => p.Take(It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(request))
.Returns(() =>
{
_cancellationTokenSource.Cancel();
throw new OperationCanceledException();
});
_jobStore.Setup(p => p.TransitionState(It.IsAny <InferenceJob>(), It.IsAny <InferenceJobStatus>(), It.IsAny <CancellationToken>()))
.Throws(new Exception("error"));
var service = new JobSubmissionService(
_instanceCleanupQueue.Object,
_logger.Object,
_serviceScopeFactory.Object,
_fileSystem.Object,
_configuration);
await service.StartAsync(_cancellationTokenSource.Token);
BlockUntilCanceled(_cancellationTokenSource.Token);
_jobStore.Verify(p => p.TransitionState(request, InferenceJobStatus.Fail, It.IsAny <CancellationToken>()), Times.Once());
_logger.VerifyLogging("Error while transitioning job state.", LogLevel.Error, Times.Once());
}
public async Task ShallHandleInvalidOperationException()
{
_jobStore.Setup(p => p.Take(It.IsAny <CancellationToken>()))
.Returns((CancellationToken token) =>
{
BlockUntilCanceled(token);
throw new InvalidOperationException("canceled");
});
_cancellationTokenSource.CancelAfter(250);
var service = new JobSubmissionService(
_instanceCleanupQueue.Object,
_logger.Object,
_jobsApi.Object,
_payloadsApi.Object,
_jobStore.Object,
_fileSystem.Object);
await service.StartAsync(_cancellationTokenSource.Token);
BlockUntilCanceled(_cancellationTokenSource.Token);
_logger.VerifyLogging("Job Submitter Hosted Service is running.", LogLevel.Information, Times.Once());
_logger.VerifyLogging("Cancellation requested.", LogLevel.Information, Times.Once());
_logger.VerifyLoggingMessageBeginsWith("Job Store Service may be disposed", LogLevel.Warning, Times.Once());
}
public async Task ShallFailJobOnException()
{
var request = new InferenceJob("/job", new Job {
JobId = "1", PayloadId = "1"
});
_jobStore.SetupSequence(p => p.Take(It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(request))
.Returns(() =>
{
_cancellationTokenSource.Cancel();
throw new OperationCanceledException();
});
_jobStore.Setup(p => p.Update(It.IsAny <InferenceJob>(), It.IsAny <InferenceJobStatus>()));
var service = new JobSubmissionService(
_instanceCleanupQueue.Object,
_logger.Object,
_jobsApi.Object,
_payloadsApi.Object,
_jobStore.Object,
_fileSystem.Object);
await service.StartAsync(_cancellationTokenSource.Token);
BlockUntilCanceled(_cancellationTokenSource.Token);
_logger.VerifyLogging("Error uploading payloads/starting job.", LogLevel.Error, Times.Once());
_jobStore.Verify(p => p.Update(request, InferenceJobStatus.Fail), Times.Once());
}
public async Task StartsJobAndTransitionsState()
{
var request = new InferenceJob
{
JobId = "1",
PayloadId = "1",
State = InferenceJobState.Starting,
Source = "Source"
};
request.SetStoragePath("/job");
_jobStore.SetupSequence(p => p.Take(It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(request))
.Returns(() =>
{
_cancellationTokenSource.Cancel();
throw new OperationCanceledException();
});
_jobStore.Setup(p => p.TransitionState(It.IsAny <InferenceJob>(), It.IsAny <InferenceJobStatus>(), It.IsAny <CancellationToken>()));
_jobsApi.Setup(p => p.Start(It.IsAny <Job>()));
var service = new JobSubmissionService(
_instanceCleanupQueue.Object,
_logger.Object,
_serviceScopeFactory.Object,
_fileSystem.Object,
_configuration);
await service.StartAsync(_cancellationTokenSource.Token);
BlockUntilCanceled(_cancellationTokenSource.Token);
_jobStore.Verify(p => p.TransitionState(request, InferenceJobStatus.Success, It.IsAny <CancellationToken>()), Times.Once());
_jobsApi.Verify(p => p.Start(It.IsAny <Job>()), Times.Once());
}
public async Task ShallStopProcessingIfCancellationRequested()
{
_cancellationTokenSource.Cancel();
var service = new JobSubmissionService(
_instanceCleanupQueue.Object,
_logger.Object,
_serviceScopeFactory.Object,
_fileSystem.Object,
_configuration);
await service.StartAsync(_cancellationTokenSource.Token);
_logger.VerifyLogging("Job Submitter Hosted Service is running.", LogLevel.Information, Times.Once());
_logger.VerifyLogging("Cancellation requested.", LogLevel.Information, Times.Once());
}
public async Task UploadsPayloadAndTransitionsState()
{
var request = new InferenceJob
{
JobId = "1",
PayloadId = "1",
State = InferenceJobState.PayloadUploading,
Source = "Source"
};
request.SetStoragePath("/job");
_jobStore.SetupSequence(p => p.Take(It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(request))
.Returns(() =>
{
_cancellationTokenSource.Cancel();
throw new OperationCanceledException();
});
_jobStore.Setup(p => p.TransitionState(It.IsAny <InferenceJob>(), It.IsAny <InferenceJobStatus>(), It.IsAny <CancellationToken>()));
_fileSystem.Setup(p => p.Directory.GetFiles(It.IsAny <string>(), It.IsAny <string>(), System.IO.SearchOption.AllDirectories))
.Returns(new string[] { "/file1", "/file2", "/file3" });
_payloadsApi.Setup(p => p.Upload(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>()));
_instanceCleanupQueue.Setup(p => p.QueueInstance(It.IsAny <string>()));
var service = new JobSubmissionService(
_instanceCleanupQueue.Object,
_logger.Object,
_serviceScopeFactory.Object,
_fileSystem.Object,
_configuration);
await service.StartAsync(_cancellationTokenSource.Token);
BlockUntilCanceled(_cancellationTokenSource.Token);
_logger.VerifyLogging("Uploading 3 files.", LogLevel.Information, Times.Once());
_logger.VerifyLogging("Upload to payload completed.", LogLevel.Information, Times.Once());
_jobStore.Verify(p => p.TransitionState(request, InferenceJobStatus.Success, It.IsAny <CancellationToken>()), Times.Once());
_jobsApi.Verify(p => p.AddMetadata(It.IsAny <Job>(), It.IsAny <Dictionary <string, string> >()), Times.Never());
_instanceCleanupQueue.Verify(p => p.QueueInstance(It.IsAny <string>()), Times.Exactly(3));
}
public async Task ShallFailJobOnPayloadUploadException()
{
var request = new InferenceJob
{
JobId = "1",
PayloadId = "1",
State = InferenceJobState.PayloadUploading,
Source = "Source"
};
request.SetStoragePath("/job");
_jobStore.SetupSequence(p => p.Take(It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(request))
.Returns(() =>
{
_cancellationTokenSource.Cancel();
throw new OperationCanceledException();
});
_jobStore.Setup(p => p.TransitionState(It.IsAny <InferenceJob>(), It.IsAny <InferenceJobStatus>(), It.IsAny <CancellationToken>()));
_fileSystem.Setup(p => p.Directory.GetFiles(It.IsAny <string>(), It.IsAny <string>(), System.IO.SearchOption.AllDirectories))
.Returns(new string[] { "/file1", "file2", "file3" });
_payloadsApi.Setup(p => p.Upload(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>()))
.Throws(new Exception("error"));
_instanceCleanupQueue.Setup(p => p.QueueInstance(It.IsAny <string>()));
var service = new JobSubmissionService(
_instanceCleanupQueue.Object,
_logger.Object,
_serviceScopeFactory.Object,
_fileSystem.Object,
_configuration);
await service.StartAsync(_cancellationTokenSource.Token);
BlockUntilCanceled(_cancellationTokenSource.Token);
_logger.VerifyLoggingMessageBeginsWith("Error uploading file:", LogLevel.Error, Times.Exactly(3));
_logger.VerifyLogging($"Failed to upload {3} files.", LogLevel.Error, Times.Once());
_jobStore.Verify(p => p.TransitionState(request, InferenceJobStatus.Fail, It.IsAny <CancellationToken>()), Times.Once());
_instanceCleanupQueue.Verify(p => p.QueueInstance(It.IsAny <string>()), Times.Never());
}
public async Task UploadsMetadataAndTransitionsState()
{
var request = new InferenceJob
{
JobId = "1",
PayloadId = "1",
State = InferenceJobState.MetadataUploading,
Source = "Source"
};
request.SetStoragePath("/job");
_jobStore.SetupSequence(p => p.Take(It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(request))
.Returns(() =>
{
_cancellationTokenSource.Cancel();
throw new OperationCanceledException();
});
_jobStore.Setup(p => p.TransitionState(It.IsAny <InferenceJob>(), It.IsAny <InferenceJobStatus>(), It.IsAny <CancellationToken>()));
_fileSystem.Setup(p => p.Directory.GetFiles(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <SearchOption>()))
.Returns(new string[] { "/file1", "/file2" });
_jobMetadataBuilderFactory.Setup(p => p.Build(It.IsAny <bool>(), It.IsAny <IReadOnlyList <string> >(), It.IsAny <IReadOnlyList <string> >()))
.Returns(new JobMetadataBuilder()
{
{ "Test", "TestValue" }
});
var service = new JobSubmissionService(
_instanceCleanupQueue.Object,
_logger.Object,
_serviceScopeFactory.Object,
_fileSystem.Object,
_configuration);
await service.StartAsync(_cancellationTokenSource.Token);
BlockUntilCanceled(_cancellationTokenSource.Token);
_jobStore.Verify(p => p.TransitionState(request, InferenceJobStatus.Success, It.IsAny <CancellationToken>()), Times.Once());
_fileSystem.Verify(p => p.Directory.GetFiles(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <SearchOption>()), Times.Once());
_jobMetadataBuilderFactory.Verify(p => p.Build(It.IsAny <bool>(), It.IsAny <IReadOnlyList <string> >(), It.IsAny <IReadOnlyList <string> >()), Times.Once());
}
public async Task ShallCompleteRequest()
{
var request = new InferenceJob("/job", new Job {
JobId = "JID", PayloadId = "PID"
});
_jobStore.SetupSequence(p => p.Take(It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(request))
.Returns(() =>
{
_cancellationTokenSource.Cancel();
throw new OperationCanceledException();
});
_jobStore.Setup(p => p.Update(It.IsAny <InferenceJob>(), It.IsAny <InferenceJobStatus>()));
_fileSystem.Setup(p => p.Directory.GetFiles(It.IsAny <string>(), It.IsAny <string>(), System.IO.SearchOption.AllDirectories))
.Returns(new string[] { "/file1", "file2", "file3" });
_payloadsApi.Setup(p => p.Upload(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <IEnumerable <string> >()));
_jobsApi.Setup(p => p.Start(It.IsAny <Job>()));
_instanceCleanupQueue.Setup(p => p.QueueInstance(It.IsAny <string>()));
var service = new JobSubmissionService(
_instanceCleanupQueue.Object,
_logger.Object,
_jobsApi.Object,
_payloadsApi.Object,
_jobStore.Object,
_fileSystem.Object);
await service.StartAsync(_cancellationTokenSource.Token);
BlockUntilCanceled(_cancellationTokenSource.Token);
_logger.VerifyLogging("Uploading 3 files.", LogLevel.Information, Times.Once());
_logger.VerifyLogging("Upload to payload completed.", LogLevel.Information, Times.Once());
_jobsApi.Verify(p => p.Start(request), Times.Once());
_jobStore.Verify(p => p.Update(request, InferenceJobStatus.Success), Times.Once());
_instanceCleanupQueue.Verify(p => p.QueueInstance(It.IsAny <string>()), Times.Exactly(3));
}
