public async Task ShallLogErrorOnJobTransitionError() { var request = new InferenceJob { JobId = "1", PayloadId = "1", State = InferenceJobState.Created, Source = "Source" }; request.SetStoragePath("/job"); _jobStore.SetupSequence(p => p.Take(It.IsAny <CancellationToken>())) .Returns(Task.FromResult(request)) .Returns(() => { _cancellationTokenSource.Cancel(); throw new OperationCanceledException(); }); _jobStore.Setup(p => p.TransitionState(It.IsAny <InferenceJob>(), It.IsAny <InferenceJobStatus>(), It.IsAny <CancellationToken>())) .Throws(new Exception("error")); var service = new JobSubmissionService( _instanceCleanupQueue.Object, _logger.Object, _serviceScopeFactory.Object, _fileSystem.Object, _configuration); await service.StartAsync(_cancellationTokenSource.Token); BlockUntilCanceled(_cancellationTokenSource.Token); _jobStore.Verify(p => p.TransitionState(request, InferenceJobStatus.Fail, It.IsAny <CancellationToken>()), Times.Once()); _logger.VerifyLogging("Error while transitioning job state.", LogLevel.Error, Times.Once()); }
public async Task ShallHandleInvalidOperationException() { _jobStore.Setup(p => p.Take(It.IsAny <CancellationToken>())) .Returns((CancellationToken token) => { BlockUntilCanceled(token); throw new InvalidOperationException("canceled"); }); _cancellationTokenSource.CancelAfter(250); var service = new JobSubmissionService( _instanceCleanupQueue.Object, _logger.Object, _jobsApi.Object, _payloadsApi.Object, _jobStore.Object, _fileSystem.Object); await service.StartAsync(_cancellationTokenSource.Token); BlockUntilCanceled(_cancellationTokenSource.Token); _logger.VerifyLogging("Job Submitter Hosted Service is running.", LogLevel.Information, Times.Once()); _logger.VerifyLogging("Cancellation requested.", LogLevel.Information, Times.Once()); _logger.VerifyLoggingMessageBeginsWith("Job Store Service may be disposed", LogLevel.Warning, Times.Once()); }
public async Task ShallFailJobOnException() { var request = new InferenceJob("/job", new Job { JobId = "1", PayloadId = "1" }); _jobStore.SetupSequence(p => p.Take(It.IsAny <CancellationToken>())) .Returns(Task.FromResult(request)) .Returns(() => { _cancellationTokenSource.Cancel(); throw new OperationCanceledException(); }); _jobStore.Setup(p => p.Update(It.IsAny <InferenceJob>(), It.IsAny <InferenceJobStatus>())); var service = new JobSubmissionService( _instanceCleanupQueue.Object, _logger.Object, _jobsApi.Object, _payloadsApi.Object, _jobStore.Object, _fileSystem.Object); await service.StartAsync(_cancellationTokenSource.Token); BlockUntilCanceled(_cancellationTokenSource.Token); _logger.VerifyLogging("Error uploading payloads/starting job.", LogLevel.Error, Times.Once()); _jobStore.Verify(p => p.Update(request, InferenceJobStatus.Fail), Times.Once()); }
public async Task StartsJobAndTransitionsState() { var request = new InferenceJob { JobId = "1", PayloadId = "1", State = InferenceJobState.Starting, Source = "Source" }; request.SetStoragePath("/job"); _jobStore.SetupSequence(p => p.Take(It.IsAny <CancellationToken>())) .Returns(Task.FromResult(request)) .Returns(() => { _cancellationTokenSource.Cancel(); throw new OperationCanceledException(); }); _jobStore.Setup(p => p.TransitionState(It.IsAny <InferenceJob>(), It.IsAny <InferenceJobStatus>(), It.IsAny <CancellationToken>())); _jobsApi.Setup(p => p.Start(It.IsAny <Job>())); var service = new JobSubmissionService( _instanceCleanupQueue.Object, _logger.Object, _serviceScopeFactory.Object, _fileSystem.Object, _configuration); await service.StartAsync(_cancellationTokenSource.Token); BlockUntilCanceled(_cancellationTokenSource.Token); _jobStore.Verify(p => p.TransitionState(request, InferenceJobStatus.Success, It.IsAny <CancellationToken>()), Times.Once()); _jobsApi.Verify(p => p.Start(It.IsAny <Job>()), Times.Once()); }
public async Task ShallStopProcessingIfCancellationRequested() { _cancellationTokenSource.Cancel(); var service = new JobSubmissionService( _instanceCleanupQueue.Object, _logger.Object, _serviceScopeFactory.Object, _fileSystem.Object, _configuration); await service.StartAsync(_cancellationTokenSource.Token); _logger.VerifyLogging("Job Submitter Hosted Service is running.", LogLevel.Information, Times.Once()); _logger.VerifyLogging("Cancellation requested.", LogLevel.Information, Times.Once()); }
public async Task UploadsPayloadAndTransitionsState() { var request = new InferenceJob { JobId = "1", PayloadId = "1", State = InferenceJobState.PayloadUploading, Source = "Source" }; request.SetStoragePath("/job"); _jobStore.SetupSequence(p => p.Take(It.IsAny <CancellationToken>())) .Returns(Task.FromResult(request)) .Returns(() => { _cancellationTokenSource.Cancel(); throw new OperationCanceledException(); }); _jobStore.Setup(p => p.TransitionState(It.IsAny <InferenceJob>(), It.IsAny <InferenceJobStatus>(), It.IsAny <CancellationToken>())); _fileSystem.Setup(p => p.Directory.GetFiles(It.IsAny <string>(), It.IsAny <string>(), System.IO.SearchOption.AllDirectories)) .Returns(new string[] { "/file1", "/file2", "/file3" }); _payloadsApi.Setup(p => p.Upload(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>())); _instanceCleanupQueue.Setup(p => p.QueueInstance(It.IsAny <string>())); var service = new JobSubmissionService( _instanceCleanupQueue.Object, _logger.Object, _serviceScopeFactory.Object, _fileSystem.Object, _configuration); await service.StartAsync(_cancellationTokenSource.Token); BlockUntilCanceled(_cancellationTokenSource.Token); _logger.VerifyLogging("Uploading 3 files.", LogLevel.Information, Times.Once()); _logger.VerifyLogging("Upload to payload completed.", LogLevel.Information, Times.Once()); _jobStore.Verify(p => p.TransitionState(request, InferenceJobStatus.Success, It.IsAny <CancellationToken>()), Times.Once()); _jobsApi.Verify(p => p.AddMetadata(It.IsAny <Job>(), It.IsAny <Dictionary <string, string> >()), Times.Never()); _instanceCleanupQueue.Verify(p => p.QueueInstance(It.IsAny <string>()), Times.Exactly(3)); }
public async Task ShallFailJobOnPayloadUploadException() { var request = new InferenceJob { JobId = "1", PayloadId = "1", State = InferenceJobState.PayloadUploading, Source = "Source" }; request.SetStoragePath("/job"); _jobStore.SetupSequence(p => p.Take(It.IsAny <CancellationToken>())) .Returns(Task.FromResult(request)) .Returns(() => { _cancellationTokenSource.Cancel(); throw new OperationCanceledException(); }); _jobStore.Setup(p => p.TransitionState(It.IsAny <InferenceJob>(), It.IsAny <InferenceJobStatus>(), It.IsAny <CancellationToken>())); _fileSystem.Setup(p => p.Directory.GetFiles(It.IsAny <string>(), It.IsAny <string>(), System.IO.SearchOption.AllDirectories)) .Returns(new string[] { "/file1", "file2", "file3" }); _payloadsApi.Setup(p => p.Upload(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>())) .Throws(new Exception("error")); _instanceCleanupQueue.Setup(p => p.QueueInstance(It.IsAny <string>())); var service = new JobSubmissionService( _instanceCleanupQueue.Object, _logger.Object, _serviceScopeFactory.Object, _fileSystem.Object, _configuration); await service.StartAsync(_cancellationTokenSource.Token); BlockUntilCanceled(_cancellationTokenSource.Token); _logger.VerifyLoggingMessageBeginsWith("Error uploading file:", LogLevel.Error, Times.Exactly(3)); _logger.VerifyLogging($"Failed to upload {3} files.", LogLevel.Error, Times.Once()); _jobStore.Verify(p => p.TransitionState(request, InferenceJobStatus.Fail, It.IsAny <CancellationToken>()), Times.Once()); _instanceCleanupQueue.Verify(p => p.QueueInstance(It.IsAny <string>()), Times.Never()); }
public async Task UploadsMetadataAndTransitionsState() { var request = new InferenceJob { JobId = "1", PayloadId = "1", State = InferenceJobState.MetadataUploading, Source = "Source" }; request.SetStoragePath("/job"); _jobStore.SetupSequence(p => p.Take(It.IsAny <CancellationToken>())) .Returns(Task.FromResult(request)) .Returns(() => { _cancellationTokenSource.Cancel(); throw new OperationCanceledException(); }); _jobStore.Setup(p => p.TransitionState(It.IsAny <InferenceJob>(), It.IsAny <InferenceJobStatus>(), It.IsAny <CancellationToken>())); _fileSystem.Setup(p => p.Directory.GetFiles(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <SearchOption>())) .Returns(new string[] { "/file1", "/file2" }); _jobMetadataBuilderFactory.Setup(p => p.Build(It.IsAny <bool>(), It.IsAny <IReadOnlyList <string> >(), It.IsAny <IReadOnlyList <string> >())) .Returns(new JobMetadataBuilder() { { "Test", "TestValue" } }); var service = new JobSubmissionService( _instanceCleanupQueue.Object, _logger.Object, _serviceScopeFactory.Object, _fileSystem.Object, _configuration); await service.StartAsync(_cancellationTokenSource.Token); BlockUntilCanceled(_cancellationTokenSource.Token); _jobStore.Verify(p => p.TransitionState(request, InferenceJobStatus.Success, It.IsAny <CancellationToken>()), Times.Once()); _fileSystem.Verify(p => p.Directory.GetFiles(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <SearchOption>()), Times.Once()); _jobMetadataBuilderFactory.Verify(p => p.Build(It.IsAny <bool>(), It.IsAny <IReadOnlyList <string> >(), It.IsAny <IReadOnlyList <string> >()), Times.Once()); }
public async Task ShallCompleteRequest() { var request = new InferenceJob("/job", new Job { JobId = "JID", PayloadId = "PID" }); _jobStore.SetupSequence(p => p.Take(It.IsAny <CancellationToken>())) .Returns(Task.FromResult(request)) .Returns(() => { _cancellationTokenSource.Cancel(); throw new OperationCanceledException(); }); _jobStore.Setup(p => p.Update(It.IsAny <InferenceJob>(), It.IsAny <InferenceJobStatus>())); _fileSystem.Setup(p => p.Directory.GetFiles(It.IsAny <string>(), It.IsAny <string>(), System.IO.SearchOption.AllDirectories)) .Returns(new string[] { "/file1", "file2", "file3" }); _payloadsApi.Setup(p => p.Upload(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <IEnumerable <string> >())); _jobsApi.Setup(p => p.Start(It.IsAny <Job>())); _instanceCleanupQueue.Setup(p => p.QueueInstance(It.IsAny <string>())); var service = new JobSubmissionService( _instanceCleanupQueue.Object, _logger.Object, _jobsApi.Object, _payloadsApi.Object, _jobStore.Object, _fileSystem.Object); await service.StartAsync(_cancellationTokenSource.Token); BlockUntilCanceled(_cancellationTokenSource.Token); _logger.VerifyLogging("Uploading 3 files.", LogLevel.Information, Times.Once()); _logger.VerifyLogging("Upload to payload completed.", LogLevel.Information, Times.Once()); _jobsApi.Verify(p => p.Start(request), Times.Once()); _jobStore.Verify(p => p.Update(request, InferenceJobStatus.Success), Times.Once()); _instanceCleanupQueue.Verify(p => p.QueueInstance(It.IsAny <string>()), Times.Exactly(3)); }