private async Task BackgroundProcessing(CancellationToken cancellationToken) { _logger.Log(LogLevel.Information, "Job Submitter Hosted Service is running."); while (!cancellationToken.IsCancellationRequested) { InferenceJob job = null; try { job = await _jobStore.Take(cancellationToken); using (_logger.BeginScope(new Dictionary <string, object> { { "JobId", job.JobId }, { "PayloadId", job.PayloadId } })) { var files = _fileSystem.Directory.GetFiles(job.JobPayloadsStoragePath, "*", System.IO.SearchOption.AllDirectories); await UploadFiles(job, job.JobPayloadsStoragePath, files); await _jobsApi.Start(job); await _jobStore.Update(job, InferenceJobStatus.Success); RemoveFiles(files); } } catch (OperationCanceledException ex) { _logger.Log(LogLevel.Warning, ex, "Job Store Service canceled: {0}"); } catch (InvalidOperationException ex) { _logger.Log(LogLevel.Warning, ex, "Job Store Service may be disposed or Jobs API returned an error: {0}"); } catch (Exception ex) { _logger.Log(LogLevel.Error, ex, "Error uploading payloads/starting job."); if (job != null) { await _jobStore.Update(job, InferenceJobStatus.Fail); } } } _logger.Log(LogLevel.Information, "Cancellation requested."); }