private async Task BackgroundProcessing(CancellationToken cancellationToken)
{
_logger.Log(LogLevel.Information, "Job Submitter Hosted Service is running.");
while (!cancellationToken.IsCancellationRequested)
{
InferenceJob job = null;
try
{
job = await _jobStore.Take(cancellationToken);
using (_logger.BeginScope(new Dictionary <string, object> {
{ "JobId", job.JobId }, { "PayloadId", job.PayloadId }
}))
{
var files = _fileSystem.Directory.GetFiles(job.JobPayloadsStoragePath, "*", System.IO.SearchOption.AllDirectories);
await UploadFiles(job, job.JobPayloadsStoragePath, files);
await _jobsApi.Start(job);
await _jobStore.Update(job, InferenceJobStatus.Success);
RemoveFiles(files);
}
}
catch (OperationCanceledException ex)
{
_logger.Log(LogLevel.Warning, ex, "Job Store Service canceled: {0}");
}
catch (InvalidOperationException ex)
{
_logger.Log(LogLevel.Warning, ex, "Job Store Service may be disposed or Jobs API returned an error: {0}");
}
catch (Exception ex)
{
_logger.Log(LogLevel.Error, ex, "Error uploading payloads/starting job.");
if (job != null)
{
await _jobStore.Update(job, InferenceJobStatus.Fail);
}
}
}
_logger.Log(LogLevel.Information, "Cancellation requested.");
}
