private async Task RetrieveViaDicomWeb(InferenceRequest inferenceRequest, RequestInputDataResource source, Dictionary <string, InstanceStorageInfo> retrievedInstance)
{
Guard.Against.Null(inferenceRequest, nameof(inferenceRequest));
Guard.Against.Null(retrievedInstance, nameof(retrievedInstance));
var authenticationHeaderValue = GenerateAuthenticationHeader(source.ConnectionDetails.AuthType, source.ConnectionDetails.AuthId);
var dicomWebClient = _dicomWebClientFactory.CreateDicomWebClient(
new Uri(source.ConnectionDetails.Uri),
authenticationHeaderValue,
null,
null,
null,
null);
switch (inferenceRequest.InputMetadata.Details.Type)
{
case InferenceRequestType.DicomUid:
await RetrieveStudies(inferenceRequest.InputMetadata.Details.Studies, inferenceRequest.StoragePath, dicomWebClient, retrievedInstance);
break;
default:
throw new InferenceRequestException($"The 'inputMetadata' type '{inferenceRequest.InputMetadata.Details.Type}' specified is not supported.");
}
}
private async Task MoveToArchive(InferenceRequest inferenceRequest)
{
Guard.Against.Null(inferenceRequest, nameof(inferenceRequest));
var crd = CreateFromRequest(inferenceRequest, true);
var operationResponse = await Policy
.Handle <HttpOperationException>()
.WaitAndRetryAsync(
3,
retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
(exception, timeSpan, retryCount, context) =>
{
_logger.Log(LogLevel.Warning, exception, $"Failed to archive inference request JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId} in CRD. Waiting {timeSpan} before next retry. Retry attempt {retryCount}. {(exception as HttpOperationException)?.Response?.Content}");
})
.ExecuteAsync(async() =>
{
var result = await _kubernetesClient.CreateNamespacedCustomObjectWithHttpMessagesAsync(CustomResourceDefinition.InferenceRequestArchivesCrd, crd);
_logger.Log(LogLevel.Information, $"Inference request archived. JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId}");
return(result);
})
.ConfigureAwait(false);
try
{
operationResponse.Response.EnsureSuccessStatusCode();
}
catch (Exception ex)
{
// drop the request if it has already reached max retries.
_logger.Log(LogLevel.Error, ex, $"Failed to archive inference request after maximum attempts. Request will be dropped. JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId} in CRD. {(ex as HttpOperationException)?.Response?.Content}");
}
}
private async Task ProcessRequest(InferenceRequest inferenceRequest, CancellationToken cancellationToken)
{
Guard.Against.Null(inferenceRequest, nameof(inferenceRequest));
var retrievedInstances = new Dictionary <string, InstanceStorageInfo>();
RestoreExistingInstances(inferenceRequest, retrievedInstances);
foreach (var source in inferenceRequest.InputResources)
{
switch (source.Interface)
{
case InputInterfaceType.DicomWeb:
await RetrieveViaDicomWeb(inferenceRequest, source, retrievedInstances);
break;
case InputInterfaceType.Algorithm:
continue;
default:
_logger.Log(LogLevel.Warning, $"Specified input interface is not supported '{source.Interface}`");
break;
}
}
if (retrievedInstances.Count == 0)
{
throw new InferenceRequestException("No DICOM instance found/retrieved with the request.");
}
await SubmitPipelineJob(inferenceRequest, retrievedInstances.Select(p => p.Value), cancellationToken);
}
public async Task UpdateSuccess_ShallRetryDeleteOnFailure()
{
_kubernetesClient
.Setup(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>()))
.Returns(Task.FromResult(new HttpOperationResponse <object>
{
Response = new HttpResponseMessage()
}));
_kubernetesClient
.Setup(p => p.DeleteNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <string>()))
.Throws(new HttpOperationException("error message")
{
Response = new HttpResponseMessageWrapper(new HttpResponseMessage(HttpStatusCode.Conflict), "error content")
});
var inferenceRequest = new InferenceRequest();
inferenceRequest.JobId = Guid.NewGuid().ToString();
inferenceRequest.PayloadId = Guid.NewGuid().ToString();
inferenceRequest.TransactionId = Guid.NewGuid().ToString();
var store = new InferenceRequestStore(_loggerFactory.Object, _configuration, _kubernetesClient.Object);
await Assert.ThrowsAsync <HttpOperationException>(async() => await store.Update(inferenceRequest, InferenceRequestStatus.Success));
_logger.VerifyLoggingMessageBeginsWith($"Failed to delete inference request JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId} in CRD.", LogLevel.Warning, Times.Exactly(3));
_logger.VerifyLoggingMessageBeginsWith($"Inference request archived. JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId}", LogLevel.Information, Times.Once());
_logger.VerifyLoggingMessageBeginsWith($"Inference request deleted. JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId}", LogLevel.Information, Times.Never());
_kubernetesClient.Verify(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>()), Times.Once());
_kubernetesClient.Verify(p => p.DeleteNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), inferenceRequest.JobId), Times.Exactly(4));
}
private void RestoreExistingInstances(InferenceRequest inferenceRequest, Dictionary <string, InstanceStorageInfo> retrievedInstances)
{
Guard.Against.Null(inferenceRequest, nameof(inferenceRequest));
Guard.Against.Null(retrievedInstances, nameof(retrievedInstances));
_logger.Log(LogLevel.Debug, $"Restoring previously retrieved DICOM instances from {inferenceRequest.StoragePath}");
foreach (var file in _fileSystem.Directory.EnumerateFiles(inferenceRequest.StoragePath, "*.dcm", System.IO.SearchOption.AllDirectories))
{
if (_dicomToolkit.HasValidHeader(file))
{
var dicomFile = _dicomToolkit.Open(file);
var instance = InstanceStorageInfo.CreateInstanceStorageInfo(dicomFile, inferenceRequest.StoragePath, _fileSystem);
if (retrievedInstances.ContainsKey(instance.SopInstanceUid))
{
continue;
}
retrievedInstances.Add(instance.SopInstanceUid, instance);
_logger.Log(LogLevel.Debug, $"Restored previously retrieved instance {instance.SopInstanceUid}");
}
else
{
_logger.Log(LogLevel.Warning, $"Unable to restore previously retrieved instance from {file}; file does not contain valid DICOM header.");
}
}
}
private async Task SubmitPipelineJob(InferenceRequest inferenceRequest, IEnumerable <InstanceStorageInfo> instances, IEnumerable <string> resources, CancellationToken cancellationToken)
{
Guard.Against.Null(inferenceRequest, nameof(inferenceRequest));
if (instances.IsNullOrEmpty() && resources.IsNullOrEmpty())
{
throw new ArgumentNullException("no instances/resources found.", nameof(instances));
}
_logger.Log(LogLevel.Information, $"Queuing a new job '{inferenceRequest.JobName}' with pipeline '{inferenceRequest.Algorithm.PipelineId}', priority={inferenceRequest.ClaraJobPriority}, instance count={instances.Count() + resources.Count()}");
using var scope = _serviceScopeFactory.CreateScope();
var repository = scope.ServiceProvider.GetRequiredService <IJobRepository>();
await repository.Add(
new InferenceJob
{
JobId = inferenceRequest.JobId,
PayloadId = inferenceRequest.PayloadId,
PipelineId = inferenceRequest.Algorithm.PipelineId,
JobName = inferenceRequest.JobName,
Instances = instances.ToList(),
State = InferenceJobState.Created,
Resources = resources.ToList(),
Source = inferenceRequest.TransactionId,
}, false);
}
public async Task UpdateSuccess_ShallArchiveAndDelete()
{
_kubernetesClient
.Setup(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>()))
.Returns(Task.FromResult(new HttpOperationResponse <object>
{
Response = new HttpResponseMessage()
}));
_kubernetesClient
.Setup(p => p.DeleteNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <string>()))
.Returns(Task.FromResult(new HttpOperationResponse <object>
{
Response = new HttpResponseMessage()
}));
var inferenceRequest = new InferenceRequest();
inferenceRequest.JobId = Guid.NewGuid().ToString();
inferenceRequest.PayloadId = Guid.NewGuid().ToString();
inferenceRequest.TransactionId = Guid.NewGuid().ToString();
var store = new InferenceRequestStore(_loggerFactory.Object, _configuration, _kubernetesClient.Object);
await store.Update(inferenceRequest, InferenceRequestStatus.Success);
_logger.VerifyLoggingMessageBeginsWith($"Inference request archived. JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId}", LogLevel.Information, Times.Once());
_logger.VerifyLoggingMessageBeginsWith($"Inference request deleted. JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId}", LogLevel.Information, Times.Once());
_kubernetesClient.Verify(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>()), Times.Once());
_kubernetesClient.Verify(p => p.DeleteNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), inferenceRequest.JobId), Times.Once());
}
public async Task <ActionResult> NewInferenceRequest([FromBody] InferenceRequest request)
{
Guard.Against.Null(request, nameof(request));
if (!request.IsValid(out string details))
{
return(Problem(title: $"Invalid request", statusCode: (int)HttpStatusCode.UnprocessableEntity, detail: details));
}
using var _ = _logger.BeginScope(new LogginDataDictionary <string, object> { { "TransactionId", request.TransactionId } });
try
{
await CreateJob(request);
_logger.Log(LogLevel.Information, $"Job created with Clara: JobId={request.JobId}, PayloadId={request.PayloadId}");
}
catch (Exception ex)
{
_logger.Log(LogLevel.Error, ex, $"Failed to create job with Clara Platform: TransactionId={request.TransactionId}");
return(Problem(title: "Failed to create job", statusCode: (int)HttpStatusCode.InternalServerError, detail: ex.Message));
}
try
{
if (_fileSystem.Directory.TryGenerateDirectory(
_fileSystem.Path.Combine(_configuration.Value.Storage.TemporaryDataDirFullPath, "irs", request.TransactionId, request.JobId),
out string storagePath))
{
_logger.Log(LogLevel.Information, $"Setting storage path to {storagePath}");
request.ConfigureTemporaryStorageLocation(storagePath);
}
else
{
throw new InferenceRequestException("Failed to generate a temporary storage location for request.");
}
}
catch (Exception ex)
{
_logger.Log(LogLevel.Error, ex, $"Failed to configure storage location for request: TransactionId={request.TransactionId}");
return(Problem(title: ex.Message, statusCode: (int)HttpStatusCode.InternalServerError, detail: ex.Message));
}
try
{
await _inferenceRequestStore.Add(request);
}
catch (Exception ex)
{
_logger.Log(LogLevel.Error, ex, $"Unable to queue the request: TransactionId={request.TransactionId}");
return(Problem(title: "Failed to save request", statusCode: (int)HttpStatusCode.InternalServerError, detail: ex.Message));
}
return(Ok(new InferenceRequestResponse
{
JobId = request.JobId,
PayloadId = request.PayloadId,
TransactionId = request.TransactionId
}));
}
public async Task Update(InferenceRequest inferenceRequest, InferenceRequestStatus status)
{
Guard.Against.Null(inferenceRequest, nameof(inferenceRequest));
if (status == InferenceRequestStatus.Success)
{
_logger.Log(LogLevel.Information, $"Archiving inference request JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId}.");
inferenceRequest.State = InferenceRequestState.Completed;
inferenceRequest.Status = InferenceRequestStatus.Success;
await MoveToArchive(inferenceRequest);
await Delete(inferenceRequest);
}
else
{
if (++inferenceRequest.TryCount > MaxRetryLimit)
{
_logger.Log(LogLevel.Information, $"Exceeded maximum retries; removing inference request JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId} from Inference Request store.");
inferenceRequest.State = InferenceRequestState.Completed;
inferenceRequest.Status = InferenceRequestStatus.Fail;
await MoveToArchive(inferenceRequest);
await Delete(inferenceRequest);
}
else
{
_logger.Log(LogLevel.Information, $"Updating inference request JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId} to Queued.");
inferenceRequest.State = InferenceRequestState.Queued;
await UpdateInferenceRequest(inferenceRequest);
_logger.Log(LogLevel.Information, $"Inference request JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId} added back to Inference Request store for retry.");
}
}
_cache.Remove(inferenceRequest.JobId);
}
private async Task CreateJob(InferenceRequest request)
{
var job = await _jobsApi.Create(request.Algorithm.PipelineId, request.JobName, request.ClaraJobPriority);
request.JobId = job.JobId;
request.PayloadId = job.PayloadId;
}
public async Task Update(InferenceRequest inferenceRequest, InferenceRequestStatus status)
{
Guard.Against.Null(inferenceRequest, nameof(inferenceRequest));
using var loggerScope = _logger.BeginScope(new LogginDataDictionary <string, object> { { "JobId", inferenceRequest.JobId }, { "TransactionId", inferenceRequest.TransactionId } });
if (status == InferenceRequestStatus.Success)
{
inferenceRequest.State = InferenceRequestState.Completed;
inferenceRequest.Status = InferenceRequestStatus.Success;
}
else
{
if (++inferenceRequest.TryCount > MaxRetryLimit)
{
_logger.Log(LogLevel.Information, $"Exceeded maximum retries.");
inferenceRequest.State = InferenceRequestState.Completed;
inferenceRequest.Status = InferenceRequestStatus.Fail;
}
else
{
_logger.Log(LogLevel.Information, $"Will retry later.");
inferenceRequest.State = InferenceRequestState.Queued;
}
}
await Save(inferenceRequest);
}
private async Task Save(InferenceRequest inferenceRequest)
{
Guard.Against.Null(inferenceRequest, nameof(inferenceRequest));
await Policy
.Handle <Exception>()
.WaitAndRetryAsync(
3,
retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
(exception, timeSpan, retryCount, context) =>
{
_logger.Log(LogLevel.Error, exception, $"Error while updating inference request. Waiting {timeSpan} before next retry. Retry attempt {retryCount}.");
})
.ExecuteAsync(async() =>
{
_logger.Log(LogLevel.Debug, $"Updating inference request.");
if (inferenceRequest.State == InferenceRequestState.Completed)
{
_inferenceRequestRepository.Detach(inferenceRequest);
}
await _inferenceRequestRepository.SaveChangesAsync();
_logger.Log(LogLevel.Information, $"Inference request updated.");
})
.ConfigureAwait(false);
}
public void IsValid_ShallReturnFalseWithEmptyStudy()
{
var request = new InferenceRequest();
request.InputResources.Add(new RequestInputDataResource
{
Interface = InputInterfaceType.Algorithm,
ConnectionDetails = new InputConnectionDetails()
});
request.InputResources.Add(new RequestInputDataResource
{
Interface = InputInterfaceType.DicomWeb,
ConnectionDetails = new InputConnectionDetails
{
Uri = "http://this.is.not.a.valid.uri\\",
AuthId = "token",
AuthType = ConnectionAuthType.Bearer,
}
});
request.InputMetadata = new InferenceRequestMetadata
{
Details = new InferenceRequestDetails
{
Type = InferenceRequestType.DicomUid
}
};
Assert.False(request.IsValid(out string _));
}
private bool ShouldRetry(InferenceRequest inferenceRequest)
{
foreach (var input in inferenceRequest.InputMetadata.Inputs)
{
if (input.Resources?.Any(p => !p.IsRetrieved) ?? false)
{
return(true);
}
if (input.Studies?.Any(p => !p.IsRetrieved) ?? false)
{
return(true);
}
if (input.Studies?.Any(s => s.Series?.Any(r => !r.IsRetrieved) ?? false) ?? false)
{
return(true);
}
if (input.Studies?.Any(s => s.Series?.Any(r => r.Instances?.Any(i => !i.IsRetrieved) ?? false) ?? false) ?? false)
{
return(true);
}
}
return(false);
}
public void IsValidate_ShallReturnTrue()
{
var request = new InferenceRequest();
request.InputResources.Add(new RequestInputDataResource
{
Interface = InputInterfaceType.Algorithm,
ConnectionDetails = new InputConnectionDetails()
});
request.InputResources.Add(new RequestInputDataResource {
Interface = InputInterfaceType.DicomWeb
});
request.InputMetadata = new InferenceRequestMetadata
{
Details = new InferenceRequestDetails
{
Type = InferenceRequestType.DicomUid,
Studies = new List <RequestedStudy>
{
new RequestedStudy
{
StudyInstanceUid = "1"
}
}
}
};
Assert.True(request.IsValid(out string _));
}
public void NewInferenceRequest_ShallAcceptInferenceRequest()
{
_jobsApi.Setup(p => p.Create(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <JobPriority>(), It.IsAny <Dictionary <string, string> >()))
.Returns(Task.FromResult(new Job
{
JobId = "JOBID",
PayloadId = "PAYLOADID"
}));
_inferenceRequestStore.Setup(p => p.Add(It.IsAny <InferenceRequest>()));
var input = new InferenceRequest();
input.TransactionId = Guid.NewGuid().ToString();
input.InputResources = new List <RequestInputDataResource>()
{
new RequestInputDataResource
{
Interface = InputInterfaceType.Algorithm,
ConnectionDetails = new InputConnectionDetails()
},
new RequestInputDataResource
{
Interface = InputInterfaceType.DicomWeb,
ConnectionDetails = new InputConnectionDetails
{
Uri = "http://my.svc/api"
}
}
};
input.InputMetadata = new InferenceRequestMetadata
{
Details = new InferenceRequestDetails
{
Type = InferenceRequestType.DicomUid,
Studies = new List <RequestedStudy>
{
new RequestedStudy
{
StudyInstanceUid = "1"
}
}
}
};
var result = _controller.NewInferenceRequest(input);
_inferenceRequestStore.Verify(p => p.Add(input), Times.Once());
Assert.NotNull(result);
var objectResult = result.Result as OkObjectResult;
Assert.NotNull(objectResult);
var response = objectResult.Value as InferenceRequestResponse;
Assert.NotNull(response);
Assert.Equal("JOBID", response.JobId);
Assert.Equal("PAYLOADID", response.PayloadId);
Assert.Equal(input.TransactionId, response.TransactionId);
}
public void NewInferenceRequest_ShallReturnProblemIfFailedToAddJob()
{
_jobsApi.Setup(p => p.Create(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <JobPriority>(), It.IsAny <Dictionary <string, string> >()))
.Returns(Task.FromResult(new Job
{
JobId = "JOBID",
PayloadId = "PAYLOADID"
}));
_inferenceRequestStore.Setup(p => p.Add(It.IsAny <InferenceRequest>()))
.Throws(new Exception("error"));
var input = new InferenceRequest();
input.TransactionId = Guid.NewGuid().ToString();
input.InputResources = new List <RequestInputDataResource>()
{
new RequestInputDataResource
{
Interface = InputInterfaceType.Algorithm,
ConnectionDetails = new InputConnectionDetails()
},
new RequestInputDataResource
{
Interface = InputInterfaceType.DicomWeb,
ConnectionDetails = new InputConnectionDetails
{
Uri = "http://my.svc/api"
}
}
};
input.InputMetadata = new InferenceRequestMetadata
{
Details = new InferenceRequestDetails
{
Type = InferenceRequestType.DicomUid,
Studies = new List <RequestedStudy>
{
new RequestedStudy
{
StudyInstanceUid = "1"
}
}
}
};
var result = _controller.NewInferenceRequest(input);
Assert.NotNull(result);
var objectResult = result.Result as ObjectResult;
Assert.NotNull(objectResult);
var problem = objectResult.Value as ProblemDetails;
Assert.NotNull(problem);
Assert.Equal("Failed to save request", problem.Title);
Assert.Equal(500, problem.Status);
}
public void IsValid_ShallReturnAllErrors()
{
var request = new InferenceRequest();
Assert.False(request.IsValid(out string _));
request.InputResources.Add(new RequestInputDataResource {
Interface = InputInterfaceType.Algorithm
});
Assert.False(request.IsValid(out string _));
}
private async Task CreateJob(InferenceRequest request)
{
var metadata = new JobMetadataBuilder();
metadata.AddSourceName($"{request.TransactionId}");
var job = await _jobsApi.Create(request.Algorithm.PipelineId, request.JobName, request.ClaraJobPriority, metadata);
request.JobId = job.JobId;
request.PayloadId = job.PayloadId;
}
public void ConfigureTemporaryStorageLocation_ShallThrowIfAlreadyConfigured()
{
var request = new InferenceRequest();
request.ConfigureTemporaryStorageLocation("/blabla");
Assert.Throws <InferenceRequestException>(() =>
{
request.ConfigureTemporaryStorageLocation("/new-location");
});
}
private async Task ProcessRequest(InferenceRequest inferenceRequest, CancellationToken cancellationToken)
{
Guard.Against.Null(inferenceRequest, nameof(inferenceRequest));
var retrievedInstances = new Dictionary <string, InstanceStorageInfo>();
RestoreExistingInstances(inferenceRequest, retrievedInstances);
var retrievedResources = new Dictionary <string, string>();
RestoreExistingResources(inferenceRequest, retrievedResources);
foreach (var source in inferenceRequest.InputResources)
{
switch (source.Interface)
{
case InputInterfaceType.DicomWeb:
_logger.Log(LogLevel.Information, $"Processing input source '{source.Interface}' from {source.ConnectionDetails.Uri}");
await RetrieveViaDicomWeb(inferenceRequest, source, retrievedInstances);
break;
case InputInterfaceType.Fhir:
_logger.Log(LogLevel.Information, $"Processing input source '{source.Interface}' from {source.ConnectionDetails.Uri}");
await RetrieveViaFhir(inferenceRequest, source, retrievedResources);
break;
case InputInterfaceType.Algorithm:
continue;
default:
_logger.Log(LogLevel.Warning, $"Specified input interface is not supported '{source.Interface}`");
break;
}
}
if (inferenceRequest.TryCount < InferenceRequestRepository.MaxRetryLimit && ShouldRetry(inferenceRequest))
{
throw new InferenceRequestException("One or more failures occurred while retrieving specified resources. Will retry later.");
}
if (retrievedInstances.Count == 0 && retrievedResources.Count == 0)
{
throw new InferenceRequestException("No DICOM instances/resources retrieved with the request.");
}
await SubmitPipelineJob(inferenceRequest, retrievedInstances.Select(p => p.Value), retrievedResources.Select(p => p.Value), cancellationToken);
RemoveInstances(retrievedInstances.Select(p => p.Value.InstanceStorageFullPath));
RemoveInstances(retrievedResources.Select(p => p.Value));
}
public void ConfigureTemporaryStorageLocation_ShallThrowWithInvalidInput()
{
var request = new InferenceRequest();
Assert.Throws <ArgumentNullException>(() =>
{
request.ConfigureTemporaryStorageLocation(null);
});
Assert.Throws <ArgumentException>(() =>
{
request.ConfigureTemporaryStorageLocation(" ");
});
}
private async Task RetrieveViaFhir(InferenceRequest inferenceRequest, RequestInputDataResource source, Dictionary <string, string> retrievedResources)
{
Guard.Against.Null(inferenceRequest, nameof(inferenceRequest));
Guard.Against.Null(retrievedResources, nameof(retrievedResources));
foreach (var input in inferenceRequest.InputMetadata.Inputs)
{
if (input.Resources.IsNullOrEmpty())
{
continue;
}
await RetrieveFhirResources(input, source, retrievedResources, _fileSystem.Path.GetFhirStoragePath(inferenceRequest.StoragePath));
}
}
private async Task BackgroundProcessing(CancellationToken cancellationToken)
{
_logger.Log(LogLevel.Information, "Data Retriever Hosted Service is running.");
while (!cancellationToken.IsCancellationRequested)
{
using var scope = _serviceScopeFactory.CreateScope();
var repository = scope.ServiceProvider.GetRequiredService <IInferenceRequestRepository>();
if (!_storageInfoProvider.HasSpaceAvailableToRetrieve)
{
_logger.Log(LogLevel.Warning, $"Data retrieval paused due to insufficient storage space. Available storage space: {_storageInfoProvider.AvailableFreeSpace:D}.");
continue;
}
InferenceRequest request = null;
try
{
request = await repository.Take(cancellationToken);
using (_logger.BeginScope(new LogginDataDictionary <string, object> {
{ "JobId", request.JobId }, { "TransactionId", request.TransactionId }
}))
{
_logger.Log(LogLevel.Information, "Processing inference request.");
await ProcessRequest(request, cancellationToken);
await repository.Update(request, InferenceRequestStatus.Success);
_logger.Log(LogLevel.Information, "Inference request completed and ready for job submission.");
}
}
catch (OperationCanceledException ex)
{
_logger.Log(LogLevel.Warning, ex, "Data Retriever Service canceled.");
}
catch (InvalidOperationException ex)
{
_logger.Log(LogLevel.Warning, ex, "Data Retriever Service may be disposed.");
}
catch (Exception ex)
{
_logger.Log(LogLevel.Error, ex, $"Error processing request: JobId = {request?.JobId}, TransactionId = {request?.TransactionId}");
if (request != null)
{
await repository.Update(request, InferenceRequestStatus.Fail);
}
}
}
Status = ServiceStatus.Cancelled;
_logger.Log(LogLevel.Information, "Cancellation requested.");
}
public async Task Add_ShallAddJob()
{
var inferenceRequest = new InferenceRequest();
inferenceRequest.JobId = Guid.NewGuid().ToString();
inferenceRequest.PayloadId = Guid.NewGuid().ToString();
inferenceRequest.TransactionId = Guid.NewGuid().ToString();
var store = new InferenceRequestRepository(_logger.Object, _jobsApi.Object, _inferenceRequestRepository.Object);
await store.Add(inferenceRequest);
_inferenceRequestRepository.Verify(p => p.AddAsync(It.IsAny <InferenceRequest>(), It.IsAny <CancellationToken>()), Times.Once());
_inferenceRequestRepository.Verify(p => p.SaveChangesAsync(It.IsAny <CancellationToken>()), Times.Once());
_logger.VerifyLoggingMessageBeginsWith($"Inference request saved.", LogLevel.Debug, Times.Once());
}
public void Algorithm_ShallReturnAlgorithm()
{
var request = new InferenceRequest();
request.InputResources.Add(new RequestInputDataResource {
Interface = InputInterfaceType.DicomWeb
});
request.InputResources.Add(new RequestInputDataResource {
Interface = InputInterfaceType.Algorithm, ConnectionDetails = new InputConnectionDetails()
});
request.InputResources.Add(new RequestInputDataResource {
Interface = InputInterfaceType.Dimse
});
Assert.NotNull(request.Algorithm);
}
public async Task UpdateSuccess_ShallSave()
{
var inferenceRequest = new InferenceRequest();
inferenceRequest.JobId = Guid.NewGuid().ToString();
inferenceRequest.PayloadId = Guid.NewGuid().ToString();
inferenceRequest.TransactionId = Guid.NewGuid().ToString();
var store = new InferenceRequestRepository(_logger.Object, _jobsApi.Object, _inferenceRequestRepository.Object);
await store.Update(inferenceRequest, InferenceRequestStatus.Fail);
_logger.VerifyLogging($"Updating inference request.", LogLevel.Debug, Times.Once());
_logger.VerifyLogging($"Inference request updated.", LogLevel.Information, Times.Once());
_inferenceRequestRepository.Verify(p => p.SaveChangesAsync(It.IsAny <CancellationToken>()), Times.Once());
}
public void NewInferenceRequest_ShallReturnProblemIfInputIsInvalid()
{
var input = new InferenceRequest();
var result = _controller.NewInferenceRequest(input);
Assert.NotNull(result);
var objectResult = result.Result as ObjectResult;
Assert.NotNull(objectResult);
var problem = objectResult.Value as ProblemDetails;
Assert.NotNull(problem);
Assert.Equal("Invalid request", problem.Title);
Assert.Equal(422, problem.Status);
}
public void ClaraJobPriority_ShallReturnMappedValue()
{
var request = new InferenceRequest();
request.Priority = 0;
Assert.Equal(JobPriority.Lower, request.ClaraJobPriority);
request.Priority = 128;
Assert.Equal(JobPriority.Normal, request.ClaraJobPriority);
request.Priority = 250;
Assert.Equal(JobPriority.Higher, request.ClaraJobPriority);
request.Priority = 255;
Assert.Equal(JobPriority.Immediate, request.ClaraJobPriority);
}
private void RestoreExistingResources(InferenceRequest inferenceRequest, Dictionary <string, string> retrievedResources)
{
Guard.Against.Null(inferenceRequest, nameof(inferenceRequest));
Guard.Against.Null(retrievedResources, nameof(retrievedResources));
_logger.Log(LogLevel.Debug, $"Restoring previously retrieved resources from {inferenceRequest.StoragePath}");
foreach (var file in _fileSystem.Directory.EnumerateFiles(inferenceRequest.StoragePath, "*", System.IO.SearchOption.AllDirectories))
{
if (file.EndsWith(".dcm") || retrievedResources.ContainsKey(file))
{
continue;
}
var key = _fileSystem.Path.GetFileName(file);
retrievedResources.Add(key, file);
_logger.Log(LogLevel.Debug, $"Restored previously retrieved resource {key}");
}
}
