private void GenerateRequests(
OutputJob job,
DicomClient client,
CountdownEvent countDownEventHandle)
{
while (job.PendingDicomFiles.Count > 0)
{
try
{
var request = new DicomCStoreRequest(job.PendingDicomFiles.Dequeue());
request.OnResponseReceived += (req, response) =>
{
if (response.Status != DicomStatus.Success)
{
job.FailureCount++;
_logger.Log(LogLevel.Error, $"Failed to export instance {request.File} with error {response.Status}");
}
else
{
job.SuccessfulExport++;
_logger.Log(LogLevel.Information, "Instance {0} sent successfully", request.File.FileMetaInfo.MediaStorageSOPInstanceUID.UID);
}
countDownEventHandle.Signal();
};
client.AddRequestAsync(request).ConfigureAwait(false);
}
catch (Exception exception)
{
_logger.LogError("Error while adding DICOM C-STORE request: {0}", exception);
}
}
}
private void HandleCStoreException(Exception ex, OutputJob job, DicomClient client)
{
var exception = ex;
if (exception is AggregateException)
{
exception = exception.InnerException;
}
if (exception is DicomAssociationAbortedException abortEx)
{
_logger.LogError("Association aborted with reason {0}, exception {1}", abortEx.AbortReason, abortEx);
}
else if (exception is DicomAssociationRejectedException rejectEx)
{
_logger.LogError("Association rejected with reason {0}, exception {1}", rejectEx.RejectReason, rejectEx);
}
else if (exception is IOException && exception?.InnerException is SocketException socketException)
{
_logger.LogError("Association aborted with error {0}, exception {1}", socketException.Message, socketException);
}
else
{
_logger.LogError("Job failed with error {0}", exception);
}
}
protected async Task ReportStatus(OutputJob outputJob, CancellationToken cancellationToken)
{
using var loggerScope = _logger.BeginScope(new LogginDataDictionary <string, object> { { "JobId", outputJob.JobId }, { "PayloadId", outputJob.PayloadId } });
if (outputJob is null)
{
return;
}
using var scope = _serviceScopeFactory.CreateScope();
var resultsService = scope.ServiceProvider.GetRequiredService <IResultsService>();
try
{
if (outputJob.ExportFailureRate > _dataExportConfiguration.FailureThreshold)
{
var retry = outputJob.Retries < _dataExportConfiguration.MaximumRetries;
await resultsService.ReportFailure(outputJob.TaskId, retry, cancellationToken);
_logger.Log(LogLevel.Warning,
$"Task marked as failed with failure rate={outputJob.ExportFailureRate}, total={outputJob.Uris.Count()}, failed={outputJob.FailureCount + outputJob.FailedFiles.Count}, processed={outputJob.SuccessfulExport}, retry={retry}");
}
else
{
await resultsService.ReportSuccess(outputJob.TaskId, cancellationToken);
_logger.LogInformation("Task marked as successful.");
}
}
catch (Exception ex)
{
_logger.Log(LogLevel.Error, ex, "Failed to report status back to Results Service.");
}
}
private async Task ReportingActionBlock(OutputJob outputJob, CancellationToken cancellationToken)
{
if (ReportActionStarted != null)
{
ReportActionStarted(this, null);
}
if (outputJob is null)
{
return;
}
using var loggerScope = _logger.BeginScope(new LogginDataDictionary <string, object> { { "JobId", outputJob.JobId }, { "PayloadId", outputJob.PayloadId } });
await ReportStatus(outputJob, cancellationToken);
}
private async Task <OutputJob> DownloadPayloadBlockCallback(OutputJob outputJob, CancellationToken cancellationToken)
{
Guard.Against.Null(outputJob, nameof(outputJob));
using var loggerScope = _logger.BeginScope(new LogginDataDictionary <string, object> { { "JobId", outputJob.JobId }, { "PayloadId", outputJob.PayloadId } });
var scope = _serviceScopeFactory.CreateScope();
var payloadsApi = scope.ServiceProvider.GetRequiredService <IPayloads>();
foreach (var url in outputJob.Uris)
{
PayloadFile file;
try
{
file = await payloadsApi.Download(outputJob.PayloadId, url);
}
catch (Exception ex)
{
_logger.Log(LogLevel.Warning, ex, "Failed to download file {0}.", url);
outputJob.FailedFiles.Add(url);
outputJob.FailureCount++;
continue;
}
try
{
var dicom = DicomFile.Open(new MemoryStream(file.Data));
outputJob.PendingDicomFiles.Enqueue(dicom);
outputJob.SuccessfulDownload++;
}
catch (Exception ex)
{
_logger.Log(LogLevel.Warning, ex, "Ignoring file; not a valid DICOM part-10 file {0}.", url);
}
}
if (outputJob.DownloadFailureRate > _dataExportConfiguration.FailureThreshold)
{
_logger.Log(LogLevel.Error, "Failure rate exceeded threshold and will not be exported.");
await ReportFailure(outputJob, cancellationToken);
return(null);
}
return(outputJob);
}
protected override async Task <OutputJob> ExportDataBlockCallback(OutputJob outputJob, CancellationToken cancellationToken)
{
using var loggerScope = _logger.BeginScope(new LogginDataDictionary <string, object> { { "JobId", outputJob.JobId }, { "PayloadId", outputJob.PayloadId } });
if (outputJob.PendingDicomFiles.Count > 0)
{
var countDownEventHandle = new CountdownEvent(outputJob.PendingDicomFiles.Count);
DicomClient client = null;
try
{
client = new DicomClient(
outputJob.HostIp,
outputJob.Port,
false,
_scuConfiguration.AeTitle,
outputJob.AeTitle);
client.AssociationAccepted += (sender, args) => _logger.LogInformation("Association accepted.");
client.AssociationRejected += (sender, args) => _logger.LogInformation("Association rejected.");
client.AssociationReleased += (sender, args) => _logger.LogInformation("Association release.");
client.Options = new DicomServiceOptions
{
LogDataPDUs = _scuConfiguration.LogDataPdus,
LogDimseDatasets = _scuConfiguration.LogDimseDatasets
};
client.NegotiateAsyncOps();
GenerateRequests(outputJob, client, countDownEventHandle);
_logger.LogInformation("Sending job to {0}@{1}:{2}", outputJob.AeTitle, outputJob.HostIp, outputJob.Port);
await client.SendAsync(cancellationToken).ConfigureAwait(false);
countDownEventHandle.Wait(cancellationToken);
_logger.LogInformation("Job sent to {0} completed", outputJob.AeTitle);
}
catch (Exception ex)
{
HandleCStoreException(ex, outputJob, client);
}
}
return(outputJob);
}
protected override async Task <OutputJob> ExportDataBlockCallback(OutputJob outputJob, CancellationToken cancellationToken)
{
using var loggerScope = _logger.BeginScope(new LogginDataDictionary <string, object> { { "TaskId", outputJob.TaskId }, { "JobId", outputJob.JobId }, { "PayloadId", outputJob.PayloadId } });
using var scope = _serviceScopeFactory.CreateScope();
var repository = scope.ServiceProvider.GetRequiredService <IInferenceRequestRepository>();
var inferenceRequest = repository.Get(outputJob.JobId, outputJob.PayloadId);
if (inferenceRequest is null)
{
_logger.Log(LogLevel.Error, "The specified job cannot be found in the inference request store and will not be exported.");
await ReportFailure(outputJob, cancellationToken);
return(null);
}
var destinations = inferenceRequest.OutputResources.Where(p => p.Interface == API.Rest.InputInterfaceType.DicomWeb);
if (destinations.Count() == 0)
{
_logger.Log(LogLevel.Error, "The inference request contains no `outputResources` nor any DICOMweb export destinations.");
await ReportFailure(outputJob, cancellationToken);
return(null);
}
foreach (var destination in destinations)
{
var authenticationHeader = AuthenticationHeaderValueExtensions.ConvertFrom(destination.ConnectionDetails.AuthType, destination.ConnectionDetails.AuthId);
var dicomWebClient = new DicomWebClient(_httpClientFactory.CreateClient("dicomweb"), _loggerFactory.CreateLogger <DicomWebClient>());
dicomWebClient.ConfigureServiceUris(new Uri(destination.ConnectionDetails.Uri, UriKind.Absolute));
dicomWebClient.ConfigureAuthentication(authenticationHeader);
_logger.Log(LogLevel.Debug, $"Exporting data to {destination.ConnectionDetails.Uri}.");
await ExportToDicomWebDestination(dicomWebClient, outputJob, destination, cancellationToken);
}
return(outputJob);
}
protected abstract Task <OutputJob> ExportDataBlockCallback(OutputJob outputJob, CancellationToken cancellationToken);
private async Task ExportToDicomWebDestination(IDicomWebClient dicomWebClient, OutputJob outputJob, API.Rest.RequestOutputDataResource destination, CancellationToken cancellationToken)
{
while (outputJob.PendingDicomFiles.Count > 0)
{
var files = new List <DicomFile>();
try
{
var counter = 10;
while (counter-- > 0 && outputJob.PendingDicomFiles.Count > 0)
{
files.Add(outputJob.PendingDicomFiles.Dequeue());
}
var result = await dicomWebClient.Stow.Store(files, cancellationToken);
CheckAndLogResult(result);
outputJob.SuccessfulExport += files.Count;
}
catch (Exception ex)
{
_logger.Log(LogLevel.Error, ex, "Failed to export data to DICOMweb destination.");
outputJob.FailureCount += files.Count;
}
finally
{
files.Clear();
}
}
}