private void GenerateRequests( OutputJob job, DicomClient client, CountdownEvent countDownEventHandle) { while (job.PendingDicomFiles.Count > 0) { try { var request = new DicomCStoreRequest(job.PendingDicomFiles.Dequeue()); request.OnResponseReceived += (req, response) => { if (response.Status != DicomStatus.Success) { job.FailureCount++; _logger.Log(LogLevel.Error, $"Failed to export instance {request.File} with error {response.Status}"); } else { job.SuccessfulExport++; _logger.Log(LogLevel.Information, "Instance {0} sent successfully", request.File.FileMetaInfo.MediaStorageSOPInstanceUID.UID); } countDownEventHandle.Signal(); }; client.AddRequestAsync(request).ConfigureAwait(false); } catch (Exception exception) { _logger.LogError("Error while adding DICOM C-STORE request: {0}", exception); } } }
private void HandleCStoreException(Exception ex, OutputJob job, DicomClient client) { var exception = ex; if (exception is AggregateException) { exception = exception.InnerException; } if (exception is DicomAssociationAbortedException abortEx) { _logger.LogError("Association aborted with reason {0}, exception {1}", abortEx.AbortReason, abortEx); } else if (exception is DicomAssociationRejectedException rejectEx) { _logger.LogError("Association rejected with reason {0}, exception {1}", rejectEx.RejectReason, rejectEx); } else if (exception is IOException && exception?.InnerException is SocketException socketException) { _logger.LogError("Association aborted with error {0}, exception {1}", socketException.Message, socketException); } else { _logger.LogError("Job failed with error {0}", exception); } }
protected async Task ReportStatus(OutputJob outputJob, CancellationToken cancellationToken) { using var loggerScope = _logger.BeginScope(new LogginDataDictionary <string, object> { { "JobId", outputJob.JobId }, { "PayloadId", outputJob.PayloadId } }); if (outputJob is null) { return; } using var scope = _serviceScopeFactory.CreateScope(); var resultsService = scope.ServiceProvider.GetRequiredService <IResultsService>(); try { if (outputJob.ExportFailureRate > _dataExportConfiguration.FailureThreshold) { var retry = outputJob.Retries < _dataExportConfiguration.MaximumRetries; await resultsService.ReportFailure(outputJob.TaskId, retry, cancellationToken); _logger.Log(LogLevel.Warning, $"Task marked as failed with failure rate={outputJob.ExportFailureRate}, total={outputJob.Uris.Count()}, failed={outputJob.FailureCount + outputJob.FailedFiles.Count}, processed={outputJob.SuccessfulExport}, retry={retry}"); } else { await resultsService.ReportSuccess(outputJob.TaskId, cancellationToken); _logger.LogInformation("Task marked as successful."); } } catch (Exception ex) { _logger.Log(LogLevel.Error, ex, "Failed to report status back to Results Service."); } }
private async Task ReportingActionBlock(OutputJob outputJob, CancellationToken cancellationToken) { if (ReportActionStarted != null) { ReportActionStarted(this, null); } if (outputJob is null) { return; } using var loggerScope = _logger.BeginScope(new LogginDataDictionary <string, object> { { "JobId", outputJob.JobId }, { "PayloadId", outputJob.PayloadId } }); await ReportStatus(outputJob, cancellationToken); }
private async Task <OutputJob> DownloadPayloadBlockCallback(OutputJob outputJob, CancellationToken cancellationToken) { Guard.Against.Null(outputJob, nameof(outputJob)); using var loggerScope = _logger.BeginScope(new LogginDataDictionary <string, object> { { "JobId", outputJob.JobId }, { "PayloadId", outputJob.PayloadId } }); var scope = _serviceScopeFactory.CreateScope(); var payloadsApi = scope.ServiceProvider.GetRequiredService <IPayloads>(); foreach (var url in outputJob.Uris) { PayloadFile file; try { file = await payloadsApi.Download(outputJob.PayloadId, url); } catch (Exception ex) { _logger.Log(LogLevel.Warning, ex, "Failed to download file {0}.", url); outputJob.FailedFiles.Add(url); outputJob.FailureCount++; continue; } try { var dicom = DicomFile.Open(new MemoryStream(file.Data)); outputJob.PendingDicomFiles.Enqueue(dicom); outputJob.SuccessfulDownload++; } catch (Exception ex) { _logger.Log(LogLevel.Warning, ex, "Ignoring file; not a valid DICOM part-10 file {0}.", url); } } if (outputJob.DownloadFailureRate > _dataExportConfiguration.FailureThreshold) { _logger.Log(LogLevel.Error, "Failure rate exceeded threshold and will not be exported."); await ReportFailure(outputJob, cancellationToken); return(null); } return(outputJob); }
protected override async Task <OutputJob> ExportDataBlockCallback(OutputJob outputJob, CancellationToken cancellationToken) { using var loggerScope = _logger.BeginScope(new LogginDataDictionary <string, object> { { "JobId", outputJob.JobId }, { "PayloadId", outputJob.PayloadId } }); if (outputJob.PendingDicomFiles.Count > 0) { var countDownEventHandle = new CountdownEvent(outputJob.PendingDicomFiles.Count); DicomClient client = null; try { client = new DicomClient( outputJob.HostIp, outputJob.Port, false, _scuConfiguration.AeTitle, outputJob.AeTitle); client.AssociationAccepted += (sender, args) => _logger.LogInformation("Association accepted."); client.AssociationRejected += (sender, args) => _logger.LogInformation("Association rejected."); client.AssociationReleased += (sender, args) => _logger.LogInformation("Association release."); client.Options = new DicomServiceOptions { LogDataPDUs = _scuConfiguration.LogDataPdus, LogDimseDatasets = _scuConfiguration.LogDimseDatasets }; client.NegotiateAsyncOps(); GenerateRequests(outputJob, client, countDownEventHandle); _logger.LogInformation("Sending job to {0}@{1}:{2}", outputJob.AeTitle, outputJob.HostIp, outputJob.Port); await client.SendAsync(cancellationToken).ConfigureAwait(false); countDownEventHandle.Wait(cancellationToken); _logger.LogInformation("Job sent to {0} completed", outputJob.AeTitle); } catch (Exception ex) { HandleCStoreException(ex, outputJob, client); } } return(outputJob); }
protected override async Task <OutputJob> ExportDataBlockCallback(OutputJob outputJob, CancellationToken cancellationToken) { using var loggerScope = _logger.BeginScope(new LogginDataDictionary <string, object> { { "TaskId", outputJob.TaskId }, { "JobId", outputJob.JobId }, { "PayloadId", outputJob.PayloadId } }); using var scope = _serviceScopeFactory.CreateScope(); var repository = scope.ServiceProvider.GetRequiredService <IInferenceRequestRepository>(); var inferenceRequest = repository.Get(outputJob.JobId, outputJob.PayloadId); if (inferenceRequest is null) { _logger.Log(LogLevel.Error, "The specified job cannot be found in the inference request store and will not be exported."); await ReportFailure(outputJob, cancellationToken); return(null); } var destinations = inferenceRequest.OutputResources.Where(p => p.Interface == API.Rest.InputInterfaceType.DicomWeb); if (destinations.Count() == 0) { _logger.Log(LogLevel.Error, "The inference request contains no `outputResources` nor any DICOMweb export destinations."); await ReportFailure(outputJob, cancellationToken); return(null); } foreach (var destination in destinations) { var authenticationHeader = AuthenticationHeaderValueExtensions.ConvertFrom(destination.ConnectionDetails.AuthType, destination.ConnectionDetails.AuthId); var dicomWebClient = new DicomWebClient(_httpClientFactory.CreateClient("dicomweb"), _loggerFactory.CreateLogger <DicomWebClient>()); dicomWebClient.ConfigureServiceUris(new Uri(destination.ConnectionDetails.Uri, UriKind.Absolute)); dicomWebClient.ConfigureAuthentication(authenticationHeader); _logger.Log(LogLevel.Debug, $"Exporting data to {destination.ConnectionDetails.Uri}."); await ExportToDicomWebDestination(dicomWebClient, outputJob, destination, cancellationToken); } return(outputJob); }
protected abstract Task <OutputJob> ExportDataBlockCallback(OutputJob outputJob, CancellationToken cancellationToken);
private async Task ExportToDicomWebDestination(IDicomWebClient dicomWebClient, OutputJob outputJob, API.Rest.RequestOutputDataResource destination, CancellationToken cancellationToken) { while (outputJob.PendingDicomFiles.Count > 0) { var files = new List <DicomFile>(); try { var counter = 10; while (counter-- > 0 && outputJob.PendingDicomFiles.Count > 0) { files.Add(outputJob.PendingDicomFiles.Dequeue()); } var result = await dicomWebClient.Stow.Store(files, cancellationToken); CheckAndLogResult(result); outputJob.SuccessfulExport += files.Count; } catch (Exception ex) { _logger.Log(LogLevel.Error, ex, "Failed to export data to DICOMweb destination."); outputJob.FailureCount += files.Count; } finally { files.Clear(); } } }