/// <summary>
/// Processes the model configuration.
/// </summary>
/// <param name="uploadQueueItem">The upload queue item.</param>
/// <param name="dicomFiles">The dicom files.</param>
/// <param name="queueTransaction">The queue transaction.</param>
/// <param name="clientConfiguration">The client configuration.</param>
/// <returns>The waitable task.</returns>
private async Task ProcessModelConfig(
IEnumerable <DicomFile> dicomFiles,
UploadQueueItem uploadQueueItem,
IQueueTransaction queueTransaction,
ClientAETConfig clientConfiguration)
{
var modelMatchResult = ApplyAETModelConfigProvider.ApplyAETModelConfig(clientConfiguration.Config.ModelsConfig, dicomFiles);
if (modelMatchResult.Matched)
{
var model = modelMatchResult.Result;
var queueItem = await StartSegmentationAsync(model.ChannelData, uploadQueueItem, model.ModelId, model.TagReplacements.ToArray(), clientConfiguration).ConfigureAwait(false);
EnqueueMessage(queueItem, _downloadQueuePath, queueTransaction);
}
else
{
var failedDicomTags = modelMatchResult.GetDicomConstraintsDicomTags();
// Log all the tags that did not match
LogError(LogEntry.Create(AssociationStatus.UploadErrorTagsDoNotMatch,
uploadQueueItem: uploadQueueItem,
failedDicomTags: string.Join(",", failedDicomTags.Select(x => x.DictionaryEntry.Name))),
new ProcessorServiceException("Failed to find a model for the received Dicom data."));
}
}
/// <summary>
/// Uploads the files.
/// </summary>
/// <param name="dicomFiles">The dicom files.</param>
/// <param name="uploadQueueItem">The upload queue item.</param>
/// <param name="queueTransaction">The queue transaction.</param>
/// <returns>The async task.</returns>
private async Task ProcessUploadQueueItem(UploadQueueItem uploadQueueItem, IQueueTransaction queueTransaction)
{
var clientConfiguration = ApplyAETModelConfigProvider.GetAETConfigs(
_aetConfigModels,
uploadQueueItem.CalledApplicationEntityTitle,
uploadQueueItem.CallingApplicationEntityTitle);
LogTrace(LogEntry.Create(AssociationStatus.UploadProcessQueueItem));
switch (clientConfiguration.Config.AETConfigType)
{
// ML Model or ML Model with Dry Run Result
case AETConfigType.Model:
case AETConfigType.ModelWithResultDryRun:
// Load all DICOM files in the received folder.
var dicomFiles = ReadDicomFiles(uploadQueueItem.AssociationFolderPath, uploadQueueItem);
await ProcessModelConfig(dicomFiles, uploadQueueItem, queueTransaction, clientConfiguration).ConfigureAwait(false);
break;
// ML Model dry run
case AETConfigType.ModelDryRun:
// Anonymize and save the files locally for the dry run using the segmentation anonymisation protocol
await AnonymiseAndSaveDicomFilesAsync(
anonymisationProtocolId : _innerEyeSegmentationClient.SegmentationAnonymisationProtocolId,
anonymisationProtocol : _innerEyeSegmentationClient.SegmentationAnonymisationProtocol,
uploadQueueItem : uploadQueueItem,
aETConfigType : clientConfiguration.Config.AETConfigType).ConfigureAwait(false);
break;
}
}
protected async Task <(string SegmentationId, string ModelId, IEnumerable <byte[]> Data)> StartFakeSegmentationAsync(string filesPath)
{
var dicomFiles = new DirectoryInfo(filesPath).GetFiles().Select(x => DicomFile.Open(x.FullName)).ToArray();
using (var segmentationClient = GetMockInnerEyeSegmentationClient())
{
var testAETConfigModel = GetTestAETConfigModel();
var matchedModel = ApplyAETModelConfigProvider.ApplyAETModelConfig(testAETConfigModel.AETConfig.Config.ModelsConfig, dicomFiles);
var modelId = matchedModel.Result.ModelId;
var(segmentationId, postedImages) = await segmentationClient.StartSegmentationAsync(
matchedModel.Result.ModelId,
matchedModel.Result.ChannelData).ConfigureAwait(false);
var referenceDicomFiles = postedImages.CreateNewDicomFileWithoutPixelData(segmentationClient.SegmentationAnonymisationProtocol.Select(x => x.DicomTagIndex.DicomTag));
return(segmentationId, modelId, referenceDicomFiles);
}
}
public async Task IntegrationTestEndToEnd()
{
var sourceDirectory = CreateTemporaryDirectory();
var resultDirectory = CreateTemporaryDirectory();
var random = new Random();
// Get file names for all in directory
var sourceDicomFileNames = new DirectoryInfo(@"Images\HN").GetFiles().ToArray();
// Load all DICOM files
var sourceDicomFiles = sourceDicomFileNames.Select(f => DicomFile.Open(f.FullName, FileReadOption.ReadAll)).ToArray();
// Add/Update random tags for the source DICOM files.
DicomAnonymisationTests.AddRandomTags(random, sourceDicomFiles);
// Save them all to the sourceDirectory.
var sourcePairs = sourceDicomFileNames.Zip(sourceDicomFiles, (f, d) => Tuple.Create(f, d)).ToArray();
foreach (var sourcePair in sourcePairs)
{
var sourceImageFilePath = Path.Combine(sourceDirectory.FullName, sourcePair.Item1.Name);
sourcePair.Item2.Save(sourceImageFilePath);
}
// Keep the first as a reference for deanonymization later.
var originalSlice = sourceDicomFiles.First();
var testAETConfigModel = GetTestAETConfigModel();
var receivePort = 160;
using (var dicomDataReceiver = new ListenerDataReceiver(new ListenerDicomSaver(resultDirectory.FullName)))
using (var deleteService = CreateDeleteService())
using (var pushService = CreatePushService())
using (var downloadService = CreateDownloadService())
using (var uploadService = CreateUploadService())
using (var receiveService = CreateReceiveService(receivePort))
{
// Start a DICOM receiver for the final DICOM-RT file
var eventCount = new ConcurrentDictionary <DicomReceiveProgressCode, int>();
var folderPath = string.Empty;
dicomDataReceiver.DataReceived += (sender, e) =>
{
folderPath = e.FolderPath;
eventCount.AddOrUpdate(e.ProgressCode, 1, (k, v) => v + 1);
};
StartDicomDataReceiver(dicomDataReceiver, testAETConfigModel.AETConfig.Destination.Port);
// Start the services.
deleteService.Start();
pushService.Start();
downloadService.Start();
uploadService.Start();
receiveService.Start();
// Try a DICOM C-ECHO
var dicomDataSender = new DicomDataSender();
var echoResult = await dicomDataSender.DicomEchoAsync(
testAETConfigModel.CallingAET,
testAETConfigModel.CalledAET,
receivePort,
"127.0.0.1").ConfigureAwait(false);
Assert.IsTrue(echoResult == DicomOperationResult.Success);
// Send the image stack
DcmtkHelpers.SendFolderUsingDCMTK(
sourceDirectory.FullName,
receivePort,
ScuProfile.LEExplicitCT,
TestContext,
applicationEntityTitle: testAETConfigModel.CallingAET,
calledAETitle: testAETConfigModel.CalledAET);
// Wait for DICOM-RT file to be received.
Func <DicomReceiveProgressCode, int, bool> TestEventCount = (progressCode, count) =>
eventCount.ContainsKey(progressCode) && eventCount[progressCode] == count;
SpinWait.SpinUntil(() => TestEventCount(DicomReceiveProgressCode.AssociationEstablished, 1));
SpinWait.SpinUntil(() => TestEventCount(DicomReceiveProgressCode.FileReceived, 1));
SpinWait.SpinUntil(() => TestEventCount(DicomReceiveProgressCode.AssociationReleased, 1));
SpinWait.SpinUntil(() => TestEventCount(DicomReceiveProgressCode.ConnectionClosed, 1));
Assert.IsTrue(eventCount[DicomReceiveProgressCode.AssociationEstablished] == 1);
Assert.IsTrue(eventCount[DicomReceiveProgressCode.FileReceived] == 1);
Assert.IsTrue(eventCount[DicomReceiveProgressCode.AssociationReleased] == 1);
Assert.IsTrue(eventCount[DicomReceiveProgressCode.ConnectionClosed] == 1);
var receivedFiles = new DirectoryInfo(folderPath).GetFiles();
Assert.AreEqual(1, receivedFiles.Length);
var receivedFilePath = receivedFiles.First().FullName;
var dicomFile = await DicomFile.OpenAsync(receivedFilePath, FileReadOption.ReadAll).ConfigureAwait(false);
Assert.IsNotNull(dicomFile);
var matchedModel = ApplyAETModelConfigProvider.ApplyAETModelConfig(testAETConfigModel.AETConfig.Config.ModelsConfig, sourceDicomFiles);
var segmentationClient = TestGatewayProcessorConfigProvider.CreateInnerEyeSegmentationClient().Invoke();
DicomAnonymisationTests.AssertDeanonymizedFile(
originalSlice,
dicomFile,
segmentationClient.TopLevelReplacements,
matchedModel.Result.TagReplacements,
false);
AssertIsDicomRtFile(DateTime.UtcNow, dicomFile, matchedModel.Result.ModelId);
}
}
/// <summary>
/// Called when [update tick] is called. This will wait for all work to execute then will pause for desired interval delay.
/// </summary>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>
/// The async task.
/// </returns>
protected override async Task OnUpdateTickAsync(CancellationToken cancellationToken)
{
using (var transaction = CreateQueueTransaction())
{
BeginMessageQueueTransaction(transaction);
PushQueueItem queueItem = null;
try
{
queueItem = await DequeueNextMessageAsync(transaction, cancellationToken).ConfigureAwait(false);
// If we have dequed this item more than once, lets check if the destination
// Dicom endpoint has been updated on the configuration service (or if the destination is null).
if (queueItem.DequeueCount > 1 || queueItem.DestinationApplicationEntity == null)
{
// Refresh the application entity config.
var applicationEntityConfig = ApplyAETModelConfigProvider.GetAETConfigs(
_aetConfigModels,
queueItem.CalledApplicationEntityTitle,
queueItem.CallingApplicationEntityTitle);
if (applicationEntityConfig.Destination == null)
{
var exception = new ArgumentNullException("applicationEntityConfig.Destination",
"The result destination is null. The destination has not been configured.");
LogError(LogEntry.Create(AssociationStatus.PushErrorDestinationEmpty, pushQueueItem: queueItem),
exception);
throw exception;
}
queueItem.DestinationApplicationEntity = new GatewayApplicationEntity(
title: applicationEntityConfig.Destination.Title,
port: applicationEntityConfig.Destination.Port,
ipAddress: applicationEntityConfig.Destination.Ip);
}
if (queueItem.FilePaths.Any())
{
var dicomFiles = ReadDicomFiles(queueItem.FilePaths, queueItem);
await PushDicomFilesAsync(
pushQueueItem : queueItem,
ownApplicationEntityTitle : queueItem.CalledApplicationEntityTitle,
destination : queueItem.DestinationApplicationEntity,
cancellationToken : cancellationToken,
dicomFiles : dicomFiles.ToArray()).ConfigureAwait(false);
}
// Enqueue delete the files.
CleanUp(queueItem, transaction);
transaction.Commit();
}
catch (MessageQueueReadException)
{
// We timed out trying to de-queue (no items on the queue).
// This exception doesn't need to be logged.
transaction.Abort();
}
catch (OperationCanceledException)
{
// Throw operation canceled exceptions up to the worker thread. It will handle
// logging correctly.
transaction.Abort();
throw;
}
#pragma warning disable CA1031 // Do not catch general exception types
catch (Exception e)
#pragma warning restore CA1031 // Do not catch general exception types
{
LogError(LogEntry.Create(AssociationStatus.PushError, pushQueueItem: queueItem),
e);
HandleExceptionForTransaction(
queueItem: queueItem,
queueTransaction: transaction,
oldQueueItemAction: () => CleanUp(queueItem, transaction));
}
}
}
