public void Test_FileCopierHost_HappyPath()
{
GlobalOptions globals = new GlobalOptionsFactory().Load();
globals.FileSystemOptions.FileSystemRoot = "root";
globals.FileSystemOptions.ExtractRoot = "exroot";
using var tester = new MicroserviceTester(globals.RabbitOptions, globals.FileCopierOptions);
string outputQueueName = globals.FileCopierOptions.CopyStatusProducerOptions.ExchangeName.Replace("Exchange", "Queue");
tester.CreateExchange(
globals.FileCopierOptions.CopyStatusProducerOptions.ExchangeName,
outputQueueName,
false,
globals.FileCopierOptions.NoVerifyRoutingKey);
var mockFileSystem = new MockFileSystem();
mockFileSystem.AddDirectory(globals.FileSystemOptions.FileSystemRoot);
mockFileSystem.AddDirectory(globals.FileSystemOptions.ExtractRoot);
mockFileSystem.AddFile(mockFileSystem.Path.Combine(globals.FileSystemOptions.FileSystemRoot, "file.dcm"), MockFileData.NullObject);
var host = new FileCopierHost(globals, mockFileSystem);
tester.StopOnDispose.Add(host);
host.Start();
var message = new ExtractFileMessage
{
ExtractionJobIdentifier = Guid.NewGuid(),
JobSubmittedAt = DateTime.UtcNow,
ProjectNumber = "1234",
ExtractionDirectory = "1234/foo",
DicomFilePath = "file.dcm",
IsIdentifiableExtraction = true,
OutputPath = "output.dcm",
};
tester.SendMessage(globals.FileCopierOptions, message);
using IConnection conn = tester.Factory.CreateConnection();
using IModel model = conn.CreateModel();
var consumer = new EventingBasicConsumer(model);
ExtractedFileStatusMessage statusMessage = null;
consumer.Received += (_, ea) => statusMessage = JsonConvert.DeserializeObject <ExtractedFileStatusMessage>(Encoding.UTF8.GetString(ea.Body.ToArray()));
model.BasicConsume(outputQueueName, true, "", consumer);
new TestTimelineAwaiter().Await(() => statusMessage != null);
Assert.AreEqual(ExtractedFileStatus.Copied, statusMessage.Status);
}
public void TestPersistMessageToStore_ExtractFileStatusMessage()
{
var testExtractJobStore = new TestExtractJobStore();
var message = new ExtractedFileStatusMessage();
var header = new MessageHeader();
message.Status = ExtractedFileStatus.None;
Assert.Throws <ApplicationException>(() => testExtractJobStore.PersistMessageToStore(message, header));
message.Status = ExtractedFileStatus.Anonymised;
Assert.Throws <ApplicationException>(() => testExtractJobStore.PersistMessageToStore(message, header));
message.Status = ExtractedFileStatus.ErrorWontRetry;
testExtractJobStore.PersistMessageToStore(message, header);
}
public void PersistMessageToStore(
[NotNull] ExtractedFileStatusMessage message,
[NotNull] IMessageHeader header)
{
if (message.Status == ExtractedFileStatus.None)
{
throw new ApplicationException("ExtractedFileStatus was None");
}
if (message.Status == ExtractedFileStatus.Anonymised)
{
throw new ApplicationException("Received an anonymisation successful message from the failure queue");
}
PersistMessageToStoreImpl(message, header);
}
public void ProcessMessage(
[NotNull] ExtractFileMessage message,
[NotNull] IMessageHeader header)
{
string fullSrc = _fileSystem.Path.Combine(_fileSystemRoot, message.DicomFilePath);
ExtractedFileStatusMessage statusMessage;
if (!_fileSystem.File.Exists(fullSrc))
{
statusMessage = new ExtractedFileStatusMessage(message)
{
DicomFilePath = message.DicomFilePath,
Status = ExtractedFileStatus.FileMissing,
StatusMessage = $"Could not find '{fullSrc}'"
};
_ = _copyStatusProducerModel.SendMessage(statusMessage, header, _options.NoVerifyRoutingKey);
return;
}
string fullDest = _fileSystem.Path.Combine(_extractionRoot, message.ExtractionDirectory, message.OutputPath);
if (_fileSystem.File.Exists(fullDest))
{
_logger.Warn($"Output file '{fullDest}' already exists. Will overwrite.");
}
IDirectoryInfo parent = _fileSystem.Directory.GetParent(fullDest);
if (!parent.Exists)
{
_logger.Debug($"Creating directory '{parent}'");
parent.Create();
}
_logger.Debug($"Copying source file to '{message.OutputPath}'");
_fileSystem.File.Copy(fullSrc, fullDest, overwrite: true);
statusMessage = new ExtractedFileStatusMessage(message)
{
DicomFilePath = message.DicomFilePath,
Status = ExtractedFileStatus.Copied,
OutputFilePath = message.OutputPath,
};
_ = _copyStatusProducerModel.SendMessage(statusMessage, header, _options.NoVerifyRoutingKey);
}
protected override void PersistMessageToStoreImpl(ExtractedFileStatusMessage message, IMessageHeader header)
{
if (InCompletedJobCollection(message.ExtractionJobIdentifier))
{
throw new ApplicationException("Received an ExtractedFileStatusMessage for a job that is already completed");
}
var newStatus = new MongoFileStatusDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(message.ExtractionJobIdentifier, header, _dateTimeProvider),
message.DicomFilePath,
message.OutputFilePath,
wasAnonymised: false,
isIdentifiable: true,
message.Status,
statusMessage: message.StatusMessage);
_database
.GetCollection <MongoFileStatusDoc>(StatusCollectionName(message.ExtractionJobIdentifier))
.InsertOne(newStatus);
}
public void TestPersistMessageToStoreImpl_ExtractFileStatusMessage()
{
var client = new TestMongoClient();
var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
Guid jobId = Guid.NewGuid();
var testExtractFileStatusMessage = new ExtractedFileStatusMessage
{
OutputFilePath = "anon.dcm",
JobSubmittedAt = _dateTimeProvider.UtcNow(),
Status = ExtractedFileStatus.ErrorWontRetry,
ProjectNumber = "1234",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = "1234/test",
StatusMessage = "Could not anonymise",
DicomFilePath = "original.dcm",
};
var header = new MessageHeader();
store.PersistMessageToStore(testExtractFileStatusMessage, header);
Dictionary <Guid, MongoFileStatusDoc> docs = client.ExtractionDatabase.StatusCollections[$"statuses_{jobId}"].Documents;
Assert.AreEqual(docs.Count, 1);
MongoFileStatusDoc statusDoc = docs.Values.ToList()[0];
var expected = new MongoFileStatusDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, header, _dateTimeProvider),
"original.dcm",
"anon.dcm",
false,
true,
ExtractedFileStatus.ErrorWontRetry,
"Could not anonymise");
Assert.True(statusDoc.Equals(expected));
}
public void Integration_IdentifiableExtraction_HappyPath()
{
using var pf = new PathFixtures("Integration_IdentifiableExtraction_HappyPath");
var jobId = Guid.NewGuid();
var testExtractionRequestInfoMessage = new ExtractionRequestInfoMessage
{
ExtractionModality = "MR",
JobSubmittedAt = _dateTimeProvider.UtcNow(),
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
KeyTag = "StudyInstanceUID",
KeyValueCount = 1,
IsIdentifiableExtraction = true,
};
var testExtractFileCollectionInfoMessage = new ExtractFileCollectionInfoMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
ExtractFileMessagesDispatched = new JsonCompatibleDictionary <MessageHeader, string>
{
{ new MessageHeader(), "out1.dcm" },
{ new MessageHeader(), "out2.dcm" },
},
RejectionReasons = new Dictionary <string, int>
{
{ "rejected - blah", 1 },
},
KeyValue = "study-1",
IsIdentifiableExtraction = true,
};
var testExtractFileStatusMessage1 = new ExtractedFileStatusMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
OutputFilePath = "src.dcm",
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
Status = ExtractedFileStatus.Copied,
StatusMessage = null,
DicomFilePath = "study-1-orig-1.dcm",
IsIdentifiableExtraction = true,
};
var testExtractFileStatusMessage2 = new ExtractedFileStatusMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
OutputFilePath = "src_missing.dcm",
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
Status = ExtractedFileStatus.FileMissing,
StatusMessage = null,
DicomFilePath = "study-1-orig-2.dcm",
IsIdentifiableExtraction = true,
};
GlobalOptions globals = new GlobalOptionsFactory().Load();
VerifyReports(
globals,
pf,
ReportFormat.Combined,
new[]
{
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.ExtractRequestInfoOptions, testExtractionRequestInfoMessage),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.FileCollectionInfoOptions, testExtractFileCollectionInfoMessage),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.NoVerifyStatusOptions, testExtractFileStatusMessage1),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.NoVerifyStatusOptions, testExtractFileStatusMessage2),
}
);
}
public void Integration_BumpyRoad(ReportFormat reportFormat)
{
// Test messages:
// - series-1
// - series-1-anon-1.dcm -> valid
// - series-1-anon-2.dcm -> rejected
// - series-2
// - series-2-anon-1.dcm -> fails anonymisation
// - series-2-anon-2.dcm -> fails validation
using var pf = new PathFixtures($"Integration_BumpyRoad_{reportFormat}");
var jobId = Guid.NewGuid();
var testExtractionRequestInfoMessage = new ExtractionRequestInfoMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
KeyTag = "SeriesInstanceUID",
KeyValueCount = 2,
};
var testExtractFileCollectionInfoMessage1 = new ExtractFileCollectionInfoMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
ExtractFileMessagesDispatched = new JsonCompatibleDictionary <MessageHeader, string>
{
{ new MessageHeader(), "series-1-anon-1.dcm" },
},
RejectionReasons = new Dictionary <string, int>
{
{ "rejected - blah", 1 },
},
KeyValue = "series-1",
};
var testExtractFileCollectionInfoMessage2 = new ExtractFileCollectionInfoMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
ExtractFileMessagesDispatched = new JsonCompatibleDictionary <MessageHeader, string>
{
{ new MessageHeader(), "series-2-anon-1.dcm" },
{ new MessageHeader(), "series-2-anon-2.dcm" },
},
RejectionReasons = new Dictionary <string, int>(),
KeyValue = "series-2",
};
var testExtractFileStatusMessage = new ExtractedFileStatusMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
OutputFilePath = "series-2-anon-1.dcm",
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
Status = ExtractedFileStatus.ErrorWontRetry,
StatusMessage = "Couldn't anonymise",
DicomFilePath = "series-2-orig-1.dcm",
};
var testIsIdentifiableMessage1 = new ExtractedFileVerificationMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
OutputFilePath = "series-1-anon-1.dcm",
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
IsIdentifiable = false,
Report = "[]",
DicomFilePath = "series-1-orig-1.dcm",
};
const string failureReport = @"
[
{
'Parts': [],
'Resource': 'series-2-anon-2.dcm',
'ResourcePrimaryKey': '1.2.3.4',
'ProblemField': 'ScanOptions',
'ProblemValue': 'FOO'
}
]";
var testIsIdentifiableMessage2 = new ExtractedFileVerificationMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
OutputFilePath = "series-2-anon-2.dcm",
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
IsIdentifiable = true,
Report = failureReport,
DicomFilePath = "series-2-orig-2.dcm",
};
GlobalOptions globals = new GlobalOptionsFactory().Load();
VerifyReports(
globals,
pf,
reportFormat,
new[]
{
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.ExtractRequestInfoOptions, testExtractionRequestInfoMessage),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.FileCollectionInfoOptions, testExtractFileCollectionInfoMessage1),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.FileCollectionInfoOptions, testExtractFileCollectionInfoMessage2),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.NoVerifyStatusOptions, testExtractFileStatusMessage),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.VerificationStatusOptions, testIsIdentifiableMessage1),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.VerificationStatusOptions, testIsIdentifiableMessage2),
}
);
}
protected override void PersistMessageToStoreImpl(ExtractedFileStatusMessage message, IMessageHeader header)
{
}
protected abstract void PersistMessageToStoreImpl(ExtractedFileStatusMessage message, IMessageHeader header);
