public void TestToExtractJobInfo()
{
Guid guid = Guid.NewGuid();
var message = new ExtractionRequestInfoMessage
{
ExtractionModality = "MR",
JobSubmittedAt = _dateTimeProvider.UtcNow(),
ProjectNumber = "1234",
ExtractionJobIdentifier = guid,
ExtractionDirectory = "test/directory",
KeyTag = "KeyTag",
KeyValueCount = 123,
IsIdentifiableExtraction = true,
IsNoFilterExtraction = true,
};
MongoExtractJobDoc doc = MongoExtractJobDoc.FromMessage(message, _messageHeader, _dateTimeProvider);
ExtractJobInfo extractJobInfo = doc.ToExtractJobInfo();
var expected = new ExtractJobInfo(
guid,
_dateTimeProvider.UtcNow(),
"1234",
"test/directory",
"KeyTag",
123,
"MR",
ExtractJobStatus.WaitingForCollectionInfo,
isIdentifiableExtraction: true,
isNoFilterExtraction: true
);
Assert.AreEqual(expected, extractJobInfo);
}
public void Integration_HappyPath(ReportFormat reportFormat)
{
// Test messages:
// - series-1
// - series-1-anon-1.dcm -> valid
using var pf = new PathFixtures($"Integration_HappyPath_{reportFormat}");
var jobId = Guid.NewGuid();
var testExtractionRequestInfoMessage = new ExtractionRequestInfoMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
KeyTag = "SeriesInstanceUID",
KeyValueCount = 1,
};
var testExtractFileCollectionInfoMessage = new ExtractFileCollectionInfoMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
ExtractFileMessagesDispatched = new JsonCompatibleDictionary <MessageHeader, string>
{
{ new MessageHeader(), "series-1-anon-1.dcm" },
},
RejectionReasons = new Dictionary <string, int>
{
{ "rejected - blah", 1 },
},
KeyValue = "series-1",
};
var testIsIdentifiableMessage = new ExtractedFileVerificationMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
OutputFilePath = "series-1-anon-1.dcm",
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
IsIdentifiable = false,
Report = "[]",
DicomFilePath = "series-1-orig-1.dcm",
};
GlobalOptions globals = new GlobalOptionsFactory().Load();
VerifyReports(
globals,
pf,
reportFormat,
new[]
{
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.ExtractRequestInfoOptions, testExtractionRequestInfoMessage),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.FileCollectionInfoOptions, testExtractFileCollectionInfoMessage),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.VerificationStatusOptions, testIsIdentifiableMessage),
}
);
}
protected override void PersistMessageToStoreImpl(ExtractionRequestInfoMessage message, IMessageHeader header)
{
if (InCompletedJobCollection(message.ExtractionJobIdentifier))
{
throw new ApplicationException("Received an ExtractionRequestInfoMessage for a job that is already completed");
}
MongoExtractJobDoc newJobInfo = MongoExtractJobDoc.FromMessage(message, header, _dateTimeProvider);
_inProgressJobCollection.InsertOne(newJobInfo);
}
public void TestPersistMessageToStoreImpl_ExtractionRequestInfoMessage()
{
Guid guid = Guid.NewGuid();
var testExtractionRequestInfoMessage = new ExtractionRequestInfoMessage
{
ExtractionJobIdentifier = guid,
ProjectNumber = "1234-5678",
ExtractionDirectory = "1234-5678/testExtract",
JobSubmittedAt = _dateTimeProvider.UtcNow(),
KeyTag = "StudyInstanceUID",
KeyValueCount = 1,
ExtractionModality = "CT",
IsIdentifiableExtraction = true,
IsNoFilterExtraction = true,
};
var testHeader = new MessageHeader
{
MessageGuid = Guid.NewGuid(),
OriginalPublishTimestamp = MessageHeader.UnixTime(_dateTimeProvider.UtcNow()),
Parents = new[] { Guid.NewGuid(), },
ProducerExecutableName = "MongoExtractStoreTests",
ProducerProcessID = 1234,
};
var client = new TestMongoClient();
var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
store.PersistMessageToStore(testExtractionRequestInfoMessage, testHeader);
Dictionary <Guid, MongoExtractJobDoc> docs = client.ExtractionDatabase.InProgressCollection.Documents;
Assert.AreEqual(docs.Count, 1);
MongoExtractJobDoc extractJob = docs.Values.ToList()[0];
var expected = new MongoExtractJobDoc(
guid,
MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, testHeader, _dateTimeProvider),
"1234-5678",
ExtractJobStatus.WaitingForCollectionInfo,
"1234-5678/testExtract",
_dateTimeProvider.UtcNow(),
"StudyInstanceUID",
1,
"CT",
isIdentifiableExtraction: true,
isNoFilterExtraction: true,
null);
Assert.AreEqual(expected, extractJob);
}
public void PersistMessageToStore(
[NotNull] ExtractionRequestInfoMessage message,
[NotNull] IMessageHeader header)
{
Logger.Info($"Received new job info {message}");
// If KeyTag is StudyInstanceUID then ExtractionModality must be specified, otherwise must be null
if (message.KeyTag == "StudyInstanceUID" ^ !string.IsNullOrWhiteSpace(message.ExtractionModality))
{
throw new ApplicationException($"Invalid combination of KeyTag and ExtractionModality (KeyTag={message.KeyTag}, ExtractionModality={message.ExtractionModality}");
}
PersistMessageToStoreImpl(message, header);
}
public static MongoExtractJobDoc FromMessage(
[NotNull] ExtractionRequestInfoMessage message,
[NotNull] IMessageHeader header,
[NotNull] DateTimeProvider dateTimeProvider)
{
return new MongoExtractJobDoc(
message.ExtractionJobIdentifier,
MongoExtractionMessageHeaderDoc.FromMessageHeader(message.ExtractionJobIdentifier, header, dateTimeProvider),
message.ProjectNumber,
ExtractJobStatus.WaitingForCollectionInfo,
message.ExtractionDirectory,
message.JobSubmittedAt,
message.KeyTag,
(uint)message.KeyValueCount,
message.ExtractionModality,
message.IsIdentifiableExtraction,
message.IsNoFilterExtraction,
null
);
}
public void TestPersistMessageToStore_ExtractionRequestInfoMessage()
{
var testExtractJobStore = new TestExtractJobStore();
var message = new ExtractionRequestInfoMessage();
var mockHeader = new MessageHeader();
message.KeyTag = "SeriesInstanceUID";
message.ExtractionModality = null;
testExtractJobStore.PersistMessageToStore(message, mockHeader);
message.KeyTag = "StudyInstanceUID";
message.ExtractionModality = "MR";
testExtractJobStore.PersistMessageToStore(message, mockHeader);
message.KeyTag = "StudyInstanceUID";
message.ExtractionModality = null;
Assert.Throws <ApplicationException>(() => testExtractJobStore.PersistMessageToStore(message, mockHeader));
message.KeyTag = "SeriesInstanceUID";
message.ExtractionModality = "MR";
Assert.Throws <ApplicationException>(() => testExtractJobStore.PersistMessageToStore(message, mockHeader));
}
public void Integration_IdentifiableExtraction_HappyPath()
{
using var pf = new PathFixtures("Integration_IdentifiableExtraction_HappyPath");
var jobId = Guid.NewGuid();
var testExtractionRequestInfoMessage = new ExtractionRequestInfoMessage
{
ExtractionModality = "MR",
JobSubmittedAt = _dateTimeProvider.UtcNow(),
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
KeyTag = "StudyInstanceUID",
KeyValueCount = 1,
IsIdentifiableExtraction = true,
};
var testExtractFileCollectionInfoMessage = new ExtractFileCollectionInfoMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
ExtractFileMessagesDispatched = new JsonCompatibleDictionary <MessageHeader, string>
{
{ new MessageHeader(), "out1.dcm" },
{ new MessageHeader(), "out2.dcm" },
},
RejectionReasons = new Dictionary <string, int>
{
{ "rejected - blah", 1 },
},
KeyValue = "study-1",
IsIdentifiableExtraction = true,
};
var testExtractFileStatusMessage1 = new ExtractedFileStatusMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
OutputFilePath = "src.dcm",
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
Status = ExtractedFileStatus.Copied,
StatusMessage = null,
DicomFilePath = "study-1-orig-1.dcm",
IsIdentifiableExtraction = true,
};
var testExtractFileStatusMessage2 = new ExtractedFileStatusMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
OutputFilePath = "src_missing.dcm",
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
Status = ExtractedFileStatus.FileMissing,
StatusMessage = null,
DicomFilePath = "study-1-orig-2.dcm",
IsIdentifiableExtraction = true,
};
GlobalOptions globals = new GlobalOptionsFactory().Load();
VerifyReports(
globals,
pf,
ReportFormat.Combined,
new[]
{
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.ExtractRequestInfoOptions, testExtractionRequestInfoMessage),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.FileCollectionInfoOptions, testExtractFileCollectionInfoMessage),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.NoVerifyStatusOptions, testExtractFileStatusMessage1),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.NoVerifyStatusOptions, testExtractFileStatusMessage2),
}
);
}
public void Integration_BumpyRoad(ReportFormat reportFormat)
{
// Test messages:
// - series-1
// - series-1-anon-1.dcm -> valid
// - series-1-anon-2.dcm -> rejected
// - series-2
// - series-2-anon-1.dcm -> fails anonymisation
// - series-2-anon-2.dcm -> fails validation
using var pf = new PathFixtures($"Integration_BumpyRoad_{reportFormat}");
var jobId = Guid.NewGuid();
var testExtractionRequestInfoMessage = new ExtractionRequestInfoMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
KeyTag = "SeriesInstanceUID",
KeyValueCount = 2,
};
var testExtractFileCollectionInfoMessage1 = new ExtractFileCollectionInfoMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
ExtractFileMessagesDispatched = new JsonCompatibleDictionary <MessageHeader, string>
{
{ new MessageHeader(), "series-1-anon-1.dcm" },
},
RejectionReasons = new Dictionary <string, int>
{
{ "rejected - blah", 1 },
},
KeyValue = "series-1",
};
var testExtractFileCollectionInfoMessage2 = new ExtractFileCollectionInfoMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
ExtractFileMessagesDispatched = new JsonCompatibleDictionary <MessageHeader, string>
{
{ new MessageHeader(), "series-2-anon-1.dcm" },
{ new MessageHeader(), "series-2-anon-2.dcm" },
},
RejectionReasons = new Dictionary <string, int>(),
KeyValue = "series-2",
};
var testExtractFileStatusMessage = new ExtractedFileStatusMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
OutputFilePath = "series-2-anon-1.dcm",
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
Status = ExtractedFileStatus.ErrorWontRetry,
StatusMessage = "Couldn't anonymise",
DicomFilePath = "series-2-orig-1.dcm",
};
var testIsIdentifiableMessage1 = new ExtractedFileVerificationMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
OutputFilePath = "series-1-anon-1.dcm",
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
IsIdentifiable = false,
Report = "[]",
DicomFilePath = "series-1-orig-1.dcm",
};
const string failureReport = @"
[
{
'Parts': [],
'Resource': 'series-2-anon-2.dcm',
'ResourcePrimaryKey': '1.2.3.4',
'ProblemField': 'ScanOptions',
'ProblemValue': 'FOO'
}
]";
var testIsIdentifiableMessage2 = new ExtractedFileVerificationMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
OutputFilePath = "series-2-anon-2.dcm",
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
IsIdentifiable = true,
Report = failureReport,
DicomFilePath = "series-2-orig-2.dcm",
};
GlobalOptions globals = new GlobalOptionsFactory().Load();
VerifyReports(
globals,
pf,
reportFormat,
new[]
{
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.ExtractRequestInfoOptions, testExtractionRequestInfoMessage),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.FileCollectionInfoOptions, testExtractFileCollectionInfoMessage1),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.FileCollectionInfoOptions, testExtractFileCollectionInfoMessage2),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.NoVerifyStatusOptions, testExtractFileStatusMessage),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.VerificationStatusOptions, testIsIdentifiableMessage1),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.VerificationStatusOptions, testIsIdentifiableMessage2),
}
);
}
protected override void PersistMessageToStoreImpl(ExtractionRequestInfoMessage message, IMessageHeader header)
{
}
protected abstract void PersistMessageToStoreImpl(ExtractionRequestInfoMessage message, IMessageHeader header);