public void TestToExtractJobInfo()
{
Guid guid = Guid.NewGuid();
var message = new ExtractionRequestInfoMessage
{
ExtractionModality = "MR",
JobSubmittedAt = _dateTimeProvider.UtcNow(),
ProjectNumber = "1234",
ExtractionJobIdentifier = guid,
ExtractionDirectory = "test/directory",
KeyTag = "KeyTag",
KeyValueCount = 123,
IsIdentifiableExtraction = true,
IsNoFilterExtraction = true,
};
MongoExtractJobDoc doc = MongoExtractJobDoc.FromMessage(message, _messageHeader, _dateTimeProvider);
ExtractJobInfo extractJobInfo = doc.ToExtractJobInfo();
var expected = new ExtractJobInfo(
guid,
_dateTimeProvider.UtcNow(),
"1234",
"test/directory",
"KeyTag",
123,
"MR",
ExtractJobStatus.WaitingForCollectionInfo,
isIdentifiableExtraction: true,
isNoFilterExtraction: true
);
Assert.AreEqual(expected, extractJobInfo);
}
public void TestExtractJobInfo_Equality()
{
Guid guid = Guid.NewGuid();
var info1 = new ExtractJobInfo(
guid,
_dateTimeProvider.UtcNow(),
"1234",
"test/directory",
"KeyTag",
123,
"MR",
ExtractJobStatus.WaitingForCollectionInfo,
isIdentifiableExtraction: true,
isNoFilterExtraction: true
);
var info2 = new ExtractJobInfo(
guid,
_dateTimeProvider.UtcNow(),
"1234",
"test/directory",
"KeyTag",
123,
"MR",
ExtractJobStatus.WaitingForCollectionInfo,
isIdentifiableExtraction: true,
isNoFilterExtraction: true
);
Assert.AreEqual(info1, info2);
}
public void TestProcessJobs()
{
Guid jobId = Guid.NewGuid();
var testJobInfo = new ExtractJobInfo(
jobId,
DateTime.UtcNow,
"123",
"test/dir",
"KeyTag",
123,
null,
ExtractJobStatus.ReadyForChecks,
isIdentifiableExtraction: true,
isNoFilterExtraction: true
);
var opts = new CohortPackagerOptions {
JobWatcherTimeoutInSeconds = 123
};
var mockJobStore = new Mock <IExtractJobStore>();
var callbackUsed = false;
var mockCallback = new Action <Exception>(_ => callbackUsed = true);
var testNotifier = new TestJobCompleteNotifier();
var testReporter = new TestJobReporter();
var watcher = new ExtractJobWatcher(opts, mockJobStore.Object, mockCallback, testNotifier, testReporter);
// Check that we can call ProcessJobs with no Guid to process all jobs
mockJobStore.Setup(x => x.GetReadyJobs(default(Guid))).Returns(new List <ExtractJobInfo>());
watcher.ProcessJobs();
mockJobStore.Verify();
// Check that we MarkJobFailed for known exceptions
mockJobStore.Reset();
mockJobStore.Setup(x => x.GetReadyJobs(It.IsAny <Guid>())).Returns(new List <ExtractJobInfo> {
testJobInfo
});
mockJobStore.Setup(x => x.MarkJobCompleted(It.IsAny <Guid>())).Throws(new ApplicationException("aah"));
watcher.ProcessJobs(jobId);
mockJobStore.Verify(x => x.MarkJobFailed(jobId, It.IsAny <ApplicationException>()), Times.Once);
// Check that we call the exception callback for unhandled exceptions
mockJobStore.Reset();
mockJobStore.Setup(x => x.GetReadyJobs(It.IsAny <Guid>())).Returns(new List <ExtractJobInfo> {
testJobInfo
});
mockJobStore.Setup(x => x.MarkJobCompleted(It.IsAny <Guid>())).Throws(new Exception("aah"));
watcher.ProcessJobs(jobId);
Assert.True(callbackUsed);
// Check happy path
mockJobStore.Reset();
mockJobStore.Setup(x => x.GetReadyJobs(It.IsAny <Guid>())).Returns(new List <ExtractJobInfo> {
testJobInfo
});
testNotifier.Notified = false;
watcher.ProcessJobs(jobId);
Assert.True(testNotifier.Notified);
Assert.True(testReporter.Reported);
}
private string AbsolutePathToProjReportsDir(ExtractJobInfo jobInfo)
=>
_fileSystem.Path.Combine(
_extractRoot,
jobInfo.ProjectExtractionDir(),
"reports"
);
private Stream GetStream(ExtractJobInfo jobInfo, string fileName)
{
if (ShouldWriteCombinedReport(jobInfo))
{
return(_currentFileStream);
}
string absReportPath = _fileSystem.Path.Combine(AbsolutePathToProjExtractionReportsDir(jobInfo), fileName);
return(_fileSystem.File.OpenWrite(absReportPath));
}
protected bool ShouldWriteCombinedReport(ExtractJobInfo jobInfo)
{
if (ReportFormat == ReportFormat.Combined || jobInfo.IsIdentifiableExtraction)
{
return(true);
}
if (ReportFormat == ReportFormat.Split)
{
return(false);
}
throw new ApplicationException($"No case for report format '{ReportFormat}'");
}
public void Test_ExtractJobInfo_ProjectExtractionDir(string extractionDir, string expected)
{
var info = new ExtractJobInfo(
Guid.NewGuid(),
DateTime.UtcNow,
"1234",
extractionDir,
"KeyTag",
123,
"MR",
ExtractJobStatus.WaitingForCollectionInfo,
isIdentifiableExtraction: false,
isNoFilterExtraction: false
);
Assert.AreEqual(expected, info.ProjectExtractionDir());
}
protected override Stream GetStreamForSummary(ExtractJobInfo jobInfo)
{
_currentFileStream = null;
Stream fileStream;
if (_createJobIdFile)
{
// Write the jobId to a file in the extraction dir to help identify the set of files if they are moved
string jobIdFile = _fileSystem.Path.Combine(_extractRoot, jobInfo.ExtractionDirectory, "jobId.txt");
_fileSystem.File.WriteAllText(jobIdFile, string.Join(ReportNewLine, jobInfo.ExtractionJobIdentifier));
}
string projReportsDirAbsolute = AbsolutePathToProjReportsDir(jobInfo);
Directory.CreateDirectory(projReportsDirAbsolute);
if (ShouldWriteCombinedReport(jobInfo))
{
// Write a single report
string jobReportPath = _fileSystem.Path.Combine(projReportsDirAbsolute, $"{jobInfo.ExtractionName()}_report.txt");
if (_fileSystem.File.Exists(jobReportPath))
{
throw new ApplicationException($"Report file '{jobReportPath}' already exists");
}
Logger.Info($"Writing report to {jobReportPath}");
fileStream = _fileSystem.File.OpenWrite(jobReportPath);
_currentFileStream = fileStream; // This is re-used
}
else
{
// Create a new directory for the report files, and return an open stream to the summary README
string jobReportDir = AbsolutePathToProjExtractionReportsDir(jobInfo);
if (_fileSystem.Directory.Exists(jobReportDir))
{
throw new ApplicationException($"Report directory '{jobReportDir}' already exists");
}
_fileSystem.Directory.CreateDirectory(jobReportDir);
Logger.Info($"Writing reports to directory {jobReportDir}");
string jobReadmePath = _fileSystem.Path.Combine(jobReportDir, "README.md");
fileStream = _fileSystem.File.OpenWrite(jobReadmePath);
}
return(fileStream);
}
private static string GetHeaderAndContents(ExtractJobInfo jobInfo, DateTimeProvider provider, string newLine)
{
string identExtraction = jobInfo.IsIdentifiableExtraction ? "Yes" : "No";
string filteredExtraction = !jobInfo.IsNoFilterExtraction ? "Yes" : "No";
var headerLines = new List<string>
{
$"# SMI extraction validation report for 1234/test",
$"",
$"Job info:",
$"- Job submitted at: {provider.UtcNow().ToString("s", CultureInfo.InvariantCulture)}",
$"- Job completed at: {(provider.UtcNow() + TimeSpan.FromHours(1)).ToString("s", CultureInfo.InvariantCulture)}",
$"- Job duration: {TimeSpan.FromHours(1)}",
$"- Job extraction id: {jobInfo.ExtractionJobIdentifier}",
$"- Extraction tag: {jobInfo.KeyTag}",
$"- Extraction modality: {jobInfo.ExtractionModality ?? "Unspecified"}",
$"- Requested identifier count: {jobInfo.KeyValueCount}",
$"- Identifiable extraction: {identExtraction}",
$"- Filtered extraction: {filteredExtraction}",
$"",
$"Report contents:",
$"",
};
if (!jobInfo.IsIdentifiableExtraction)
{
headerLines.AddRange(new List<string>
{
$"- Verification failures",
$" - Summary",
$" - Full Details",
$"- Blocked files",
$"- Anonymisation failures",
});
}
else
{
headerLines.AddRange(new List<string>
{
$"- Missing file list (files which were selected from an input ID but could not be found)",
});
}
return string.Join(newLine, headerLines);
}
private void DoJobCompletionTasks(ExtractJobInfo jobInfo)
{
Guid jobId = jobInfo.ExtractionJobIdentifier;
if (jobInfo.JobStatus != ExtractJobStatus.ReadyForChecks)
{
throw new ApplicationException($"Job {jobId} is not ready for checks");
}
_logger.Info($"All files for job {jobId} present, running completion tasks");
_jobStore.MarkJobCompleted(jobId);
_reporter.CreateReport(jobId);
_logger.Info($"Report for {jobId} created");
_notifier.NotifyJobCompleted(jobInfo);
}
protected override Stream GetStreamForTagDataSummary(ExtractJobInfo jobInfo) => GetStream(jobInfo, "tag_data_summary.csv");
protected override Stream GetStreamForPixelDataWordLengthFrequencies(ExtractJobInfo jobInfo) => GetStream(jobInfo, "pixel_data_word_frequencies.csv");
protected override Stream GetStreamForPixelDataFull(ExtractJobInfo jobInfo) => GetStream(jobInfo, "pixel_data_full.csv");
protected override Stream GetStreamForPixelDataSummary(ExtractJobInfo jobInfo) => GetStream(jobInfo, "pixel_data_summary.csv");
protected abstract Stream GetStreamForPixelDataWordLengthFrequencies(ExtractJobInfo jobInfo);
protected override Stream GetStreamForPixelDataWordLengthFrequencies(ExtractJobInfo jobInfo)
{
_currentReportName = "pixel word length frequencies";
return(new MemoryStream());
}
protected abstract Stream GetStreamForTagDataSummary(ExtractJobInfo jobInfo);
protected override Stream GetStreamForTagDataFull(ExtractJobInfo jobInfo) => new MemoryStream();
public void TestGetReadJobsImpl()
{
Guid jobId = Guid.NewGuid();
var testJob = new MongoExtractJobDoc(
jobId,
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider),
"1234",
ExtractJobStatus.Failed,
"test/dir",
_dateTimeProvider.UtcNow(),
"SeriesInstanceUID",
1,
"MR",
isIdentifiableExtraction: true,
isNoFilterExtraction: true,
null);
var testMongoExpectedFilesDoc = new MongoExpectedFilesDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider),
"1.2.3.4",
new HashSet <MongoExpectedFileInfoDoc>
{
new MongoExpectedFileInfoDoc(Guid.NewGuid(), "anon1.dcm"),
},
new MongoRejectedKeyInfoDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider),
new Dictionary <string, int>())
);
var testMongoFileStatusDoc = new MongoFileStatusDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider),
"input.dcm",
"anon1.dcm",
true,
false,
ExtractedFileStatus.Anonymised,
"Verified");
var client = new TestMongoClient();
var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
// Assert that jobs marked as failed are not returned
client.ExtractionDatabase.InProgressCollection.InsertOne(testJob);
client.ExtractionDatabase.InProgressCollection.RejectChanges = true;
Assert.AreEqual(0, store.GetReadyJobs().Count);
// Assert that an in progress job is not returned
client = new TestMongoClient();
store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
testJob.JobStatus = ExtractJobStatus.WaitingForCollectionInfo;
client.ExtractionDatabase.InProgressCollection.InsertOne(testJob);
client.ExtractionDatabase.InProgressCollection.RejectChanges = true;
Assert.AreEqual(0, store.GetReadyJobs().Count);
// Check we handle a bad ReplaceOneResult
client = new TestMongoClient();
store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
testJob.JobStatus = ExtractJobStatus.WaitingForCollectionInfo;
client.ExtractionDatabase.InProgressCollection.InsertOne(testJob);
client.ExtractionDatabase.InProgressCollection.RejectChanges = true;
client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"] = new MockExtractCollection <Guid, MongoExpectedFilesDoc>();
client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"].InsertOne(testMongoExpectedFilesDoc);
Assert.Throws <ApplicationException>(() => store.GetReadyJobs());
// Check happy path
client = new TestMongoClient();
store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
testJob.JobStatus = ExtractJobStatus.WaitingForCollectionInfo;
client.ExtractionDatabase.InProgressCollection.InsertOne(testJob);
client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"] = new MockExtractCollection <Guid, MongoExpectedFilesDoc>();
client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"].InsertOne(testMongoExpectedFilesDoc);
Assert.AreEqual(0, store.GetReadyJobs().Count);
Assert.AreEqual(ExtractJobStatus.WaitingForStatuses, client.ExtractionDatabase.InProgressCollection.Documents.Single().Value.JobStatus);
client.ExtractionDatabase.StatusCollections[$"statuses_{jobId}"] = new MockExtractCollection <Guid, MongoFileStatusDoc>();
client.ExtractionDatabase.StatusCollections[$"statuses_{jobId}"].InsertOne(testMongoFileStatusDoc);
ExtractJobInfo job = store.GetReadyJobs().Single();
Assert.AreEqual(ExtractJobStatus.ReadyForChecks, job.JobStatus);
Assert.AreEqual(ExtractJobStatus.ReadyForChecks, client.ExtractionDatabase.InProgressCollection.Documents.Single().Value.JobStatus);
}
protected override Stream GetStreamForPixelDataFull(ExtractJobInfo jobInfo)
{
_currentReportName = "pixel full";
return(new MemoryStream());
}
protected override Stream GetStreamForSummary(ExtractJobInfo jobInfo)
{
_currentReportName = "summary";
_isCombinedReport = ShouldWriteCombinedReport(jobInfo);
return(new MemoryStream());
}
protected override Stream GetStreamForTagDataSummary(ExtractJobInfo jobInfo)
{
_currentReportName = "tag summary";
return(new MemoryStream());
}
protected override Stream GetStreamForTagDataFull(ExtractJobInfo jobInfo) => GetStream(jobInfo, "tag_data_full.csv");
private string AbsolutePathToProjExtractionReportsDir(ExtractJobInfo jobInfo)
=>
_fileSystem.Path.Combine(
AbsolutePathToProjReportsDir(jobInfo),
jobInfo.ExtractionName()
);
protected abstract Stream GetStreamForTagDataFull(ExtractJobInfo jobInfo);
protected override Stream GetStreamForPixelDataSummary(ExtractJobInfo jobInfo) => new MemoryStream();
protected override Stream GetStreamForPixelDataWordLengthFrequencies(ExtractJobInfo jobInfo) => new MemoryStream();
public void NotifyJobCompleted(ExtractJobInfo jobInfo)
{
Notified = true;
}
public void NotifyJobCompleted(ExtractJobInfo jobInfo)
{
_logger.Info("Job " + jobInfo.ExtractionJobIdentifier + " completed!");
}
