public void TestToExtractJobInfo() { Guid guid = Guid.NewGuid(); var message = new ExtractionRequestInfoMessage { ExtractionModality = "MR", JobSubmittedAt = _dateTimeProvider.UtcNow(), ProjectNumber = "1234", ExtractionJobIdentifier = guid, ExtractionDirectory = "test/directory", KeyTag = "KeyTag", KeyValueCount = 123, IsIdentifiableExtraction = true, IsNoFilterExtraction = true, }; MongoExtractJobDoc doc = MongoExtractJobDoc.FromMessage(message, _messageHeader, _dateTimeProvider); ExtractJobInfo extractJobInfo = doc.ToExtractJobInfo(); var expected = new ExtractJobInfo( guid, _dateTimeProvider.UtcNow(), "1234", "test/directory", "KeyTag", 123, "MR", ExtractJobStatus.WaitingForCollectionInfo, isIdentifiableExtraction: true, isNoFilterExtraction: true ); Assert.AreEqual(expected, extractJobInfo); }
public void TestExtractJobInfo_Equality() { Guid guid = Guid.NewGuid(); var info1 = new ExtractJobInfo( guid, _dateTimeProvider.UtcNow(), "1234", "test/directory", "KeyTag", 123, "MR", ExtractJobStatus.WaitingForCollectionInfo, isIdentifiableExtraction: true, isNoFilterExtraction: true ); var info2 = new ExtractJobInfo( guid, _dateTimeProvider.UtcNow(), "1234", "test/directory", "KeyTag", 123, "MR", ExtractJobStatus.WaitingForCollectionInfo, isIdentifiableExtraction: true, isNoFilterExtraction: true ); Assert.AreEqual(info1, info2); }
public void TestProcessJobs() { Guid jobId = Guid.NewGuid(); var testJobInfo = new ExtractJobInfo( jobId, DateTime.UtcNow, "123", "test/dir", "KeyTag", 123, null, ExtractJobStatus.ReadyForChecks, isIdentifiableExtraction: true, isNoFilterExtraction: true ); var opts = new CohortPackagerOptions { JobWatcherTimeoutInSeconds = 123 }; var mockJobStore = new Mock <IExtractJobStore>(); var callbackUsed = false; var mockCallback = new Action <Exception>(_ => callbackUsed = true); var testNotifier = new TestJobCompleteNotifier(); var testReporter = new TestJobReporter(); var watcher = new ExtractJobWatcher(opts, mockJobStore.Object, mockCallback, testNotifier, testReporter); // Check that we can call ProcessJobs with no Guid to process all jobs mockJobStore.Setup(x => x.GetReadyJobs(default(Guid))).Returns(new List <ExtractJobInfo>()); watcher.ProcessJobs(); mockJobStore.Verify(); // Check that we MarkJobFailed for known exceptions mockJobStore.Reset(); mockJobStore.Setup(x => x.GetReadyJobs(It.IsAny <Guid>())).Returns(new List <ExtractJobInfo> { testJobInfo }); mockJobStore.Setup(x => x.MarkJobCompleted(It.IsAny <Guid>())).Throws(new ApplicationException("aah")); watcher.ProcessJobs(jobId); mockJobStore.Verify(x => x.MarkJobFailed(jobId, It.IsAny <ApplicationException>()), Times.Once); // Check that we call the exception callback for unhandled exceptions mockJobStore.Reset(); mockJobStore.Setup(x => x.GetReadyJobs(It.IsAny <Guid>())).Returns(new List <ExtractJobInfo> { testJobInfo }); mockJobStore.Setup(x => x.MarkJobCompleted(It.IsAny <Guid>())).Throws(new Exception("aah")); watcher.ProcessJobs(jobId); Assert.True(callbackUsed); // Check happy path mockJobStore.Reset(); mockJobStore.Setup(x => x.GetReadyJobs(It.IsAny <Guid>())).Returns(new List <ExtractJobInfo> { testJobInfo }); testNotifier.Notified = false; watcher.ProcessJobs(jobId); Assert.True(testNotifier.Notified); Assert.True(testReporter.Reported); }
private string AbsolutePathToProjReportsDir(ExtractJobInfo jobInfo) => _fileSystem.Path.Combine( _extractRoot, jobInfo.ProjectExtractionDir(), "reports" );
private Stream GetStream(ExtractJobInfo jobInfo, string fileName) { if (ShouldWriteCombinedReport(jobInfo)) { return(_currentFileStream); } string absReportPath = _fileSystem.Path.Combine(AbsolutePathToProjExtractionReportsDir(jobInfo), fileName); return(_fileSystem.File.OpenWrite(absReportPath)); }
protected bool ShouldWriteCombinedReport(ExtractJobInfo jobInfo) { if (ReportFormat == ReportFormat.Combined || jobInfo.IsIdentifiableExtraction) { return(true); } if (ReportFormat == ReportFormat.Split) { return(false); } throw new ApplicationException($"No case for report format '{ReportFormat}'"); }
public void Test_ExtractJobInfo_ProjectExtractionDir(string extractionDir, string expected) { var info = new ExtractJobInfo( Guid.NewGuid(), DateTime.UtcNow, "1234", extractionDir, "KeyTag", 123, "MR", ExtractJobStatus.WaitingForCollectionInfo, isIdentifiableExtraction: false, isNoFilterExtraction: false ); Assert.AreEqual(expected, info.ProjectExtractionDir()); }
protected override Stream GetStreamForSummary(ExtractJobInfo jobInfo) { _currentFileStream = null; Stream fileStream; if (_createJobIdFile) { // Write the jobId to a file in the extraction dir to help identify the set of files if they are moved string jobIdFile = _fileSystem.Path.Combine(_extractRoot, jobInfo.ExtractionDirectory, "jobId.txt"); _fileSystem.File.WriteAllText(jobIdFile, string.Join(ReportNewLine, jobInfo.ExtractionJobIdentifier)); } string projReportsDirAbsolute = AbsolutePathToProjReportsDir(jobInfo); Directory.CreateDirectory(projReportsDirAbsolute); if (ShouldWriteCombinedReport(jobInfo)) { // Write a single report string jobReportPath = _fileSystem.Path.Combine(projReportsDirAbsolute, $"{jobInfo.ExtractionName()}_report.txt"); if (_fileSystem.File.Exists(jobReportPath)) { throw new ApplicationException($"Report file '{jobReportPath}' already exists"); } Logger.Info($"Writing report to {jobReportPath}"); fileStream = _fileSystem.File.OpenWrite(jobReportPath); _currentFileStream = fileStream; // This is re-used } else { // Create a new directory for the report files, and return an open stream to the summary README string jobReportDir = AbsolutePathToProjExtractionReportsDir(jobInfo); if (_fileSystem.Directory.Exists(jobReportDir)) { throw new ApplicationException($"Report directory '{jobReportDir}' already exists"); } _fileSystem.Directory.CreateDirectory(jobReportDir); Logger.Info($"Writing reports to directory {jobReportDir}"); string jobReadmePath = _fileSystem.Path.Combine(jobReportDir, "README.md"); fileStream = _fileSystem.File.OpenWrite(jobReadmePath); } return(fileStream); }
private static string GetHeaderAndContents(ExtractJobInfo jobInfo, DateTimeProvider provider, string newLine) { string identExtraction = jobInfo.IsIdentifiableExtraction ? "Yes" : "No"; string filteredExtraction = !jobInfo.IsNoFilterExtraction ? "Yes" : "No"; var headerLines = new List<string> { $"# SMI extraction validation report for 1234/test", $"", $"Job info:", $"- Job submitted at: {provider.UtcNow().ToString("s", CultureInfo.InvariantCulture)}", $"- Job completed at: {(provider.UtcNow() + TimeSpan.FromHours(1)).ToString("s", CultureInfo.InvariantCulture)}", $"- Job duration: {TimeSpan.FromHours(1)}", $"- Job extraction id: {jobInfo.ExtractionJobIdentifier}", $"- Extraction tag: {jobInfo.KeyTag}", $"- Extraction modality: {jobInfo.ExtractionModality ?? "Unspecified"}", $"- Requested identifier count: {jobInfo.KeyValueCount}", $"- Identifiable extraction: {identExtraction}", $"- Filtered extraction: {filteredExtraction}", $"", $"Report contents:", $"", }; if (!jobInfo.IsIdentifiableExtraction) { headerLines.AddRange(new List<string> { $"- Verification failures", $" - Summary", $" - Full Details", $"- Blocked files", $"- Anonymisation failures", }); } else { headerLines.AddRange(new List<string> { $"- Missing file list (files which were selected from an input ID but could not be found)", }); } return string.Join(newLine, headerLines); }
private void DoJobCompletionTasks(ExtractJobInfo jobInfo) { Guid jobId = jobInfo.ExtractionJobIdentifier; if (jobInfo.JobStatus != ExtractJobStatus.ReadyForChecks) { throw new ApplicationException($"Job {jobId} is not ready for checks"); } _logger.Info($"All files for job {jobId} present, running completion tasks"); _jobStore.MarkJobCompleted(jobId); _reporter.CreateReport(jobId); _logger.Info($"Report for {jobId} created"); _notifier.NotifyJobCompleted(jobInfo); }
protected override Stream GetStreamForTagDataSummary(ExtractJobInfo jobInfo) => GetStream(jobInfo, "tag_data_summary.csv");
protected override Stream GetStreamForPixelDataWordLengthFrequencies(ExtractJobInfo jobInfo) => GetStream(jobInfo, "pixel_data_word_frequencies.csv");
protected override Stream GetStreamForPixelDataFull(ExtractJobInfo jobInfo) => GetStream(jobInfo, "pixel_data_full.csv");
protected override Stream GetStreamForPixelDataSummary(ExtractJobInfo jobInfo) => GetStream(jobInfo, "pixel_data_summary.csv");
protected abstract Stream GetStreamForPixelDataWordLengthFrequencies(ExtractJobInfo jobInfo);
protected override Stream GetStreamForPixelDataWordLengthFrequencies(ExtractJobInfo jobInfo) { _currentReportName = "pixel word length frequencies"; return(new MemoryStream()); }
protected abstract Stream GetStreamForTagDataSummary(ExtractJobInfo jobInfo);
protected override Stream GetStreamForTagDataFull(ExtractJobInfo jobInfo) => new MemoryStream();
public void TestGetReadJobsImpl() { Guid jobId = Guid.NewGuid(); var testJob = new MongoExtractJobDoc( jobId, MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), "1234", ExtractJobStatus.Failed, "test/dir", _dateTimeProvider.UtcNow(), "SeriesInstanceUID", 1, "MR", isIdentifiableExtraction: true, isNoFilterExtraction: true, null); var testMongoExpectedFilesDoc = new MongoExpectedFilesDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), "1.2.3.4", new HashSet <MongoExpectedFileInfoDoc> { new MongoExpectedFileInfoDoc(Guid.NewGuid(), "anon1.dcm"), }, new MongoRejectedKeyInfoDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), new Dictionary <string, int>()) ); var testMongoFileStatusDoc = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), "input.dcm", "anon1.dcm", true, false, ExtractedFileStatus.Anonymised, "Verified"); var client = new TestMongoClient(); var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); // Assert that jobs marked as failed are not returned client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.ExtractionDatabase.InProgressCollection.RejectChanges = true; Assert.AreEqual(0, store.GetReadyJobs().Count); // Assert that an in progress job is not returned client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); testJob.JobStatus = ExtractJobStatus.WaitingForCollectionInfo; client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.ExtractionDatabase.InProgressCollection.RejectChanges = true; Assert.AreEqual(0, store.GetReadyJobs().Count); // Check we handle a bad ReplaceOneResult client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); testJob.JobStatus = ExtractJobStatus.WaitingForCollectionInfo; client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.ExtractionDatabase.InProgressCollection.RejectChanges = true; client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"] = new MockExtractCollection <Guid, MongoExpectedFilesDoc>(); client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"].InsertOne(testMongoExpectedFilesDoc); Assert.Throws <ApplicationException>(() => store.GetReadyJobs()); // Check happy path client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); testJob.JobStatus = ExtractJobStatus.WaitingForCollectionInfo; client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"] = new MockExtractCollection <Guid, MongoExpectedFilesDoc>(); client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"].InsertOne(testMongoExpectedFilesDoc); Assert.AreEqual(0, store.GetReadyJobs().Count); Assert.AreEqual(ExtractJobStatus.WaitingForStatuses, client.ExtractionDatabase.InProgressCollection.Documents.Single().Value.JobStatus); client.ExtractionDatabase.StatusCollections[$"statuses_{jobId}"] = new MockExtractCollection <Guid, MongoFileStatusDoc>(); client.ExtractionDatabase.StatusCollections[$"statuses_{jobId}"].InsertOne(testMongoFileStatusDoc); ExtractJobInfo job = store.GetReadyJobs().Single(); Assert.AreEqual(ExtractJobStatus.ReadyForChecks, job.JobStatus); Assert.AreEqual(ExtractJobStatus.ReadyForChecks, client.ExtractionDatabase.InProgressCollection.Documents.Single().Value.JobStatus); }
protected override Stream GetStreamForPixelDataFull(ExtractJobInfo jobInfo) { _currentReportName = "pixel full"; return(new MemoryStream()); }
protected override Stream GetStreamForSummary(ExtractJobInfo jobInfo) { _currentReportName = "summary"; _isCombinedReport = ShouldWriteCombinedReport(jobInfo); return(new MemoryStream()); }
protected override Stream GetStreamForTagDataSummary(ExtractJobInfo jobInfo) { _currentReportName = "tag summary"; return(new MemoryStream()); }
protected override Stream GetStreamForTagDataFull(ExtractJobInfo jobInfo) => GetStream(jobInfo, "tag_data_full.csv");
private string AbsolutePathToProjExtractionReportsDir(ExtractJobInfo jobInfo) => _fileSystem.Path.Combine( AbsolutePathToProjReportsDir(jobInfo), jobInfo.ExtractionName() );
protected abstract Stream GetStreamForTagDataFull(ExtractJobInfo jobInfo);
protected override Stream GetStreamForPixelDataSummary(ExtractJobInfo jobInfo) => new MemoryStream();
protected override Stream GetStreamForPixelDataWordLengthFrequencies(ExtractJobInfo jobInfo) => new MemoryStream();
public void NotifyJobCompleted(ExtractJobInfo jobInfo) { Notified = true; }
public void NotifyJobCompleted(ExtractJobInfo jobInfo) { _logger.Info("Job " + jobInfo.ExtractionJobIdentifier + " completed!"); }