public void TestPersistMessageToStoreImpl_ExtractFileCollectionInfoMessage_NoIdentifiers() { Guid jobId = Guid.NewGuid(); var testExtractFileCollectionInfoMessage = new ExtractFileCollectionInfoMessage { ExtractionJobIdentifier = jobId, ProjectNumber = "1234-5678", RejectionReasons = new Dictionary <string, int> { { "ImageType is not ORIGINAL", 1 }, }, JobSubmittedAt = DateTime.UtcNow, ExtractionDirectory = "1234-5678/testExtract", ExtractFileMessagesDispatched = new JsonCompatibleDictionary <MessageHeader, string>(), // No files were extractable for this key KeyValue = "series-1", }; var header = new MessageHeader { MessageGuid = Guid.NewGuid(), OriginalPublishTimestamp = MessageHeader.UnixTimeNow(), Parents = new[] { Guid.NewGuid(), }, ProducerExecutableName = "MongoExtractStoreTests", ProducerProcessID = 1234, }; var client = new TestMongoClient(); var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); Assert.DoesNotThrow(() => store.PersistMessageToStore(testExtractFileCollectionInfoMessage, header)); }
public void TestPersistMessageToStoreImpl_ExtractionRequestInfoMessage() { Guid guid = Guid.NewGuid(); var testExtractionRequestInfoMessage = new ExtractionRequestInfoMessage { ExtractionJobIdentifier = guid, ProjectNumber = "1234-5678", ExtractionDirectory = "1234-5678/testExtract", JobSubmittedAt = _dateTimeProvider.UtcNow(), KeyTag = "StudyInstanceUID", KeyValueCount = 1, ExtractionModality = "CT", IsIdentifiableExtraction = true, IsNoFilterExtraction = true, }; var testHeader = new MessageHeader { MessageGuid = Guid.NewGuid(), OriginalPublishTimestamp = MessageHeader.UnixTime(_dateTimeProvider.UtcNow()), Parents = new[] { Guid.NewGuid(), }, ProducerExecutableName = "MongoExtractStoreTests", ProducerProcessID = 1234, }; var client = new TestMongoClient(); var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); store.PersistMessageToStore(testExtractionRequestInfoMessage, testHeader); Dictionary <Guid, MongoExtractJobDoc> docs = client.ExtractionDatabase.InProgressCollection.Documents; Assert.AreEqual(docs.Count, 1); MongoExtractJobDoc extractJob = docs.Values.ToList()[0]; var expected = new MongoExtractJobDoc( guid, MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, testHeader, _dateTimeProvider), "1234-5678", ExtractJobStatus.WaitingForCollectionInfo, "1234-5678/testExtract", _dateTimeProvider.UtcNow(), "StudyInstanceUID", 1, "CT", isIdentifiableExtraction: true, isNoFilterExtraction: true, null); Assert.AreEqual(expected, extractJob); }
public void TestPersistMessageToStoreImpl_IsIdentifiableMessage() { var client = new TestMongoClient(); var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); Guid jobId = Guid.NewGuid(); var testIsIdentifiableMessage = new ExtractedFileVerificationMessage { OutputFilePath = "anon.dcm", JobSubmittedAt = _dateTimeProvider.UtcNow(), ProjectNumber = "1234", ExtractionJobIdentifier = jobId, ExtractionDirectory = "1234/test", DicomFilePath = "original.dcm", IsIdentifiable = false, Report = "[]", // NOTE(rkm 2020-03-10) An "empty" report from IsIdentifiable }; var header = new MessageHeader(); store.PersistMessageToStore(testIsIdentifiableMessage, header); Dictionary <Guid, MongoFileStatusDoc> docs = client.ExtractionDatabase.StatusCollections[$"statuses_{jobId}"].Documents; Assert.AreEqual(docs.Count, 1); MongoFileStatusDoc statusDoc = docs.Values.ToList()[0]; var expected = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, header, _dateTimeProvider), "original.dcm", "anon.dcm", true, false, ExtractedFileStatus.Anonymised, "[]"); Assert.True(statusDoc.Equals(expected)); }
public void TestPersistMessageToStoreImpl_ExtractFileStatusMessage() { var client = new TestMongoClient(); var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); Guid jobId = Guid.NewGuid(); var testExtractFileStatusMessage = new ExtractedFileStatusMessage { OutputFilePath = "anon.dcm", JobSubmittedAt = _dateTimeProvider.UtcNow(), Status = ExtractedFileStatus.ErrorWontRetry, ProjectNumber = "1234", ExtractionJobIdentifier = jobId, ExtractionDirectory = "1234/test", StatusMessage = "Could not anonymise", DicomFilePath = "original.dcm", }; var header = new MessageHeader(); store.PersistMessageToStore(testExtractFileStatusMessage, header); Dictionary <Guid, MongoFileStatusDoc> docs = client.ExtractionDatabase.StatusCollections[$"statuses_{jobId}"].Documents; Assert.AreEqual(docs.Count, 1); MongoFileStatusDoc statusDoc = docs.Values.ToList()[0]; var expected = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, header, _dateTimeProvider), "original.dcm", "anon.dcm", false, true, ExtractedFileStatus.ErrorWontRetry, "Could not anonymise"); Assert.True(statusDoc.Equals(expected)); }
public void TestMarkJobFailedImpl() { Guid jobId = Guid.NewGuid(); var testJob = new MongoExtractJobDoc( jobId, MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), "1234", ExtractJobStatus.Failed, "test/dir", _dateTimeProvider.UtcNow(), "1.2.3.4", 123, "MR", isIdentifiableExtraction: true, isNoFilterExtraction: true, null); var client = new TestMongoClient(); var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); // Assert that an exception is thrown for a non-existent job Assert.Throws <ApplicationException>(() => store.MarkJobFailed(Guid.NewGuid(), new Exception())); client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once); client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never); // Assert that a job can't be failed twice client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.MockSessionHandle.Reset(); Assert.Throws <ApplicationException>(() => store.MarkJobFailed(jobId, new Exception())); client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once); client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never); // Check we handle a bad ReplaceOneResult client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); testJob.JobStatus = ExtractJobStatus.WaitingForCollectionInfo; client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.ExtractionDatabase.InProgressCollection.RejectChanges = true; client.MockSessionHandle.Reset(); Assert.Throws <ApplicationException>(() => store.MarkJobFailed(jobId, new Exception())); client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once); client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never); // Check happy path client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); testJob.JobStatus = ExtractJobStatus.WaitingForCollectionInfo; testJob.FailedJobInfoDoc = null; client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.MockSessionHandle.Reset(); store.MarkJobFailed(jobId, new Exception("TestMarkJobFailedImpl")); client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Never); client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Once); Dictionary <Guid, MongoExtractJobDoc> docs = client.ExtractionDatabase.InProgressCollection.Documents; Assert.AreEqual(1, docs.Count); MongoExtractJobDoc failedDoc = docs[jobId]; Assert.AreEqual(ExtractJobStatus.Failed, failedDoc.JobStatus); Assert.NotNull(failedDoc.FailedJobInfoDoc); Assert.AreEqual("TestMarkJobFailedImpl", failedDoc.FailedJobInfoDoc.ExceptionMessage); }
public void TestCompleteJobImpl() { Guid jobId = Guid.NewGuid(); var testJob = new MongoExtractJobDoc( jobId, MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), "1234", ExtractJobStatus.Failed, "test/dir", _dateTimeProvider.UtcNow(), "SeriesInstanceUID", 1, "MR", isIdentifiableExtraction: true, isNoFilterExtraction: true, null); var testMongoExpectedFilesDoc = new MongoExpectedFilesDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), "1.2.3.4", new HashSet <MongoExpectedFileInfoDoc> { new MongoExpectedFileInfoDoc(Guid.NewGuid(), "anon1.dcm"), }, new MongoRejectedKeyInfoDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), new Dictionary <string, int>()) ); var testMongoFileStatusDoc = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), "input.dcm", "anon1.dcm", true, false, ExtractedFileStatus.Anonymised, "Verified"); var client = new TestMongoClient(); var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); // Assert that an exception is thrown for a non-existent job Assert.Throws <ApplicationException>(() => store.MarkJobCompleted(Guid.NewGuid())); client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once); client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never); // Assert that an exception is thrown for a job which is marked as failed client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.MockSessionHandle.Reset(); Assert.Throws <ApplicationException>(() => store.MarkJobCompleted(Guid.NewGuid())); client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once); client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never); // Check that we handle a failed insertion client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); testJob.JobStatus = ExtractJobStatus.Completed; client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.ExtractionDatabase.CompletedJobCollection.RejectChanges = true; client.MockSessionHandle.Reset(); Assert.Throws <Exception>(() => store.MarkJobCompleted(jobId)); client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once); client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never); // Check we handle a bad DeleteResult client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); client.ExtractionDatabase.InProgressCollection.RejectChanges = true; client.MockSessionHandle.Reset(); Assert.Throws <ApplicationException>(() => store.MarkJobCompleted(jobId)); client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once); client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never); // Check we handle missing expectedFiles collection client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.MockSessionHandle.Reset(); Assert.Throws <ApplicationException>(() => store.MarkJobCompleted(jobId)); client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once); client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never); // Check we handle missing statuses collection client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"] = new MockExtractCollection <Guid, MongoExpectedFilesDoc>(); client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"].InsertOne(testMongoExpectedFilesDoc); client.MockSessionHandle.Reset(); Assert.Throws <ApplicationException>(() => store.MarkJobCompleted(jobId)); client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once); client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never); // Check happy path client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"] = new MockExtractCollection <Guid, MongoExpectedFilesDoc>(); client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"].InsertOne(testMongoExpectedFilesDoc); client.ExtractionDatabase.StatusCollections[$"statuses_{jobId}"] = new MockExtractCollection <Guid, MongoFileStatusDoc>(); client.ExtractionDatabase.StatusCollections[$"statuses_{jobId}"].InsertOne(testMongoFileStatusDoc); client.MockSessionHandle.Reset(); store.MarkJobCompleted(jobId); client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Never); client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Once); Assert.AreEqual(1, client.ExtractionDatabase.ExpectedFilesCollections.Count); Assert.AreEqual(1, client.ExtractionDatabase.StatusCollections.Count); }
public void TestGetReadJobsImpl() { Guid jobId = Guid.NewGuid(); var testJob = new MongoExtractJobDoc( jobId, MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), "1234", ExtractJobStatus.Failed, "test/dir", _dateTimeProvider.UtcNow(), "SeriesInstanceUID", 1, "MR", isIdentifiableExtraction: true, isNoFilterExtraction: true, null); var testMongoExpectedFilesDoc = new MongoExpectedFilesDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), "1.2.3.4", new HashSet <MongoExpectedFileInfoDoc> { new MongoExpectedFileInfoDoc(Guid.NewGuid(), "anon1.dcm"), }, new MongoRejectedKeyInfoDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), new Dictionary <string, int>()) ); var testMongoFileStatusDoc = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), "input.dcm", "anon1.dcm", true, false, ExtractedFileStatus.Anonymised, "Verified"); var client = new TestMongoClient(); var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); // Assert that jobs marked as failed are not returned client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.ExtractionDatabase.InProgressCollection.RejectChanges = true; Assert.AreEqual(0, store.GetReadyJobs().Count); // Assert that an in progress job is not returned client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); testJob.JobStatus = ExtractJobStatus.WaitingForCollectionInfo; client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.ExtractionDatabase.InProgressCollection.RejectChanges = true; Assert.AreEqual(0, store.GetReadyJobs().Count); // Check we handle a bad ReplaceOneResult client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); testJob.JobStatus = ExtractJobStatus.WaitingForCollectionInfo; client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.ExtractionDatabase.InProgressCollection.RejectChanges = true; client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"] = new MockExtractCollection <Guid, MongoExpectedFilesDoc>(); client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"].InsertOne(testMongoExpectedFilesDoc); Assert.Throws <ApplicationException>(() => store.GetReadyJobs()); // Check happy path client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); testJob.JobStatus = ExtractJobStatus.WaitingForCollectionInfo; client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"] = new MockExtractCollection <Guid, MongoExpectedFilesDoc>(); client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"].InsertOne(testMongoExpectedFilesDoc); Assert.AreEqual(0, store.GetReadyJobs().Count); Assert.AreEqual(ExtractJobStatus.WaitingForStatuses, client.ExtractionDatabase.InProgressCollection.Documents.Single().Value.JobStatus); client.ExtractionDatabase.StatusCollections[$"statuses_{jobId}"] = new MockExtractCollection <Guid, MongoFileStatusDoc>(); client.ExtractionDatabase.StatusCollections[$"statuses_{jobId}"].InsertOne(testMongoFileStatusDoc); ExtractJobInfo job = store.GetReadyJobs().Single(); Assert.AreEqual(ExtractJobStatus.ReadyForChecks, job.JobStatus); Assert.AreEqual(ExtractJobStatus.ReadyForChecks, client.ExtractionDatabase.InProgressCollection.Documents.Single().Value.JobStatus); }
public void TestPersistMessageToStoreImpl_ExtractFileCollectionInfoMessage() { Guid jobId = Guid.NewGuid(); var header1 = new MessageHeader(); var header2 = new MessageHeader(); var testExtractFileCollectionInfoMessage = new ExtractFileCollectionInfoMessage { ExtractionJobIdentifier = jobId, ProjectNumber = "1234-5678", RejectionReasons = new Dictionary <string, int> { { "reject1", 1 }, { "reject2", 2 }, }, JobSubmittedAt = DateTime.UtcNow, ExtractionDirectory = "1234-5678/testExtract", ExtractFileMessagesDispatched = new JsonCompatibleDictionary <MessageHeader, string> { { header1, "file1" }, { header2, "file2" } }, KeyValue = "series-1", }; var header = new MessageHeader { MessageGuid = Guid.NewGuid(), OriginalPublishTimestamp = MessageHeader.UnixTimeNow(), Parents = new[] { Guid.NewGuid(), }, ProducerExecutableName = "MongoExtractStoreTests", ProducerProcessID = 1234, }; var client = new TestMongoClient(); var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); store.PersistMessageToStore(testExtractFileCollectionInfoMessage, header); Dictionary <Guid, MongoExpectedFilesDoc> docs = client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"].Documents; Assert.AreEqual(docs.Count, 1); MongoExpectedFilesDoc extractJob = docs.Values.ToList()[0]; var expected = new MongoExpectedFilesDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, header, _dateTimeProvider), "series-1", new HashSet <MongoExpectedFileInfoDoc> { new MongoExpectedFileInfoDoc(header1.MessageGuid, "file1"), new MongoExpectedFileInfoDoc(header2.MessageGuid, "file2"), }, new MongoRejectedKeyInfoDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, header, _dateTimeProvider), new Dictionary <string, int> { { "reject1", 1 }, { "reject2", 2 }, }) ); Assert.True(extractJob.Equals(expected)); }