private bool TryGetMongoExtractJobDoc(Guid extractionJobIdentifier, out MongoExtractJobDoc mongoExtractJobDoc) { mongoExtractJobDoc = _inProgressJobCollection .Find(GetFilterForSpecificJob <MongoExtractJobDoc>(extractionJobIdentifier)) .SingleOrDefault(); return(mongoExtractJobDoc != null); }
public void TestToExtractJobInfo() { Guid guid = Guid.NewGuid(); var message = new ExtractionRequestInfoMessage { ExtractionModality = "MR", JobSubmittedAt = _dateTimeProvider.UtcNow(), ProjectNumber = "1234", ExtractionJobIdentifier = guid, ExtractionDirectory = "test/directory", KeyTag = "KeyTag", KeyValueCount = 123, IsIdentifiableExtraction = true, IsNoFilterExtraction = true, }; MongoExtractJobDoc doc = MongoExtractJobDoc.FromMessage(message, _messageHeader, _dateTimeProvider); ExtractJobInfo extractJobInfo = doc.ToExtractJobInfo(); var expected = new ExtractJobInfo( guid, _dateTimeProvider.UtcNow(), "1234", "test/directory", "KeyTag", 123, "MR", ExtractJobStatus.WaitingForCollectionInfo, isIdentifiableExtraction: true, isNoFilterExtraction: true ); Assert.AreEqual(expected, extractJobInfo); }
public void TestMongoExtractJobDoc_GetHashCode() { Guid guid = Guid.NewGuid(); var doc1 = new MongoExtractJobDoc( guid, MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, _messageHeader, _dateTimeProvider), "1234", ExtractJobStatus.WaitingForCollectionInfo, "test/directory", _dateTimeProvider.UtcNow(), "KeyTag", 123, "MR", isIdentifiableExtraction: true, isNoFilterExtraction: true, null); var doc2 = new MongoExtractJobDoc( guid, MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, _messageHeader, _dateTimeProvider), "1234", ExtractJobStatus.WaitingForCollectionInfo, "test/directory", _dateTimeProvider.UtcNow(), "KeyTag", 123, "MR", isIdentifiableExtraction: true, isNoFilterExtraction: true, null); Assert.AreEqual(doc1.GetHashCode(), doc2.GetHashCode()); }
public void TestMongoExtractJobDoc_Equality() { Guid guid = Guid.NewGuid(); var failedInfoDoc = new MongoFailedJobInfoDoc(new TestException("aaah"), _dateTimeProvider); var doc1 = new MongoExtractJobDoc( guid, MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, _messageHeader, _dateTimeProvider), "1234", ExtractJobStatus.WaitingForCollectionInfo, "test/directory", _dateTimeProvider.UtcNow(), "KeyTag", 123, "MR", isIdentifiableExtraction: true, isNoFilterExtraction: true, failedInfoDoc); var doc2 = new MongoExtractJobDoc( guid, MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, _messageHeader, _dateTimeProvider), "1234", ExtractJobStatus.WaitingForCollectionInfo, "test/directory", _dateTimeProvider.UtcNow(), "KeyTag", 123, "MR", isIdentifiableExtraction: true, isNoFilterExtraction: true, failedInfoDoc); Assert.AreEqual(doc1, doc2); }
protected override void PersistMessageToStoreImpl(ExtractionRequestInfoMessage message, IMessageHeader header) { if (InCompletedJobCollection(message.ExtractionJobIdentifier)) { throw new ApplicationException("Received an ExtractionRequestInfoMessage for a job that is already completed"); } MongoExtractJobDoc newJobInfo = MongoExtractJobDoc.FromMessage(message, header, _dateTimeProvider); _inProgressJobCollection.InsertOne(newJobInfo); }
public static ExtractJobInfo ToExtractJobInfo(this MongoExtractJobDoc mongoExtractJobDoc) => new ExtractJobInfo( mongoExtractJobDoc.ExtractionJobIdentifier, mongoExtractJobDoc.JobSubmittedAt, mongoExtractJobDoc.ProjectNumber, mongoExtractJobDoc.ExtractionDirectory, mongoExtractJobDoc.KeyTag, mongoExtractJobDoc.KeyCount, mongoExtractJobDoc.ExtractionModality, mongoExtractJobDoc.JobStatus, mongoExtractJobDoc.IsIdentifiableExtraction, mongoExtractJobDoc.IsNoFilterExtraction );
public void TestPersistMessageToStoreImpl_ExtractionRequestInfoMessage() { Guid guid = Guid.NewGuid(); var testExtractionRequestInfoMessage = new ExtractionRequestInfoMessage { ExtractionJobIdentifier = guid, ProjectNumber = "1234-5678", ExtractionDirectory = "1234-5678/testExtract", JobSubmittedAt = _dateTimeProvider.UtcNow(), KeyTag = "StudyInstanceUID", KeyValueCount = 1, ExtractionModality = "CT", IsIdentifiableExtraction = true, IsNoFilterExtraction = true, }; var testHeader = new MessageHeader { MessageGuid = Guid.NewGuid(), OriginalPublishTimestamp = MessageHeader.UnixTime(_dateTimeProvider.UtcNow()), Parents = new[] { Guid.NewGuid(), }, ProducerExecutableName = "MongoExtractStoreTests", ProducerProcessID = 1234, }; var client = new TestMongoClient(); var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); store.PersistMessageToStore(testExtractionRequestInfoMessage, testHeader); Dictionary <Guid, MongoExtractJobDoc> docs = client.ExtractionDatabase.InProgressCollection.Documents; Assert.AreEqual(docs.Count, 1); MongoExtractJobDoc extractJob = docs.Values.ToList()[0]; var expected = new MongoExtractJobDoc( guid, MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, testHeader, _dateTimeProvider), "1234-5678", ExtractJobStatus.WaitingForCollectionInfo, "1234-5678/testExtract", _dateTimeProvider.UtcNow(), "StudyInstanceUID", 1, "CT", isIdentifiableExtraction: true, isNoFilterExtraction: true, null); Assert.AreEqual(expected, extractJob); }
public void Test_MongoExtractJobDoc_CopyConstructor() { var jobid = Guid.NewGuid(); var original = new MongoExtractJobDoc( jobid, MongoExtractionMessageHeaderDoc.FromMessageHeader(jobid, _messageHeader, _dateTimeProvider), "1234", ExtractJobStatus.WaitingForCollectionInfo, "test/directory", _dateTimeProvider.UtcNow(), "KeyTag", 123, "MR", isIdentifiableExtraction: true, isNoFilterExtraction: true, new MongoFailedJobInfoDoc(new Exception("foo"), _dateTimeProvider) ); var copied = new MongoExtractJobDoc(original); Assert.AreEqual(original, copied); }
public void TestMarkJobFailedImpl() { Guid jobId = Guid.NewGuid(); var testJob = new MongoExtractJobDoc( jobId, MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), "1234", ExtractJobStatus.Failed, "test/dir", _dateTimeProvider.UtcNow(), "1.2.3.4", 123, "MR", isIdentifiableExtraction: true, isNoFilterExtraction: true, null); var client = new TestMongoClient(); var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); // Assert that an exception is thrown for a non-existent job Assert.Throws <ApplicationException>(() => store.MarkJobFailed(Guid.NewGuid(), new Exception())); client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once); client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never); // Assert that a job can't be failed twice client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.MockSessionHandle.Reset(); Assert.Throws <ApplicationException>(() => store.MarkJobFailed(jobId, new Exception())); client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once); client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never); // Check we handle a bad ReplaceOneResult client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); testJob.JobStatus = ExtractJobStatus.WaitingForCollectionInfo; client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.ExtractionDatabase.InProgressCollection.RejectChanges = true; client.MockSessionHandle.Reset(); Assert.Throws <ApplicationException>(() => store.MarkJobFailed(jobId, new Exception())); client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once); client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never); // Check happy path client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); testJob.JobStatus = ExtractJobStatus.WaitingForCollectionInfo; testJob.FailedJobInfoDoc = null; client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.MockSessionHandle.Reset(); store.MarkJobFailed(jobId, new Exception("TestMarkJobFailedImpl")); client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Never); client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Once); Dictionary <Guid, MongoExtractJobDoc> docs = client.ExtractionDatabase.InProgressCollection.Documents; Assert.AreEqual(1, docs.Count); MongoExtractJobDoc failedDoc = docs[jobId]; Assert.AreEqual(ExtractJobStatus.Failed, failedDoc.JobStatus); Assert.NotNull(failedDoc.FailedJobInfoDoc); Assert.AreEqual("TestMarkJobFailedImpl", failedDoc.FailedJobInfoDoc.ExceptionMessage); }
public void TestCompleteJobImpl() { Guid jobId = Guid.NewGuid(); var testJob = new MongoExtractJobDoc( jobId, MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), "1234", ExtractJobStatus.Failed, "test/dir", _dateTimeProvider.UtcNow(), "SeriesInstanceUID", 1, "MR", isIdentifiableExtraction: true, isNoFilterExtraction: true, null); var testMongoExpectedFilesDoc = new MongoExpectedFilesDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), "1.2.3.4", new HashSet <MongoExpectedFileInfoDoc> { new MongoExpectedFileInfoDoc(Guid.NewGuid(), "anon1.dcm"), }, new MongoRejectedKeyInfoDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), new Dictionary <string, int>()) ); var testMongoFileStatusDoc = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), "input.dcm", "anon1.dcm", true, false, ExtractedFileStatus.Anonymised, "Verified"); var client = new TestMongoClient(); var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); // Assert that an exception is thrown for a non-existent job Assert.Throws <ApplicationException>(() => store.MarkJobCompleted(Guid.NewGuid())); client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once); client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never); // Assert that an exception is thrown for a job which is marked as failed client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.MockSessionHandle.Reset(); Assert.Throws <ApplicationException>(() => store.MarkJobCompleted(Guid.NewGuid())); client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once); client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never); // Check that we handle a failed insertion client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); testJob.JobStatus = ExtractJobStatus.Completed; client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.ExtractionDatabase.CompletedJobCollection.RejectChanges = true; client.MockSessionHandle.Reset(); Assert.Throws <Exception>(() => store.MarkJobCompleted(jobId)); client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once); client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never); // Check we handle a bad DeleteResult client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); client.ExtractionDatabase.InProgressCollection.RejectChanges = true; client.MockSessionHandle.Reset(); Assert.Throws <ApplicationException>(() => store.MarkJobCompleted(jobId)); client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once); client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never); // Check we handle missing expectedFiles collection client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.MockSessionHandle.Reset(); Assert.Throws <ApplicationException>(() => store.MarkJobCompleted(jobId)); client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once); client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never); // Check we handle missing statuses collection client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"] = new MockExtractCollection <Guid, MongoExpectedFilesDoc>(); client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"].InsertOne(testMongoExpectedFilesDoc); client.MockSessionHandle.Reset(); Assert.Throws <ApplicationException>(() => store.MarkJobCompleted(jobId)); client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once); client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never); // Check happy path client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"] = new MockExtractCollection <Guid, MongoExpectedFilesDoc>(); client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"].InsertOne(testMongoExpectedFilesDoc); client.ExtractionDatabase.StatusCollections[$"statuses_{jobId}"] = new MockExtractCollection <Guid, MongoFileStatusDoc>(); client.ExtractionDatabase.StatusCollections[$"statuses_{jobId}"].InsertOne(testMongoFileStatusDoc); client.MockSessionHandle.Reset(); store.MarkJobCompleted(jobId); client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Never); client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Once); Assert.AreEqual(1, client.ExtractionDatabase.ExpectedFilesCollections.Count); Assert.AreEqual(1, client.ExtractionDatabase.StatusCollections.Count); }
public void TestGetReadJobsImpl() { Guid jobId = Guid.NewGuid(); var testJob = new MongoExtractJobDoc( jobId, MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), "1234", ExtractJobStatus.Failed, "test/dir", _dateTimeProvider.UtcNow(), "SeriesInstanceUID", 1, "MR", isIdentifiableExtraction: true, isNoFilterExtraction: true, null); var testMongoExpectedFilesDoc = new MongoExpectedFilesDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), "1.2.3.4", new HashSet <MongoExpectedFileInfoDoc> { new MongoExpectedFileInfoDoc(Guid.NewGuid(), "anon1.dcm"), }, new MongoRejectedKeyInfoDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), new Dictionary <string, int>()) ); var testMongoFileStatusDoc = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), "input.dcm", "anon1.dcm", true, false, ExtractedFileStatus.Anonymised, "Verified"); var client = new TestMongoClient(); var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); // Assert that jobs marked as failed are not returned client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.ExtractionDatabase.InProgressCollection.RejectChanges = true; Assert.AreEqual(0, store.GetReadyJobs().Count); // Assert that an in progress job is not returned client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); testJob.JobStatus = ExtractJobStatus.WaitingForCollectionInfo; client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.ExtractionDatabase.InProgressCollection.RejectChanges = true; Assert.AreEqual(0, store.GetReadyJobs().Count); // Check we handle a bad ReplaceOneResult client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); testJob.JobStatus = ExtractJobStatus.WaitingForCollectionInfo; client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.ExtractionDatabase.InProgressCollection.RejectChanges = true; client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"] = new MockExtractCollection <Guid, MongoExpectedFilesDoc>(); client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"].InsertOne(testMongoExpectedFilesDoc); Assert.Throws <ApplicationException>(() => store.GetReadyJobs()); // Check happy path client = new TestMongoClient(); store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); testJob.JobStatus = ExtractJobStatus.WaitingForCollectionInfo; client.ExtractionDatabase.InProgressCollection.InsertOne(testJob); client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"] = new MockExtractCollection <Guid, MongoExpectedFilesDoc>(); client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"].InsertOne(testMongoExpectedFilesDoc); Assert.AreEqual(0, store.GetReadyJobs().Count); Assert.AreEqual(ExtractJobStatus.WaitingForStatuses, client.ExtractionDatabase.InProgressCollection.Documents.Single().Value.JobStatus); client.ExtractionDatabase.StatusCollections[$"statuses_{jobId}"] = new MockExtractCollection <Guid, MongoFileStatusDoc>(); client.ExtractionDatabase.StatusCollections[$"statuses_{jobId}"].InsertOne(testMongoFileStatusDoc); ExtractJobInfo job = store.GetReadyJobs().Single(); Assert.AreEqual(ExtractJobStatus.ReadyForChecks, job.JobStatus); Assert.AreEqual(ExtractJobStatus.ReadyForChecks, client.ExtractionDatabase.InProgressCollection.Documents.Single().Value.JobStatus); }