public void TestMongoExtractJobDoc_GetHashCode()
{
Guid guid = Guid.NewGuid();
var doc1 = new MongoExtractJobDoc(
guid,
MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, _messageHeader, _dateTimeProvider),
"1234",
ExtractJobStatus.WaitingForCollectionInfo,
"test/directory",
_dateTimeProvider.UtcNow(),
"KeyTag",
123,
"MR",
isIdentifiableExtraction: true,
isNoFilterExtraction: true,
null);
var doc2 = new MongoExtractJobDoc(
guid,
MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, _messageHeader, _dateTimeProvider),
"1234",
ExtractJobStatus.WaitingForCollectionInfo,
"test/directory",
_dateTimeProvider.UtcNow(),
"KeyTag",
123,
"MR",
isIdentifiableExtraction: true,
isNoFilterExtraction: true,
null);
Assert.AreEqual(doc1.GetHashCode(), doc2.GetHashCode());
}
public void TestMongoExpectedFilesDoc_GetHashCode()
{
var expectedFiles = new HashSet <MongoExpectedFileInfoDoc>
{
new MongoExpectedFileInfoDoc(Guid.NewGuid(), "anon1.dcm"),
new MongoExpectedFileInfoDoc(Guid.NewGuid(), "anon2.dcm"),
};
Guid jobId = Guid.NewGuid();
var rejectedKeys = new MongoRejectedKeyInfoDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, _testHeader, _dateTimeProvider),
new Dictionary <string, int>
{
{ "reject-1", 1 },
{ "reject-2", 2 },
}
);
var doc1 = new MongoExpectedFilesDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, _testHeader, _dateTimeProvider),
"TestKey",
expectedFiles,
rejectedKeys);
var doc2 = new MongoExpectedFilesDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, _testHeader, _dateTimeProvider),
"TestKey",
expectedFiles,
rejectedKeys);
Assert.AreEqual(doc1.GetHashCode(), doc2.GetHashCode());
}
public void TestMongoExtractJobDoc_Equality()
{
Guid guid = Guid.NewGuid();
var failedInfoDoc = new MongoFailedJobInfoDoc(new TestException("aaah"), _dateTimeProvider);
var doc1 = new MongoExtractJobDoc(
guid,
MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, _messageHeader, _dateTimeProvider),
"1234",
ExtractJobStatus.WaitingForCollectionInfo,
"test/directory",
_dateTimeProvider.UtcNow(),
"KeyTag",
123,
"MR",
isIdentifiableExtraction: true,
isNoFilterExtraction: true,
failedInfoDoc);
var doc2 = new MongoExtractJobDoc(
guid,
MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, _messageHeader, _dateTimeProvider),
"1234",
ExtractJobStatus.WaitingForCollectionInfo,
"test/directory",
_dateTimeProvider.UtcNow(),
"KeyTag",
123,
"MR",
isIdentifiableExtraction: true,
isNoFilterExtraction: true,
failedInfoDoc);
Assert.AreEqual(doc1, doc2);
}
public void TestMongoExtractionMessageHeaderDoc_FromMessageHeader()
{
Guid guid = Guid.NewGuid();
Guid p1 = Guid.NewGuid();
Guid p2 = Guid.NewGuid();
const long unixTimeNow = 1234;
var dateTimeProvider = new TestDateTimeProvider();
var header = new MessageHeader
{
MessageGuid = guid,
ProducerExecutableName = "TestFromMessageHeader",
ProducerProcessID = 1234,
Parents = new[] { p1, p2 },
OriginalPublishTimestamp = unixTimeNow,
};
MongoExtractionMessageHeaderDoc doc = MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, header, dateTimeProvider);
var expected = new MongoExtractionMessageHeaderDoc(
guid,
guid,
"TestFromMessageHeader",
1234,
DateTime.UnixEpoch + TimeSpan.FromSeconds(unixTimeNow),
$"{p1}->{p2}",
dateTimeProvider.UtcNow()
);
Assert.AreEqual(expected, doc);
}
public void TestMongoExtractJobDoc_FromMessage()
{
Guid guid = Guid.NewGuid();
var message = new ExtractionRequestInfoMessage
{
ExtractionModality = "MR",
JobSubmittedAt = _dateTimeProvider.UtcNow(),
ProjectNumber = "1234",
ExtractionJobIdentifier = guid,
ExtractionDirectory = "test/directory",
KeyTag = "KeyTag",
KeyValueCount = 123,
IsIdentifiableExtraction = true,
IsNoFilterExtraction = true,
};
MongoExtractJobDoc doc = MongoExtractJobDoc.FromMessage(message, _messageHeader, _dateTimeProvider);
var expected = new MongoExtractJobDoc(
guid,
MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, _messageHeader, _dateTimeProvider),
"1234",
ExtractJobStatus.WaitingForCollectionInfo,
"test/directory",
_dateTimeProvider.UtcNow(),
"KeyTag",
123,
"MR",
isIdentifiableExtraction: true,
isNoFilterExtraction: true,
null);
Assert.AreEqual(expected, doc);
}
public void TestMongoExtractionMessageHeaderDoc_GetHashCode()
{
Guid guid = Guid.NewGuid();
DateTime now = DateTime.UtcNow;
var doc1 = new MongoExtractionMessageHeaderDoc(guid, guid, "Test1", 123, now, "parents", now);
var doc2 = new MongoExtractionMessageHeaderDoc(guid, guid, "Test1", 123, now, "parents", now);
Assert.AreEqual(doc1.GetHashCode(), doc2.GetHashCode());
}
public void TestMongoRejectedKeyInfoDoc_GetHashCode()
{
Guid guid = Guid.NewGuid();
var rejectReasons = new Dictionary <string, int>
{
{ "Reject1", 1 },
{ "Reject2", 2 },
};
var doc1 = new MongoRejectedKeyInfoDoc(MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, _testHeader, _dateTimeProvider), rejectReasons);
var doc2 = new MongoRejectedKeyInfoDoc(MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, _testHeader, _dateTimeProvider), rejectReasons);
Assert.AreEqual(doc1.GetHashCode(), doc2.GetHashCode());
}
public void TestPersistMessageToStoreImpl_ExtractionRequestInfoMessage()
{
Guid guid = Guid.NewGuid();
var testExtractionRequestInfoMessage = new ExtractionRequestInfoMessage
{
ExtractionJobIdentifier = guid,
ProjectNumber = "1234-5678",
ExtractionDirectory = "1234-5678/testExtract",
JobSubmittedAt = _dateTimeProvider.UtcNow(),
KeyTag = "StudyInstanceUID",
KeyValueCount = 1,
ExtractionModality = "CT",
IsIdentifiableExtraction = true,
IsNoFilterExtraction = true,
};
var testHeader = new MessageHeader
{
MessageGuid = Guid.NewGuid(),
OriginalPublishTimestamp = MessageHeader.UnixTime(_dateTimeProvider.UtcNow()),
Parents = new[] { Guid.NewGuid(), },
ProducerExecutableName = "MongoExtractStoreTests",
ProducerProcessID = 1234,
};
var client = new TestMongoClient();
var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
store.PersistMessageToStore(testExtractionRequestInfoMessage, testHeader);
Dictionary <Guid, MongoExtractJobDoc> docs = client.ExtractionDatabase.InProgressCollection.Documents;
Assert.AreEqual(docs.Count, 1);
MongoExtractJobDoc extractJob = docs.Values.ToList()[0];
var expected = new MongoExtractJobDoc(
guid,
MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, testHeader, _dateTimeProvider),
"1234-5678",
ExtractJobStatus.WaitingForCollectionInfo,
"1234-5678/testExtract",
_dateTimeProvider.UtcNow(),
"StudyInstanceUID",
1,
"CT",
isIdentifiableExtraction: true,
isNoFilterExtraction: true,
null);
Assert.AreEqual(expected, extractJob);
}
protected override void PersistMessageToStoreImpl(ExtractedFileVerificationMessage message, IMessageHeader header)
{
if (InCompletedJobCollection(message.ExtractionJobIdentifier))
{
throw new ApplicationException("Received an ExtractedFileVerificationMessage for a job that is already completed");
}
var newStatus = new MongoFileStatusDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(message.ExtractionJobIdentifier, header, _dateTimeProvider),
message.DicomFilePath,
message.OutputFilePath,
wasAnonymised: true,
isIdentifiable: message.IsIdentifiable,
ExtractedFileStatus.Anonymised,
statusMessage: message.Report);
_database
.GetCollection <MongoFileStatusDoc>(StatusCollectionName(message.ExtractionJobIdentifier))
.InsertOne(newStatus);
}
public void Test_MongoFileStatusDoc_IsIdentifiable_StatusMessage()
{
Assert.Throws <ArgumentNullException>(() =>
new MongoFileStatusDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(Guid.NewGuid(), new MessageHeader(), new DateTimeProvider()),
"input.dcm",
"anon.dcm",
true,
false,
ExtractedFileStatus.Anonymised,
null));
Assert.DoesNotThrow(() =>
new MongoFileStatusDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(Guid.NewGuid(), new MessageHeader(), new DateTimeProvider()),
"input.dcm",
"anon.dcm",
true,
true,
ExtractedFileStatus.Anonymised,
null));
}
public void Test_MongoExtractJobDoc_CopyConstructor()
{
var jobid = Guid.NewGuid();
var original = new MongoExtractJobDoc(
jobid,
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobid, _messageHeader, _dateTimeProvider),
"1234",
ExtractJobStatus.WaitingForCollectionInfo,
"test/directory",
_dateTimeProvider.UtcNow(),
"KeyTag",
123,
"MR",
isIdentifiableExtraction: true,
isNoFilterExtraction: true,
new MongoFailedJobInfoDoc(new Exception("foo"), _dateTimeProvider)
);
var copied = new MongoExtractJobDoc(original);
Assert.AreEqual(original, copied);
}
public void TestMongoFileStatusDoc_GetHashCode()
{
Guid guid = Guid.NewGuid();
var doc1 = new MongoFileStatusDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, _messageHeader, _dateTimeProvider),
"input.dcm",
"anon.dcm",
true,
false,
ExtractedFileStatus.Anonymised,
"anonymised");
var doc2 = new MongoFileStatusDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, _messageHeader, _dateTimeProvider),
"input.dcm",
"anon.dcm",
true,
false,
ExtractedFileStatus.Anonymised,
"anonymised");
Assert.AreEqual(doc1.GetHashCode(), doc2.GetHashCode());
}
public void TestPersistMessageToStoreImpl_IsIdentifiableMessage()
{
var client = new TestMongoClient();
var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
Guid jobId = Guid.NewGuid();
var testIsIdentifiableMessage = new ExtractedFileVerificationMessage
{
OutputFilePath = "anon.dcm",
JobSubmittedAt = _dateTimeProvider.UtcNow(),
ProjectNumber = "1234",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = "1234/test",
DicomFilePath = "original.dcm",
IsIdentifiable = false,
Report = "[]", // NOTE(rkm 2020-03-10) An "empty" report from IsIdentifiable
};
var header = new MessageHeader();
store.PersistMessageToStore(testIsIdentifiableMessage, header);
Dictionary <Guid, MongoFileStatusDoc> docs = client.ExtractionDatabase.StatusCollections[$"statuses_{jobId}"].Documents;
Assert.AreEqual(docs.Count, 1);
MongoFileStatusDoc statusDoc = docs.Values.ToList()[0];
var expected = new MongoFileStatusDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, header, _dateTimeProvider),
"original.dcm",
"anon.dcm",
true,
false,
ExtractedFileStatus.Anonymised,
"[]");
Assert.True(statusDoc.Equals(expected));
}
public void TestPersistMessageToStoreImpl_ExtractFileStatusMessage()
{
var client = new TestMongoClient();
var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
Guid jobId = Guid.NewGuid();
var testExtractFileStatusMessage = new ExtractedFileStatusMessage
{
OutputFilePath = "anon.dcm",
JobSubmittedAt = _dateTimeProvider.UtcNow(),
Status = ExtractedFileStatus.ErrorWontRetry,
ProjectNumber = "1234",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = "1234/test",
StatusMessage = "Could not anonymise",
DicomFilePath = "original.dcm",
};
var header = new MessageHeader();
store.PersistMessageToStore(testExtractFileStatusMessage, header);
Dictionary <Guid, MongoFileStatusDoc> docs = client.ExtractionDatabase.StatusCollections[$"statuses_{jobId}"].Documents;
Assert.AreEqual(docs.Count, 1);
MongoFileStatusDoc statusDoc = docs.Values.ToList()[0];
var expected = new MongoFileStatusDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, header, _dateTimeProvider),
"original.dcm",
"anon.dcm",
false,
true,
ExtractedFileStatus.ErrorWontRetry,
"Could not anonymise");
Assert.True(statusDoc.Equals(expected));
}
public void TestMongoRejectedKeyInfoDoc_FromMessage()
{
var mockMessage = new Mock <ExtractFileCollectionInfoMessage>();
Guid jobId = Guid.NewGuid();
mockMessage.Object.ExtractionJobIdentifier = jobId;
mockMessage.Object.RejectionReasons = new Dictionary <string, int>
{
{ "Reject1", 1 },
{ "Reject2", 2 },
};
MongoRejectedKeyInfoDoc doc = MongoRejectedKeyInfoDoc.FromMessage(mockMessage.Object, _testHeader, _dateTimeProvider);
var expected = new MongoRejectedKeyInfoDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, _testHeader, _dateTimeProvider),
new Dictionary <string, int>
{
{ "Reject1", 1 },
{ "Reject2", 2 },
});
Assert.AreEqual(expected, doc);
}
public void TestMongoExpectedFilesDoc_FromMessage()
{
var mockMessage = new Mock <ExtractFileCollectionInfoMessage>();
mockMessage.Object.KeyValue = "TestKey";
Guid jobId = Guid.NewGuid();
mockMessage.Object.ExtractionJobIdentifier = jobId;
var header1 = new MessageHeader();
var header2 = new MessageHeader();
mockMessage.Object.ExtractFileMessagesDispatched = new JsonCompatibleDictionary <MessageHeader, string>
{
{ header1, "AnonFile1.dcm" },
{ header2, "AnonFile2.dcm" },
};
mockMessage.Object.RejectionReasons = new Dictionary <string, int>
{
{ "Reject1", 1 },
{ "Reject2", 2 },
};
MongoExpectedFilesDoc doc = MongoExpectedFilesDoc.FromMessage(mockMessage.Object, _testHeader, _dateTimeProvider);
var expected = new MongoExpectedFilesDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, _testHeader, _dateTimeProvider),
"TestKey",
new HashSet <MongoExpectedFileInfoDoc>
{
new MongoExpectedFileInfoDoc(header1.MessageGuid, "AnonFile1.dcm"),
new MongoExpectedFileInfoDoc(header2.MessageGuid, "AnonFile2.dcm"),
},
MongoRejectedKeyInfoDoc.FromMessage(mockMessage.Object, _testHeader, _dateTimeProvider));
Assert.AreEqual(expected, doc);
}
public void TestMarkJobFailedImpl()
{
Guid jobId = Guid.NewGuid();
var testJob = new MongoExtractJobDoc(
jobId,
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider),
"1234",
ExtractJobStatus.Failed,
"test/dir",
_dateTimeProvider.UtcNow(),
"1.2.3.4",
123,
"MR",
isIdentifiableExtraction: true,
isNoFilterExtraction: true,
null);
var client = new TestMongoClient();
var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
// Assert that an exception is thrown for a non-existent job
Assert.Throws <ApplicationException>(() => store.MarkJobFailed(Guid.NewGuid(), new Exception()));
client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once);
client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never);
// Assert that a job can't be failed twice
client = new TestMongoClient();
store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
client.ExtractionDatabase.InProgressCollection.InsertOne(testJob);
client.MockSessionHandle.Reset();
Assert.Throws <ApplicationException>(() => store.MarkJobFailed(jobId, new Exception()));
client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once);
client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never);
// Check we handle a bad ReplaceOneResult
client = new TestMongoClient();
store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
testJob.JobStatus = ExtractJobStatus.WaitingForCollectionInfo;
client.ExtractionDatabase.InProgressCollection.InsertOne(testJob);
client.ExtractionDatabase.InProgressCollection.RejectChanges = true;
client.MockSessionHandle.Reset();
Assert.Throws <ApplicationException>(() => store.MarkJobFailed(jobId, new Exception()));
client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once);
client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never);
// Check happy path
client = new TestMongoClient();
store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
testJob.JobStatus = ExtractJobStatus.WaitingForCollectionInfo;
testJob.FailedJobInfoDoc = null;
client.ExtractionDatabase.InProgressCollection.InsertOne(testJob);
client.MockSessionHandle.Reset();
store.MarkJobFailed(jobId, new Exception("TestMarkJobFailedImpl"));
client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Never);
client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Once);
Dictionary <Guid, MongoExtractJobDoc> docs = client.ExtractionDatabase.InProgressCollection.Documents;
Assert.AreEqual(1, docs.Count);
MongoExtractJobDoc failedDoc = docs[jobId];
Assert.AreEqual(ExtractJobStatus.Failed, failedDoc.JobStatus);
Assert.NotNull(failedDoc.FailedJobInfoDoc);
Assert.AreEqual("TestMarkJobFailedImpl", failedDoc.FailedJobInfoDoc.ExceptionMessage);
}
public void TestCompleteJobImpl()
{
Guid jobId = Guid.NewGuid();
var testJob = new MongoExtractJobDoc(
jobId,
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider),
"1234",
ExtractJobStatus.Failed,
"test/dir",
_dateTimeProvider.UtcNow(),
"SeriesInstanceUID",
1,
"MR",
isIdentifiableExtraction: true,
isNoFilterExtraction: true,
null);
var testMongoExpectedFilesDoc = new MongoExpectedFilesDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider),
"1.2.3.4",
new HashSet <MongoExpectedFileInfoDoc>
{
new MongoExpectedFileInfoDoc(Guid.NewGuid(), "anon1.dcm"),
},
new MongoRejectedKeyInfoDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider),
new Dictionary <string, int>())
);
var testMongoFileStatusDoc = new MongoFileStatusDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider),
"input.dcm",
"anon1.dcm",
true,
false,
ExtractedFileStatus.Anonymised,
"Verified");
var client = new TestMongoClient();
var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
// Assert that an exception is thrown for a non-existent job
Assert.Throws <ApplicationException>(() => store.MarkJobCompleted(Guid.NewGuid()));
client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once);
client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never);
// Assert that an exception is thrown for a job which is marked as failed
client = new TestMongoClient();
store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
client.ExtractionDatabase.InProgressCollection.InsertOne(testJob);
client.MockSessionHandle.Reset();
Assert.Throws <ApplicationException>(() => store.MarkJobCompleted(Guid.NewGuid()));
client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once);
client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never);
// Check that we handle a failed insertion
client = new TestMongoClient();
store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
testJob.JobStatus = ExtractJobStatus.Completed;
client.ExtractionDatabase.InProgressCollection.InsertOne(testJob);
client.ExtractionDatabase.CompletedJobCollection.RejectChanges = true;
client.MockSessionHandle.Reset();
Assert.Throws <Exception>(() => store.MarkJobCompleted(jobId));
client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once);
client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never);
// Check we handle a bad DeleteResult
client = new TestMongoClient();
store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
client.ExtractionDatabase.InProgressCollection.RejectChanges = true;
client.MockSessionHandle.Reset();
Assert.Throws <ApplicationException>(() => store.MarkJobCompleted(jobId));
client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once);
client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never);
// Check we handle missing expectedFiles collection
client = new TestMongoClient();
store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
client.ExtractionDatabase.InProgressCollection.InsertOne(testJob);
client.MockSessionHandle.Reset();
Assert.Throws <ApplicationException>(() => store.MarkJobCompleted(jobId));
client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once);
client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never);
// Check we handle missing statuses collection
client = new TestMongoClient();
store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
client.ExtractionDatabase.InProgressCollection.InsertOne(testJob);
client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"] = new MockExtractCollection <Guid, MongoExpectedFilesDoc>();
client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"].InsertOne(testMongoExpectedFilesDoc);
client.MockSessionHandle.Reset();
Assert.Throws <ApplicationException>(() => store.MarkJobCompleted(jobId));
client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Once);
client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Never);
// Check happy path
client = new TestMongoClient();
store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
client.ExtractionDatabase.InProgressCollection.InsertOne(testJob);
client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"] = new MockExtractCollection <Guid, MongoExpectedFilesDoc>();
client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"].InsertOne(testMongoExpectedFilesDoc);
client.ExtractionDatabase.StatusCollections[$"statuses_{jobId}"] = new MockExtractCollection <Guid, MongoFileStatusDoc>();
client.ExtractionDatabase.StatusCollections[$"statuses_{jobId}"].InsertOne(testMongoFileStatusDoc);
client.MockSessionHandle.Reset();
store.MarkJobCompleted(jobId);
client.MockSessionHandle.Verify(x => x.AbortTransaction(It.IsAny <CancellationToken>()), Times.Never);
client.MockSessionHandle.Verify(x => x.CommitTransaction(It.IsAny <CancellationToken>()), Times.Once);
Assert.AreEqual(1, client.ExtractionDatabase.ExpectedFilesCollections.Count);
Assert.AreEqual(1, client.ExtractionDatabase.StatusCollections.Count);
}
public void TestGetReadJobsImpl()
{
Guid jobId = Guid.NewGuid();
var testJob = new MongoExtractJobDoc(
jobId,
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider),
"1234",
ExtractJobStatus.Failed,
"test/dir",
_dateTimeProvider.UtcNow(),
"SeriesInstanceUID",
1,
"MR",
isIdentifiableExtraction: true,
isNoFilterExtraction: true,
null);
var testMongoExpectedFilesDoc = new MongoExpectedFilesDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider),
"1.2.3.4",
new HashSet <MongoExpectedFileInfoDoc>
{
new MongoExpectedFileInfoDoc(Guid.NewGuid(), "anon1.dcm"),
},
new MongoRejectedKeyInfoDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider),
new Dictionary <string, int>())
);
var testMongoFileStatusDoc = new MongoFileStatusDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider),
"input.dcm",
"anon1.dcm",
true,
false,
ExtractedFileStatus.Anonymised,
"Verified");
var client = new TestMongoClient();
var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
// Assert that jobs marked as failed are not returned
client.ExtractionDatabase.InProgressCollection.InsertOne(testJob);
client.ExtractionDatabase.InProgressCollection.RejectChanges = true;
Assert.AreEqual(0, store.GetReadyJobs().Count);
// Assert that an in progress job is not returned
client = new TestMongoClient();
store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
testJob.JobStatus = ExtractJobStatus.WaitingForCollectionInfo;
client.ExtractionDatabase.InProgressCollection.InsertOne(testJob);
client.ExtractionDatabase.InProgressCollection.RejectChanges = true;
Assert.AreEqual(0, store.GetReadyJobs().Count);
// Check we handle a bad ReplaceOneResult
client = new TestMongoClient();
store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
testJob.JobStatus = ExtractJobStatus.WaitingForCollectionInfo;
client.ExtractionDatabase.InProgressCollection.InsertOne(testJob);
client.ExtractionDatabase.InProgressCollection.RejectChanges = true;
client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"] = new MockExtractCollection <Guid, MongoExpectedFilesDoc>();
client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"].InsertOne(testMongoExpectedFilesDoc);
Assert.Throws <ApplicationException>(() => store.GetReadyJobs());
// Check happy path
client = new TestMongoClient();
store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
testJob.JobStatus = ExtractJobStatus.WaitingForCollectionInfo;
client.ExtractionDatabase.InProgressCollection.InsertOne(testJob);
client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"] = new MockExtractCollection <Guid, MongoExpectedFilesDoc>();
client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"].InsertOne(testMongoExpectedFilesDoc);
Assert.AreEqual(0, store.GetReadyJobs().Count);
Assert.AreEqual(ExtractJobStatus.WaitingForStatuses, client.ExtractionDatabase.InProgressCollection.Documents.Single().Value.JobStatus);
client.ExtractionDatabase.StatusCollections[$"statuses_{jobId}"] = new MockExtractCollection <Guid, MongoFileStatusDoc>();
client.ExtractionDatabase.StatusCollections[$"statuses_{jobId}"].InsertOne(testMongoFileStatusDoc);
ExtractJobInfo job = store.GetReadyJobs().Single();
Assert.AreEqual(ExtractJobStatus.ReadyForChecks, job.JobStatus);
Assert.AreEqual(ExtractJobStatus.ReadyForChecks, client.ExtractionDatabase.InProgressCollection.Documents.Single().Value.JobStatus);
}
public void TestPersistMessageToStoreImpl_ExtractFileCollectionInfoMessage()
{
Guid jobId = Guid.NewGuid();
var header1 = new MessageHeader();
var header2 = new MessageHeader();
var testExtractFileCollectionInfoMessage = new ExtractFileCollectionInfoMessage
{
ExtractionJobIdentifier = jobId,
ProjectNumber = "1234-5678",
RejectionReasons = new Dictionary <string, int>
{
{ "reject1", 1 },
{ "reject2", 2 },
},
JobSubmittedAt = DateTime.UtcNow,
ExtractionDirectory = "1234-5678/testExtract",
ExtractFileMessagesDispatched = new JsonCompatibleDictionary <MessageHeader, string>
{
{ header1, "file1" },
{ header2, "file2" }
},
KeyValue = "series-1",
};
var header = new MessageHeader
{
MessageGuid = Guid.NewGuid(),
OriginalPublishTimestamp = MessageHeader.UnixTimeNow(),
Parents = new[] { Guid.NewGuid(), },
ProducerExecutableName = "MongoExtractStoreTests",
ProducerProcessID = 1234,
};
var client = new TestMongoClient();
var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider);
store.PersistMessageToStore(testExtractFileCollectionInfoMessage, header);
Dictionary <Guid, MongoExpectedFilesDoc> docs = client.ExtractionDatabase.ExpectedFilesCollections[$"expectedFiles_{jobId}"].Documents;
Assert.AreEqual(docs.Count, 1);
MongoExpectedFilesDoc extractJob = docs.Values.ToList()[0];
var expected = new MongoExpectedFilesDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, header, _dateTimeProvider),
"series-1",
new HashSet <MongoExpectedFileInfoDoc>
{
new MongoExpectedFileInfoDoc(header1.MessageGuid, "file1"),
new MongoExpectedFileInfoDoc(header2.MessageGuid, "file2"),
},
new MongoRejectedKeyInfoDoc(
MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, header, _dateTimeProvider),
new Dictionary <string, int>
{
{ "reject1", 1 },
{ "reject2", 2 },
})
);
Assert.True(extractJob.Equals(expected));
}
