public void SendHeader()
{
var o = new GlobalOptionsFactory().Load();
var consumerOptions = new ConsumerOptions();
consumerOptions.QueueName = "TEST.HeaderPreservationTest_Read1";
consumerOptions.AutoAck = false;
consumerOptions.QoSPrefetchCount = 1;
TestConsumer consumer;
using (var tester = new MicroserviceTester(o.RabbitOptions, consumerOptions))
{
var header = new MessageHeader();
header.MessageGuid = Guid.Parse("5afce68f-c270-4bf3-b327-756f6038bb76");
header.Parents = new[] { Guid.Parse("12345678-c270-4bf3-b327-756f6038bb76"), Guid.Parse("87654321-c270-4bf3-b327-756f6038bb76") };
tester.SendMessage(consumerOptions, header, new TestMessage()
{
Message = "hi"
});
consumer = new TestConsumer();
var a = new RabbitMqAdapter(o.RabbitOptions.CreateConnectionFactory(), "TestHost");
a.StartConsumer(consumerOptions, consumer);
TestTimelineAwaiter awaiter = new TestTimelineAwaiter();
awaiter.Await(() => consumer.Failed || consumer.Passed, "timed out", 5000);
a.Shutdown(RabbitMqAdapter.DefaultOperationTimeout);
}
Assert.IsTrue(consumer.Passed);
}
public void TestClassifierName_ValidClassifier()
{
var options = new GlobalOptionsFactory().Load();
var testDcm = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(TestClassifierName_ValidClassifier), "f1.dcm")); Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(TestClassifierName_ValidClassifier), "f1.dcm");
TestData.Create(testDcm);
using (var tester = new MicroserviceTester(options.RabbitOptions, options.IsIdentifiableOptions))
{
tester.CreateExchange(options.IsIdentifiableOptions.IsIdentifiableProducerOptions.ExchangeName, null);
options.IsIdentifiableOptions.ClassifierType = typeof(RejectAllClassifier).FullName;
options.IsIdentifiableOptions.DataDirectory = TestContext.CurrentContext.TestDirectory;
var host = new IsIdentifiableHost(options, new IsIdentifiableServiceOptions());
Assert.IsNotNull(host);
host.Start();
tester.SendMessage(options.IsIdentifiableOptions, new ExtractedFileStatusMessage()
{
DicomFilePath = "yay.dcm",
OutputFilePath = testDcm.FullName,
ProjectNumber = "100",
ExtractionDirectory = "./fish",
StatusMessage = "yay!",
Status = ExtractedFileStatus.Anonymised
});
var awaiter = new TestTimelineAwaiter();
awaiter.Await(() => host.Consumer.AckCount == 1);
}
}
public void Integration_HappyPath(ReportFormat reportFormat)
{
// Test messages:
// - series-1
// - series-1-anon-1.dcm -> valid
using var pf = new PathFixtures($"Integration_HappyPath_{reportFormat}");
var jobId = Guid.NewGuid();
var testExtractionRequestInfoMessage = new ExtractionRequestInfoMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
KeyTag = "SeriesInstanceUID",
KeyValueCount = 1,
};
var testExtractFileCollectionInfoMessage = new ExtractFileCollectionInfoMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
ExtractFileMessagesDispatched = new JsonCompatibleDictionary <MessageHeader, string>
{
{ new MessageHeader(), "series-1-anon-1.dcm" },
},
RejectionReasons = new Dictionary <string, int>
{
{ "rejected - blah", 1 },
},
KeyValue = "series-1",
};
var testIsIdentifiableMessage = new ExtractedFileVerificationMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
OutputFilePath = "series-1-anon-1.dcm",
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
IsIdentifiable = false,
Report = "[]",
DicomFilePath = "series-1-orig-1.dcm",
};
GlobalOptions globals = new GlobalOptionsFactory().Load();
VerifyReports(
globals,
pf,
reportFormat,
new[]
{
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.ExtractRequestInfoOptions, testExtractionRequestInfoMessage),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.FileCollectionInfoOptions, testExtractFileCollectionInfoMessage),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.VerificationStatusOptions, testIsIdentifiableMessage),
}
);
}
public void Test_ExtractionRequestQueueConsumer_IdentExtraction_RoutingKey()
{
GlobalOptions globals = new GlobalOptionsFactory().Load();
globals.CohortExtractorOptions.ExtractAnonRoutingKey = "";
globals.CohortExtractorOptions.ExtractIdentRoutingKey = "ident";
AssertMessagePublishedWithSpecifiedKey(globals, true, "ident");
}
public void OneTimeSetUp()
{
TestLogger.Setup();
GlobalOptions globalOptions = new GlobalOptionsFactory().Load();
_mongoOptions = globalOptions.MongoDatabases.DicomStoreOptions;
}
public void Test_GlobalOptions_FileReadOption_ThrowsException()
{
GlobalOptions g = new GlobalOptionsFactory().Load();
g.DicomTagReaderOptions.FileReadOption = "SkipLargeTags";
Assert.Throws <ApplicationException>(() => g.DicomTagReaderOptions.GetReadOption());
}
public void Test_GlobalOptionsUseTestValues_Nulls()
{
GlobalOptions g = new GlobalOptionsFactory().Load();
Assert.IsNotNull(g.RabbitOptions.RabbitMqHostName);
g.UseTestValues(null, null, null, null, null);
Assert.IsNull(g.RabbitOptions.RabbitMqHostName);
}
public void TestClassifierName_NoClassifier()
{
var options = new GlobalOptionsFactory().Load();
options.IsIdentifiableOptions.ClassifierType = "";
var ex = Assert.Throws <ArgumentException>(() => new IsIdentifiableHost(options, new IsIdentifiableServiceOptions()));
StringAssert.Contains("No IClassifier has been set in options. Enter a value for " + nameof(options.IsIdentifiableOptions.ClassifierType), ex.Message);
}
public void GlobalOptions_Test()
{
GlobalOptions globals = new GlobalOptionsFactory().Load();
Assert.IsFalse(string.IsNullOrWhiteSpace(globals.RabbitOptions.RabbitMqHostName));
Assert.IsFalse(string.IsNullOrWhiteSpace(globals.FileSystemOptions.FileSystemRoot));
Assert.IsFalse(string.IsNullOrWhiteSpace(globals.RDMPOptions.CatalogueConnectionString));
Assert.IsFalse(string.IsNullOrWhiteSpace(globals.RDMPOptions.DataExportConnectionString));
}
public static GlobalOptions GetNewMongoDbPopulatorOptions()
{
GlobalOptions options = new GlobalOptionsFactory().Load();
options.MongoDatabases.DicomStoreOptions.DatabaseName = TestDbName;
options.MongoDbPopulatorOptions.MongoDbFlushTime = 1; //1 second
return(options);
}
public void TestClassifierName_NotRecognized()
{
var options = new GlobalOptionsFactory().Load();
options.IsIdentifiableOptions.DataDirectory = TestContext.CurrentContext.WorkDirectory;
options.IsIdentifiableOptions.ClassifierType = "HappyFunTimes";
var ex = Assert.Throws <TypeLoadException>(() => new IsIdentifiableHost(options, new IsIdentifiableServiceOptions()));
StringAssert.Contains("Could not load type 'HappyFunTimes' from", ex.Message);
}
public void TestDecorators()
{
var factory = new GlobalOptionsFactory(new List <IOptionsDecorator> {
new TestDecorator()
});
var g = factory.Load();
Assert.AreEqual("FFFFF", g.MongoDatabases.DeadLetterStoreOptions.DatabaseName);
Assert.AreEqual("FFFFF", g.MongoDatabases.DicomStoreOptions.DatabaseName);
Assert.AreEqual("FFFFF", g.MongoDatabases.ExtractionStoreOptions.DatabaseName);
}
public void TestSwapCache()
{
var mappingDataTable = new DataTable("IdMap");
mappingDataTable.Columns.Add("priv");
mappingDataTable.Columns.Add("pub");
mappingDataTable.Rows.Add("CHI-1", "REP-1");
mappingDataTable.Rows.Add("CHI-2", "REP-2");
DiscoveredDatabase db = GetCleanedServer(DatabaseType.MicrosoftSQLServer);
GlobalOptions options = new GlobalOptionsFactory().Load();
options.IdentifierMapperOptions = new IdentifierMapperOptions
{
MappingConnectionString = db.Server.Builder.ConnectionString,
MappingTableName = db.CreateTable("IdMap", mappingDataTable).GetFullyQualifiedName(),
SwapColumnName = "priv",
ReplacementColumnName = "pub",
MappingDatabaseType = DatabaseType.MicrosoftSQLServer,
TimeoutInSeconds = 500
};
var swapper = new TableLookupSwapper();
swapper.Setup(options.IdentifierMapperOptions);
string _;
string swapped = swapper.GetSubstitutionFor("CHI-1", out _);
Assert.AreEqual("REP-1", swapped);
swapped = swapper.GetSubstitutionFor("CHI-1", out _);
Assert.AreEqual("REP-1", swapped);
Assert.AreEqual(2, swapper.Success);
Assert.AreEqual(1, swapper.CacheHit);
swapped = swapper.GetSubstitutionFor("CHI-2", out _);
Assert.AreEqual("REP-2", swapped);
swapped = swapper.GetSubstitutionFor("CHI-2", out _);
Assert.AreEqual("REP-2", swapped);
Assert.AreEqual(4, swapper.Success);
Assert.AreEqual(2, swapper.CacheHit);
// Just to make sure...
swapped = swapper.GetSubstitutionFor("CHI-1", out _);
Assert.AreEqual("REP-1", swapped);
Assert.AreEqual(5, swapper.Success);
Assert.AreEqual(2, swapper.CacheHit);
}
public void Test_FileCopierHost_HappyPath()
{
GlobalOptions globals = new GlobalOptionsFactory().Load();
globals.FileSystemOptions.FileSystemRoot = "root";
globals.FileSystemOptions.ExtractRoot = "exroot";
using var tester = new MicroserviceTester(globals.RabbitOptions, globals.FileCopierOptions);
string outputQueueName = globals.FileCopierOptions.CopyStatusProducerOptions.ExchangeName.Replace("Exchange", "Queue");
tester.CreateExchange(
globals.FileCopierOptions.CopyStatusProducerOptions.ExchangeName,
outputQueueName,
false,
globals.FileCopierOptions.NoVerifyRoutingKey);
var mockFileSystem = new MockFileSystem();
mockFileSystem.AddDirectory(globals.FileSystemOptions.FileSystemRoot);
mockFileSystem.AddDirectory(globals.FileSystemOptions.ExtractRoot);
mockFileSystem.AddFile(mockFileSystem.Path.Combine(globals.FileSystemOptions.FileSystemRoot, "file.dcm"), MockFileData.NullObject);
var host = new FileCopierHost(globals, mockFileSystem);
tester.StopOnDispose.Add(host);
host.Start();
var message = new ExtractFileMessage
{
ExtractionJobIdentifier = Guid.NewGuid(),
JobSubmittedAt = DateTime.UtcNow,
ProjectNumber = "1234",
ExtractionDirectory = "1234/foo",
DicomFilePath = "file.dcm",
IsIdentifiableExtraction = true,
OutputPath = "output.dcm",
};
tester.SendMessage(globals.FileCopierOptions, message);
using IConnection conn = tester.Factory.CreateConnection();
using IModel model = conn.CreateModel();
var consumer = new EventingBasicConsumer(model);
ExtractedFileStatusMessage statusMessage = null;
consumer.Received += (_, ea) => statusMessage = JsonConvert.DeserializeObject <ExtractedFileStatusMessage>(Encoding.UTF8.GetString(ea.Body.ToArray()));
model.BasicConsume(outputQueueName, true, "", consumer);
new TestTimelineAwaiter().Await(() => statusMessage != null);
Assert.AreEqual(ExtractedFileStatus.Copied, statusMessage.Status);
}
public void ReportNewLine_LoadFromYaml_EscapesNewlines()
{
string yaml = @"
CurrentDirectory:
CohortPackagerOptions:
ReportNewLine: '\r\n'
";
string tmpConfig = Path.GetTempFileName() + ".yaml";
File.WriteAllText(tmpConfig, yaml);
GlobalOptions globals = new GlobalOptionsFactory().Load(tmpConfig);
Assert.AreEqual(WindowsNewLine, globals.CohortPackagerOptions.ReportNewLine);
}
public void TestIsIdentifiable_TesseractStanfordDicomFileClassifier()
{
var options = new GlobalOptionsFactory().Load();
// Create a test data directory containing IsIdentifiableRules with 0 rules, and tessdata with the eng.traineddata classifier
// TODO(rkm 2020-04-14) This is a stop-gap solution until the tests are properly refactored
var testRulesDir = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, "data", "IsIdentifiableRules"));
testRulesDir.Create();
options.IsIdentifiableOptions.DataDirectory = testRulesDir.Parent.FullName;
var tessDir = new DirectoryInfo(Path.Combine(testRulesDir.Parent.FullName, "tessdata"));
tessDir.Create();
var dest = Path.Combine(tessDir.FullName, "eng.traineddata");
if (!File.Exists(dest))
{
File.Copy(Path.Combine(DataDirectory, "tessdata", "eng.traineddata"), dest);
}
var testDcm = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(TestIsIdentifiable_TesseractStanfordDicomFileClassifier), "f1.dcm"));
Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(TestClassifierName_ValidClassifier), "f1.dcm");
TestData.Create(testDcm);
using (var tester = new MicroserviceTester(options.RabbitOptions, options.IsIdentifiableOptions))
{
options.IsIdentifiableOptions.ClassifierType = typeof(TesseractStanfordDicomFileClassifier).FullName;
var host = new IsIdentifiableHost(options, new IsIdentifiableServiceOptions());
host.Start();
tester.SendMessage(options.IsIdentifiableOptions, new ExtractedFileStatusMessage
{
DicomFilePath = "yay.dcm",
OutputFilePath = testDcm.FullName,
ProjectNumber = "100",
ExtractionDirectory = "./fish",
StatusMessage = "yay!",
Status = ExtractedFileStatus.Anonymised
});
var awaiter = new TestTimelineAwaiter();
awaiter.Await(() => host.Consumer.AckCount == 1 || host.Consumer.NackCount == 1);
Assert.AreEqual(1, host.Consumer.AckCount, "Tesseract not acking");
}
}
public void RabbitAvailable()
{
var options = new GlobalOptionsFactory().Load();
var rabbitOptions = options.RabbitOptions;
Console.WriteLine("Checking the following configuration:" + Environment.NewLine + rabbitOptions);
try
{
var adapter = new RabbitMqAdapter(rabbitOptions.CreateConnectionFactory(), "TestHost");
}
catch (Exception)
{
Assert.Fail("Could not access Rabbit MQ Server");
}
Assert.Pass();
}
public void OneTimeSetUp()
{
TestLogger.Setup();
GlobalOptions globals = new GlobalOptionsFactory().Load();
var testRulesDir = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, "data", "IsIdentifiableRules"));
testRulesDir.Create();
globals.IsIdentifiableOptions.DataDirectory = testRulesDir.Parent.FullName;
_tessDir = new DirectoryInfo(Path.Combine(testRulesDir.Parent.FullName, "tessdata"));
_tessDir.Create();
string dest = Path.Combine(_tessDir.FullName, "eng.traineddata");
if (!File.Exists(dest))
{
File.Copy(Path.Combine(DataDirectory, "tessdata", "eng.traineddata"), dest);
}
}
public void TestCreatingNamer_CorrectType(DatabaseType dbType, string typeName, Type expectedType)
{
var db = GetCleanedServer(dbType);
var dt = new DataTable();
dt.Columns.Add("Hi");
dt.Rows.Add("There");
var tbl = db.CreateTable("DicomRelationalMapperHostTests", dt);
var cata = Import(tbl);
var globals = new GlobalOptionsFactory().Load();
var consumerOptions = globals.DicomRelationalMapperOptions;
var lmd = new LoadMetadata(CatalogueRepository, "MyLoad");
cata.LoadMetadata_ID = lmd.ID;
cata.SaveToDatabase();
consumerOptions.LoadMetadataId = lmd.ID;
consumerOptions.DatabaseNamerType = typeName;
consumerOptions.Guid = Guid.Empty;
globals.RDMPOptions.CatalogueConnectionString = CatalogueRepository.DiscoveredServer.Builder.ConnectionString;
globals.RDMPOptions.DataExportConnectionString = DataExportRepository.DiscoveredServer.Builder.ConnectionString;
using (new MicroserviceTester(globals.RabbitOptions, globals.DicomRelationalMapperOptions))
{
using (var host = new DicomRelationalMapperHost(globals))
{
host.Start();
Assert.AreEqual(expectedType, host.Consumer.DatabaseNamer.GetType());
Assert.IsNotNull(host);
host.Stop("Test finished");
}
}
}
public void Test_Shutdown(Type consumerType)
{
MemoryTarget target = new MemoryTarget();
target.Layout = "${message}";
NLog.Config.SimpleConfigurator.ConfigureForTargetLogging(target, LogLevel.Debug);
var o = new GlobalOptionsFactory().Load();
var consumer = (IConsumer)Activator.CreateInstance(consumerType);
//connect to rabbit with a new consumer
using (var tester = new MicroserviceTester(o.RabbitOptions, new [] { _testConsumerOptions }))
{
_testAdapter.StartConsumer(_testConsumerOptions, consumer, true);
//send a message to trigger consumer behaviour
tester.SendMessage(_testConsumerOptions, new TestMessage());
//give the message time to get picked up
Thread.Sleep(3000);
//now attempt to shut down adapter
_testAdapter.Shutdown(RabbitMqAdapter.DefaultOperationTimeout);
string expectedErrorMessage = "nothing to see here";
if (consumer is SelfClosingConsumer)
{
expectedErrorMessage = "exiting (channel is closed)";
}
if (consumer is DoNothingConsumer)
{
expectedErrorMessage = "exiting (cancellation was requested)";
}
Assert.IsTrue(target.Logs.Any(s => s.Contains(expectedErrorMessage)), "Expected message was not found, messages were:" + string.Join(Environment.NewLine, target.Logs));
}
}
public void Integration_IdentifiableExtraction_HappyPath()
{
using var pf = new PathFixtures("Integration_IdentifiableExtraction_HappyPath");
var jobId = Guid.NewGuid();
var testExtractionRequestInfoMessage = new ExtractionRequestInfoMessage
{
ExtractionModality = "MR",
JobSubmittedAt = _dateTimeProvider.UtcNow(),
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
KeyTag = "StudyInstanceUID",
KeyValueCount = 1,
IsIdentifiableExtraction = true,
};
var testExtractFileCollectionInfoMessage = new ExtractFileCollectionInfoMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
ExtractFileMessagesDispatched = new JsonCompatibleDictionary <MessageHeader, string>
{
{ new MessageHeader(), "out1.dcm" },
{ new MessageHeader(), "out2.dcm" },
},
RejectionReasons = new Dictionary <string, int>
{
{ "rejected - blah", 1 },
},
KeyValue = "study-1",
IsIdentifiableExtraction = true,
};
var testExtractFileStatusMessage1 = new ExtractedFileStatusMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
OutputFilePath = "src.dcm",
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
Status = ExtractedFileStatus.Copied,
StatusMessage = null,
DicomFilePath = "study-1-orig-1.dcm",
IsIdentifiableExtraction = true,
};
var testExtractFileStatusMessage2 = new ExtractedFileStatusMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
OutputFilePath = "src_missing.dcm",
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
Status = ExtractedFileStatus.FileMissing,
StatusMessage = null,
DicomFilePath = "study-1-orig-2.dcm",
IsIdentifiableExtraction = true,
};
GlobalOptions globals = new GlobalOptionsFactory().Load();
VerifyReports(
globals,
pf,
ReportFormat.Combined,
new[]
{
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.ExtractRequestInfoOptions, testExtractionRequestInfoMessage),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.FileCollectionInfoOptions, testExtractFileCollectionInfoMessage),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.NoVerifyStatusOptions, testExtractFileStatusMessage1),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.NoVerifyStatusOptions, testExtractFileStatusMessage2),
}
);
}
public void Integration_BumpyRoad(ReportFormat reportFormat)
{
// Test messages:
// - series-1
// - series-1-anon-1.dcm -> valid
// - series-1-anon-2.dcm -> rejected
// - series-2
// - series-2-anon-1.dcm -> fails anonymisation
// - series-2-anon-2.dcm -> fails validation
using var pf = new PathFixtures($"Integration_BumpyRoad_{reportFormat}");
var jobId = Guid.NewGuid();
var testExtractionRequestInfoMessage = new ExtractionRequestInfoMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
KeyTag = "SeriesInstanceUID",
KeyValueCount = 2,
};
var testExtractFileCollectionInfoMessage1 = new ExtractFileCollectionInfoMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
ExtractFileMessagesDispatched = new JsonCompatibleDictionary <MessageHeader, string>
{
{ new MessageHeader(), "series-1-anon-1.dcm" },
},
RejectionReasons = new Dictionary <string, int>
{
{ "rejected - blah", 1 },
},
KeyValue = "series-1",
};
var testExtractFileCollectionInfoMessage2 = new ExtractFileCollectionInfoMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
ExtractFileMessagesDispatched = new JsonCompatibleDictionary <MessageHeader, string>
{
{ new MessageHeader(), "series-2-anon-1.dcm" },
{ new MessageHeader(), "series-2-anon-2.dcm" },
},
RejectionReasons = new Dictionary <string, int>(),
KeyValue = "series-2",
};
var testExtractFileStatusMessage = new ExtractedFileStatusMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
OutputFilePath = "series-2-anon-1.dcm",
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
Status = ExtractedFileStatus.ErrorWontRetry,
StatusMessage = "Couldn't anonymise",
DicomFilePath = "series-2-orig-1.dcm",
};
var testIsIdentifiableMessage1 = new ExtractedFileVerificationMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
OutputFilePath = "series-1-anon-1.dcm",
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
IsIdentifiable = false,
Report = "[]",
DicomFilePath = "series-1-orig-1.dcm",
};
const string failureReport = @"
[
{
'Parts': [],
'Resource': 'series-2-anon-2.dcm',
'ResourcePrimaryKey': '1.2.3.4',
'ProblemField': 'ScanOptions',
'ProblemValue': 'FOO'
}
]";
var testIsIdentifiableMessage2 = new ExtractedFileVerificationMessage
{
JobSubmittedAt = _dateTimeProvider.UtcNow(),
OutputFilePath = "series-2-anon-2.dcm",
ProjectNumber = "testProj1",
ExtractionJobIdentifier = jobId,
ExtractionDirectory = pf.ProjExtractDirRelative,
IsIdentifiable = true,
Report = failureReport,
DicomFilePath = "series-2-orig-2.dcm",
};
GlobalOptions globals = new GlobalOptionsFactory().Load();
VerifyReports(
globals,
pf,
reportFormat,
new[]
{
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.ExtractRequestInfoOptions, testExtractionRequestInfoMessage),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.FileCollectionInfoOptions, testExtractFileCollectionInfoMessage1),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.FileCollectionInfoOptions, testExtractFileCollectionInfoMessage2),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.NoVerifyStatusOptions, testExtractFileStatusMessage),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.VerificationStatusOptions, testIsIdentifiableMessage1),
new Tuple <ConsumerOptions, IMessage>(globals.CohortPackagerOptions.VerificationStatusOptions, testIsIdentifiableMessage2),
}
);
}
public void TestIdentifierSwap_RegexVsDeserialize(DatabaseType type, int batchSize, int numberOfRandomTagsPerDicom)
{
var options = new GlobalOptionsFactory().Load();
var mappingDataTable = new DataTable("IdMap");
mappingDataTable.Columns.Add("priv");
mappingDataTable.Columns.Add("pub");
mappingDataTable.Rows.Add("010101", "020202");
mappingDataTable.Rows.Add("0101010101", "0202020202");
var db = GetCleanedServer(type);
options.IdentifierMapperOptions.MappingConnectionString = db.Server.Builder.ConnectionString;
options.IdentifierMapperOptions.MappingTableName = db.CreateTable("IdMap", mappingDataTable).GetFullyQualifiedName();
options.IdentifierMapperOptions.SwapColumnName = "priv";
options.IdentifierMapperOptions.ReplacementColumnName = "pub";
options.IdentifierMapperOptions.MappingDatabaseType = type;
options.IdentifierMapperOptions.TimeoutInSeconds = 500;
var swapper = new PreloadTableSwapper();
swapper.Setup(options.IdentifierMapperOptions);
var goodChis = new List <DicomFileMessage>();
var badChis = new List <DicomFileMessage>();
Console.WriteLine("Generating Test data...");
List <Task> tasks = new List <Task>();
object oTaskLock = new object();
for (int i = 0; i < batchSize; i++)
{
var t = new Task(() =>
{
var a = GetTestDicomFileMessage(Test.ProperlyFormatedChi, numberOfRandomTagsPerDicom);
var b = GetTestDicomFileMessage(Test.ProperlyFormatedChi, numberOfRandomTagsPerDicom);
lock (oTaskLock)
{
goodChis.Add(a);
badChis.Add(b);
}
});
t.Start();
tasks.Add(t);
if (i % Environment.ProcessorCount == 0)
{
Task.WaitAll(tasks.ToArray());
tasks.Clear();
}
if (i % 100 == 0)
{
Console.WriteLine(i + " pairs done");
}
}
Task.WaitAll(tasks.ToArray());
options.IdentifierMapperOptions.AllowRegexMatching = true;
using (var tester = new MicroserviceTester(options.RabbitOptions, options.IdentifierMapperOptions))
{
tester.CreateExchange(options.IdentifierMapperOptions.AnonImagesProducerOptions.ExchangeName, null);
Console.WriteLine("Pushing good messages to Rabbit...");
tester.SendMessages(options.IdentifierMapperOptions, goodChis, true);
var host = new IdentifierMapperHost(options, swapper);
tester.StopOnDispose.Add(host);
Console.WriteLine("Starting host");
Stopwatch sw = Stopwatch.StartNew();
host.Start();
new TestTimelineAwaiter().Await(() => host.Consumer.AckCount == batchSize);
Console.WriteLine("Good message processing (" + batchSize + ") took:" + sw.ElapsedMilliseconds + "ms");
host.Stop("Test finished");
}
options.IdentifierMapperOptions.AllowRegexMatching = false;
using (var tester = new MicroserviceTester(options.RabbitOptions, options.IdentifierMapperOptions))
{
tester.CreateExchange(options.IdentifierMapperOptions.AnonImagesProducerOptions.ExchangeName, null);
Console.WriteLine("Pushing bad messages to Rabbit...");
tester.SendMessages(options.IdentifierMapperOptions, badChis, true);
var host = new IdentifierMapperHost(options, swapper);
tester.StopOnDispose.Add(host);
Console.WriteLine("Starting host");
Stopwatch sw = Stopwatch.StartNew();
host.Start();
new TestTimelineAwaiter().Await(() => host.Consumer.AckCount == batchSize);
Console.WriteLine("Bad message processing (" + batchSize + ") took:" + sw.ElapsedMilliseconds + "ms");
host.Stop("Test finished");
}
}
public void MapperSource_IntegrationTest(DatabaseType dbType)
{
var db = GetCleanedServer(dbType);
DataTable dt = new DataTable();
dt.Columns.Add("PatientID", typeof(string));
dt.Columns.Add("StudyDescription", typeof(string));
dt.SetDoNotReType(true);
// We have a live table with anonymised data. There is one person with a known ECHI 0101010101=0A0A0A0A0A
dt.Rows.Add("0A0A0A0A0A", "CT Head");
//There are 2 people for whome we have added temporary identifiers
dt.Rows.Add("bbb-bbb-bbb", "CT Tail");
dt.Rows.Add("ccc-ccc-ccc", "CT Wings");
var liveTable = db.CreateTable("MyLiveTable", dt);
DiscoveredTable map;
using (var dtMap = new DataTable())
{
dtMap.Columns.Add("CHI");
dtMap.Columns.Add("ECHI");
dtMap.PrimaryKey = new [] { dtMap.Columns["CHI"] };
dtMap.Rows.Add("0101010101", "0A0A0A0A0A");
map = db.CreateTable("Map", dtMap);
}
// Import into RDMP the live table so we have a TableInfo pointer to it floating around
Import(liveTable);
var mapperOptions = new IdentifierMapperOptions()
{
MappingTableName = map.GetFullyQualifiedName(),
MappingConnectionString = db.Server.Builder.ConnectionString,
SwapColumnName = "CHI",
ReplacementColumnName = "ECHI",
MappingDatabaseType = db.Server.DatabaseType,
SwapperType = typeof(TableLookupWithGuidFallbackSwapper).FullName
};
var swapper = new TableLookupWithGuidFallbackSwapper();
swapper.Setup(mapperOptions);
var guidTable = swapper.GetGuidTableIfAny(mapperOptions);
Assert.AreEqual(0, guidTable.GetRowCount(), "No temporary guids should exist yet");
Assert.AreEqual(1, map.GetRowCount(), "We should have a mapping table with 1 entry");
guidTable.Insert(new Dictionary <string, object>()
{
{ "CHI", "0202020202" },
{ TableLookupWithGuidFallbackSwapper.GuidColumnName, "bbb-bbb-bbb" }
});
guidTable.Insert(new Dictionary <string, object>()
{
{ "CHI", "0303030303" },
{ TableLookupWithGuidFallbackSwapper.GuidColumnName, "ccc-ccc-ccc" }
});
Assert.AreEqual(1, map.GetRowCount(), "We should have a mapping table with 1 entry");
Assert.AreEqual(2, guidTable.GetRowCount(), "We should have a temporary guid for 0202020202");
// make a fake data load into this table (create trigger and insert/update)
var triggerImplementer = new TriggerImplementerFactory(dbType).Create(map);
triggerImplementer.CreateTrigger(new ThrowImmediatelyCheckNotifier());
//create a brand new mapping
map.Insert(new Dictionary <string, object>()
{
{ "CHI", "0303030303" },
{ "ECHI", "0C0C0C0C0C" },
{ SpecialFieldNames.ValidFrom, DateTime.Now },
{ SpecialFieldNames.DataLoadRunID, 55 },
});
var globals = new GlobalOptionsFactory().Load();
var cliOptions = new TriggerUpdatesFromMapperOptions()
{
DateOfLastUpdate = new DateTime(2020, 01, 01),
LiveDatabaseFieldName = "PatientID",
Qualifier = '\'',
};
globals.UseTestValues(
RequiresRabbit.GetConnectionFactory(),
RequiresMongoDb.GetMongoClientSettings(),
RequiresRelationalDb.GetRelationalDatabaseConnectionStrings(),
((TableRepository)RepositoryLocator.CatalogueRepository).ConnectionStringBuilder,
((TableRepository)RepositoryLocator.DataExportRepository).ConnectionStringBuilder);
//make sure the identifier mapper goes to the right table
globals.IdentifierMapperOptions.MappingConnectionString = db.Server.Builder.ConnectionString;
globals.IdentifierMapperOptions.MappingDatabaseType = dbType;
globals.IdentifierMapperOptions.MappingTableName = map.GetFullyQualifiedName();
globals.IdentifierMapperOptions.SwapperType = typeof(TableLookupWithGuidFallbackSwapper).FullName;
using (var tester = new MicroserviceTester(globals.RabbitOptions, globals.CohortExtractorOptions))
{
tester.CreateExchange(globals.TriggerUpdatesOptions.ExchangeName, globals.UpdateValuesOptions.QueueName);
var sourceHost = new TriggerUpdatesHost(globals, new MapperSource(globals, cliOptions));
var destHost = new UpdateValuesHost(globals);
sourceHost.Start();
tester.StopOnDispose.Add(sourceHost);
destHost.Start();
tester.StopOnDispose.Add(destHost);
//wait till updater is done updating the live table
new TestTimelineAwaiter().Await(() => destHost.Consumer.AckCount == 1);
}
var liveDtAfter = liveTable.GetDataTable();
Assert.AreEqual(1, liveDtAfter.Rows.Cast <DataRow>().Count(r => (string)r["PatientID"] == "0A0A0A0A0A"), "Expected original data to still be intact");
Assert.AreEqual(1, liveDtAfter.Rows.Cast <DataRow>().Count(r => (string)r["PatientID"] == "bbb-bbb-bbb"), "Expected unknown CHI with guid bbb to still be unknown");
Assert.AreEqual(1, liveDtAfter.Rows.Cast <DataRow>().Count(r => (string)r["PatientID"] == "0C0C0C0C0C"), "Expected the unknown CHI ccc to be now known as 0C0C0C0C0C");
}
