public void Test_DodgyTagNames()
{
_helper.TruncateTablesIfExists();
DirectoryInfo d = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_DodgyTagNames)));
d.Create();
var fi = TestData.Create(new FileInfo(Path.Combine(d.FullName, "MyTestFile.dcm")));
var fi2 = TestData.Create(new FileInfo(Path.Combine(d.FullName, "MyTestFile2.dcm")));
DicomFile dcm;
using (var stream = File.OpenRead(fi.FullName))
{
dcm = DicomFile.Open(stream);
dcm.Dataset.AddOrUpdate(DicomTag.PrintRETIRED, "FISH");
dcm.Dataset.AddOrUpdate(DicomTag.Date, new DateTime(2001, 01, 01));
dcm.Save(fi2.FullName);
}
var adder = new TagColumnAdder(DicomTypeTranslaterReader.GetColumnNameForTag(DicomTag.Date, false), "datetime2", _helper.ImageTableInfo, new AcceptAllCheckNotifier());
adder.Execute();
adder = new TagColumnAdder(DicomTypeTranslaterReader.GetColumnNameForTag(DicomTag.PrintRETIRED, false), "datetime2", _helper.ImageTableInfo, new AcceptAllCheckNotifier());
adder.Execute();
fi.Delete();
File.Move(fi2.FullName, fi.FullName);
//creates the queues, exchanges and bindings
var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.DicomRelationalMapperOptions);
tester.CreateExchange(_globals.RabbitOptions.FatalLoggingExchange, null);
using (var host = new DicomRelationalMapperHost(_globals))
{
host.Start();
using (var timeline = new TestTimeline(tester))
{
timeline.SendMessage(_globals.DicomRelationalMapperOptions, _helper.GetDicomFileMessage(_globals.FileSystemOptions.FileSystemRoot, fi));
//start the timeline
timeline.StartTimeline();
Thread.Sleep(TimeSpan.FromSeconds(10));
new TestTimelineAwaiter().Await(() => host.Consumer.AckCount >= 1, null, 30000, () => host.Consumer.DleErrors);
Assert.AreEqual(1, _helper.SeriesTable.GetRowCount(), "SeriesTable did not have the expected number of rows in LIVE");
Assert.AreEqual(1, _helper.StudyTable.GetRowCount(), "StudyTable did not have the expected number of rows in LIVE");
Assert.AreEqual(1, _helper.ImageTable.GetRowCount(), "ImageTable did not have the expected number of rows in LIVE");
host.Stop("Test end");
}
}
tester.Shutdown();
}
public void SetUpSuite()
{
SetUpDefaults();
// Create the test Series/Image exchanges
Options.RabbitOptions.RabbitMqControlExchangeName = "TEST.ControlExchange";
var tester = new MicroserviceTester(Options.RabbitOptions);
tester.CreateExchange(Options.DicomTagReaderOptions.ImageProducerOptions.ExchangeName, TestImageQueueName);
tester.CreateExchange(Options.DicomTagReaderOptions.SeriesProducerOptions.ExchangeName, TestSeriesQueueName);
tester.CreateExchange(Options.RabbitOptions.FatalLoggingExchange, null);
tester.CreateExchange(Options.RabbitOptions.RabbitMqControlExchangeName, null);
tester.Shutdown();
_testConnection = new ConnectionFactory
{
HostName = Options.RabbitOptions.RabbitMqHostName,
Port = Options.RabbitOptions.RabbitMqHostPort,
VirtualHost = Options.RabbitOptions.RabbitMqVirtualHost,
UserName = Options.RabbitOptions.RabbitMqUserName,
Password = Options.RabbitOptions.RabbitMqPassword
}.CreateConnection("TestConnection");
_testModel = _testConnection.CreateModel();
MockHost = Mock.Of <IMicroserviceHost>();
}
public void TestBasicOperation()
{
_helper.Options.FileSystemOptions.FileSystemRoot = _helper.TestDir.FullName;
_helper.TestAccessionDirectoryMessage.DirectoryPath = _helper.TestDir.FullName;
var tester = new MicroserviceTester(_helper.Options.RabbitOptions, _helper.AccessionConsumerOptions);
var host = new DicomTagReaderHost(_helper.Options);
host.Start();
tester.SendMessage(_helper.AccessionConsumerOptions, new MessageHeader(), _helper.TestAccessionDirectoryMessage);
var timeout = 30000;
const int stepSize = 500;
while (!_helper.CheckQueues(1, 1) && timeout > 0)
{
timeout -= 500;
Thread.Sleep(stepSize);
}
host.Stop("Test end");
tester.Dispose();
if (timeout <= 0)
{
Assert.Fail("Failed to process expected number of messages within the timeout");
}
}
public void TestClassifierName_ValidClassifier()
{
var options = new GlobalOptionsFactory().Load();
var testDcm = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(TestClassifierName_ValidClassifier), "f1.dcm")); Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(TestClassifierName_ValidClassifier), "f1.dcm");
TestData.Create(testDcm);
using (var tester = new MicroserviceTester(options.RabbitOptions, options.IsIdentifiableOptions))
{
tester.CreateExchange(options.IsIdentifiableOptions.IsIdentifiableProducerOptions.ExchangeName, null);
options.IsIdentifiableOptions.ClassifierType = typeof(RejectAllClassifier).FullName;
options.IsIdentifiableOptions.DataDirectory = TestContext.CurrentContext.TestDirectory;
var host = new IsIdentifiableHost(options, new IsIdentifiableServiceOptions());
Assert.IsNotNull(host);
host.Start();
tester.SendMessage(options.IsIdentifiableOptions, new ExtractedFileStatusMessage()
{
DicomFilePath = "yay.dcm",
OutputFilePath = testDcm.FullName,
ProjectNumber = "100",
ExtractionDirectory = "./fish",
StatusMessage = "yay!",
Status = ExtractedFileStatus.Anonymised
});
var awaiter = new TestTimelineAwaiter();
awaiter.Await(() => host.Consumer.AckCount == 1);
}
}
public void TestLargeImageDatasets(DatabaseType databaseType, int numberOfImages)
{
foreach (Pipeline p in CatalogueRepository.GetAllObjects <Pipeline>())
{
p.DeleteInDatabase();
}
var db = GetCleanedServer(databaseType);
var d = CatalogueRepository.GetServerDefaults();
d.ClearDefault(PermissableDefaults.RAWDataLoadServer);
var template = ImageTableTemplateCollection.LoadFrom(_templateXml);
_globals = new GlobalOptionsFactory().Load();
_globals.DicomRelationalMapperOptions.DatabaseNamerType = typeof(MyFixedStagingDatabaseNamer).FullName;
_globals.DicomRelationalMapperOptions.QoSPrefetchCount = ushort.MaxValue;
_globals.DicomRelationalMapperOptions.MinimumBatchSize = numberOfImages;
_globals.DicomRelationalMapperOptions.UseInsertIntoForRAWMigration = true;
_helper = new DicomRelationalMapperTestHelper();
_helper.SetupSuite(db, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource), root: null, template: template, persistentRaw: true);
//do not use an explicit RAW data load server
d.ClearDefault(PermissableDefaults.RAWDataLoadServer);
Random r = new Random(123);
List <DicomDataset> allImages;
using (var generator = new DicomDataGenerator(r, null, "CT"))
allImages = generator.GenerateImages(numberOfImages, r);
Assert.AreEqual(numberOfImages, allImages.Count);
using (var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.DicomRelationalMapperOptions))
{
using (var host = new DicomRelationalMapperHost(_globals))
{
tester.SendMessages(_globals.DicomRelationalMapperOptions, allImages.Select(GetFileMessageForDataset), true);
Console.WriteLine("Starting Host");
host.Start();
Stopwatch sw = Stopwatch.StartNew();
new TestTimelineAwaiter().Await(() => host.Consumer.AckCount == numberOfImages, null, 20 * 60 * 100); //1 minute
Console.Write("Time For DLE:" + sw.Elapsed.TotalSeconds + "s");
host.Stop("Test finished");
}
}
foreach (Pipeline allObject in CatalogueRepository.GetAllObjects <Pipeline>())
{
allObject.DeleteInDatabase();
}
}
public void Test_FileCopierHost_HappyPath()
{
GlobalOptions globals = new GlobalOptionsFactory().Load();
globals.FileSystemOptions.FileSystemRoot = "root";
globals.FileSystemOptions.ExtractRoot = "exroot";
using var tester = new MicroserviceTester(globals.RabbitOptions, globals.FileCopierOptions);
string outputQueueName = globals.FileCopierOptions.CopyStatusProducerOptions.ExchangeName.Replace("Exchange", "Queue");
tester.CreateExchange(
globals.FileCopierOptions.CopyStatusProducerOptions.ExchangeName,
outputQueueName,
false,
globals.FileCopierOptions.NoVerifyRoutingKey);
var mockFileSystem = new MockFileSystem();
mockFileSystem.AddDirectory(globals.FileSystemOptions.FileSystemRoot);
mockFileSystem.AddDirectory(globals.FileSystemOptions.ExtractRoot);
mockFileSystem.AddFile(mockFileSystem.Path.Combine(globals.FileSystemOptions.FileSystemRoot, "file.dcm"), MockFileData.NullObject);
var host = new FileCopierHost(globals, mockFileSystem);
tester.StopOnDispose.Add(host);
host.Start();
var message = new ExtractFileMessage
{
ExtractionJobIdentifier = Guid.NewGuid(),
JobSubmittedAt = DateTime.UtcNow,
ProjectNumber = "1234",
ExtractionDirectory = "1234/foo",
DicomFilePath = "file.dcm",
IsIdentifiableExtraction = true,
OutputPath = "output.dcm",
};
tester.SendMessage(globals.FileCopierOptions, message);
using IConnection conn = tester.Factory.CreateConnection();
using IModel model = conn.CreateModel();
var consumer = new EventingBasicConsumer(model);
ExtractedFileStatusMessage statusMessage = null;
consumer.Received += (_, ea) => statusMessage = JsonConvert.DeserializeObject <ExtractedFileStatusMessage>(Encoding.UTF8.GetString(ea.Body.ToArray()));
model.BasicConsume(outputQueueName, true, "", consumer);
new TestTimelineAwaiter().Await(() => statusMessage != null);
Assert.AreEqual(ExtractedFileStatus.Copied, statusMessage.Status);
}
public void TestLoadingOneImage_SingleFileMessage(int numberOfMessagesToSend, bool mixInATextFile)
{
_helper.TruncateTablesIfExists();
DirectoryInfo d = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(TestLoadingOneImage_SingleFileMessage)));
d.Create();
var fi = TestData.Create(new FileInfo(Path.Combine(d.FullName, "MyTestFile.dcm")));
if (mixInATextFile)
{
var randomText = new FileInfo(Path.Combine(d.FullName, "RandomTextFile.dcm"));
File.WriteAllLines(randomText.FullName, new[] { "I love dancing", "all around the world", "boy the world is a big place eh?" });
}
//creates the queues, exchanges and bindings
var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.DicomRelationalMapperOptions);
tester.CreateExchange(_globals.RabbitOptions.FatalLoggingExchange, null);
using (var host = new DicomRelationalMapperHost(_globals))
{
host.Start();
using (var timeline = new TestTimeline(tester))
{
//send the message 10 times over a 10 second period
for (int i = 0; i < numberOfMessagesToSend; i++)
{
timeline
.SendMessage(_globals.DicomRelationalMapperOptions, _helper.GetDicomFileMessage(_globals.FileSystemOptions.FileSystemRoot, fi))
.Wait(1000);
}
//start the timeline
timeline.StartTimeline();
Thread.Sleep(TimeSpan.FromSeconds(10));
new TestTimelineAwaiter().Await(() => host.Consumer.AckCount >= numberOfMessagesToSend, null, 30000, () => host.Consumer.DleErrors);
Assert.AreEqual(1, _helper.SeriesTable.GetRowCount(), "SeriesTable did not have the expected number of rows in LIVE");
Assert.AreEqual(1, _helper.StudyTable.GetRowCount(), "StudyTable did not have the expected number of rows in LIVE");
Assert.AreEqual(1, _helper.ImageTable.GetRowCount(), "ImageTable did not have the expected number of rows in LIVE");
host.Stop("Test end");
}
}
tester.Shutdown();
}
public void TestIsIdentifiable_TesseractStanfordDicomFileClassifier()
{
var options = new GlobalOptionsFactory().Load();
// Create a test data directory containing IsIdentifiableRules with 0 rules, and tessdata with the eng.traineddata classifier
// TODO(rkm 2020-04-14) This is a stop-gap solution until the tests are properly refactored
var testRulesDir = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, "data", "IsIdentifiableRules"));
testRulesDir.Create();
options.IsIdentifiableOptions.DataDirectory = testRulesDir.Parent.FullName;
var tessDir = new DirectoryInfo(Path.Combine(testRulesDir.Parent.FullName, "tessdata"));
tessDir.Create();
var dest = Path.Combine(tessDir.FullName, "eng.traineddata");
if (!File.Exists(dest))
{
File.Copy(Path.Combine(DataDirectory, "tessdata", "eng.traineddata"), dest);
}
var testDcm = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(TestIsIdentifiable_TesseractStanfordDicomFileClassifier), "f1.dcm"));
Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(TestClassifierName_ValidClassifier), "f1.dcm");
TestData.Create(testDcm);
using (var tester = new MicroserviceTester(options.RabbitOptions, options.IsIdentifiableOptions))
{
options.IsIdentifiableOptions.ClassifierType = typeof(TesseractStanfordDicomFileClassifier).FullName;
var host = new IsIdentifiableHost(options, new IsIdentifiableServiceOptions());
host.Start();
tester.SendMessage(options.IsIdentifiableOptions, new ExtractedFileStatusMessage
{
DicomFilePath = "yay.dcm",
OutputFilePath = testDcm.FullName,
ProjectNumber = "100",
ExtractionDirectory = "./fish",
StatusMessage = "yay!",
Status = ExtractedFileStatus.Anonymised
});
var awaiter = new TestTimelineAwaiter();
awaiter.Await(() => host.Consumer.AckCount == 1 || host.Consumer.NackCount == 1);
Assert.AreEqual(1, host.Consumer.AckCount, "Tesseract not acking");
}
}
public void IdenticalDatasetsTest()
{
_helper.TruncateTablesIfExists();
var ds = new DicomDataset();
ds.AddOrUpdate(DicomTag.SeriesInstanceUID, "123");
ds.AddOrUpdate(DicomTag.SOPInstanceUID, "123");
ds.AddOrUpdate(DicomTag.StudyInstanceUID, "123");
ds.AddOrUpdate(DicomTag.PatientID, "123");
var msg1 = _helper.GetDicomFileMessage(ds, _globals.FileSystemOptions.FileSystemRoot, Path.Combine(_globals.FileSystemOptions.FileSystemRoot, "mydicom.dcm"));
var msg2 = _helper.GetDicomFileMessage(ds, _globals.FileSystemOptions.FileSystemRoot, Path.Combine(_globals.FileSystemOptions.FileSystemRoot, "mydicom.dcm"));
//creates the queues, exchanges and bindings
using (var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.DicomRelationalMapperOptions))
{
tester.CreateExchange(_globals.RabbitOptions.FatalLoggingExchange, null);
tester.SendMessage(_globals.DicomRelationalMapperOptions, msg1);
tester.SendMessage(_globals.DicomRelationalMapperOptions, msg2);
_globals.DicomRelationalMapperOptions.RunChecks = true;
using (var host = new DicomRelationalMapperHost(_globals))
{
host.Start();
new TestTimelineAwaiter().Await(() => host.Consumer.MessagesProcessed == 2, null, 30000, () => host.Consumer.DleErrors);
Assert.GreaterOrEqual(1, _helper.SeriesTable.GetRowCount(), "SeriesTable did not have the expected number of rows in LIVE");
Assert.GreaterOrEqual(1, _helper.StudyTable.GetRowCount(), "StudyTable did not have the expected number of rows in LIVE");
Assert.AreEqual(1, _helper.ImageTable.GetRowCount(), "ImageTable did not have the expected number of rows in LIVE");
host.Stop("Test end");
}
tester.Shutdown();
}
}
private void VerifyReports(GlobalOptions globals, PathFixtures pf, ReportFormat reportFormat, IEnumerable <Tuple <ConsumerOptions, IMessage> > toSend)
{
globals.FileSystemOptions.ExtractRoot = pf.ExtractRootAbsolute;
globals.CohortPackagerOptions.JobWatcherTimeoutInSeconds = 5;
globals.CohortPackagerOptions.ReporterType = "FileReporter";
globals.CohortPackagerOptions.ReportFormat = reportFormat.ToString();
MongoClient client = MongoClientHelpers.GetMongoClient(globals.MongoDatabases.ExtractionStoreOptions, "test", true);
globals.MongoDatabases.ExtractionStoreOptions.DatabaseName += "-" + Guid.NewGuid().ToString().Split('-')[0];
client.DropDatabase(globals.MongoDatabases.ExtractionStoreOptions.DatabaseName);
using (var tester = new MicroserviceTester(
globals.RabbitOptions,
globals.CohortPackagerOptions.ExtractRequestInfoOptions,
globals.CohortPackagerOptions.FileCollectionInfoOptions,
globals.CohortPackagerOptions.NoVerifyStatusOptions,
globals.CohortPackagerOptions.VerificationStatusOptions))
{
foreach ((ConsumerOptions consumerOptions, IMessage message) in toSend)
{
tester.SendMessage(consumerOptions, new MessageHeader(), message);
}
var host = new CohortPackagerHost(globals);
host.Start();
var timeoutSecs = 10;
while (!HaveFiles(pf) && timeoutSecs > 0)
{
--timeoutSecs;
Thread.Sleep(TimeSpan.FromSeconds(1));
}
host.Stop("Test end");
}
var firstLine = $"# SMI extraction validation report for testProj1/{pf.ExtractName}";
switch (reportFormat)
{
case ReportFormat.Combined:
{
string reportContent = File.ReadAllText(Path.Combine(pf.ProjReportsDirAbsolute, $"{pf.ExtractName}_report.txt"));
Assert.True(reportContent.StartsWith(firstLine));
break;
}
case ReportFormat.Split:
{
string extractReportsDirAbsolute = Path.Combine(pf.ProjReportsDirAbsolute, pf.ExtractName);
Assert.AreEqual(6, Directory.GetFiles(extractReportsDirAbsolute).Length);
string reportContent = File.ReadAllText(Path.Combine(extractReportsDirAbsolute, "README.md"));
Assert.True(reportContent.StartsWith(firstLine));
break;
}
default:
Assert.Fail($"No case for ReportFormat {reportFormat}");
break;
}
}
public void MapperSource_IntegrationTest(DatabaseType dbType)
{
var db = GetCleanedServer(dbType);
DataTable dt = new DataTable();
dt.Columns.Add("PatientID", typeof(string));
dt.Columns.Add("StudyDescription", typeof(string));
dt.SetDoNotReType(true);
// We have a live table with anonymised data. There is one person with a known ECHI 0101010101=0A0A0A0A0A
dt.Rows.Add("0A0A0A0A0A", "CT Head");
//There are 2 people for whome we have added temporary identifiers
dt.Rows.Add("bbb-bbb-bbb", "CT Tail");
dt.Rows.Add("ccc-ccc-ccc", "CT Wings");
var liveTable = db.CreateTable("MyLiveTable", dt);
DiscoveredTable map;
using (var dtMap = new DataTable())
{
dtMap.Columns.Add("CHI");
dtMap.Columns.Add("ECHI");
dtMap.PrimaryKey = new [] { dtMap.Columns["CHI"] };
dtMap.Rows.Add("0101010101", "0A0A0A0A0A");
map = db.CreateTable("Map", dtMap);
}
// Import into RDMP the live table so we have a TableInfo pointer to it floating around
Import(liveTable);
var mapperOptions = new IdentifierMapperOptions()
{
MappingTableName = map.GetFullyQualifiedName(),
MappingConnectionString = db.Server.Builder.ConnectionString,
SwapColumnName = "CHI",
ReplacementColumnName = "ECHI",
MappingDatabaseType = db.Server.DatabaseType,
SwapperType = typeof(TableLookupWithGuidFallbackSwapper).FullName
};
var swapper = new TableLookupWithGuidFallbackSwapper();
swapper.Setup(mapperOptions);
var guidTable = swapper.GetGuidTableIfAny(mapperOptions);
Assert.AreEqual(0, guidTable.GetRowCount(), "No temporary guids should exist yet");
Assert.AreEqual(1, map.GetRowCount(), "We should have a mapping table with 1 entry");
guidTable.Insert(new Dictionary <string, object>()
{
{ "CHI", "0202020202" },
{ TableLookupWithGuidFallbackSwapper.GuidColumnName, "bbb-bbb-bbb" }
});
guidTable.Insert(new Dictionary <string, object>()
{
{ "CHI", "0303030303" },
{ TableLookupWithGuidFallbackSwapper.GuidColumnName, "ccc-ccc-ccc" }
});
Assert.AreEqual(1, map.GetRowCount(), "We should have a mapping table with 1 entry");
Assert.AreEqual(2, guidTable.GetRowCount(), "We should have a temporary guid for 0202020202");
// make a fake data load into this table (create trigger and insert/update)
var triggerImplementer = new TriggerImplementerFactory(dbType).Create(map);
triggerImplementer.CreateTrigger(new ThrowImmediatelyCheckNotifier());
//create a brand new mapping
map.Insert(new Dictionary <string, object>()
{
{ "CHI", "0303030303" },
{ "ECHI", "0C0C0C0C0C" },
{ SpecialFieldNames.ValidFrom, DateTime.Now },
{ SpecialFieldNames.DataLoadRunID, 55 },
});
var globals = new GlobalOptionsFactory().Load();
var cliOptions = new TriggerUpdatesFromMapperOptions()
{
DateOfLastUpdate = new DateTime(2020, 01, 01),
LiveDatabaseFieldName = "PatientID",
Qualifier = '\'',
};
globals.UseTestValues(
RequiresRabbit.GetConnectionFactory(),
RequiresMongoDb.GetMongoClientSettings(),
RequiresRelationalDb.GetRelationalDatabaseConnectionStrings(),
((TableRepository)RepositoryLocator.CatalogueRepository).ConnectionStringBuilder,
((TableRepository)RepositoryLocator.DataExportRepository).ConnectionStringBuilder);
//make sure the identifier mapper goes to the right table
globals.IdentifierMapperOptions.MappingConnectionString = db.Server.Builder.ConnectionString;
globals.IdentifierMapperOptions.MappingDatabaseType = dbType;
globals.IdentifierMapperOptions.MappingTableName = map.GetFullyQualifiedName();
globals.IdentifierMapperOptions.SwapperType = typeof(TableLookupWithGuidFallbackSwapper).FullName;
using (var tester = new MicroserviceTester(globals.RabbitOptions, globals.CohortExtractorOptions))
{
tester.CreateExchange(globals.TriggerUpdatesOptions.ExchangeName, globals.UpdateValuesOptions.QueueName);
var sourceHost = new TriggerUpdatesHost(globals, new MapperSource(globals, cliOptions));
var destHost = new UpdateValuesHost(globals);
sourceHost.Start();
tester.StopOnDispose.Add(sourceHost);
destHost.Start();
tester.StopOnDispose.Add(destHost);
//wait till updater is done updating the live table
new TestTimelineAwaiter().Await(() => destHost.Consumer.AckCount == 1);
}
var liveDtAfter = liveTable.GetDataTable();
Assert.AreEqual(1, liveDtAfter.Rows.Cast <DataRow>().Count(r => (string)r["PatientID"] == "0A0A0A0A0A"), "Expected original data to still be intact");
Assert.AreEqual(1, liveDtAfter.Rows.Cast <DataRow>().Count(r => (string)r["PatientID"] == "bbb-bbb-bbb"), "Expected unknown CHI with guid bbb to still be unknown");
Assert.AreEqual(1, liveDtAfter.Rows.Cast <DataRow>().Count(r => (string)r["PatientID"] == "0C0C0C0C0C"), "Expected the unknown CHI ccc to be now known as 0C0C0C0C0C");
}
public void TestIdentifierSwap_RegexVsDeserialize(DatabaseType type, int batchSize, int numberOfRandomTagsPerDicom)
{
var options = new GlobalOptionsFactory().Load();
var mappingDataTable = new DataTable("IdMap");
mappingDataTable.Columns.Add("priv");
mappingDataTable.Columns.Add("pub");
mappingDataTable.Rows.Add("010101", "020202");
mappingDataTable.Rows.Add("0101010101", "0202020202");
var db = GetCleanedServer(type);
options.IdentifierMapperOptions.MappingConnectionString = db.Server.Builder.ConnectionString;
options.IdentifierMapperOptions.MappingTableName = db.CreateTable("IdMap", mappingDataTable).GetFullyQualifiedName();
options.IdentifierMapperOptions.SwapColumnName = "priv";
options.IdentifierMapperOptions.ReplacementColumnName = "pub";
options.IdentifierMapperOptions.MappingDatabaseType = type;
options.IdentifierMapperOptions.TimeoutInSeconds = 500;
var swapper = new PreloadTableSwapper();
swapper.Setup(options.IdentifierMapperOptions);
var goodChis = new List <DicomFileMessage>();
var badChis = new List <DicomFileMessage>();
Console.WriteLine("Generating Test data...");
List <Task> tasks = new List <Task>();
object oTaskLock = new object();
for (int i = 0; i < batchSize; i++)
{
var t = new Task(() =>
{
var a = GetTestDicomFileMessage(Test.ProperlyFormatedChi, numberOfRandomTagsPerDicom);
var b = GetTestDicomFileMessage(Test.ProperlyFormatedChi, numberOfRandomTagsPerDicom);
lock (oTaskLock)
{
goodChis.Add(a);
badChis.Add(b);
}
});
t.Start();
tasks.Add(t);
if (i % Environment.ProcessorCount == 0)
{
Task.WaitAll(tasks.ToArray());
tasks.Clear();
}
if (i % 100 == 0)
{
Console.WriteLine(i + " pairs done");
}
}
Task.WaitAll(tasks.ToArray());
options.IdentifierMapperOptions.AllowRegexMatching = true;
using (var tester = new MicroserviceTester(options.RabbitOptions, options.IdentifierMapperOptions))
{
tester.CreateExchange(options.IdentifierMapperOptions.AnonImagesProducerOptions.ExchangeName, null);
Console.WriteLine("Pushing good messages to Rabbit...");
tester.SendMessages(options.IdentifierMapperOptions, goodChis, true);
var host = new IdentifierMapperHost(options, swapper);
tester.StopOnDispose.Add(host);
Console.WriteLine("Starting host");
Stopwatch sw = Stopwatch.StartNew();
host.Start();
new TestTimelineAwaiter().Await(() => host.Consumer.AckCount == batchSize);
Console.WriteLine("Good message processing (" + batchSize + ") took:" + sw.ElapsedMilliseconds + "ms");
host.Stop("Test finished");
}
options.IdentifierMapperOptions.AllowRegexMatching = false;
using (var tester = new MicroserviceTester(options.RabbitOptions, options.IdentifierMapperOptions))
{
tester.CreateExchange(options.IdentifierMapperOptions.AnonImagesProducerOptions.ExchangeName, null);
Console.WriteLine("Pushing bad messages to Rabbit...");
tester.SendMessages(options.IdentifierMapperOptions, badChis, true);
var host = new IdentifierMapperHost(options, swapper);
tester.StopOnDispose.Add(host);
Console.WriteLine("Starting host");
Stopwatch sw = Stopwatch.StartNew();
host.Start();
new TestTimelineAwaiter().Await(() => host.Consumer.AckCount == batchSize);
Console.WriteLine("Bad message processing (" + batchSize + ") took:" + sw.ElapsedMilliseconds + "ms");
host.Stop("Test finished");
}
}
public void TestPopulatorBasic(int nMessages)
{
// Arrange
string currentCollectionName = MongoDbPopulatorTestHelper.GetCollectionNameForTest(string.Format("TestPopulatorBasic({0})", nMessages));
_helper.Globals.MongoDbPopulatorOptions.SeriesCollection = currentCollectionName;
var tester = new MicroserviceTester(_helper.Globals.RabbitOptions, _helper.Globals.MongoDbPopulatorOptions.SeriesQueueConsumerOptions, _helper.Globals.MongoDbPopulatorOptions.ImageQueueConsumerOptions);
var host = new MongoDbPopulatorHost(_helper.Globals);
host.Start();
using (var timeline = new TestTimeline(tester))
{
var ds = new DicomDataset
{
new DicomUniqueIdentifier(DicomTag.SOPInstanceUID, "1.2.3.4")
};
var message = new SeriesMessage
{
NationalPACSAccessionNumber = "NationalPACSAccessionNumber-test",
DirectoryPath = "DirectoryPath-test",
StudyInstanceUID = "StudyInstanceUID-test",
SeriesInstanceUID = "SeriesInstanceUID-test",
ImagesInSeries = 123,
DicomDataset = DicomTypeTranslater.SerializeDatasetToJson(ds)
};
// Act
for (var i = 0; i < nMessages; i++)
{
timeline.SendMessage(_helper.Globals.MongoDbPopulatorOptions.SeriesQueueConsumerOptions, message);
}
timeline.StartTimeline();
var timeout = 30000;
const int stepSize = 500;
if (Debugger.IsAttached)
{
timeout = int.MaxValue;
}
var nWritten = 0L;
while (nWritten < nMessages && timeout > 0)
{
nWritten = _helper.TestDatabase.GetCollection <BsonDocument>(currentCollectionName).CountDocuments(new BsonDocument());
Thread.Sleep(stepSize);
timeout -= stepSize;
}
// Assert
if (timeout <= 0)
{
Assert.Fail("Failed to process expected number of messages within the timeout");
}
host.Stop("Test end");
tester.Shutdown();
}
}
public void TestLoadingOneImage_MileWideTest()
{
_helper.TruncateTablesIfExists();
DirectoryInfo d = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(TestLoadingOneImage_MileWideTest)));
d.Create();
var r = new Random(5000);
FileInfo[] files;
using (var g = new DicomDataGenerator(r, d, "CT"))
files = g.GenerateImageFiles(1, r).ToArray();
Assert.AreEqual(1, files.Length);
var existingColumns = _helper.ImageTable.DiscoverColumns();
//Add 200 random tags
foreach (string tag in TagColumnAdder.GetAvailableTags().OrderBy(a => r.Next()).Take(200))
{
string dataType;
try
{
dataType = TagColumnAdder.GetDataTypeForTag(tag, new MicrosoftSQLTypeTranslater());
}
catch (Exception)
{
continue;
}
if (existingColumns.Any(c => c.GetRuntimeName().Equals(tag)))
{
continue;
}
var adder = new TagColumnAdder(tag, dataType, _helper.ImageTableInfo, new AcceptAllCheckNotifier());
adder.SkipChecksAndSynchronization = true;
adder.Execute();
}
new TableInfoSynchronizer(_helper.ImageTableInfo).Synchronize(new AcceptAllCheckNotifier());
//creates the queues, exchanges and bindings
var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.DicomRelationalMapperOptions);
tester.CreateExchange(_globals.RabbitOptions.FatalLoggingExchange, null);
using (var host = new DicomRelationalMapperHost(_globals))
{
host.Start();
using (var timeline = new TestTimeline(tester))
{
foreach (var f in files)
{
timeline.SendMessage(_globals.DicomRelationalMapperOptions,
_helper.GetDicomFileMessage(_globals.FileSystemOptions.FileSystemRoot, f));
}
//start the timeline
timeline.StartTimeline();
new TestTimelineAwaiter().Await(() => host.Consumer.MessagesProcessed == 1, null, 30000, () => host.Consumer.DleErrors);
Assert.GreaterOrEqual(1, _helper.SeriesTable.GetRowCount(), "SeriesTable did not have the expected number of rows in LIVE");
Assert.GreaterOrEqual(1, _helper.StudyTable.GetRowCount(), "StudyTable did not have the expected number of rows in LIVE");
Assert.AreEqual(1, _helper.ImageTable.GetRowCount(), "ImageTable did not have the expected number of rows in LIVE");
host.Stop("Test end");
}
}
tester.Shutdown();
}
private void RunTest(DirectoryInfo dir, int numberOfExpectedRows, Action <FileSystemOptions> adjustFileSystemOptions)
{
TestLogger.Setup();
var logger = LogManager.GetLogger("MicroservicesIntegrationTest");
_globals.FileSystemOptions.FileSystemRoot = TestContext.CurrentContext.TestDirectory;
var readFromFatalErrors = new ConsumerOptions
{
QueueName = "TEST.FatalLoggingQueue"
};
///////////////////////////////////// Directory //////////////////////////
var processDirectoryOptions = new DicomDirectoryProcessorCliOptions();
processDirectoryOptions.ToProcessDir = dir;
processDirectoryOptions.DirectoryFormat = "Default";
adjustFileSystemOptions?.Invoke(_globals.FileSystemOptions);
//////////////////////////////////////////////// Mongo Db Populator ////////////////////////
// Make this a GUID or something, should be unique per test
var currentSeriesCollectionName = "Integration_HappyPath_Series" + DateTime.Now.Ticks;
var currentImageCollectionName = "Integration_HappyPath_Image" + DateTime.Now.Ticks;
_globals.MongoDbPopulatorOptions.SeriesCollection = currentSeriesCollectionName;
_globals.MongoDbPopulatorOptions.ImageCollection = currentImageCollectionName;
//use the test catalogue not the one in the combined app.config
_globals.RDMPOptions.CatalogueConnectionString = ((TableRepository)RepositoryLocator.CatalogueRepository).DiscoveredServer.Builder.ConnectionString;
_globals.RDMPOptions.DataExportConnectionString = ((TableRepository)RepositoryLocator.DataExportRepository).DiscoveredServer.Builder.ConnectionString;
_globals.DicomRelationalMapperOptions.RunChecks = true;
if (_globals.DicomRelationalMapperOptions.MinimumBatchSize < 1)
{
_globals.DicomRelationalMapperOptions.MinimumBatchSize = 1;
}
using (var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.CohortExtractorOptions))
{
tester.CreateExchange(_globals.ProcessDirectoryOptions.AccessionDirectoryProducerOptions.ExchangeName, _globals.DicomTagReaderOptions.QueueName);
tester.CreateExchange(_globals.DicomTagReaderOptions.SeriesProducerOptions.ExchangeName, _globals.MongoDbPopulatorOptions.SeriesQueueConsumerOptions.QueueName);
tester.CreateExchange(_globals.DicomTagReaderOptions.ImageProducerOptions.ExchangeName, _globals.IdentifierMapperOptions.QueueName);
tester.CreateExchange(_globals.DicomTagReaderOptions.ImageProducerOptions.ExchangeName, _globals.MongoDbPopulatorOptions.ImageQueueConsumerOptions.QueueName, true);
tester.CreateExchange(_globals.IdentifierMapperOptions.AnonImagesProducerOptions.ExchangeName, _globals.DicomRelationalMapperOptions.QueueName);
tester.CreateExchange(_globals.RabbitOptions.FatalLoggingExchange, readFromFatalErrors.QueueName);
tester.CreateExchange(_globals.CohortExtractorOptions.ExtractFilesProducerOptions.ExchangeName, null, false, _globals.CohortExtractorOptions.ExtractIdentRoutingKey);
tester.CreateExchange(_globals.CohortExtractorOptions.ExtractFilesProducerOptions.ExchangeName, null, true, _globals.CohortExtractorOptions.ExtractAnonRoutingKey);
tester.CreateExchange(_globals.CohortExtractorOptions.ExtractFilesInfoProducerOptions.ExchangeName, null);
#region Running Microservices
var processDirectory = new DicomDirectoryProcessorHost(_globals, processDirectoryOptions);
processDirectory.Start();
tester.StopOnDispose.Add(processDirectory);
var dicomTagReaderHost = new DicomTagReaderHost(_globals);
dicomTagReaderHost.Start();
tester.StopOnDispose.Add(dicomTagReaderHost);
var mongoDbPopulatorHost = new MongoDbPopulatorHost(_globals);
mongoDbPopulatorHost.Start();
tester.StopOnDispose.Add(mongoDbPopulatorHost);
var identifierMapperHost = new IdentifierMapperHost(_globals, new SwapForFixedValueTester("FISHFISH"));
identifierMapperHost.Start();
tester.StopOnDispose.Add(identifierMapperHost);
new TestTimelineAwaiter().Await(() => dicomTagReaderHost.AccessionDirectoryMessageConsumer.AckCount >= 1);
logger.Info("\n### DicomTagReader has processed its messages ###\n");
// FIXME: This isn't exactly how the pipeline runs
new TestTimelineAwaiter().Await(() => identifierMapperHost.Consumer.AckCount >= 1);
logger.Info("\n### IdentifierMapper has processed its messages ###\n");
using (var relationalMapperHost = new DicomRelationalMapperHost(_globals))
{
var start = DateTime.Now;
relationalMapperHost.Start();
tester.StopOnDispose.Add(relationalMapperHost);
Assert.True(mongoDbPopulatorHost.Consumers.Count == 2);
new TestTimelineAwaiter().Await(() => mongoDbPopulatorHost.Consumers[0].Processor.AckCount >= 1);
new TestTimelineAwaiter().Await(() => mongoDbPopulatorHost.Consumers[1].Processor.AckCount >= 1);
logger.Info("\n### MongoDbPopulator has processed its messages ###\n");
new TestTimelineAwaiter().Await(() => identifierMapperHost.Consumer.AckCount >= 1);//number of series
logger.Info("\n### IdentifierMapper has processed its messages ###\n");
Assert.AreEqual(0, dicomTagReaderHost.AccessionDirectoryMessageConsumer.NackCount);
Assert.AreEqual(0, identifierMapperHost.Consumer.NackCount);
Assert.AreEqual(0, ((Consumer <SeriesMessage>)mongoDbPopulatorHost.Consumers[0]).NackCount);
Assert.AreEqual(0, ((Consumer <DicomFileMessage>)mongoDbPopulatorHost.Consumers[1]).NackCount);
try
{
Thread.Sleep(TimeSpan.FromSeconds(10));
new TestTimelineAwaiter().Await(() => relationalMapperHost.Consumer.AckCount >= numberOfExpectedRows, null, 30000, () => relationalMapperHost.Consumer.DleErrors); //number of image files
logger.Info("\n### DicomRelationalMapper has processed its messages ###\n");
}
finally
{
//find out what happens from the logging database
var rdmpLogging = new Rdmp.Core.Logging.LogManager(_helper.LoadMetadata.GetDistinctLoggingDatabase());
//if error was reported during the dicom relational mapper run
foreach (var dli in rdmpLogging.GetArchivalDataLoadInfos(_helper.LoadMetadata.GetDistinctLoggingTask(), null, null))
{
if (dli.StartTime > start)
{
foreach (ArchivalFatalError e in dli.Errors)
{
logger.Error(e.Date.TimeOfDay + ":" + e.Source + ":" + e.Description);
}
}
}
}
Assert.AreEqual(numberOfExpectedRows, _helper.ImageTable.GetRowCount(), "All images should appear in the image table");
Assert.LessOrEqual(_helper.SeriesTable.GetRowCount(), numberOfExpectedRows, "Only unique series data should appear in series table, there should be less unique series than images (or equal)");
Assert.LessOrEqual(_helper.StudyTable.GetRowCount(), numberOfExpectedRows, "Only unique study data should appear in study table, there should be less unique studies than images (or equal)");
Assert.LessOrEqual(_helper.StudyTable.GetRowCount(), _helper.SeriesTable.GetRowCount(), "There should be less studies than series (or equal)");
//make sure that the substitution identifier (that replaces old the PatientId) is the correct substitution (FISHFISH)/
Assert.AreEqual("FISHFISH", _helper.StudyTable.GetDataTable().Rows.OfType <DataRow>().First()["PatientId"]);
//The file size in the final table should be more than 0
Assert.Greater((long)_helper.ImageTable.GetDataTable().Rows.OfType <DataRow>().First()["DicomFileSize"], 0);
dicomTagReaderHost.Stop("TestIsFinished");
mongoDbPopulatorHost.Stop("TestIsFinished");
DropMongoTestDb(_globals.MongoDatabases.DicomStoreOptions.HostName, _globals.MongoDatabases.DicomStoreOptions.Port);
identifierMapperHost.Stop("TestIsFinished");
relationalMapperHost.Stop("Test end");
}
//Now do extraction
var extractorHost = new CohortExtractorHost(_globals, null, null);
extractorHost.Start();
var extract = new ExtractionRequestMessage
{
ExtractionJobIdentifier = Guid.NewGuid(),
ProjectNumber = "1234-5678",
ExtractionDirectory = "1234-5678_P1",
KeyTag = "SeriesInstanceUID",
};
foreach (DataRow row in _helper.ImageTable.GetDataTable().Rows)
{
var ser = (string)row["SeriesInstanceUID"];
if (!extract.ExtractionIdentifiers.Contains(ser))
{
extract.ExtractionIdentifiers.Add(ser);
}
}
tester.SendMessage(_globals.CohortExtractorOptions, extract);
//wait till extractor picked up the messages and dispatched the responses
new TestTimelineAwaiter().Await(() => extractorHost.Consumer.AckCount == 1);
extractorHost.Stop("TestIsFinished");
tester.Shutdown();
}
#endregion
}
