public void TestIdentifierSwap2ForGuids(DatabaseType dbType)
{
var db = GetCleanedServer(dbType);
var mapTbl = db.ExpectTable("Map");
//the declaration of what the guid namer table should be
var options = new IdentifierMapperOptions();
options.MappingConnectionString = db.Server.Builder.ConnectionString;
options.MappingTableName = mapTbl.GetFullyQualifiedName();
options.SwapColumnName = "priv";
options.ReplacementColumnName = "pub";
options.MappingDatabaseType = dbType;
var swapper = new ForGuidIdentifierSwapper();
swapper.Setup(options);
string reason;
Assert.AreEqual(36, swapper.GetSubstitutionFor("01010101", out reason).Length);
Assert.AreEqual(36, swapper.GetSubstitutionFor("02020202", out reason).Length);
var answer1 = swapper.GetSubstitutionFor("03030303", out reason);
var answer2 = swapper.GetSubstitutionFor("04040404", out reason);
var answer3 = swapper.GetSubstitutionFor("03030303", out reason);
Assert.AreEqual(answer1, answer3);
Assert.AreNotEqual(answer1, answer2);
}
public void TestIdentifierSwap2ForGuids_WithSeperateSwappers(DatabaseType dbType)
{
var db = GetCleanedServer(dbType);
var mapTbl = db.ExpectTable("Map");
//the declaration of what the guid namer table should be
var options = new IdentifierMapperOptions();
options.MappingConnectionString = db.Server.Builder.ConnectionString;
options.MappingTableName = mapTbl.GetFullyQualifiedName();
options.SwapColumnName = "priv";
options.ReplacementColumnName = "pub";
options.MappingDatabaseType = dbType;
var swapper1 = new ForGuidIdentifierSwapper();
swapper1.Setup(options);
var swapper2 = new ForGuidIdentifierSwapper();
swapper2.Setup(options);
var answer1 = swapper1.GetSubstitutionFor("01010101", out _);
var answer2 = swapper2.GetSubstitutionFor("01010101", out _);
Assert.AreEqual(answer1, answer2);
Assert.IsNotNull(answer1);
Assert.IsNotNull(answer2);
}
public void TestIdentifierSwap(DatabaseType type)
{
var mappingDataTable = new DataTable("IdMap");
mappingDataTable.Columns.Add("priv");
mappingDataTable.Columns.Add("pub");
mappingDataTable.Rows.Add("010101", "020202");
var db = GetCleanedServer(type);
var options = new IdentifierMapperOptions();
options.MappingConnectionString = db.Server.Builder.ConnectionString;
options.MappingTableName = db.CreateTable("IdMap", mappingDataTable).GetFullyQualifiedName();
options.SwapColumnName = "priv";
options.ReplacementColumnName = "pub";
options.MappingDatabaseType = type;
options.TimeoutInSeconds = 500;
var swapper = new PreloadTableSwapper();
swapper.Setup(options);
var consumer = new IdentifierMapperQueueConsumer(Mock.Of <IProducerModel>(), swapper);
var msg = GetTestDicomFileMessage();
string reason;
consumer.SwapIdentifier(msg, out reason);
AssertDicomFileMessageHasPatientID(msg, "020202");
}
private ImageTableTemplateCollection GetDefaultTemplate(FAnsi.DatabaseType databaseType)
{
var collection = ImageTableTemplateCollection.LoadFrom(DefaultTemplateYaml);
collection.DatabaseType = databaseType;
return(collection);
}
public void IntegrationTest_BumpyRide(DatabaseType databaseType)
{
var server = GetCleanedServer(databaseType, ScratchDatabaseName);
SetupSuite(server);
_globals.DicomRelationalMapperOptions.Guid = new Guid("6c7cfbce-1af6-4101-ade7-6537eea72e03");
_globals.DicomRelationalMapperOptions.QoSPrefetchCount = 5000;
_globals.IdentifierMapperOptions.QoSPrefetchCount = 50;
_globals.DicomTagReaderOptions.NackIfAnyFileErrors = false;
_helper.TruncateTablesIfExists();
//Create test directory
var dir = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(IntegrationTest_BumpyRide)));
var r = new Random(500);
//create a generator
using (var generator = new DicomDataGenerator(r, dir, "CT"))
{
generator.GenerateImageFiles(40, r);
RunTest(dir, 40);
}
}
public void TestLargeImageDatasets(DatabaseType databaseType, int numberOfImages)
{
foreach (Pipeline p in CatalogueRepository.GetAllObjects <Pipeline>())
{
p.DeleteInDatabase();
}
var db = GetCleanedServer(databaseType);
var d = CatalogueRepository.GetServerDefaults();
d.ClearDefault(PermissableDefaults.RAWDataLoadServer);
var template = ImageTableTemplateCollection.LoadFrom(_templateXml);
_globals = new GlobalOptionsFactory().Load();
_globals.DicomRelationalMapperOptions.DatabaseNamerType = typeof(MyFixedStagingDatabaseNamer).FullName;
_globals.DicomRelationalMapperOptions.QoSPrefetchCount = ushort.MaxValue;
_globals.DicomRelationalMapperOptions.MinimumBatchSize = numberOfImages;
_globals.DicomRelationalMapperOptions.UseInsertIntoForRAWMigration = true;
_helper = new DicomRelationalMapperTestHelper();
_helper.SetupSuite(db, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource), root: null, template: template, persistentRaw: true);
//do not use an explicit RAW data load server
d.ClearDefault(PermissableDefaults.RAWDataLoadServer);
Random r = new Random(123);
List <DicomDataset> allImages;
using (var generator = new DicomDataGenerator(r, null, "CT"))
allImages = generator.GenerateImages(numberOfImages, r);
Assert.AreEqual(numberOfImages, allImages.Count);
using (var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.DicomRelationalMapperOptions))
{
using (var host = new DicomRelationalMapperHost(_globals))
{
tester.SendMessages(_globals.DicomRelationalMapperOptions, allImages.Select(GetFileMessageForDataset), true);
Console.WriteLine("Starting Host");
host.Start();
Stopwatch sw = Stopwatch.StartNew();
new TestTimelineAwaiter().Await(() => host.Consumer.AckCount == numberOfImages, null, 20 * 60 * 100); //1 minute
Console.Write("Time For DLE:" + sw.Elapsed.TotalSeconds + "s");
host.Stop("Test finished");
}
}
foreach (Pipeline allObject in CatalogueRepository.GetAllObjects <Pipeline>())
{
allObject.DeleteInDatabase();
}
}
public void IntegrationTest_Rejector(DatabaseType databaseType, Type rejector)
{
var server = GetCleanedServer(databaseType, ScratchDatabaseName);
SetupSuite(server, false, "MR_");
//this ensures that the ExtractionConfiguration.ID and Project.ID properties are out of sync (they are automnums). Its just a precaution since we are using both IDs in places if we
//had any bugs where we used the wrong one but they were the same then it would be obscured until production
var p = new Project(DataExportRepository, "delme");
p.DeleteInDatabase();
_globals.DicomRelationalMapperOptions.Guid = new Guid("fc229fc3-f700-4515-86e8-e3d38b3d1823");
_globals.DicomRelationalMapperOptions.QoSPrefetchCount = 5000;
_globals.DicomRelationalMapperOptions.DatabaseNamerType = typeof(GuidDatabaseNamer).FullName;
_globals.CohortExtractorOptions.RejectorType = rejector?.FullName;
_globals.FileSystemOptions.DicomSearchPattern = "*";
_helper.TruncateTablesIfExists();
//Create test directory with a single image
var dir = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(IntegrationTest_NoFileExtensions)));
dir.Create();
var arg = _helper.LoadMetadata.ProcessTasks.SelectMany(a => a.ProcessTaskArguments).Single(a => a.Name.Equals("ModalityMatchingRegex"));
arg.SetValue(new Regex("([A-z]*)_.*$"));
arg.SaveToDatabase();
//clean up the directory
foreach (FileInfo f in dir.GetFiles())
{
f.Delete();
}
TestData.Create(new FileInfo(Path.Combine(dir.FullName, "Mr.010101"))); //this is legit a dicom file
try
{
RunTest(dir, 1, (o) => o.DicomSearchPattern = "*");
}
finally
{
// Reset this in case it breaks other tests
_globals.FileSystemOptions.DicomSearchPattern = "*.dcm";
}
}
public void TestIdentifierSwap_NoCache(DatabaseType type, Test test)
{
var mappingDataTable = new DataTable("IdMap");
mappingDataTable.Columns.Add("priv");
mappingDataTable.Columns.Add("pub");
mappingDataTable.Rows.Add("010101", "020202");
mappingDataTable.Rows.Add("0101010101", "0202020202");
var db = GetCleanedServer(type);
var options = new IdentifierMapperOptions();
options.MappingConnectionString = db.Server.Builder.ConnectionString;
options.MappingTableName = db.CreateTable("IdMap", mappingDataTable).GetFullyQualifiedName();
options.SwapColumnName = "priv";
options.ReplacementColumnName = "pub";
options.MappingDatabaseType = type;
options.TimeoutInSeconds = 500;
var swapper = new TableLookupSwapper();
swapper.Setup(options);
var consumer = new IdentifierMapperQueueConsumer(Mock.Of <IProducerModel>(), swapper);
consumer.AllowRegexMatching = true;
var msg = GetTestDicomFileMessage(test);
string reason;
consumer.SwapIdentifier(msg, out reason);
if (test == Test.Normal)
{
AssertDicomFileMessageHasPatientID(msg, "020202");
}
else if (test == Test.ProperlyFormatedChi)
{
AssertDicomFileMessageHasPatientID(msg, "0202020202");
}
else
{
Assert.Fail("Wrong test case?");
}
}
public void TestIdentifierSwapForGuid(DatabaseType dbType)
{
var db = GetCleanedServer(dbType);
var mapTbl = db.ExpectTable("Map");
//the declaration of what the guid namer table should be
var options = new IdentifierMapperOptions();
options.MappingConnectionString = db.Server.Builder.ConnectionString;
options.MappingTableName = mapTbl.GetFullyQualifiedName();
options.SwapColumnName = "priv";
options.ReplacementColumnName = "pub";
options.MappingDatabaseType = dbType;
var swapper = new ForGuidIdentifierSwapper();
swapper.Setup(options);
swapper.Setup(options);
swapper.Setup(options); //this isn't just for the lols, this will test both the 'create it mode' and the 'discover it mode'
var consumer = new IdentifierMapperQueueConsumer(Mock.Of <IProducerModel>(), swapper);
var msg = GetTestDicomFileMessage();
string reason;
consumer.SwapIdentifier(msg, out reason);
var newDs = DicomTypeTranslater.DeserializeJsonToDataset(msg.DicomDataset);
var guidAllocated = newDs.GetValue <string>(DicomTag.PatientID, 0);
var dt = mapTbl.GetDataTable();
Assert.AreEqual(1, dt.Rows.Count);
//e.g. '841A2E3E-B7C9-410C-A5D1-816B95C0E806'
Assert.AreEqual(36, guidAllocated.Length);
}
public void TestBulkInsertOnly(DatabaseType databaseType, int numberOfImages)
{
foreach (Pipeline p in CatalogueRepository.GetAllObjects <Pipeline>())
{
p.DeleteInDatabase();
}
var db = GetCleanedServer(databaseType);
var template = ImageTableTemplateCollection.LoadFrom(_templateXml);
_globals = new GlobalOptionsFactory().Load();
_globals.DicomRelationalMapperOptions.DatabaseNamerType = typeof(MyFixedStagingDatabaseNamer).FullName;
_globals.DicomRelationalMapperOptions.QoSPrefetchCount = ushort.MaxValue;
_globals.DicomRelationalMapperOptions.MinimumBatchSize = numberOfImages;
_globals.DicomRelationalMapperOptions.UseInsertIntoForRAWMigration = true;
_helper = new DicomRelationalMapperTestHelper();
_helper.SetupSuite(db, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource), root: null, template: template, persistentRaw: true);
//do not use an explicit RAW data load server
CatalogueRepository.GetServerDefaults().ClearDefault(PermissableDefaults.RAWDataLoadServer);
Random r = new Random(123);
List <DicomDataset> allImages;
using (var generator = new DicomDataGenerator(r, null, "CT"))
allImages = generator.GenerateImages(numberOfImages, r);
DicomDatasetCollectionSource source = new DicomDatasetCollectionSource();
source.PreInitialize(new ExplicitListDicomDatasetWorklist(allImages.ToArray(), "amagad.dcm", new Dictionary <string, string>()
{
{ "MessageGuid", "0x123" }
}), new ThrowImmediatelyDataLoadEventListener());;
source.FilenameField = "gggg";
var dt = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
source.Dispose(new ThrowImmediatelyDataLoadEventListener()
{
WriteToConsole = true
}, null);
Assert.AreEqual(numberOfImages, allImages.Count);
Assert.AreEqual(numberOfImages, dt.Rows.Count);
var tables = _helper.LoadMetadata.GetDistinctTableInfoList(false);
var config = new HICDatabaseConfiguration(_helper.LoadMetadata, new SuffixBasedNamer());
var job = Mock.Of <IDataLoadJob>(
j => j.RegularTablesToLoad == tables.Cast <ITableInfo>().ToList() &&
j.DataLoadInfo == _dli &&
j.Configuration == config);
var attacher = new AutoRoutingAttacher();
attacher.Job = job;
//Drop Primary Keys
using (var con = db.Server.GetConnection())
{
con.Open();
var cmd = db.Server.GetCommand(
databaseType == DatabaseType.MicrosoftSQLServer ?
@"ALTER TABLE ImageTable DROP CONSTRAINT PK_ImageTable
ALTER TABLE SeriesTable DROP CONSTRAINT PK_SeriesTable
ALTER TABLE StudyTable DROP CONSTRAINT PK_StudyTable" :
@"ALTER TABLE ImageTable DROP PRIMARY KEY;
ALTER TABLE SeriesTable DROP PRIMARY KEY;
ALTER TABLE StudyTable DROP PRIMARY KEY;"
, con);
cmd.ExecuteNonQuery();
}
attacher.Initialize(LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "IgnoreMe", true), db);
try
{
attacher.ProcessPipelineData(dt, new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
attacher.Dispose(new ThrowImmediatelyDataLoadEventListener(), null);
}
catch (Exception e)
{
attacher.Dispose(new ThrowImmediatelyDataLoadEventListener(), e);
throw;
}
foreach (var tableInfo in tables)
{
Assert.AreEqual(numberOfImages,
tableInfo.Discover(DataAccessContext.InternalDataProcessing).GetRowCount(),
"Row count was wrong for " + tableInfo);
}
foreach (Pipeline allObject in CatalogueRepository.GetAllObjects <Pipeline>())
{
allObject.DeleteInDatabase();
}
}
public void Test_OnlyExtractableImages(DatabaseType dbType, bool useDynamic)
{
var db = GetCleanedServer(dbType);
//create table with 300 rows to ensure at least two studies
const int testrows = 300;
var tbl = BuildExampleExtractionTable(db, "CT", testrows, true);
Assert.AreEqual(testrows, tbl.GetRowCount());
var cata = Import(tbl);
List <string> studies;
//fetch all unique studies from the database
using (var dt = tbl.GetDataTable())
studies = dt.Rows.Cast <DataRow>().Select(r => r["StudyInstanceUID"]).Cast <string>().Distinct().ToList();
Assert.GreaterOrEqual(studies.Count, 2, "Expected at least 2 studies to be randomly generated in database");
//Create message to extract all the studies by StudyInstanceUID
var msgIn = new ExtractionRequestMessage();
msgIn.KeyTag = DicomTag.StudyInstanceUID.DictionaryEntry.Keyword;
msgIn.ExtractionIdentifiers = studies;
int matches = 0;
//The strategy pattern implementation that goes to the database but also considers reason
var fulfiller = new FromCataloguesExtractionRequestFulfiller(new[] { cata });
fulfiller.Rejectors.Add(useDynamic? (IRejector) new DynamicRejector():new TestRejector());
foreach (ExtractImageCollection msgOut in fulfiller.GetAllMatchingFiles(msgIn, new NullAuditExtractions()))
{
matches += msgOut.Accepted.Count();
Assert.IsEmpty(msgOut.Rejected);
}
//currently all images are extractable
Assert.AreEqual(testrows, matches);
//now make 10 not extractable
using (var con = tbl.Database.Server.GetConnection())
{
con.Open();
string sql = GetUpdateTopXSql(tbl, 10, "Set IsExtractableToDisk=0, IsExtractableToDisk_Reason = 'We decided NO!'");
//make the top 10 not extractable
using (var cmd = tbl.Database.Server.GetCommand(sql, con))
{
cmd.ExecuteNonQuery();
}
}
matches = 0;
int rejections = 0;
foreach (ExtractImageCollection msgOut in fulfiller.GetAllMatchingFiles(msgIn, new NullAuditExtractions()))
{
matches += msgOut.Accepted.Count;
rejections += msgOut.Rejected.Count;
Assert.IsTrue(msgOut.Rejected.All(v => v.RejectReason.Equals("We decided NO!")));
}
Assert.AreEqual(testrows - 10, matches);
Assert.AreEqual(10, rejections);
}
public void Test_OnlyListedModalities(DatabaseType dbType)
{
var db = GetCleanedServer(dbType);
//create table with 100 rows
var tblCT = BuildExampleExtractionTable(db, "CT", 70, true);
var tblMR = BuildExampleExtractionTable(db, "MR", 30, true);
var cataCT = Import(tblCT);
var cataMR = Import(tblMR);
List <string> studies = new List <string>();
//fetch all unique studies from the database
using (var dt = tblCT.GetDataTable())
studies.AddRange(dt.Rows.Cast <DataRow>().Select(r => r["StudyInstanceUID"]).Cast <string>().Distinct());
using (var dt = tblMR.GetDataTable())
studies.AddRange(dt.Rows.Cast <DataRow>().Select(r => r["StudyInstanceUID"]).Cast <string>().Distinct());
//Create message to extract all the series by StudyInstanceUID
var msgIn = new ExtractionRequestMessage();
msgIn.KeyTag = DicomTag.StudyInstanceUID.DictionaryEntry.Keyword;
//extract only MR (this is what we are actually testing).
msgIn.Modality = "MR";
msgIn.ExtractionIdentifiers = studies;
int matches = 0;
//The strategy pattern implementation that goes to the database but also considers reason
var fulfiller = new FromCataloguesExtractionRequestFulfiller(new[] { cataCT, cataMR })
{
//Give it the default modality pattern (table prefix)
ModalityRoutingRegex = new Regex(new CohortExtractorOptions().ModalityRoutingRegex)
};
foreach (ExtractImageCollection msgOut in fulfiller.GetAllMatchingFiles(msgIn, new NullAuditExtractions()))
{
matches += msgOut.Accepted.Count();
Assert.IsEmpty(msgOut.Rejected);
}
//expect only the MR images to be returned
Assert.AreEqual(30, matches);
// Ask for something that doesn't exist
msgIn.Modality = "Hello";
var ex = Assert.Throws <Exception>(() => fulfiller.GetAllMatchingFiles(msgIn, new NullAuditExtractions()).ToArray());
StringAssert.Contains("Modality=Hello", ex.Message);
// Ask for all modalities at once by not specifying any
msgIn.Modality = null;
Assert.AreEqual(100, fulfiller.GetAllMatchingFiles(msgIn, new NullAuditExtractions()).Sum(r => r.Accepted.Count));
// Ask for both modalities specifically
msgIn.Modality = "CT,Hello";
Assert.AreEqual(70, fulfiller.GetAllMatchingFiles(msgIn, new NullAuditExtractions()).Sum(r => r.Accepted.Count));
// Ask for both modalities specifically
msgIn.Modality = "CT,MR";
Assert.AreEqual(100, fulfiller.GetAllMatchingFiles(msgIn, new NullAuditExtractions()).Sum(r => r.Accepted.Count));
//when we don't have that flag anymore the error should tell us that
tblCT.DropColumn(tblCT.DiscoverColumn("IsOriginal"));
msgIn.Modality = "CT,MR";
ex = Assert.Throws(Is.AssignableTo(typeof(Exception)), () => fulfiller.GetAllMatchingFiles(msgIn, new NullAuditExtractions()).ToArray());
StringAssert.Contains("IsOriginal", ex.Message);
}
public void TestIdentifierSwap_MillionsOfRows(DatabaseType type)
{
Console.WriteLine("DatabaseType:" + type);
var mappingDataTable = new DataTable("IdMap");
mappingDataTable.Columns.Add("priv");
mappingDataTable.Columns.Add("pub");
mappingDataTable.Rows.Add("abclkjlkjdefghijiklaskdf", Guid.NewGuid().ToString());
var db = GetCleanedServer(type);
DiscoveredTable tbl;
var options = new IdentifierMapperOptions();
options.MappingConnectionString = db.Server.Builder.ConnectionString;
options.MappingTableName = (tbl = db.CreateTable("IdMap", mappingDataTable)).GetFullyQualifiedName();
options.SwapColumnName = "priv";
options.ReplacementColumnName = "pub";
options.MappingDatabaseType = type;
Stopwatch sw = new Stopwatch();
sw.Start();
mappingDataTable.Rows.Clear();
using (var blk = tbl.BeginBulkInsert())
for (int i = 0; i < 9999999; i++) //9 million
{
mappingDataTable.Rows.Add(i.ToString(), Guid.NewGuid().ToString());
if (i % 100000 == 0)
{
blk.Upload(mappingDataTable);
mappingDataTable.Rows.Clear();
Console.WriteLine("Upload Table " + i + " rows " + sw.ElapsedMilliseconds);
}
}
sw.Stop();
sw.Reset();
sw.Start();
var swapper = new PreloadTableSwapper();
swapper.Setup(options);
sw.Stop();
Console.WriteLine("PreloadTableSwapper.Setup:" + sw.ElapsedMilliseconds);
sw.Reset();
sw.Start();
string reason;
var answer = swapper.GetSubstitutionFor("12325", out reason);
sw.Stop();
Console.WriteLine("Lookup Key:" + sw.ElapsedMilliseconds);
sw.Reset();
Assert.IsNotNull(answer);
Assert.IsTrue(answer.Length > 20);
}
public void TestIdentifierSwap_RegexVsDeserialize(DatabaseType type, int batchSize, int numberOfRandomTagsPerDicom)
{
var options = new GlobalOptionsFactory().Load();
var mappingDataTable = new DataTable("IdMap");
mappingDataTable.Columns.Add("priv");
mappingDataTable.Columns.Add("pub");
mappingDataTable.Rows.Add("010101", "020202");
mappingDataTable.Rows.Add("0101010101", "0202020202");
var db = GetCleanedServer(type);
options.IdentifierMapperOptions.MappingConnectionString = db.Server.Builder.ConnectionString;
options.IdentifierMapperOptions.MappingTableName = db.CreateTable("IdMap", mappingDataTable).GetFullyQualifiedName();
options.IdentifierMapperOptions.SwapColumnName = "priv";
options.IdentifierMapperOptions.ReplacementColumnName = "pub";
options.IdentifierMapperOptions.MappingDatabaseType = type;
options.IdentifierMapperOptions.TimeoutInSeconds = 500;
var swapper = new PreloadTableSwapper();
swapper.Setup(options.IdentifierMapperOptions);
var goodChis = new List <DicomFileMessage>();
var badChis = new List <DicomFileMessage>();
Console.WriteLine("Generating Test data...");
List <Task> tasks = new List <Task>();
object oTaskLock = new object();
for (int i = 0; i < batchSize; i++)
{
var t = new Task(() =>
{
var a = GetTestDicomFileMessage(Test.ProperlyFormatedChi, numberOfRandomTagsPerDicom);
var b = GetTestDicomFileMessage(Test.ProperlyFormatedChi, numberOfRandomTagsPerDicom);
lock (oTaskLock)
{
goodChis.Add(a);
badChis.Add(b);
}
});
t.Start();
tasks.Add(t);
if (i % Environment.ProcessorCount == 0)
{
Task.WaitAll(tasks.ToArray());
tasks.Clear();
}
if (i % 100 == 0)
{
Console.WriteLine(i + " pairs done");
}
}
Task.WaitAll(tasks.ToArray());
options.IdentifierMapperOptions.AllowRegexMatching = true;
using (var tester = new MicroserviceTester(options.RabbitOptions, options.IdentifierMapperOptions))
{
tester.CreateExchange(options.IdentifierMapperOptions.AnonImagesProducerOptions.ExchangeName, null);
Console.WriteLine("Pushing good messages to Rabbit...");
tester.SendMessages(options.IdentifierMapperOptions, goodChis, true);
var host = new IdentifierMapperHost(options, swapper);
tester.StopOnDispose.Add(host);
Console.WriteLine("Starting host");
Stopwatch sw = Stopwatch.StartNew();
host.Start();
new TestTimelineAwaiter().Await(() => host.Consumer.AckCount == batchSize);
Console.WriteLine("Good message processing (" + batchSize + ") took:" + sw.ElapsedMilliseconds + "ms");
host.Stop("Test finished");
}
options.IdentifierMapperOptions.AllowRegexMatching = false;
using (var tester = new MicroserviceTester(options.RabbitOptions, options.IdentifierMapperOptions))
{
tester.CreateExchange(options.IdentifierMapperOptions.AnonImagesProducerOptions.ExchangeName, null);
Console.WriteLine("Pushing bad messages to Rabbit...");
tester.SendMessages(options.IdentifierMapperOptions, badChis, true);
var host = new IdentifierMapperHost(options, swapper);
tester.StopOnDispose.Add(host);
Console.WriteLine("Starting host");
Stopwatch sw = Stopwatch.StartNew();
host.Start();
new TestTimelineAwaiter().Await(() => host.Consumer.AckCount == batchSize);
Console.WriteLine("Bad message processing (" + batchSize + ") took:" + sw.ElapsedMilliseconds + "ms");
host.Stop("Test finished");
}
}
public void IntegrationTest_HappyPath_WithElevation(DatabaseType databaseType, bool persistentRaw)
{
var server = GetCleanedServer(databaseType, ScratchDatabaseName);
SetupSuite(server, persistentRaw: persistentRaw);
_globals.DicomRelationalMapperOptions.Guid = new Guid("fc229fc3-f888-4515-86e8-e3d38b3d1823");
_globals.DicomRelationalMapperOptions.QoSPrefetchCount = 5000;
_helper.TruncateTablesIfExists();
//add the column to the table
_helper.ImageTable.AddColumn("d_DerivationCodeMeaning", "varchar(100)", true, 5);
var archiveTable = _helper.ImageTable.Database.ExpectTable(_helper.ImageTable.GetRuntimeName() + "_Archive");
if (archiveTable.Exists())
{
archiveTable.AddColumn("d_DerivationCodeMeaning", "varchar(100)", true, 5);
}
new TableInfoSynchronizer(_helper.ImageTableInfo).Synchronize(new AcceptAllCheckNotifier());
var elevationRules = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, "ElevationConfig.xml"));
File.WriteAllText(elevationRules.FullName,
@"<!DOCTYPE TagElevationRequestCollection
[
<!ELEMENT TagElevationRequestCollection (TagElevationRequest*)>
<!ELEMENT TagElevationRequest (ColumnName,ElevationPathway,Conditional?)>
<!ELEMENT ColumnName (#PCDATA)>
<!ELEMENT ElevationPathway (#PCDATA)>
<!ELEMENT Conditional (ConditionalPathway,ConditionalRegex)>
<!ELEMENT ConditionalPathway (#PCDATA)>
<!ELEMENT ConditionalRegex (#PCDATA)>
]>
<TagElevationRequestCollection>
<TagElevationRequest>
<ColumnName>d_DerivationCodeMeaning</ColumnName>
<ElevationPathway>DerivationCodeSequence->CodeMeaning</ElevationPathway>
</TagElevationRequest>
</TagElevationRequestCollection>");
var arg = _helper.DicomSourcePipelineComponent.PipelineComponentArguments.Single(a => a.Name.Equals("TagElevationConfigurationFile"));
arg.SetValue(elevationRules);
arg.SaveToDatabase();
//Create test directory with a single image
var dir = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(IntegrationTest_HappyPath_WithElevation)));
dir.Create();
//clean up the directory
foreach (FileInfo f in dir.GetFiles())
{
f.Delete();
}
TestData.Create(new FileInfo(Path.Combine(dir.FullName, "MyTestFile.dcm")));
RunTest(dir, 1);
var tbl = _helper.ImageTable.GetDataTable();
Assert.AreEqual(1, tbl.Rows.Count);
Assert.AreEqual("Full fidelity image, uncompressed or lossless compressed", tbl.Rows[0]["d_DerivationCodeMeaning"]);
_helper.ImageTable.DropColumn(_helper.ImageTable.DiscoverColumn("d_DerivationCodeMeaning"));
if (archiveTable.Exists())
{
archiveTable.DropColumn(archiveTable.DiscoverColumn("d_DerivationCodeMeaning"));
}
}
public void IntegrationTest_HappyPath_WithIsolation(DatabaseType databaseType, Type namerType)
{
var server = GetCleanedServer(databaseType, ScratchDatabaseName);
SetupSuite(server, false, "MR_");
//this ensures that the ExtractionConfiguration.ID and Project.ID properties are out of sync (they are automnums). Its just a precaution since we are using both IDs in places if we
//had any bugs where we used the wrong one but they were the same then it would be obscured until production
var p = new Project(DataExportRepository, "delme");
p.DeleteInDatabase();
_globals.DicomRelationalMapperOptions.Guid = new Guid("fc229fc3-f700-4515-86e8-e3d38b3d1823");
_globals.DicomRelationalMapperOptions.QoSPrefetchCount = 5000;
_globals.DicomRelationalMapperOptions.MinimumBatchSize = 3;
_globals.DicomRelationalMapperOptions.DatabaseNamerType = namerType.FullName;
_helper.TruncateTablesIfExists();
//Create test directory with a single image
var dir = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(IntegrationTest_HappyPath_WithIsolation)));
dir.Create();
var ptIsolation = new ProcessTask(CatalogueRepository, _helper.LoadMetadata, LoadStage.AdjustRaw);
ptIsolation.CreateArgumentsForClassIfNotExists <PrimaryKeyCollisionIsolationMutilation>();
ptIsolation.Path = typeof(PrimaryKeyCollisionIsolationMutilation).FullName;
ptIsolation.ProcessTaskType = ProcessTaskType.MutilateDataTable;
ptIsolation.SaveToDatabase();
var arg1 = _helper.LoadMetadata.ProcessTasks.SelectMany(a => a.ProcessTaskArguments).Single(a => a.Name.Equals("TablesToIsolate"));
arg1.SetValue(new[] { _helper.StudyTableInfo, _helper.SeriesTableInfo, _helper.ImageTableInfo });
arg1.SaveToDatabase();
var db = new ExternalDatabaseServer(CatalogueRepository, "IsolationDatabase(live)", null);
db.SetProperties(server);
var arg2 = _helper.LoadMetadata.ProcessTasks.SelectMany(a => a.ProcessTaskArguments).Single(a => a.Name.Equals("IsolationDatabase"));
arg2.SetValue(db);
arg2.SaveToDatabase();
var arg3 = _helper.LoadMetadata.ProcessTasks.SelectMany(a => a.ProcessTaskArguments).Single(a => a.Name.Equals("ModalityMatchingRegex"));
arg3.SetValue(new Regex("([A-z]*)_.*$"));
arg3.SaveToDatabase();
//build the joins
new JoinInfo(CatalogueRepository,
_helper.ImageTableInfo.ColumnInfos.Single(c => c.GetRuntimeName().Equals("SeriesInstanceUID")),
_helper.SeriesTableInfo.ColumnInfos.Single(c => c.GetRuntimeName().Equals("SeriesInstanceUID")),
ExtractionJoinType.Right, null);
new JoinInfo(CatalogueRepository,
_helper.SeriesTableInfo.ColumnInfos.Single(c => c.GetRuntimeName().Equals("StudyInstanceUID")),
_helper.StudyTableInfo.ColumnInfos.Single(c => c.GetRuntimeName().Equals("StudyInstanceUID")),
ExtractionJoinType.Right, null);
//start with Study table
_helper.StudyTableInfo.IsPrimaryExtractionTable = true;
_helper.StudyTableInfo.SaveToDatabase();
//clean up the directory
foreach (FileInfo f in dir.GetFiles())
{
f.Delete();
}
TestData.Create(new FileInfo(Path.Combine(dir.FullName, "MyTestFile.dcm")));
var ds1 = new DicomDataset();
ds1.Add(DicomTag.StudyInstanceUID, "1.2.3");
ds1.Add(DicomTag.SeriesInstanceUID, "1.2.2");
ds1.Add(DicomTag.SOPInstanceUID, "1.2.3");
ds1.Add(DicomTag.PatientAge, "030Y");
ds1.Add(DicomTag.PatientID, "123");
ds1.Add(DicomTag.SOPClassUID, "1");
ds1.Add(DicomTag.Modality, "MR");
new DicomFile(ds1).Save(Path.Combine(dir.FullName, "abc.dcm"));
var ds2 = new DicomDataset();
ds2.Add(DicomTag.StudyInstanceUID, "1.2.3");
ds2.Add(DicomTag.SeriesInstanceUID, "1.2.4");
ds2.Add(DicomTag.SOPInstanceUID, "1.2.7");
ds2.Add(DicomTag.PatientAge, "040Y"); //age is replicated but should be unique at study level so gets isolated
ds2.Add(DicomTag.PatientID, "123");
ds2.Add(DicomTag.SOPClassUID, "1");
ds2.Add(DicomTag.Modality, "MR");
new DicomFile(ds2).Save(Path.Combine(dir.FullName, "def.dcm"));
var checks = new ProcessTaskChecks(_helper.LoadMetadata);
checks.Check(new AcceptAllCheckNotifier());
RunTest(dir, 1);
Assert.AreEqual(1, _helper.ImageTable.GetRowCount());
var isoTable = server.ExpectTable(_helper.ImageTable.GetRuntimeName() + "_Isolation");
Assert.AreEqual(2, isoTable.GetRowCount());
}
