public void Test_BlankPatientIdentifier(Test testCase) { var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); //the declaration of what the guid namer table should be var options = new IdentifierMapperOptions(); options.MappingConnectionString = db.Server.Builder.ConnectionString; var swapper = new SwapForFixedValueTester("meeee"); swapper.Setup(options); var consumer = new IdentifierMapperQueueConsumer(Mock.Of <IProducerModel>(), swapper); var msg = GetTestDicomFileMessage(testCase: testCase); string reason; Assert.False(consumer.SwapIdentifier(msg, out reason)); switch (testCase) { case Test.EmptyInPatientTag: Assert.AreEqual("PatientID was blank", reason); break; case Test.NoPatientTag: Assert.AreEqual("Dataset did not contain PatientID", reason); break; } }
public void Test_Redist_CacheUsage(DatabaseType dbType) { var db = GetCleanedServer(dbType); DiscoveredTable map; using (var dt = new DataTable()) { dt.Columns.Add("CHI"); dt.Columns.Add("ECHI"); dt.Rows.Add("0101010101", "0A0A0A0A0A"); map = db.CreateTable("Map", dt); } var options = new IdentifierMapperOptions() { MappingTableName = map.GetFullyQualifiedName(), MappingConnectionString = db.Server.Builder.ConnectionString, SwapColumnName = "CHI", ReplacementColumnName = "ECHI", MappingDatabaseType = db.Server.DatabaseType }; RedisSwapper swapper; try { swapper = new RedisSwapper(TestRedisServer, new TableLookupWithGuidFallbackSwapper()); swapper.Setup(options); ClearRedisServer(); } catch (RedisConnectionException) { Assert.Inconclusive(); throw new Exception("To keep static analysis happy, btw Redis was unavailable"); } //hit on the lookup table string answer = swapper.GetSubstitutionFor("0101010101", out string reason); Assert.AreEqual("0A0A0A0A0A", answer); Assert.IsNull(reason); //hit didn't come from Redis Assert.AreEqual(0, swapper.CacheHit); Assert.AreEqual(1, swapper.Success); //hit from Redis string answer2 = swapper.GetSubstitutionFor("0101010101", out string reason2); Assert.AreEqual("0A0A0A0A0A", answer); Assert.IsNull(reason); //hit must come from Redis Assert.AreEqual(1, swapper.CacheHit); Assert.AreEqual(2, swapper.Success); }
public void TestIdentifierSwap2ForGuids_WithSeperateSwappers(DatabaseType dbType) { var db = GetCleanedServer(dbType); var mapTbl = db.ExpectTable("Map"); //the declaration of what the guid namer table should be var options = new IdentifierMapperOptions(); options.MappingConnectionString = db.Server.Builder.ConnectionString; options.MappingTableName = mapTbl.GetFullyQualifiedName(); options.SwapColumnName = "priv"; options.ReplacementColumnName = "pub"; options.MappingDatabaseType = dbType; var swapper1 = new ForGuidIdentifierSwapper(); swapper1.Setup(options); var swapper2 = new ForGuidIdentifierSwapper(); swapper2.Setup(options); var answer1 = swapper1.GetSubstitutionFor("01010101", out _); var answer2 = swapper2.GetSubstitutionFor("01010101", out _); Assert.AreEqual(answer1, answer2); Assert.IsNotNull(answer1); Assert.IsNotNull(answer2); }
public void TestIdentifierSwap2ForGuids(DatabaseType dbType) { var db = GetCleanedServer(dbType); var mapTbl = db.ExpectTable("Map"); //the declaration of what the guid namer table should be var options = new IdentifierMapperOptions(); options.MappingConnectionString = db.Server.Builder.ConnectionString; options.MappingTableName = mapTbl.GetFullyQualifiedName(); options.SwapColumnName = "priv"; options.ReplacementColumnName = "pub"; options.MappingDatabaseType = dbType; var swapper = new ForGuidIdentifierSwapper(); swapper.Setup(options); string reason; Assert.AreEqual(36, swapper.GetSubstitutionFor("01010101", out reason).Length); Assert.AreEqual(36, swapper.GetSubstitutionFor("02020202", out reason).Length); var answer1 = swapper.GetSubstitutionFor("03030303", out reason); var answer2 = swapper.GetSubstitutionFor("04040404", out reason); var answer3 = swapper.GetSubstitutionFor("03030303", out reason); Assert.AreEqual(answer1, answer3); Assert.AreNotEqual(answer1, answer2); }
public void TestIdentifierSwap(DatabaseType type) { var mappingDataTable = new DataTable("IdMap"); mappingDataTable.Columns.Add("priv"); mappingDataTable.Columns.Add("pub"); mappingDataTable.Rows.Add("010101", "020202"); var db = GetCleanedServer(type); var options = new IdentifierMapperOptions(); options.MappingConnectionString = db.Server.Builder.ConnectionString; options.MappingTableName = db.CreateTable("IdMap", mappingDataTable).GetFullyQualifiedName(); options.SwapColumnName = "priv"; options.ReplacementColumnName = "pub"; options.MappingDatabaseType = type; options.TimeoutInSeconds = 500; var swapper = new PreloadTableSwapper(); swapper.Setup(options); var consumer = new IdentifierMapperQueueConsumer(Mock.Of <IProducerModel>(), swapper); var msg = GetTestDicomFileMessage(); string reason; consumer.SwapIdentifier(msg, out reason); AssertDicomFileMessageHasPatientID(msg, "020202"); }
public void Test_SwapValueTooLong(DatabaseType dbType, bool createGuidTableUpFront) { var db = GetCleanedServer(dbType); DiscoveredTable map; using (var dt = new DataTable()) { dt.Columns.Add("CHI"); dt.Columns.Add("ECHI"); dt.Rows.Add("0101010101", "0A0A0A0A0A"); map = db.CreateTable("Map", dt); } using (var dt = new DataTable()) { dt.Columns.Add("CHI"); dt.Columns.Add("guid"); } if (createGuidTableUpFront) { db.CreateTable("Map_guid", new DatabaseColumnRequest[] { new DatabaseColumnRequest("CHI", new DatabaseTypeRequest(typeof(string), 30, null)), new DatabaseColumnRequest("Guid", new DatabaseTypeRequest(typeof(string), 36, null)), }); } var options = new IdentifierMapperOptions() { MappingTableName = map.GetFullyQualifiedName(), MappingConnectionString = db.Server.Builder.ConnectionString, SwapColumnName = "CHI", ReplacementColumnName = "ECHI", MappingDatabaseType = db.Server.DatabaseType }; var swapper = new TableLookupWithGuidFallbackSwapper(); swapper.Setup(options); //cache hit string answer = swapper.GetSubstitutionFor("010101010031002300020320402054240204022433040301", out string reason); Assert.IsNull(answer); if (createGuidTableUpFront) { StringAssert.AreEqualIgnoringCase("Supplied value was too long (48) - max allowed is (30)", reason); } else { StringAssert.AreEqualIgnoringCase("Supplied value was too long (48) - max allowed is (10)", reason); } }
/// <summary> /// Sets up a CHI/ECHI mapping table with fallback guid and populates each table with a single record. /// 0101010101 is a known CHI and 0202020202 is an known one (which was assigned a temporary guid mapping). /// Also prepares the main map table for DLE loading (<see cref="TriggerImplementer"/>) /// </summary> /// <param name="dbType"></param> /// <param name="map"></param> /// <param name="guidTable"></param> /// <param name="mapperOptions"></param> /// <param name="guids">true to create a <see cref="TableLookupWithGuidFallbackSwapper"/> otherwise creates a <see cref="TableLookupSwapper"/></param> private void SetupMappers(DatabaseType dbType, out DiscoveredTable map, out DiscoveredTable guidTable, out IdentifierMapperOptions mapperOptions, bool guids = true) { var db = GetCleanedServer(dbType); using (var dt = new DataTable()) { dt.Columns.Add("CHI"); dt.Columns.Add("ECHI"); dt.PrimaryKey = new [] { dt.Columns["CHI"] }; dt.Rows.Add("0101010101", "0A0A0A0A0A"); map = db.CreateTable("Map", dt); } mapperOptions = new IdentifierMapperOptions() { MappingTableName = map.GetFullyQualifiedName(), MappingConnectionString = db.Server.Builder.ConnectionString, SwapColumnName = "CHI", ReplacementColumnName = "ECHI", MappingDatabaseType = db.Server.DatabaseType, SwapperType = (guids ? typeof(TableLookupWithGuidFallbackSwapper):typeof(TableLookupSwapper)).FullName }; if (guids) { var swapper = new TableLookupWithGuidFallbackSwapper(); swapper.Setup(mapperOptions); guidTable = swapper.GetGuidTableIfAny(mapperOptions); Assert.AreEqual(0, guidTable.GetRowCount(), "No temporary guids should exist yet"); Assert.AreEqual(1, map.GetRowCount(), "We should have a mapping table with 1 entry"); //lookup an as yet unknown value swapper.GetSubstitutionFor("0202020202", out _); Assert.AreEqual(1, map.GetRowCount(), "We should have a mapping table with 1 entry"); Assert.AreEqual(1, guidTable.GetRowCount(), "We should have a temporary guid for 0202020202"); } else { guidTable = null; } // make a fake data load into this table (create trigger and insert/update) var triggerImplementer = new TriggerImplementerFactory(dbType).Create(map); triggerImplementer.CreateTrigger(new ThrowImmediatelyCheckNotifier()); }
public IdentifierMapperHost(GlobalOptions options, ISwapIdentifiers swapper = null) : base(options) { _consumerOptions = options.IdentifierMapperOptions; FansiImplementations.Load(); if (swapper == null) { Logger.Info("Not passed a swapper, creating one of type " + options.IdentifierMapperOptions.SwapperType); _swapper = ObjectFactory.CreateInstance <ISwapIdentifiers>(options.IdentifierMapperOptions.SwapperType, typeof(ISwapIdentifiers).Assembly); } else { _swapper = swapper; } // If we want to use a Redis server to cache answers then wrap the mapper in a Redis caching swapper if (!string.IsNullOrWhiteSpace(options.IdentifierMapperOptions.RedisConnectionString)) { try { _swapper = new RedisSwapper(options.IdentifierMapperOptions.RedisConnectionString, _swapper); } catch (RedisConnectionException e) { // NOTE(rkm 2020-03-30) Log & throw! I hate this, but if we don't log here using NLog, then the exception will bubble-up // and only be printed to STDERR instead of to the log file and may be lost Logger.Error(e, "Could not connect to Redis"); throw; } } _swapper.Setup(_consumerOptions); Logger.Info($"Swapper of type {_swapper.GetType()} created"); // Batching now handled implicitly as backlog demands _producerModel = RabbitMqAdapter.SetupProducer(options.IdentifierMapperOptions.AnonImagesProducerOptions, isBatch: true); Consumer = new IdentifierMapperQueueConsumer(_producerModel, _swapper) { AllowRegexMatching = options.IdentifierMapperOptions.AllowRegexMatching }; // Add our event handler for control messages AddControlHandler(new IdentifierMapperControlMessageHandler(_swapper)); }
public void TestIdentifierSwap_NoCache(DatabaseType type, Test test) { var mappingDataTable = new DataTable("IdMap"); mappingDataTable.Columns.Add("priv"); mappingDataTable.Columns.Add("pub"); mappingDataTable.Rows.Add("010101", "020202"); mappingDataTable.Rows.Add("0101010101", "0202020202"); var db = GetCleanedServer(type); var options = new IdentifierMapperOptions(); options.MappingConnectionString = db.Server.Builder.ConnectionString; options.MappingTableName = db.CreateTable("IdMap", mappingDataTable).GetFullyQualifiedName(); options.SwapColumnName = "priv"; options.ReplacementColumnName = "pub"; options.MappingDatabaseType = type; options.TimeoutInSeconds = 500; var swapper = new TableLookupSwapper(); swapper.Setup(options); var consumer = new IdentifierMapperQueueConsumer(Mock.Of <IProducerModel>(), swapper); consumer.AllowRegexMatching = true; var msg = GetTestDicomFileMessage(test); string reason; consumer.SwapIdentifier(msg, out reason); if (test == Test.Normal) { AssertDicomFileMessageHasPatientID(msg, "020202"); } else if (test == Test.ProperlyFormatedChi) { AssertDicomFileMessageHasPatientID(msg, "0202020202"); } else { Assert.Fail("Wrong test case?"); } }
public void Test_NoMatchingIdentifierFound() { var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); //the declaration of what the guid namer table should be var options = new IdentifierMapperOptions(); options.MappingConnectionString = db.Server.Builder.ConnectionString; //null here means it will never find any identifier var swapper = new SwapForFixedValueTester(null); swapper.Setup(options); var consumer = new IdentifierMapperQueueConsumer(Mock.Of <IProducerModel>(), swapper); var msg = GetTestDicomFileMessage(); string reason; Assert.False(consumer.SwapIdentifier(msg, out reason)); Assert.AreEqual("Swapper Microservices.IdentifierMapper.Tests.SwapForFixedValueTester returned null", reason); }
public void TestIdentifierSwapForGuid(DatabaseType dbType) { var db = GetCleanedServer(dbType); var mapTbl = db.ExpectTable("Map"); //the declaration of what the guid namer table should be var options = new IdentifierMapperOptions(); options.MappingConnectionString = db.Server.Builder.ConnectionString; options.MappingTableName = mapTbl.GetFullyQualifiedName(); options.SwapColumnName = "priv"; options.ReplacementColumnName = "pub"; options.MappingDatabaseType = dbType; var swapper = new ForGuidIdentifierSwapper(); swapper.Setup(options); swapper.Setup(options); swapper.Setup(options); //this isn't just for the lols, this will test both the 'create it mode' and the 'discover it mode' var consumer = new IdentifierMapperQueueConsumer(Mock.Of <IProducerModel>(), swapper); var msg = GetTestDicomFileMessage(); string reason; consumer.SwapIdentifier(msg, out reason); var newDs = DicomTypeTranslater.DeserializeJsonToDataset(msg.DicomDataset); var guidAllocated = newDs.GetValue <string>(DicomTag.PatientID, 0); var dt = mapTbl.GetDataTable(); Assert.AreEqual(1, dt.Rows.Count); //e.g. '841A2E3E-B7C9-410C-A5D1-816B95C0E806' Assert.AreEqual(36, guidAllocated.Length); }
public void Test_Cache1Hit1Miss(DatabaseType dbType) { var db = GetCleanedServer(dbType); DiscoveredTable map; using (var dt = new DataTable()) { dt.Columns.Add("CHI"); dt.Columns.Add("ECHI"); dt.Rows.Add("0101010101", "0A0A0A0A0A"); map = db.CreateTable("Map", dt); } var options = new IdentifierMapperOptions() { MappingTableName = map.GetFullyQualifiedName(), MappingConnectionString = db.Server.Builder.ConnectionString, SwapColumnName = "CHI", ReplacementColumnName = "ECHI", MappingDatabaseType = db.Server.DatabaseType }; var swapper = new TableLookupWithGuidFallbackSwapper(); swapper.Setup(options); //cache hit string answer = swapper.GetSubstitutionFor("0101010101", out string reason); Assert.AreEqual("0A0A0A0A0A", answer); Assert.IsNull(reason); var guidTable = swapper.GetGuidTableIfAny(options); Assert.AreEqual("Map_guid", guidTable.GetRuntimeName()); //The key column should match the SwapColumnName Assert.IsNotNull(guidTable.DiscoverColumn("CHI")); //but the swap column should always be called guid Assert.IsNotNull(guidTable.DiscoverColumn("guid")); string answer2 = swapper.GetSubstitutionFor("0202020202", out reason); //should be a guid e.g. like "bc70d07d-4c77-4086-be1c-2971fd66ccf2" Assert.IsNotNull(answer2); Assert.AreEqual(4, answer2.Count(c => c == '-'), $"Answer '{answer2}' did not look like a guid"); Assert.IsNull(reason); //make sure the guid mapping table has the correct row persisted for repeated calls Assert.IsTrue(guidTable.Exists()); Assert.AreEqual(1, guidTable.GetRowCount()); Assert.AreEqual("0202020202", guidTable.GetDataTable().Rows[0]["CHI"]); Assert.AreEqual(answer2, guidTable.GetDataTable().Rows[0]["guid"]); //repeated misses should not result in more rows and should return the same guid (obviously) Assert.AreEqual(answer2, swapper.GetSubstitutionFor("0202020202", out reason)); Assert.AreEqual(answer2, swapper.GetSubstitutionFor("0202020202", out reason)); Assert.AreEqual(answer2, swapper.GetSubstitutionFor("0202020202", out reason)); Assert.AreEqual(1, guidTable.GetRowCount()); Assert.AreEqual("0202020202", guidTable.GetDataTable().Rows[0]["CHI"]); Assert.AreEqual(answer2, guidTable.GetDataTable().Rows[0]["guid"]); //now insert a legit mapping for 0202020202 map.Insert(new Dictionary <string, object> { { "CHI", "0202020202" }, { "ECHI", "0B0B0B0B0B" } }); //note that the below line could fail if we ever implement miss caching (i.e. cache that we looked up the value and failed in the lookup swapper in which case this test would need to clearcache) //now that we have a cache hit we can lookup the good value Assert.AreEqual("0B0B0B0B0B", swapper.GetSubstitutionFor("0202020202", out reason)); }
public void MapperSource_IntegrationTest(DatabaseType dbType) { var db = GetCleanedServer(dbType); DataTable dt = new DataTable(); dt.Columns.Add("PatientID", typeof(string)); dt.Columns.Add("StudyDescription", typeof(string)); dt.SetDoNotReType(true); // We have a live table with anonymised data. There is one person with a known ECHI 0101010101=0A0A0A0A0A dt.Rows.Add("0A0A0A0A0A", "CT Head"); //There are 2 people for whome we have added temporary identifiers dt.Rows.Add("bbb-bbb-bbb", "CT Tail"); dt.Rows.Add("ccc-ccc-ccc", "CT Wings"); var liveTable = db.CreateTable("MyLiveTable", dt); DiscoveredTable map; using (var dtMap = new DataTable()) { dtMap.Columns.Add("CHI"); dtMap.Columns.Add("ECHI"); dtMap.PrimaryKey = new [] { dtMap.Columns["CHI"] }; dtMap.Rows.Add("0101010101", "0A0A0A0A0A"); map = db.CreateTable("Map", dtMap); } // Import into RDMP the live table so we have a TableInfo pointer to it floating around Import(liveTable); var mapperOptions = new IdentifierMapperOptions() { MappingTableName = map.GetFullyQualifiedName(), MappingConnectionString = db.Server.Builder.ConnectionString, SwapColumnName = "CHI", ReplacementColumnName = "ECHI", MappingDatabaseType = db.Server.DatabaseType, SwapperType = typeof(TableLookupWithGuidFallbackSwapper).FullName }; var swapper = new TableLookupWithGuidFallbackSwapper(); swapper.Setup(mapperOptions); var guidTable = swapper.GetGuidTableIfAny(mapperOptions); Assert.AreEqual(0, guidTable.GetRowCount(), "No temporary guids should exist yet"); Assert.AreEqual(1, map.GetRowCount(), "We should have a mapping table with 1 entry"); guidTable.Insert(new Dictionary <string, object>() { { "CHI", "0202020202" }, { TableLookupWithGuidFallbackSwapper.GuidColumnName, "bbb-bbb-bbb" } }); guidTable.Insert(new Dictionary <string, object>() { { "CHI", "0303030303" }, { TableLookupWithGuidFallbackSwapper.GuidColumnName, "ccc-ccc-ccc" } }); Assert.AreEqual(1, map.GetRowCount(), "We should have a mapping table with 1 entry"); Assert.AreEqual(2, guidTable.GetRowCount(), "We should have a temporary guid for 0202020202"); // make a fake data load into this table (create trigger and insert/update) var triggerImplementer = new TriggerImplementerFactory(dbType).Create(map); triggerImplementer.CreateTrigger(new ThrowImmediatelyCheckNotifier()); //create a brand new mapping map.Insert(new Dictionary <string, object>() { { "CHI", "0303030303" }, { "ECHI", "0C0C0C0C0C" }, { SpecialFieldNames.ValidFrom, DateTime.Now }, { SpecialFieldNames.DataLoadRunID, 55 }, }); var globals = new GlobalOptionsFactory().Load(); var cliOptions = new TriggerUpdatesFromMapperOptions() { DateOfLastUpdate = new DateTime(2020, 01, 01), LiveDatabaseFieldName = "PatientID", Qualifier = '\'', }; globals.UseTestValues( RequiresRabbit.GetConnectionFactory(), RequiresMongoDb.GetMongoClientSettings(), RequiresRelationalDb.GetRelationalDatabaseConnectionStrings(), ((TableRepository)RepositoryLocator.CatalogueRepository).ConnectionStringBuilder, ((TableRepository)RepositoryLocator.DataExportRepository).ConnectionStringBuilder); //make sure the identifier mapper goes to the right table globals.IdentifierMapperOptions.MappingConnectionString = db.Server.Builder.ConnectionString; globals.IdentifierMapperOptions.MappingDatabaseType = dbType; globals.IdentifierMapperOptions.MappingTableName = map.GetFullyQualifiedName(); globals.IdentifierMapperOptions.SwapperType = typeof(TableLookupWithGuidFallbackSwapper).FullName; using (var tester = new MicroserviceTester(globals.RabbitOptions, globals.CohortExtractorOptions)) { tester.CreateExchange(globals.TriggerUpdatesOptions.ExchangeName, globals.UpdateValuesOptions.QueueName); var sourceHost = new TriggerUpdatesHost(globals, new MapperSource(globals, cliOptions)); var destHost = new UpdateValuesHost(globals); sourceHost.Start(); tester.StopOnDispose.Add(sourceHost); destHost.Start(); tester.StopOnDispose.Add(destHost); //wait till updater is done updating the live table new TestTimelineAwaiter().Await(() => destHost.Consumer.AckCount == 1); } var liveDtAfter = liveTable.GetDataTable(); Assert.AreEqual(1, liveDtAfter.Rows.Cast <DataRow>().Count(r => (string)r["PatientID"] == "0A0A0A0A0A"), "Expected original data to still be intact"); Assert.AreEqual(1, liveDtAfter.Rows.Cast <DataRow>().Count(r => (string)r["PatientID"] == "bbb-bbb-bbb"), "Expected unknown CHI with guid bbb to still be unknown"); Assert.AreEqual(1, liveDtAfter.Rows.Cast <DataRow>().Count(r => (string)r["PatientID"] == "0C0C0C0C0C"), "Expected the unknown CHI ccc to be now known as 0C0C0C0C0C"); }
public void TestIdentifierSwap_MillionsOfRows(DatabaseType type) { Console.WriteLine("DatabaseType:" + type); var mappingDataTable = new DataTable("IdMap"); mappingDataTable.Columns.Add("priv"); mappingDataTable.Columns.Add("pub"); mappingDataTable.Rows.Add("abclkjlkjdefghijiklaskdf", Guid.NewGuid().ToString()); var db = GetCleanedServer(type); DiscoveredTable tbl; var options = new IdentifierMapperOptions(); options.MappingConnectionString = db.Server.Builder.ConnectionString; options.MappingTableName = (tbl = db.CreateTable("IdMap", mappingDataTable)).GetFullyQualifiedName(); options.SwapColumnName = "priv"; options.ReplacementColumnName = "pub"; options.MappingDatabaseType = type; Stopwatch sw = new Stopwatch(); sw.Start(); mappingDataTable.Rows.Clear(); using (var blk = tbl.BeginBulkInsert()) for (int i = 0; i < 9999999; i++) //9 million { mappingDataTable.Rows.Add(i.ToString(), Guid.NewGuid().ToString()); if (i % 100000 == 0) { blk.Upload(mappingDataTable); mappingDataTable.Rows.Clear(); Console.WriteLine("Upload Table " + i + " rows " + sw.ElapsedMilliseconds); } } sw.Stop(); sw.Reset(); sw.Start(); var swapper = new PreloadTableSwapper(); swapper.Setup(options); sw.Stop(); Console.WriteLine("PreloadTableSwapper.Setup:" + sw.ElapsedMilliseconds); sw.Reset(); sw.Start(); string reason; var answer = swapper.GetSubstitutionFor("12325", out reason); sw.Stop(); Console.WriteLine("Lookup Key:" + sw.ElapsedMilliseconds); sw.Reset(); Assert.IsNotNull(answer); Assert.IsTrue(answer.Length > 20); }