public void Test_SwapValueTooLong(DatabaseType dbType, bool createGuidTableUpFront)
{
var db = GetCleanedServer(dbType);
DiscoveredTable map;
using (var dt = new DataTable())
{
dt.Columns.Add("CHI");
dt.Columns.Add("ECHI");
dt.Rows.Add("0101010101", "0A0A0A0A0A");
map = db.CreateTable("Map", dt);
}
using (var dt = new DataTable())
{
dt.Columns.Add("CHI");
dt.Columns.Add("guid");
}
if (createGuidTableUpFront)
{
db.CreateTable("Map_guid", new DatabaseColumnRequest[]
{
new DatabaseColumnRequest("CHI", new DatabaseTypeRequest(typeof(string), 30, null)),
new DatabaseColumnRequest("Guid", new DatabaseTypeRequest(typeof(string), 36, null)),
});
}
var options = new IdentifierMapperOptions()
{
MappingTableName = map.GetFullyQualifiedName(),
MappingConnectionString = db.Server.Builder.ConnectionString,
SwapColumnName = "CHI",
ReplacementColumnName = "ECHI",
MappingDatabaseType = db.Server.DatabaseType
};
var swapper = new TableLookupWithGuidFallbackSwapper();
swapper.Setup(options);
//cache hit
string answer = swapper.GetSubstitutionFor("010101010031002300020320402054240204022433040301", out string reason);
Assert.IsNull(answer);
if (createGuidTableUpFront)
{
StringAssert.AreEqualIgnoringCase("Supplied value was too long (48) - max allowed is (30)", reason);
}
else
{
StringAssert.AreEqualIgnoringCase("Supplied value was too long (48) - max allowed is (10)", reason);
}
}
/// <summary>
/// Sets up a CHI/ECHI mapping table with fallback guid and populates each table with a single record.
/// 0101010101 is a known CHI and 0202020202 is an known one (which was assigned a temporary guid mapping).
/// Also prepares the main map table for DLE loading (<see cref="TriggerImplementer"/>)
/// </summary>
/// <param name="dbType"></param>
/// <param name="map"></param>
/// <param name="guidTable"></param>
/// <param name="mapperOptions"></param>
/// <param name="guids">true to create a <see cref="TableLookupWithGuidFallbackSwapper"/> otherwise creates a <see cref="TableLookupSwapper"/></param>
private void SetupMappers(DatabaseType dbType, out DiscoveredTable map, out DiscoveredTable guidTable, out IdentifierMapperOptions mapperOptions, bool guids = true)
{
var db = GetCleanedServer(dbType);
using (var dt = new DataTable())
{
dt.Columns.Add("CHI");
dt.Columns.Add("ECHI");
dt.PrimaryKey = new [] { dt.Columns["CHI"] };
dt.Rows.Add("0101010101", "0A0A0A0A0A");
map = db.CreateTable("Map", dt);
}
mapperOptions = new IdentifierMapperOptions()
{
MappingTableName = map.GetFullyQualifiedName(),
MappingConnectionString = db.Server.Builder.ConnectionString,
SwapColumnName = "CHI",
ReplacementColumnName = "ECHI",
MappingDatabaseType = db.Server.DatabaseType,
SwapperType = (guids ? typeof(TableLookupWithGuidFallbackSwapper):typeof(TableLookupSwapper)).FullName
};
if (guids)
{
var swapper = new TableLookupWithGuidFallbackSwapper();
swapper.Setup(mapperOptions);
guidTable = swapper.GetGuidTableIfAny(mapperOptions);
Assert.AreEqual(0, guidTable.GetRowCount(), "No temporary guids should exist yet");
Assert.AreEqual(1, map.GetRowCount(), "We should have a mapping table with 1 entry");
//lookup an as yet unknown value
swapper.GetSubstitutionFor("0202020202", out _);
Assert.AreEqual(1, map.GetRowCount(), "We should have a mapping table with 1 entry");
Assert.AreEqual(1, guidTable.GetRowCount(), "We should have a temporary guid for 0202020202");
}
else
{
guidTable = null;
}
// make a fake data load into this table (create trigger and insert/update)
var triggerImplementer = new TriggerImplementerFactory(dbType).Create(map);
triggerImplementer.CreateTrigger(new ThrowImmediatelyCheckNotifier());
}
public void Test_Cache1Hit1Miss(DatabaseType dbType)
{
var db = GetCleanedServer(dbType);
DiscoveredTable map;
using (var dt = new DataTable())
{
dt.Columns.Add("CHI");
dt.Columns.Add("ECHI");
dt.Rows.Add("0101010101", "0A0A0A0A0A");
map = db.CreateTable("Map", dt);
}
var options = new IdentifierMapperOptions()
{
MappingTableName = map.GetFullyQualifiedName(),
MappingConnectionString = db.Server.Builder.ConnectionString,
SwapColumnName = "CHI",
ReplacementColumnName = "ECHI",
MappingDatabaseType = db.Server.DatabaseType
};
var swapper = new TableLookupWithGuidFallbackSwapper();
swapper.Setup(options);
//cache hit
string answer = swapper.GetSubstitutionFor("0101010101", out string reason);
Assert.AreEqual("0A0A0A0A0A", answer);
Assert.IsNull(reason);
var guidTable = swapper.GetGuidTableIfAny(options);
Assert.AreEqual("Map_guid", guidTable.GetRuntimeName());
//The key column should match the SwapColumnName
Assert.IsNotNull(guidTable.DiscoverColumn("CHI"));
//but the swap column should always be called guid
Assert.IsNotNull(guidTable.DiscoverColumn("guid"));
string answer2 = swapper.GetSubstitutionFor("0202020202", out reason);
//should be a guid e.g. like "bc70d07d-4c77-4086-be1c-2971fd66ccf2"
Assert.IsNotNull(answer2);
Assert.AreEqual(4, answer2.Count(c => c == '-'), $"Answer '{answer2}' did not look like a guid");
Assert.IsNull(reason);
//make sure the guid mapping table has the correct row persisted for repeated calls
Assert.IsTrue(guidTable.Exists());
Assert.AreEqual(1, guidTable.GetRowCount());
Assert.AreEqual("0202020202", guidTable.GetDataTable().Rows[0]["CHI"]);
Assert.AreEqual(answer2, guidTable.GetDataTable().Rows[0]["guid"]);
//repeated misses should not result in more rows and should return the same guid (obviously)
Assert.AreEqual(answer2, swapper.GetSubstitutionFor("0202020202", out reason));
Assert.AreEqual(answer2, swapper.GetSubstitutionFor("0202020202", out reason));
Assert.AreEqual(answer2, swapper.GetSubstitutionFor("0202020202", out reason));
Assert.AreEqual(1, guidTable.GetRowCount());
Assert.AreEqual("0202020202", guidTable.GetDataTable().Rows[0]["CHI"]);
Assert.AreEqual(answer2, guidTable.GetDataTable().Rows[0]["guid"]);
//now insert a legit mapping for 0202020202
map.Insert(new Dictionary <string, object>
{
{ "CHI", "0202020202" }, { "ECHI", "0B0B0B0B0B" }
});
//note that the below line could fail if we ever implement miss caching (i.e. cache that we looked up the value and failed in the lookup swapper in which case this test would need to clearcache)
//now that we have a cache hit we can lookup the good value
Assert.AreEqual("0B0B0B0B0B", swapper.GetSubstitutionFor("0202020202", out reason));
}
public void MapperSource_IntegrationTest(DatabaseType dbType)
{
var db = GetCleanedServer(dbType);
DataTable dt = new DataTable();
dt.Columns.Add("PatientID", typeof(string));
dt.Columns.Add("StudyDescription", typeof(string));
dt.SetDoNotReType(true);
// We have a live table with anonymised data. There is one person with a known ECHI 0101010101=0A0A0A0A0A
dt.Rows.Add("0A0A0A0A0A", "CT Head");
//There are 2 people for whome we have added temporary identifiers
dt.Rows.Add("bbb-bbb-bbb", "CT Tail");
dt.Rows.Add("ccc-ccc-ccc", "CT Wings");
var liveTable = db.CreateTable("MyLiveTable", dt);
DiscoveredTable map;
using (var dtMap = new DataTable())
{
dtMap.Columns.Add("CHI");
dtMap.Columns.Add("ECHI");
dtMap.PrimaryKey = new [] { dtMap.Columns["CHI"] };
dtMap.Rows.Add("0101010101", "0A0A0A0A0A");
map = db.CreateTable("Map", dtMap);
}
// Import into RDMP the live table so we have a TableInfo pointer to it floating around
Import(liveTable);
var mapperOptions = new IdentifierMapperOptions()
{
MappingTableName = map.GetFullyQualifiedName(),
MappingConnectionString = db.Server.Builder.ConnectionString,
SwapColumnName = "CHI",
ReplacementColumnName = "ECHI",
MappingDatabaseType = db.Server.DatabaseType,
SwapperType = typeof(TableLookupWithGuidFallbackSwapper).FullName
};
var swapper = new TableLookupWithGuidFallbackSwapper();
swapper.Setup(mapperOptions);
var guidTable = swapper.GetGuidTableIfAny(mapperOptions);
Assert.AreEqual(0, guidTable.GetRowCount(), "No temporary guids should exist yet");
Assert.AreEqual(1, map.GetRowCount(), "We should have a mapping table with 1 entry");
guidTable.Insert(new Dictionary <string, object>()
{
{ "CHI", "0202020202" },
{ TableLookupWithGuidFallbackSwapper.GuidColumnName, "bbb-bbb-bbb" }
});
guidTable.Insert(new Dictionary <string, object>()
{
{ "CHI", "0303030303" },
{ TableLookupWithGuidFallbackSwapper.GuidColumnName, "ccc-ccc-ccc" }
});
Assert.AreEqual(1, map.GetRowCount(), "We should have a mapping table with 1 entry");
Assert.AreEqual(2, guidTable.GetRowCount(), "We should have a temporary guid for 0202020202");
// make a fake data load into this table (create trigger and insert/update)
var triggerImplementer = new TriggerImplementerFactory(dbType).Create(map);
triggerImplementer.CreateTrigger(new ThrowImmediatelyCheckNotifier());
//create a brand new mapping
map.Insert(new Dictionary <string, object>()
{
{ "CHI", "0303030303" },
{ "ECHI", "0C0C0C0C0C" },
{ SpecialFieldNames.ValidFrom, DateTime.Now },
{ SpecialFieldNames.DataLoadRunID, 55 },
});
var globals = new GlobalOptionsFactory().Load();
var cliOptions = new TriggerUpdatesFromMapperOptions()
{
DateOfLastUpdate = new DateTime(2020, 01, 01),
LiveDatabaseFieldName = "PatientID",
Qualifier = '\'',
};
globals.UseTestValues(
RequiresRabbit.GetConnectionFactory(),
RequiresMongoDb.GetMongoClientSettings(),
RequiresRelationalDb.GetRelationalDatabaseConnectionStrings(),
((TableRepository)RepositoryLocator.CatalogueRepository).ConnectionStringBuilder,
((TableRepository)RepositoryLocator.DataExportRepository).ConnectionStringBuilder);
//make sure the identifier mapper goes to the right table
globals.IdentifierMapperOptions.MappingConnectionString = db.Server.Builder.ConnectionString;
globals.IdentifierMapperOptions.MappingDatabaseType = dbType;
globals.IdentifierMapperOptions.MappingTableName = map.GetFullyQualifiedName();
globals.IdentifierMapperOptions.SwapperType = typeof(TableLookupWithGuidFallbackSwapper).FullName;
using (var tester = new MicroserviceTester(globals.RabbitOptions, globals.CohortExtractorOptions))
{
tester.CreateExchange(globals.TriggerUpdatesOptions.ExchangeName, globals.UpdateValuesOptions.QueueName);
var sourceHost = new TriggerUpdatesHost(globals, new MapperSource(globals, cliOptions));
var destHost = new UpdateValuesHost(globals);
sourceHost.Start();
tester.StopOnDispose.Add(sourceHost);
destHost.Start();
tester.StopOnDispose.Add(destHost);
//wait till updater is done updating the live table
new TestTimelineAwaiter().Await(() => destHost.Consumer.AckCount == 1);
}
var liveDtAfter = liveTable.GetDataTable();
Assert.AreEqual(1, liveDtAfter.Rows.Cast <DataRow>().Count(r => (string)r["PatientID"] == "0A0A0A0A0A"), "Expected original data to still be intact");
Assert.AreEqual(1, liveDtAfter.Rows.Cast <DataRow>().Count(r => (string)r["PatientID"] == "bbb-bbb-bbb"), "Expected unknown CHI with guid bbb to still be unknown");
Assert.AreEqual(1, liveDtAfter.Rows.Cast <DataRow>().Count(r => (string)r["PatientID"] == "0C0C0C0C0C"), "Expected the unknown CHI ccc to be now known as 0C0C0C0C0C");
}
