/// <summary>
/// Tests that the logging database for the load is reachable and that it has an appropriate logging task for the load (if not a new task will be created 'Loading X')
/// </summary>
/// <param name="catalogue"></param>
public void EnsureLoggingWorksFor(ICatalogue catalogue)
{
//if there's no logging task / logging server set them up with the same name as the lmd
IExternalDatabaseServer loggingServer;
if (catalogue.LiveLoggingServer_ID == null)
{
loggingServer = CatalogueRepository.GetServerDefaults().GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID);
if (loggingServer != null)
{
catalogue.LiveLoggingServer_ID = loggingServer.ID;
}
else
{
throw new NotSupportedException("You do not yet have any logging servers configured so cannot create data loads");
}
}
else
{
loggingServer = Repository.GetObjectByID <ExternalDatabaseServer>(catalogue.LiveLoggingServer_ID.Value);
}
//if there's no logging task yet and there's a logging server
if (string.IsNullOrWhiteSpace(catalogue.LoggingDataTask))
{
var lm = new LogManager(loggingServer);
var loggingTaskName = Name;
lm.CreateNewLoggingTaskIfNotExists(loggingTaskName);
catalogue.LoggingDataTask = loggingTaskName;
catalogue.SaveToDatabase();
}
}
public void SupportsValidation_NoDQE()
{
IServerDefaults defaults = CatalogueRepository.GetServerDefaults();
var before = defaults.GetDefaultFor(PermissableDefaults.DQE);
//cannot run test because it didn't have a value to clear!
Assert.IsNotNull(before);
//clear the default value
defaults.ClearDefault(PermissableDefaults.DQE);
try
{
CatalogueConstraintReport report = new CatalogueConstraintReport(_catalogue, SpecialFieldNames.DataLoadRunID);
var e = Assert.Throws <Exception>(() => report.Check(new ThrowImmediatelyCheckNotifier()));
Assert.IsTrue(
e.Message.StartsWith(
"Failed to create DQE Repository, possibly there is no DataQualityEngine Reporting Server (ExternalDatabaseServer). You will need to create/set one in CatalogueManager")
);
}
finally
{
defaults.SetDefault(PermissableDefaults.DQE, before);
}
}
public DatabaseTests()
{
var opts = new PlatformDatabaseCreationOptions()
{
ServerName = TestDatabaseSettings.ServerName,
Prefix = TestDatabaseNames.Prefix,
Username = TestDatabaseSettings.Username,
Password = TestDatabaseSettings.Password
};
RepositoryLocator = new PlatformDatabaseCreationRepositoryFinder(opts);
Console.WriteLine("Expecting Unit Test Catalogue To Be At Server=" + CatalogueRepository.DiscoveredServer.Name + " Database=" + CatalogueRepository.DiscoveredServer.GetCurrentDatabase());
Assert.IsTrue(CatalogueRepository.DiscoveredServer.Exists(), "Catalogue database does not exist, run 'rdmp.exe install ...' to create it (Ensure that servername and prefix in TestDatabases.txt match those you provide to CreateDatabases.exe e.g. 'rdmp.exe install localhost\\sqlexpress TEST_')");
Console.WriteLine("Found Catalogue");
Console.WriteLine("Expecting Unit Test Data Export To Be At Server=" + DataExportRepository.DiscoveredServer.Name + " Database= " + DataExportRepository.DiscoveredServer.GetCurrentDatabase());
Assert.IsTrue(DataExportRepository.DiscoveredServer.Exists(), "Data Export database does not exist, run 'rdmp.exe install ...' to create it (Ensure that servername and prefix in TestDatabases.txt match those you provide to CreateDatabases.exe e.g. 'rdmp.exe install localhost\\sqlexpress TEST_')");
Console.WriteLine("Found DataExport");
Console.Write(Environment.NewLine + Environment.NewLine + Environment.NewLine);
RunBlitzDatabases(RepositoryLocator);
var defaults = CatalogueRepository.GetServerDefaults();
DataQualityEngineConnectionString = CreateServerPointerInCatalogue(defaults, TestDatabaseNames.Prefix, PlatformDatabaseCreation.DefaultDQEDatabaseName, PermissableDefaults.DQE, new DataQualityEnginePatcher());
UnitTestLoggingConnectionString = CreateServerPointerInCatalogue(defaults, TestDatabaseNames.Prefix, PlatformDatabaseCreation.DefaultLoggingDatabaseName, PermissableDefaults.LiveLoggingServer_ID, new LoggingDatabasePatcher());
DiscoveredServerICanCreateRandomDatabasesAndTablesOn = new DiscoveredServer(CreateServerPointerInCatalogue(defaults, TestDatabaseNames.Prefix, null, PermissableDefaults.RAWDataLoadServer, null));
_discoveredSqlServer = new DiscoveredServer(TestDatabaseSettings.ServerName, null, DatabaseType.MicrosoftSQLServer, TestDatabaseSettings.Username, TestDatabaseSettings.Password);
if (TestDatabaseSettings.MySql != null)
{
var builder = new MySqlConnectionStringBuilder(TestDatabaseSettings.MySql);
foreach (string k in builder.Keys)
{
if (k == "server" || k == "database" || k == "user id" || k == "password")
{
continue;
}
new ConnectionStringKeyword(CatalogueRepository, DatabaseType.MySql, k, builder[k].ToString());
}
_discoveredMySqlServer = new DiscoveredServer(builder);
}
if (TestDatabaseSettings.Oracle != null)
{
_discoveredOracleServer = new DiscoveredServer(TestDatabaseSettings.Oracle, DatabaseType.Oracle);
}
if (TestDatabaseSettings.PostgreSql != null)
{
_discoveredPostgresServer = new DiscoveredServer(TestDatabaseSettings.PostgreSql, DatabaseType.PostgreSql);
}
}
public void SupportsValidation_NoLoggingServer()
{
IServerDefaults defaults = CatalogueRepository.GetServerDefaults();
var before = defaults.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID);
//cannot run test because it didn't have a value to clear!
Assert.IsNotNull(before);
//clear the default value
defaults.ClearDefault(PermissableDefaults.LiveLoggingServer_ID);
try
{
CatalogueConstraintReport report = new CatalogueConstraintReport(_catalogue, SpecialFieldNames.DataLoadRunID);
var e = Assert.Throws <Exception>(() => report.Check(new ThrowImmediatelyCheckNotifier()));
Assert.IsTrue(
e.Message.StartsWith(
"Failed to setup logging of DQE runs")
);
}
finally
{
defaults.SetDefault(PermissableDefaults.LiveLoggingServer_ID, before);
}
}
public void FinalTearDown()
{
//clear anostore default
CatalogueRepository.GetServerDefaults().ClearDefault(PermissableDefaults.ANOStore);
//delete the external server reference
ANOStore_ExternalDatabaseServer.DeleteInDatabase();
}
public void TestLargeImageDatasets(DatabaseType databaseType, int numberOfImages)
{
foreach (Pipeline p in CatalogueRepository.GetAllObjects <Pipeline>())
{
p.DeleteInDatabase();
}
var db = GetCleanedServer(databaseType);
var d = CatalogueRepository.GetServerDefaults();
d.ClearDefault(PermissableDefaults.RAWDataLoadServer);
var template = ImageTableTemplateCollection.LoadFrom(_templateXml);
_globals = new GlobalOptionsFactory().Load();
_globals.DicomRelationalMapperOptions.DatabaseNamerType = typeof(MyFixedStagingDatabaseNamer).FullName;
_globals.DicomRelationalMapperOptions.QoSPrefetchCount = ushort.MaxValue;
_globals.DicomRelationalMapperOptions.MinimumBatchSize = numberOfImages;
_globals.DicomRelationalMapperOptions.UseInsertIntoForRAWMigration = true;
_helper = new DicomRelationalMapperTestHelper();
_helper.SetupSuite(db, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource), root: null, template: template, persistentRaw: true);
//do not use an explicit RAW data load server
d.ClearDefault(PermissableDefaults.RAWDataLoadServer);
Random r = new Random(123);
List <DicomDataset> allImages;
using (var generator = new DicomDataGenerator(r, null, "CT"))
allImages = generator.GenerateImages(numberOfImages, r);
Assert.AreEqual(numberOfImages, allImages.Count);
using (var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.DicomRelationalMapperOptions))
{
using (var host = new DicomRelationalMapperHost(_globals))
{
tester.SendMessages(_globals.DicomRelationalMapperOptions, allImages.Select(GetFileMessageForDataset), true);
Console.WriteLine("Starting Host");
host.Start();
Stopwatch sw = Stopwatch.StartNew();
new TestTimelineAwaiter().Await(() => host.Consumer.AckCount == numberOfImages, null, 20 * 60 * 100); //1 minute
Console.Write("Time For DLE:" + sw.Elapsed.TotalSeconds + "s");
host.Stop("Test finished");
}
}
foreach (Pipeline allObject in CatalogueRepository.GetAllObjects <Pipeline>())
{
allObject.DeleteInDatabase();
}
}
public void TestMemoryVsDatabaseRepository_CatalogueConstructor()
{
var memoryRepository = new MemoryCatalogueRepository(CatalogueRepository.GetServerDefaults());
Catalogue memCatalogue = new Catalogue(memoryRepository, "My New Catalogue");
Catalogue dbCatalogue = new Catalogue(CatalogueRepository, "My New Catalogue");
UnitTests.AssertAreEqual(memCatalogue, dbCatalogue);
}
private void CreateReferenceInCatalogueToANODatabase()
{
RemovePreExistingReference();
//now create a new reference!
ANOStore_ExternalDatabaseServer = new ExternalDatabaseServer(CatalogueRepository, ANOStore_DatabaseName, new ANOStorePatcher());
ANOStore_ExternalDatabaseServer.SetProperties(ANOStore_Database);
CatalogueRepository.GetServerDefaults().SetDefault(PermissableDefaults.ANOStore, ANOStore_ExternalDatabaseServer);
}
public void TestImportingATable(DatabaseType dbType)
{
DataTable dt = new DataTable();
dt.Columns.Add("Do");
dt.Columns.Add("Ray");
dt.Columns.Add("Me");
dt.Columns.Add("Fa");
dt.Columns.Add("So");
var db = GetCleanedServer(dbType);
var tbl = db.CreateTable("OmgTables", dt);
var memoryRepository = new MemoryCatalogueRepository(CatalogueRepository.GetServerDefaults());
var importer1 = new TableInfoImporter(memoryRepository, tbl, DataAccessContext.Any);
TableInfo memTableInfo;
ColumnInfo[] memColumnInfos;
Catalogue memCatalogue;
CatalogueItem[] memCatalogueItems;
ExtractionInformation[] memExtractionInformations;
importer1.DoImport(out memTableInfo, out memColumnInfos);
var forwardEngineer1 = new ForwardEngineerCatalogue(memTableInfo, memColumnInfos);
forwardEngineer1.ExecuteForwardEngineering(out memCatalogue, out memCatalogueItems, out memExtractionInformations);
TableInfo dbTableInfo;
ColumnInfo[] dbColumnInfos;
Catalogue dbCatalogue;
CatalogueItem[] dbCatalogueItems;
ExtractionInformation[] dbExtractionInformations;
var importerdb = new TableInfoImporter(CatalogueRepository, tbl, DataAccessContext.Any);
importerdb.DoImport(out dbTableInfo, out dbColumnInfos);
var forwardEngineer2 = new ForwardEngineerCatalogue(dbTableInfo, dbColumnInfos);
forwardEngineer2.ExecuteForwardEngineering(out dbCatalogue, out dbCatalogueItems, out dbExtractionInformations);
UnitTests.AssertAreEqual(memCatalogue, dbCatalogue);
UnitTests.AssertAreEqual(memTableInfo, dbTableInfo);
UnitTests.AssertAreEqual(memCatalogue.CatalogueItems, dbCatalogue.CatalogueItems);
UnitTests.AssertAreEqual(memCatalogue.GetAllExtractionInformation(ExtractionCategory.Any), dbCatalogue.GetAllExtractionInformation(ExtractionCategory.Any));
UnitTests.AssertAreEqual(memCatalogue.CatalogueItems.Select(ci => ci.ColumnInfo), dbCatalogue.CatalogueItems.Select(ci => ci.ColumnInfo));
}
public void TestMemoryRepository_AggregateConfigurationConstructor()
{
var memoryRepository = new MemoryCatalogueRepository(CatalogueRepository.GetServerDefaults());
Catalogue memCatalogue = new Catalogue(memoryRepository, "My New Catalogue");
Catalogue dbCatalogue = new Catalogue(CatalogueRepository, "My New Catalogue");
var memAggregate = new AggregateConfiguration(memoryRepository, memCatalogue, "My New Aggregate");
var dbAggregate = new AggregateConfiguration(CatalogueRepository, dbCatalogue, "My New Aggregate");
UnitTests.AssertAreEqual(memAggregate, dbAggregate);
}
public void Setup_IdentifierDump()
{
IdentifierDump_Database = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(IdentifierDump_DatabaseName);
if (IdentifierDump_Database.Exists())
{
IdentifierDump_Database.Drop();
}
var scriptCreate = new MasterDatabaseScriptExecutor(IdentifierDump_Database);
var p = new IdentifierDumpDatabasePatcher();
scriptCreate.CreateAndPatchDatabase(p, new ThrowImmediatelyCheckNotifier());
//now create a new reference!
IdentifierDump_ExternalDatabaseServer = new ExternalDatabaseServer(CatalogueRepository, IdentifierDump_DatabaseName, p);
IdentifierDump_ExternalDatabaseServer.SetProperties(IdentifierDump_Database);
CatalogueRepository.GetServerDefaults().SetDefault(PermissableDefaults.IdentifierDumpServer_ID, IdentifierDump_ExternalDatabaseServer);
}
public void TestMemoryVsDatabaseRepository_ProcessTaskConstructor()
{
var memoryRepository = new MemoryCatalogueRepository(CatalogueRepository.GetServerDefaults());
var memLmd = new LoadMetadata(memoryRepository, "My New Load");
var dbLmd = new LoadMetadata(CatalogueRepository, "My New Load");
UnitTests.AssertAreEqual(memLmd, dbLmd);
var memPt = new ProcessTask(memoryRepository, memLmd, LoadStage.AdjustRaw)
{
Name = "MyPt"
};
var dbPt = new ProcessTask(CatalogueRepository, dbLmd, LoadStage.AdjustRaw)
{
Name = "MyPt"
};
UnitTests.AssertAreEqual(memPt, dbPt);
}
public void SetupSuite(DiscoveredDatabase server, bool persistentRaw = false, string modalityPrefix = null)
{
TestLogger.Setup();
_globals = new GlobalOptionsFactory().Load();
_globals.UseTestValues(
RequiresRabbit.GetConnectionFactory(),
RequiresMongoDb.GetMongoClientSettings(),
RequiresRelationalDb.GetRelationalDatabaseConnectionStrings(),
((TableRepository)RepositoryLocator.CatalogueRepository).ConnectionStringBuilder,
((TableRepository)RepositoryLocator.DataExportRepository).ConnectionStringBuilder);
_helper = new DicomRelationalMapperTestHelper();
_helper.SetupSuite(server, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource), null, null, persistentRaw, modalityPrefix);
_globals.DicomRelationalMapperOptions.RetryOnFailureCount = 0;
_globals.DicomRelationalMapperOptions.RetryDelayInSeconds = 0;
//do not use an explicit RAW data load server
CatalogueRepository.GetServerDefaults().ClearDefault(PermissableDefaults.RAWDataLoadServer);
}
public void TestBulkInsertOnly(DatabaseType databaseType, int numberOfImages)
{
foreach (Pipeline p in CatalogueRepository.GetAllObjects <Pipeline>())
{
p.DeleteInDatabase();
}
var db = GetCleanedServer(databaseType);
var template = ImageTableTemplateCollection.LoadFrom(_templateXml);
_globals = new GlobalOptionsFactory().Load();
_globals.DicomRelationalMapperOptions.DatabaseNamerType = typeof(MyFixedStagingDatabaseNamer).FullName;
_globals.DicomRelationalMapperOptions.QoSPrefetchCount = ushort.MaxValue;
_globals.DicomRelationalMapperOptions.MinimumBatchSize = numberOfImages;
_globals.DicomRelationalMapperOptions.UseInsertIntoForRAWMigration = true;
_helper = new DicomRelationalMapperTestHelper();
_helper.SetupSuite(db, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource), root: null, template: template, persistentRaw: true);
//do not use an explicit RAW data load server
CatalogueRepository.GetServerDefaults().ClearDefault(PermissableDefaults.RAWDataLoadServer);
Random r = new Random(123);
List <DicomDataset> allImages;
using (var generator = new DicomDataGenerator(r, null, "CT"))
allImages = generator.GenerateImages(numberOfImages, r);
DicomDatasetCollectionSource source = new DicomDatasetCollectionSource();
source.PreInitialize(new ExplicitListDicomDatasetWorklist(allImages.ToArray(), "amagad.dcm", new Dictionary <string, string>()
{
{ "MessageGuid", "0x123" }
}), new ThrowImmediatelyDataLoadEventListener());;
source.FilenameField = "gggg";
var dt = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
source.Dispose(new ThrowImmediatelyDataLoadEventListener()
{
WriteToConsole = true
}, null);
Assert.AreEqual(numberOfImages, allImages.Count);
Assert.AreEqual(numberOfImages, dt.Rows.Count);
var tables = _helper.LoadMetadata.GetDistinctTableInfoList(false);
var config = new HICDatabaseConfiguration(_helper.LoadMetadata, new SuffixBasedNamer());
var job = Mock.Of <IDataLoadJob>(
j => j.RegularTablesToLoad == tables.Cast <ITableInfo>().ToList() &&
j.DataLoadInfo == _dli &&
j.Configuration == config);
var attacher = new AutoRoutingAttacher();
attacher.Job = job;
//Drop Primary Keys
using (var con = db.Server.GetConnection())
{
con.Open();
var cmd = db.Server.GetCommand(
databaseType == DatabaseType.MicrosoftSQLServer ?
@"ALTER TABLE ImageTable DROP CONSTRAINT PK_ImageTable
ALTER TABLE SeriesTable DROP CONSTRAINT PK_SeriesTable
ALTER TABLE StudyTable DROP CONSTRAINT PK_StudyTable" :
@"ALTER TABLE ImageTable DROP PRIMARY KEY;
ALTER TABLE SeriesTable DROP PRIMARY KEY;
ALTER TABLE StudyTable DROP PRIMARY KEY;"
, con);
cmd.ExecuteNonQuery();
}
attacher.Initialize(LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "IgnoreMe", true), db);
try
{
attacher.ProcessPipelineData(dt, new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
attacher.Dispose(new ThrowImmediatelyDataLoadEventListener(), null);
}
catch (Exception e)
{
attacher.Dispose(new ThrowImmediatelyDataLoadEventListener(), e);
throw;
}
foreach (var tableInfo in tables)
{
Assert.AreEqual(numberOfImages,
tableInfo.Discover(DataAccessContext.InternalDataProcessing).GetRowCount(),
"Row count was wrong for " + tableInfo);
}
foreach (Pipeline allObject in CatalogueRepository.GetAllObjects <Pipeline>())
{
allObject.DeleteInDatabase();
}
}
public void ValidateBulkTestData(bool testCancellingValiationEarly)
{
int numberOfRecordsToGenerate = 10000;
DateTime startTime = DateTime.Now;
BulkTestsData testData = new BulkTestsData(CatalogueRepository, DiscoveredDatabaseICanCreateRandomTablesIn, numberOfRecordsToGenerate);
testData.SetupTestData();
testData.ImportAsCatalogue();
DQERepository dqeRepository = new DQERepository(CatalogueRepository);
//the shouldn't be any lingering results in the database
Assert.IsNull(dqeRepository.GetMostRecentEvaluationFor(_catalogue));
//set some validation rules
testData.catalogue.ValidatorXML = bulkTestDataValidation;
//set the time periodicity field
var toBeTimePeriodicityCol = testData.catalogue.GetAllExtractionInformation(ExtractionCategory.Any).Single(e => e.GetRuntimeName().Equals("dtCreated"));
testData.catalogue.TimeCoverage_ExtractionInformation_ID = toBeTimePeriodicityCol.ID;
//do the validation
CatalogueConstraintReport report = new CatalogueConstraintReport(testData.catalogue, SpecialFieldNames.DataLoadRunID);
report.Check(new ThrowImmediatelyCheckNotifier());
CancellationTokenSource source = new CancellationTokenSource();
if (testCancellingValiationEarly)
{
source.Cancel();
}
ToMemoryDataLoadEventListener listener = new ToMemoryDataLoadEventListener(false);
report.GenerateReport(testData.catalogue, listener, source.Token);
if (testCancellingValiationEarly)
{
Assert.IsTrue(listener.EventsReceivedBySender[report].Count(m => m.Exception is OperationCanceledException) == 1);
testData.Destroy();
testData.DeleteCatalogue();
return;
}
Assert.IsTrue(listener.EventsReceivedBySender[report].All(m => m.Exception == null));//all messages must have null exceptions
//get the reuslts now
var results = dqeRepository.GetMostRecentEvaluationFor(testData.catalogue);
Assert.IsNotNull(results);
//the sum of all consquences across all data load run ids should be the record count
Assert.AreEqual(10000, results.RowStates.Sum(r => r.Missing + r.Invalid + r.Wrong + r.Correct));
//there should be at least 5 data load run ids (should be around 12 actually - see BulkTestData but theoretically everyone could magically - all 10,000 into 5 decades - or even less but those statistics must be astronomical)
Assert.GreaterOrEqual(results.RowStates.Count(), 5);
//there should be lots of column results too
Assert.GreaterOrEqual(results.ColumnStates.Count(), 5);
//Did it log?
LogManager logManager = new LogManager(CatalogueRepository.GetServerDefaults().GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID));
var log = logManager.GetArchivalDataLoadInfos("DQE").FirstOrDefault();
Assert.IsNotNull(log);
Assert.GreaterOrEqual(log.StartTime, startTime);
Assert.AreEqual(0, log.Errors.Count);
Assert.AreEqual(numberOfRecordsToGenerate, log.TableLoadInfos.Single().Inserts);
testData.Destroy();
testData.DeleteCatalogue();
}
public DiscoveredServer GetDistinctLoggingDatabase(out IExternalDatabaseServer serverChosen)
{
serverChosen = CatalogueRepository.GetServerDefaults().GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID);
return(GetDistinctLoggingDatabase());
}
