public void CreateCSV(int numberOfRecords)
{
var r = new Random(500);
var people = new PersonCollection();
people.GeneratePeople(100, r);
var f = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, "DeleteMeTestBiochemistry.csv"));
bool finished = false;
int finishedWithRecords = -1;
var biochem = new Biochemistry(new Random(500));
biochem.RowsGenerated += (s, e) =>
{
finished = e.IsFinished;
finishedWithRecords = e.RowsWritten;
};
biochem.GenerateTestDataFile(people, f, numberOfRecords);
//one progress task only, should have reported creating the correct number of rows
Assert.IsTrue(finished);
Assert.AreEqual(numberOfRecords, finishedWithRecords);
Assert.GreaterOrEqual(File.ReadAllLines(f.FullName).Length, numberOfRecords);//can be newlines in middle of file
Console.WriteLine("Created file: " + f.FullName);
f.Delete();
}
static void Main(string[] args)
{
Console.WriteLine("Hello World!");
var rand = new Random(500 /*seed*/);
var pc = new PersonCollection();
// Generate 100 people
pc.GeneratePeople(100, rand);
// Generate a Biochemistry record
var biochemRecord = new BiochemistryRecord(rand);
//Get a random person from the collection
var person = pc.People[rand.Next(pc.People.Length)];
// Output some custom Xml format
using (var writer = new StreamWriter("out.xml"))
{
writer.WriteLine("<Record>");
writer.WriteLine($"<Date>{person.GetRandomDateDuringLifetime(rand)}</Date>");
writer.WriteLine($"<SampleType>{biochemRecord.SampleType}</SampleType>");
writer.WriteLine($"<TestCode>{biochemRecord.TestCode}</TestCode>");
writer.WriteLine($"<Result>{biochemRecord.Result}</Result>");
writer.WriteLine($"<Identifier>{biochemRecord.LabNumber}</Identifier>");
writer.WriteLine($"<ReadCode>{biochemRecord.ReadCodeValue}</ReadCode>");
writer.WriteLine("</Record>");
}
}
private void btnGenerate_Click(object sender, EventArgs e)
{
var uis = pDatasets.Controls.OfType <DataGeneratorUI>().Where(ui => ui.Generate).ToArray();
if (!uis.Any())
{
MessageBox.Show("At least one dataset must be selected");
return;
}
try
{
if (started)
{
MessageBox.Show("Generation already in progress");
return;
}
started = true;
var r = _seed.HasValue ? new Random(_seed.Value):new Random();
var identifiers = new PersonCollection();
identifiers.GeneratePeople(populationSize, r);
if (cbLookups.Checked)
{
DataGenerator.WriteLookups(_extractDirectory);
}
//run them at the same time
if (!_seed.HasValue)
{
foreach (DataGeneratorUI ui in uis)
{
Executing.Add(ui);
ui.BeginGeneration(identifiers, _extractDirectory);
DataGeneratorUI ui1 = ui;
ui.Completed += () => { Executing.Remove(ui1); AnnounceIfComplete(); };
}
}
else
{
Queue <DataGeneratorUI> queue = new Queue <DataGeneratorUI>(uis);
Execute(identifiers, queue, queue.Dequeue(), r);
}
UsefulStuff.GetInstance().ShowFolderInWindowsExplorer(_extractDirectory);
}
catch (Exception exception)
{
ExceptionViewer.Show(exception);
}
}
public void Join_PatientIndexTable_OptionalCacheOnSameServer(DatabaseType dbType, bool createQueryCache)
{
/*
* Server1
* _____________
* |Biochemistry|
* ↓
* ___________________
* | Cache (optional) |
* ↓ join ↓
* _____________________
* | Hospital Admissions|
*
*/
var db = GetCleanedServer(dbType);
ExternalDatabaseServer cache = null;
if (createQueryCache)
{
cache = CreateCache(db);
}
var r = new Random(500);
var people = new PersonCollection();
people.GeneratePeople(5000, r);
var cic = new CohortIdentificationConfiguration(CatalogueRepository, "cic");
var joinable = SetupPatientIndexTable(db, people, r, cic);
cic.CreateRootContainerIfNotExists();
cic.QueryCachingServer_ID = cache?.ID;
cic.SaveToDatabase();
var hospitalAdmissions = SetupPatientIndexTableUser(db, people, r, cic, joinable);
cic.RootCohortAggregateContainer.AddChild(hospitalAdmissions, 0);
var compiler = new CohortCompiler(cic);
var runner = new CohortCompilerRunner(compiler, 50000);
runner.Run(new CancellationToken());
AssertNoErrors(compiler);
if (createQueryCache)
{
Assert.IsTrue(compiler.Tasks.Any(t => t.Key.GetCachedQueryUseCount().Equals("1/1")), "Expected cache to be used for the joinable");
}
else
{
Assert.IsTrue(compiler.Tasks.Any(t => t.Key.GetCachedQueryUseCount().Equals("0/1")), "Did not create cache so expected cache usage to be 0");
}
}
private void RunAsIdentifierList(AggregateConfiguration ac, CachedAggregateConfigurationResultsManager cache, CancellationToken token)
{
var pc = new PersonCollection();
var requiredNumber = GetNumberToGenerate(ac);
var rand = new Random();
pc.GeneratePeople(requiredNumber, rand);
var set = new HashSet <string>(pc.People.Select(p => p.CHI));
// there may be duplicates, if so we need to bump up the number to match the required count
while (set.Count < requiredNumber)
{
pc.GeneratePeople(1, rand);
set.Add(pc.People[0].CHI);
}
// generate a list of random chis
SubmitIdentifierList("chi", set, ac, cache);
}
private static IPersonCollection GetPeople(ProgramOptions opts, out Random r)
{
r = opts.Seed == -1 ? new Random() : new Random(opts.Seed);
//create a cohort of people
IPersonCollection identifiers = new PersonCollection();
identifiers.GeneratePeople(opts.NumberOfPatients, r);
return(identifiers);
}
protected DiscoveredTable CreateDataset <T>(int people, int rows, Random r, out PersonCollection peopleGenerated) where T : IDataGenerator
{
var f = new DataGeneratorFactory();
T instance = f.Create <T>(r);
peopleGenerated = new PersonCollection();
peopleGenerated.GeneratePeople(people, r);
var dt = instance.GetDataTable(peopleGenerated, rows);
return(DiscoveredDatabaseICanCreateRandomTablesIn.CreateTable(typeof(T).Name, dt, null, false, this));
}
/// <summary>
/// Creates a new demography file ready for loading in the ForLoading directory of the load with the specified number of <paramref name="rows"/>
/// </summary>
/// <param name="filename">Filename to generate in ForLoading e.g. "bob.csv" (cannot be relative)</param>
/// <param name="rows"></param>
/// <param name="r">Seed random to ensure tests are reproducible</param>
protected FileInfo CreateFileInForLoading(string filename, int rows, Random r)
{
var fi = new FileInfo(Path.Combine(LoadDirectory.ForLoading.FullName, Path.GetFileName(filename)));
var demog = new Demography(r);
var people = new PersonCollection();
people.GeneratePeople(500, r);
demog.GenerateTestDataFile(people, fi, rows);
return(fi);
}
public void Join_PatientIndexTable_DoNotUseCacheOnDifferentServer(DatabaseType dbType)
{
/*
* Server1 Server 2
* _____________ _________
* |Biochemistry| → | Cache | (cache is still populated but not used in the resulting join).
*
* ↓ join ↓ (do not use cache)
* _____________________
* | Hospital Admissions|
*
*/
//get the data database
var db = GetCleanedServer(dbType);
//create the cache on the other server type (doesn't matter what type just as long as it's different).
var dbCache =
GetCleanedServer(Enum.GetValues(typeof(DatabaseType)).Cast <DatabaseType>().First(t => t != dbType));
ExternalDatabaseServer cache = CreateCache(dbCache);
var r = new Random(500);
var people = new PersonCollection();
people.GeneratePeople(5000, r);
var cic = new CohortIdentificationConfiguration(CatalogueRepository, "cic");
var joinable = SetupPatientIndexTable(db, people, r, cic);
cic.CreateRootContainerIfNotExists();
cic.QueryCachingServer_ID = cache?.ID;
cic.SaveToDatabase();
var hospitalAdmissions = SetupPatientIndexTableUser(db, people, r, cic, joinable);
cic.RootCohortAggregateContainer.AddChild(hospitalAdmissions, 0);
var compiler = new CohortCompiler(cic);
var runner = new CohortCompilerRunner(compiler, 50000);
runner.Run(new CancellationToken());
AssertNoErrors(compiler);
Assert.IsTrue(compiler.Tasks.Any(t => t.Key.GetCachedQueryUseCount().Equals("1/1")), "Expected cache to be used only for the final UNION");
}
public void Join_PatientIndexTable_ThreeServers()
{
/*
* Server1 Server 2 Server 3
* _____________ _________
* |Biochemistry| → (successfully caches joinable bit) | Cache |
*
* ↘ join ↘ (should crash)
* _____________________
* | Hospital Admissions|
*
*/
var server1 = GetCleanedServer(DatabaseType.MySql);
var server2 = GetCleanedServer(DatabaseType.MicrosoftSQLServer);
var server3 = GetCleanedServer(DatabaseType.Oracle);
ExternalDatabaseServer cache = CreateCache(server3);
var r = new Random(500);
var people = new PersonCollection();
people.GeneratePeople(5000, r);
var cic = new CohortIdentificationConfiguration(CatalogueRepository, "cic");
var joinable = SetupPatientIndexTable(server1, people, r, cic);
cic.CreateRootContainerIfNotExists();
cic.QueryCachingServer_ID = cache?.ID;
cic.SaveToDatabase();
var hospitalAdmissions = SetupPatientIndexTableUser(server2, people, r, cic, joinable);
cic.RootCohortAggregateContainer.AddChild(hospitalAdmissions, 0);
var compiler = new CohortCompiler(cic);
var runner = new CohortCompilerRunner(compiler, 50000);
runner.Run(new CancellationToken());
var hospitalAdmissionsTask = compiler.Tasks.Keys.OfType <AggregationTask>().Single(t => t.Aggregate.Equals(hospitalAdmissions));
Assert.AreEqual(CompilationState.Crashed, hospitalAdmissionsTask.State);
StringAssert.Contains("is not fully cached and CacheUsageDecision is MustUse", hospitalAdmissionsTask.CrashMessage.ToString());
}
public static IEnumerable <FileInfo> GenerateImageFiles(this DicomDataGenerator g, int numberOfImages, Random r)
{
var p = new PersonCollection();
p.GeneratePeople(5000, r);
if (g.OutputDir.Exists)
{
g.OutputDir.Delete(true);
}
var inventory = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "inventory.csv")); Path.Combine(TestContext.CurrentContext.WorkDirectory, "inventory.csv");
g.MaximumImages = numberOfImages;
g.GenerateTestDataFile(p, inventory, numberOfImages);
return(g.OutputDir.GetFiles("*.dcm", SearchOption.AllDirectories));
}
public void Test_CsvOption()
{
var r = new Random(500);
var outputDir = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "TestCsv"));
outputDir.Create();
var people = new PersonCollection();
people.GeneratePeople(100, r);
using (var generator = new DicomDataGenerator(r, outputDir, "CT"))
{
generator.Csv = true;
generator.NoPixels = true;
generator.MaximumImages = 500;
generator.GenerateTestDataFile(people, new FileInfo(Path.Combine(outputDir.FullName, "index.csv")), 500);
}
//3 csv files + index.csv (the default one
Assert.AreEqual(4, outputDir.GetFiles().Length);
foreach (FileInfo f in outputDir.GetFiles())
{
using (var reader = new CsvReader(new StreamReader(f.FullName), CultureInfo.CurrentCulture))
{
int rowcount = 0;
//confirms that the CSV is intact (no dodgy commas, unquoted newlines etc)
while (reader.Read())
{
rowcount++;
}
//should be 1 row per image + 1 for header
if (f.Name == DicomDataGenerator.ImageCsvFilename)
{
Assert.AreEqual(501, rowcount);
}
}
}
}
public void Join_PatientIndexTable_NotOnCacheServer()
{
/*
* Server1 Server 2
* _____________ _________
* |Biochemistry| → | Cache | (cache must first be populated)
*
* ↘ join ↘ (must use cache)
* _____________________
* | Hospital Admissions|
*
*/
var server1 = GetCleanedServer(DatabaseType.MySql);
var server2 = GetCleanedServer(DatabaseType.MicrosoftSQLServer);
ExternalDatabaseServer cache = CreateCache(server2);
var r = new Random(500);
var people = new PersonCollection();
people.GeneratePeople(5000, r);
var cic = new CohortIdentificationConfiguration(CatalogueRepository, "cic");
var joinable = SetupPatientIndexTable(server1, people, r, cic);
cic.CreateRootContainerIfNotExists();
cic.QueryCachingServer_ID = cache?.ID;
cic.SaveToDatabase();
var hospitalAdmissions = SetupPatientIndexTableUser(server2, people, r, cic, joinable);
cic.RootCohortAggregateContainer.AddChild(hospitalAdmissions, 0);
var compiler = new CohortCompiler(cic);
var runner = new CohortCompilerRunner(compiler, 50000);
runner.Run(new CancellationToken());
AssertNoErrors(compiler);
Assert.IsTrue(compiler.Tasks.Any(t => t.Key.GetCachedQueryUseCount().Equals("1/1")), "Expected cache to be used only for the final UNION");
}
private static void RunOptionsAndReturnExitCode(ProgramOptions opts)
{
if (opts.NumberOfPatients <= 0)
{
opts.NumberOfPatients = 500;
}
if (opts.NumberOfStudies <= 0)
{
opts.NumberOfStudies = 2000;
}
var dir = Directory.CreateDirectory(opts.OutputDirectory);
try
{
Random r = opts.Seed == -1 ? new Random() : new Random(opts.Seed);
//create a cohort of people
IPersonCollection identifiers = new PersonCollection();
identifiers.GeneratePeople(opts.NumberOfPatients, r);
//Generate the dicom files (of the modalities that the user requested)
string[] modalities = !string.IsNullOrWhiteSpace(opts.Modalities)? opts.Modalities.Split(",") :new string[0];
var dicomGenerator = new DicomDataGenerator(r, dir, modalities)
{
NoPixels = opts.NoPixels,
Layout = opts.Layout,
MaximumImages = opts.MaximumImages,
};
var targetFile = new FileInfo(Path.Combine(dir.FullName, "DicomFiles.csv"));
dicomGenerator.GenerateTestDataFile(identifiers, targetFile, opts.NumberOfStudies);
}
catch (Exception e)
{
Console.WriteLine(e);
returnCode = 2;
return;
}
returnCode = 0;
}
private void RunAsPatientIndexTable(AggregateConfiguration ac, CachedAggregateConfigurationResultsManager cache, CancellationToken token)
{
using DataTable dt = new DataTable();
dt.Columns.Add("chi", typeof(string));
dt.Columns.Add("dateOfBirth", typeof(DateTime));
dt.Columns.Add("dateOfDeath", typeof(DateTime));
// generate a list of random chis + date of birth/death
var pc = new PersonCollection();
pc.GeneratePeople(GetNumberToGenerate(ac), new Random());
foreach (var p in pc.People)
{
dt.Rows.Add(p.CHI, p.DateOfBirth, p.DateOfDeath ?? (object)DBNull.Value);
}
SubmitPatientIndexTable(dt, ac, cache, true);
}
//[TestCase(DatabaseType.Oracle,true)] //Oracle FAnsi doesn't currently support parameters
//[TestCase(DatabaseType.Oracle,false)]
public void Test_SingleServer_WithOneParameter(DatabaseType dbType, bool useParameter)
{
/*
* Server1 Server2
* _____________________ _____________________
* |HospitalAdmissions | → | Cache |
* @date_of_max
*
*/
var server1 = GetCleanedServer(dbType);
var server2 = GetCleanedServer(DatabaseType.MicrosoftSQLServer);
var cache = CreateCache(server2);
var r = new Random(500);
var people = new PersonCollection();
people.GeneratePeople(5000, r);
var cic = new CohortIdentificationConfiguration(CatalogueRepository, "cic");
var ac1 = SetupAggregateConfigurationWithFilter(server1, people, r, cic, useParameter, "@date_of_max", "'2001-01-01'");
cic.CreateRootContainerIfNotExists();
cic.QueryCachingServer_ID = cache.ID;
cic.SaveToDatabase();
var root = cic.RootCohortAggregateContainer;
root.AddChild(ac1, 0);
var compiler = new CohortCompiler(cic);
var runner = new CohortCompilerRunner(compiler, 50000);
runner.Run(new CancellationToken());
AssertNoErrors(compiler);
Assert.IsTrue(compiler.Tasks.Where(t => t.Key is AggregationContainerTask).Any(t => t.Key.GetCachedQueryUseCount().Equals("1/1")), "Expected UNION container to use the cache");
}
/// <summary>
/// Creates the <see cref="BulkDataTable"/> in the <see cref="BulkDataDatabase"/> and uploads test data. Use <see cref="ImportAsCatalogue"/> to get
/// rdmp metadata objects pointing at the table.
/// </summary>
public void SetupTestData()
{
//make sure database exists
if (!BulkDataDatabase.Exists())
{
BulkDataDatabase.Create();
}
//generate some people
var people = new PersonCollection();
people.GeneratePeople(5000, r);
//generate the test data
var dt = _dataGenerator.GetDataTable(people, ExpectedNumberOfRowsInTestData);
var tbl = BulkDataDatabase.ExpectTable(BulkDataTable);
if (tbl.Exists())
{
tbl.Drop();
}
//create the table but make sure the chi is a primary key and the correct data type and that we have a sensible primary key
Table = BulkDataDatabase.CreateTable(BulkDataTable, dt, new DatabaseColumnRequest[] {
new DatabaseColumnRequest("chi", new DatabaseTypeRequest(typeof(string), 10))
{
IsPrimaryKey = true
},
new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime)))
{
IsPrimaryKey = true
},
new DatabaseColumnRequest("hb_extract", new DatabaseTypeRequest(typeof(string), 1))
{
IsPrimaryKey = true
}
});
}
public void Join_PatientIndexTable_ThenShipToCacheForSets(DatabaseType dbType)
{
/*
* Server1 Server 2
* _____________ _________
* |Biochemistry| → | Cache | (cache is still populated but not used in the resulting join).
*
* ↓ join ↓ (do not use cache)
* _______________________
* | Hospital Admissions 1| → results1
*
* EXCEPT
* _______________________
* | Hospital Admissions 2| → results 2
* ↓ result = 0 records
*/
//get the data database
var db = GetCleanedServer(dbType);
//create the cache on the other server type (either Sql Server or Oracle) since MySql can't do EXCEPT or UNION etc)
var dbCache =
GetCleanedServer(Enum.GetValues(typeof(DatabaseType)).Cast <DatabaseType>().First(t =>
t != dbType && t != DatabaseType.MySql));
ExternalDatabaseServer cache = CreateCache(dbCache);
var r = new Random(500);
var people = new PersonCollection();
people.GeneratePeople(5000, r);
var cic = new CohortIdentificationConfiguration(CatalogueRepository, "cic");
var joinable = SetupPatientIndexTable(db, people, r, cic);
cic.CreateRootContainerIfNotExists();
cic.QueryCachingServer_ID = cache?.ID;
cic.SaveToDatabase();
var hospitalAdmissions = SetupPatientIndexTableUser(db, people, r, cic, joinable);
var hospitalAdmissions2 = SetupAggregateConfiguration(db, people, r, cic);
var root = cic.RootCohortAggregateContainer;
root.AddChild(hospitalAdmissions, 0);
root.AddChild(hospitalAdmissions2, 1);
root.Name = "EXCEPT";
root.Operation = SetOperation.EXCEPT;
root.SaveToDatabase();
var compiler = new CohortCompiler(cic);
var runner = new CohortCompilerRunner(compiler, 50000);
runner.Run(new CancellationToken());
AssertNoErrors(compiler);
Assert.IsTrue(compiler.Tasks.Any(t => t.Key.GetCachedQueryUseCount().Equals("2/2")), "Expected cache to be used for both top level operations in the EXCEPT");
}
public void Test_SingleServerPatientIndexTable_WithTwoParameters(DatabaseType dbType, bool useSameName, bool useCache)
{
/*
* Server1 (Also Server1 - if useCache is true)
* ______________________________ __________________________
* | Patient Index Table | → | Cache |
* | (NA by date ) | ↗
* @date_of_max '2001-01-01'
* ↗
* JOIN (should use cache)
* ___________________________ ↗
* | Hospitalised after NA (ac) |
* @date_of_max (or @maximum) '2005-01-01'
*
* (has different value so rename operations come into effect)
*/
var server1 = GetCleanedServer(dbType);
var cache = useCache? CreateCache(server1): null;
var r = new Random(500);
var people = new PersonCollection();
people.GeneratePeople(5000, r);
var cic = new CohortIdentificationConfiguration(CatalogueRepository, "cic");
var patientIndexTable = SetupPatientIndexTableWithFilter(server1, people, r, cic, true, "@date_of_max", "'2001-01-01'");
var ac = SetupPatientIndexTableUserWithFilter(server1, people, r, cic, patientIndexTable, true,
useSameName ? "@date_of_max": "@maximum", "'2005-01-01'");
cic.CreateRootContainerIfNotExists();
cic.QueryCachingServer_ID = cache?.ID;
cic.SaveToDatabase();
var root = cic.RootCohortAggregateContainer;
root.AddChild(ac, 0);
var compiler = new CohortCompiler(cic);
var runner = new CohortCompilerRunner(compiler, 50000);
runner.Run(new CancellationToken());
if (useSameName)
{
AssertCrashed(compiler, ac, "PatientIndexTables cannot have parameters with the same name as their users. Offending parameter(s) were @date_of_max");
}
else
{
if (useCache)
{
//we should hit up the cache for the interior of the query and therefore not need the parameter
AssertNoErrors(compiler, ac, "@maximum='2005-01-01'", "JoinableInceptionQuery_AggregateConfiguration", "AdmissionDate. < @maximum");
AssertCacheUsed(compiler, root, "1/1");
}
else
{
AssertNoErrors(compiler, ac, "@date_of_max='2001-01-01'", "@maximum='2005-01-01'", "SampleDate. < @date_of_max", "AdmissionDate. < @maximum");
}
AssertNoErrors(compiler);
}
}
protected DiscoveredTable CreateDataset <T>(DiscoveredDatabase db, int people, int rows, Random r, out PersonCollection peopleGenerated) where T : IDataGenerator
{
peopleGenerated = new PersonCollection();
peopleGenerated.GeneratePeople(people, r);
return(CreateDataset <T>(db, peopleGenerated, rows, r));
}
internal void Create(DiscoveredDatabase db, ICheckNotifier notifier, PlatformDatabaseCreationOptions options)
{
if (db.Exists())
{
if (options.DropDatabases)
{
db.Drop();
}
else
{
throw new Exception("Database " + db.GetRuntimeName() + " already exists and allowDrop option was not specified");
}
}
notifier.OnCheckPerformed(new CheckEventArgs("About to create " + db.GetRuntimeName(), CheckResult.Success));
//create a new database for the datasets
db.Create();
notifier.OnCheckPerformed(new CheckEventArgs("Succesfully created " + db.GetRuntimeName(), CheckResult.Success));
//fixed seed so everyone gets the same datasets
var r = new Random(options.Seed);
notifier.OnCheckPerformed(new CheckEventArgs("Generating people", CheckResult.Success));
//people
var people = new PersonCollection();
people.GeneratePeople(options.NumberOfPeople, r);
//datasets
var biochem = ImportCatalogue(Create <Biochemistry>(db, people, r, notifier, options.NumberOfRowsPerDataset, "chi", "Healthboard", "SampleDate", "TestCode"));
var demography = ImportCatalogue(Create <Demography>(db, people, r, notifier, options.NumberOfRowsPerDataset, "chi", "dtCreated", "hb_extract"));
var prescribing = ImportCatalogue(Create <Prescribing>(db, people, r, notifier, options.NumberOfRowsPerDataset, "chi", "PrescribedDate", "Name")); //<- this is slooo!
var admissions = ImportCatalogue(Create <HospitalAdmissions>(db, people, r, notifier, options.NumberOfRowsPerDataset, "chi", "AdmissionDate"));
var carotid = Create <CarotidArteryScan>(db, people, r, notifier, options.NumberOfRowsPerDataset, "RECORD_NUMBER");
//the following should not be extractable
ForExtractionInformations(demography,
e => e.DeleteInDatabase(),
"chi_num_of_curr_record",
"surname",
"forename",
"current_address_L1",
"current_address_L2",
"current_address_L3",
"current_address_L4",
"birth_surname",
"previous_surname",
"midname",
"alt_forename",
"other_initials",
"previous_address_L1",
"previous_address_L2",
"previous_address_L3",
"previous_address_L4",
"previous_postcode",
"date_address_changed",
"adr",
"previous_gp_accept_date",
"hic_dataLoadRunID");
//the following should be special approval only
ForExtractionInformations(demography,
e => {
e.ExtractionCategory = ExtractionCategory.SpecialApprovalRequired;
e.SaveToDatabase();
},
"current_postcode",
"current_gp",
"previous_gp",
"date_of_birth");
CreateAdmissionsViews(db);
var vConditions = ImportCatalogue(db.ExpectTable("vConditions"));
var vOperations = ImportCatalogue(db.ExpectTable("vOperations"));
CreateGraph(biochem, "Test Codes", "TestCode", false, null);
CreateGraph(biochem, "Test Codes By Date", "SampleDate", true, "TestCode");
CreateFilter(biochem, "Creatinine", "TestCode", "TestCode like '%CRE%'", @"Serum creatinine is a blood measurement. It is an indicator of renal health.");
CreateFilter(biochem, "Test Code", "TestCode", "TestCode like @code", "Filters any test code set");
CreateExtractionInformation(demography, "Age", "date_of_birth", "FLOOR(DATEDIFF(DAY, date_of_birth, GETDATE()) / 365.25) As Age");
var fAge = CreateFilter(demography, "Older at least x years", "Age", "FLOOR(DATEDIFF(DAY, date_of_birth, GETDATE()) / 365.25) >= @age", "Patients age is greater than or equal to the provided @age");
SetParameter(fAge, "@age", "int", "16");
CreateGraph(demography, "Patient Ages", "Age", false, null);
CreateGraph(prescribing, "Approved Name", "ApprovedName", false, null);
CreateGraph(prescribing, "Approved Name Over Time", "PrescribedDate", true, "ApprovedName");
CreateGraph(prescribing, "Bnf", "FormattedBnfCode", false, null);
CreateGraph(prescribing, "Bnf Over Time", "PrescribedDate", true, "FormattedBnfCode");
CreateFilter(
CreateGraph(vConditions, "Conditions frequency", "Field", false, "Condition"),
"Common Conditions Only",
@"(Condition in
(select top 40 Condition from vConditions c
WHERE Condition <> 'NULL' AND Condition <> 'Nul'
group by Condition order by count(*) desc))");
CreateFilter(
CreateGraph(vOperations, "Operation frequency", "Field", false, "Operation"),
"Common Operation Only",
@"(Operation in
(select top 40 Operation from vOperations c
WHERE Operation <> 'NULL' AND Operation <> 'Nul'
group by Operation order by count(*) desc))");
//group these all into the same folder
admissions.Folder = new CatalogueFolder(admissions, @"\admissions");
admissions.SaveToDatabase();
vConditions.Folder = new CatalogueFolder(vConditions, @"\admissions");
vConditions.SaveToDatabase();
vOperations.Folder = new CatalogueFolder(vOperations, @"\admissions");
vOperations.SaveToDatabase();
//Create cohort store database
var wizard = new CreateNewCohortDatabaseWizard(db, _repos.CatalogueRepository, _repos.DataExportRepository, false);
var externalCohortTable = wizard.CreateDatabase(new PrivateIdentifierPrototype("chi", "varchar(10)"), new ThrowImmediatelyCheckNotifier());
//Find the pipeline for committing cohorts
var cohortCreationPipeline = _repos.CatalogueRepository.GetAllObjects <Pipeline>().FirstOrDefault(p => p?.Source?.Class == typeof(CohortIdentificationConfigurationSource).FullName);
if (cohortCreationPipeline == null)
{
throw new Exception("Could not find a cohort committing pipeline");
}
//A cohort creation query
var f = CreateFilter(vConditions, "Lung Cancer Condition", "Condition", "Condition like 'C349'", "ICD-10-CM Diagnosis Code C34.9 Malignant neoplasm of unspecified part of bronchus or lung");
var cic = CreateCohortIdentificationConfiguration((ExtractionFilter)f);
var cohort = CommitCohortToNewProject(cic, externalCohortTable, cohortCreationPipeline, "Lung Cancer Project", "P1 Lung Cancer Patients", 123, out Project project);
var cohortTable = cohort.ExternalCohortTable.DiscoverCohortTable();
using (var con = cohortTable.Database.Server.GetConnection())
{
con.Open();
//delete half the records (so we can simulate cohort refresh)
var cmd = cohortTable.Database.Server.GetCommand(string.Format("DELETE TOP (10) PERCENT from {0}", cohortTable.GetFullyQualifiedName()), con);
cmd.ExecuteNonQuery();
}
var ec1 = CreateExtractionConfiguration(project, cohort, "First Extraction (2016 - project 123)", true, notifier, biochem, prescribing, demography);
var ec2 = CreateExtractionConfiguration(project, cohort, "Project 123 - 2017 Refresh", true, notifier, biochem, prescribing, demography, admissions);
var ec3 = CreateExtractionConfiguration(project, cohort, "Project 123 - 2018 Refresh", true, notifier, biochem, prescribing, demography, admissions);
ReleaseAllConfigurations(notifier, ec1, ec2, ec3);
}
private DiscoveredTable BuildExampleExtractionTable(DiscoveredDatabase db, string modality, int recordCount, bool useDcmFileExtension)
{
var tbl = db.CreateTable(modality + "_IsExtractable",
new[]
{
new DatabaseColumnRequest("StudyInstanceUID", new DatabaseTypeRequest(typeof(string), 64), false),
new DatabaseColumnRequest("SeriesInstanceUID", new DatabaseTypeRequest(typeof(string), 64), false),
new DatabaseColumnRequest("SOPInstanceUID", new DatabaseTypeRequest(typeof(string), 64), false)
{
IsPrimaryKey = true
},
new DatabaseColumnRequest("IsExtractableToDisk", new DatabaseTypeRequest(typeof(bool))),
new DatabaseColumnRequest("IsExtractableToDisk_Reason", new DatabaseTypeRequest(typeof(string), 512)),
new DatabaseColumnRequest("RelativeFileArchiveURI", new DatabaseTypeRequest(typeof(string), 512), false),
new DatabaseColumnRequest("IsOriginal", new DatabaseTypeRequest(typeof(bool)), false),
new DatabaseColumnRequest("IsPrimary", new DatabaseTypeRequest(typeof(bool)), false),
new DatabaseColumnRequest(SpecialFieldNames.DataLoadRunID, new DatabaseTypeRequest(typeof(int))),
new DatabaseColumnRequest(SpecialFieldNames.ValidFrom, new DatabaseTypeRequest(typeof(DateTime))),
});
if (recordCount > 0)
{
var r = new Random(500);
DicomDataGenerator g = new DicomDataGenerator(r, null);
g.MaximumImages = recordCount;
var persons = new PersonCollection();
persons.GeneratePeople(500, r);
while (recordCount > 0)
{
foreach (var image in g.GenerateStudyImages(persons.People[r.Next(persons.People.Length)], out var study))
{
tbl.Insert(new Dictionary <string, object>()
{
{ "StudyInstanceUID", image.GetSingleValue <string>(DicomTag.StudyInstanceUID) },
{ "SeriesInstanceUID", image.GetSingleValue <string>(DicomTag.SeriesInstanceUID) },
{ "SOPInstanceUID", image.GetSingleValue <string>(DicomTag.SOPInstanceUID) },
{ "IsExtractableToDisk", true },
{ "IsExtractableToDisk_Reason", DBNull.Value },
{ "RelativeFileArchiveURI", image.GetSingleValue <string>(DicomTag.SOPInstanceUID) + (useDcmFileExtension ? ".dcm" :"") },
{ "IsOriginal", image.GetValues <string>(DicomTag.ImageType)[0] == "ORIGINAL" },
{ "IsPrimary", image.GetValues <string>(DicomTag.ImageType)[1] == "PRIMARY" },
{ SpecialFieldNames.DataLoadRunID, 1 },
{ SpecialFieldNames.ValidFrom, DateTime.Now },
});
recordCount--;
if (recordCount <= 0)
{
break;
}
}
}
}
return(tbl);
}
//[TestCase(DatabaseType.Oracle)] //Oracle FAnsi doesn't currently support parameters
public void Test_SingleServer_WithTwoParameters(DatabaseType dbType, bool useCache)
{
/*
* Server1 Server2
* ____________________ _____________________
* |HospitalAdmissions | → | Cache |
* @date_of_max
* ____________________ ↗
* |HospitalAdmissions |
* @date_of_max
*
* (has different value so rename operations come into effect)
*/
var server1 = GetCleanedServer(dbType);
var server2 = GetCleanedServer(DatabaseType.MicrosoftSQLServer);
var cache = useCache ? CreateCache(server2): null;
var r = new Random(500);
var people = new PersonCollection();
people.GeneratePeople(5000, r);
var cic = new CohortIdentificationConfiguration(CatalogueRepository, "cic");
var ac1 = SetupAggregateConfigurationWithFilter(server1, people, r, cic, true, "@date_of_max", "'2001-01-01'");
var ac2 = SetupAggregateConfigurationWithFilter(server1, people, r, cic, true, "@date_of_max", "'2005-01-01'");
cic.CreateRootContainerIfNotExists();
cic.QueryCachingServer_ID = cache?.ID;
cic.SaveToDatabase();
var root = cic.RootCohortAggregateContainer;
root.AddChild(ac1, 0);
root.AddChild(ac2, 1);
var compiler = new CohortCompiler(cic);
var runner = new CohortCompilerRunner(compiler, 50000);
runner.Run(new CancellationToken());
AssertNoErrors(compiler);
//each container on it's own ran with the normal SQL
AssertNoErrors(compiler, ac1, "@date_of_max='2001-01-01'");
AssertNoErrors(compiler, ac2, "@date_of_max='2005-01-01'");
if (useCache)
{
//the root container run with dual cache fetch and no parameters
AssertNoErrors(compiler, root, "IndexedExtractionIdentifierList");
AssertCacheUsed(compiler, root, "2/2");
}
else
{
//the root container ran with a rename operations (no cache available)
AssertNoErrors(compiler, root, "@date_of_max='2001-01-01'", "@date_of_max_2='2005-01-01'");
AssertCacheUsed(compiler, root, "0/2");
}
}
public void Test_EXCEPT_TwoAggregates(DatabaseType dbType)
{
/*
* Server1
* _____________________
* |HospitalAdmissions x2|
* ↓ both run into ↓
* ___________________
* | Cache |
*
*
* HospitalAdmissions
* EXCEPT
* HospitalAdmissions (copy)
* = 0
*/
var db = GetCleanedServer(dbType);
var cache = CreateCache(db);
var r = new Random(500);
var people = new PersonCollection();
people.GeneratePeople(5000, r);
var cic = new CohortIdentificationConfiguration(CatalogueRepository, "cic");
var ac1 = SetupAggregateConfiguration(db, people, r, cic);
var ac2 = SetupAggregateConfiguration(db, people, r, cic);
cic.CreateRootContainerIfNotExists();
cic.QueryCachingServer_ID = cache.ID;
cic.SaveToDatabase();
var root = cic.RootCohortAggregateContainer;
root.Operation = SetOperation.EXCEPT;
root.Name = "EXCEPT";
root.SaveToDatabase();
root.AddChild(ac1, 0);
root.AddChild(ac2, 1);
var compiler = new CohortCompiler(cic);
var runner = new CohortCompilerRunner(compiler, 50000);
runner.Run(new CancellationToken());
if (dbType == DatabaseType.MySql)
{
var crashed = compiler.Tasks.Single(t => t.Key.State == CompilationState.Crashed);
StringAssert.Contains("INTERSECT / UNION / EXCEPT are not supported by MySql", crashed.Key.CrashMessage.Message);
return;
}
AssertNoErrors(compiler);
Assert.AreEqual(compiler.Tasks.Single(t => t.Value != null && t.Value.IsResultsForRootContainer).Key.FinalRowCount, 0);
Assert.Greater(compiler.Tasks.Single(t => t.Key is AggregationTask at && at.Aggregate.Equals(ac1)).Key.FinalRowCount, 0); //both ac should have the same total
Assert.Greater(compiler.Tasks.Single(t => t.Key is AggregationTask at && at.Aggregate.Equals(ac2)).Key.FinalRowCount, 0); // that is not 0
Assert.IsTrue(compiler.Tasks.Any(t => t.Key.GetCachedQueryUseCount().Equals("2/2")), "Expected EXCEPT container to use the cache");
}
public void Test_ZipFileNotation(bool expressRelative)
{
//get a clean database to upload to
var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer);
//create a folder in which to generate some dicoms
var dirToLoad = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFileNotation)));
if (dirToLoad.Exists)
{
dirToLoad.Delete(true);
}
dirToLoad.Create();
//generate some random dicoms
var r = new Random(999);
DicomDataGenerator generator = new DicomDataGenerator(r, dirToLoad, "CT")
{
MaximumImages = 5
};
var people = new PersonCollection();
people.GeneratePeople(1, r);
generator.GenerateTestDataFile(people, new FileInfo("./inventory.csv"), 1);
//This generates
// Test_ZipFile
// 2015
// 3
// 18
// 751140 2.25.166922918107154891877498685128076062226.dcm
// 751140 2.25.179610809676265137473873365625829826423.dcm
// 751140 2.25.201969634959506849065133495434871450465.dcm
// 751140 2.25.237492679533001779093365416814254319890.dcm
// 751140 2.25.316241631782653383510844072713132248731.dcm
var yearDir = dirToLoad.GetDirectories().Single();
StringAssert.IsMatch("\\d{4}", yearDir.Name);
//should be 5 images in the zip file
var dicomFiles = yearDir.GetFiles("*.dcm", SearchOption.AllDirectories);
Assert.AreEqual(5, dicomFiles.Length);
//e.g. \2015\3\18\2.25.223398837779449245317520567111874824918.dcm
//e.g. \2015\3\18\2.25.179610809676265137473873365625829826423.dcm
var relativePathWithinZip1 = dicomFiles[0].FullName.Substring(dirToLoad.FullName.Length);
var relativePathWithinZip2 = dicomFiles[1].FullName.Substring(dirToLoad.FullName.Length);
//zip them up
FileInfo zip = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip")); Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip");
if (zip.Exists)
{
zip.Delete();
}
ZipFile.CreateFromDirectory(dirToLoad.FullName, zip.FullName);
//e.g. E:\RdmpDicom\Rdmp.Dicom.Tests\bin\Debug\netcoreapp2.2\Test_ZipFile.zip!\2015\3\18\2.25.223398837779449245317520567111874824918.dcm
string pathToLoad1 = zip.FullName + "!" + relativePathWithinZip1;
string pathToLoad2 = zip.FullName + "!" + relativePathWithinZip2;
var loadMeTextFile = new FileInfo(Path.Combine(dirToLoad.FullName, "LoadMe.txt"));
//tell the source to load the zip
File.WriteAllText(loadMeTextFile.FullName, string.Join(Environment.NewLine, pathToLoad1, pathToLoad2));
var f = new FlatFileToLoad(loadMeTextFile);
//Setup source
var source = new DicomFileCollectionSource {
FilenameField = "RelativeFileArchiveURI"
};
if (expressRelative)
{
source.ArchiveRoot = TestContext.CurrentContext.TestDirectory;
}
var worklist = new FlatFileToLoadDicomFileWorklist(f);
//Setup destination
var destination = new DataTableUploadDestination {
AllowResizingColumnsAtUploadTime = true
};
//setup pipeline
var contextFactory = new DataFlowPipelineContextFactory <DataTable>();
var context = contextFactory.Create(PipelineUsage.FixedDestination | PipelineUsage.FixedDestination);
//run pipeline
var pipe = new DataFlowPipelineEngine <DataTable>(context, source, destination, new ThrowImmediatelyDataLoadEventListener());
pipe.Initialize(db, worklist);
pipe.ExecutePipeline(new GracefulCancellationToken());
var finalTable = db.ExpectTable(destination.TargetTableName);
using (var dt = finalTable.GetDataTable())
{
//should be 2 rows (since we told it to only load 2 files out of the zip)
Assert.AreEqual(2, dt.Rows.Count);
string pathInDbToDicomFile = (string)dt.Rows[0]["RelativeFileArchiveURI"];
//We expect either something like:
// E:/RdmpDicom/Rdmp.Dicom.Tests/bin/Debug/netcoreapp2.2/Test_ZipFile.zip!2015/3/18/2.25.160787663560951826149226183314694084702.dcm
// ./Test_ZipFile.zip!2015/3/18/2.25.105592977437473375573190160334447272386.dcm
//the path referenced should be the file read in relative/absolute format
StringAssert.IsMatch(
expressRelative ? $@"./{zip.Name}![\d./]*.dcm":
$@"{Regex.Escape(zip.FullName.Replace('\\','/'))}![\d./]*.dcm",
pathInDbToDicomFile);
StringAssert.Contains(yearDir.Name, pathInDbToDicomFile, "Expected zip file to have subdirectories and for them to be loaded correctly");
//confirm we can read that out again
using (var pool = new ZipPool())
{
var path = new AmbiguousFilePath(TestContext.CurrentContext.TestDirectory, pathInDbToDicomFile);
Assert.IsNotNull(path.GetDataset(pool));
}
}
Assert.IsTrue(finalTable.Exists());
finalTable.Drop();
}
public void Test_ZipFile(bool expressRelative)
{
//get a clean database to upload to
var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer);
//create a folder in which to generate some dicoms
var dirToLoad = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile)));
if (dirToLoad.Exists)
{
dirToLoad.Delete(true);
}
dirToLoad.Create();
//generate some random dicoms
var r = new Random(999);
DicomDataGenerator generator = new DicomDataGenerator(r, dirToLoad, "CT")
{
MaximumImages = 5
};
var people = new PersonCollection();
people.GeneratePeople(1, r);
generator.GenerateTestDataFile(people, new FileInfo("./inventory.csv"), 1);
//This generates
// Test_ZipFile
// 2015
// 3
// 18
// 751140 2.25.166922918107154891877498685128076062226.dcm
// 751140 2.25.179610809676265137473873365625829826423.dcm
// 751140 2.25.201969634959506849065133495434871450465.dcm
// 751140 2.25.237492679533001779093365416814254319890.dcm
// 751140 2.25.316241631782653383510844072713132248731.dcm
var yearDir = dirToLoad.GetDirectories().Single();
StringAssert.IsMatch("\\d{4}", yearDir.Name);
//zip them up
FileInfo zip = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip")); Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip");
if (zip.Exists)
{
zip.Delete();
}
ZipFile.CreateFromDirectory(dirToLoad.FullName, zip.FullName);
//tell the source to load the zip
var f = new FlatFileToLoad(zip);
var source = new DicomFileCollectionSource {
FilenameField = "RelativeFileArchiveURI"
};
if (expressRelative)
{
source.ArchiveRoot = TestContext.CurrentContext.TestDirectory;
}
source.PreInitialize(new FlatFileToLoadDicomFileWorklist(f), new ThrowImmediatelyDataLoadEventListener());
var tbl = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
var destination = new DataTableUploadDestination();
destination.PreInitialize(db, new ThrowImmediatelyDataLoadEventListener());
destination.AllowResizingColumnsAtUploadTime = true;
destination.ProcessPipelineData(tbl, new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
destination.Dispose(new ThrowImmediatelyDataLoadEventListener(), null);
var finalTable = db.ExpectTable(destination.TargetTableName);
using (var dt = finalTable.GetDataTable())
{
//should be 5 rows in the final table (5 images)
Assert.AreEqual(5, dt.Rows.Count);
string pathInDbToDicomFile = (string)dt.Rows[0]["RelativeFileArchiveURI"];
//We expect either something like:
// E:/RdmpDicom/Rdmp.Dicom.Tests/bin/Debug/netcoreapp2.2/Test_ZipFile.zip!2015/3/18/2.25.160787663560951826149226183314694084702.dcm
// ./Test_ZipFile.zip!2015/3/18/2.25.105592977437473375573190160334447272386.dcm
//the path referenced should be the file read in relative/absolute format
StringAssert.IsMatch(
expressRelative ? $@"./{zip.Name}![\d./]*.dcm":
$@"{Regex.Escape(zip.FullName.Replace('\\','/'))}![\d./]*.dcm",
pathInDbToDicomFile);
StringAssert.Contains(yearDir.Name, pathInDbToDicomFile, "Expected zip file to have subdirectories and for them to be loaded correctly");
//confirm we can read that out again
using (var pool = new ZipPool())
{
var path = new AmbiguousFilePath(TestContext.CurrentContext.TestDirectory, pathInDbToDicomFile);
Assert.IsNotNull(path.GetDataset(pool));
}
}
Assert.IsTrue(finalTable.Exists());
finalTable.Drop();
}
