public void TestGetChunktOnly(int numberOfImages)
{
Random r = new Random(123);
List <DicomDataset> allImages;
using (DicomDataGenerator g = new DicomDataGenerator(r, null, "CT", "MR"))
allImages = g.GenerateImages(numberOfImages, r);
DicomDatasetCollectionSource source = new DicomDatasetCollectionSource();
source.PreInitialize(new ExplicitListDicomDatasetWorklist(allImages.ToArray(), "amagad.dcm", new Dictionary <string, string>()
{
{ "MessageGuid", "0x123" }
}), new ThrowImmediatelyDataLoadEventListener());;
source.FilenameField = "gggg";
Stopwatch sw = new Stopwatch();
sw.Start();
var dt = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
source.Dispose(new ThrowImmediatelyDataLoadEventListener()
{
WriteToConsole = true
}, null);
sw.Stop();
Console.WriteLine("GetChunk took " + sw.ElapsedMilliseconds);
Assert.AreEqual(numberOfImages, dt.Rows.Count);
Assert.AreEqual(numberOfImages, dt.Rows.Cast <DataRow>().Select(w => w["SOPInstanceUID"]).Distinct().Count());
}
public void Test_CreatingOnDisk_OneFile()
{
var r = new Random(500);
var root = new DirectoryInfo(TestContext.CurrentContext.WorkDirectory);
var generator = new DicomDataGenerator(r, root)
{
Layout = FileSystemLayout.StudyUID, MaximumImages = 1
};
var person = new Person(r);
//generates a study but because of maximum images 1 we should only get 1 image being generated
string studyUid = (string)generator.GenerateTestDataRow(person)[0];
//should be a directory named after the Study UID
Assert.IsTrue(Directory.Exists(Path.Combine(root.FullName, studyUid)));
//should be a single file
var f = new FileInfo(Directory.GetFiles(Path.Combine(root.FullName, studyUid)).Single());
Assert.IsTrue(f.Exists);
var datasetCreated = DicomFile.Open(f.FullName);
Assert.AreEqual(studyUid,
datasetCreated.Dataset.GetValues <DicomUID>(DicomTag.StudyInstanceUID)[0].UID,
"UID in the dicom file generated did not match the one output into the CSV inventory file"
);
Console.WriteLine("Created file " + f.FullName);
generator.Dispose();
}
public void IntegrationTest_BumpyRide(DatabaseType databaseType)
{
var server = GetCleanedServer(databaseType, ScratchDatabaseName);
SetupSuite(server);
_globals.DicomRelationalMapperOptions.Guid = new Guid("6c7cfbce-1af6-4101-ade7-6537eea72e03");
_globals.DicomRelationalMapperOptions.QoSPrefetchCount = 5000;
_globals.IdentifierMapperOptions.QoSPrefetchCount = 50;
_globals.DicomTagReaderOptions.NackIfAnyFileErrors = false;
_helper.TruncateTablesIfExists();
//Create test directory
var dir = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(IntegrationTest_BumpyRide)));
var r = new Random(500);
//create a generator
using (var generator = new DicomDataGenerator(r, dir, "CT"))
{
generator.GenerateImageFiles(40, r);
RunTest(dir, 40);
}
}
public void TestLargeImageDatasets(DatabaseType databaseType, int numberOfImages)
{
foreach (Pipeline p in CatalogueRepository.GetAllObjects <Pipeline>())
{
p.DeleteInDatabase();
}
var db = GetCleanedServer(databaseType);
var d = CatalogueRepository.GetServerDefaults();
d.ClearDefault(PermissableDefaults.RAWDataLoadServer);
var template = ImageTableTemplateCollection.LoadFrom(_templateXml);
_globals = new GlobalOptionsFactory().Load();
_globals.DicomRelationalMapperOptions.DatabaseNamerType = typeof(MyFixedStagingDatabaseNamer).FullName;
_globals.DicomRelationalMapperOptions.QoSPrefetchCount = ushort.MaxValue;
_globals.DicomRelationalMapperOptions.MinimumBatchSize = numberOfImages;
_globals.DicomRelationalMapperOptions.UseInsertIntoForRAWMigration = true;
_helper = new DicomRelationalMapperTestHelper();
_helper.SetupSuite(db, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource), root: null, template: template, persistentRaw: true);
//do not use an explicit RAW data load server
d.ClearDefault(PermissableDefaults.RAWDataLoadServer);
Random r = new Random(123);
List <DicomDataset> allImages;
using (var generator = new DicomDataGenerator(r, null, "CT"))
allImages = generator.GenerateImages(numberOfImages, r);
Assert.AreEqual(numberOfImages, allImages.Count);
using (var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.DicomRelationalMapperOptions))
{
using (var host = new DicomRelationalMapperHost(_globals))
{
tester.SendMessages(_globals.DicomRelationalMapperOptions, allImages.Select(GetFileMessageForDataset), true);
Console.WriteLine("Starting Host");
host.Start();
Stopwatch sw = Stopwatch.StartNew();
new TestTimelineAwaiter().Await(() => host.Consumer.AckCount == numberOfImages, null, 20 * 60 * 100); //1 minute
Console.Write("Time For DLE:" + sw.Elapsed.TotalSeconds + "s");
host.Stop("Test finished");
}
}
foreach (Pipeline allObject in CatalogueRepository.GetAllObjects <Pipeline>())
{
allObject.DeleteInDatabase();
}
}
private DicomFileMessage GetTestDicomFileMessage(Test testCase = Test.Normal, int numberOfRandomTagsPerDicom = 0)
{
var msg = new DicomFileMessage()
{
DicomFilePath = "Path/To/The/File.dcm",
NationalPACSAccessionNumber = "1234",
SOPInstanceUID = "1.2.3.4",
SeriesInstanceUID = "1.2.3.4",
StudyInstanceUID = "1.2.3.4",
};
DicomDataset ds;
Random r = new Random(123);
using (var generator = new DicomDataGenerator(r, null, "CT"))
ds = generator.GenerateTestDataset(new Person(r), r);
ds.AddOrUpdate(DicomTag.AccessionNumber, "1234");
ds.AddOrUpdate(DicomTag.SOPInstanceUID, "1.2.3.4");
ds.AddOrUpdate(DicomTag.SeriesInstanceUID, "1.2.3.4");
ds.AddOrUpdate(DicomTag.StudyInstanceUID, "1.2.3.4");
switch (testCase)
{
case Test.Normal:
ds.AddOrUpdate(DicomTag.PatientID, "010101");
break;
case Test.NoPatientTag:
ds.Remove(DicomTag.PatientID);
break;
case Test.EmptyInPatientTag:
ds.AddOrUpdate(DicomTag.PatientID, string.Empty);
break;
case Test.ProperlyFormatedChi:
ds.AddOrUpdate(DicomTag.PatientID, "0101010101");
break;
default:
throw new ArgumentOutOfRangeException("testCase");
}
msg.DicomDataset = DicomTypeTranslater.SerializeDatasetToJson(ds);
return(msg);
}
public void Test_CreatingInMemory_ModalityCT()
{
var r = new Random(23);
var person = new Person(r);
var generator = new DicomDataGenerator(r, new DirectoryInfo(TestContext.CurrentContext.WorkDirectory), "CT");
//generate 100 images
for (int i = 0; i < 100; i++)
{
//all should be CT because we said CT only
var ds = generator.GenerateTestDataset(person);
Assert.AreEqual("CT", ds.GetSingleValue <string>(DicomTag.Modality));
}
}
public void Test_CreatingInMemory_Modality_CTAndMR()
{
var r = new Random(23);
var person = new Person(r);
var generator = new DicomDataGenerator(r, new DirectoryInfo(TestContext.CurrentContext.WorkDirectory), "CT", "MR");
//generate 100 images
for (int i = 0; i < 100; i++)
{
//all should be CT because we said CT only
var ds = generator.GenerateTestDataset(person);
var modality = ds.GetSingleValue <string>(DicomTag.Modality);
Assert.IsTrue(modality == "CT" || modality == "MR", "Unexpected modality {0}", modality);
}
}
public static IEnumerable <FileInfo> GenerateImageFiles(this DicomDataGenerator g, int numberOfImages, Random r)
{
var p = new PersonCollection();
p.GeneratePeople(5000, r);
if (g.OutputDir.Exists)
{
g.OutputDir.Delete(true);
}
var inventory = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "inventory.csv")); Path.Combine(TestContext.CurrentContext.WorkDirectory, "inventory.csv");
g.MaximumImages = numberOfImages;
g.GenerateTestDataFile(p, inventory, numberOfImages);
return(g.OutputDir.GetFiles("*.dcm", SearchOption.AllDirectories));
}
public static List <DicomDataset> GenerateImages(this DicomDataGenerator g, int numberOfImages, Random r)
{
var toReturn = new List <DicomDataset>();
g.MaximumImages = numberOfImages;
while (toReturn.Count <= numberOfImages)
{
toReturn.AddRange(g.GenerateStudyImages(new Person(r), out _));
}
//trim off extras
toReturn = toReturn.Take(numberOfImages).ToList();
Assert.AreEqual(numberOfImages, toReturn.Count);
return(toReturn);
}
public void Test_CsvOption()
{
var r = new Random(500);
var outputDir = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "TestCsv"));
outputDir.Create();
var people = new PersonCollection();
people.GeneratePeople(100, r);
using (var generator = new DicomDataGenerator(r, outputDir, "CT"))
{
generator.Csv = true;
generator.NoPixels = true;
generator.MaximumImages = 500;
generator.GenerateTestDataFile(people, new FileInfo(Path.Combine(outputDir.FullName, "index.csv")), 500);
}
//3 csv files + index.csv (the default one
Assert.AreEqual(4, outputDir.GetFiles().Length);
foreach (FileInfo f in outputDir.GetFiles())
{
using (var reader = new CsvReader(new StreamReader(f.FullName), CultureInfo.CurrentCulture))
{
int rowcount = 0;
//confirms that the CSV is intact (no dodgy commas, unquoted newlines etc)
while (reader.Read())
{
rowcount++;
}
//should be 1 row per image + 1 for header
if (f.Name == DicomDataGenerator.ImageCsvFilename)
{
Assert.AreEqual(501, rowcount);
}
}
}
}
private static void RunOptionsAndReturnExitCode(ProgramOptions opts)
{
if (opts.NumberOfPatients <= 0)
{
opts.NumberOfPatients = 500;
}
if (opts.NumberOfStudies <= 0)
{
opts.NumberOfStudies = 2000;
}
var dir = Directory.CreateDirectory(opts.OutputDirectory);
try
{
Random r = opts.Seed == -1 ? new Random() : new Random(opts.Seed);
//create a cohort of people
IPersonCollection identifiers = new PersonCollection();
identifiers.GeneratePeople(opts.NumberOfPatients, r);
//Generate the dicom files (of the modalities that the user requested)
string[] modalities = !string.IsNullOrWhiteSpace(opts.Modalities)? opts.Modalities.Split(",") :new string[0];
var dicomGenerator = new DicomDataGenerator(r, dir, modalities)
{
NoPixels = opts.NoPixels,
Layout = opts.Layout,
MaximumImages = opts.MaximumImages,
};
var targetFile = new FileInfo(Path.Combine(dir.FullName, "DicomFiles.csv"));
dicomGenerator.GenerateTestDataFile(identifiers, targetFile, opts.NumberOfStudies);
}
catch (Exception e)
{
Console.WriteLine(e);
returnCode = 2;
return;
}
returnCode = 0;
}
public void Test_CreatingNewStudy_HasSomeImages()
{
var r = new Random(100);
var generator = new DicomDataGenerator(r, null);
var p = new Person(r);
Study study = new Study(generator, p, new ModalityStats("MR", 2, 0, 50, 0, r), r);
Assert.AreEqual(2, study.Series.Count);
Assert.AreEqual(50, study.Series[0].Datasets.Count);
foreach (DicomDataset ds in study.Series[0])
{
Assert.AreEqual("MR", ds.GetValues <string>(DicomTag.Modality)[0]);
Assert.AreEqual(study.StudyTime, ds.GetSingleValue <DateTime>(DicomTag.StudyTime).TimeOfDay);
}
}
public void ExampleUsage()
{
//create a test person
var r = new Random(23);
var person = new Person(r);
//create a generator
var generator = new DicomDataGenerator(r, null, "CT");
//create a dataset in memory
DicomDataset dataset = generator.GenerateTestDataset(person);
//values should match the patient details
Assert.AreEqual(person.CHI, dataset.GetValue <string>(DicomTag.PatientID, 0));
Assert.GreaterOrEqual(dataset.GetValue <DateTime>(DicomTag.StudyDate, 0), person.DateOfBirth);
//should have a study description
Assert.IsNotNull(dataset.GetValue <string>(DicomTag.StudyDescription, 0));
//should have a study description
Assert.IsNotNull(dataset.GetSingleValue <DateTime>(DicomTag.StudyTime).TimeOfDay);
}
public void TestTagReader_SingleFileMode()
{
var dirRoot = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "TestTagReader_SingleFileMode"));
if (dirRoot.Exists)
{
dirRoot.Delete(true);
}
dirRoot.Create();
var julyFolder = dirRoot.CreateSubdirectory("July");
_helper.Options.FileSystemOptions.FileSystemRoot = dirRoot.FullName;
new MicroserviceTester(_helper.Options.RabbitOptions, _helper.AccessionConsumerOptions);
var host = new DicomTagReaderHost(_helper.Options);
var r = new Random(5);
var generator = new DicomDataGenerator(r, julyFolder, "CT");
var files = generator.GenerateImageFiles(10, r).ToArray();
host.AccessionDirectoryMessageConsumer.RunSingleFile(files[2]);
Assert.AreEqual(1, _helper.ImageCount);
Assert.AreEqual(1, _helper.SeriesCount);
var julyZip = Path.Combine(dirRoot.FullName, "july.zip");
ZipFile.CreateFromDirectory(julyFolder.FullName, julyZip);
host.AccessionDirectoryMessageConsumer.RunSingleFile(new FileInfo(julyZip));
Assert.AreEqual(11, _helper.ImageCount);
Assert.GreaterOrEqual(_helper.SeriesCount, 1);
}
public void Test_Anonymise()
{
var r = new Random(23);
var person = new Person(r);
var generator = new DicomDataGenerator(r, new DirectoryInfo(TestContext.CurrentContext.WorkDirectory), "CT");
// without anonymisation (default) we get the normal patient ID
var ds = generator.GenerateTestDataset(person, r);
Assert.IsTrue(ds.Contains(DicomTag.PatientID));
Assert.AreEqual(person.CHI, ds.GetValue <string>(DicomTag.PatientID, 0));
// with anonymisation
generator.Anonymise = true;
var ds2 = generator.GenerateTestDataset(person, r);
// we get a blank patient ID
Assert.IsTrue(ds2.Contains(DicomTag.PatientID));
Assert.AreEqual(string.Empty, ds2.GetString(DicomTag.PatientID));
generator.Dispose();
}
public void Test_ZipFileNotation(bool expressRelative)
{
//get a clean database to upload to
var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer);
//create a folder in which to generate some dicoms
var dirToLoad = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFileNotation)));
if (dirToLoad.Exists)
{
dirToLoad.Delete(true);
}
dirToLoad.Create();
//generate some random dicoms
var r = new Random(999);
DicomDataGenerator generator = new DicomDataGenerator(r, dirToLoad, "CT")
{
MaximumImages = 5
};
var people = new PersonCollection();
people.GeneratePeople(1, r);
generator.GenerateTestDataFile(people, new FileInfo("./inventory.csv"), 1);
//This generates
// Test_ZipFile
// 2015
// 3
// 18
// 751140 2.25.166922918107154891877498685128076062226.dcm
// 751140 2.25.179610809676265137473873365625829826423.dcm
// 751140 2.25.201969634959506849065133495434871450465.dcm
// 751140 2.25.237492679533001779093365416814254319890.dcm
// 751140 2.25.316241631782653383510844072713132248731.dcm
var yearDir = dirToLoad.GetDirectories().Single();
StringAssert.IsMatch("\\d{4}", yearDir.Name);
//should be 5 images in the zip file
var dicomFiles = yearDir.GetFiles("*.dcm", SearchOption.AllDirectories);
Assert.AreEqual(5, dicomFiles.Length);
//e.g. \2015\3\18\2.25.223398837779449245317520567111874824918.dcm
//e.g. \2015\3\18\2.25.179610809676265137473873365625829826423.dcm
var relativePathWithinZip1 = dicomFiles[0].FullName.Substring(dirToLoad.FullName.Length);
var relativePathWithinZip2 = dicomFiles[1].FullName.Substring(dirToLoad.FullName.Length);
//zip them up
FileInfo zip = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip")); Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip");
if (zip.Exists)
{
zip.Delete();
}
ZipFile.CreateFromDirectory(dirToLoad.FullName, zip.FullName);
//e.g. E:\RdmpDicom\Rdmp.Dicom.Tests\bin\Debug\netcoreapp2.2\Test_ZipFile.zip!\2015\3\18\2.25.223398837779449245317520567111874824918.dcm
string pathToLoad1 = zip.FullName + "!" + relativePathWithinZip1;
string pathToLoad2 = zip.FullName + "!" + relativePathWithinZip2;
var loadMeTextFile = new FileInfo(Path.Combine(dirToLoad.FullName, "LoadMe.txt"));
//tell the source to load the zip
File.WriteAllText(loadMeTextFile.FullName, string.Join(Environment.NewLine, pathToLoad1, pathToLoad2));
var f = new FlatFileToLoad(loadMeTextFile);
//Setup source
var source = new DicomFileCollectionSource {
FilenameField = "RelativeFileArchiveURI"
};
if (expressRelative)
{
source.ArchiveRoot = TestContext.CurrentContext.TestDirectory;
}
var worklist = new FlatFileToLoadDicomFileWorklist(f);
//Setup destination
var destination = new DataTableUploadDestination {
AllowResizingColumnsAtUploadTime = true
};
//setup pipeline
var contextFactory = new DataFlowPipelineContextFactory <DataTable>();
var context = contextFactory.Create(PipelineUsage.FixedDestination | PipelineUsage.FixedDestination);
//run pipeline
var pipe = new DataFlowPipelineEngine <DataTable>(context, source, destination, new ThrowImmediatelyDataLoadEventListener());
pipe.Initialize(db, worklist);
pipe.ExecutePipeline(new GracefulCancellationToken());
var finalTable = db.ExpectTable(destination.TargetTableName);
using (var dt = finalTable.GetDataTable())
{
//should be 2 rows (since we told it to only load 2 files out of the zip)
Assert.AreEqual(2, dt.Rows.Count);
string pathInDbToDicomFile = (string)dt.Rows[0]["RelativeFileArchiveURI"];
//We expect either something like:
// E:/RdmpDicom/Rdmp.Dicom.Tests/bin/Debug/netcoreapp2.2/Test_ZipFile.zip!2015/3/18/2.25.160787663560951826149226183314694084702.dcm
// ./Test_ZipFile.zip!2015/3/18/2.25.105592977437473375573190160334447272386.dcm
//the path referenced should be the file read in relative/absolute format
StringAssert.IsMatch(
expressRelative ? $@"./{zip.Name}![\d./]*.dcm":
$@"{Regex.Escape(zip.FullName.Replace('\\','/'))}![\d./]*.dcm",
pathInDbToDicomFile);
StringAssert.Contains(yearDir.Name, pathInDbToDicomFile, "Expected zip file to have subdirectories and for them to be loaded correctly");
//confirm we can read that out again
using (var pool = new ZipPool())
{
var path = new AmbiguousFilePath(TestContext.CurrentContext.TestDirectory, pathInDbToDicomFile);
Assert.IsNotNull(path.GetDataset(pool));
}
}
Assert.IsTrue(finalTable.Exists());
finalTable.Drop();
}
public void Test_ZipFile(bool expressRelative)
{
//get a clean database to upload to
var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer);
//create a folder in which to generate some dicoms
var dirToLoad = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile)));
if (dirToLoad.Exists)
{
dirToLoad.Delete(true);
}
dirToLoad.Create();
//generate some random dicoms
var r = new Random(999);
DicomDataGenerator generator = new DicomDataGenerator(r, dirToLoad, "CT")
{
MaximumImages = 5
};
var people = new PersonCollection();
people.GeneratePeople(1, r);
generator.GenerateTestDataFile(people, new FileInfo("./inventory.csv"), 1);
//This generates
// Test_ZipFile
// 2015
// 3
// 18
// 751140 2.25.166922918107154891877498685128076062226.dcm
// 751140 2.25.179610809676265137473873365625829826423.dcm
// 751140 2.25.201969634959506849065133495434871450465.dcm
// 751140 2.25.237492679533001779093365416814254319890.dcm
// 751140 2.25.316241631782653383510844072713132248731.dcm
var yearDir = dirToLoad.GetDirectories().Single();
StringAssert.IsMatch("\\d{4}", yearDir.Name);
//zip them up
FileInfo zip = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip")); Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip");
if (zip.Exists)
{
zip.Delete();
}
ZipFile.CreateFromDirectory(dirToLoad.FullName, zip.FullName);
//tell the source to load the zip
var f = new FlatFileToLoad(zip);
var source = new DicomFileCollectionSource {
FilenameField = "RelativeFileArchiveURI"
};
if (expressRelative)
{
source.ArchiveRoot = TestContext.CurrentContext.TestDirectory;
}
source.PreInitialize(new FlatFileToLoadDicomFileWorklist(f), new ThrowImmediatelyDataLoadEventListener());
var tbl = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
var destination = new DataTableUploadDestination();
destination.PreInitialize(db, new ThrowImmediatelyDataLoadEventListener());
destination.AllowResizingColumnsAtUploadTime = true;
destination.ProcessPipelineData(tbl, new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
destination.Dispose(new ThrowImmediatelyDataLoadEventListener(), null);
var finalTable = db.ExpectTable(destination.TargetTableName);
using (var dt = finalTable.GetDataTable())
{
//should be 5 rows in the final table (5 images)
Assert.AreEqual(5, dt.Rows.Count);
string pathInDbToDicomFile = (string)dt.Rows[0]["RelativeFileArchiveURI"];
//We expect either something like:
// E:/RdmpDicom/Rdmp.Dicom.Tests/bin/Debug/netcoreapp2.2/Test_ZipFile.zip!2015/3/18/2.25.160787663560951826149226183314694084702.dcm
// ./Test_ZipFile.zip!2015/3/18/2.25.105592977437473375573190160334447272386.dcm
//the path referenced should be the file read in relative/absolute format
StringAssert.IsMatch(
expressRelative ? $@"./{zip.Name}![\d./]*.dcm":
$@"{Regex.Escape(zip.FullName.Replace('\\','/'))}![\d./]*.dcm",
pathInDbToDicomFile);
StringAssert.Contains(yearDir.Name, pathInDbToDicomFile, "Expected zip file to have subdirectories and for them to be loaded correctly");
//confirm we can read that out again
using (var pool = new ZipPool())
{
var path = new AmbiguousFilePath(TestContext.CurrentContext.TestDirectory, pathInDbToDicomFile);
Assert.IsNotNull(path.GetDataset(pool));
}
}
Assert.IsTrue(finalTable.Exists());
finalTable.Drop();
}
public void TestBulkInsertOnly(DatabaseType databaseType, int numberOfImages)
{
foreach (Pipeline p in CatalogueRepository.GetAllObjects <Pipeline>())
{
p.DeleteInDatabase();
}
var db = GetCleanedServer(databaseType);
var template = ImageTableTemplateCollection.LoadFrom(_templateXml);
_globals = new GlobalOptionsFactory().Load();
_globals.DicomRelationalMapperOptions.DatabaseNamerType = typeof(MyFixedStagingDatabaseNamer).FullName;
_globals.DicomRelationalMapperOptions.QoSPrefetchCount = ushort.MaxValue;
_globals.DicomRelationalMapperOptions.MinimumBatchSize = numberOfImages;
_globals.DicomRelationalMapperOptions.UseInsertIntoForRAWMigration = true;
_helper = new DicomRelationalMapperTestHelper();
_helper.SetupSuite(db, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource), root: null, template: template, persistentRaw: true);
//do not use an explicit RAW data load server
CatalogueRepository.GetServerDefaults().ClearDefault(PermissableDefaults.RAWDataLoadServer);
Random r = new Random(123);
List <DicomDataset> allImages;
using (var generator = new DicomDataGenerator(r, null, "CT"))
allImages = generator.GenerateImages(numberOfImages, r);
DicomDatasetCollectionSource source = new DicomDatasetCollectionSource();
source.PreInitialize(new ExplicitListDicomDatasetWorklist(allImages.ToArray(), "amagad.dcm", new Dictionary <string, string>()
{
{ "MessageGuid", "0x123" }
}), new ThrowImmediatelyDataLoadEventListener());;
source.FilenameField = "gggg";
var dt = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
source.Dispose(new ThrowImmediatelyDataLoadEventListener()
{
WriteToConsole = true
}, null);
Assert.AreEqual(numberOfImages, allImages.Count);
Assert.AreEqual(numberOfImages, dt.Rows.Count);
var tables = _helper.LoadMetadata.GetDistinctTableInfoList(false);
var config = new HICDatabaseConfiguration(_helper.LoadMetadata, new SuffixBasedNamer());
var job = Mock.Of <IDataLoadJob>(
j => j.RegularTablesToLoad == tables.Cast <ITableInfo>().ToList() &&
j.DataLoadInfo == _dli &&
j.Configuration == config);
var attacher = new AutoRoutingAttacher();
attacher.Job = job;
//Drop Primary Keys
using (var con = db.Server.GetConnection())
{
con.Open();
var cmd = db.Server.GetCommand(
databaseType == DatabaseType.MicrosoftSQLServer ?
@"ALTER TABLE ImageTable DROP CONSTRAINT PK_ImageTable
ALTER TABLE SeriesTable DROP CONSTRAINT PK_SeriesTable
ALTER TABLE StudyTable DROP CONSTRAINT PK_StudyTable" :
@"ALTER TABLE ImageTable DROP PRIMARY KEY;
ALTER TABLE SeriesTable DROP PRIMARY KEY;
ALTER TABLE StudyTable DROP PRIMARY KEY;"
, con);
cmd.ExecuteNonQuery();
}
attacher.Initialize(LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "IgnoreMe", true), db);
try
{
attacher.ProcessPipelineData(dt, new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
attacher.Dispose(new ThrowImmediatelyDataLoadEventListener(), null);
}
catch (Exception e)
{
attacher.Dispose(new ThrowImmediatelyDataLoadEventListener(), e);
throw;
}
foreach (var tableInfo in tables)
{
Assert.AreEqual(numberOfImages,
tableInfo.Discover(DataAccessContext.InternalDataProcessing).GetRowCount(),
"Row count was wrong for " + tableInfo);
}
foreach (Pipeline allObject in CatalogueRepository.GetAllObjects <Pipeline>())
{
allObject.DeleteInDatabase();
}
}
private DiscoveredTable BuildExampleExtractionTable(DiscoveredDatabase db, string modality, int recordCount, bool useDcmFileExtension)
{
var tbl = db.CreateTable(modality + "_IsExtractable",
new[]
{
new DatabaseColumnRequest("StudyInstanceUID", new DatabaseTypeRequest(typeof(string), 64), false),
new DatabaseColumnRequest("SeriesInstanceUID", new DatabaseTypeRequest(typeof(string), 64), false),
new DatabaseColumnRequest("SOPInstanceUID", new DatabaseTypeRequest(typeof(string), 64), false)
{
IsPrimaryKey = true
},
new DatabaseColumnRequest("IsExtractableToDisk", new DatabaseTypeRequest(typeof(bool))),
new DatabaseColumnRequest("IsExtractableToDisk_Reason", new DatabaseTypeRequest(typeof(string), 512)),
new DatabaseColumnRequest("RelativeFileArchiveURI", new DatabaseTypeRequest(typeof(string), 512), false),
new DatabaseColumnRequest("IsOriginal", new DatabaseTypeRequest(typeof(bool)), false),
new DatabaseColumnRequest("IsPrimary", new DatabaseTypeRequest(typeof(bool)), false),
new DatabaseColumnRequest(SpecialFieldNames.DataLoadRunID, new DatabaseTypeRequest(typeof(int))),
new DatabaseColumnRequest(SpecialFieldNames.ValidFrom, new DatabaseTypeRequest(typeof(DateTime))),
});
if (recordCount > 0)
{
var r = new Random(500);
DicomDataGenerator g = new DicomDataGenerator(r, null);
g.MaximumImages = recordCount;
var persons = new PersonCollection();
persons.GeneratePeople(500, r);
while (recordCount > 0)
{
foreach (var image in g.GenerateStudyImages(persons.People[r.Next(persons.People.Length)], out var study))
{
tbl.Insert(new Dictionary <string, object>()
{
{ "StudyInstanceUID", image.GetSingleValue <string>(DicomTag.StudyInstanceUID) },
{ "SeriesInstanceUID", image.GetSingleValue <string>(DicomTag.SeriesInstanceUID) },
{ "SOPInstanceUID", image.GetSingleValue <string>(DicomTag.SOPInstanceUID) },
{ "IsExtractableToDisk", true },
{ "IsExtractableToDisk_Reason", DBNull.Value },
{ "RelativeFileArchiveURI", image.GetSingleValue <string>(DicomTag.SOPInstanceUID) + (useDcmFileExtension ? ".dcm" :"") },
{ "IsOriginal", image.GetValues <string>(DicomTag.ImageType)[0] == "ORIGINAL" },
{ "IsPrimary", image.GetValues <string>(DicomTag.ImageType)[1] == "PRIMARY" },
{ SpecialFieldNames.DataLoadRunID, 1 },
{ SpecialFieldNames.ValidFrom, DateTime.Now },
});
recordCount--;
if (recordCount <= 0)
{
break;
}
}
}
}
return(tbl);
}
public void TestLoadingOneImage_MileWideTest()
{
_helper.TruncateTablesIfExists();
DirectoryInfo d = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(TestLoadingOneImage_MileWideTest)));
d.Create();
var r = new Random(5000);
FileInfo[] files;
using (var g = new DicomDataGenerator(r, d, "CT"))
files = g.GenerateImageFiles(1, r).ToArray();
Assert.AreEqual(1, files.Length);
var existingColumns = _helper.ImageTable.DiscoverColumns();
//Add 200 random tags
foreach (string tag in TagColumnAdder.GetAvailableTags().OrderBy(a => r.Next()).Take(200))
{
string dataType;
try
{
dataType = TagColumnAdder.GetDataTypeForTag(tag, new MicrosoftSQLTypeTranslater());
}
catch (Exception)
{
continue;
}
if (existingColumns.Any(c => c.GetRuntimeName().Equals(tag)))
{
continue;
}
var adder = new TagColumnAdder(tag, dataType, _helper.ImageTableInfo, new AcceptAllCheckNotifier());
adder.SkipChecksAndSynchronization = true;
adder.Execute();
}
new TableInfoSynchronizer(_helper.ImageTableInfo).Synchronize(new AcceptAllCheckNotifier());
//creates the queues, exchanges and bindings
var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.DicomRelationalMapperOptions);
tester.CreateExchange(_globals.RabbitOptions.FatalLoggingExchange, null);
using (var host = new DicomRelationalMapperHost(_globals))
{
host.Start();
using (var timeline = new TestTimeline(tester))
{
foreach (var f in files)
{
timeline.SendMessage(_globals.DicomRelationalMapperOptions,
_helper.GetDicomFileMessage(_globals.FileSystemOptions.FileSystemRoot, f));
}
//start the timeline
timeline.StartTimeline();
new TestTimelineAwaiter().Await(() => host.Consumer.MessagesProcessed == 1, null, 30000, () => host.Consumer.DleErrors);
Assert.GreaterOrEqual(1, _helper.SeriesTable.GetRowCount(), "SeriesTable did not have the expected number of rows in LIVE");
Assert.GreaterOrEqual(1, _helper.StudyTable.GetRowCount(), "StudyTable did not have the expected number of rows in LIVE");
Assert.AreEqual(1, _helper.ImageTable.GetRowCount(), "ImageTable did not have the expected number of rows in LIVE");
host.Stop("Test end");
}
}
tester.Shutdown();
}
