public void TestGetChunktOnly(int numberOfImages) { Random r = new Random(123); List <DicomDataset> allImages; using (DicomDataGenerator g = new DicomDataGenerator(r, null, "CT", "MR")) allImages = g.GenerateImages(numberOfImages, r); DicomDatasetCollectionSource source = new DicomDatasetCollectionSource(); source.PreInitialize(new ExplicitListDicomDatasetWorklist(allImages.ToArray(), "amagad.dcm", new Dictionary <string, string>() { { "MessageGuid", "0x123" } }), new ThrowImmediatelyDataLoadEventListener());; source.FilenameField = "gggg"; Stopwatch sw = new Stopwatch(); sw.Start(); var dt = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); source.Dispose(new ThrowImmediatelyDataLoadEventListener() { WriteToConsole = true }, null); sw.Stop(); Console.WriteLine("GetChunk took " + sw.ElapsedMilliseconds); Assert.AreEqual(numberOfImages, dt.Rows.Count); Assert.AreEqual(numberOfImages, dt.Rows.Cast <DataRow>().Select(w => w["SOPInstanceUID"]).Distinct().Count()); }
public void Test_CreatingOnDisk_OneFile() { var r = new Random(500); var root = new DirectoryInfo(TestContext.CurrentContext.WorkDirectory); var generator = new DicomDataGenerator(r, root) { Layout = FileSystemLayout.StudyUID, MaximumImages = 1 }; var person = new Person(r); //generates a study but because of maximum images 1 we should only get 1 image being generated string studyUid = (string)generator.GenerateTestDataRow(person)[0]; //should be a directory named after the Study UID Assert.IsTrue(Directory.Exists(Path.Combine(root.FullName, studyUid))); //should be a single file var f = new FileInfo(Directory.GetFiles(Path.Combine(root.FullName, studyUid)).Single()); Assert.IsTrue(f.Exists); var datasetCreated = DicomFile.Open(f.FullName); Assert.AreEqual(studyUid, datasetCreated.Dataset.GetValues <DicomUID>(DicomTag.StudyInstanceUID)[0].UID, "UID in the dicom file generated did not match the one output into the CSV inventory file" ); Console.WriteLine("Created file " + f.FullName); generator.Dispose(); }
public void IntegrationTest_BumpyRide(DatabaseType databaseType) { var server = GetCleanedServer(databaseType, ScratchDatabaseName); SetupSuite(server); _globals.DicomRelationalMapperOptions.Guid = new Guid("6c7cfbce-1af6-4101-ade7-6537eea72e03"); _globals.DicomRelationalMapperOptions.QoSPrefetchCount = 5000; _globals.IdentifierMapperOptions.QoSPrefetchCount = 50; _globals.DicomTagReaderOptions.NackIfAnyFileErrors = false; _helper.TruncateTablesIfExists(); //Create test directory var dir = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(IntegrationTest_BumpyRide))); var r = new Random(500); //create a generator using (var generator = new DicomDataGenerator(r, dir, "CT")) { generator.GenerateImageFiles(40, r); RunTest(dir, 40); } }
public void TestLargeImageDatasets(DatabaseType databaseType, int numberOfImages) { foreach (Pipeline p in CatalogueRepository.GetAllObjects <Pipeline>()) { p.DeleteInDatabase(); } var db = GetCleanedServer(databaseType); var d = CatalogueRepository.GetServerDefaults(); d.ClearDefault(PermissableDefaults.RAWDataLoadServer); var template = ImageTableTemplateCollection.LoadFrom(_templateXml); _globals = new GlobalOptionsFactory().Load(); _globals.DicomRelationalMapperOptions.DatabaseNamerType = typeof(MyFixedStagingDatabaseNamer).FullName; _globals.DicomRelationalMapperOptions.QoSPrefetchCount = ushort.MaxValue; _globals.DicomRelationalMapperOptions.MinimumBatchSize = numberOfImages; _globals.DicomRelationalMapperOptions.UseInsertIntoForRAWMigration = true; _helper = new DicomRelationalMapperTestHelper(); _helper.SetupSuite(db, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource), root: null, template: template, persistentRaw: true); //do not use an explicit RAW data load server d.ClearDefault(PermissableDefaults.RAWDataLoadServer); Random r = new Random(123); List <DicomDataset> allImages; using (var generator = new DicomDataGenerator(r, null, "CT")) allImages = generator.GenerateImages(numberOfImages, r); Assert.AreEqual(numberOfImages, allImages.Count); using (var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.DicomRelationalMapperOptions)) { using (var host = new DicomRelationalMapperHost(_globals)) { tester.SendMessages(_globals.DicomRelationalMapperOptions, allImages.Select(GetFileMessageForDataset), true); Console.WriteLine("Starting Host"); host.Start(); Stopwatch sw = Stopwatch.StartNew(); new TestTimelineAwaiter().Await(() => host.Consumer.AckCount == numberOfImages, null, 20 * 60 * 100); //1 minute Console.Write("Time For DLE:" + sw.Elapsed.TotalSeconds + "s"); host.Stop("Test finished"); } } foreach (Pipeline allObject in CatalogueRepository.GetAllObjects <Pipeline>()) { allObject.DeleteInDatabase(); } }
private DicomFileMessage GetTestDicomFileMessage(Test testCase = Test.Normal, int numberOfRandomTagsPerDicom = 0) { var msg = new DicomFileMessage() { DicomFilePath = "Path/To/The/File.dcm", NationalPACSAccessionNumber = "1234", SOPInstanceUID = "1.2.3.4", SeriesInstanceUID = "1.2.3.4", StudyInstanceUID = "1.2.3.4", }; DicomDataset ds; Random r = new Random(123); using (var generator = new DicomDataGenerator(r, null, "CT")) ds = generator.GenerateTestDataset(new Person(r), r); ds.AddOrUpdate(DicomTag.AccessionNumber, "1234"); ds.AddOrUpdate(DicomTag.SOPInstanceUID, "1.2.3.4"); ds.AddOrUpdate(DicomTag.SeriesInstanceUID, "1.2.3.4"); ds.AddOrUpdate(DicomTag.StudyInstanceUID, "1.2.3.4"); switch (testCase) { case Test.Normal: ds.AddOrUpdate(DicomTag.PatientID, "010101"); break; case Test.NoPatientTag: ds.Remove(DicomTag.PatientID); break; case Test.EmptyInPatientTag: ds.AddOrUpdate(DicomTag.PatientID, string.Empty); break; case Test.ProperlyFormatedChi: ds.AddOrUpdate(DicomTag.PatientID, "0101010101"); break; default: throw new ArgumentOutOfRangeException("testCase"); } msg.DicomDataset = DicomTypeTranslater.SerializeDatasetToJson(ds); return(msg); }
public void Test_CreatingInMemory_ModalityCT() { var r = new Random(23); var person = new Person(r); var generator = new DicomDataGenerator(r, new DirectoryInfo(TestContext.CurrentContext.WorkDirectory), "CT"); //generate 100 images for (int i = 0; i < 100; i++) { //all should be CT because we said CT only var ds = generator.GenerateTestDataset(person); Assert.AreEqual("CT", ds.GetSingleValue <string>(DicomTag.Modality)); } }
public void Test_CreatingInMemory_Modality_CTAndMR() { var r = new Random(23); var person = new Person(r); var generator = new DicomDataGenerator(r, new DirectoryInfo(TestContext.CurrentContext.WorkDirectory), "CT", "MR"); //generate 100 images for (int i = 0; i < 100; i++) { //all should be CT because we said CT only var ds = generator.GenerateTestDataset(person); var modality = ds.GetSingleValue <string>(DicomTag.Modality); Assert.IsTrue(modality == "CT" || modality == "MR", "Unexpected modality {0}", modality); } }
public static IEnumerable <FileInfo> GenerateImageFiles(this DicomDataGenerator g, int numberOfImages, Random r) { var p = new PersonCollection(); p.GeneratePeople(5000, r); if (g.OutputDir.Exists) { g.OutputDir.Delete(true); } var inventory = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "inventory.csv")); Path.Combine(TestContext.CurrentContext.WorkDirectory, "inventory.csv"); g.MaximumImages = numberOfImages; g.GenerateTestDataFile(p, inventory, numberOfImages); return(g.OutputDir.GetFiles("*.dcm", SearchOption.AllDirectories)); }
public static List <DicomDataset> GenerateImages(this DicomDataGenerator g, int numberOfImages, Random r) { var toReturn = new List <DicomDataset>(); g.MaximumImages = numberOfImages; while (toReturn.Count <= numberOfImages) { toReturn.AddRange(g.GenerateStudyImages(new Person(r), out _)); } //trim off extras toReturn = toReturn.Take(numberOfImages).ToList(); Assert.AreEqual(numberOfImages, toReturn.Count); return(toReturn); }
public void Test_CsvOption() { var r = new Random(500); var outputDir = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "TestCsv")); outputDir.Create(); var people = new PersonCollection(); people.GeneratePeople(100, r); using (var generator = new DicomDataGenerator(r, outputDir, "CT")) { generator.Csv = true; generator.NoPixels = true; generator.MaximumImages = 500; generator.GenerateTestDataFile(people, new FileInfo(Path.Combine(outputDir.FullName, "index.csv")), 500); } //3 csv files + index.csv (the default one Assert.AreEqual(4, outputDir.GetFiles().Length); foreach (FileInfo f in outputDir.GetFiles()) { using (var reader = new CsvReader(new StreamReader(f.FullName), CultureInfo.CurrentCulture)) { int rowcount = 0; //confirms that the CSV is intact (no dodgy commas, unquoted newlines etc) while (reader.Read()) { rowcount++; } //should be 1 row per image + 1 for header if (f.Name == DicomDataGenerator.ImageCsvFilename) { Assert.AreEqual(501, rowcount); } } } }
private static void RunOptionsAndReturnExitCode(ProgramOptions opts) { if (opts.NumberOfPatients <= 0) { opts.NumberOfPatients = 500; } if (opts.NumberOfStudies <= 0) { opts.NumberOfStudies = 2000; } var dir = Directory.CreateDirectory(opts.OutputDirectory); try { Random r = opts.Seed == -1 ? new Random() : new Random(opts.Seed); //create a cohort of people IPersonCollection identifiers = new PersonCollection(); identifiers.GeneratePeople(opts.NumberOfPatients, r); //Generate the dicom files (of the modalities that the user requested) string[] modalities = !string.IsNullOrWhiteSpace(opts.Modalities)? opts.Modalities.Split(",") :new string[0]; var dicomGenerator = new DicomDataGenerator(r, dir, modalities) { NoPixels = opts.NoPixels, Layout = opts.Layout, MaximumImages = opts.MaximumImages, }; var targetFile = new FileInfo(Path.Combine(dir.FullName, "DicomFiles.csv")); dicomGenerator.GenerateTestDataFile(identifiers, targetFile, opts.NumberOfStudies); } catch (Exception e) { Console.WriteLine(e); returnCode = 2; return; } returnCode = 0; }
public void Test_CreatingNewStudy_HasSomeImages() { var r = new Random(100); var generator = new DicomDataGenerator(r, null); var p = new Person(r); Study study = new Study(generator, p, new ModalityStats("MR", 2, 0, 50, 0, r), r); Assert.AreEqual(2, study.Series.Count); Assert.AreEqual(50, study.Series[0].Datasets.Count); foreach (DicomDataset ds in study.Series[0]) { Assert.AreEqual("MR", ds.GetValues <string>(DicomTag.Modality)[0]); Assert.AreEqual(study.StudyTime, ds.GetSingleValue <DateTime>(DicomTag.StudyTime).TimeOfDay); } }
public void ExampleUsage() { //create a test person var r = new Random(23); var person = new Person(r); //create a generator var generator = new DicomDataGenerator(r, null, "CT"); //create a dataset in memory DicomDataset dataset = generator.GenerateTestDataset(person); //values should match the patient details Assert.AreEqual(person.CHI, dataset.GetValue <string>(DicomTag.PatientID, 0)); Assert.GreaterOrEqual(dataset.GetValue <DateTime>(DicomTag.StudyDate, 0), person.DateOfBirth); //should have a study description Assert.IsNotNull(dataset.GetValue <string>(DicomTag.StudyDescription, 0)); //should have a study description Assert.IsNotNull(dataset.GetSingleValue <DateTime>(DicomTag.StudyTime).TimeOfDay); }
public void TestTagReader_SingleFileMode() { var dirRoot = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "TestTagReader_SingleFileMode")); if (dirRoot.Exists) { dirRoot.Delete(true); } dirRoot.Create(); var julyFolder = dirRoot.CreateSubdirectory("July"); _helper.Options.FileSystemOptions.FileSystemRoot = dirRoot.FullName; new MicroserviceTester(_helper.Options.RabbitOptions, _helper.AccessionConsumerOptions); var host = new DicomTagReaderHost(_helper.Options); var r = new Random(5); var generator = new DicomDataGenerator(r, julyFolder, "CT"); var files = generator.GenerateImageFiles(10, r).ToArray(); host.AccessionDirectoryMessageConsumer.RunSingleFile(files[2]); Assert.AreEqual(1, _helper.ImageCount); Assert.AreEqual(1, _helper.SeriesCount); var julyZip = Path.Combine(dirRoot.FullName, "july.zip"); ZipFile.CreateFromDirectory(julyFolder.FullName, julyZip); host.AccessionDirectoryMessageConsumer.RunSingleFile(new FileInfo(julyZip)); Assert.AreEqual(11, _helper.ImageCount); Assert.GreaterOrEqual(_helper.SeriesCount, 1); }
public void Test_Anonymise() { var r = new Random(23); var person = new Person(r); var generator = new DicomDataGenerator(r, new DirectoryInfo(TestContext.CurrentContext.WorkDirectory), "CT"); // without anonymisation (default) we get the normal patient ID var ds = generator.GenerateTestDataset(person, r); Assert.IsTrue(ds.Contains(DicomTag.PatientID)); Assert.AreEqual(person.CHI, ds.GetValue <string>(DicomTag.PatientID, 0)); // with anonymisation generator.Anonymise = true; var ds2 = generator.GenerateTestDataset(person, r); // we get a blank patient ID Assert.IsTrue(ds2.Contains(DicomTag.PatientID)); Assert.AreEqual(string.Empty, ds2.GetString(DicomTag.PatientID)); generator.Dispose(); }
public void Test_ZipFileNotation(bool expressRelative) { //get a clean database to upload to var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); //create a folder in which to generate some dicoms var dirToLoad = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFileNotation))); if (dirToLoad.Exists) { dirToLoad.Delete(true); } dirToLoad.Create(); //generate some random dicoms var r = new Random(999); DicomDataGenerator generator = new DicomDataGenerator(r, dirToLoad, "CT") { MaximumImages = 5 }; var people = new PersonCollection(); people.GeneratePeople(1, r); generator.GenerateTestDataFile(people, new FileInfo("./inventory.csv"), 1); //This generates // Test_ZipFile // 2015 // 3 // 18 // 751140 2.25.166922918107154891877498685128076062226.dcm // 751140 2.25.179610809676265137473873365625829826423.dcm // 751140 2.25.201969634959506849065133495434871450465.dcm // 751140 2.25.237492679533001779093365416814254319890.dcm // 751140 2.25.316241631782653383510844072713132248731.dcm var yearDir = dirToLoad.GetDirectories().Single(); StringAssert.IsMatch("\\d{4}", yearDir.Name); //should be 5 images in the zip file var dicomFiles = yearDir.GetFiles("*.dcm", SearchOption.AllDirectories); Assert.AreEqual(5, dicomFiles.Length); //e.g. \2015\3\18\2.25.223398837779449245317520567111874824918.dcm //e.g. \2015\3\18\2.25.179610809676265137473873365625829826423.dcm var relativePathWithinZip1 = dicomFiles[0].FullName.Substring(dirToLoad.FullName.Length); var relativePathWithinZip2 = dicomFiles[1].FullName.Substring(dirToLoad.FullName.Length); //zip them up FileInfo zip = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip")); Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip"); if (zip.Exists) { zip.Delete(); } ZipFile.CreateFromDirectory(dirToLoad.FullName, zip.FullName); //e.g. E:\RdmpDicom\Rdmp.Dicom.Tests\bin\Debug\netcoreapp2.2\Test_ZipFile.zip!\2015\3\18\2.25.223398837779449245317520567111874824918.dcm string pathToLoad1 = zip.FullName + "!" + relativePathWithinZip1; string pathToLoad2 = zip.FullName + "!" + relativePathWithinZip2; var loadMeTextFile = new FileInfo(Path.Combine(dirToLoad.FullName, "LoadMe.txt")); //tell the source to load the zip File.WriteAllText(loadMeTextFile.FullName, string.Join(Environment.NewLine, pathToLoad1, pathToLoad2)); var f = new FlatFileToLoad(loadMeTextFile); //Setup source var source = new DicomFileCollectionSource { FilenameField = "RelativeFileArchiveURI" }; if (expressRelative) { source.ArchiveRoot = TestContext.CurrentContext.TestDirectory; } var worklist = new FlatFileToLoadDicomFileWorklist(f); //Setup destination var destination = new DataTableUploadDestination { AllowResizingColumnsAtUploadTime = true }; //setup pipeline var contextFactory = new DataFlowPipelineContextFactory <DataTable>(); var context = contextFactory.Create(PipelineUsage.FixedDestination | PipelineUsage.FixedDestination); //run pipeline var pipe = new DataFlowPipelineEngine <DataTable>(context, source, destination, new ThrowImmediatelyDataLoadEventListener()); pipe.Initialize(db, worklist); pipe.ExecutePipeline(new GracefulCancellationToken()); var finalTable = db.ExpectTable(destination.TargetTableName); using (var dt = finalTable.GetDataTable()) { //should be 2 rows (since we told it to only load 2 files out of the zip) Assert.AreEqual(2, dt.Rows.Count); string pathInDbToDicomFile = (string)dt.Rows[0]["RelativeFileArchiveURI"]; //We expect either something like: // E:/RdmpDicom/Rdmp.Dicom.Tests/bin/Debug/netcoreapp2.2/Test_ZipFile.zip!2015/3/18/2.25.160787663560951826149226183314694084702.dcm // ./Test_ZipFile.zip!2015/3/18/2.25.105592977437473375573190160334447272386.dcm //the path referenced should be the file read in relative/absolute format StringAssert.IsMatch( expressRelative ? $@"./{zip.Name}![\d./]*.dcm": $@"{Regex.Escape(zip.FullName.Replace('\\','/'))}![\d./]*.dcm", pathInDbToDicomFile); StringAssert.Contains(yearDir.Name, pathInDbToDicomFile, "Expected zip file to have subdirectories and for them to be loaded correctly"); //confirm we can read that out again using (var pool = new ZipPool()) { var path = new AmbiguousFilePath(TestContext.CurrentContext.TestDirectory, pathInDbToDicomFile); Assert.IsNotNull(path.GetDataset(pool)); } } Assert.IsTrue(finalTable.Exists()); finalTable.Drop(); }
public void Test_ZipFile(bool expressRelative) { //get a clean database to upload to var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); //create a folder in which to generate some dicoms var dirToLoad = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile))); if (dirToLoad.Exists) { dirToLoad.Delete(true); } dirToLoad.Create(); //generate some random dicoms var r = new Random(999); DicomDataGenerator generator = new DicomDataGenerator(r, dirToLoad, "CT") { MaximumImages = 5 }; var people = new PersonCollection(); people.GeneratePeople(1, r); generator.GenerateTestDataFile(people, new FileInfo("./inventory.csv"), 1); //This generates // Test_ZipFile // 2015 // 3 // 18 // 751140 2.25.166922918107154891877498685128076062226.dcm // 751140 2.25.179610809676265137473873365625829826423.dcm // 751140 2.25.201969634959506849065133495434871450465.dcm // 751140 2.25.237492679533001779093365416814254319890.dcm // 751140 2.25.316241631782653383510844072713132248731.dcm var yearDir = dirToLoad.GetDirectories().Single(); StringAssert.IsMatch("\\d{4}", yearDir.Name); //zip them up FileInfo zip = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip")); Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_ZipFile) + ".zip"); if (zip.Exists) { zip.Delete(); } ZipFile.CreateFromDirectory(dirToLoad.FullName, zip.FullName); //tell the source to load the zip var f = new FlatFileToLoad(zip); var source = new DicomFileCollectionSource { FilenameField = "RelativeFileArchiveURI" }; if (expressRelative) { source.ArchiveRoot = TestContext.CurrentContext.TestDirectory; } source.PreInitialize(new FlatFileToLoadDicomFileWorklist(f), new ThrowImmediatelyDataLoadEventListener()); var tbl = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); var destination = new DataTableUploadDestination(); destination.PreInitialize(db, new ThrowImmediatelyDataLoadEventListener()); destination.AllowResizingColumnsAtUploadTime = true; destination.ProcessPipelineData(tbl, new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); destination.Dispose(new ThrowImmediatelyDataLoadEventListener(), null); var finalTable = db.ExpectTable(destination.TargetTableName); using (var dt = finalTable.GetDataTable()) { //should be 5 rows in the final table (5 images) Assert.AreEqual(5, dt.Rows.Count); string pathInDbToDicomFile = (string)dt.Rows[0]["RelativeFileArchiveURI"]; //We expect either something like: // E:/RdmpDicom/Rdmp.Dicom.Tests/bin/Debug/netcoreapp2.2/Test_ZipFile.zip!2015/3/18/2.25.160787663560951826149226183314694084702.dcm // ./Test_ZipFile.zip!2015/3/18/2.25.105592977437473375573190160334447272386.dcm //the path referenced should be the file read in relative/absolute format StringAssert.IsMatch( expressRelative ? $@"./{zip.Name}![\d./]*.dcm": $@"{Regex.Escape(zip.FullName.Replace('\\','/'))}![\d./]*.dcm", pathInDbToDicomFile); StringAssert.Contains(yearDir.Name, pathInDbToDicomFile, "Expected zip file to have subdirectories and for them to be loaded correctly"); //confirm we can read that out again using (var pool = new ZipPool()) { var path = new AmbiguousFilePath(TestContext.CurrentContext.TestDirectory, pathInDbToDicomFile); Assert.IsNotNull(path.GetDataset(pool)); } } Assert.IsTrue(finalTable.Exists()); finalTable.Drop(); }
public void TestBulkInsertOnly(DatabaseType databaseType, int numberOfImages) { foreach (Pipeline p in CatalogueRepository.GetAllObjects <Pipeline>()) { p.DeleteInDatabase(); } var db = GetCleanedServer(databaseType); var template = ImageTableTemplateCollection.LoadFrom(_templateXml); _globals = new GlobalOptionsFactory().Load(); _globals.DicomRelationalMapperOptions.DatabaseNamerType = typeof(MyFixedStagingDatabaseNamer).FullName; _globals.DicomRelationalMapperOptions.QoSPrefetchCount = ushort.MaxValue; _globals.DicomRelationalMapperOptions.MinimumBatchSize = numberOfImages; _globals.DicomRelationalMapperOptions.UseInsertIntoForRAWMigration = true; _helper = new DicomRelationalMapperTestHelper(); _helper.SetupSuite(db, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource), root: null, template: template, persistentRaw: true); //do not use an explicit RAW data load server CatalogueRepository.GetServerDefaults().ClearDefault(PermissableDefaults.RAWDataLoadServer); Random r = new Random(123); List <DicomDataset> allImages; using (var generator = new DicomDataGenerator(r, null, "CT")) allImages = generator.GenerateImages(numberOfImages, r); DicomDatasetCollectionSource source = new DicomDatasetCollectionSource(); source.PreInitialize(new ExplicitListDicomDatasetWorklist(allImages.ToArray(), "amagad.dcm", new Dictionary <string, string>() { { "MessageGuid", "0x123" } }), new ThrowImmediatelyDataLoadEventListener());; source.FilenameField = "gggg"; var dt = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); source.Dispose(new ThrowImmediatelyDataLoadEventListener() { WriteToConsole = true }, null); Assert.AreEqual(numberOfImages, allImages.Count); Assert.AreEqual(numberOfImages, dt.Rows.Count); var tables = _helper.LoadMetadata.GetDistinctTableInfoList(false); var config = new HICDatabaseConfiguration(_helper.LoadMetadata, new SuffixBasedNamer()); var job = Mock.Of <IDataLoadJob>( j => j.RegularTablesToLoad == tables.Cast <ITableInfo>().ToList() && j.DataLoadInfo == _dli && j.Configuration == config); var attacher = new AutoRoutingAttacher(); attacher.Job = job; //Drop Primary Keys using (var con = db.Server.GetConnection()) { con.Open(); var cmd = db.Server.GetCommand( databaseType == DatabaseType.MicrosoftSQLServer ? @"ALTER TABLE ImageTable DROP CONSTRAINT PK_ImageTable ALTER TABLE SeriesTable DROP CONSTRAINT PK_SeriesTable ALTER TABLE StudyTable DROP CONSTRAINT PK_StudyTable" : @"ALTER TABLE ImageTable DROP PRIMARY KEY; ALTER TABLE SeriesTable DROP PRIMARY KEY; ALTER TABLE StudyTable DROP PRIMARY KEY;" , con); cmd.ExecuteNonQuery(); } attacher.Initialize(LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "IgnoreMe", true), db); try { attacher.ProcessPipelineData(dt, new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); attacher.Dispose(new ThrowImmediatelyDataLoadEventListener(), null); } catch (Exception e) { attacher.Dispose(new ThrowImmediatelyDataLoadEventListener(), e); throw; } foreach (var tableInfo in tables) { Assert.AreEqual(numberOfImages, tableInfo.Discover(DataAccessContext.InternalDataProcessing).GetRowCount(), "Row count was wrong for " + tableInfo); } foreach (Pipeline allObject in CatalogueRepository.GetAllObjects <Pipeline>()) { allObject.DeleteInDatabase(); } }
private DiscoveredTable BuildExampleExtractionTable(DiscoveredDatabase db, string modality, int recordCount, bool useDcmFileExtension) { var tbl = db.CreateTable(modality + "_IsExtractable", new[] { new DatabaseColumnRequest("StudyInstanceUID", new DatabaseTypeRequest(typeof(string), 64), false), new DatabaseColumnRequest("SeriesInstanceUID", new DatabaseTypeRequest(typeof(string), 64), false), new DatabaseColumnRequest("SOPInstanceUID", new DatabaseTypeRequest(typeof(string), 64), false) { IsPrimaryKey = true }, new DatabaseColumnRequest("IsExtractableToDisk", new DatabaseTypeRequest(typeof(bool))), new DatabaseColumnRequest("IsExtractableToDisk_Reason", new DatabaseTypeRequest(typeof(string), 512)), new DatabaseColumnRequest("RelativeFileArchiveURI", new DatabaseTypeRequest(typeof(string), 512), false), new DatabaseColumnRequest("IsOriginal", new DatabaseTypeRequest(typeof(bool)), false), new DatabaseColumnRequest("IsPrimary", new DatabaseTypeRequest(typeof(bool)), false), new DatabaseColumnRequest(SpecialFieldNames.DataLoadRunID, new DatabaseTypeRequest(typeof(int))), new DatabaseColumnRequest(SpecialFieldNames.ValidFrom, new DatabaseTypeRequest(typeof(DateTime))), }); if (recordCount > 0) { var r = new Random(500); DicomDataGenerator g = new DicomDataGenerator(r, null); g.MaximumImages = recordCount; var persons = new PersonCollection(); persons.GeneratePeople(500, r); while (recordCount > 0) { foreach (var image in g.GenerateStudyImages(persons.People[r.Next(persons.People.Length)], out var study)) { tbl.Insert(new Dictionary <string, object>() { { "StudyInstanceUID", image.GetSingleValue <string>(DicomTag.StudyInstanceUID) }, { "SeriesInstanceUID", image.GetSingleValue <string>(DicomTag.SeriesInstanceUID) }, { "SOPInstanceUID", image.GetSingleValue <string>(DicomTag.SOPInstanceUID) }, { "IsExtractableToDisk", true }, { "IsExtractableToDisk_Reason", DBNull.Value }, { "RelativeFileArchiveURI", image.GetSingleValue <string>(DicomTag.SOPInstanceUID) + (useDcmFileExtension ? ".dcm" :"") }, { "IsOriginal", image.GetValues <string>(DicomTag.ImageType)[0] == "ORIGINAL" }, { "IsPrimary", image.GetValues <string>(DicomTag.ImageType)[1] == "PRIMARY" }, { SpecialFieldNames.DataLoadRunID, 1 }, { SpecialFieldNames.ValidFrom, DateTime.Now }, }); recordCount--; if (recordCount <= 0) { break; } } } } return(tbl); }
public void TestLoadingOneImage_MileWideTest() { _helper.TruncateTablesIfExists(); DirectoryInfo d = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(TestLoadingOneImage_MileWideTest))); d.Create(); var r = new Random(5000); FileInfo[] files; using (var g = new DicomDataGenerator(r, d, "CT")) files = g.GenerateImageFiles(1, r).ToArray(); Assert.AreEqual(1, files.Length); var existingColumns = _helper.ImageTable.DiscoverColumns(); //Add 200 random tags foreach (string tag in TagColumnAdder.GetAvailableTags().OrderBy(a => r.Next()).Take(200)) { string dataType; try { dataType = TagColumnAdder.GetDataTypeForTag(tag, new MicrosoftSQLTypeTranslater()); } catch (Exception) { continue; } if (existingColumns.Any(c => c.GetRuntimeName().Equals(tag))) { continue; } var adder = new TagColumnAdder(tag, dataType, _helper.ImageTableInfo, new AcceptAllCheckNotifier()); adder.SkipChecksAndSynchronization = true; adder.Execute(); } new TableInfoSynchronizer(_helper.ImageTableInfo).Synchronize(new AcceptAllCheckNotifier()); //creates the queues, exchanges and bindings var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.DicomRelationalMapperOptions); tester.CreateExchange(_globals.RabbitOptions.FatalLoggingExchange, null); using (var host = new DicomRelationalMapperHost(_globals)) { host.Start(); using (var timeline = new TestTimeline(tester)) { foreach (var f in files) { timeline.SendMessage(_globals.DicomRelationalMapperOptions, _helper.GetDicomFileMessage(_globals.FileSystemOptions.FileSystemRoot, f)); } //start the timeline timeline.StartTimeline(); new TestTimelineAwaiter().Await(() => host.Consumer.MessagesProcessed == 1, null, 30000, () => host.Consumer.DleErrors); Assert.GreaterOrEqual(1, _helper.SeriesTable.GetRowCount(), "SeriesTable did not have the expected number of rows in LIVE"); Assert.GreaterOrEqual(1, _helper.StudyTable.GetRowCount(), "StudyTable did not have the expected number of rows in LIVE"); Assert.AreEqual(1, _helper.ImageTable.GetRowCount(), "ImageTable did not have the expected number of rows in LIVE"); host.Stop("Test end"); } } tester.Shutdown(); }