public void TestGetChunktOnly(int numberOfImages) { Random r = new Random(123); List <DicomDataset> allImages; using (DicomDataGenerator g = new DicomDataGenerator(r, null, "CT", "MR")) allImages = g.GenerateImages(numberOfImages, r); DicomDatasetCollectionSource source = new DicomDatasetCollectionSource(); source.PreInitialize(new ExplicitListDicomDatasetWorklist(allImages.ToArray(), "amagad.dcm", new Dictionary <string, string>() { { "MessageGuid", "0x123" } }), new ThrowImmediatelyDataLoadEventListener());; source.FilenameField = "gggg"; Stopwatch sw = new Stopwatch(); sw.Start(); var dt = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); source.Dispose(new ThrowImmediatelyDataLoadEventListener() { WriteToConsole = true }, null); sw.Stop(); Console.WriteLine("GetChunk took " + sw.ElapsedMilliseconds); Assert.AreEqual(numberOfImages, dt.Rows.Count); Assert.AreEqual(numberOfImages, dt.Rows.Cast <DataRow>().Select(w => w["SOPInstanceUID"]).Distinct().Count()); }
public void SourceRead_InvalidFloatInSequence_ToTable(InvalidDataHandling dataHandlingStrategy) { var source = new DicomDatasetCollectionSource(); source.InvalidDataHandlingStrategy = dataHandlingStrategy; //when we have a dicom file with an invalid Float number var ds = new DicomDataset(); ds.Add(DicomTag.PatientAge, "123Y"); var sequence = new DicomSequence(DicomTag.AcquisitionContextSequence, new DicomDataset() { { DicomTag.WedgeAngleFloat, "3.40282347e+038" } }); ds.Add(sequence); var worklist = new ExplicitListDicomDatasetWorklist(new[] { ds }, "fish.dcm"); source.PreInitialize(worklist, new ThrowImmediatelyDataLoadEventListener()); source.FilenameField = "RelFileName"; DataTable dt = null; switch (dataHandlingStrategy) { case InvalidDataHandling.MarkCorrupt: dt = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); //row was not processed (which leaves data table with 0 rows and hence component returns null) Assert.IsNull(dt); //corrupt message should appear in the worklist Assert.AreEqual(1, worklist.CorruptMessages.Count); return; case InvalidDataHandling.ConvertToNullAndWarn: dt = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); Assert.AreEqual("123Y", dt.Rows[0]["PatientAge"]); Assert.AreEqual("fish.dcm", dt.Rows[0]["RelFileName"]); Assert.AreEqual(DBNull.Value, dt.Rows[0]["AcquisitionContextSequence"]); Assert.AreEqual(0, worklist.CorruptMessages.Count); break; case InvalidDataHandling.ThrowException: Assert.Throws <OverflowException>(() => source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken())); return; default: throw new ArgumentOutOfRangeException("dataHandlingStrategy"); } }
public void SourceRead_ToTable_IgnoringSuperflousColumn_LoadMetadata() { var repo = new MemoryCatalogueRepository(); var lmd = new LoadMetadata(repo, "MyLoad"); var cata1 = new Catalogue(repo, "PatientCatalogue"); var ci1 = new CatalogueItem(repo, cata1, "PatientAge"); var ti1 = new TableInfo(repo, "PatientTableInfo"); var colInfo1 = new ColumnInfo(repo, "PatientAge", "varchar(100)", ti1); ci1.ColumnInfo_ID = colInfo1.ID; ci1.SaveToDatabase(); cata1.LoadMetadata_ID = lmd.ID; cata1.SaveToDatabase(); var cata2 = new Catalogue(repo, "FileCatalogue"); var ci2 = new CatalogueItem(repo, cata2, "RelFileName"); var ti2 = new TableInfo(repo, "FileTableInfo"); var colInfo2 = new ColumnInfo(repo, "RelFileName", "varchar(100)", ti2); ci2.ColumnInfo_ID = colInfo2.ID; ci2.SaveToDatabase(); cata2.LoadMetadata_ID = lmd.ID; cata2.SaveToDatabase(); var source = new DicomDatasetCollectionSource(); source.InvalidDataHandlingStrategy = InvalidDataHandling.ThrowException; var ds = new DicomDataset(); ds.Add(DicomTag.PatientAge, "123Y"); var sequence = new DicomSequence(DicomTag.AcquisitionContextSequence, new DicomDataset() { { DicomTag.WedgeAngleFloat, "3.40282347e+038" } }); ds.Add(sequence); var worklist = new ExplicitListDicomDatasetWorklist(new[] { ds }, "fish.dcm"); source.PreInitialize(worklist, new ThrowImmediatelyDataLoadEventListener()); source.FilenameField = "RelFileName"; source.UseAllTableInfoInLoadAsFieldMap = lmd; var dt = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); Assert.AreEqual("123Y", dt.Rows[0]["PatientAge"]); Assert.AreEqual("fish.dcm", dt.Rows[0]["RelFileName"]); Assert.AreEqual(2, dt.Columns.Count); }
public void SourceRead_InvalidFloat_ToTable(InvalidDataHandling dataHandlingStrategy) { var source = new DicomDatasetCollectionSource(); source.InvalidDataHandlingStrategy = dataHandlingStrategy; var ds = new DicomDataset(); ds.Add(DicomTag.PatientAge, "123Y"); ds.Add(DicomTag.WedgeAngleFloat, "3.40282347e+038"); var worklist = new ExplicitListDicomDatasetWorklist(new[] { ds }, "fish.dcm", new Dictionary <string, string> { { "MessageGuid", "123x321" } }); source.PreInitialize(worklist, new ThrowImmediatelyDataLoadEventListener()); source.FilenameField = "RelFileName"; DataTable dt = null; switch (dataHandlingStrategy) { case InvalidDataHandling.ThrowException: Assert.Throws <OverflowException>(() => source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken())); return; case InvalidDataHandling.ConvertToNullAndWarn: var tomem = new ToMemoryDataLoadEventListener(true); dt = source.GetChunk(tomem, new GracefulCancellationToken()); Assert.AreEqual(DBNull.Value, dt.Rows[0]["WedgeAngleFloat"]); //should be a warning about WedgeAngleFloat logged var warning = tomem.EventsReceivedBySender.SelectMany(e => e.Value).Single(v => v.ProgressEventType == ProgressEventType.Warning); Assert.IsTrue(warning.Message.Contains("WedgeAngleFloat")); Assert.IsTrue(warning.Message.Contains("MessageGuid")); Assert.IsTrue(warning.Message.Contains("123x321")); Assert.IsTrue(warning.Message.Contains("fish.dcm")); break; default: throw new ArgumentOutOfRangeException("dataHandlingStrategy"); } Assert.AreEqual("123Y", dt.Rows[0]["PatientAge"]); Assert.AreEqual("fish.dcm", dt.Rows[0]["RelFileName"]); }
public void TestStringTooLong(DataTooWideHandling strategy) { var ds = new DicomDataset(); #pragma warning disable CS0618 // Obsolete ds.AutoValidate = false; #pragma warning restore CS0618 ds.AddOrUpdate(DicomTag.AccessionNumber, "1342340123129473279427572495349757459347839479375974"); ds.GetValues <string>(DicomTag.AccessionNumber); var source = new DicomDatasetCollectionSource(); var worklist = new ExplicitListDicomDatasetWorklist(new[] { ds }, "fish.dcm", new Dictionary <string, string>()); source.DataTooLongHandlingStrategy = strategy; source.FilenameField = "abc"; source.PreInitialize(worklist, new ThrowImmediatelyDataLoadEventListener()); var dt = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); switch (strategy) { case DataTooWideHandling.None: Assert.AreEqual("1342340123129473279427572495349757459347839479375974", dt.Rows[0]["AccessionNumber"]); Assert.AreEqual(0, worklist.CorruptMessages.Count); break; case DataTooWideHandling.TruncateAndWarn: Assert.AreEqual("1342340123129473", dt.Rows[0]["AccessionNumber"]); Assert.AreEqual(0, worklist.CorruptMessages.Count); break; case DataTooWideHandling.MarkCorrupt: Assert.IsNull(dt); //since dt has no rows it just returns null Assert.AreEqual(1, worklist.CorruptMessages.Count); break; case DataTooWideHandling.ConvertToNullAndWarn: Assert.AreEqual(DBNull.Value, dt.Rows[0]["AccessionNumber"]); Assert.AreEqual(0, worklist.CorruptMessages.Count); break; default: throw new ArgumentOutOfRangeException("strategy"); } }
public void SourceReadSimpleTagToTable() { var source = new DicomDatasetCollectionSource(); var ds = new DicomDataset(); ds.Add(DicomTag.PatientAge, "123Y"); var worklist = new ExplicitListDicomDatasetWorklist(new [] { ds }, "fish.dcm"); source.PreInitialize(worklist, new ThrowImmediatelyDataLoadEventListener()); source.FilenameField = "RelFileName"; var dt = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); Assert.AreEqual("123Y", dt.Rows[0]["PatientAge"]); Assert.AreEqual("fish.dcm", dt.Rows[0]["RelFileName"]); }
public void SourceRead_ToTable_IgnoringSuperflousColumn_TableInfo() { var repo = new MemoryCatalogueRepository(); var ti = new TableInfo(repo, "MyTable"); new ColumnInfo(repo, "PatientAge", "varchar(100)", ti); new ColumnInfo(repo, "RelFileName", "varchar(100)", ti); var source = new DicomDatasetCollectionSource(); source.InvalidDataHandlingStrategy = InvalidDataHandling.ThrowException; var ds = new DicomDataset(); ds.Add(DicomTag.PatientAge, "123Y"); var sequence = new DicomSequence(DicomTag.AcquisitionContextSequence, new DicomDataset() { { DicomTag.WedgeAngleFloat, "3.40282347e+038" } }); ds.Add(sequence); var worklist = new ExplicitListDicomDatasetWorklist(new[] { ds }, "fish.dcm"); source.PreInitialize(worklist, new ThrowImmediatelyDataLoadEventListener()); source.FilenameField = "RelFileName"; source.FieldMapTableIfAny = ti; var dt = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); Assert.AreEqual("123Y", dt.Rows[0]["PatientAge"]); Assert.AreEqual("fish.dcm", dt.Rows[0]["RelFileName"]); Assert.AreEqual(2, dt.Columns.Count); }
public void Test_NullRoot() { var s = new DicomDatasetCollectionSource(); s.ArchiveRoot = null; }
public void TestBulkInsertOnly(DatabaseType databaseType, int numberOfImages) { foreach (Pipeline p in CatalogueRepository.GetAllObjects <Pipeline>()) { p.DeleteInDatabase(); } var db = GetCleanedServer(databaseType); var template = ImageTableTemplateCollection.LoadFrom(_templateXml); _globals = new GlobalOptionsFactory().Load(); _globals.DicomRelationalMapperOptions.DatabaseNamerType = typeof(MyFixedStagingDatabaseNamer).FullName; _globals.DicomRelationalMapperOptions.QoSPrefetchCount = ushort.MaxValue; _globals.DicomRelationalMapperOptions.MinimumBatchSize = numberOfImages; _globals.DicomRelationalMapperOptions.UseInsertIntoForRAWMigration = true; _helper = new DicomRelationalMapperTestHelper(); _helper.SetupSuite(db, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource), root: null, template: template, persistentRaw: true); //do not use an explicit RAW data load server CatalogueRepository.GetServerDefaults().ClearDefault(PermissableDefaults.RAWDataLoadServer); Random r = new Random(123); List <DicomDataset> allImages; using (var generator = new DicomDataGenerator(r, null, "CT")) allImages = generator.GenerateImages(numberOfImages, r); DicomDatasetCollectionSource source = new DicomDatasetCollectionSource(); source.PreInitialize(new ExplicitListDicomDatasetWorklist(allImages.ToArray(), "amagad.dcm", new Dictionary <string, string>() { { "MessageGuid", "0x123" } }), new ThrowImmediatelyDataLoadEventListener());; source.FilenameField = "gggg"; var dt = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); source.Dispose(new ThrowImmediatelyDataLoadEventListener() { WriteToConsole = true }, null); Assert.AreEqual(numberOfImages, allImages.Count); Assert.AreEqual(numberOfImages, dt.Rows.Count); var tables = _helper.LoadMetadata.GetDistinctTableInfoList(false); var config = new HICDatabaseConfiguration(_helper.LoadMetadata, new SuffixBasedNamer()); var job = Mock.Of <IDataLoadJob>( j => j.RegularTablesToLoad == tables.Cast <ITableInfo>().ToList() && j.DataLoadInfo == _dli && j.Configuration == config); var attacher = new AutoRoutingAttacher(); attacher.Job = job; //Drop Primary Keys using (var con = db.Server.GetConnection()) { con.Open(); var cmd = db.Server.GetCommand( databaseType == DatabaseType.MicrosoftSQLServer ? @"ALTER TABLE ImageTable DROP CONSTRAINT PK_ImageTable ALTER TABLE SeriesTable DROP CONSTRAINT PK_SeriesTable ALTER TABLE StudyTable DROP CONSTRAINT PK_StudyTable" : @"ALTER TABLE ImageTable DROP PRIMARY KEY; ALTER TABLE SeriesTable DROP PRIMARY KEY; ALTER TABLE StudyTable DROP PRIMARY KEY;" , con); cmd.ExecuteNonQuery(); } attacher.Initialize(LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "IgnoreMe", true), db); try { attacher.ProcessPipelineData(dt, new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); attacher.Dispose(new ThrowImmediatelyDataLoadEventListener(), null); } catch (Exception e) { attacher.Dispose(new ThrowImmediatelyDataLoadEventListener(), e); throw; } foreach (var tableInfo in tables) { Assert.AreEqual(numberOfImages, tableInfo.Discover(DataAccessContext.InternalDataProcessing).GetRowCount(), "Row count was wrong for " + tableInfo); } foreach (Pipeline allObject in CatalogueRepository.GetAllObjects <Pipeline>()) { allObject.DeleteInDatabase(); } }
public void SourceRead_InvalidFloatInSequence_WithElevation_ToTable(InvalidDataHandling dataHandlingStrategy) { //create the elevation configuration var elevationRules = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, "ElevationConfig.xml")); File.WriteAllText(elevationRules.FullName, @"<!DOCTYPE TagElevationRequestCollection [ <!ELEMENT TagElevationRequestCollection (TagElevationRequest*)> <!ELEMENT TagElevationRequest (ColumnName,ElevationPathway,Conditional?)> <!ELEMENT ColumnName (#PCDATA)> <!ELEMENT ElevationPathway (#PCDATA)> <!ELEMENT Conditional (ConditionalPathway,ConditionalRegex)> <!ELEMENT ConditionalPathway (#PCDATA)> <!ELEMENT ConditionalRegex (#PCDATA)> ]> <TagElevationRequestCollection> <TagElevationRequest> <ColumnName>WedgeAngleFloat</ColumnName> <ElevationPathway>AcquisitionContextSequence->WedgeAngleFloat</ElevationPathway> </TagElevationRequest> </TagElevationRequestCollection>"); //setup the source reader var source = new DicomDatasetCollectionSource(); source.InvalidDataHandlingStrategy = dataHandlingStrategy; source.TagElevationConfigurationFile = elevationRules; //don't load the sequence, just the elevation source.TagBlacklist = new Regex("AcquisitionContextSequence"); //The dataset we are trying to load var ds = new DicomDataset(); ds.Add(DicomTag.PatientAge, "123Y"); var sequence = new DicomSequence(DicomTag.AcquisitionContextSequence, new DicomDataset() { { DicomTag.WedgeAngleFloat, "3.40282347e+038" } //dodgy float in sequence (the sequence we are trying to elevate) }); ds.Add(sequence); var worklist = new ExplicitListDicomDatasetWorklist(new[] { ds }, "fish.dcm", new Dictionary <string, string> { { "MessageGuid", "123x321" } }); source.PreInitialize(worklist, new ThrowImmediatelyDataLoadEventListener()); source.FilenameField = "RelFileName"; DataTable dt = null; switch (dataHandlingStrategy) { case InvalidDataHandling.ThrowException: Assert.Throws <OverflowException>(() => source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken())); return; case InvalidDataHandling.ConvertToNullAndWarn: var tomem = new ToMemoryDataLoadEventListener(true); dt = source.GetChunk(tomem, new GracefulCancellationToken()); Assert.AreEqual(DBNull.Value, dt.Rows[0]["WedgeAngleFloat"]); //should be a warning about WedgeAngleFloat logged var warning = tomem.EventsReceivedBySender.SelectMany(e => e.Value).Single(v => v.ProgressEventType == ProgressEventType.Warning); Assert.IsTrue(warning.Message.Contains("WedgeAngleFloat")); Assert.IsTrue(warning.Message.Contains("MessageGuid")); Assert.IsTrue(warning.Message.Contains("123x321")); Assert.IsTrue(warning.Message.Contains("fish.dcm")); break; default: throw new ArgumentOutOfRangeException("dataHandlingStrategy"); } Assert.AreEqual("123Y", dt.Rows[0]["PatientAge"]); Assert.AreEqual("fish.dcm", dt.Rows[0]["RelFileName"]); }