Example #1
0
        public void Test_DodgyTagNames()
        {
            _helper.TruncateTablesIfExists();

            DirectoryInfo d = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(Test_DodgyTagNames)));

            d.Create();

            var fi  = TestData.Create(new FileInfo(Path.Combine(d.FullName, "MyTestFile.dcm")));
            var fi2 = TestData.Create(new FileInfo(Path.Combine(d.FullName, "MyTestFile2.dcm")));

            DicomFile dcm;

            using (var stream = File.OpenRead(fi.FullName))
            {
                dcm = DicomFile.Open(stream);
                dcm.Dataset.AddOrUpdate(DicomTag.PrintRETIRED, "FISH");
                dcm.Dataset.AddOrUpdate(DicomTag.Date, new DateTime(2001, 01, 01));
                dcm.Save(fi2.FullName);
            }

            var adder = new TagColumnAdder(DicomTypeTranslaterReader.GetColumnNameForTag(DicomTag.Date, false), "datetime2", _helper.ImageTableInfo, new AcceptAllCheckNotifier());

            adder.Execute();

            adder = new TagColumnAdder(DicomTypeTranslaterReader.GetColumnNameForTag(DicomTag.PrintRETIRED, false), "datetime2", _helper.ImageTableInfo, new AcceptAllCheckNotifier());
            adder.Execute();

            fi.Delete();
            File.Move(fi2.FullName, fi.FullName);

            //creates the queues, exchanges and bindings
            var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.DicomRelationalMapperOptions);

            tester.CreateExchange(_globals.RabbitOptions.FatalLoggingExchange, null);

            using (var host = new DicomRelationalMapperHost(_globals))
            {
                host.Start();

                using (var timeline = new TestTimeline(tester))
                {
                    timeline.SendMessage(_globals.DicomRelationalMapperOptions, _helper.GetDicomFileMessage(_globals.FileSystemOptions.FileSystemRoot, fi));

                    //start the timeline
                    timeline.StartTimeline();

                    Thread.Sleep(TimeSpan.FromSeconds(10));
                    new TestTimelineAwaiter().Await(() => host.Consumer.AckCount >= 1, null, 30000, () => host.Consumer.DleErrors);

                    Assert.AreEqual(1, _helper.SeriesTable.GetRowCount(), "SeriesTable did not have the expected number of rows in LIVE");
                    Assert.AreEqual(1, _helper.StudyTable.GetRowCount(), "StudyTable did not have the expected number of rows in LIVE");
                    Assert.AreEqual(1, _helper.ImageTable.GetRowCount(), "ImageTable did not have the expected number of rows in LIVE");

                    host.Stop("Test end");
                }
            }

            tester.Shutdown();
        }
Example #2
0
        public void TestLargeImageDatasets(DatabaseType databaseType, int numberOfImages)
        {
            foreach (Pipeline p in CatalogueRepository.GetAllObjects <Pipeline>())
            {
                p.DeleteInDatabase();
            }

            var db = GetCleanedServer(databaseType);

            var d = CatalogueRepository.GetServerDefaults();

            d.ClearDefault(PermissableDefaults.RAWDataLoadServer);

            var template = ImageTableTemplateCollection.LoadFrom(_templateXml);

            _globals = new GlobalOptionsFactory().Load();

            _globals.DicomRelationalMapperOptions.DatabaseNamerType            = typeof(MyFixedStagingDatabaseNamer).FullName;
            _globals.DicomRelationalMapperOptions.QoSPrefetchCount             = ushort.MaxValue;
            _globals.DicomRelationalMapperOptions.MinimumBatchSize             = numberOfImages;
            _globals.DicomRelationalMapperOptions.UseInsertIntoForRAWMigration = true;

            _helper = new DicomRelationalMapperTestHelper();
            _helper.SetupSuite(db, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource), root: null, template: template, persistentRaw: true);

            //do not use an explicit RAW data load server
            d.ClearDefault(PermissableDefaults.RAWDataLoadServer);

            Random r = new Random(123);

            List <DicomDataset> allImages;

            using (var generator = new DicomDataGenerator(r, null, "CT"))
                allImages = generator.GenerateImages(numberOfImages, r);

            Assert.AreEqual(numberOfImages, allImages.Count);

            using (var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.DicomRelationalMapperOptions))
            {
                using (var host = new DicomRelationalMapperHost(_globals))
                {
                    tester.SendMessages(_globals.DicomRelationalMapperOptions, allImages.Select(GetFileMessageForDataset), true);

                    Console.WriteLine("Starting Host");
                    host.Start();

                    Stopwatch sw = Stopwatch.StartNew();
                    new TestTimelineAwaiter().Await(() => host.Consumer.AckCount == numberOfImages, null, 20 * 60 * 100); //1 minute

                    Console.Write("Time For DLE:" + sw.Elapsed.TotalSeconds + "s");
                    host.Stop("Test finished");
                }
            }

            foreach (Pipeline allObject in CatalogueRepository.GetAllObjects <Pipeline>())
            {
                allObject.DeleteInDatabase();
            }
        }
Example #3
0
        public void TestLoadingOneImage_SingleFileMessage(int numberOfMessagesToSend, bool mixInATextFile)
        {
            _helper.TruncateTablesIfExists();

            DirectoryInfo d = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(TestLoadingOneImage_SingleFileMessage)));

            d.Create();

            var fi = TestData.Create(new FileInfo(Path.Combine(d.FullName, "MyTestFile.dcm")));

            if (mixInATextFile)
            {
                var randomText = new FileInfo(Path.Combine(d.FullName, "RandomTextFile.dcm"));
                File.WriteAllLines(randomText.FullName, new[] { "I love dancing", "all around the world", "boy the world is a big place eh?" });
            }

            //creates the queues, exchanges and bindings
            var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.DicomRelationalMapperOptions);

            tester.CreateExchange(_globals.RabbitOptions.FatalLoggingExchange, null);

            using (var host = new DicomRelationalMapperHost(_globals))
            {
                host.Start();

                using (var timeline = new TestTimeline(tester))
                {
                    //send the message 10 times over a 10 second period
                    for (int i = 0; i < numberOfMessagesToSend; i++)
                    {
                        timeline
                        .SendMessage(_globals.DicomRelationalMapperOptions, _helper.GetDicomFileMessage(_globals.FileSystemOptions.FileSystemRoot, fi))
                        .Wait(1000);
                    }

                    //start the timeline
                    timeline.StartTimeline();

                    Thread.Sleep(TimeSpan.FromSeconds(10));
                    new TestTimelineAwaiter().Await(() => host.Consumer.AckCount >= numberOfMessagesToSend, null, 30000, () => host.Consumer.DleErrors);

                    Assert.AreEqual(1, _helper.SeriesTable.GetRowCount(), "SeriesTable did not have the expected number of rows in LIVE");
                    Assert.AreEqual(1, _helper.StudyTable.GetRowCount(), "StudyTable did not have the expected number of rows in LIVE");
                    Assert.AreEqual(1, _helper.ImageTable.GetRowCount(), "ImageTable did not have the expected number of rows in LIVE");

                    host.Stop("Test end");
                }
            }

            tester.Shutdown();
        }
Example #4
0
        public void TestCreatingNamer_CorrectType(DatabaseType dbType, string typeName, Type expectedType)
        {
            var db = GetCleanedServer(dbType);

            var dt = new DataTable();

            dt.Columns.Add("Hi");
            dt.Rows.Add("There");

            var tbl = db.CreateTable("DicomRelationalMapperHostTests", dt);

            var cata = Import(tbl);

            var globals         = new GlobalOptionsFactory().Load();
            var consumerOptions = globals.DicomRelationalMapperOptions;

            var lmd = new LoadMetadata(CatalogueRepository, "MyLoad");

            cata.LoadMetadata_ID = lmd.ID;
            cata.SaveToDatabase();

            consumerOptions.LoadMetadataId    = lmd.ID;
            consumerOptions.DatabaseNamerType = typeName;
            consumerOptions.Guid = Guid.Empty;

            globals.RDMPOptions.CatalogueConnectionString  = CatalogueRepository.DiscoveredServer.Builder.ConnectionString;
            globals.RDMPOptions.DataExportConnectionString = DataExportRepository.DiscoveredServer.Builder.ConnectionString;

            using (new MicroserviceTester(globals.RabbitOptions, globals.DicomRelationalMapperOptions))
            {
                using (var host = new DicomRelationalMapperHost(globals))
                {
                    host.Start();

                    Assert.AreEqual(expectedType, host.Consumer.DatabaseNamer.GetType());
                    Assert.IsNotNull(host);

                    host.Stop("Test finished");
                }
            }
        }
Example #5
0
        public void IdenticalDatasetsTest()
        {
            _helper.TruncateTablesIfExists();

            var ds = new DicomDataset();

            ds.AddOrUpdate(DicomTag.SeriesInstanceUID, "123");
            ds.AddOrUpdate(DicomTag.SOPInstanceUID, "123");
            ds.AddOrUpdate(DicomTag.StudyInstanceUID, "123");
            ds.AddOrUpdate(DicomTag.PatientID, "123");

            var msg1 = _helper.GetDicomFileMessage(ds, _globals.FileSystemOptions.FileSystemRoot, Path.Combine(_globals.FileSystemOptions.FileSystemRoot, "mydicom.dcm"));
            var msg2 = _helper.GetDicomFileMessage(ds, _globals.FileSystemOptions.FileSystemRoot, Path.Combine(_globals.FileSystemOptions.FileSystemRoot, "mydicom.dcm"));


            //creates the queues, exchanges and bindings
            using (var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.DicomRelationalMapperOptions))
            {
                tester.CreateExchange(_globals.RabbitOptions.FatalLoggingExchange, null);
                tester.SendMessage(_globals.DicomRelationalMapperOptions, msg1);
                tester.SendMessage(_globals.DicomRelationalMapperOptions, msg2);

                _globals.DicomRelationalMapperOptions.RunChecks = true;

                using (var host = new DicomRelationalMapperHost(_globals))
                {
                    host.Start();

                    new TestTimelineAwaiter().Await(() => host.Consumer.MessagesProcessed == 2, null, 30000, () => host.Consumer.DleErrors);

                    Assert.GreaterOrEqual(1, _helper.SeriesTable.GetRowCount(), "SeriesTable did not have the expected number of rows in LIVE");
                    Assert.GreaterOrEqual(1, _helper.StudyTable.GetRowCount(), "StudyTable did not have the expected number of rows in LIVE");
                    Assert.AreEqual(1, _helper.ImageTable.GetRowCount(), "ImageTable did not have the expected number of rows in LIVE");

                    host.Stop("Test end");
                }
                tester.Shutdown();
            }
        }
Example #6
0
        public void TestLoadingOneImage_MileWideTest()
        {
            _helper.TruncateTablesIfExists();

            DirectoryInfo d = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(TestLoadingOneImage_MileWideTest)));

            d.Create();

            var r = new Random(5000);

            FileInfo[] files;

            using (var g = new DicomDataGenerator(r, d, "CT"))
                files = g.GenerateImageFiles(1, r).ToArray();

            Assert.AreEqual(1, files.Length);

            var existingColumns = _helper.ImageTable.DiscoverColumns();

            //Add 200 random tags
            foreach (string tag in TagColumnAdder.GetAvailableTags().OrderBy(a => r.Next()).Take(200))
            {
                string dataType;

                try
                {
                    dataType = TagColumnAdder.GetDataTypeForTag(tag, new MicrosoftSQLTypeTranslater());
                }
                catch (Exception)
                {
                    continue;
                }

                if (existingColumns.Any(c => c.GetRuntimeName().Equals(tag)))
                {
                    continue;
                }

                var adder = new TagColumnAdder(tag, dataType, _helper.ImageTableInfo, new AcceptAllCheckNotifier());
                adder.SkipChecksAndSynchronization = true;
                adder.Execute();
            }

            new TableInfoSynchronizer(_helper.ImageTableInfo).Synchronize(new AcceptAllCheckNotifier());

            //creates the queues, exchanges and bindings
            var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.DicomRelationalMapperOptions);

            tester.CreateExchange(_globals.RabbitOptions.FatalLoggingExchange, null);

            using (var host = new DicomRelationalMapperHost(_globals))
            {
                host.Start();

                using (var timeline = new TestTimeline(tester))
                {
                    foreach (var f in files)
                    {
                        timeline.SendMessage(_globals.DicomRelationalMapperOptions,
                                             _helper.GetDicomFileMessage(_globals.FileSystemOptions.FileSystemRoot, f));
                    }

                    //start the timeline
                    timeline.StartTimeline();

                    new TestTimelineAwaiter().Await(() => host.Consumer.MessagesProcessed == 1, null, 30000, () => host.Consumer.DleErrors);

                    Assert.GreaterOrEqual(1, _helper.SeriesTable.GetRowCount(), "SeriesTable did not have the expected number of rows in LIVE");
                    Assert.GreaterOrEqual(1, _helper.StudyTable.GetRowCount(), "StudyTable did not have the expected number of rows in LIVE");
                    Assert.AreEqual(1, _helper.ImageTable.GetRowCount(), "ImageTable did not have the expected number of rows in LIVE");

                    host.Stop("Test end");
                }
            }

            tester.Shutdown();
        }
Example #7
0
        private void RunTest(DirectoryInfo dir, int numberOfExpectedRows, Action <FileSystemOptions> adjustFileSystemOptions)
        {
            TestLogger.Setup();
            var logger = LogManager.GetLogger("MicroservicesIntegrationTest");

            _globals.FileSystemOptions.FileSystemRoot = TestContext.CurrentContext.TestDirectory;

            var readFromFatalErrors = new ConsumerOptions
            {
                QueueName = "TEST.FatalLoggingQueue"
            };

            ///////////////////////////////////// Directory //////////////////////////
            var processDirectoryOptions = new DicomDirectoryProcessorCliOptions();

            processDirectoryOptions.ToProcessDir    = dir;
            processDirectoryOptions.DirectoryFormat = "Default";

            adjustFileSystemOptions?.Invoke(_globals.FileSystemOptions);

            //////////////////////////////////////////////// Mongo Db Populator ////////////////////////
            // Make this a GUID or something, should be unique per test
            var currentSeriesCollectionName = "Integration_HappyPath_Series" + DateTime.Now.Ticks;
            var currentImageCollectionName  = "Integration_HappyPath_Image" + DateTime.Now.Ticks;

            _globals.MongoDbPopulatorOptions.SeriesCollection = currentSeriesCollectionName;
            _globals.MongoDbPopulatorOptions.ImageCollection  = currentImageCollectionName;

            //use the test catalogue not the one in the combined app.config

            _globals.RDMPOptions.CatalogueConnectionString  = ((TableRepository)RepositoryLocator.CatalogueRepository).DiscoveredServer.Builder.ConnectionString;
            _globals.RDMPOptions.DataExportConnectionString = ((TableRepository)RepositoryLocator.DataExportRepository).DiscoveredServer.Builder.ConnectionString;
            _globals.DicomRelationalMapperOptions.RunChecks = true;

            if (_globals.DicomRelationalMapperOptions.MinimumBatchSize < 1)
            {
                _globals.DicomRelationalMapperOptions.MinimumBatchSize = 1;
            }

            using (var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.CohortExtractorOptions))
            {
                tester.CreateExchange(_globals.ProcessDirectoryOptions.AccessionDirectoryProducerOptions.ExchangeName, _globals.DicomTagReaderOptions.QueueName);
                tester.CreateExchange(_globals.DicomTagReaderOptions.SeriesProducerOptions.ExchangeName, _globals.MongoDbPopulatorOptions.SeriesQueueConsumerOptions.QueueName);
                tester.CreateExchange(_globals.DicomTagReaderOptions.ImageProducerOptions.ExchangeName, _globals.IdentifierMapperOptions.QueueName);
                tester.CreateExchange(_globals.DicomTagReaderOptions.ImageProducerOptions.ExchangeName, _globals.MongoDbPopulatorOptions.ImageQueueConsumerOptions.QueueName, true);
                tester.CreateExchange(_globals.IdentifierMapperOptions.AnonImagesProducerOptions.ExchangeName, _globals.DicomRelationalMapperOptions.QueueName);
                tester.CreateExchange(_globals.RabbitOptions.FatalLoggingExchange, readFromFatalErrors.QueueName);

                tester.CreateExchange(_globals.CohortExtractorOptions.ExtractFilesProducerOptions.ExchangeName, null, false, _globals.CohortExtractorOptions.ExtractIdentRoutingKey);
                tester.CreateExchange(_globals.CohortExtractorOptions.ExtractFilesProducerOptions.ExchangeName, null, true, _globals.CohortExtractorOptions.ExtractAnonRoutingKey);
                tester.CreateExchange(_globals.CohortExtractorOptions.ExtractFilesInfoProducerOptions.ExchangeName, null);

                #region Running Microservices

                var processDirectory = new DicomDirectoryProcessorHost(_globals, processDirectoryOptions);
                processDirectory.Start();
                tester.StopOnDispose.Add(processDirectory);

                var dicomTagReaderHost = new DicomTagReaderHost(_globals);
                dicomTagReaderHost.Start();
                tester.StopOnDispose.Add(dicomTagReaderHost);

                var mongoDbPopulatorHost = new MongoDbPopulatorHost(_globals);
                mongoDbPopulatorHost.Start();
                tester.StopOnDispose.Add(mongoDbPopulatorHost);

                var identifierMapperHost = new IdentifierMapperHost(_globals, new SwapForFixedValueTester("FISHFISH"));
                identifierMapperHost.Start();
                tester.StopOnDispose.Add(identifierMapperHost);

                new TestTimelineAwaiter().Await(() => dicomTagReaderHost.AccessionDirectoryMessageConsumer.AckCount >= 1);
                logger.Info("\n### DicomTagReader has processed its messages ###\n");

                // FIXME: This isn't exactly how the pipeline runs
                new TestTimelineAwaiter().Await(() => identifierMapperHost.Consumer.AckCount >= 1);
                logger.Info("\n### IdentifierMapper has processed its messages ###\n");

                using (var relationalMapperHost = new DicomRelationalMapperHost(_globals))
                {
                    var start = DateTime.Now;

                    relationalMapperHost.Start();
                    tester.StopOnDispose.Add(relationalMapperHost);

                    Assert.True(mongoDbPopulatorHost.Consumers.Count == 2);
                    new TestTimelineAwaiter().Await(() => mongoDbPopulatorHost.Consumers[0].Processor.AckCount >= 1);
                    new TestTimelineAwaiter().Await(() => mongoDbPopulatorHost.Consumers[1].Processor.AckCount >= 1);
                    logger.Info("\n### MongoDbPopulator has processed its messages ###\n");

                    new TestTimelineAwaiter().Await(() => identifierMapperHost.Consumer.AckCount >= 1);//number of series
                    logger.Info("\n### IdentifierMapper has processed its messages ###\n");

                    Assert.AreEqual(0, dicomTagReaderHost.AccessionDirectoryMessageConsumer.NackCount);
                    Assert.AreEqual(0, identifierMapperHost.Consumer.NackCount);
                    Assert.AreEqual(0, ((Consumer <SeriesMessage>)mongoDbPopulatorHost.Consumers[0]).NackCount);
                    Assert.AreEqual(0, ((Consumer <DicomFileMessage>)mongoDbPopulatorHost.Consumers[1]).NackCount);


                    try
                    {
                        Thread.Sleep(TimeSpan.FromSeconds(10));
                        new TestTimelineAwaiter().Await(() => relationalMapperHost.Consumer.AckCount >= numberOfExpectedRows, null, 30000, () => relationalMapperHost.Consumer.DleErrors); //number of image files
                        logger.Info("\n### DicomRelationalMapper has processed its messages ###\n");
                    }
                    finally
                    {
                        //find out what happens from the logging database
                        var rdmpLogging = new Rdmp.Core.Logging.LogManager(_helper.LoadMetadata.GetDistinctLoggingDatabase());

                        //if error was reported during the dicom relational mapper run
                        foreach (var dli in rdmpLogging.GetArchivalDataLoadInfos(_helper.LoadMetadata.GetDistinctLoggingTask(), null, null))
                        {
                            if (dli.StartTime > start)
                            {
                                foreach (ArchivalFatalError e in dli.Errors)
                                {
                                    logger.Error(e.Date.TimeOfDay + ":" + e.Source + ":" + e.Description);
                                }
                            }
                        }
                    }

                    Assert.AreEqual(numberOfExpectedRows, _helper.ImageTable.GetRowCount(), "All images should appear in the image table");
                    Assert.LessOrEqual(_helper.SeriesTable.GetRowCount(), numberOfExpectedRows, "Only unique series data should appear in series table, there should be less unique series than images (or equal)");
                    Assert.LessOrEqual(_helper.StudyTable.GetRowCount(), numberOfExpectedRows, "Only unique study data should appear in study table, there should be less unique studies than images (or equal)");
                    Assert.LessOrEqual(_helper.StudyTable.GetRowCount(), _helper.SeriesTable.GetRowCount(), "There should be less studies than series (or equal)");

                    //make sure that the substitution identifier (that replaces old the PatientId) is the correct substitution (FISHFISH)/
                    Assert.AreEqual("FISHFISH", _helper.StudyTable.GetDataTable().Rows.OfType <DataRow>().First()["PatientId"]);

                    //The file size in the final table should be more than 0
                    Assert.Greater((long)_helper.ImageTable.GetDataTable().Rows.OfType <DataRow>().First()["DicomFileSize"], 0);

                    dicomTagReaderHost.Stop("TestIsFinished");

                    mongoDbPopulatorHost.Stop("TestIsFinished");
                    DropMongoTestDb(_globals.MongoDatabases.DicomStoreOptions.HostName, _globals.MongoDatabases.DicomStoreOptions.Port);

                    identifierMapperHost.Stop("TestIsFinished");

                    relationalMapperHost.Stop("Test end");
                }

                //Now do extraction
                var extractorHost = new CohortExtractorHost(_globals, null, null);

                extractorHost.Start();

                var extract = new ExtractionRequestMessage
                {
                    ExtractionJobIdentifier = Guid.NewGuid(),
                    ProjectNumber           = "1234-5678",
                    ExtractionDirectory     = "1234-5678_P1",
                    KeyTag = "SeriesInstanceUID",
                };

                foreach (DataRow row in _helper.ImageTable.GetDataTable().Rows)
                {
                    var ser = (string)row["SeriesInstanceUID"];

                    if (!extract.ExtractionIdentifiers.Contains(ser))
                    {
                        extract.ExtractionIdentifiers.Add(ser);
                    }
                }

                tester.SendMessage(_globals.CohortExtractorOptions, extract);

                //wait till extractor picked up the messages and dispatched the responses
                new TestTimelineAwaiter().Await(() => extractorHost.Consumer.AckCount == 1);

                extractorHost.Stop("TestIsFinished");

                tester.Shutdown();
            }


            #endregion
        }