private static int OnParse(GlobalOptions globals, DicomDirectoryProcessorCliOptions parsedOptions) { var bootstrapper = new MicroserviceHostBootstrapper(() => new DicomDirectoryProcessorHost(globals, parsedOptions)); int ret = bootstrapper.Main(); return(ret); }
/// <summary> /// Constructor /// </summary> /// <param name="cliOptions">Common microservices options. Must contain details for an message exchange labelled as "accessionDirectories"</param> /// <param name="globals">Configuration settings for the program</param> public DicomDirectoryProcessorHost(GlobalOptions globals, DicomDirectoryProcessorCliOptions cliOptions) : base(globals) { _cliOptions = cliOptions; if (!cliOptions.DirectoryFormat.ToLower().Equals("list")) { // TODO(rkm 2020-02-12) I think we want to check this regardless of the mode // (bp 2020-02-13) By not doing this check on list means that the list of paths is not required to be in PACS and can be imported from anywhere if (!Directory.Exists(globals.FileSystemOptions.FileSystemRoot)) { throw new ArgumentException("Cannot find the FileSystemRoot specified in the given MicroservicesOptions (" + globals.FileSystemOptions.FileSystemRoot + ")"); } if (!cliOptions.ToProcessDir.Exists) { throw new ArgumentException("Could not find directory " + cliOptions.ToProcessDir.FullName); } if (!cliOptions.ToProcessDir.FullName.StartsWith(globals.FileSystemOptions.FileSystemRoot, true, CultureInfo.CurrentCulture)) { throw new ArgumentException("Directory parameter (" + cliOptions.ToProcessDir.FullName + ") must be below the FileSystemRoot (" + globals.FileSystemOptions.FileSystemRoot + ")"); } } else { if (!File.Exists(cliOptions.ToProcessDir.FullName)) { throw new ArgumentException("Could not find accession directory list file (" + cliOptions.ToProcessDir.FullName + ")"); } if (!Path.GetExtension(cliOptions.ToProcessDir.FullName).Equals(".csv")) { throw new ArgumentException("When in 'list' mode, path to accession directory file of format .csv expected (" + cliOptions.ToProcessDir.FullName + ")"); } } if (cliOptions.DirectoryFormat.ToLower().Equals("pacs")) { Logger.Info("Creating PACS directory finder"); _ddf = new PacsDirectoryFinder(globals.FileSystemOptions.FileSystemRoot, globals.FileSystemOptions.DicomSearchPattern, RabbitMqAdapter.SetupProducer(globals.ProcessDirectoryOptions.AccessionDirectoryProducerOptions, isBatch: false)); } else if (cliOptions.DirectoryFormat.ToLower().Equals("list")) { Logger.Info("Creating accession directory lister"); _ddf = new AccessionDirectoryLister(globals.FileSystemOptions.FileSystemRoot, globals.FileSystemOptions.DicomSearchPattern, RabbitMqAdapter.SetupProducer(globals.ProcessDirectoryOptions.AccessionDirectoryProducerOptions, isBatch: false)); } else if (cliOptions.DirectoryFormat.ToLower().Equals("default")) { Logger.Info("Creating basic directory finder"); _ddf = new BasicDicomDirectoryFinder(globals.FileSystemOptions.FileSystemRoot, globals.FileSystemOptions.DicomSearchPattern, RabbitMqAdapter.SetupProducer(globals.ProcessDirectoryOptions.AccessionDirectoryProducerOptions, isBatch: false)); } else if (cliOptions.DirectoryFormat.ToLower().Equals("zips")) { Logger.Info("Creating zip directory finder"); _ddf = new ZipDicomDirectoryFinder(globals.FileSystemOptions.FileSystemRoot, globals.FileSystemOptions.DicomSearchPattern, RabbitMqAdapter.SetupProducer(globals.ProcessDirectoryOptions.AccessionDirectoryProducerOptions, isBatch: false)); } else { throw new ArgumentException("Could not match directory format " + cliOptions.DirectoryFormat + " to an directory scan implementation"); } }
private void RunTest(DirectoryInfo dir, int numberOfExpectedRows, Action <FileSystemOptions> adjustFileSystemOptions) { TestLogger.Setup(); var logger = LogManager.GetLogger("MicroservicesIntegrationTest"); _globals.FileSystemOptions.FileSystemRoot = TestContext.CurrentContext.TestDirectory; var readFromFatalErrors = new ConsumerOptions { QueueName = "TEST.FatalLoggingQueue" }; ///////////////////////////////////// Directory ////////////////////////// var processDirectoryOptions = new DicomDirectoryProcessorCliOptions(); processDirectoryOptions.ToProcessDir = dir; processDirectoryOptions.DirectoryFormat = "Default"; adjustFileSystemOptions?.Invoke(_globals.FileSystemOptions); //////////////////////////////////////////////// Mongo Db Populator //////////////////////// // Make this a GUID or something, should be unique per test var currentSeriesCollectionName = "Integration_HappyPath_Series" + DateTime.Now.Ticks; var currentImageCollectionName = "Integration_HappyPath_Image" + DateTime.Now.Ticks; _globals.MongoDbPopulatorOptions.SeriesCollection = currentSeriesCollectionName; _globals.MongoDbPopulatorOptions.ImageCollection = currentImageCollectionName; //use the test catalogue not the one in the combined app.config _globals.RDMPOptions.CatalogueConnectionString = ((TableRepository)RepositoryLocator.CatalogueRepository).DiscoveredServer.Builder.ConnectionString; _globals.RDMPOptions.DataExportConnectionString = ((TableRepository)RepositoryLocator.DataExportRepository).DiscoveredServer.Builder.ConnectionString; _globals.DicomRelationalMapperOptions.RunChecks = true; if (_globals.DicomRelationalMapperOptions.MinimumBatchSize < 1) { _globals.DicomRelationalMapperOptions.MinimumBatchSize = 1; } using (var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.CohortExtractorOptions)) { tester.CreateExchange(_globals.ProcessDirectoryOptions.AccessionDirectoryProducerOptions.ExchangeName, _globals.DicomTagReaderOptions.QueueName); tester.CreateExchange(_globals.DicomTagReaderOptions.SeriesProducerOptions.ExchangeName, _globals.MongoDbPopulatorOptions.SeriesQueueConsumerOptions.QueueName); tester.CreateExchange(_globals.DicomTagReaderOptions.ImageProducerOptions.ExchangeName, _globals.IdentifierMapperOptions.QueueName); tester.CreateExchange(_globals.DicomTagReaderOptions.ImageProducerOptions.ExchangeName, _globals.MongoDbPopulatorOptions.ImageQueueConsumerOptions.QueueName, true); tester.CreateExchange(_globals.IdentifierMapperOptions.AnonImagesProducerOptions.ExchangeName, _globals.DicomRelationalMapperOptions.QueueName); tester.CreateExchange(_globals.RabbitOptions.FatalLoggingExchange, readFromFatalErrors.QueueName); tester.CreateExchange(_globals.CohortExtractorOptions.ExtractFilesProducerOptions.ExchangeName, null, false, _globals.CohortExtractorOptions.ExtractIdentRoutingKey); tester.CreateExchange(_globals.CohortExtractorOptions.ExtractFilesProducerOptions.ExchangeName, null, true, _globals.CohortExtractorOptions.ExtractAnonRoutingKey); tester.CreateExchange(_globals.CohortExtractorOptions.ExtractFilesInfoProducerOptions.ExchangeName, null); #region Running Microservices var processDirectory = new DicomDirectoryProcessorHost(_globals, processDirectoryOptions); processDirectory.Start(); tester.StopOnDispose.Add(processDirectory); var dicomTagReaderHost = new DicomTagReaderHost(_globals); dicomTagReaderHost.Start(); tester.StopOnDispose.Add(dicomTagReaderHost); var mongoDbPopulatorHost = new MongoDbPopulatorHost(_globals); mongoDbPopulatorHost.Start(); tester.StopOnDispose.Add(mongoDbPopulatorHost); var identifierMapperHost = new IdentifierMapperHost(_globals, new SwapForFixedValueTester("FISHFISH")); identifierMapperHost.Start(); tester.StopOnDispose.Add(identifierMapperHost); new TestTimelineAwaiter().Await(() => dicomTagReaderHost.AccessionDirectoryMessageConsumer.AckCount >= 1); logger.Info("\n### DicomTagReader has processed its messages ###\n"); // FIXME: This isn't exactly how the pipeline runs new TestTimelineAwaiter().Await(() => identifierMapperHost.Consumer.AckCount >= 1); logger.Info("\n### IdentifierMapper has processed its messages ###\n"); using (var relationalMapperHost = new DicomRelationalMapperHost(_globals)) { var start = DateTime.Now; relationalMapperHost.Start(); tester.StopOnDispose.Add(relationalMapperHost); Assert.True(mongoDbPopulatorHost.Consumers.Count == 2); new TestTimelineAwaiter().Await(() => mongoDbPopulatorHost.Consumers[0].Processor.AckCount >= 1); new TestTimelineAwaiter().Await(() => mongoDbPopulatorHost.Consumers[1].Processor.AckCount >= 1); logger.Info("\n### MongoDbPopulator has processed its messages ###\n"); new TestTimelineAwaiter().Await(() => identifierMapperHost.Consumer.AckCount >= 1);//number of series logger.Info("\n### IdentifierMapper has processed its messages ###\n"); Assert.AreEqual(0, dicomTagReaderHost.AccessionDirectoryMessageConsumer.NackCount); Assert.AreEqual(0, identifierMapperHost.Consumer.NackCount); Assert.AreEqual(0, ((Consumer <SeriesMessage>)mongoDbPopulatorHost.Consumers[0]).NackCount); Assert.AreEqual(0, ((Consumer <DicomFileMessage>)mongoDbPopulatorHost.Consumers[1]).NackCount); try { Thread.Sleep(TimeSpan.FromSeconds(10)); new TestTimelineAwaiter().Await(() => relationalMapperHost.Consumer.AckCount >= numberOfExpectedRows, null, 30000, () => relationalMapperHost.Consumer.DleErrors); //number of image files logger.Info("\n### DicomRelationalMapper has processed its messages ###\n"); } finally { //find out what happens from the logging database var rdmpLogging = new Rdmp.Core.Logging.LogManager(_helper.LoadMetadata.GetDistinctLoggingDatabase()); //if error was reported during the dicom relational mapper run foreach (var dli in rdmpLogging.GetArchivalDataLoadInfos(_helper.LoadMetadata.GetDistinctLoggingTask(), null, null)) { if (dli.StartTime > start) { foreach (ArchivalFatalError e in dli.Errors) { logger.Error(e.Date.TimeOfDay + ":" + e.Source + ":" + e.Description); } } } } Assert.AreEqual(numberOfExpectedRows, _helper.ImageTable.GetRowCount(), "All images should appear in the image table"); Assert.LessOrEqual(_helper.SeriesTable.GetRowCount(), numberOfExpectedRows, "Only unique series data should appear in series table, there should be less unique series than images (or equal)"); Assert.LessOrEqual(_helper.StudyTable.GetRowCount(), numberOfExpectedRows, "Only unique study data should appear in study table, there should be less unique studies than images (or equal)"); Assert.LessOrEqual(_helper.StudyTable.GetRowCount(), _helper.SeriesTable.GetRowCount(), "There should be less studies than series (or equal)"); //make sure that the substitution identifier (that replaces old the PatientId) is the correct substitution (FISHFISH)/ Assert.AreEqual("FISHFISH", _helper.StudyTable.GetDataTable().Rows.OfType <DataRow>().First()["PatientId"]); //The file size in the final table should be more than 0 Assert.Greater((long)_helper.ImageTable.GetDataTable().Rows.OfType <DataRow>().First()["DicomFileSize"], 0); dicomTagReaderHost.Stop("TestIsFinished"); mongoDbPopulatorHost.Stop("TestIsFinished"); DropMongoTestDb(_globals.MongoDatabases.DicomStoreOptions.HostName, _globals.MongoDatabases.DicomStoreOptions.Port); identifierMapperHost.Stop("TestIsFinished"); relationalMapperHost.Stop("Test end"); } //Now do extraction var extractorHost = new CohortExtractorHost(_globals, null, null); extractorHost.Start(); var extract = new ExtractionRequestMessage { ExtractionJobIdentifier = Guid.NewGuid(), ProjectNumber = "1234-5678", ExtractionDirectory = "1234-5678_P1", KeyTag = "SeriesInstanceUID", }; foreach (DataRow row in _helper.ImageTable.GetDataTable().Rows) { var ser = (string)row["SeriesInstanceUID"]; if (!extract.ExtractionIdentifiers.Contains(ser)) { extract.ExtractionIdentifiers.Add(ser); } } tester.SendMessage(_globals.CohortExtractorOptions, extract); //wait till extractor picked up the messages and dispatched the responses new TestTimelineAwaiter().Await(() => extractorHost.Consumer.AckCount == 1); extractorHost.Stop("TestIsFinished"); tester.Shutdown(); } #endregion }