public void TestBasicOperation() { _helper.Options.FileSystemOptions.FileSystemRoot = _helper.TestDir.FullName; _helper.TestAccessionDirectoryMessage.DirectoryPath = _helper.TestDir.FullName; var tester = new MicroserviceTester(_helper.Options.RabbitOptions, _helper.AccessionConsumerOptions); var host = new DicomTagReaderHost(_helper.Options); host.Start(); tester.SendMessage(_helper.AccessionConsumerOptions, new MessageHeader(), _helper.TestAccessionDirectoryMessage); var timeout = 30000; const int stepSize = 500; while (!_helper.CheckQueues(1, 1) && timeout > 0) { timeout -= 500; Thread.Sleep(stepSize); } host.Stop("Test end"); tester.Dispose(); if (timeout <= 0) { Assert.Fail("Failed to process expected number of messages within the timeout"); } }
public void TestClassifierName_ValidClassifier() { var options = new GlobalOptionsFactory().Load(); var testDcm = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(TestClassifierName_ValidClassifier), "f1.dcm")); Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(TestClassifierName_ValidClassifier), "f1.dcm"); TestData.Create(testDcm); using (var tester = new MicroserviceTester(options.RabbitOptions, options.IsIdentifiableOptions)) { tester.CreateExchange(options.IsIdentifiableOptions.IsIdentifiableProducerOptions.ExchangeName, null); options.IsIdentifiableOptions.ClassifierType = typeof(RejectAllClassifier).FullName; options.IsIdentifiableOptions.DataDirectory = TestContext.CurrentContext.TestDirectory; var host = new IsIdentifiableHost(options, new IsIdentifiableServiceOptions()); Assert.IsNotNull(host); host.Start(); tester.SendMessage(options.IsIdentifiableOptions, new ExtractedFileStatusMessage() { DicomFilePath = "yay.dcm", OutputFilePath = testDcm.FullName, ProjectNumber = "100", ExtractionDirectory = "./fish", StatusMessage = "yay!", Status = ExtractedFileStatus.Anonymised }); var awaiter = new TestTimelineAwaiter(); awaiter.Await(() => host.Consumer.AckCount == 1); } }
public void Test_FileCopierHost_HappyPath() { GlobalOptions globals = new GlobalOptionsFactory().Load(); globals.FileSystemOptions.FileSystemRoot = "root"; globals.FileSystemOptions.ExtractRoot = "exroot"; using var tester = new MicroserviceTester(globals.RabbitOptions, globals.FileCopierOptions); string outputQueueName = globals.FileCopierOptions.CopyStatusProducerOptions.ExchangeName.Replace("Exchange", "Queue"); tester.CreateExchange( globals.FileCopierOptions.CopyStatusProducerOptions.ExchangeName, outputQueueName, false, globals.FileCopierOptions.NoVerifyRoutingKey); var mockFileSystem = new MockFileSystem(); mockFileSystem.AddDirectory(globals.FileSystemOptions.FileSystemRoot); mockFileSystem.AddDirectory(globals.FileSystemOptions.ExtractRoot); mockFileSystem.AddFile(mockFileSystem.Path.Combine(globals.FileSystemOptions.FileSystemRoot, "file.dcm"), MockFileData.NullObject); var host = new FileCopierHost(globals, mockFileSystem); tester.StopOnDispose.Add(host); host.Start(); var message = new ExtractFileMessage { ExtractionJobIdentifier = Guid.NewGuid(), JobSubmittedAt = DateTime.UtcNow, ProjectNumber = "1234", ExtractionDirectory = "1234/foo", DicomFilePath = "file.dcm", IsIdentifiableExtraction = true, OutputPath = "output.dcm", }; tester.SendMessage(globals.FileCopierOptions, message); using IConnection conn = tester.Factory.CreateConnection(); using IModel model = conn.CreateModel(); var consumer = new EventingBasicConsumer(model); ExtractedFileStatusMessage statusMessage = null; consumer.Received += (_, ea) => statusMessage = JsonConvert.DeserializeObject <ExtractedFileStatusMessage>(Encoding.UTF8.GetString(ea.Body.ToArray())); model.BasicConsume(outputQueueName, true, "", consumer); new TestTimelineAwaiter().Await(() => statusMessage != null); Assert.AreEqual(ExtractedFileStatus.Copied, statusMessage.Status); }
public void IdenticalDatasetsTest() { _helper.TruncateTablesIfExists(); var ds = new DicomDataset(); ds.AddOrUpdate(DicomTag.SeriesInstanceUID, "123"); ds.AddOrUpdate(DicomTag.SOPInstanceUID, "123"); ds.AddOrUpdate(DicomTag.StudyInstanceUID, "123"); ds.AddOrUpdate(DicomTag.PatientID, "123"); var msg1 = _helper.GetDicomFileMessage(ds, _globals.FileSystemOptions.FileSystemRoot, Path.Combine(_globals.FileSystemOptions.FileSystemRoot, "mydicom.dcm")); var msg2 = _helper.GetDicomFileMessage(ds, _globals.FileSystemOptions.FileSystemRoot, Path.Combine(_globals.FileSystemOptions.FileSystemRoot, "mydicom.dcm")); //creates the queues, exchanges and bindings using (var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.DicomRelationalMapperOptions)) { tester.CreateExchange(_globals.RabbitOptions.FatalLoggingExchange, null); tester.SendMessage(_globals.DicomRelationalMapperOptions, msg1); tester.SendMessage(_globals.DicomRelationalMapperOptions, msg2); _globals.DicomRelationalMapperOptions.RunChecks = true; using (var host = new DicomRelationalMapperHost(_globals)) { host.Start(); new TestTimelineAwaiter().Await(() => host.Consumer.MessagesProcessed == 2, null, 30000, () => host.Consumer.DleErrors); Assert.GreaterOrEqual(1, _helper.SeriesTable.GetRowCount(), "SeriesTable did not have the expected number of rows in LIVE"); Assert.GreaterOrEqual(1, _helper.StudyTable.GetRowCount(), "StudyTable did not have the expected number of rows in LIVE"); Assert.AreEqual(1, _helper.ImageTable.GetRowCount(), "ImageTable did not have the expected number of rows in LIVE"); host.Stop("Test end"); } tester.Shutdown(); } }
public void TestIsIdentifiable_TesseractStanfordDicomFileClassifier() { var options = new GlobalOptionsFactory().Load(); // Create a test data directory containing IsIdentifiableRules with 0 rules, and tessdata with the eng.traineddata classifier // TODO(rkm 2020-04-14) This is a stop-gap solution until the tests are properly refactored var testRulesDir = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, "data", "IsIdentifiableRules")); testRulesDir.Create(); options.IsIdentifiableOptions.DataDirectory = testRulesDir.Parent.FullName; var tessDir = new DirectoryInfo(Path.Combine(testRulesDir.Parent.FullName, "tessdata")); tessDir.Create(); var dest = Path.Combine(tessDir.FullName, "eng.traineddata"); if (!File.Exists(dest)) { File.Copy(Path.Combine(DataDirectory, "tessdata", "eng.traineddata"), dest); } var testDcm = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(TestIsIdentifiable_TesseractStanfordDicomFileClassifier), "f1.dcm")); Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(TestClassifierName_ValidClassifier), "f1.dcm"); TestData.Create(testDcm); using (var tester = new MicroserviceTester(options.RabbitOptions, options.IsIdentifiableOptions)) { options.IsIdentifiableOptions.ClassifierType = typeof(TesseractStanfordDicomFileClassifier).FullName; var host = new IsIdentifiableHost(options, new IsIdentifiableServiceOptions()); host.Start(); tester.SendMessage(options.IsIdentifiableOptions, new ExtractedFileStatusMessage { DicomFilePath = "yay.dcm", OutputFilePath = testDcm.FullName, ProjectNumber = "100", ExtractionDirectory = "./fish", StatusMessage = "yay!", Status = ExtractedFileStatus.Anonymised }); var awaiter = new TestTimelineAwaiter(); awaiter.Await(() => host.Consumer.AckCount == 1 || host.Consumer.NackCount == 1); Assert.AreEqual(1, host.Consumer.AckCount, "Tesseract not acking"); } }
private void VerifyReports(GlobalOptions globals, PathFixtures pf, ReportFormat reportFormat, IEnumerable <Tuple <ConsumerOptions, IMessage> > toSend) { globals.FileSystemOptions.ExtractRoot = pf.ExtractRootAbsolute; globals.CohortPackagerOptions.JobWatcherTimeoutInSeconds = 5; globals.CohortPackagerOptions.ReporterType = "FileReporter"; globals.CohortPackagerOptions.ReportFormat = reportFormat.ToString(); MongoClient client = MongoClientHelpers.GetMongoClient(globals.MongoDatabases.ExtractionStoreOptions, "test", true); globals.MongoDatabases.ExtractionStoreOptions.DatabaseName += "-" + Guid.NewGuid().ToString().Split('-')[0]; client.DropDatabase(globals.MongoDatabases.ExtractionStoreOptions.DatabaseName); using (var tester = new MicroserviceTester( globals.RabbitOptions, globals.CohortPackagerOptions.ExtractRequestInfoOptions, globals.CohortPackagerOptions.FileCollectionInfoOptions, globals.CohortPackagerOptions.NoVerifyStatusOptions, globals.CohortPackagerOptions.VerificationStatusOptions)) { foreach ((ConsumerOptions consumerOptions, IMessage message) in toSend) { tester.SendMessage(consumerOptions, new MessageHeader(), message); } var host = new CohortPackagerHost(globals); host.Start(); var timeoutSecs = 10; while (!HaveFiles(pf) && timeoutSecs > 0) { --timeoutSecs; Thread.Sleep(TimeSpan.FromSeconds(1)); } host.Stop("Test end"); } var firstLine = $"# SMI extraction validation report for testProj1/{pf.ExtractName}"; switch (reportFormat) { case ReportFormat.Combined: { string reportContent = File.ReadAllText(Path.Combine(pf.ProjReportsDirAbsolute, $"{pf.ExtractName}_report.txt")); Assert.True(reportContent.StartsWith(firstLine)); break; } case ReportFormat.Split: { string extractReportsDirAbsolute = Path.Combine(pf.ProjReportsDirAbsolute, pf.ExtractName); Assert.AreEqual(6, Directory.GetFiles(extractReportsDirAbsolute).Length); string reportContent = File.ReadAllText(Path.Combine(extractReportsDirAbsolute, "README.md")); Assert.True(reportContent.StartsWith(firstLine)); break; } default: Assert.Fail($"No case for ReportFormat {reportFormat}"); break; } }
private void RunTest(DirectoryInfo dir, int numberOfExpectedRows, Action <FileSystemOptions> adjustFileSystemOptions) { TestLogger.Setup(); var logger = LogManager.GetLogger("MicroservicesIntegrationTest"); _globals.FileSystemOptions.FileSystemRoot = TestContext.CurrentContext.TestDirectory; var readFromFatalErrors = new ConsumerOptions { QueueName = "TEST.FatalLoggingQueue" }; ///////////////////////////////////// Directory ////////////////////////// var processDirectoryOptions = new DicomDirectoryProcessorCliOptions(); processDirectoryOptions.ToProcessDir = dir; processDirectoryOptions.DirectoryFormat = "Default"; adjustFileSystemOptions?.Invoke(_globals.FileSystemOptions); //////////////////////////////////////////////// Mongo Db Populator //////////////////////// // Make this a GUID or something, should be unique per test var currentSeriesCollectionName = "Integration_HappyPath_Series" + DateTime.Now.Ticks; var currentImageCollectionName = "Integration_HappyPath_Image" + DateTime.Now.Ticks; _globals.MongoDbPopulatorOptions.SeriesCollection = currentSeriesCollectionName; _globals.MongoDbPopulatorOptions.ImageCollection = currentImageCollectionName; //use the test catalogue not the one in the combined app.config _globals.RDMPOptions.CatalogueConnectionString = ((TableRepository)RepositoryLocator.CatalogueRepository).DiscoveredServer.Builder.ConnectionString; _globals.RDMPOptions.DataExportConnectionString = ((TableRepository)RepositoryLocator.DataExportRepository).DiscoveredServer.Builder.ConnectionString; _globals.DicomRelationalMapperOptions.RunChecks = true; if (_globals.DicomRelationalMapperOptions.MinimumBatchSize < 1) { _globals.DicomRelationalMapperOptions.MinimumBatchSize = 1; } using (var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.CohortExtractorOptions)) { tester.CreateExchange(_globals.ProcessDirectoryOptions.AccessionDirectoryProducerOptions.ExchangeName, _globals.DicomTagReaderOptions.QueueName); tester.CreateExchange(_globals.DicomTagReaderOptions.SeriesProducerOptions.ExchangeName, _globals.MongoDbPopulatorOptions.SeriesQueueConsumerOptions.QueueName); tester.CreateExchange(_globals.DicomTagReaderOptions.ImageProducerOptions.ExchangeName, _globals.IdentifierMapperOptions.QueueName); tester.CreateExchange(_globals.DicomTagReaderOptions.ImageProducerOptions.ExchangeName, _globals.MongoDbPopulatorOptions.ImageQueueConsumerOptions.QueueName, true); tester.CreateExchange(_globals.IdentifierMapperOptions.AnonImagesProducerOptions.ExchangeName, _globals.DicomRelationalMapperOptions.QueueName); tester.CreateExchange(_globals.RabbitOptions.FatalLoggingExchange, readFromFatalErrors.QueueName); tester.CreateExchange(_globals.CohortExtractorOptions.ExtractFilesProducerOptions.ExchangeName, null, false, _globals.CohortExtractorOptions.ExtractIdentRoutingKey); tester.CreateExchange(_globals.CohortExtractorOptions.ExtractFilesProducerOptions.ExchangeName, null, true, _globals.CohortExtractorOptions.ExtractAnonRoutingKey); tester.CreateExchange(_globals.CohortExtractorOptions.ExtractFilesInfoProducerOptions.ExchangeName, null); #region Running Microservices var processDirectory = new DicomDirectoryProcessorHost(_globals, processDirectoryOptions); processDirectory.Start(); tester.StopOnDispose.Add(processDirectory); var dicomTagReaderHost = new DicomTagReaderHost(_globals); dicomTagReaderHost.Start(); tester.StopOnDispose.Add(dicomTagReaderHost); var mongoDbPopulatorHost = new MongoDbPopulatorHost(_globals); mongoDbPopulatorHost.Start(); tester.StopOnDispose.Add(mongoDbPopulatorHost); var identifierMapperHost = new IdentifierMapperHost(_globals, new SwapForFixedValueTester("FISHFISH")); identifierMapperHost.Start(); tester.StopOnDispose.Add(identifierMapperHost); new TestTimelineAwaiter().Await(() => dicomTagReaderHost.AccessionDirectoryMessageConsumer.AckCount >= 1); logger.Info("\n### DicomTagReader has processed its messages ###\n"); // FIXME: This isn't exactly how the pipeline runs new TestTimelineAwaiter().Await(() => identifierMapperHost.Consumer.AckCount >= 1); logger.Info("\n### IdentifierMapper has processed its messages ###\n"); using (var relationalMapperHost = new DicomRelationalMapperHost(_globals)) { var start = DateTime.Now; relationalMapperHost.Start(); tester.StopOnDispose.Add(relationalMapperHost); Assert.True(mongoDbPopulatorHost.Consumers.Count == 2); new TestTimelineAwaiter().Await(() => mongoDbPopulatorHost.Consumers[0].Processor.AckCount >= 1); new TestTimelineAwaiter().Await(() => mongoDbPopulatorHost.Consumers[1].Processor.AckCount >= 1); logger.Info("\n### MongoDbPopulator has processed its messages ###\n"); new TestTimelineAwaiter().Await(() => identifierMapperHost.Consumer.AckCount >= 1);//number of series logger.Info("\n### IdentifierMapper has processed its messages ###\n"); Assert.AreEqual(0, dicomTagReaderHost.AccessionDirectoryMessageConsumer.NackCount); Assert.AreEqual(0, identifierMapperHost.Consumer.NackCount); Assert.AreEqual(0, ((Consumer <SeriesMessage>)mongoDbPopulatorHost.Consumers[0]).NackCount); Assert.AreEqual(0, ((Consumer <DicomFileMessage>)mongoDbPopulatorHost.Consumers[1]).NackCount); try { Thread.Sleep(TimeSpan.FromSeconds(10)); new TestTimelineAwaiter().Await(() => relationalMapperHost.Consumer.AckCount >= numberOfExpectedRows, null, 30000, () => relationalMapperHost.Consumer.DleErrors); //number of image files logger.Info("\n### DicomRelationalMapper has processed its messages ###\n"); } finally { //find out what happens from the logging database var rdmpLogging = new Rdmp.Core.Logging.LogManager(_helper.LoadMetadata.GetDistinctLoggingDatabase()); //if error was reported during the dicom relational mapper run foreach (var dli in rdmpLogging.GetArchivalDataLoadInfos(_helper.LoadMetadata.GetDistinctLoggingTask(), null, null)) { if (dli.StartTime > start) { foreach (ArchivalFatalError e in dli.Errors) { logger.Error(e.Date.TimeOfDay + ":" + e.Source + ":" + e.Description); } } } } Assert.AreEqual(numberOfExpectedRows, _helper.ImageTable.GetRowCount(), "All images should appear in the image table"); Assert.LessOrEqual(_helper.SeriesTable.GetRowCount(), numberOfExpectedRows, "Only unique series data should appear in series table, there should be less unique series than images (or equal)"); Assert.LessOrEqual(_helper.StudyTable.GetRowCount(), numberOfExpectedRows, "Only unique study data should appear in study table, there should be less unique studies than images (or equal)"); Assert.LessOrEqual(_helper.StudyTable.GetRowCount(), _helper.SeriesTable.GetRowCount(), "There should be less studies than series (or equal)"); //make sure that the substitution identifier (that replaces old the PatientId) is the correct substitution (FISHFISH)/ Assert.AreEqual("FISHFISH", _helper.StudyTable.GetDataTable().Rows.OfType <DataRow>().First()["PatientId"]); //The file size in the final table should be more than 0 Assert.Greater((long)_helper.ImageTable.GetDataTable().Rows.OfType <DataRow>().First()["DicomFileSize"], 0); dicomTagReaderHost.Stop("TestIsFinished"); mongoDbPopulatorHost.Stop("TestIsFinished"); DropMongoTestDb(_globals.MongoDatabases.DicomStoreOptions.HostName, _globals.MongoDatabases.DicomStoreOptions.Port); identifierMapperHost.Stop("TestIsFinished"); relationalMapperHost.Stop("Test end"); } //Now do extraction var extractorHost = new CohortExtractorHost(_globals, null, null); extractorHost.Start(); var extract = new ExtractionRequestMessage { ExtractionJobIdentifier = Guid.NewGuid(), ProjectNumber = "1234-5678", ExtractionDirectory = "1234-5678_P1", KeyTag = "SeriesInstanceUID", }; foreach (DataRow row in _helper.ImageTable.GetDataTable().Rows) { var ser = (string)row["SeriesInstanceUID"]; if (!extract.ExtractionIdentifiers.Contains(ser)) { extract.ExtractionIdentifiers.Add(ser); } } tester.SendMessage(_globals.CohortExtractorOptions, extract); //wait till extractor picked up the messages and dispatched the responses new TestTimelineAwaiter().Await(() => extractorHost.Consumer.AckCount == 1); extractorHost.Stop("TestIsFinished"); tester.Shutdown(); } #endregion }