private void CreateNewLoggingTaskFor(ICheckNotifier notifier, ICatalogue[] catalogues, string proposedName) { var catarepo = _loadMetadata.CatalogueRepository; var serverIds = catalogues.Select(c => c.LiveLoggingServer_ID).Where(i => i.HasValue).Distinct().ToArray(); IExternalDatabaseServer loggingServer; if (serverIds.Any()) { //a server is specified if (serverIds.Length != 1) { throw new Exception("Catalogues do not agree on a single logging server"); } //we checked for HasValue above see the WHERE in the linq loggingServer = catarepo.GetObjectByID <ExternalDatabaseServer>(serverIds[0].Value); } else { loggingServer = catarepo.GetServerDefaults().GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID); if (loggingServer == null) { throw new Exception("There is no default logging server!"); } } var logManager = new LogManager(loggingServer); logManager.CreateNewLoggingTaskIfNotExists(proposedName); notifier.OnCheckPerformed(new CheckEventArgs("Created Logging Task '" + proposedName + "'", CheckResult.Success)); foreach (Catalogue catalogue in catalogues.Cast <Catalogue>()) { catalogue.LiveLoggingServer_ID = loggingServer.ID; catalogue.LoggingDataTask = proposedName; catalogue.SaveToDatabase(); } }
public void Check(ICheckNotifier notifier) { var catalogues = _loadMetadata.GetAllCatalogues().ToArray(); //if there are no logging tasks defined on any Catalogues if (catalogues.Any() && catalogues.All(c => string.IsNullOrWhiteSpace(c.LoggingDataTask))) { string proposedName; bool fix; if (catalogues.Length == 1) { proposedName = "Loading '" + catalogues[0] + "'"; fix = notifier.OnCheckPerformed( new CheckEventArgs( "Catalogue " + catalogues[0] + " does not have a logging task specified", CheckResult.Fail, null, "Create a new Logging Task called '" + proposedName + "'?")); } else { proposedName = _loadMetadata.Name; fix = notifier.OnCheckPerformed( new CheckEventArgs( "Catalogues " + string.Join(",", catalogues.Select(c => c.Name)) + " do not have a logging task specified", CheckResult.Fail, null, "Create a new Logging Task called '" + proposedName + "'?")); } if (fix) { CreateNewLoggingTaskFor(notifier, catalogues, proposedName); } else { return; } } #region Fix missing LoggingDataTask var missingTasks = catalogues.Where(c => string.IsNullOrWhiteSpace(c.LoggingDataTask)).ToArray(); var potentialTasks = catalogues.Except(missingTasks).Select(c => c.LoggingDataTask).Distinct().ToArray(); //If any Catalogues are missing tasks if (missingTasks.Any()) { //but there is consensus for those that are not missing tasks if (potentialTasks.Length == 1) { var fix = notifier.OnCheckPerformed(new CheckEventArgs("Some catalogues have NULL LoggingDataTasks", CheckResult.Fail, null, $"Set task to {potentialTasks.Single()}")); if (fix) { foreach (var cata in missingTasks) { cata.LoggingDataTask = potentialTasks.Single(); cata.SaveToDatabase(); } } } } #endregion #region Fix missing LiveLoggingServer_ID var missingServer = catalogues.Where(c => c.LiveLoggingServer_ID == null).ToArray(); var potentialServer = catalogues.Except(missingServer).Select(c => c.LiveLoggingServer_ID).Distinct().ToArray(); if (missingServer.Any()) { if (potentialServer.Length == 1) { var fix = notifier.OnCheckPerformed(new CheckEventArgs("Some catalogues have NULL LiveLoggingServer_ID", CheckResult.Fail, null, $"Set LiveLoggingServer_ID to {potentialServer.Single()}")); if (fix) { foreach (var cata in missingServer) { cata.LiveLoggingServer_ID = potentialServer.Single(); cata.SaveToDatabase(); } } } else { var defaults = _loadMetadata.CatalogueRepository.GetServerDefaults(); var defaultLoggingServer = defaults.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID); if (defaultLoggingServer != null) { var fix = notifier.OnCheckPerformed(new CheckEventArgs("Some catalogues have NULL LiveLoggingServer_ID", CheckResult.Fail, null, $"Set LiveLoggingServer_ID to '{defaultLoggingServer}' (the default)")); if (fix) { foreach (var cata in missingServer) { cata.LiveLoggingServer_ID = defaultLoggingServer.ID; cata.SaveToDatabase(); } } } } } #endregion string distinctLoggingTask = null; try { distinctLoggingTask = _loadMetadata.GetDistinctLoggingTask(); notifier.OnCheckPerformed(new CheckEventArgs("All Catalogues agreed on a single Logging Task:" + distinctLoggingTask, CheckResult.Success, null)); } catch (Exception e) { notifier.OnCheckPerformed(new CheckEventArgs("Catalogues could not agreed on a single Logging Task", CheckResult.Fail, e)); } try { var settings = _loadMetadata.GetDistinctLoggingDatabase(); settings.TestConnection(); notifier.OnCheckPerformed(new CheckEventArgs("Connected to logging architecture successfully", CheckResult.Success, null)); if (distinctLoggingTask != null) { LogManager lm = new LogManager(settings); string[] dataTasks = lm.ListDataTasks(); if (dataTasks.Contains(distinctLoggingTask)) { notifier.OnCheckPerformed(new CheckEventArgs("Found Logging Task " + distinctLoggingTask + " in Logging database", CheckResult.Success, null)); } else { var fix = notifier.OnCheckPerformed(new CheckEventArgs("Could not find Logging Task " + distinctLoggingTask + " in Logging database", CheckResult.Fail, null, "Create Logging Task '" + distinctLoggingTask + "'")); if (fix) { lm.CreateNewLoggingTaskIfNotExists(distinctLoggingTask); } } } } catch (Exception e) { notifier.OnCheckPerformed(new CheckEventArgs("Could reach default logging server", CheckResult.Fail, e)); } }
public void Check(ICheckNotifier notifier) { var catalogues = _loadMetadata.GetAllCatalogues().ToArray(); //if there are no logging tasks defined on any Catalogues if (catalogues.Any() && catalogues.All(c => string.IsNullOrWhiteSpace(c.LoggingDataTask))) { string proposedName; bool fix; if (catalogues.Length == 1) { proposedName = "Loading '" + catalogues[0] + "'"; fix = notifier.OnCheckPerformed( new CheckEventArgs( "Catalogue " + catalogues[0] + " does not have a logging task specified", CheckResult.Fail, null, "Create a new Logging Task called '" + proposedName + "'?")); } else { proposedName = _loadMetadata.Name; fix = notifier.OnCheckPerformed( new CheckEventArgs( "Catalogues " + string.Join(",", catalogues.Select(c => c.Name)) + " do not have a logging task specified", CheckResult.Fail, null, "Create a new Logging Task called '" + proposedName + "'?")); } if (fix) { CreateNewLoggingTaskFor(notifier, catalogues, proposedName); } else { return; } } string distinctLoggingTask = null; try { distinctLoggingTask = _loadMetadata.GetDistinctLoggingTask(); notifier.OnCheckPerformed(new CheckEventArgs("All Catalogues agreed on a single Logging Task:" + distinctLoggingTask, CheckResult.Success, null)); } catch (Exception e) { notifier.OnCheckPerformed(new CheckEventArgs("Catalogues could not agreed on a single Logging Task", CheckResult.Fail, e)); } try { var settings = _loadMetadata.GetDistinctLoggingDatabase(); settings.TestConnection(); notifier.OnCheckPerformed(new CheckEventArgs("Connected to logging architecture successfully", CheckResult.Success, null)); if (distinctLoggingTask != null) { LogManager lm = new LogManager(settings); string[] dataTasks = lm.ListDataTasks(); if (dataTasks.Contains(distinctLoggingTask)) { notifier.OnCheckPerformed(new CheckEventArgs("Found Logging Task " + distinctLoggingTask + " in Logging database", CheckResult.Success, null)); } else { var fix = notifier.OnCheckPerformed(new CheckEventArgs("Could not find Logging Task " + distinctLoggingTask + " in Logging database", CheckResult.Fail, null, "Create Logging Task '" + distinctLoggingTask + "'")); if (fix) { lm.CreateNewLoggingTaskIfNotExists(distinctLoggingTask); } } } } catch (Exception e) { notifier.OnCheckPerformed(new CheckEventArgs("Could reach default logging server", CheckResult.Fail, e)); } }
private void RunTest(DirectoryInfo dir, int numberOfExpectedRows, Action <FileSystemOptions> adjustFileSystemOptions) { TestLogger.Setup(); var logger = LogManager.GetLogger("MicroservicesIntegrationTest"); _globals.FileSystemOptions.FileSystemRoot = TestContext.CurrentContext.TestDirectory; var readFromFatalErrors = new ConsumerOptions { QueueName = "TEST.FatalLoggingQueue" }; ///////////////////////////////////// Directory ////////////////////////// var processDirectoryOptions = new DicomDirectoryProcessorCliOptions(); processDirectoryOptions.ToProcessDir = dir; processDirectoryOptions.DirectoryFormat = "Default"; adjustFileSystemOptions?.Invoke(_globals.FileSystemOptions); //////////////////////////////////////////////// Mongo Db Populator //////////////////////// // Make this a GUID or something, should be unique per test var currentSeriesCollectionName = "Integration_HappyPath_Series" + DateTime.Now.Ticks; var currentImageCollectionName = "Integration_HappyPath_Image" + DateTime.Now.Ticks; _globals.MongoDbPopulatorOptions.SeriesCollection = currentSeriesCollectionName; _globals.MongoDbPopulatorOptions.ImageCollection = currentImageCollectionName; //use the test catalogue not the one in the combined app.config _globals.RDMPOptions.CatalogueConnectionString = ((TableRepository)RepositoryLocator.CatalogueRepository).DiscoveredServer.Builder.ConnectionString; _globals.RDMPOptions.DataExportConnectionString = ((TableRepository)RepositoryLocator.DataExportRepository).DiscoveredServer.Builder.ConnectionString; _globals.DicomRelationalMapperOptions.RunChecks = true; if (_globals.DicomRelationalMapperOptions.MinimumBatchSize < 1) { _globals.DicomRelationalMapperOptions.MinimumBatchSize = 1; } using (var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.CohortExtractorOptions)) { tester.CreateExchange(_globals.ProcessDirectoryOptions.AccessionDirectoryProducerOptions.ExchangeName, _globals.DicomTagReaderOptions.QueueName); tester.CreateExchange(_globals.DicomTagReaderOptions.SeriesProducerOptions.ExchangeName, _globals.MongoDbPopulatorOptions.SeriesQueueConsumerOptions.QueueName); tester.CreateExchange(_globals.DicomTagReaderOptions.ImageProducerOptions.ExchangeName, _globals.IdentifierMapperOptions.QueueName); tester.CreateExchange(_globals.DicomTagReaderOptions.ImageProducerOptions.ExchangeName, _globals.MongoDbPopulatorOptions.ImageQueueConsumerOptions.QueueName, true); tester.CreateExchange(_globals.IdentifierMapperOptions.AnonImagesProducerOptions.ExchangeName, _globals.DicomRelationalMapperOptions.QueueName); tester.CreateExchange(_globals.RabbitOptions.FatalLoggingExchange, readFromFatalErrors.QueueName); tester.CreateExchange(_globals.CohortExtractorOptions.ExtractFilesProducerOptions.ExchangeName, null, false, _globals.CohortExtractorOptions.ExtractIdentRoutingKey); tester.CreateExchange(_globals.CohortExtractorOptions.ExtractFilesProducerOptions.ExchangeName, null, true, _globals.CohortExtractorOptions.ExtractAnonRoutingKey); tester.CreateExchange(_globals.CohortExtractorOptions.ExtractFilesInfoProducerOptions.ExchangeName, null); #region Running Microservices var processDirectory = new DicomDirectoryProcessorHost(_globals, processDirectoryOptions); processDirectory.Start(); tester.StopOnDispose.Add(processDirectory); var dicomTagReaderHost = new DicomTagReaderHost(_globals); dicomTagReaderHost.Start(); tester.StopOnDispose.Add(dicomTagReaderHost); var mongoDbPopulatorHost = new MongoDbPopulatorHost(_globals); mongoDbPopulatorHost.Start(); tester.StopOnDispose.Add(mongoDbPopulatorHost); var identifierMapperHost = new IdentifierMapperHost(_globals, new SwapForFixedValueTester("FISHFISH")); identifierMapperHost.Start(); tester.StopOnDispose.Add(identifierMapperHost); new TestTimelineAwaiter().Await(() => dicomTagReaderHost.AccessionDirectoryMessageConsumer.AckCount >= 1); logger.Info("\n### DicomTagReader has processed its messages ###\n"); // FIXME: This isn't exactly how the pipeline runs new TestTimelineAwaiter().Await(() => identifierMapperHost.Consumer.AckCount >= 1); logger.Info("\n### IdentifierMapper has processed its messages ###\n"); using (var relationalMapperHost = new DicomRelationalMapperHost(_globals)) { var start = DateTime.Now; relationalMapperHost.Start(); tester.StopOnDispose.Add(relationalMapperHost); Assert.True(mongoDbPopulatorHost.Consumers.Count == 2); new TestTimelineAwaiter().Await(() => mongoDbPopulatorHost.Consumers[0].Processor.AckCount >= 1); new TestTimelineAwaiter().Await(() => mongoDbPopulatorHost.Consumers[1].Processor.AckCount >= 1); logger.Info("\n### MongoDbPopulator has processed its messages ###\n"); new TestTimelineAwaiter().Await(() => identifierMapperHost.Consumer.AckCount >= 1);//number of series logger.Info("\n### IdentifierMapper has processed its messages ###\n"); Assert.AreEqual(0, dicomTagReaderHost.AccessionDirectoryMessageConsumer.NackCount); Assert.AreEqual(0, identifierMapperHost.Consumer.NackCount); Assert.AreEqual(0, ((Consumer <SeriesMessage>)mongoDbPopulatorHost.Consumers[0]).NackCount); Assert.AreEqual(0, ((Consumer <DicomFileMessage>)mongoDbPopulatorHost.Consumers[1]).NackCount); try { Thread.Sleep(TimeSpan.FromSeconds(10)); new TestTimelineAwaiter().Await(() => relationalMapperHost.Consumer.AckCount >= numberOfExpectedRows, null, 30000, () => relationalMapperHost.Consumer.DleErrors); //number of image files logger.Info("\n### DicomRelationalMapper has processed its messages ###\n"); } finally { //find out what happens from the logging database var rdmpLogging = new Rdmp.Core.Logging.LogManager(_helper.LoadMetadata.GetDistinctLoggingDatabase()); //if error was reported during the dicom relational mapper run foreach (var dli in rdmpLogging.GetArchivalDataLoadInfos(_helper.LoadMetadata.GetDistinctLoggingTask(), null, null)) { if (dli.StartTime > start) { foreach (ArchivalFatalError e in dli.Errors) { logger.Error(e.Date.TimeOfDay + ":" + e.Source + ":" + e.Description); } } } } Assert.AreEqual(numberOfExpectedRows, _helper.ImageTable.GetRowCount(), "All images should appear in the image table"); Assert.LessOrEqual(_helper.SeriesTable.GetRowCount(), numberOfExpectedRows, "Only unique series data should appear in series table, there should be less unique series than images (or equal)"); Assert.LessOrEqual(_helper.StudyTable.GetRowCount(), numberOfExpectedRows, "Only unique study data should appear in study table, there should be less unique studies than images (or equal)"); Assert.LessOrEqual(_helper.StudyTable.GetRowCount(), _helper.SeriesTable.GetRowCount(), "There should be less studies than series (or equal)"); //make sure that the substitution identifier (that replaces old the PatientId) is the correct substitution (FISHFISH)/ Assert.AreEqual("FISHFISH", _helper.StudyTable.GetDataTable().Rows.OfType <DataRow>().First()["PatientId"]); //The file size in the final table should be more than 0 Assert.Greater((long)_helper.ImageTable.GetDataTable().Rows.OfType <DataRow>().First()["DicomFileSize"], 0); dicomTagReaderHost.Stop("TestIsFinished"); mongoDbPopulatorHost.Stop("TestIsFinished"); DropMongoTestDb(_globals.MongoDatabases.DicomStoreOptions.HostName, _globals.MongoDatabases.DicomStoreOptions.Port); identifierMapperHost.Stop("TestIsFinished"); relationalMapperHost.Stop("Test end"); } //Now do extraction var extractorHost = new CohortExtractorHost(_globals, null, null); extractorHost.Start(); var extract = new ExtractionRequestMessage { ExtractionJobIdentifier = Guid.NewGuid(), ProjectNumber = "1234-5678", ExtractionDirectory = "1234-5678_P1", KeyTag = "SeriesInstanceUID", }; foreach (DataRow row in _helper.ImageTable.GetDataTable().Rows) { var ser = (string)row["SeriesInstanceUID"]; if (!extract.ExtractionIdentifiers.Contains(ser)) { extract.ExtractionIdentifiers.Add(ser); } } tester.SendMessage(_globals.CohortExtractorOptions, extract); //wait till extractor picked up the messages and dispatched the responses new TestTimelineAwaiter().Await(() => extractorHost.Consumer.AckCount == 1); extractorHost.Stop("TestIsFinished"); tester.Shutdown(); } #endregion }