public void TestLargeImageDatasets(DatabaseType databaseType, int numberOfImages) { foreach (Pipeline p in CatalogueRepository.GetAllObjects <Pipeline>()) { p.DeleteInDatabase(); } var db = GetCleanedServer(databaseType); var d = CatalogueRepository.GetServerDefaults(); d.ClearDefault(PermissableDefaults.RAWDataLoadServer); var template = ImageTableTemplateCollection.LoadFrom(_templateXml); _globals = new GlobalOptionsFactory().Load(); _globals.DicomRelationalMapperOptions.DatabaseNamerType = typeof(MyFixedStagingDatabaseNamer).FullName; _globals.DicomRelationalMapperOptions.QoSPrefetchCount = ushort.MaxValue; _globals.DicomRelationalMapperOptions.MinimumBatchSize = numberOfImages; _globals.DicomRelationalMapperOptions.UseInsertIntoForRAWMigration = true; _helper = new DicomRelationalMapperTestHelper(); _helper.SetupSuite(db, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource), root: null, template: template, persistentRaw: true); //do not use an explicit RAW data load server d.ClearDefault(PermissableDefaults.RAWDataLoadServer); Random r = new Random(123); List <DicomDataset> allImages; using (var generator = new DicomDataGenerator(r, null, "CT")) allImages = generator.GenerateImages(numberOfImages, r); Assert.AreEqual(numberOfImages, allImages.Count); using (var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.DicomRelationalMapperOptions)) { using (var host = new DicomRelationalMapperHost(_globals)) { tester.SendMessages(_globals.DicomRelationalMapperOptions, allImages.Select(GetFileMessageForDataset), true); Console.WriteLine("Starting Host"); host.Start(); Stopwatch sw = Stopwatch.StartNew(); new TestTimelineAwaiter().Await(() => host.Consumer.AckCount == numberOfImages, null, 20 * 60 * 100); //1 minute Console.Write("Time For DLE:" + sw.Elapsed.TotalSeconds + "s"); host.Stop("Test finished"); } } foreach (Pipeline allObject in CatalogueRepository.GetAllObjects <Pipeline>()) { allObject.DeleteInDatabase(); } }
public void Setup() { BlitzMainDataTables(); _globals = new GlobalOptionsFactory().Load(); var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); _helper = new DicomRelationalMapperTestHelper(); _helper.SetupSuite(db, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource)); TestLogger.Setup(); }
public void SetupSuite(DiscoveredDatabase server, bool persistentRaw = false, string modalityPrefix = null) { TestLogger.Setup(); _globals = new GlobalOptionsFactory().Load(); _globals.UseTestValues( RequiresRabbit.GetConnectionFactory(), RequiresMongoDb.GetMongoClientSettings(), RequiresRelationalDb.GetRelationalDatabaseConnectionStrings(), ((TableRepository)RepositoryLocator.CatalogueRepository).ConnectionStringBuilder, ((TableRepository)RepositoryLocator.DataExportRepository).ConnectionStringBuilder); _helper = new DicomRelationalMapperTestHelper(); _helper.SetupSuite(server, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource), null, null, persistentRaw, modalityPrefix); _globals.DicomRelationalMapperOptions.RetryOnFailureCount = 0; _globals.DicomRelationalMapperOptions.RetryDelayInSeconds = 0; //do not use an explicit RAW data load server CatalogueRepository.GetServerDefaults().ClearDefault(PermissableDefaults.RAWDataLoadServer); }
public void TestBulkInsertOnly(DatabaseType databaseType, int numberOfImages) { foreach (Pipeline p in CatalogueRepository.GetAllObjects <Pipeline>()) { p.DeleteInDatabase(); } var db = GetCleanedServer(databaseType); var template = ImageTableTemplateCollection.LoadFrom(_templateXml); _globals = new GlobalOptionsFactory().Load(); _globals.DicomRelationalMapperOptions.DatabaseNamerType = typeof(MyFixedStagingDatabaseNamer).FullName; _globals.DicomRelationalMapperOptions.QoSPrefetchCount = ushort.MaxValue; _globals.DicomRelationalMapperOptions.MinimumBatchSize = numberOfImages; _globals.DicomRelationalMapperOptions.UseInsertIntoForRAWMigration = true; _helper = new DicomRelationalMapperTestHelper(); _helper.SetupSuite(db, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource), root: null, template: template, persistentRaw: true); //do not use an explicit RAW data load server CatalogueRepository.GetServerDefaults().ClearDefault(PermissableDefaults.RAWDataLoadServer); Random r = new Random(123); List <DicomDataset> allImages; using (var generator = new DicomDataGenerator(r, null, "CT")) allImages = generator.GenerateImages(numberOfImages, r); DicomDatasetCollectionSource source = new DicomDatasetCollectionSource(); source.PreInitialize(new ExplicitListDicomDatasetWorklist(allImages.ToArray(), "amagad.dcm", new Dictionary <string, string>() { { "MessageGuid", "0x123" } }), new ThrowImmediatelyDataLoadEventListener());; source.FilenameField = "gggg"; var dt = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); source.Dispose(new ThrowImmediatelyDataLoadEventListener() { WriteToConsole = true }, null); Assert.AreEqual(numberOfImages, allImages.Count); Assert.AreEqual(numberOfImages, dt.Rows.Count); var tables = _helper.LoadMetadata.GetDistinctTableInfoList(false); var config = new HICDatabaseConfiguration(_helper.LoadMetadata, new SuffixBasedNamer()); var job = Mock.Of <IDataLoadJob>( j => j.RegularTablesToLoad == tables.Cast <ITableInfo>().ToList() && j.DataLoadInfo == _dli && j.Configuration == config); var attacher = new AutoRoutingAttacher(); attacher.Job = job; //Drop Primary Keys using (var con = db.Server.GetConnection()) { con.Open(); var cmd = db.Server.GetCommand( databaseType == DatabaseType.MicrosoftSQLServer ? @"ALTER TABLE ImageTable DROP CONSTRAINT PK_ImageTable ALTER TABLE SeriesTable DROP CONSTRAINT PK_SeriesTable ALTER TABLE StudyTable DROP CONSTRAINT PK_StudyTable" : @"ALTER TABLE ImageTable DROP PRIMARY KEY; ALTER TABLE SeriesTable DROP PRIMARY KEY; ALTER TABLE StudyTable DROP PRIMARY KEY;" , con); cmd.ExecuteNonQuery(); } attacher.Initialize(LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "IgnoreMe", true), db); try { attacher.ProcessPipelineData(dt, new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()); attacher.Dispose(new ThrowImmediatelyDataLoadEventListener(), null); } catch (Exception e) { attacher.Dispose(new ThrowImmediatelyDataLoadEventListener(), e); throw; } foreach (var tableInfo in tables) { Assert.AreEqual(numberOfImages, tableInfo.Discover(DataAccessContext.InternalDataProcessing).GetRowCount(), "Row count was wrong for " + tableInfo); } foreach (Pipeline allObject in CatalogueRepository.GetAllObjects <Pipeline>()) { allObject.DeleteInDatabase(); } }