public void TestLargeImageDatasets(DatabaseType databaseType, int numberOfImages)
{
foreach (Pipeline p in CatalogueRepository.GetAllObjects <Pipeline>())
{
p.DeleteInDatabase();
}
var db = GetCleanedServer(databaseType);
var d = CatalogueRepository.GetServerDefaults();
d.ClearDefault(PermissableDefaults.RAWDataLoadServer);
var template = ImageTableTemplateCollection.LoadFrom(_templateXml);
_globals = new GlobalOptionsFactory().Load();
_globals.DicomRelationalMapperOptions.DatabaseNamerType = typeof(MyFixedStagingDatabaseNamer).FullName;
_globals.DicomRelationalMapperOptions.QoSPrefetchCount = ushort.MaxValue;
_globals.DicomRelationalMapperOptions.MinimumBatchSize = numberOfImages;
_globals.DicomRelationalMapperOptions.UseInsertIntoForRAWMigration = true;
_helper = new DicomRelationalMapperTestHelper();
_helper.SetupSuite(db, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource), root: null, template: template, persistentRaw: true);
//do not use an explicit RAW data load server
d.ClearDefault(PermissableDefaults.RAWDataLoadServer);
Random r = new Random(123);
List <DicomDataset> allImages;
using (var generator = new DicomDataGenerator(r, null, "CT"))
allImages = generator.GenerateImages(numberOfImages, r);
Assert.AreEqual(numberOfImages, allImages.Count);
using (var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.DicomRelationalMapperOptions))
{
using (var host = new DicomRelationalMapperHost(_globals))
{
tester.SendMessages(_globals.DicomRelationalMapperOptions, allImages.Select(GetFileMessageForDataset), true);
Console.WriteLine("Starting Host");
host.Start();
Stopwatch sw = Stopwatch.StartNew();
new TestTimelineAwaiter().Await(() => host.Consumer.AckCount == numberOfImages, null, 20 * 60 * 100); //1 minute
Console.Write("Time For DLE:" + sw.Elapsed.TotalSeconds + "s");
host.Stop("Test finished");
}
}
foreach (Pipeline allObject in CatalogueRepository.GetAllObjects <Pipeline>())
{
allObject.DeleteInDatabase();
}
}
public void Setup()
{
BlitzMainDataTables();
_globals = new GlobalOptionsFactory().Load();
var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer);
_helper = new DicomRelationalMapperTestHelper();
_helper.SetupSuite(db, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource));
TestLogger.Setup();
}
public void SetupSuite(DiscoveredDatabase server, bool persistentRaw = false, string modalityPrefix = null)
{
TestLogger.Setup();
_globals = new GlobalOptionsFactory().Load();
_globals.UseTestValues(
RequiresRabbit.GetConnectionFactory(),
RequiresMongoDb.GetMongoClientSettings(),
RequiresRelationalDb.GetRelationalDatabaseConnectionStrings(),
((TableRepository)RepositoryLocator.CatalogueRepository).ConnectionStringBuilder,
((TableRepository)RepositoryLocator.DataExportRepository).ConnectionStringBuilder);
_helper = new DicomRelationalMapperTestHelper();
_helper.SetupSuite(server, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource), null, null, persistentRaw, modalityPrefix);
_globals.DicomRelationalMapperOptions.RetryOnFailureCount = 0;
_globals.DicomRelationalMapperOptions.RetryDelayInSeconds = 0;
//do not use an explicit RAW data load server
CatalogueRepository.GetServerDefaults().ClearDefault(PermissableDefaults.RAWDataLoadServer);
}
public void TestBulkInsertOnly(DatabaseType databaseType, int numberOfImages)
{
foreach (Pipeline p in CatalogueRepository.GetAllObjects <Pipeline>())
{
p.DeleteInDatabase();
}
var db = GetCleanedServer(databaseType);
var template = ImageTableTemplateCollection.LoadFrom(_templateXml);
_globals = new GlobalOptionsFactory().Load();
_globals.DicomRelationalMapperOptions.DatabaseNamerType = typeof(MyFixedStagingDatabaseNamer).FullName;
_globals.DicomRelationalMapperOptions.QoSPrefetchCount = ushort.MaxValue;
_globals.DicomRelationalMapperOptions.MinimumBatchSize = numberOfImages;
_globals.DicomRelationalMapperOptions.UseInsertIntoForRAWMigration = true;
_helper = new DicomRelationalMapperTestHelper();
_helper.SetupSuite(db, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource), root: null, template: template, persistentRaw: true);
//do not use an explicit RAW data load server
CatalogueRepository.GetServerDefaults().ClearDefault(PermissableDefaults.RAWDataLoadServer);
Random r = new Random(123);
List <DicomDataset> allImages;
using (var generator = new DicomDataGenerator(r, null, "CT"))
allImages = generator.GenerateImages(numberOfImages, r);
DicomDatasetCollectionSource source = new DicomDatasetCollectionSource();
source.PreInitialize(new ExplicitListDicomDatasetWorklist(allImages.ToArray(), "amagad.dcm", new Dictionary <string, string>()
{
{ "MessageGuid", "0x123" }
}), new ThrowImmediatelyDataLoadEventListener());;
source.FilenameField = "gggg";
var dt = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
source.Dispose(new ThrowImmediatelyDataLoadEventListener()
{
WriteToConsole = true
}, null);
Assert.AreEqual(numberOfImages, allImages.Count);
Assert.AreEqual(numberOfImages, dt.Rows.Count);
var tables = _helper.LoadMetadata.GetDistinctTableInfoList(false);
var config = new HICDatabaseConfiguration(_helper.LoadMetadata, new SuffixBasedNamer());
var job = Mock.Of <IDataLoadJob>(
j => j.RegularTablesToLoad == tables.Cast <ITableInfo>().ToList() &&
j.DataLoadInfo == _dli &&
j.Configuration == config);
var attacher = new AutoRoutingAttacher();
attacher.Job = job;
//Drop Primary Keys
using (var con = db.Server.GetConnection())
{
con.Open();
var cmd = db.Server.GetCommand(
databaseType == DatabaseType.MicrosoftSQLServer ?
@"ALTER TABLE ImageTable DROP CONSTRAINT PK_ImageTable
ALTER TABLE SeriesTable DROP CONSTRAINT PK_SeriesTable
ALTER TABLE StudyTable DROP CONSTRAINT PK_StudyTable" :
@"ALTER TABLE ImageTable DROP PRIMARY KEY;
ALTER TABLE SeriesTable DROP PRIMARY KEY;
ALTER TABLE StudyTable DROP PRIMARY KEY;"
, con);
cmd.ExecuteNonQuery();
}
attacher.Initialize(LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "IgnoreMe", true), db);
try
{
attacher.ProcessPipelineData(dt, new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
attacher.Dispose(new ThrowImmediatelyDataLoadEventListener(), null);
}
catch (Exception e)
{
attacher.Dispose(new ThrowImmediatelyDataLoadEventListener(), e);
throw;
}
foreach (var tableInfo in tables)
{
Assert.AreEqual(numberOfImages,
tableInfo.Discover(DataAccessContext.InternalDataProcessing).GetRowCount(),
"Row count was wrong for " + tableInfo);
}
foreach (Pipeline allObject in CatalogueRepository.GetAllObjects <Pipeline>())
{
allObject.DeleteInDatabase();
}
}