public void TestLargeImageDatasets(DatabaseType databaseType, int numberOfImages)
{
foreach (Pipeline p in CatalogueRepository.GetAllObjects <Pipeline>())
{
p.DeleteInDatabase();
}
var db = GetCleanedServer(databaseType);
var d = CatalogueRepository.GetServerDefaults();
d.ClearDefault(PermissableDefaults.RAWDataLoadServer);
var template = ImageTableTemplateCollection.LoadFrom(_templateXml);
_globals = new GlobalOptionsFactory().Load();
_globals.DicomRelationalMapperOptions.DatabaseNamerType = typeof(MyFixedStagingDatabaseNamer).FullName;
_globals.DicomRelationalMapperOptions.QoSPrefetchCount = ushort.MaxValue;
_globals.DicomRelationalMapperOptions.MinimumBatchSize = numberOfImages;
_globals.DicomRelationalMapperOptions.UseInsertIntoForRAWMigration = true;
_helper = new DicomRelationalMapperTestHelper();
_helper.SetupSuite(db, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource), root: null, template: template, persistentRaw: true);
//do not use an explicit RAW data load server
d.ClearDefault(PermissableDefaults.RAWDataLoadServer);
Random r = new Random(123);
List <DicomDataset> allImages;
using (var generator = new DicomDataGenerator(r, null, "CT"))
allImages = generator.GenerateImages(numberOfImages, r);
Assert.AreEqual(numberOfImages, allImages.Count);
using (var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.DicomRelationalMapperOptions))
{
using (var host = new DicomRelationalMapperHost(_globals))
{
tester.SendMessages(_globals.DicomRelationalMapperOptions, allImages.Select(GetFileMessageForDataset), true);
Console.WriteLine("Starting Host");
host.Start();
Stopwatch sw = Stopwatch.StartNew();
new TestTimelineAwaiter().Await(() => host.Consumer.AckCount == numberOfImages, null, 20 * 60 * 100); //1 minute
Console.Write("Time For DLE:" + sw.Elapsed.TotalSeconds + "s");
host.Stop("Test finished");
}
}
foreach (Pipeline allObject in CatalogueRepository.GetAllObjects <Pipeline>())
{
allObject.DeleteInDatabase();
}
}
public void TestIdentifierSwap_RegexVsDeserialize(DatabaseType type, int batchSize, int numberOfRandomTagsPerDicom)
{
var options = new GlobalOptionsFactory().Load();
var mappingDataTable = new DataTable("IdMap");
mappingDataTable.Columns.Add("priv");
mappingDataTable.Columns.Add("pub");
mappingDataTable.Rows.Add("010101", "020202");
mappingDataTable.Rows.Add("0101010101", "0202020202");
var db = GetCleanedServer(type);
options.IdentifierMapperOptions.MappingConnectionString = db.Server.Builder.ConnectionString;
options.IdentifierMapperOptions.MappingTableName = db.CreateTable("IdMap", mappingDataTable).GetFullyQualifiedName();
options.IdentifierMapperOptions.SwapColumnName = "priv";
options.IdentifierMapperOptions.ReplacementColumnName = "pub";
options.IdentifierMapperOptions.MappingDatabaseType = type;
options.IdentifierMapperOptions.TimeoutInSeconds = 500;
var swapper = new PreloadTableSwapper();
swapper.Setup(options.IdentifierMapperOptions);
var goodChis = new List <DicomFileMessage>();
var badChis = new List <DicomFileMessage>();
Console.WriteLine("Generating Test data...");
List <Task> tasks = new List <Task>();
object oTaskLock = new object();
for (int i = 0; i < batchSize; i++)
{
var t = new Task(() =>
{
var a = GetTestDicomFileMessage(Test.ProperlyFormatedChi, numberOfRandomTagsPerDicom);
var b = GetTestDicomFileMessage(Test.ProperlyFormatedChi, numberOfRandomTagsPerDicom);
lock (oTaskLock)
{
goodChis.Add(a);
badChis.Add(b);
}
});
t.Start();
tasks.Add(t);
if (i % Environment.ProcessorCount == 0)
{
Task.WaitAll(tasks.ToArray());
tasks.Clear();
}
if (i % 100 == 0)
{
Console.WriteLine(i + " pairs done");
}
}
Task.WaitAll(tasks.ToArray());
options.IdentifierMapperOptions.AllowRegexMatching = true;
using (var tester = new MicroserviceTester(options.RabbitOptions, options.IdentifierMapperOptions))
{
tester.CreateExchange(options.IdentifierMapperOptions.AnonImagesProducerOptions.ExchangeName, null);
Console.WriteLine("Pushing good messages to Rabbit...");
tester.SendMessages(options.IdentifierMapperOptions, goodChis, true);
var host = new IdentifierMapperHost(options, swapper);
tester.StopOnDispose.Add(host);
Console.WriteLine("Starting host");
Stopwatch sw = Stopwatch.StartNew();
host.Start();
new TestTimelineAwaiter().Await(() => host.Consumer.AckCount == batchSize);
Console.WriteLine("Good message processing (" + batchSize + ") took:" + sw.ElapsedMilliseconds + "ms");
host.Stop("Test finished");
}
options.IdentifierMapperOptions.AllowRegexMatching = false;
using (var tester = new MicroserviceTester(options.RabbitOptions, options.IdentifierMapperOptions))
{
tester.CreateExchange(options.IdentifierMapperOptions.AnonImagesProducerOptions.ExchangeName, null);
Console.WriteLine("Pushing bad messages to Rabbit...");
tester.SendMessages(options.IdentifierMapperOptions, badChis, true);
var host = new IdentifierMapperHost(options, swapper);
tester.StopOnDispose.Add(host);
Console.WriteLine("Starting host");
Stopwatch sw = Stopwatch.StartNew();
host.Start();
new TestTimelineAwaiter().Await(() => host.Consumer.AckCount == batchSize);
Console.WriteLine("Bad message processing (" + batchSize + ") took:" + sw.ElapsedMilliseconds + "ms");
host.Stop("Test finished");
}
}
