public LiveVsTemplateComparer(ITableInfo table, ImageTableTemplateCollection templateCollection)
{
// locate the live table and script it as it stands today
var discoveredTable = table.Discover(DataAccessContext.InternalDataProcessing);
LiveSql = discoveredTable.ScriptTableCreation(false, false, false);
LiveSql = TailorLiveSql(LiveSql, discoveredTable.Database.Server.DatabaseType);
// The live table name e.g. CT_StudyTable
var liveTableName = discoveredTable.GetRuntimeName();
// Without the prefix e.g. StudyTable
var liveTableNameWithoutPrefix = liveTableName.Substring(liveTableName.IndexOf("_", StringComparison.Ordinal) + 1);
var template = templateCollection.Tables.FirstOrDefault(
c => c.TableName.Equals(liveTableName, StringComparison.CurrentCultureIgnoreCase) ||
c.TableName.Equals(liveTableNameWithoutPrefix, StringComparison.CurrentCultureIgnoreCase));
if (template == null)
{
throw new Exception($"Could not find a Template called '{liveTableName}' or '{liveTableNameWithoutPrefix}'. Templates in file were {string.Join(",",templateCollection.Tables.Select(t=>t.TableName))}");
}
//script the template
var creator = new ImagingTableCreation(discoveredTable.Database.Server.GetQuerySyntaxHelper());
TemplateSql = creator.GetCreateTableSql(discoveredTable.Database, liveTableName, template, discoveredTable.Schema);
TemplateSql = TailorTemplateSql(TemplateSql);
}
private void btnCreateSuiteWithTemplate_Click(object sender, EventArgs e)
{
string filename;
using (OpenFileDialog ofd = new OpenFileDialog
{
Filter = "Imaging Template|*.it"
})
{
if (ofd.ShowDialog() != DialogResult.OK)
{
return;
}
filename = ofd.FileName;
}
try
{
var yaml = File.ReadAllText(filename);
var template = ImageTableTemplateCollection.LoadFrom(yaml);
CreateSuite(template);
}
catch (Exception exception)
{
ExceptionViewer.Show(exception);
}
}
private ImageTableTemplateCollection GetDefaultTemplate(FAnsi.DatabaseType databaseType)
{
var collection = ImageTableTemplateCollection.LoadFrom(DefaultTemplateYaml);
collection.DatabaseType = databaseType;
return(collection);
}
public void TestLargeImageDatasets(DatabaseType databaseType, int numberOfImages)
{
foreach (Pipeline p in CatalogueRepository.GetAllObjects <Pipeline>())
{
p.DeleteInDatabase();
}
var db = GetCleanedServer(databaseType);
var d = CatalogueRepository.GetServerDefaults();
d.ClearDefault(PermissableDefaults.RAWDataLoadServer);
var template = ImageTableTemplateCollection.LoadFrom(_templateXml);
_globals = new GlobalOptionsFactory().Load();
_globals.DicomRelationalMapperOptions.DatabaseNamerType = typeof(MyFixedStagingDatabaseNamer).FullName;
_globals.DicomRelationalMapperOptions.QoSPrefetchCount = ushort.MaxValue;
_globals.DicomRelationalMapperOptions.MinimumBatchSize = numberOfImages;
_globals.DicomRelationalMapperOptions.UseInsertIntoForRAWMigration = true;
_helper = new DicomRelationalMapperTestHelper();
_helper.SetupSuite(db, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource), root: null, template: template, persistentRaw: true);
//do not use an explicit RAW data load server
d.ClearDefault(PermissableDefaults.RAWDataLoadServer);
Random r = new Random(123);
List <DicomDataset> allImages;
using (var generator = new DicomDataGenerator(r, null, "CT"))
allImages = generator.GenerateImages(numberOfImages, r);
Assert.AreEqual(numberOfImages, allImages.Count);
using (var tester = new MicroserviceTester(_globals.RabbitOptions, _globals.DicomRelationalMapperOptions))
{
using (var host = new DicomRelationalMapperHost(_globals))
{
tester.SendMessages(_globals.DicomRelationalMapperOptions, allImages.Select(GetFileMessageForDataset), true);
Console.WriteLine("Starting Host");
host.Start();
Stopwatch sw = Stopwatch.StartNew();
new TestTimelineAwaiter().Await(() => host.Consumer.AckCount == numberOfImages, null, 20 * 60 * 100); //1 minute
Console.Write("Time For DLE:" + sw.Elapsed.TotalSeconds + "s");
host.Stop("Test finished");
}
}
foreach (Pipeline allObject in CatalogueRepository.GetAllObjects <Pipeline>())
{
allObject.DeleteInDatabase();
}
}
private void CreateSuite(ImageTableTemplateCollection template)
{
var db = serverDatabaseTableSelector1.GetDiscoveredDatabase();
if (!CreateDatabaseIfNotExists(db))
{
return;
}
DirectoryInfo dir = null;
using (FolderBrowserDialog dialog = new FolderBrowserDialog
{
Description = "Select Project Directory (For Sql scripts/Executables etc)"
})
{
//if we are creating a load we need to know where to store load scripts etc
if (cbCreateLoad.Checked)
{
if (dialog.ShowDialog() == DialogResult.OK)
{
dir = new DirectoryInfo(dialog.SelectedPath);
}
else
{
return;
}
}
}
var cmd = new ExecuteCommandCreateNewImagingDatasetSuite(_activator.RepositoryLocator, db, dir)
{
DicomSourceType = rbJsonSources.Checked
? typeof(DicomDatasetCollectionSource)
: typeof(DicomFileCollectionSource),
CreateCoalescer = cbMergeNullability.Checked,
CreateLoad = cbCreateLoad.Checked,
TablePrefix = tbPrefix.Text,
Template = template
};
cmd.Execute();
var firstCata = cmd.NewCataloguesCreated.First();
if (firstCata != null)
{
_activator.Publish(firstCata);
}
MessageBox.Show("Create Suite Completed");
}
public ExecuteCommandCreateNewImagingDatasetSuite(
IRDMPPlatformRepositoryServiceLocator repositoryLocator,
DiscoveredDatabase databaseToCreateInto,
DirectoryInfo projectDirectory,
[DemandsInitialization("The pipeline source for reading dicom tags from e.g. from files or from serialized JSON", TypeOf = typeof(DicomSource))]
Type dicomSourceType,
string tablePrefix,
FileInfo templateFile,
bool persistentRaw,
bool createLoad
) : this(repositoryLocator, databaseToCreateInto, projectDirectory)
{
DicomSourceType = dicomSourceType ?? typeof(DicomFileCollectionSource);
TablePrefix = tablePrefix;
Template = ImageTableTemplateCollection.LoadFrom(File.ReadAllText(templateFile.FullName));
PersistentRaw = persistentRaw;
CreateLoad = createLoad;
}
public override void Execute()
{
base.Execute();
// Get the template for diff
var file = Activator.SelectFile("Template File", "Imaging Template", "*.it");
if (file == null)
{
return;
}
var templateCollection = ImageTableTemplateCollection.LoadFrom(File.ReadAllText(file.FullName));
var comparer = new LiveVsTemplateComparer(_tableInfo, templateCollection);
var viewer = new SQLBeforeAndAfterViewer(comparer.LiveSql, comparer.TemplateSql, "Your Database", "Template", "Differences between live table and image template", MessageBoxButtons.OK); // lgtm[cs/local-not-disposed]
viewer.Show();
}
private void btnCreateSuiteWithTemplate_Click(object sender, EventArgs e)
{
OpenFileDialog ofd = new OpenFileDialog();
ofd.Filter = "Imaging Template|*.it";
if (ofd.ShowDialog() == DialogResult.OK)
{
try
{
var yaml = File.ReadAllText(ofd.FileName);
var template = ImageTableTemplateCollection.LoadFrom(yaml);
CreateSuite(template);
}
catch (Exception exception)
{
ExceptionViewer.Show(exception);
}
}
}
public void TestBulkInsertOnly(DatabaseType databaseType, int numberOfImages)
{
foreach (Pipeline p in CatalogueRepository.GetAllObjects <Pipeline>())
{
p.DeleteInDatabase();
}
var db = GetCleanedServer(databaseType);
var template = ImageTableTemplateCollection.LoadFrom(_templateXml);
_globals = new GlobalOptionsFactory().Load();
_globals.DicomRelationalMapperOptions.DatabaseNamerType = typeof(MyFixedStagingDatabaseNamer).FullName;
_globals.DicomRelationalMapperOptions.QoSPrefetchCount = ushort.MaxValue;
_globals.DicomRelationalMapperOptions.MinimumBatchSize = numberOfImages;
_globals.DicomRelationalMapperOptions.UseInsertIntoForRAWMigration = true;
_helper = new DicomRelationalMapperTestHelper();
_helper.SetupSuite(db, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource), root: null, template: template, persistentRaw: true);
//do not use an explicit RAW data load server
CatalogueRepository.GetServerDefaults().ClearDefault(PermissableDefaults.RAWDataLoadServer);
Random r = new Random(123);
List <DicomDataset> allImages;
using (var generator = new DicomDataGenerator(r, null, "CT"))
allImages = generator.GenerateImages(numberOfImages, r);
DicomDatasetCollectionSource source = new DicomDatasetCollectionSource();
source.PreInitialize(new ExplicitListDicomDatasetWorklist(allImages.ToArray(), "amagad.dcm", new Dictionary <string, string>()
{
{ "MessageGuid", "0x123" }
}), new ThrowImmediatelyDataLoadEventListener());;
source.FilenameField = "gggg";
var dt = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
source.Dispose(new ThrowImmediatelyDataLoadEventListener()
{
WriteToConsole = true
}, null);
Assert.AreEqual(numberOfImages, allImages.Count);
Assert.AreEqual(numberOfImages, dt.Rows.Count);
var tables = _helper.LoadMetadata.GetDistinctTableInfoList(false);
var config = new HICDatabaseConfiguration(_helper.LoadMetadata, new SuffixBasedNamer());
var job = Mock.Of <IDataLoadJob>(
j => j.RegularTablesToLoad == tables.Cast <ITableInfo>().ToList() &&
j.DataLoadInfo == _dli &&
j.Configuration == config);
var attacher = new AutoRoutingAttacher();
attacher.Job = job;
//Drop Primary Keys
using (var con = db.Server.GetConnection())
{
con.Open();
var cmd = db.Server.GetCommand(
databaseType == DatabaseType.MicrosoftSQLServer ?
@"ALTER TABLE ImageTable DROP CONSTRAINT PK_ImageTable
ALTER TABLE SeriesTable DROP CONSTRAINT PK_SeriesTable
ALTER TABLE StudyTable DROP CONSTRAINT PK_StudyTable" :
@"ALTER TABLE ImageTable DROP PRIMARY KEY;
ALTER TABLE SeriesTable DROP PRIMARY KEY;
ALTER TABLE StudyTable DROP PRIMARY KEY;"
, con);
cmd.ExecuteNonQuery();
}
attacher.Initialize(LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "IgnoreMe", true), db);
try
{
attacher.ProcessPipelineData(dt, new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
attacher.Dispose(new ThrowImmediatelyDataLoadEventListener(), null);
}
catch (Exception e)
{
attacher.Dispose(new ThrowImmediatelyDataLoadEventListener(), e);
throw;
}
foreach (var tableInfo in tables)
{
Assert.AreEqual(numberOfImages,
tableInfo.Discover(DataAccessContext.InternalDataProcessing).GetRowCount(),
"Row count was wrong for " + tableInfo);
}
foreach (Pipeline allObject in CatalogueRepository.GetAllObjects <Pipeline>())
{
allObject.DeleteInDatabase();
}
}
private static int RunDatabaseTarget(TargetDatabase configDatabase, ProgramOptions opts)
{
var batchSize = Math.Max(1, configDatabase.Batches);
//if we are going into a database we definitely do not need pixels!
if (opts.NoPixels == false)
{
opts.NoPixels = true;
}
Stopwatch swTotal = new Stopwatch();
swTotal.Start();
string neverDistinct = "SOPInstanceUID";
if (!File.Exists(configDatabase.Template))
{
Console.WriteLine($"Listed template file '{configDatabase.Template}' does not exist");
return(-1);
}
ImageTableTemplateCollection template;
try
{
template = ImageTableTemplateCollection.LoadFrom(File.ReadAllText(configDatabase.Template));
}
catch (Exception e)
{
Console.WriteLine($"Error reading yaml from '{configDatabase.Template}'");
Console.WriteLine(e.ToString());
return(-2);
}
ImplementationManager.Load <MySqlImplementation>();
ImplementationManager.Load <PostgreSqlImplementation>();
ImplementationManager.Load <OracleImplementation>();
ImplementationManager.Load <MicrosoftSQLImplementation>();
var server = new DiscoveredServer(configDatabase.ConnectionString, configDatabase.DatabaseType);
try
{
server.TestConnection();
}
catch (Exception e)
{
Console.WriteLine($"Could not reach target server '{server.Name}'");
Console.WriteLine(e);
return(-2);
}
var db = server.ExpectDatabase(configDatabase.DatabaseName);
if (!db.Exists())
{
Console.WriteLine($"Creating Database '{db.GetRuntimeName()}'");
db.Create();
Console.WriteLine("Database Created");
}
else
{
Console.WriteLine($"Found Database '{db.GetRuntimeName()}'");
}
var creator = new ImagingTableCreation(db.Server.GetQuerySyntaxHelper());
Console.WriteLine($"Image template contained schemas for {template.Tables.Count} tables. Looking for existing tables..");
//setting up bulk inserters
DiscoveredTable[] tables = new DiscoveredTable[template.Tables.Count];
DataTable[][] batches = new DataTable[batchSize][];
for (var i = 0; i < batches.Length; i++)
{
batches[i] = new DataTable[template.Tables.Count];
}
IBulkCopy[][] uploaders = new IBulkCopy[batchSize][];
for (int i = 0; i < uploaders.Length; i++)
{
uploaders[i] = new IBulkCopy[template.Tables.Count];
}
string[] pks = new string[template.Tables.Count];
for (var i = 0; i < template.Tables.Count; i++)
{
var tableSchema = template.Tables[i];
var tbl = db.ExpectTable(tableSchema.TableName);
tables[i] = tbl;
if (configDatabase.MakeDistinct)
{
var col = tableSchema.Columns.Where(c => c.IsPrimaryKey).ToArray();
if (col.Length > 1)
{
Console.WriteLine("MakeDistinct only works with single column primary keys e.g. StudyInstanceUID / SeriesInstanceUID");
}
pks[i] = col.SingleOrDefault()?.ColumnName;
if (pks[i] != null)
{
//if it is sop instance uid then we shouldn't be trying to deduplicate
if (string.Equals(pks[i], neverDistinct, StringComparison.CurrentCultureIgnoreCase))
{
pks[i] = null;
}
else
{
//we will make this a primary key later on
col.Single().IsPrimaryKey = false;
Console.WriteLine($"MakeDistinct will apply to '{pks[i]}' on '{tbl.GetFullyQualifiedName()}'");
}
}
}
bool create = true;
if (tbl.Exists())
{
if (configDatabase.DropTables)
{
Console.WriteLine($"Dropping existing table '{tbl.GetFullyQualifiedName()}'");
tbl.Drop();
}
else
{
Console.WriteLine($"Table '{tbl.GetFullyQualifiedName()}' already existed (so will not be created)");
create = false;
}
}
if (create)
{
Console.WriteLine($"About to create '{tbl.GetFullyQualifiedName()}'");
creator.CreateTable(tbl, tableSchema);
Console.WriteLine($"Successfully created create '{tbl.GetFullyQualifiedName()}'");
}
Console.WriteLine($"Creating uploader for '{tbl.GetRuntimeName()}''");
for (int j = 0; j < batchSize; j++)
{
//fetch schema
var dt = tbl.GetDataTable();
dt.Rows.Clear();
batches[j][i] = dt;
uploaders[j][i] = tbl.BeginBulkInsert();
}
}
var tasks = new Task[batchSize];
IPersonCollection identifiers = GetPeople(opts, out Random r);
for (int i = 0; i < batchSize; i++)
{
var batch = i;
tasks[i] = new Task(() => // lgtm[cs/local-not-disposed]
{
RunBatch(identifiers, opts, r, batches[batch], uploaders[batch]);
});
tasks[i].Start();
}
Task.WaitAll(tasks);
swTotal.Stop();
for (var i = 0; i < tables.Length; i++)
{
if (pks[i] == null)
{
continue;
}
Console.WriteLine($"{DateTime.Now} Making table '{tables[i]}' distinct (this may take a long time)");
var tbl = tables[i];
tbl.MakeDistinct(500000000);
Console.WriteLine($"{DateTime.Now} Creating primary key on '{tables[i]}' of '{pks[i]}'");
tbl.CreatePrimaryKey(500000000, tbl.DiscoverColumn(pks[i]));
}
Console.WriteLine("Final Row Counts:");
foreach (DiscoveredTable t in tables)
{
Console.WriteLine($"{t.GetFullyQualifiedName()}: {t.GetRowCount():0,0}");
}
Console.WriteLine("Total Running Time:" + swTotal.Elapsed);
return(0);
}
public void SetupSuite(DiscoveredDatabase databaseToCreateInto, IRDMPPlatformRepositoryServiceLocator repositoryLocator, GlobalOptions globalOptions, Type pipelineDicomSourceType, string root = null, ImageTableTemplateCollection template = null, bool persistentRaw = false, string modalityPrefix = null)
{
ImageTable = databaseToCreateInto.ExpectTable(modalityPrefix + "ImageTable");
SeriesTable = databaseToCreateInto.ExpectTable(modalityPrefix + "SeriesTable");
StudyTable = databaseToCreateInto.ExpectTable(modalityPrefix + "StudyTable");
try
{
File.Copy(typeof(InvalidDataHandling).Assembly.Location, Path.Combine(TestContext.CurrentContext.TestDirectory, "Rdmp.Dicom.dll"), true);
}
catch (System.IO.IOException)
{
//nevermind, it's probably locked
}
//The Rdmp.Dicom assembly should be loaded as a plugin, this simulates it.
foreach (var type in typeof(InvalidDataHandling).Assembly.GetTypes())
{
repositoryLocator.CatalogueRepository.MEF.AddTypeToCatalogForTesting(type);
}
ICatalogueRepository catalogueRepository = repositoryLocator.CatalogueRepository;
IDataExportRepository dataExportRepository = repositoryLocator.DataExportRepository;
foreach (var t in new[] { ImageTable, SeriesTable, StudyTable })
{
if (t.Exists())
{
t.Drop();
}
}
var suite = new ExecuteCommandCreateNewImagingDatasetSuite(repositoryLocator, databaseToCreateInto, new DirectoryInfo(TestContext.CurrentContext.TestDirectory));
suite.Template = template ?? GetDefaultTemplate(databaseToCreateInto.Server.DatabaseType);
suite.PersistentRaw = persistentRaw;
suite.TablePrefix = modalityPrefix;
suite.DicomSourceType = pipelineDicomSourceType;
suite.CreateCoalescer = true;
suite.Execute();
DicomSourcePipelineComponent = suite.DicomSourcePipelineComponent; //store the component created so we can inject/adjust the arguments e.g. adding ElevationRequests to it
LoadMetadata = suite.NewLoadMetadata;
var tableInfos = LoadMetadata.GetAllCatalogues().SelectMany(c => c.GetTableInfoList(false)).Distinct().ToArray();
ImageTableInfo = (TableInfo)tableInfos.Single(t => t.GetRuntimeName().Equals(ImageTable.GetRuntimeName()));
SeriesTableInfo = (TableInfo)tableInfos.Single(t => t.GetRuntimeName().Equals(SeriesTable.GetRuntimeName()));
StudyTableInfo = (TableInfo)tableInfos.Single(t => t.GetRuntimeName().Equals(StudyTable.GetRuntimeName()));
// Override the options with stuff coming from Core RDMP DatabaseTests (TestDatabases.txt)
globalOptions.FileSystemOptions.FileSystemRoot = root ?? TestContext.CurrentContext.TestDirectory;
globalOptions.RDMPOptions.CatalogueConnectionString = ((TableRepository)catalogueRepository).DiscoveredServer.Builder.ConnectionString;
globalOptions.RDMPOptions.DataExportConnectionString = ((TableRepository)dataExportRepository).DiscoveredServer.Builder.ConnectionString;
globalOptions.DicomRelationalMapperOptions.LoadMetadataId = LoadMetadata.ID;
globalOptions.DicomRelationalMapperOptions.MinimumBatchSize = 1;
globalOptions.DicomRelationalMapperOptions.UseInsertIntoForRAWMigration = true;
//Image table now needs all the UIDs in order to be extractable
var adder = new TagColumnAdder("StudyInstanceUID", "varchar(100)", ImageTableInfo, new AcceptAllCheckNotifier());
adder.Execute();
}
