public override void Execute()
{
if (_type == null)
{
if (BasicActivator.SelectType("Process Type", GetProcessTaskTypes(), out Type chosen))
{
SetType(chosen);
}
else
{
return;
}
}
ProcessTask newTask = new ProcessTask(BasicActivator.RepositoryLocator.CatalogueRepository, _loadMetadata, _loadStage);
newTask.Path = _type.FullName;
newTask.ProcessTaskType = _processTaskType;
newTask.Name = _type.Name;
newTask.SaveToDatabase();
newTask.CreateArgumentsForClassIfNotExists(_type);
Publish(_loadMetadata);
Activate(newTask);
}
public void CacheProvider_NoPipeline()
{
var pt1 = new ProcessTask(CatalogueRepository, _lmd, LoadStage.GetFiles);
pt1.Path = typeof(TestCachedFileRetriever).FullName;
pt1.ProcessTaskType = ProcessTaskType.DataProvider;
pt1.Name = "Cache1";
pt1.SaveToDatabase();
_cp.CacheFillProgress = new DateTime(1999, 1, 1);
_cp.Name = "MyTestCp";
_cp.SaveToDatabase();
pt1.CreateArgumentsForClassIfNotExists <TestCachedFileRetriever>();
var projDir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "delme", true);
_lmd.LocationOfFlatFiles = projDir.RootPath.FullName;
_lmd.SaveToDatabase();
try
{
var ex = Assert.Throws <Exception>(() => _factory.Create(_lp, new ThrowImmediatelyDataLoadEventListener()));
Assert.AreEqual("CacheProgress MyTestCp does not have a Pipeline configured on it", ex.Message);
}
finally
{
projDir.RootPath.Delete(true);
}
}
/// <summary>
/// Creates a <see cref="AnySeparatorFileAttacher"/> parcelled into a <see cref="ProcessTask"/> that reads CSV files in ForLoading
/// in the mounting stage of the load
/// </summary>
/// <param name="lmd">The load to create the <see cref="ProcessTask"/> in</param>
/// <param name="pattern">File pattern to load e.g. *.csv</param>
/// <param name="ti">The table to load (must be part of the <paramref name="lmd"/></param>
/// <param name="separator">The separator of the files e.g. ','</param>
/// <param name="ignoreColumns">Columns to ignore in the load</param>
/// <returns></returns>
protected ProcessTask CreateFlatFileAttacher(LoadMetadata lmd, string pattern, ITableInfo ti, string separator = ",", string ignoreColumns = "hic_dataLoadRunID")
{
var csvProcessTask = new ProcessTask(CatalogueRepository, lmd, LoadStage.Mounting);
var args = csvProcessTask.CreateArgumentsForClassIfNotExists <AnySeparatorFileAttacher>();
csvProcessTask.Path = typeof(AnySeparatorFileAttacher).FullName;
csvProcessTask.ProcessTaskType = ProcessTaskType.Attacher;
csvProcessTask.SaveToDatabase();
var filePattern = args.Single(a => a.Name == "FilePattern");
filePattern.SetValue(pattern);
filePattern.SaveToDatabase();
var tableToLoad = args.Single(a => a.Name == "TableToLoad");
tableToLoad.SetValue(ti);
tableToLoad.SaveToDatabase();
var separatorArg = args.Single(a => a.Name == "Separator");
separatorArg.SetValue(separator);
separatorArg.SaveToDatabase();
var ignoreDataLoadRunIDCol = args.Single(a => a.Name == "IgnoreColumns");
ignoreDataLoadRunIDCol.SetValue(ignoreColumns);
ignoreDataLoadRunIDCol.SaveToDatabase();
return(csvProcessTask);
}
private void CreateAttacher(ITableInfo t, QueryBuilder qb, LoadMetadata lmd, LoadProgress loadProgressIfAny)
{
var pt = new ProcessTask(Activator.RepositoryLocator.CatalogueRepository, lmd, LoadStage.Mounting);
pt.ProcessTaskType = ProcessTaskType.Attacher;
pt.Name = "Read from " + t;
pt.Path = typeof(RemoteTableAttacher).FullName;
pt.SaveToDatabase();
pt.CreateArgumentsForClassIfNotExists <RemoteTableAttacher>();
pt.SetArgumentValue("RemoteServer", t.Server);
pt.SetArgumentValue("RemoteDatabaseName", t.GetDatabaseRuntimeName(LoadStage.PostLoad));
pt.SetArgumentValue("RemoteTableName", t.GetRuntimeName());
pt.SetArgumentValue("DatabaseType", DatabaseType.MicrosoftSQLServer);
pt.SetArgumentValue("RemoteSelectSQL", qb.SQL);
pt.SetArgumentValue("RAWTableName", t.GetRuntimeName(LoadBubble.Raw));
if (loadProgressIfAny != null)
{
pt.SetArgumentValue("Progress", loadProgressIfAny);
// pt.SetArgumentValue("ProgressUpdateStrategy", DataLoadProgressUpdateStrategy.UseMaxRequestedDay);
pt.SetArgumentValue("LoadNotRequiredIfNoRowsRead", true);
}
/*
*
* public DataLoadProgressUpdateInfo { get; set; }
*/
}
private void CreateDilutionMutilation(KeyValuePair <PreLoadDiscardedColumn, IDilutionOperation> dilutionOp, LoadMetadata lmd)
{
var pt = new ProcessTask(Activator.RepositoryLocator.CatalogueRepository, lmd, LoadStage.AdjustStaging);
pt.CreateArgumentsForClassIfNotExists <Dilution>();
pt.ProcessTaskType = ProcessTaskType.MutilateDataTable;
pt.Name = "Dilute " + dilutionOp.Key.GetRuntimeName();
pt.Path = typeof(Dilution).FullName;
pt.SaveToDatabase();
pt.SetArgumentValue("ColumnToDilute", dilutionOp.Key);
pt.SetArgumentValue("Operation", dilutionOp.Value.GetType());
}
public override void Execute()
{
ProcessTask newTask = new ProcessTask(BasicActivator.RepositoryLocator.CatalogueRepository, _loadMetadata, _loadStage);
newTask.Path = _type.FullName;
newTask.ProcessTaskType = _processTaskType;
newTask.Name = _type.Name;
newTask.SaveToDatabase();
newTask.CreateArgumentsForClassIfNotExists(_type);
Publish(_loadMetadata);
Activate(newTask);
}
private void CreateCSVProcessTask(LoadMetadata lmd, TableInfo ti, string regex)
{
var pt = new ProcessTask(CatalogueRepository, lmd, LoadStage.Mounting);
pt.Path = typeof(AnySeparatorFileAttacher).FullName;
pt.ProcessTaskType = ProcessTaskType.Attacher;
pt.Name = "Load " + ti.GetRuntimeName();
pt.SaveToDatabase();
pt.CreateArgumentsForClassIfNotExists <AnySeparatorFileAttacher>();
pt.SetArgumentValue("FilePattern", regex);
pt.SetArgumentValue("Separator", ",");
pt.SetArgumentValue("TableToLoad", ti);
pt.Check(new ThrowImmediatelyCheckNotifier());
}
public void GatherAndShare_LoadMetadata_WithReferenceProcessTaskArgument()
{
//create an object
LoadMetadata lmd1 = WhenIHaveA <LoadMetadata>();
//setup Reflection / MEF
SetupMEF();
RuntimeTaskFactory f = new RuntimeTaskFactory(Repository);
var stg = Mock.Of <IStageArgs>(x =>
x.LoadStage == LoadStage.Mounting &&
x.DbInfo == new DiscoveredServer(new SqlConnectionStringBuilder()).ExpectDatabase("d"));
//create a single process task for the load
var pt1 = new ProcessTask(Repository, lmd1, LoadStage.Mounting);
pt1.ProcessTaskType = ProcessTaskType.MutilateDataTable;
pt1.LoadStage = LoadStage.AdjustRaw;
pt1.Path = typeof(SafePrimaryKeyCollisionResolverMutilation).FullName;
pt1.SaveToDatabase();
//give it a reference to an (unshared) object (ColumnInfo)
pt1.CreateArgumentsForClassIfNotExists(typeof(SafePrimaryKeyCollisionResolverMutilation));
var pta = pt1.ProcessTaskArguments.Single(pt => pt.Name == "ColumnToResolveOn");
pta.SetValue(WhenIHaveA <ColumnInfo>());
pta.SaveToDatabase();
//check that reflection can assemble the master ProcessTask
MutilateDataTablesRuntimeTask t = (MutilateDataTablesRuntimeTask)f.Create(pt1, stg);
Assert.IsNotNull(((SafePrimaryKeyCollisionResolverMutilation)t.MEFPluginClassInstance).ColumnToResolveOn);
//share to the second repository (which won't have that ColumnInfo)
var lmd2 = ShareToNewRepository(lmd1);
//create a new reflection factory for the new repo
RuntimeTaskFactory f2 = new RuntimeTaskFactory(lmd2.CatalogueRepository);
lmd2.CatalogueRepository.MEF = MEF;
//when we create the shared instance it should not have a valid value for ColumnInfo (since it wasn't - and shouldn't be shared)
MutilateDataTablesRuntimeTask t2 = (MutilateDataTablesRuntimeTask)f2.Create(lmd2.ProcessTasks.Single(), stg);
Assert.IsNull(((SafePrimaryKeyCollisionResolverMutilation)t2.MEFPluginClassInstance).ColumnToResolveOn);
}
public void MEFCompatibleType_NoProjectDirectory()
{
_lmd.LocationOfFlatFiles = null;
_lmd.SaveToDatabase();
_task.ProcessTaskType = ProcessTaskType.Attacher;
_task.LoadStage = LoadStage.Mounting;
_task.Path = typeof(AnySeparatorFileAttacher).FullName;
_task.SaveToDatabase();
_task.CreateArgumentsForClassIfNotExists <AnySeparatorFileAttacher>();
var ex = Assert.Throws <Exception>(() => _checker.Check(new ThrowImmediatelyCheckNotifier()
{
ThrowOnWarning = true
}));
Assert.AreEqual(@"No Project Directory (LocationOfFlatFiles) has been configured on LoadMetadata " + _lmd.Name, ex.InnerException.Message);
}
public void GatherAndShare_LoadMetadata_WithRealProcessTask()
{
//create an object
LoadMetadata lmd1 = WhenIHaveA <LoadMetadata>();
SetupMEF();
var pt1 = new ProcessTask(Repository, lmd1, LoadStage.Mounting);
pt1.ProcessTaskType = ProcessTaskType.Attacher;
pt1.LoadStage = LoadStage.Mounting;
pt1.Path = typeof(AnySeparatorFileAttacher).FullName;
pt1.SaveToDatabase();
pt1.CreateArgumentsForClassIfNotExists(typeof(AnySeparatorFileAttacher));
var pta = pt1.ProcessTaskArguments.Single(pt => pt.Name == "Separator");
pta.SetValue(",");
pta.SaveToDatabase();
var lmd2 = ShareToNewRepository(lmd1);
//different repos so not identical
Assert.IsFalse(ReferenceEquals(lmd1, lmd2));
AssertAreEqual(lmd1, lmd2);
var pt2 = lmd2.ProcessTasks.Single();
Assert.IsFalse(ReferenceEquals(pt1, pt2));
AssertAreEqual(pt1, pt2);
AssertAreEqual(pt1.GetAllArguments(), pt2.GetAllArguments());
RuntimeTaskFactory f = new RuntimeTaskFactory(Repository);
var stg = Mock.Of <IStageArgs>(x => x.LoadStage == LoadStage.Mounting);
f.Create(pt1, stg);
}
public void TestArgumentCreation()
{
LoadMetadata lmd = new LoadMetadata(CatalogueRepository, "TestArgumentCreation");
var pt = new ProcessTask(CatalogueRepository, lmd, LoadStage.AdjustRaw);
pt.CreateArgumentsForClassIfNotExists <TestArgumentedClass>();
try
{
var arg = pt.ProcessTaskArguments.Single();
Assert.AreEqual("MyBool", arg.Name);
Assert.AreEqual("System.Boolean", arg.Type);
Assert.AreEqual("Fishes", arg.Description);
Assert.AreEqual("True", arg.Value);
Assert.AreEqual(true, arg.GetValueAsSystemType());
}
finally
{
pt.DeleteInDatabase();
lmd.DeleteInDatabase();
}
}
public void TestFindTables(DatabaseType dbType)
{
var db = GetCleanedServer(dbType);
var dt = new DataTable();
dt.Columns.Add("A");
dt.Columns.Add("B");
dt.Columns.Add("C");
// 'pk' 1 differs on col B AND col C
dt.Rows.Add(1, 2, 3);
dt.Rows.Add(1, 3, 2);
//novel (should not appear in diff table)
dt.Rows.Add(4, 1, 1);
//novel (should not appear in diff table)
dt.Rows.Add(5, 1, 1);
// 'pk' 2 differs on col C
dt.Rows.Add(2, 1, 1);
dt.Rows.Add(2, 1, 2);
//novel (should not appear in diff table)
dt.Rows.Add(6, 1, 1);
// 'pk' 3 differs on col B
dt.Rows.Add(3, 1, 1);
dt.Rows.Add(3, 2, 1);
db.CreateTable("mytbl_Isolation", dt);
var lmd = new LoadMetadata(CatalogueRepository, "ExampleLoad");
var pt = new ProcessTask(CatalogueRepository, lmd, LoadStage.AdjustRaw);
pt.ProcessTaskType = ProcessTaskType.MutilateDataTable;
pt.Path = typeof(PrimaryKeyCollisionIsolationMutilation).FullName;
pt.SaveToDatabase();
//make an isolation db that is the
var eds = new ExternalDatabaseServer(CatalogueRepository, "Isolation db", null);
eds.SetProperties(db);
var args = pt.CreateArgumentsForClassIfNotExists(typeof(PrimaryKeyCollisionIsolationMutilation));
var ti = new TableInfo(CatalogueRepository, "mytbl");
var ci = new ColumnInfo(CatalogueRepository, "A", "varchar(1)", ti);
ci.IsPrimaryKey = true;
ci.SaveToDatabase();
SetArg(args, "IsolationDatabase", eds);
SetArg(args, "TablesToIsolate", new [] { ti });
var reviewer = new IsolationReview(pt);
//no error since it is configured correctly
Assert.IsNull(reviewer.Error);
//tables should exist
var isolationTables = reviewer.GetIsolationTables();
Assert.IsTrue(isolationTables.Single().Value.Exists());
var diffDataTable = reviewer.GetDifferences(isolationTables.Single(), out List <IsolationDifference> diffs);
Assert.AreEqual(6, diffDataTable.Rows.Count);
Assert.AreEqual(6, diffs.Count);
}
protected override void SetUp()
{
base.SetUp();
Database = GetCleanedServer(FAnsi.DatabaseType.MicrosoftSQLServer);
var rootFolder = new DirectoryInfo(TestContext.CurrentContext.TestDirectory);
var subdir = rootFolder.CreateSubdirectory("TestsRequiringADle");
LoadDirectory = LoadDirectory.CreateDirectoryStructure(rootFolder, subdir.FullName, true);
Clear(LoadDirectory);
LiveTable = CreateDataset <Demography>(Database, 500, 5000, new Random(190));
LiveTable.CreatePrimaryKey(new DiscoveredColumn[] {
LiveTable.DiscoverColumn("chi"),
LiveTable.DiscoverColumn("dtCreated"),
LiveTable.DiscoverColumn("hb_extract")
});
TestCatalogue = Import(LiveTable);
RowsBefore = 5000;
TestLoadMetadata = new LoadMetadata(CatalogueRepository, "Loading Test Catalogue");
TestLoadMetadata.LocationOfFlatFiles = LoadDirectory.RootPath.FullName;
TestLoadMetadata.SaveToDatabase();
//make the load load the table
TestCatalogue.LoadMetadata_ID = TestLoadMetadata.ID;
TestCatalogue.SaveToDatabase();
var csvProcessTask = new ProcessTask(CatalogueRepository, TestLoadMetadata, LoadStage.Mounting);
var args = csvProcessTask.CreateArgumentsForClassIfNotExists <AnySeparatorFileAttacher>();
csvProcessTask.Path = typeof(AnySeparatorFileAttacher).FullName;
csvProcessTask.ProcessTaskType = ProcessTaskType.Attacher;
csvProcessTask.SaveToDatabase();
var filePattern = args.Single(a => a.Name == "FilePattern");
filePattern.SetValue("*.csv");
filePattern.SaveToDatabase();
var tableToLoad = args.Single(a => a.Name == "TableToLoad");
tableToLoad.SetValue(TestCatalogue.GetTableInfoList(false).Single());
tableToLoad.SaveToDatabase();
var separator = args.Single(a => a.Name == "Separator");
separator.SetValue(",");
separator.SaveToDatabase();
var ignoreDataLoadRunIDCol = args.Single(a => a.Name == "IgnoreColumns");
ignoreDataLoadRunIDCol.SetValue("hic_dataLoadRunID");
ignoreDataLoadRunIDCol.SaveToDatabase();
//Get DleRunner to run pre load checks (includes trigger creation etc)
var runner = new DleRunner(new DleOptions()
{
LoadMetadata = TestLoadMetadata.ID, Command = CommandLineActivity.check
});
runner.Run(RepositoryLocator, new ThrowImmediatelyDataLoadEventListener(), new AcceptAllCheckNotifier(), new GracefulCancellationToken());
}
public void RunEndToEndDLECacheTest()
{
RepositoryLocator.CatalogueRepository.MEF.AddTypeToCatalogForTesting(typeof(TestDataWriter));
RepositoryLocator.CatalogueRepository.MEF.AddTypeToCatalogForTesting(typeof(TestDataInventor));
int timeoutInMilliseconds = 120000;
var lmd = TestLoadMetadata;
LoadProgress lp = new LoadProgress(CatalogueRepository, lmd);
lp.DataLoadProgress = new DateTime(2001, 1, 1);
lp.DefaultNumberOfDaysToLoadEachTime = 10;
lp.SaveToDatabase();
var cp = new CacheProgress(CatalogueRepository, lp);
cp.CacheFillProgress = new DateTime(2001, 1, 11); //10 days available to load
cp.SaveToDatabase();
var assembler = new TestDataPipelineAssembler("RunEndToEndDLECacheTest pipe", CatalogueRepository);
assembler.ConfigureCacheProgressToUseThePipeline(cp);
//setup the cache process task
var pt = new ProcessTask(CatalogueRepository, lmd, LoadStage.GetFiles);
pt.Path = typeof(BasicCacheDataProvider).FullName;
pt.ProcessTaskType = ProcessTaskType.DataProvider;
pt.SaveToDatabase();
pt.CreateArgumentsForClassIfNotExists <BasicCacheDataProvider>();
var attacher = lmd.ProcessTasks.Single(p => p.ProcessTaskType == ProcessTaskType.Attacher);
var patternArgument = (ProcessTaskArgument)attacher.GetAllArguments().Single(a => a.Name.Equals("FilePattern"));
patternArgument.SetValue("*.csv");
patternArgument.SaveToDatabase();
//take the forLoading file
var csvFile = CreateFileInForLoading("bob.csv", 10, new Random(5000));
//and move it to the cache and give it a date in the range we expect for the cached data
csvFile.MoveTo(Path.Combine(LoadDirectory.Cache.FullName, "2001-01-09.csv"));
RunDLE(timeoutInMilliseconds);
Assert.AreEqual(10, RowsNow - RowsBefore);
Assert.AreEqual(0, LoadDirectory.Cache.GetFiles().Count());
Assert.AreEqual(0, LoadDirectory.ForLoading.GetFiles().Count());
Assert.AreEqual(1, LoadDirectory.ForArchiving.GetFiles().Count());
var archiveFile = LoadDirectory.ForArchiving.GetFiles()[0];
Assert.AreEqual(".zip", archiveFile.Extension);
//load progress should be updated to the largest date in the cache (2001-01-09)
lp.RevertToDatabaseState();
Assert.AreEqual(lp.DataLoadProgress, new DateTime(2001, 01, 09));
cp.DeleteInDatabase();
lp.DeleteInDatabase();
assembler.Destroy();
}
public override void Execute()
{
if (DicomSourceType == null)
{
SetImpossible("You must specify a Type for DicomSourceType");
throw new ImpossibleCommandException(this, ReasonCommandImpossible);
}
base.Execute();
List <DiscoveredTable> tablesCreated = new List <DiscoveredTable>();
//Create with template?
if (Template != null)
{
foreach (ImageTableTemplate table in Template.Tables)
{
string tblName = GetNameWithPrefix(table.TableName);
var tbl = _databaseToCreateInto.ExpectTable(tblName);
var cmd = new ExecuteCommandCreateNewImagingDataset(_repositoryLocator, tbl, table);
cmd.Execute();
NewCataloguesCreated.Add(cmd.NewCatalogueCreated);
tablesCreated.Add(tbl);
}
}
else
{
throw new Exception("No Template provided");
}
//that's us done if we aren't creating a load
if (!CreateLoad)
{
return;
}
string loadName = GetNameWithPrefixInBracketsIfAny("SMI Image Loading");
NewLoadMetadata = new LoadMetadata(_catalogueRepository, loadName);
//tell all the catalogues that they are part of this load and where to log under the same task
foreach (Catalogue c in NewCataloguesCreated)
{
c.LoadMetadata_ID = NewLoadMetadata.ID;
c.LoggingDataTask = loadName;
c.LiveLoggingServer_ID = _loggingServer.ID;
c.SaveToDatabase();
}
//create the logging task
new Core.Logging.LogManager(_loggingServer).CreateNewLoggingTaskIfNotExists(loadName);
var projDir = LoadDirectory.CreateDirectoryStructure(_projectDirectory, "ImageLoading", true);
NewLoadMetadata.LocationOfFlatFiles = projDir.RootPath.FullName;
NewLoadMetadata.SaveToDatabase();
/////////////////////////////////////////////Attacher////////////////////////////
//Create a pipeline for reading from Dicom files and writing to any destination component (which must be fixed)
var pipe = new Pipeline(_catalogueRepository, GetNameWithPrefixInBracketsIfAny("Image Loading Pipe"));
DicomSourcePipelineComponent = new PipelineComponent(_catalogueRepository, pipe, DicomSourceType, 0, DicomSourceType.Name);
DicomSourcePipelineComponent.CreateArgumentsForClassIfNotExists(DicomSourceType);
pipe.SourcePipelineComponent_ID = DicomSourcePipelineComponent.ID;
pipe.SaveToDatabase();
//Create the load process task that uses the pipe to load RAW tables with data from the dicom files
var pt = new ProcessTask(_catalogueRepository, NewLoadMetadata, LoadStage.Mounting);
pt.Name = "Auto Routing Attacher";
pt.ProcessTaskType = ProcessTaskType.Attacher;
pt.Path = PersistentRaw? typeof(AutoRoutingAttacherWithPersistentRaw).FullName: typeof(AutoRoutingAttacher).FullName;
pt.Order = 1;
pt.SaveToDatabase();
var args = PersistentRaw? pt.CreateArgumentsForClassIfNotExists <AutoRoutingAttacherWithPersistentRaw>() : pt.CreateArgumentsForClassIfNotExists <AutoRoutingAttacher>();
SetArgument(args, "LoadPipeline", pipe);
/////////////////////////////////////// Distinct tables on load /////////////////////////
var distincter = new ProcessTask(_catalogueRepository, NewLoadMetadata, LoadStage.AdjustRaw);
var distincterArgs = distincter.CreateArgumentsForClassIfNotExists <Distincter>();
distincter.Name = "Distincter";
distincter.ProcessTaskType = ProcessTaskType.MutilateDataTable;
distincter.Path = typeof(Distincter).FullName;
distincter.Order = 2;
distincter.SaveToDatabase();
SetArgument(distincterArgs, "TableRegexPattern", ".*");
/////////////////////////////////////////////////////////////////////////////////////
if (CreateCoalescer)
{
var coalescer = new ProcessTask(_catalogueRepository, NewLoadMetadata, LoadStage.AdjustRaw);
coalescer.Name = "Coalescer";
coalescer.ProcessTaskType = ProcessTaskType.MutilateDataTable;
coalescer.Path = typeof(Coalescer).FullName;
coalescer.Order = 3;
coalescer.SaveToDatabase();
StringBuilder regexPattern = new StringBuilder();
foreach (var tbl in tablesCreated)
{
if (!tbl.DiscoverColumns().Any(c => c.GetRuntimeName().Equals("SOPInstanceUID", StringComparison.CurrentCultureIgnoreCase)))
{
regexPattern.Append("(" + tbl.GetRuntimeName() + ")|");
}
}
var coalArgs = coalescer.CreateArgumentsForClassIfNotExists <Coalescer>();
SetArgument(coalArgs, "TableRegexPattern", regexPattern.ToString().TrimEnd('|'));
SetArgument(coalArgs, "CreateIndex", true);
}
////////////////////////////////Load Ender (if no rows in load) ////////////////////////////
var prematureLoadEnder = new ProcessTask(_catalogueRepository, NewLoadMetadata, LoadStage.Mounting);
prematureLoadEnder.Name = "Premature Load Ender";
prematureLoadEnder.ProcessTaskType = ProcessTaskType.MutilateDataTable;
prematureLoadEnder.Path = typeof(PrematureLoadEnder).FullName;
prematureLoadEnder.Order = 4;
prematureLoadEnder.SaveToDatabase();
args = prematureLoadEnder.CreateArgumentsForClassIfNotExists <PrematureLoadEnder>();
SetArgument(args, "ExitCodeToReturnIfConditionMet", ExitCodeType.OperationNotRequired);
SetArgument(args, "ConditionsToTerminateUnder", PrematureLoadEndCondition.NoRecordsInAnyTablesInDatabase);
////////////////////////////////////////////////////////////////////////////////////////////////
var checker = new CheckEntireDataLoadProcess(NewLoadMetadata, new HICDatabaseConfiguration(NewLoadMetadata), new HICLoadConfigurationFlags(), _catalogueRepository.MEF);
checker.Check(new AcceptAllCheckNotifier());
}
public void IntegrationTest_HappyPath_WithIsolation(DatabaseType databaseType, Type namerType)
{
var server = GetCleanedServer(databaseType, ScratchDatabaseName);
SetupSuite(server, false, "MR_");
//this ensures that the ExtractionConfiguration.ID and Project.ID properties are out of sync (they are automnums). Its just a precaution since we are using both IDs in places if we
//had any bugs where we used the wrong one but they were the same then it would be obscured until production
var p = new Project(DataExportRepository, "delme");
p.DeleteInDatabase();
_globals.DicomRelationalMapperOptions.Guid = new Guid("fc229fc3-f700-4515-86e8-e3d38b3d1823");
_globals.DicomRelationalMapperOptions.QoSPrefetchCount = 5000;
_globals.DicomRelationalMapperOptions.MinimumBatchSize = 3;
_globals.DicomRelationalMapperOptions.DatabaseNamerType = namerType.FullName;
_helper.TruncateTablesIfExists();
//Create test directory with a single image
var dir = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(IntegrationTest_HappyPath_WithIsolation)));
dir.Create();
var ptIsolation = new ProcessTask(CatalogueRepository, _helper.LoadMetadata, LoadStage.AdjustRaw);
ptIsolation.CreateArgumentsForClassIfNotExists <PrimaryKeyCollisionIsolationMutilation>();
ptIsolation.Path = typeof(PrimaryKeyCollisionIsolationMutilation).FullName;
ptIsolation.ProcessTaskType = ProcessTaskType.MutilateDataTable;
ptIsolation.SaveToDatabase();
var arg1 = _helper.LoadMetadata.ProcessTasks.SelectMany(a => a.ProcessTaskArguments).Single(a => a.Name.Equals("TablesToIsolate"));
arg1.SetValue(new[] { _helper.StudyTableInfo, _helper.SeriesTableInfo, _helper.ImageTableInfo });
arg1.SaveToDatabase();
var db = new ExternalDatabaseServer(CatalogueRepository, "IsolationDatabase(live)", null);
db.SetProperties(server);
var arg2 = _helper.LoadMetadata.ProcessTasks.SelectMany(a => a.ProcessTaskArguments).Single(a => a.Name.Equals("IsolationDatabase"));
arg2.SetValue(db);
arg2.SaveToDatabase();
var arg3 = _helper.LoadMetadata.ProcessTasks.SelectMany(a => a.ProcessTaskArguments).Single(a => a.Name.Equals("ModalityMatchingRegex"));
arg3.SetValue(new Regex("([A-z]*)_.*$"));
arg3.SaveToDatabase();
//build the joins
new JoinInfo(CatalogueRepository,
_helper.ImageTableInfo.ColumnInfos.Single(c => c.GetRuntimeName().Equals("SeriesInstanceUID")),
_helper.SeriesTableInfo.ColumnInfos.Single(c => c.GetRuntimeName().Equals("SeriesInstanceUID")),
ExtractionJoinType.Right, null);
new JoinInfo(CatalogueRepository,
_helper.SeriesTableInfo.ColumnInfos.Single(c => c.GetRuntimeName().Equals("StudyInstanceUID")),
_helper.StudyTableInfo.ColumnInfos.Single(c => c.GetRuntimeName().Equals("StudyInstanceUID")),
ExtractionJoinType.Right, null);
//start with Study table
_helper.StudyTableInfo.IsPrimaryExtractionTable = true;
_helper.StudyTableInfo.SaveToDatabase();
//clean up the directory
foreach (FileInfo f in dir.GetFiles())
{
f.Delete();
}
TestData.Create(new FileInfo(Path.Combine(dir.FullName, "MyTestFile.dcm")));
var ds1 = new DicomDataset();
ds1.Add(DicomTag.StudyInstanceUID, "1.2.3");
ds1.Add(DicomTag.SeriesInstanceUID, "1.2.2");
ds1.Add(DicomTag.SOPInstanceUID, "1.2.3");
ds1.Add(DicomTag.PatientAge, "030Y");
ds1.Add(DicomTag.PatientID, "123");
ds1.Add(DicomTag.SOPClassUID, "1");
ds1.Add(DicomTag.Modality, "MR");
new DicomFile(ds1).Save(Path.Combine(dir.FullName, "abc.dcm"));
var ds2 = new DicomDataset();
ds2.Add(DicomTag.StudyInstanceUID, "1.2.3");
ds2.Add(DicomTag.SeriesInstanceUID, "1.2.4");
ds2.Add(DicomTag.SOPInstanceUID, "1.2.7");
ds2.Add(DicomTag.PatientAge, "040Y"); //age is replicated but should be unique at study level so gets isolated
ds2.Add(DicomTag.PatientID, "123");
ds2.Add(DicomTag.SOPClassUID, "1");
ds2.Add(DicomTag.Modality, "MR");
new DicomFile(ds2).Save(Path.Combine(dir.FullName, "def.dcm"));
var checks = new ProcessTaskChecks(_helper.LoadMetadata);
checks.Check(new AcceptAllCheckNotifier());
RunTest(dir, 1);
Assert.AreEqual(1, _helper.ImageTable.GetRowCount());
var isoTable = server.ExpectTable(_helper.ImageTable.GetRuntimeName() + "_Isolation");
Assert.AreEqual(2, isoTable.GetRowCount());
}
