protected virtual Pipeline SetupPipeline()
{
var repository = RepositoryLocator.CatalogueRepository;
var pipeline = new Pipeline(repository, "Empty extraction pipeline");
var component = new PipelineComponent(repository, pipeline, typeof(ExecuteDatasetExtractionFlatFileDestination), 0, "Destination");
var arguments = component.CreateArgumentsForClassIfNotExists <ExecuteDatasetExtractionFlatFileDestination>().ToArray();
if (arguments.Length < 3)
{
throw new Exception("Expected only 2 arguments for type ExecuteDatasetExtractionFlatFileDestination, did somebody add another [DemandsInitialization]? if so handle it below");
}
arguments.Single(a => a.Name.Equals("DateFormat")).SetValue("yyyy-MM-dd");
arguments.Single(a => a.Name.Equals("DateFormat")).SaveToDatabase();
arguments.Single(a => a.Name.Equals("FlatFileType")).SetValue(ExecuteExtractionToFlatFileType.CSV);
arguments.Single(a => a.Name.Equals("FlatFileType")).SaveToDatabase();
var component2 = new PipelineComponent(repository, pipeline, typeof(ExecuteDatasetExtractionSource), -1, "Source");
var arguments2 = component2.CreateArgumentsForClassIfNotExists <ExecuteDatasetExtractionSource>().ToArray();
arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(AllowEmptyExtractions);
arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase();
//configure the component as the destination
pipeline.DestinationPipelineComponent_ID = component.ID;
pipeline.SourcePipelineComponent_ID = component2.ID;
pipeline.SaveToDatabase();
return(pipeline);
}
private void AddAdvertisedComponent(DragEventArgs e, DividerLineControl divider)
{
var advert = GetAdvertisedObjectFromDragOperation(e);
if (
//if user is trying to add a source
advert.GetRole() == DataFlowComponentVisualisation.Role.Source
//and there is already an explicit source or a configured one
&& (_useCase.ExplicitSource != null || _pipeline.SourcePipelineComponent_ID != null))
{
MessageBox.Show("There is already a source in this pipeline");
return;
}
if (
//if user is trying to add a destination
advert.GetRole() == DataFlowComponentVisualisation.Role.Destination
//and there is already an explicit destination or a configured one
&& (_useCase.ExplicitDestination != null || _pipeline.DestinationPipelineComponent_ID != null))
{
MessageBox.Show("There is already a destination in this pipeline");
return;
}
Type underlyingComponentType = advert.GetComponentType();
//add the component to the pipeline
var repository = (ICatalogueRepository)_pipeline.Repository;
var newcomp = new PipelineComponent(repository, _pipeline, underlyingComponentType, -999, advert.ToString())
{
Order = GetOrderMakingSpaceIfNessesary(null, divider)
};
newcomp.CreateArgumentsForClassIfNotExists(underlyingComponentType);
newcomp.SaveToDatabase();
if (advert.GetRole() == DataFlowComponentVisualisation.Role.Source)
{
_pipeline.SourcePipelineComponent_ID = newcomp.ID;
_pipeline.SaveToDatabase();
}
if (advert.GetRole() == DataFlowComponentVisualisation.Role.Destination)
{
_pipeline.DestinationPipelineComponent_ID = newcomp.ID;
_pipeline.SaveToDatabase();
}
RefreshUIFromDatabase();
var viz = flpPipelineDiagram.Controls.OfType <PipelineComponentVisualisation>().Single(v => Equals(v.PipelineComponent, newcomp));
component_Selected(viz, newcomp);
}
public TestDataPipelineAssembler(string pipeName, CatalogueRepository catalogueRepository)
{
Pipeline = new Pipeline(catalogueRepository, pipeName);
Source = new PipelineComponent(catalogueRepository, Pipeline, typeof(TestDataInventor), 1, "DataInventorSource");
Destination = new PipelineComponent(catalogueRepository, Pipeline, typeof(TestDataWriter), 2, "DataInventorDestination");
Destination.CreateArgumentsForClassIfNotExists <TestDataWriter>();
Pipeline.SourcePipelineComponent_ID = Source.ID;
Pipeline.DestinationPipelineComponent_ID = Destination.ID;
Pipeline.SaveToDatabase();
var args = Source.CreateArgumentsForClassIfNotExists <TestDataInventor>();
args[0].SetValue(TestContext.CurrentContext.TestDirectory);
args[0].SaveToDatabase();
}
protected override Pipeline SetupPipeline()
{
//create a target server pointer
_extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null);
_extractionServer.Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name;
_extractionServer.Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny;
_extractionServer.Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny;
_extractionServer.SaveToDatabase();
//create a pipeline
_pipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline");
//set the destination pipeline
var component = new PipelineComponent(CatalogueRepository, _pipeline, typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination");
var destinationArguments = component.CreateArgumentsForClassIfNotExists <ExecuteFullExtractionToDatabaseMSSql>().ToList();
IArgument argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer");
IArgument argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern");
IArgument argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern");
Assert.AreEqual("TargetDatabaseServer", argumentServer.Name);
argumentServer.SetValue(_extractionServer);
argumentServer.SaveToDatabase();
argumentDbNamePattern.SetValue(TestDatabaseNames.Prefix + "$p_$n");
argumentDbNamePattern.SaveToDatabase();
argumentTblNamePattern.SetValue("$c_$d");
argumentTblNamePattern.SaveToDatabase();
AdjustPipelineComponentDelegate?.Invoke(component);
var component2 = new PipelineComponent(CatalogueRepository, _pipeline, typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source");
var arguments2 = component2.CreateArgumentsForClassIfNotExists <ExecuteCrossServerDatasetExtractionSource>().ToArray();
arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false);
arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase();
AdjustPipelineComponentDelegate?.Invoke(component2);
//configure the component as the destination
_pipeline.DestinationPipelineComponent_ID = component.ID;
_pipeline.SourcePipelineComponent_ID = component2.ID;
_pipeline.SaveToDatabase();
return(_pipeline);
}
private Pipeline CreatePipeline(string nameOfPipe, Type sourceType, Type destinationTypeIfAny, params Type[] componentTypes)
{
if (componentTypes == null || componentTypes.Length == 0)
{
return(CreatePipeline(nameOfPipe, sourceType, destinationTypeIfAny));
}
var pipeline = CreatePipeline(nameOfPipe, sourceType, destinationTypeIfAny);
int i = 1;
foreach (var componentType in componentTypes)
{
var component = new PipelineComponent(_repositoryLocator.CatalogueRepository, pipeline, componentType, i++);
component.CreateArgumentsForClassIfNotExists(componentType);
component.Pipeline_ID = pipeline.ID;
}
return(pipeline);
}
private void PopulateCohortDatabaseWithRecordsFromNonTvfCatalogue()
{
//create a cohort identification configuration (identifies people from datasets using set operations - see CohortManager)
_cic = new CohortIdentificationConfiguration(CatalogueRepository, "TbvfCIC");
_cic.CreateRootContainerIfNotExists();
//turn the catalogue _nonTvfCatalogue into a cohort set and add it to the root container
var newAggregate = _cic.CreateNewEmptyConfigurationForCatalogue(_nonTvfCatalogue, (s, e) => { throw new Exception("Did not expect there to be more than 1!"); });
var root = _cic.RootCohortAggregateContainer;
root.AddChild(newAggregate, 0);
//create a pipeline for executing this CIC and turning it into a cohort
_pipe = new Pipeline(CatalogueRepository, "CREATE COHORT:By Executing CIC");
var source = new PipelineComponent(CatalogueRepository, _pipe, typeof(CohortIdentificationConfigurationSource), 0, "CIC Source");
_project = new Project(DataExportRepository, "TvfProject");
_project.ProjectNumber = 12;
_project.ExtractionDirectory = TestContext.CurrentContext.TestDirectory;
_project.SaveToDatabase();
var destination = new PipelineComponent(CatalogueRepository, _pipe, typeof(BasicCohortDestination), 1, "Destination");
_pipe.SourcePipelineComponent_ID = source.ID;
_pipe.DestinationPipelineComponent_ID = destination.ID;
_pipe.SaveToDatabase();
//create pipeline arguments
source.CreateArgumentsForClassIfNotExists <CohortIdentificationConfigurationSource>();
destination.CreateArgumentsForClassIfNotExists <BasicCohortDestination>();
//create pipeline initialization objects
var request = new CohortCreationRequest(_project, new CohortDefinition(null, "MyFirstCohortForTvfTest", 1, 12, _externalCohortTable), (DataExportRepository)DataExportRepository, "Here goes nothing");
request.CohortIdentificationConfiguration = _cic;
var engine = request.GetEngine(_pipe, new ThrowImmediatelyDataLoadEventListener());
engine.ExecutePipeline(new GracefulCancellationToken());
}
private Pipeline CreatePipeline(string nameOfPipe, Type sourceType, Type destinationTypeIfAny)
{
var pipe = new Pipeline(_repositoryLocator.CatalogueRepository, nameOfPipe);
if (sourceType != null)
{
var source = new PipelineComponent(_repositoryLocator.CatalogueRepository, pipe, sourceType, 0);
source.CreateArgumentsForClassIfNotExists(sourceType);
pipe.SourcePipelineComponent_ID = source.ID;
}
if (destinationTypeIfAny != null)
{
var destination = new PipelineComponent(_repositoryLocator.CatalogueRepository, pipe, destinationTypeIfAny, 100);
destination.CreateArgumentsForClassIfNotExists(destinationTypeIfAny);
pipe.DestinationPipelineComponent_ID = destination.ID;
}
pipe.SaveToDatabase();
return(pipe);
}
public void TestNestedDemandsGetPutIntoDatabaseAndCanBeBroughtBack()
{
var pipe = new Pipeline(CatalogueRepository, "NestedPipe");
var pc = new PipelineComponent(CatalogueRepository, pipe, typeof(BasicDataReleaseDestination), -1,
"Coconuts");
pipe.DestinationPipelineComponent_ID = pc.ID;
pipe.SaveToDatabase();
//some of the DemandsInitialization on BasicDataReleaseDestination should be nested
var f = new ArgumentFactory();
Assert.True(
f.GetRequiredProperties(typeof(BasicDataReleaseDestination)).Any(r => r.ParentPropertyInfo != null));
//new pc should have no arguments
Assert.That(pc.GetAllArguments(), Is.Empty);
//we create them (the root and nested ones!)
var args = pc.CreateArgumentsForClassIfNotExists <BasicDataReleaseDestination>();
//and get all arguments / create arguments for class should have handled that
Assert.That(pc.GetAllArguments().Any());
var match = args.Single(a => a.Name == "ReleaseSettings.DeleteFilesOnSuccess");
match.SetValue(true);
match.SaveToDatabase();
var useCase = ReleaseUseCase.DesignTime();
var factory = new DataFlowPipelineEngineFactory(useCase, RepositoryLocator.CatalogueRepository.MEF);
var destInstance = factory.CreateDestinationIfExists(pipe);
Assert.AreEqual(true, ((BasicDataReleaseDestination)destInstance).ReleaseSettings.DeleteFilesOnSuccess);
}
public void RefreshCohort_WithCaching()
{
ExtractionPipelineUseCase useCase;
IExecuteDatasetExtractionDestination results;
var pipe = new Pipeline(CatalogueRepository, "RefreshPipeWithCaching");
var source = new PipelineComponent(CatalogueRepository, pipe, typeof(CohortIdentificationConfigurationSource), 0);
var args = source.CreateArgumentsForClassIfNotExists <CohortIdentificationConfigurationSource>();
var freezeArg = args.Single(a => a.Name.Equals("FreezeAfterSuccessfulImport"));
freezeArg.SetValue(false);
freezeArg.SaveToDatabase();
var dest = new PipelineComponent(CatalogueRepository, pipe, typeof(BasicCohortDestination), 0);
var argsDest = dest.CreateArgumentsForClassIfNotExists <BasicCohortDestination>();
var allocatorArg = argsDest.Single(a => a.Name.Equals("ReleaseIdentifierAllocator"));
allocatorArg.SetValue(null);
allocatorArg.SaveToDatabase();
pipe.SourcePipelineComponent_ID = source.ID;
pipe.DestinationPipelineComponent_ID = dest.ID;
pipe.SaveToDatabase();
Execute(out useCase, out results);
var oldcohort = _configuration.Cohort;
//Create a query cache
var p = new QueryCachingPatcher();
ExternalDatabaseServer queryCacheServer = new ExternalDatabaseServer(CatalogueRepository, "TestCohortRefreshing_CacheTest", p);
DiscoveredDatabase cachedb = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase("TestCohortRefreshing_CacheTest");
if (cachedb.Exists())
{
cachedb.Drop();
}
new MasterDatabaseScriptExecutor(cachedb).CreateAndPatchDatabase(p, new ThrowImmediatelyCheckNotifier());
queryCacheServer.SetProperties(cachedb);
//Create a Cohort Identification configuration (query) that will identify the cohort
CohortIdentificationConfiguration cic = new CohortIdentificationConfiguration(RepositoryLocator.CatalogueRepository, "RefreshCohort.cs");;
try
{
//make it use the cache
cic.QueryCachingServer_ID = queryCacheServer.ID;
cic.SaveToDatabase();
//give it a single table query to fetch distinct chi from test data
var agg = cic.CreateNewEmptyConfigurationForCatalogue(_catalogue, null);
//add the sub query as the only entry in the cic (in the root container)
cic.CreateRootContainerIfNotExists();
cic.RootCohortAggregateContainer.AddChild(agg, 1);
//make the ExtractionConfiguration refresh cohort query be the cic
_configuration.CohortIdentificationConfiguration_ID = cic.ID;
_configuration.CohortRefreshPipeline_ID = pipe.ID;
_configuration.SaveToDatabase();
//get a refreshing engine
var engine = new CohortRefreshEngine(new ThrowImmediatelyDataLoadEventListener(), _configuration);
engine.Execute();
Assert.NotNull(engine.Request.NewCohortDefinition);
var oldData = oldcohort.GetExternalData();
Assert.AreEqual(oldData.ExternalDescription, engine.Request.NewCohortDefinition.Description);
Assert.AreEqual(oldData.ExternalVersion + 1, engine.Request.NewCohortDefinition.Version);
Assert.AreNotEqual(oldcohort.CountDistinct, engine.Request.CohortCreatedIfAny.CountDistinct);
//now nuke all data in the catalogue so the cic returns nobody (except that the identifiers are cached eh?)
DataAccessPortal.GetInstance().ExpectDatabase(_tableInfo, DataAccessContext.InternalDataProcessing).ExpectTable(_tableInfo.GetRuntimeName()).Truncate();
var toMem = new ToMemoryDataLoadEventListener(false);
//get a new engine
engine = new CohortRefreshEngine(toMem, _configuration);
//execute it
var ex = Assert.Throws <Exception>(() => engine.Execute());
Assert.IsTrue(ex.InnerException.InnerException.Message.Contains("CohortIdentificationCriteria execution resulted in an empty dataset"));
//expected this message to happen
//that it did clear the cache
Assert.AreEqual(1, toMem.EventsReceivedBySender.SelectMany(kvp => kvp.Value).Count(msg => msg.Message.Equals("Clearing Cohort Identifier Cache")));
}
finally
{
//make the ExtractionConfiguration not use the cic query
_configuration.CohortRefreshPipeline_ID = null;
_configuration.CohortIdentificationConfiguration_ID = null;
_configuration.SaveToDatabase();
//delete the cic query
cic.QueryCachingServer_ID = null;
cic.SaveToDatabase();
cic.DeleteInDatabase();
//delete the caching database
queryCacheServer.DeleteInDatabase();
cachedb.Drop();
}
}
public void KVPAttacherTest_Attach(KVPAttacherTestCase testCase)
{
bool hasPk = testCase != KVPAttacherTestCase.OneFileWithoutPrimaryKey;
var db = GetCleanedServer(FAnsi.DatabaseType.MicrosoftSQLServer);
var attacher = new KVPAttacher();
var tbl = db.ExpectTable("KVPTestTable");
var workingDir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory);
var parentDir = workingDir.CreateSubdirectory("KVPAttacherTestProjectDirectory");
var projectDir = LoadDirectory.CreateDirectoryStructure(parentDir, "KVPAttacherTest", true);
string filepk = "kvpTestFilePK.csv";
string filepk2 = "kvpTestFilePK2.csv";
string fileNoPk = "kvpTestFile_NoPK.csv";
if (testCase == KVPAttacherTestCase.OneFileWithPrimaryKey || testCase == KVPAttacherTestCase.TwoFilesWithPrimaryKey)
{
CopyToBin(projectDir, filepk);
}
if (testCase == KVPAttacherTestCase.TwoFilesWithPrimaryKey)
{
CopyToBin(projectDir, filepk2);
}
if (testCase == KVPAttacherTestCase.OneFileWithoutPrimaryKey)
{
CopyToBin(projectDir, fileNoPk);
}
if (tbl.Exists())
{
tbl.Drop();
}
//Create destination data table on server (where the data will ultimately end SetUp)
using (var con = (SqlConnection)tbl.Database.Server.GetConnection())
{
con.Open();
string sql = hasPk
? "CREATE TABLE KVPTestTable (Person varchar(100), Test varchar(50), Result int)"
: "CREATE TABLE KVPTestTable (Test varchar(50), Result int)";
new SqlCommand(sql, con).ExecuteNonQuery();
}
var remnantPipeline =
CatalogueRepository.GetAllObjects <Pipeline>("WHERE Name='KVPAttacherTestPipeline'").SingleOrDefault();
if (remnantPipeline != null)
{
remnantPipeline.DeleteInDatabase();
}
//Setup the Pipeline
var p = new Pipeline(CatalogueRepository, "KVPAttacherTestPipeline");
//With a CSV source
var flatFileLoad = new PipelineComponent(CatalogueRepository, p, typeof(DelimitedFlatFileDataFlowSource), 0, "Data Flow Source");
//followed by a Transpose that turns columns to rows (see how the test file grows right with new records instead of down, this is common in KVP input files but not always)
var transpose = new PipelineComponent(CatalogueRepository, p, typeof(Transposer), 1, "Transposer");
var saneHeaders = transpose.CreateArgumentsForClassIfNotExists(typeof(Transposer)).Single(a => a.Name.Equals("MakeHeaderNamesSane"));
saneHeaders.SetValue(false);
saneHeaders.SaveToDatabase();
//set the source separator to comma
flatFileLoad.CreateArgumentsForClassIfNotExists(typeof(DelimitedFlatFileDataFlowSource));
var arg = flatFileLoad.PipelineComponentArguments.Single(a => a.Name.Equals("Separator"));
arg.SetValue(",");
arg.SaveToDatabase();
arg = flatFileLoad.PipelineComponentArguments.Single(a => a.Name.Equals("MakeHeaderNamesSane"));
arg.SetValue(false);
arg.SaveToDatabase();
p.SourcePipelineComponent_ID = flatFileLoad.ID;
p.SaveToDatabase();
try
{
attacher.PipelineForReadingFromFlatFile = p;
attacher.TableName = "KVPTestTable";
switch (testCase)
{
case KVPAttacherTestCase.OneFileWithPrimaryKey:
attacher.FilePattern = filepk;
break;
case KVPAttacherTestCase.OneFileWithoutPrimaryKey:
attacher.FilePattern = fileNoPk;
break;
case KVPAttacherTestCase.TwoFilesWithPrimaryKey:
attacher.FilePattern = "kvpTestFilePK*.*";
break;
default:
throw new ArgumentOutOfRangeException("testCase");
}
if (hasPk)
{
attacher.PrimaryKeyColumns = "Person";
}
attacher.TargetDataTableKeyColumnName = "Test";
attacher.TargetDataTableValueColumnName = "Result";
attacher.Initialize(projectDir, db);
attacher.Attach(new ThrowImmediatelyDataLoadJob(), new GracefulCancellationToken());
//test file contains 291 values belonging to 3 different people
int expectedRows = 291;
//if we loaded two files (or should have done) then add the number of values in that file (54)
if (testCase == KVPAttacherTestCase.TwoFilesWithPrimaryKey)
{
expectedRows += 54;
}
Assert.AreEqual(expectedRows, tbl.GetRowCount());
}
finally
{
p.DeleteInDatabase();
tbl.Drop();
}
}
public void CloneAPipeline(bool revertAfterClone)
{
Pipeline p = new Pipeline(CatalogueRepository);
var source = new PipelineComponent(CatalogueRepository, p, typeof(DelimitedFlatFileAttacher), 0);
source.CreateArgumentsForClassIfNotExists <DelimitedFlatFileAttacher>();
var middle = new PipelineComponent(CatalogueRepository, p, typeof(ColumnRenamer), 1);
middle.CreateArgumentsForClassIfNotExists <ColumnRenamer>();
var middle2 = new PipelineComponent(CatalogueRepository, p, typeof(ColumnBlacklister), 1);
middle2.CreateArgumentsForClassIfNotExists <ColumnBlacklister>();
var destination = new PipelineComponent(CatalogueRepository, p, typeof(DataTableUploadDestination), 2);
destination.CreateArgumentsForClassIfNotExists <DataTableUploadDestination>();
p.SourcePipelineComponent_ID = source.ID;
p.DestinationPipelineComponent_ID = destination.ID;
p.SaveToDatabase();
int componentsBefore = RepositoryLocator.CatalogueRepository.GetAllObjects <PipelineComponent>().Count();
int argumentsBefore = RepositoryLocator.CatalogueRepository.GetAllObjects <PipelineComponentArgument>().Count();
var arg = p.PipelineComponents.Single(c => c.Class == typeof(ColumnRenamer).ToString()).PipelineComponentArguments.Single(a => a.Name == "ColumnNameToFind");
arg.SetValue("MyMostCoolestColumnEver");
arg.SaveToDatabase();
//Execute the cloning process
var p2 = p.Clone();
if (revertAfterClone)
{
p2.RevertToDatabaseState();
}
Assert.AreNotEqual(p2, p);
Assert.AreNotEqual(p2.ID, p.ID);
Assert.AreEqual(p2.Name, p.Name + " (Clone)");
Assert.AreEqual(componentsBefore * 2, RepositoryLocator.CatalogueRepository.GetAllObjects <PipelineComponent>().Count());
Assert.AreEqual(argumentsBefore * 2, RepositoryLocator.CatalogueRepository.GetAllObjects <PipelineComponentArgument>().Count());
//p the original should have a pipeline component that has the value we set earlier
Assert.AreEqual(
p.PipelineComponents.Single(c => c.Class == typeof(ColumnRenamer).ToString()).PipelineComponentArguments.Single(a => a.Name == "ColumnNameToFind").Value,
"MyMostCoolestColumnEver"
);
//p2 the clone should have a pipeline component too since it's a clone
Assert.AreEqual(
p2.PipelineComponents.Single(c => c.Class == typeof(ColumnRenamer).ToString()).PipelineComponentArguments.Single(a => a.Name == "ColumnNameToFind").Value,
"MyMostCoolestColumnEver"
);
//both should have source and destination components
Assert.NotNull(p2.DestinationPipelineComponent_ID);
Assert.NotNull(p2.SourcePipelineComponent_ID);
//but with different IDs because they are clones
Assert.AreNotEqual(p.DestinationPipelineComponent_ID, p2.DestinationPipelineComponent_ID);
Assert.AreNotEqual(p.SourcePipelineComponent_ID, p2.SourcePipelineComponent_ID);
p.DeleteInDatabase();
p2.DeleteInDatabase();
}
public override void Execute()
{
if (DicomSourceType == null)
{
SetImpossible("You must specify a Type for DicomSourceType");
throw new ImpossibleCommandException(this, ReasonCommandImpossible);
}
base.Execute();
List <DiscoveredTable> tablesCreated = new List <DiscoveredTable>();
//Create with template?
if (Template != null)
{
foreach (ImageTableTemplate table in Template.Tables)
{
string tblName = GetNameWithPrefix(table.TableName);
var tbl = _databaseToCreateInto.ExpectTable(tblName);
var cmd = new ExecuteCommandCreateNewImagingDataset(_repositoryLocator, tbl, table);
cmd.Execute();
NewCataloguesCreated.Add(cmd.NewCatalogueCreated);
tablesCreated.Add(tbl);
}
}
else
{
throw new Exception("No Template provided");
}
//that's us done if we aren't creating a load
if (!CreateLoad)
{
return;
}
string loadName = GetNameWithPrefixInBracketsIfAny("SMI Image Loading");
NewLoadMetadata = new LoadMetadata(_catalogueRepository, loadName);
//tell all the catalogues that they are part of this load and where to log under the same task
foreach (Catalogue c in NewCataloguesCreated)
{
c.LoadMetadata_ID = NewLoadMetadata.ID;
c.LoggingDataTask = loadName;
c.LiveLoggingServer_ID = _loggingServer.ID;
c.SaveToDatabase();
}
//create the logging task
new Core.Logging.LogManager(_loggingServer).CreateNewLoggingTaskIfNotExists(loadName);
var projDir = LoadDirectory.CreateDirectoryStructure(_projectDirectory, "ImageLoading", true);
NewLoadMetadata.LocationOfFlatFiles = projDir.RootPath.FullName;
NewLoadMetadata.SaveToDatabase();
/////////////////////////////////////////////Attacher////////////////////////////
//Create a pipeline for reading from Dicom files and writing to any destination component (which must be fixed)
var pipe = new Pipeline(_catalogueRepository, GetNameWithPrefixInBracketsIfAny("Image Loading Pipe"));
DicomSourcePipelineComponent = new PipelineComponent(_catalogueRepository, pipe, DicomSourceType, 0, DicomSourceType.Name);
DicomSourcePipelineComponent.CreateArgumentsForClassIfNotExists(DicomSourceType);
pipe.SourcePipelineComponent_ID = DicomSourcePipelineComponent.ID;
pipe.SaveToDatabase();
//Create the load process task that uses the pipe to load RAW tables with data from the dicom files
var pt = new ProcessTask(_catalogueRepository, NewLoadMetadata, LoadStage.Mounting);
pt.Name = "Auto Routing Attacher";
pt.ProcessTaskType = ProcessTaskType.Attacher;
pt.Path = PersistentRaw? typeof(AutoRoutingAttacherWithPersistentRaw).FullName: typeof(AutoRoutingAttacher).FullName;
pt.Order = 1;
pt.SaveToDatabase();
var args = PersistentRaw? pt.CreateArgumentsForClassIfNotExists <AutoRoutingAttacherWithPersistentRaw>() : pt.CreateArgumentsForClassIfNotExists <AutoRoutingAttacher>();
SetArgument(args, "LoadPipeline", pipe);
/////////////////////////////////////// Distinct tables on load /////////////////////////
var distincter = new ProcessTask(_catalogueRepository, NewLoadMetadata, LoadStage.AdjustRaw);
var distincterArgs = distincter.CreateArgumentsForClassIfNotExists <Distincter>();
distincter.Name = "Distincter";
distincter.ProcessTaskType = ProcessTaskType.MutilateDataTable;
distincter.Path = typeof(Distincter).FullName;
distincter.Order = 2;
distincter.SaveToDatabase();
SetArgument(distincterArgs, "TableRegexPattern", ".*");
/////////////////////////////////////////////////////////////////////////////////////
if (CreateCoalescer)
{
var coalescer = new ProcessTask(_catalogueRepository, NewLoadMetadata, LoadStage.AdjustRaw);
coalescer.Name = "Coalescer";
coalescer.ProcessTaskType = ProcessTaskType.MutilateDataTable;
coalescer.Path = typeof(Coalescer).FullName;
coalescer.Order = 3;
coalescer.SaveToDatabase();
StringBuilder regexPattern = new StringBuilder();
foreach (var tbl in tablesCreated)
{
if (!tbl.DiscoverColumns().Any(c => c.GetRuntimeName().Equals("SOPInstanceUID", StringComparison.CurrentCultureIgnoreCase)))
{
regexPattern.Append("(" + tbl.GetRuntimeName() + ")|");
}
}
var coalArgs = coalescer.CreateArgumentsForClassIfNotExists <Coalescer>();
SetArgument(coalArgs, "TableRegexPattern", regexPattern.ToString().TrimEnd('|'));
SetArgument(coalArgs, "CreateIndex", true);
}
////////////////////////////////Load Ender (if no rows in load) ////////////////////////////
var prematureLoadEnder = new ProcessTask(_catalogueRepository, NewLoadMetadata, LoadStage.Mounting);
prematureLoadEnder.Name = "Premature Load Ender";
prematureLoadEnder.ProcessTaskType = ProcessTaskType.MutilateDataTable;
prematureLoadEnder.Path = typeof(PrematureLoadEnder).FullName;
prematureLoadEnder.Order = 4;
prematureLoadEnder.SaveToDatabase();
args = prematureLoadEnder.CreateArgumentsForClassIfNotExists <PrematureLoadEnder>();
SetArgument(args, "ExitCodeToReturnIfConditionMet", ExitCodeType.OperationNotRequired);
SetArgument(args, "ConditionsToTerminateUnder", PrematureLoadEndCondition.NoRecordsInAnyTablesInDatabase);
////////////////////////////////////////////////////////////////////////////////////////////////
var checker = new CheckEntireDataLoadProcess(NewLoadMetadata, new HICDatabaseConfiguration(NewLoadMetadata), new HICLoadConfigurationFlags(), _catalogueRepository.MEF);
checker.Check(new AcceptAllCheckNotifier());
}
public override void Execute()
{
base.Execute();
var add = _toAdd;
var order = _order;
// if command doesn't know which to add, ask user
if (add == null)
{
var mef = BasicActivator.RepositoryLocator.CatalogueRepository.MEF;
var context = _useCaseIfAny?.GetContext();
List <Type> offer = new List <Type>();
TypeFilter filter = (t, o) => t.IsGenericType &&
(t.GetGenericTypeDefinition() == typeof(IDataFlowComponent <>) ||
t.GetGenericTypeDefinition() == typeof(IDataFlowSource <>));
//get any source and flow components compatible with any context
offer.AddRange(
mef.GetAllTypes()
.Where(t => !t.IsInterface && !t.IsAbstract)
.Where(t => t.FindInterfaces(filter, null).Any())
.Where(t => context == null || context.IsAllowable(t))
);
if (!BasicActivator.SelectObject("Add", offer.ToArray(), out add))
{
return;
}
}
// Only proceed if we have a component type to add to the pipe
if (add == null)
{
return;
}
// check if it is a source or destination (or if both are false it is a middle component)
TypeFilter sourceFilter = (t, o) =>
t.IsGenericType && t.GetGenericTypeDefinition() == typeof(IDataFlowSource <>);
TypeFilter destFilter = (t, o) =>
t.IsGenericType && t.GetGenericTypeDefinition() == typeof(IDataFlowDestination <>);
var isSource = add.FindInterfaces(sourceFilter, null).Any();
var isDest = add.FindInterfaces(destFilter, null).Any();
if (isSource)
{
order = int.MinValue;
if (_pipeline.SourcePipelineComponent_ID.HasValue)
{
throw new Exception($"Pipeline '{_pipeline}' already has a source");
}
}
if (isDest)
{
order = int.MaxValue;
if (_pipeline.DestinationPipelineComponent_ID.HasValue)
{
throw new Exception($"Pipeline '{_pipeline}' already has a destination");
}
}
// if we don't know the order yet and it's important
if (!order.HasValue && !isDest && !isSource)
{
if (BasicActivator.SelectValueType("Order", typeof(int), 0, out object chosen))
{
order = (int)chosen;
}
else
{
return;
}
}
var newComponent = new PipelineComponent(BasicActivator.RepositoryLocator.CatalogueRepository, _pipeline,
add, order ?? 0);
newComponent.CreateArgumentsForClassIfNotExists(add);
if (isSource)
{
_pipeline.SourcePipelineComponent_ID = newComponent.ID;
_pipeline.SaveToDatabase();
}
if (isDest)
{
_pipeline.DestinationPipelineComponent_ID = newComponent.ID;
_pipeline.SaveToDatabase();
}
Publish(newComponent);
Emphasise(newComponent);
}
