private void SetupPipelineSelectionExtraction()
{
//already set i tup
if (_extractionPipelineSelectionUI != null)
{
return;
}
//the use case is extracting a dataset
var useCase = ExtractionPipelineUseCase.DesignTime();
//the user is DefaultPipeline_ID field of ExtractionConfiguration
var user = new PipelineUser(typeof(ExtractionConfiguration).GetProperty("DefaultPipeline_ID"), _extractionConfiguration);
//create the UI for this situation
var factory = new PipelineSelectionUIFactory(Activator.RepositoryLocator.CatalogueRepository, user, useCase);
_extractionPipelineSelectionUI = factory.Create(Activator, "Extraction Pipeline", DockStyle.Fill, pChooseExtractionPipeline);
_extractionPipelineSelectionUI.CollapseToSingleLineMode();
}
protected override void AdjustCommand(ExtractionOptions opts, CommandLineActivity activity)
{
base.AdjustCommand(opts, activity);
var useCase = ExtractionPipelineUseCase.DesignTime();
var compatible = useCase.FilterCompatiblePipelines(BasicActivator.RepositoryLocator.CatalogueRepository.GetAllObjects <Pipeline>()).ToArray();
if (!compatible.Any())
{
throw new Exception("No compatible pipelines");
}
var pipe = BasicActivator.SelectOne("Extraction Pipeline", compatible, null, true);
if (pipe == null)
{
throw new OperationCanceledException();
}
opts.Pipeline = pipe.ID;
}
public override void SetDatabaseObject(IActivateItems activator, ExtractionConfiguration databaseObject)
{
base.SetDatabaseObject(activator, databaseObject);
_extractionConfiguration = databaseObject;
if (!_commonFunctionality.IsSetup)
{
_commonFunctionality.SetUp(RDMPCollection.None, tlvDatasets, activator, olvName, null, new RDMPCollectionCommonFunctionalitySettings()
{
AddFavouriteColumn = false,
AllowPinning = false,
SuppressChildrenAdder = true,
SuppressActivate = true,
AddCheckColumn = false
});
}
var checkedBefore = tlvDatasets.CheckedObjects;
tlvDatasets.ClearObjects();
_globals = _extractionConfiguration.GetGlobals();
_datasets = databaseObject.SelectedDataSets.ToArray();
GetBundledStuff();
//add the folders
tlvDatasets.AddObjects(new object[] { _globalsFolder, _coreDatasetsFolder, _projectSpecificDatasetsFolder });
//enable all to start with
tlvDatasets.EnableObjects(tlvDatasets.Objects);
tlvDatasets.DisableObjects(_globals);
tlvDatasets.DisableObjects(_bundledStuff);
//if there are no globals disable this option
if (!_globals.Any())
{
tlvDatasets.DisableObject(_globalsFolder);
}
//if there are no project specific datasets
if (_datasets.All(sds => sds.ExtractableDataSet.Project_ID == null))
{
tlvDatasets.DisableObject(_projectSpecificDatasetsFolder); //disable this option
}
//if all the datasets are project specific
if (_datasets.All(sds => sds.ExtractableDataSet.Project_ID != null))
{
tlvDatasets.DisableObject(_coreDatasetsFolder);
}
//don't accept refresh while executing
if (checkAndExecuteUI1.IsExecuting)
{
return;
}
if (_pipelineSelectionUI1 == null)
{
//create a new selection UI (pick an extraction pipeliene UI)
var useCase = ExtractionPipelineUseCase.DesignTime();
var factory = new PipelineSelectionUIFactory(Activator.RepositoryLocator.CatalogueRepository, null, useCase);
_pipelineSelectionUI1 = factory.Create("Extraction Pipeline", DockStyle.Fill);
_pipelineSelectionUI1.CollapseToSingleLineMode();
//if the configuration has a default then use that pipeline
if (_extractionConfiguration.DefaultPipeline_ID != null)
{
_pipelineSelectionUI1.Pipeline = _extractionConfiguration.DefaultPipeline;
}
_pipelineSelectionUI1.PipelineChanged += ResetChecksUI;
_pipelinePanel = new ToolStripControlHost((Control)_pipelineSelectionUI1);
helpIcon1.SetHelpText("Extraction", "It is a wise idea to click here if you don't know what this screen can do for you...", BuildHelpFlow());
}
CommonFunctionality.Add(new ToolStripLabel("Extraction Pipeline:"));
CommonFunctionality.Add(_pipelinePanel);
CommonFunctionality.AddHelpStringToToolStrip("Extraction Pipeline", "The sequence of components that will be executed in order to enable the datasets to be extracted. This will start with a source component that performs the linkage against the cohort followed by subsequent components (if any) and then a destination component that writes the final records (e.g. to database / csv file etc).");
CommonFunctionality.AddToMenu(new ExecuteCommandRelease(activator).SetTarget(_extractionConfiguration));
CommonFunctionality.Add(lblMaxConcurrent);
CommonFunctionality.Add(tbMaxConcurrent);
CommonFunctionality.AddHelpStringToToolStrip("Concurrent", "The maximum number of datasets to extract at once. Once this number is reached the remainder will be queued and only started when one of the other extracting datasets completes.");
checkAndExecuteUI1.SetItemActivator(activator);
tlvDatasets.ExpandAll();
if (_isFirstTime)
{
tlvDatasets.CheckAll();
foreach (var disabledObject in tlvDatasets.DisabledObjects.OfType <ArbitraryFolderNode>())
{
tlvDatasets.UncheckObject(disabledObject);
}
}
else if (checkedBefore.Count > 0)
{
tlvDatasets.CheckObjects(checkedBefore);
}
_isFirstTime = false;
}
public DataExportChildProvider(IRDMPPlatformRepositoryServiceLocator repositoryLocator, IChildProvider[] pluginChildProviders, ICheckNotifier errorsCheckNotifier, DataExportChildProvider previousStateIfKnown) : base(repositoryLocator.CatalogueRepository, pluginChildProviders, errorsCheckNotifier, previousStateIfKnown)
{
BlackListedSources = previousStateIfKnown?.BlackListedSources ?? new List <ExternalCohortTable>();
_errorsCheckNotifier = errorsCheckNotifier;
dataExportRepository = repositoryLocator.DataExportRepository;
AllProjectAssociatedCics = GetAllObjects <ProjectCohortIdentificationConfigurationAssociation>(dataExportRepository);
_cicAssociations = new HashSet <int>(AllProjectAssociatedCics.Select(a => a.CohortIdentificationConfiguration_ID));
CohortSources = GetAllObjects <ExternalCohortTable>(dataExportRepository);
ExtractableDataSets = GetAllObjects <ExtractableDataSet>(dataExportRepository);
AddToDictionaries(new HashSet <object>(AllCohortIdentificationConfigurations.Where(cic => _cicAssociations.Contains(cic.ID))), new DescendancyList(AllProjectCohortIdentificationConfigurationsNode));
AddToDictionaries(new HashSet <object>(AllCohortIdentificationConfigurations.Where(cic => !_cicAssociations.Contains(cic.ID))), new DescendancyList(AllFreeCohortIdentificationConfigurationsNode));
_selectedDataSetsWithNoIsExtractionIdentifier = new HashSet <ISelectedDataSets>(dataExportRepository.GetSelectedDatasetsWithNoExtractionIdentifiers());
SelectedDataSets = GetAllObjects <SelectedDataSets>(dataExportRepository);
ReportProgress("Fetching data export objects");
var dsDictionary = ExtractableDataSets.ToDictionary(ds => ds.ID, d => d);
foreach (SelectedDataSets s in SelectedDataSets)
{
s.InjectKnown(dsDictionary[s.ExtractableDataSet_ID]);
}
ReportProgress("Injecting SelectedDataSets");
//This means that the ToString method in ExtractableDataSet doesn't need to go lookup catalogue info
var catalogueIdDict = AllCatalogues.ToDictionary(c => c.ID, c2 => c2);
foreach (ExtractableDataSet ds in ExtractableDataSets)
{
if (catalogueIdDict.TryGetValue(ds.Catalogue_ID, out Catalogue cata))
{
ds.InjectKnown(cata);
}
}
ReportProgress("Injecting ExtractableDataSet");
AllPackages = GetAllObjects <ExtractableDataSetPackage>(dataExportRepository);
Projects = GetAllObjects <Project>(dataExportRepository);
ExtractionConfigurations = GetAllObjects <ExtractionConfiguration>(dataExportRepository);
ReportProgress("Get Projects and Configurations");
ExtractionConfigurationsByProject = ExtractionConfigurations.GroupBy(k => k.Project_ID).ToDictionary(gdc => gdc.Key, gdc => gdc.ToList());
ReportProgress("Grouping Extractions by Project");
AllGlobalExtractionFilterParameters = GetAllObjects <GlobalExtractionFilterParameter>(dataExportRepository);
AllContainers = GetAllObjects <FilterContainer>(dataExportRepository).ToDictionary(o => o.ID, o => o);
AllDeployedExtractionFilters = GetAllObjects <DeployedExtractionFilter>(dataExportRepository);
_allParameters = GetAllObjects <DeployedExtractionFilterParameter>(dataExportRepository);
ReportProgress("Getting Filters");
//if we are using a database repository then we can make use of the caching class DataExportFilterManagerFromChildProvider to speed up
//filter contents
var dbRepo = dataExportRepository as DataExportRepository;
_dataExportFilterManager = dbRepo == null ? dataExportRepository.FilterManager : new DataExportFilterManagerFromChildProvider(dbRepo, this);
ReportProgress("Building FilterManager");
Cohorts = GetAllObjects <ExtractableCohort>(dataExportRepository);
_cohortsByOriginId = new Dictionary <int, HashSet <ExtractableCohort> >();
foreach (ExtractableCohort c in Cohorts)
{
if (!_cohortsByOriginId.ContainsKey(c.OriginID))
{
_cohortsByOriginId.Add(c.OriginID, new HashSet <ExtractableCohort>());
}
_cohortsByOriginId[c.OriginID].Add(c);
}
_configurationToDatasetMapping = new Dictionary <ExtractionConfiguration, List <SelectedDataSets> >();
ReportProgress("Fetching Cohorts");
GetCohortAvailability();
ReportProgress("GetCohortAvailability");
var configToSds = SelectedDataSets.GroupBy(k => k.ExtractionConfiguration_ID).ToDictionary(gdc => gdc.Key, gdc => gdc.ToList());
foreach (ExtractionConfiguration configuration in ExtractionConfigurations)
{
if (configToSds.TryGetValue(configuration.ID, out List <SelectedDataSets> result))
{
_configurationToDatasetMapping.Add(configuration, result);
}
}
ReportProgress("Mapping configurations to datasets");
RootCohortsNode = new AllCohortsNode();
AddChildren(RootCohortsNode, new DescendancyList(RootCohortsNode));
foreach (ExtractableDataSetPackage package in AllPackages)
{
AddChildren(package, new DescendancyList(package));
}
ReportProgress("Packages and Cohorts");
foreach (Project p in Projects)
{
AddChildren(p, new DescendancyList(p));
}
ReportProgress("Projects");
//work out all the Catalogues that are extractable (Catalogues are extractable if there is an ExtractableDataSet with the Catalogue_ID that matches them)
var cataToEds = new Dictionary <int, ExtractableDataSet>(ExtractableDataSets.ToDictionary(k => k.Catalogue_ID));
//inject extractability into Catalogues
foreach (Catalogue catalogue in AllCatalogues)
{
if (cataToEds.TryGetValue(catalogue.ID, out ExtractableDataSet result))
{
catalogue.InjectKnown(result.GetCatalogueExtractabilityStatus());
}
else
{
catalogue.InjectKnown(new CatalogueExtractabilityStatus(false, false));
}
}
ReportProgress("Catalogue extractability injection");
try
{
AddPipelineUseCases(new Dictionary <string, PipelineUseCase>
{
{ "File Import", UploadFileUseCase.DesignTime() },
{ "Extraction", ExtractionPipelineUseCase.DesignTime() },
{ "Release", ReleaseUseCase.DesignTime() },
{ "Cohort Creation", CohortCreationRequest.DesignTime() },
{ "Caching", CachingPipelineUseCase.DesignTime() },
{ "Aggregate Committing", CreateTableFromAggregateUseCase.DesignTime(repositoryLocator.CatalogueRepository) }
});
}
catch (Exception ex)
{
_errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs("Failed to build DesignTime PipelineUseCases", CheckResult.Fail, ex));
}
ReportProgress("Pipeline adding");
}