private void SetupPipelineSelectionExtraction() { //already set i tup if (_extractionPipelineSelectionUI != null) { return; } //the use case is extracting a dataset var useCase = ExtractionPipelineUseCase.DesignTime(); //the user is DefaultPipeline_ID field of ExtractionConfiguration var user = new PipelineUser(typeof(ExtractionConfiguration).GetProperty("DefaultPipeline_ID"), _extractionConfiguration); //create the UI for this situation var factory = new PipelineSelectionUIFactory(Activator.RepositoryLocator.CatalogueRepository, user, useCase); _extractionPipelineSelectionUI = factory.Create(Activator, "Extraction Pipeline", DockStyle.Fill, pChooseExtractionPipeline); _extractionPipelineSelectionUI.CollapseToSingleLineMode(); }
protected override void AdjustCommand(ExtractionOptions opts, CommandLineActivity activity) { base.AdjustCommand(opts, activity); var useCase = ExtractionPipelineUseCase.DesignTime(); var compatible = useCase.FilterCompatiblePipelines(BasicActivator.RepositoryLocator.CatalogueRepository.GetAllObjects <Pipeline>()).ToArray(); if (!compatible.Any()) { throw new Exception("No compatible pipelines"); } var pipe = BasicActivator.SelectOne("Extraction Pipeline", compatible, null, true); if (pipe == null) { throw new OperationCanceledException(); } opts.Pipeline = pipe.ID; }
public override void SetDatabaseObject(IActivateItems activator, ExtractionConfiguration databaseObject) { base.SetDatabaseObject(activator, databaseObject); _extractionConfiguration = databaseObject; if (!_commonFunctionality.IsSetup) { _commonFunctionality.SetUp(RDMPCollection.None, tlvDatasets, activator, olvName, null, new RDMPCollectionCommonFunctionalitySettings() { AddFavouriteColumn = false, AllowPinning = false, SuppressChildrenAdder = true, SuppressActivate = true, AddCheckColumn = false }); } var checkedBefore = tlvDatasets.CheckedObjects; tlvDatasets.ClearObjects(); _globals = _extractionConfiguration.GetGlobals(); _datasets = databaseObject.SelectedDataSets.ToArray(); GetBundledStuff(); //add the folders tlvDatasets.AddObjects(new object[] { _globalsFolder, _coreDatasetsFolder, _projectSpecificDatasetsFolder }); //enable all to start with tlvDatasets.EnableObjects(tlvDatasets.Objects); tlvDatasets.DisableObjects(_globals); tlvDatasets.DisableObjects(_bundledStuff); //if there are no globals disable this option if (!_globals.Any()) { tlvDatasets.DisableObject(_globalsFolder); } //if there are no project specific datasets if (_datasets.All(sds => sds.ExtractableDataSet.Project_ID == null)) { tlvDatasets.DisableObject(_projectSpecificDatasetsFolder); //disable this option } //if all the datasets are project specific if (_datasets.All(sds => sds.ExtractableDataSet.Project_ID != null)) { tlvDatasets.DisableObject(_coreDatasetsFolder); } //don't accept refresh while executing if (checkAndExecuteUI1.IsExecuting) { return; } if (_pipelineSelectionUI1 == null) { //create a new selection UI (pick an extraction pipeliene UI) var useCase = ExtractionPipelineUseCase.DesignTime(); var factory = new PipelineSelectionUIFactory(Activator.RepositoryLocator.CatalogueRepository, null, useCase); _pipelineSelectionUI1 = factory.Create("Extraction Pipeline", DockStyle.Fill); _pipelineSelectionUI1.CollapseToSingleLineMode(); //if the configuration has a default then use that pipeline if (_extractionConfiguration.DefaultPipeline_ID != null) { _pipelineSelectionUI1.Pipeline = _extractionConfiguration.DefaultPipeline; } _pipelineSelectionUI1.PipelineChanged += ResetChecksUI; _pipelinePanel = new ToolStripControlHost((Control)_pipelineSelectionUI1); helpIcon1.SetHelpText("Extraction", "It is a wise idea to click here if you don't know what this screen can do for you...", BuildHelpFlow()); } CommonFunctionality.Add(new ToolStripLabel("Extraction Pipeline:")); CommonFunctionality.Add(_pipelinePanel); CommonFunctionality.AddHelpStringToToolStrip("Extraction Pipeline", "The sequence of components that will be executed in order to enable the datasets to be extracted. This will start with a source component that performs the linkage against the cohort followed by subsequent components (if any) and then a destination component that writes the final records (e.g. to database / csv file etc)."); CommonFunctionality.AddToMenu(new ExecuteCommandRelease(activator).SetTarget(_extractionConfiguration)); CommonFunctionality.Add(lblMaxConcurrent); CommonFunctionality.Add(tbMaxConcurrent); CommonFunctionality.AddHelpStringToToolStrip("Concurrent", "The maximum number of datasets to extract at once. Once this number is reached the remainder will be queued and only started when one of the other extracting datasets completes."); checkAndExecuteUI1.SetItemActivator(activator); tlvDatasets.ExpandAll(); if (_isFirstTime) { tlvDatasets.CheckAll(); foreach (var disabledObject in tlvDatasets.DisabledObjects.OfType <ArbitraryFolderNode>()) { tlvDatasets.UncheckObject(disabledObject); } } else if (checkedBefore.Count > 0) { tlvDatasets.CheckObjects(checkedBefore); } _isFirstTime = false; }
public DataExportChildProvider(IRDMPPlatformRepositoryServiceLocator repositoryLocator, IChildProvider[] pluginChildProviders, ICheckNotifier errorsCheckNotifier, DataExportChildProvider previousStateIfKnown) : base(repositoryLocator.CatalogueRepository, pluginChildProviders, errorsCheckNotifier, previousStateIfKnown) { BlackListedSources = previousStateIfKnown?.BlackListedSources ?? new List <ExternalCohortTable>(); _errorsCheckNotifier = errorsCheckNotifier; dataExportRepository = repositoryLocator.DataExportRepository; AllProjectAssociatedCics = GetAllObjects <ProjectCohortIdentificationConfigurationAssociation>(dataExportRepository); _cicAssociations = new HashSet <int>(AllProjectAssociatedCics.Select(a => a.CohortIdentificationConfiguration_ID)); CohortSources = GetAllObjects <ExternalCohortTable>(dataExportRepository); ExtractableDataSets = GetAllObjects <ExtractableDataSet>(dataExportRepository); AddToDictionaries(new HashSet <object>(AllCohortIdentificationConfigurations.Where(cic => _cicAssociations.Contains(cic.ID))), new DescendancyList(AllProjectCohortIdentificationConfigurationsNode)); AddToDictionaries(new HashSet <object>(AllCohortIdentificationConfigurations.Where(cic => !_cicAssociations.Contains(cic.ID))), new DescendancyList(AllFreeCohortIdentificationConfigurationsNode)); _selectedDataSetsWithNoIsExtractionIdentifier = new HashSet <ISelectedDataSets>(dataExportRepository.GetSelectedDatasetsWithNoExtractionIdentifiers()); SelectedDataSets = GetAllObjects <SelectedDataSets>(dataExportRepository); ReportProgress("Fetching data export objects"); var dsDictionary = ExtractableDataSets.ToDictionary(ds => ds.ID, d => d); foreach (SelectedDataSets s in SelectedDataSets) { s.InjectKnown(dsDictionary[s.ExtractableDataSet_ID]); } ReportProgress("Injecting SelectedDataSets"); //This means that the ToString method in ExtractableDataSet doesn't need to go lookup catalogue info var catalogueIdDict = AllCatalogues.ToDictionary(c => c.ID, c2 => c2); foreach (ExtractableDataSet ds in ExtractableDataSets) { if (catalogueIdDict.TryGetValue(ds.Catalogue_ID, out Catalogue cata)) { ds.InjectKnown(cata); } } ReportProgress("Injecting ExtractableDataSet"); AllPackages = GetAllObjects <ExtractableDataSetPackage>(dataExportRepository); Projects = GetAllObjects <Project>(dataExportRepository); ExtractionConfigurations = GetAllObjects <ExtractionConfiguration>(dataExportRepository); ReportProgress("Get Projects and Configurations"); ExtractionConfigurationsByProject = ExtractionConfigurations.GroupBy(k => k.Project_ID).ToDictionary(gdc => gdc.Key, gdc => gdc.ToList()); ReportProgress("Grouping Extractions by Project"); AllGlobalExtractionFilterParameters = GetAllObjects <GlobalExtractionFilterParameter>(dataExportRepository); AllContainers = GetAllObjects <FilterContainer>(dataExportRepository).ToDictionary(o => o.ID, o => o); AllDeployedExtractionFilters = GetAllObjects <DeployedExtractionFilter>(dataExportRepository); _allParameters = GetAllObjects <DeployedExtractionFilterParameter>(dataExportRepository); ReportProgress("Getting Filters"); //if we are using a database repository then we can make use of the caching class DataExportFilterManagerFromChildProvider to speed up //filter contents var dbRepo = dataExportRepository as DataExportRepository; _dataExportFilterManager = dbRepo == null ? dataExportRepository.FilterManager : new DataExportFilterManagerFromChildProvider(dbRepo, this); ReportProgress("Building FilterManager"); Cohorts = GetAllObjects <ExtractableCohort>(dataExportRepository); _cohortsByOriginId = new Dictionary <int, HashSet <ExtractableCohort> >(); foreach (ExtractableCohort c in Cohorts) { if (!_cohortsByOriginId.ContainsKey(c.OriginID)) { _cohortsByOriginId.Add(c.OriginID, new HashSet <ExtractableCohort>()); } _cohortsByOriginId[c.OriginID].Add(c); } _configurationToDatasetMapping = new Dictionary <ExtractionConfiguration, List <SelectedDataSets> >(); ReportProgress("Fetching Cohorts"); GetCohortAvailability(); ReportProgress("GetCohortAvailability"); var configToSds = SelectedDataSets.GroupBy(k => k.ExtractionConfiguration_ID).ToDictionary(gdc => gdc.Key, gdc => gdc.ToList()); foreach (ExtractionConfiguration configuration in ExtractionConfigurations) { if (configToSds.TryGetValue(configuration.ID, out List <SelectedDataSets> result)) { _configurationToDatasetMapping.Add(configuration, result); } } ReportProgress("Mapping configurations to datasets"); RootCohortsNode = new AllCohortsNode(); AddChildren(RootCohortsNode, new DescendancyList(RootCohortsNode)); foreach (ExtractableDataSetPackage package in AllPackages) { AddChildren(package, new DescendancyList(package)); } ReportProgress("Packages and Cohorts"); foreach (Project p in Projects) { AddChildren(p, new DescendancyList(p)); } ReportProgress("Projects"); //work out all the Catalogues that are extractable (Catalogues are extractable if there is an ExtractableDataSet with the Catalogue_ID that matches them) var cataToEds = new Dictionary <int, ExtractableDataSet>(ExtractableDataSets.ToDictionary(k => k.Catalogue_ID)); //inject extractability into Catalogues foreach (Catalogue catalogue in AllCatalogues) { if (cataToEds.TryGetValue(catalogue.ID, out ExtractableDataSet result)) { catalogue.InjectKnown(result.GetCatalogueExtractabilityStatus()); } else { catalogue.InjectKnown(new CatalogueExtractabilityStatus(false, false)); } } ReportProgress("Catalogue extractability injection"); try { AddPipelineUseCases(new Dictionary <string, PipelineUseCase> { { "File Import", UploadFileUseCase.DesignTime() }, { "Extraction", ExtractionPipelineUseCase.DesignTime() }, { "Release", ReleaseUseCase.DesignTime() }, { "Cohort Creation", CohortCreationRequest.DesignTime() }, { "Caching", CachingPipelineUseCase.DesignTime() }, { "Aggregate Committing", CreateTableFromAggregateUseCase.DesignTime(repositoryLocator.CatalogueRepository) } }); } catch (Exception ex) { _errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs("Failed to build DesignTime PipelineUseCases", CheckResult.Fail, ex)); } ReportProgress("Pipeline adding"); }