private CohortIdentificationConfiguration CreateCohortIdentificationConfiguration(ExtractionFilter inclusionFilter1) { //Create the top level configuration object var cic = new CohortIdentificationConfiguration(_repos.CatalogueRepository, "Tayside Lung Cancer Cohort"); //create a UNION container for Inclusion Criteria var container = new CohortAggregateContainer(_repos.CatalogueRepository, SetOperation.UNION); container.Name = "Inclusion Criteria"; container.SaveToDatabase(); cic.RootCohortAggregateContainer_ID = container.ID; cic.SaveToDatabase(); //Create a new cohort set to the 'Inclusion Criteria' based on the filters Catalogue var cata = inclusionFilter1.ExtractionInformation.CatalogueItem.Catalogue; var ac = cic.CreateNewEmptyConfigurationForCatalogue(cata, (a, b) => { throw new Exception("Problem encountered with chi column(s)"); }, false); container.AddChild(ac, 0); //Add the filter to the WHERE logic of the cohort set var whereContainer = new AggregateFilterContainer(_repos.CatalogueRepository, FilterContainerOperation.OR); ac.Name = "People with " + inclusionFilter1.Name; ac.RootFilterContainer_ID = whereContainer.ID; cic.EnsureNamingConvention(ac); //this will put cicx at the front and cause implicit SaveToDatabase FilterImporter filterImporter = new FilterImporter(new AggregateFilterFactory(_repos.CatalogueRepository), null); var cloneFilter = filterImporter.ImportFilter(inclusionFilter1, null); whereContainer.AddChild(cloneFilter); return(cic); }
public AggregateConfigurationCombineable GenerateAggregateConfigurationFor(IBasicActivateItems activator, CohortIdentificationConfiguration cic, bool importMandatoryFilters = true, [CallerMemberName] string caller = null) { var newAggregate = cic.CreateNewEmptyConfigurationForCatalogue(Catalogue, ResolveMultipleExtractionIdentifiers ?? ((a,b)=> CohortCombineToCreateCommandHelper.PickOneExtractionIdentifier(activator,a, b)), importMandatoryFilters); return new AggregateConfigurationCombineable(newAggregate); }
private void PopulateCohortDatabaseWithRecordsFromNonTvfCatalogue() { //create a cohort identification configuration (identifies people from datasets using set operations - see CohortManager) _cic = new CohortIdentificationConfiguration(CatalogueRepository, "TbvfCIC"); _cic.CreateRootContainerIfNotExists(); //turn the catalogue _nonTvfCatalogue into a cohort set and add it to the root container var newAggregate = _cic.CreateNewEmptyConfigurationForCatalogue(_nonTvfCatalogue, (s, e) => { throw new Exception("Did not expect there to be more than 1!"); }); var root = _cic.RootCohortAggregateContainer; root.AddChild(newAggregate, 0); //create a pipeline for executing this CIC and turning it into a cohort _pipe = new Pipeline(CatalogueRepository, "CREATE COHORT:By Executing CIC"); var source = new PipelineComponent(CatalogueRepository, _pipe, typeof(CohortIdentificationConfigurationSource), 0, "CIC Source"); _project = new Project(DataExportRepository, "TvfProject"); _project.ProjectNumber = 12; _project.ExtractionDirectory = TestContext.CurrentContext.TestDirectory; _project.SaveToDatabase(); var destination = new PipelineComponent(CatalogueRepository, _pipe, typeof(BasicCohortDestination), 1, "Destination"); _pipe.SourcePipelineComponent_ID = source.ID; _pipe.DestinationPipelineComponent_ID = destination.ID; _pipe.SaveToDatabase(); //create pipeline arguments source.CreateArgumentsForClassIfNotExists <CohortIdentificationConfigurationSource>(); destination.CreateArgumentsForClassIfNotExists <BasicCohortDestination>(); //create pipeline initialization objects var request = new CohortCreationRequest(_project, new CohortDefinition(null, "MyFirstCohortForTvfTest", 1, 12, _externalCohortTable), (DataExportRepository)DataExportRepository, "Here goes nothing"); request.CohortIdentificationConfiguration = _cic; var engine = request.GetEngine(_pipe, new ThrowImmediatelyDataLoadEventListener()); engine.ExecutePipeline(new GracefulCancellationToken()); }
public void RefreshCohort_WithCaching() { ExtractionPipelineUseCase useCase; IExecuteDatasetExtractionDestination results; var pipe = new Pipeline(CatalogueRepository, "RefreshPipeWithCaching"); var source = new PipelineComponent(CatalogueRepository, pipe, typeof(CohortIdentificationConfigurationSource), 0); var args = source.CreateArgumentsForClassIfNotExists <CohortIdentificationConfigurationSource>(); var freezeArg = args.Single(a => a.Name.Equals("FreezeAfterSuccessfulImport")); freezeArg.SetValue(false); freezeArg.SaveToDatabase(); var dest = new PipelineComponent(CatalogueRepository, pipe, typeof(BasicCohortDestination), 0); var argsDest = dest.CreateArgumentsForClassIfNotExists <BasicCohortDestination>(); var allocatorArg = argsDest.Single(a => a.Name.Equals("ReleaseIdentifierAllocator")); allocatorArg.SetValue(null); allocatorArg.SaveToDatabase(); pipe.SourcePipelineComponent_ID = source.ID; pipe.DestinationPipelineComponent_ID = dest.ID; pipe.SaveToDatabase(); Execute(out useCase, out results); var oldcohort = _configuration.Cohort; //Create a query cache var p = new QueryCachingPatcher(); ExternalDatabaseServer queryCacheServer = new ExternalDatabaseServer(CatalogueRepository, "TestCohortRefreshing_CacheTest", p); DiscoveredDatabase cachedb = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase("TestCohortRefreshing_CacheTest"); if (cachedb.Exists()) { cachedb.Drop(); } new MasterDatabaseScriptExecutor(cachedb).CreateAndPatchDatabase(p, new ThrowImmediatelyCheckNotifier()); queryCacheServer.SetProperties(cachedb); //Create a Cohort Identification configuration (query) that will identify the cohort CohortIdentificationConfiguration cic = new CohortIdentificationConfiguration(RepositoryLocator.CatalogueRepository, "RefreshCohort.cs");; try { //make it use the cache cic.QueryCachingServer_ID = queryCacheServer.ID; cic.SaveToDatabase(); //give it a single table query to fetch distinct chi from test data var agg = cic.CreateNewEmptyConfigurationForCatalogue(_catalogue, null); //add the sub query as the only entry in the cic (in the root container) cic.CreateRootContainerIfNotExists(); cic.RootCohortAggregateContainer.AddChild(agg, 1); //make the ExtractionConfiguration refresh cohort query be the cic _configuration.CohortIdentificationConfiguration_ID = cic.ID; _configuration.CohortRefreshPipeline_ID = pipe.ID; _configuration.SaveToDatabase(); //get a refreshing engine var engine = new CohortRefreshEngine(new ThrowImmediatelyDataLoadEventListener(), _configuration); engine.Execute(); Assert.NotNull(engine.Request.NewCohortDefinition); var oldData = oldcohort.GetExternalData(); Assert.AreEqual(oldData.ExternalDescription, engine.Request.NewCohortDefinition.Description); Assert.AreEqual(oldData.ExternalVersion + 1, engine.Request.NewCohortDefinition.Version); Assert.AreNotEqual(oldcohort.CountDistinct, engine.Request.CohortCreatedIfAny.CountDistinct); //now nuke all data in the catalogue so the cic returns nobody (except that the identifiers are cached eh?) DataAccessPortal.GetInstance().ExpectDatabase(_tableInfo, DataAccessContext.InternalDataProcessing).ExpectTable(_tableInfo.GetRuntimeName()).Truncate(); var toMem = new ToMemoryDataLoadEventListener(false); //get a new engine engine = new CohortRefreshEngine(toMem, _configuration); //execute it var ex = Assert.Throws <Exception>(() => engine.Execute()); Assert.IsTrue(ex.InnerException.InnerException.Message.Contains("CohortIdentificationCriteria execution resulted in an empty dataset")); //expected this message to happen //that it did clear the cache Assert.AreEqual(1, toMem.EventsReceivedBySender.SelectMany(kvp => kvp.Value).Count(msg => msg.Message.Equals("Clearing Cohort Identifier Cache"))); } finally { //make the ExtractionConfiguration not use the cic query _configuration.CohortRefreshPipeline_ID = null; _configuration.CohortIdentificationConfiguration_ID = null; _configuration.SaveToDatabase(); //delete the cic query cic.QueryCachingServer_ID = null; cic.SaveToDatabase(); cic.DeleteInDatabase(); //delete the caching database queryCacheServer.DeleteInDatabase(); cachedb.Drop(); } }
public AggregateConfigurationCommand GenerateAggregateConfigurationFor(CohortIdentificationConfiguration cic, bool importMandatoryFilters = true, [CallerMemberName] string caller = null) { var newAggregate = cic.CreateNewEmptyConfigurationForCatalogue(Catalogue, ResolveMultipleExtractionIdentifiers ?? CohortCommandHelper.PickOneExtractionIdentifier, importMandatoryFilters); return(new AggregateConfigurationCommand(newAggregate)); }