public void StartAll() { //only allow starting all if we are not mid execution already if (IsExecutingGlobalOperations()) { return; } _cancelGlobalOperations = new CancellationTokenSource(); _runner = new CohortCompilerRunner(Compiler, _timeoutControls.Timeout); _runner.PhaseChanged += RunnerOnPhaseChanged; new Task(() => { try { _runner.Run(_cancelGlobalOperations.Token); } catch (Exception e) { ExceptionViewer.Show(e); } }).Start(); }
public void Join_PatientIndexTable_OptionalCacheOnSameServer(DatabaseType dbType, bool createQueryCache) { /* * Server1 * _____________ * |Biochemistry| * ↓ * ___________________ * | Cache (optional) | * ↓ join ↓ * _____________________ * | Hospital Admissions| * */ var db = GetCleanedServer(dbType); ExternalDatabaseServer cache = null; if (createQueryCache) { cache = CreateCache(db); } var r = new Random(500); var people = new PersonCollection(); people.GeneratePeople(5000, r); var cic = new CohortIdentificationConfiguration(CatalogueRepository, "cic"); var joinable = SetupPatientIndexTable(db, people, r, cic); cic.CreateRootContainerIfNotExists(); cic.QueryCachingServer_ID = cache?.ID; cic.SaveToDatabase(); var hospitalAdmissions = SetupPatientIndexTableUser(db, people, r, cic, joinable); cic.RootCohortAggregateContainer.AddChild(hospitalAdmissions, 0); var compiler = new CohortCompiler(cic); var runner = new CohortCompilerRunner(compiler, 50000); runner.Run(new CancellationToken()); AssertNoErrors(compiler); if (createQueryCache) { Assert.IsTrue(compiler.Tasks.Any(t => t.Key.GetCachedQueryUseCount().Equals("1/1")), "Expected cache to be used for the joinable"); } else { Assert.IsTrue(compiler.Tasks.Any(t => t.Key.GetCachedQueryUseCount().Equals("0/1")), "Did not create cache so expected cache usage to be 0"); } }
public void Join_PatientIndexTable_DoNotUseCacheOnDifferentServer(DatabaseType dbType) { /* * Server1 Server 2 * _____________ _________ * |Biochemistry| → | Cache | (cache is still populated but not used in the resulting join). * * ↓ join ↓ (do not use cache) * _____________________ * | Hospital Admissions| * */ //get the data database var db = GetCleanedServer(dbType); //create the cache on the other server type (doesn't matter what type just as long as it's different). var dbCache = GetCleanedServer(Enum.GetValues(typeof(DatabaseType)).Cast <DatabaseType>().First(t => t != dbType)); ExternalDatabaseServer cache = CreateCache(dbCache); var r = new Random(500); var people = new PersonCollection(); people.GeneratePeople(5000, r); var cic = new CohortIdentificationConfiguration(CatalogueRepository, "cic"); var joinable = SetupPatientIndexTable(db, people, r, cic); cic.CreateRootContainerIfNotExists(); cic.QueryCachingServer_ID = cache?.ID; cic.SaveToDatabase(); var hospitalAdmissions = SetupPatientIndexTableUser(db, people, r, cic, joinable); cic.RootCohortAggregateContainer.AddChild(hospitalAdmissions, 0); var compiler = new CohortCompiler(cic); var runner = new CohortCompilerRunner(compiler, 50000); runner.Run(new CancellationToken()); AssertNoErrors(compiler); Assert.IsTrue(compiler.Tasks.Any(t => t.Key.GetCachedQueryUseCount().Equals("1/1")), "Expected cache to be used only for the final UNION"); }
public void Join_PatientIndexTable_ThreeServers() { /* * Server1 Server 2 Server 3 * _____________ _________ * |Biochemistry| → (successfully caches joinable bit) | Cache | * * ↘ join ↘ (should crash) * _____________________ * | Hospital Admissions| * */ var server1 = GetCleanedServer(DatabaseType.MySql); var server2 = GetCleanedServer(DatabaseType.MicrosoftSQLServer); var server3 = GetCleanedServer(DatabaseType.Oracle); ExternalDatabaseServer cache = CreateCache(server3); var r = new Random(500); var people = new PersonCollection(); people.GeneratePeople(5000, r); var cic = new CohortIdentificationConfiguration(CatalogueRepository, "cic"); var joinable = SetupPatientIndexTable(server1, people, r, cic); cic.CreateRootContainerIfNotExists(); cic.QueryCachingServer_ID = cache?.ID; cic.SaveToDatabase(); var hospitalAdmissions = SetupPatientIndexTableUser(server2, people, r, cic, joinable); cic.RootCohortAggregateContainer.AddChild(hospitalAdmissions, 0); var compiler = new CohortCompiler(cic); var runner = new CohortCompilerRunner(compiler, 50000); runner.Run(new CancellationToken()); var hospitalAdmissionsTask = compiler.Tasks.Keys.OfType <AggregationTask>().Single(t => t.Aggregate.Equals(hospitalAdmissions)); Assert.AreEqual(CompilationState.Crashed, hospitalAdmissionsTask.State); StringAssert.Contains("is not fully cached and CacheUsageDecision is MustUse", hospitalAdmissionsTask.CrashMessage.ToString()); }
public void Join_PatientIndexTable_NotOnCacheServer() { /* * Server1 Server 2 * _____________ _________ * |Biochemistry| → | Cache | (cache must first be populated) * * ↘ join ↘ (must use cache) * _____________________ * | Hospital Admissions| * */ var server1 = GetCleanedServer(DatabaseType.MySql); var server2 = GetCleanedServer(DatabaseType.MicrosoftSQLServer); ExternalDatabaseServer cache = CreateCache(server2); var r = new Random(500); var people = new PersonCollection(); people.GeneratePeople(5000, r); var cic = new CohortIdentificationConfiguration(CatalogueRepository, "cic"); var joinable = SetupPatientIndexTable(server1, people, r, cic); cic.CreateRootContainerIfNotExists(); cic.QueryCachingServer_ID = cache?.ID; cic.SaveToDatabase(); var hospitalAdmissions = SetupPatientIndexTableUser(server2, people, r, cic, joinable); cic.RootCohortAggregateContainer.AddChild(hospitalAdmissions, 0); var compiler = new CohortCompiler(cic); var runner = new CohortCompilerRunner(compiler, 50000); runner.Run(new CancellationToken()); AssertNoErrors(compiler); Assert.IsTrue(compiler.Tasks.Any(t => t.Key.GetCachedQueryUseCount().Equals("1/1")), "Expected cache to be used only for the final UNION"); }
private ICompileable RunAllTasksWithRunner(CohortCompiler cohortCompiler, IDataLoadEventListener listener) { if (ClearCohortIdentificationConfigurationCacheBeforeRunning) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Clearing Cohort Identifier Cache")); var cacheManager = new CachedAggregateConfigurationResultsManager(_cohortIdentificationConfiguration.QueryCachingServer); cohortCompiler.AddAllTasks(false); foreach (var cacheable in cohortCompiler.Tasks.Keys.OfType <ICacheableTask>()) { cacheable.ClearYourselfFromCache(cacheManager); } } var runner = new CohortCompilerRunner(cohortCompiler, Timeout); runner.RunSubcontainers = false; runner.PhaseChanged += (s, e) => listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "CohortCompilerRunner entered Phase '" + runner.ExecutionPhase + "'")); return(runner.Run(_cancelGlobalOperations.Token)); }
//[TestCase(DatabaseType.Oracle,true)] //Oracle FAnsi doesn't currently support parameters //[TestCase(DatabaseType.Oracle,false)] public void Test_SingleServer_WithOneParameter(DatabaseType dbType, bool useParameter) { /* * Server1 Server2 * _____________________ _____________________ * |HospitalAdmissions | → | Cache | * @date_of_max * */ var server1 = GetCleanedServer(dbType); var server2 = GetCleanedServer(DatabaseType.MicrosoftSQLServer); var cache = CreateCache(server2); var r = new Random(500); var people = new PersonCollection(); people.GeneratePeople(5000, r); var cic = new CohortIdentificationConfiguration(CatalogueRepository, "cic"); var ac1 = SetupAggregateConfigurationWithFilter(server1, people, r, cic, useParameter, "@date_of_max", "'2001-01-01'"); cic.CreateRootContainerIfNotExists(); cic.QueryCachingServer_ID = cache.ID; cic.SaveToDatabase(); var root = cic.RootCohortAggregateContainer; root.AddChild(ac1, 0); var compiler = new CohortCompiler(cic); var runner = new CohortCompilerRunner(compiler, 50000); runner.Run(new CancellationToken()); AssertNoErrors(compiler); Assert.IsTrue(compiler.Tasks.Where(t => t.Key is AggregationContainerTask).Any(t => t.Key.GetCachedQueryUseCount().Equals("1/1")), "Expected UNION container to use the cache"); }
public void CacheIdentifierListWithRunner_WithCaching() { DiscoveredDatabase db; CohortIdentificationConfiguration cic; DataTable dt; SetupCohort(out db, out cic, out dt); MasterDatabaseScriptExecutor e = new MasterDatabaseScriptExecutor(db); var p = new QueryCachingPatcher(); e.CreateAndPatchDatabase(p, new AcceptAllCheckNotifier()); var serverReference = new ExternalDatabaseServer(CatalogueRepository, "Cache", p); serverReference.SetProperties(db); cic.QueryCachingServer_ID = serverReference.ID; cic.SaveToDatabase(); var compiler = new CohortCompiler(cic); var runner = new CohortCompilerRunner(compiler, 5000); runner.Run(new CancellationToken()); Assert.AreEqual(CohortCompilerRunner.Phase.Finished, runner.ExecutionPhase); var rootTask = runner.Compiler.Tasks.Single(t => t.Key is AggregationContainerTask); Assert.IsTrue(rootTask.Value.IsResultsForRootContainer); Assert.IsNull(rootTask.Key.CrashMessage); Assert.AreEqual(CompilationState.Finished, rootTask.Key.State); Assert.IsTrue(runner.Compiler.AreaAllQueriesCached(rootTask.Key)); Assert.AreEqual(dt.Rows.Count, rootTask.Value.Identifiers.Rows.Count); }
public void CacheIdentifierListWithRunner_SimpleCase() { DiscoveredDatabase db; CohortIdentificationConfiguration cic; DataTable dt; SetupCohort(out db, out cic, out dt); var compiler = new CohortCompiler(cic); var runner = new CohortCompilerRunner(compiler, 5000); runner.Run(new CancellationToken()); Assert.AreEqual(CohortCompilerRunner.Phase.Finished, runner.ExecutionPhase); var rootTask = runner.Compiler.Tasks.Single(t => t.Key is AggregationContainerTask); Assert.IsTrue(rootTask.Value.IsResultsForRootContainer); Assert.IsNull(rootTask.Key.CrashMessage); Assert.AreEqual(CompilationState.Finished, rootTask.Key.State); Assert.AreEqual(dt.Rows.Count, rootTask.Value.Identifiers.Rows.Count); }
public void Join_PatientIndexTable_ThenShipToCacheForSets(DatabaseType dbType) { /* * Server1 Server 2 * _____________ _________ * |Biochemistry| → | Cache | (cache is still populated but not used in the resulting join). * * ↓ join ↓ (do not use cache) * _______________________ * | Hospital Admissions 1| → results1 * * EXCEPT * _______________________ * | Hospital Admissions 2| → results 2 * ↓ result = 0 records */ //get the data database var db = GetCleanedServer(dbType); //create the cache on the other server type (either Sql Server or Oracle) since MySql can't do EXCEPT or UNION etc) var dbCache = GetCleanedServer(Enum.GetValues(typeof(DatabaseType)).Cast <DatabaseType>().First(t => t != dbType && t != DatabaseType.MySql)); ExternalDatabaseServer cache = CreateCache(dbCache); var r = new Random(500); var people = new PersonCollection(); people.GeneratePeople(5000, r); var cic = new CohortIdentificationConfiguration(CatalogueRepository, "cic"); var joinable = SetupPatientIndexTable(db, people, r, cic); cic.CreateRootContainerIfNotExists(); cic.QueryCachingServer_ID = cache?.ID; cic.SaveToDatabase(); var hospitalAdmissions = SetupPatientIndexTableUser(db, people, r, cic, joinable); var hospitalAdmissions2 = SetupAggregateConfiguration(db, people, r, cic); var root = cic.RootCohortAggregateContainer; root.AddChild(hospitalAdmissions, 0); root.AddChild(hospitalAdmissions2, 1); root.Name = "EXCEPT"; root.Operation = SetOperation.EXCEPT; root.SaveToDatabase(); var compiler = new CohortCompiler(cic); var runner = new CohortCompilerRunner(compiler, 50000); runner.Run(new CancellationToken()); AssertNoErrors(compiler); Assert.IsTrue(compiler.Tasks.Any(t => t.Key.GetCachedQueryUseCount().Equals("2/2")), "Expected cache to be used for both top level operations in the EXCEPT"); }
public void Test_EXCEPT_TwoAggregates(DatabaseType dbType) { /* * Server1 * _____________________ * |HospitalAdmissions x2| * ↓ both run into ↓ * ___________________ * | Cache | * * * HospitalAdmissions * EXCEPT * HospitalAdmissions (copy) * = 0 */ var db = GetCleanedServer(dbType); var cache = CreateCache(db); var r = new Random(500); var people = new PersonCollection(); people.GeneratePeople(5000, r); var cic = new CohortIdentificationConfiguration(CatalogueRepository, "cic"); var ac1 = SetupAggregateConfiguration(db, people, r, cic); var ac2 = SetupAggregateConfiguration(db, people, r, cic); cic.CreateRootContainerIfNotExists(); cic.QueryCachingServer_ID = cache.ID; cic.SaveToDatabase(); var root = cic.RootCohortAggregateContainer; root.Operation = SetOperation.EXCEPT; root.Name = "EXCEPT"; root.SaveToDatabase(); root.AddChild(ac1, 0); root.AddChild(ac2, 1); var compiler = new CohortCompiler(cic); var runner = new CohortCompilerRunner(compiler, 50000); runner.Run(new CancellationToken()); if (dbType == DatabaseType.MySql) { var crashed = compiler.Tasks.Single(t => t.Key.State == CompilationState.Crashed); StringAssert.Contains("INTERSECT / UNION / EXCEPT are not supported by MySql", crashed.Key.CrashMessage.Message); return; } AssertNoErrors(compiler); Assert.AreEqual(compiler.Tasks.Single(t => t.Value != null && t.Value.IsResultsForRootContainer).Key.FinalRowCount, 0); Assert.Greater(compiler.Tasks.Single(t => t.Key is AggregationTask at && at.Aggregate.Equals(ac1)).Key.FinalRowCount, 0); //both ac should have the same total Assert.Greater(compiler.Tasks.Single(t => t.Key is AggregationTask at && at.Aggregate.Equals(ac2)).Key.FinalRowCount, 0); // that is not 0 Assert.IsTrue(compiler.Tasks.Any(t => t.Key.GetCachedQueryUseCount().Equals("2/2")), "Expected EXCEPT container to use the cache"); }
public void Test_SingleServerPatientIndexTable_WithTwoParameters(DatabaseType dbType, bool useSameName, bool useCache) { /* * Server1 (Also Server1 - if useCache is true) * ______________________________ __________________________ * | Patient Index Table | → | Cache | * | (NA by date ) | ↗ * @date_of_max '2001-01-01' * ↗ * JOIN (should use cache) * ___________________________ ↗ * | Hospitalised after NA (ac) | * @date_of_max (or @maximum) '2005-01-01' * * (has different value so rename operations come into effect) */ var server1 = GetCleanedServer(dbType); var cache = useCache? CreateCache(server1): null; var r = new Random(500); var people = new PersonCollection(); people.GeneratePeople(5000, r); var cic = new CohortIdentificationConfiguration(CatalogueRepository, "cic"); var patientIndexTable = SetupPatientIndexTableWithFilter(server1, people, r, cic, true, "@date_of_max", "'2001-01-01'"); var ac = SetupPatientIndexTableUserWithFilter(server1, people, r, cic, patientIndexTable, true, useSameName ? "@date_of_max": "@maximum", "'2005-01-01'"); cic.CreateRootContainerIfNotExists(); cic.QueryCachingServer_ID = cache?.ID; cic.SaveToDatabase(); var root = cic.RootCohortAggregateContainer; root.AddChild(ac, 0); var compiler = new CohortCompiler(cic); var runner = new CohortCompilerRunner(compiler, 50000); runner.Run(new CancellationToken()); if (useSameName) { AssertCrashed(compiler, ac, "PatientIndexTables cannot have parameters with the same name as their users. Offending parameter(s) were @date_of_max"); } else { if (useCache) { //we should hit up the cache for the interior of the query and therefore not need the parameter AssertNoErrors(compiler, ac, "@maximum='2005-01-01'", "JoinableInceptionQuery_AggregateConfiguration", "AdmissionDate. < @maximum"); AssertCacheUsed(compiler, root, "1/1"); } else { AssertNoErrors(compiler, ac, "@date_of_max='2001-01-01'", "@maximum='2005-01-01'", "SampleDate. < @date_of_max", "AdmissionDate. < @maximum"); } AssertNoErrors(compiler); } }
//[TestCase(DatabaseType.Oracle)] //Oracle FAnsi doesn't currently support parameters public void Test_SingleServer_WithTwoParameters(DatabaseType dbType, bool useCache) { /* * Server1 Server2 * ____________________ _____________________ * |HospitalAdmissions | → | Cache | * @date_of_max * ____________________ ↗ * |HospitalAdmissions | * @date_of_max * * (has different value so rename operations come into effect) */ var server1 = GetCleanedServer(dbType); var server2 = GetCleanedServer(DatabaseType.MicrosoftSQLServer); var cache = useCache ? CreateCache(server2): null; var r = new Random(500); var people = new PersonCollection(); people.GeneratePeople(5000, r); var cic = new CohortIdentificationConfiguration(CatalogueRepository, "cic"); var ac1 = SetupAggregateConfigurationWithFilter(server1, people, r, cic, true, "@date_of_max", "'2001-01-01'"); var ac2 = SetupAggregateConfigurationWithFilter(server1, people, r, cic, true, "@date_of_max", "'2005-01-01'"); cic.CreateRootContainerIfNotExists(); cic.QueryCachingServer_ID = cache?.ID; cic.SaveToDatabase(); var root = cic.RootCohortAggregateContainer; root.AddChild(ac1, 0); root.AddChild(ac2, 1); var compiler = new CohortCompiler(cic); var runner = new CohortCompilerRunner(compiler, 50000); runner.Run(new CancellationToken()); AssertNoErrors(compiler); //each container on it's own ran with the normal SQL AssertNoErrors(compiler, ac1, "@date_of_max='2001-01-01'"); AssertNoErrors(compiler, ac2, "@date_of_max='2005-01-01'"); if (useCache) { //the root container run with dual cache fetch and no parameters AssertNoErrors(compiler, root, "IndexedExtractionIdentifierList"); AssertCacheUsed(compiler, root, "2/2"); } else { //the root container ran with a rename operations (no cache available) AssertNoErrors(compiler, root, "@date_of_max='2001-01-01'", "@date_of_max_2='2005-01-01'"); AssertCacheUsed(compiler, root, "0/2"); } }
public void CohortIdentificationConfiguration_Join_PatientIndexTable() { DataTable header = new DataTable(); header.Columns.Add("ID"); header.Columns.Add("Chi"); header.Columns.Add("Age"); header.Columns.Add("Date"); header.Columns.Add("Healthboard"); header.PrimaryKey = new [] { header.Columns["ID"] }; header.Rows.Add("1", "0101010101", 50, new DateTime(2001, 1, 1), "T"); header.Rows.Add("2", "0202020202", 50, new DateTime(2002, 2, 2), "T"); var hTbl = From.CreateTable("header", header); var cata = Import(hTbl, out TableInfo hTi, out _); cata.Name = "My Combo Join Catalogue"; cata.SaveToDatabase(); var scripter = new MasterDatabaseScriptExecutor(To); var patcher = new QueryCachingPatcher(); scripter.CreateAndPatchDatabase(patcher, new AcceptAllCheckNotifier()); var edsCache = new ExternalDatabaseServer(CatalogueRepository, "Cache", new QueryCachingPatcher()); edsCache.SetProperties(To); DataTable results = new DataTable(); results.Columns.Add("Header_ID"); results.Columns.Add("TestCode"); results.Columns.Add("Result"); results.Rows.Add("1", "HBA1C", 50); results.Rows.Add("1", "ECOM", "Hi fellas"); results.Rows.Add("1", "ALB", 100); results.Rows.Add("2", "ALB", 50); var rTbl = From.CreateTable("results", results); var importer = new TableInfoImporter(CatalogueRepository, rTbl); importer.DoImport(out TableInfo rTi, out ColumnInfo[] rColInfos); var fe = new ForwardEngineerCatalogue(rTi, rColInfos, true); fe.ExecuteForwardEngineering(cata); //Should now be 1 Catalogue with all the columns (tables will have to be joined to build the query though) Assert.AreEqual(8, cata.GetAllExtractionInformation(ExtractionCategory.Core).Length); var ji = new JoinInfo(CatalogueRepository, rTi.ColumnInfos.Single(ci => ci.GetRuntimeName().Equals("Header_ID", StringComparison.CurrentCultureIgnoreCase)), hTi.ColumnInfos.Single(ci => ci.GetRuntimeName().Equals("ID", StringComparison.CurrentCultureIgnoreCase)), ExtractionJoinType.Right, null ); //setup a cic that uses the cache var cic = new CohortIdentificationConfiguration(CatalogueRepository, "MyCic"); cic.CreateRootContainerIfNotExists(); cic.QueryCachingServer_ID = edsCache.ID; cic.SaveToDatabase(); //create a patient index table that shows all the times that they had a test in any HB (with the HB being part of the result set) var acPatIndex = new AggregateConfiguration(CatalogueRepository, cata, "My PatIndes"); var eiChi = cata.GetAllExtractionInformation(ExtractionCategory.Core).Single(ei => ei.GetRuntimeName().Equals("Chi")); eiChi.IsExtractionIdentifier = true; acPatIndex.CountSQL = null; eiChi.SaveToDatabase(); acPatIndex.AddDimension(eiChi); acPatIndex.AddDimension(cata.GetAllExtractionInformation(ExtractionCategory.Core).Single(ei => ei.GetRuntimeName().Equals("Date"))); acPatIndex.AddDimension(cata.GetAllExtractionInformation(ExtractionCategory.Core).Single(ei => ei.GetRuntimeName().Equals("Healthboard"))); cic.EnsureNamingConvention(acPatIndex); var joinable = new JoinableCohortAggregateConfiguration(CatalogueRepository, cic, acPatIndex); Assert.IsTrue(acPatIndex.IsCohortIdentificationAggregate); Assert.IsTrue(acPatIndex.IsJoinablePatientIndexTable()); var compiler = new CohortCompiler(cic); var runner = new CohortCompilerRunner(compiler, 50); var cancellation = new System.Threading.CancellationToken(); runner.Run(cancellation); //they should not be executing and should be completed Assert.IsFalse(compiler.Tasks.Any(t => t.Value.IsExecuting)); Assert.AreEqual(Phase.Finished, runner.ExecutionPhase); var manager = new CachedAggregateConfigurationResultsManager(edsCache); var cacheTableName = manager.GetLatestResultsTableUnsafe(acPatIndex, AggregateOperation.JoinableInceptionQuery); Assert.IsNotNull(cacheTableName, "No results were cached!"); var cacheTable = To.ExpectTable(cacheTableName.GetRuntimeName()); //chi, Date and TestCode Assert.AreEqual(3, cacheTable.DiscoverColumns().Length); //healthboard should be a string Assert.AreEqual(typeof(string), cacheTable.DiscoverColumn("Healthboard").DataType.GetCSharpDataType()); /* Query Cache contains this: * * Chi Date Healthboard * 0101010101 2001-01-01 00:00:00.0000000 T * 0202020202 2002-02-02 00:00:00.0000000 T */ Assert.AreEqual(2, cacheTable.GetRowCount()); //Now we could add a new AggregateConfiguration that uses the joinable! }