/// <summary> /// Checks that all globals pass thier respective checkers (<see cref="SupportingSQLTableChecker"/> and <see cref="SupportingDocumentsFetcher"/>) and that /// the <see cref="Pipeline"/> (if any) is capable of extracting the globals. /// </summary> /// <param name="notifier"></param> public void Check(ICheckNotifier notifier) { foreach (SupportingSQLTable table in _configuration.GetGlobals().OfType <SupportingSQLTable>()) { new SupportingSQLTableChecker(table).Check(notifier); } foreach (SupportingDocument document in _configuration.GetGlobals().OfType <SupportingDocument>()) { new SupportingDocumentsFetcher(document).Check(notifier); } if (_alsoCheckPipeline != null && _command != null) { var engine = new ExtractionPipelineUseCase(_configuration.Project, _command, _alsoCheckPipeline, DataLoadInfo.Empty) .GetEngine(_alsoCheckPipeline, new FromCheckNotifierToDataLoadEventListener(notifier)); engine.Check(notifier); } }
/// <summary> /// Checks the <see cref="SelectedDataSet"/> and reports success/failures to the <paramref name="notifier"/> /// </summary> /// <param name="notifier"></param> public void Check(ICheckNotifier notifier) { var ds = SelectedDataSet.ExtractableDataSet; var config = SelectedDataSet.ExtractionConfiguration; var cohort = config.Cohort; var project = config.Project; const int timeout = 5; notifier.OnCheckPerformed(new CheckEventArgs("Inspecting dataset " + ds, CheckResult.Success)); var selectedcols = new List <IColumn>(config.GetAllExtractableColumnsFor(ds)); if (!selectedcols.Any()) { notifier.OnCheckPerformed( new CheckEventArgs( "Dataset " + ds + " in configuration '" + config + "' has no selected columns", CheckResult.Fail)); return; } ICatalogue cata; try { cata = ds.Catalogue; } catch (Exception e) { notifier.OnCheckPerformed(new CheckEventArgs("Unable to find Catalogue for ExtractableDataSet", CheckResult.Fail, e)); return; } if (cata.IsInternalDataset) { notifier.OnCheckPerformed(new CheckEventArgs($"Dataset '{ds}' is marked {nameof(ICatalogue.IsInternalDataset)} so should not be extracted", CheckResult.Fail)); } var request = new ExtractDatasetCommand(config, cohort, new ExtractableDatasetBundle(ds), selectedcols, new HICProjectSalt(project), new ExtractionDirectory(project.ExtractionDirectory, config)) { TopX = 1 }; try { request.GenerateQueryBuilder(); } catch (Exception e) { notifier.OnCheckPerformed( new CheckEventArgs( "Could not generate valid extraction SQL for dataset " + ds + " in configuration " + config, CheckResult.Fail, e)); return; } var server = request.GetDistinctLiveDatabaseServer(); bool serverExists = server.Exists(); notifier.OnCheckPerformed(new CheckEventArgs("Server " + server + " Exists:" + serverExists, serverExists ? CheckResult.Success : CheckResult.Fail)); var cohortServer = request.ExtractableCohort.ExternalCohortTable.Discover(); if (cohortServer == null || !cohortServer.Exists()) { notifier.OnCheckPerformed(new CheckEventArgs("Cohort server did not exist or was unreachable", CheckResult.Fail)); return; } //when 2+ columns have the same Name it's a problem foreach (IGrouping <string, IColumn> grouping in request.ColumnsToExtract.GroupBy(c => c.GetRuntimeName()).Where(g => g.Count() > 1)) { notifier.OnCheckPerformed(new CheckEventArgs($"There are { grouping.Count() } columns in the extract ({request.DatasetBundle?.DataSet}) called '{ grouping.Key }'", CheckResult.Fail)); } //when 2+ columns have the same Order it's a problem because foreach (IGrouping <int, IColumn> grouping in request.ColumnsToExtract.GroupBy(c => c.Order).Where(g => g.Count() > 1)) { notifier.OnCheckPerformed(new CheckEventArgs($"There are { grouping.Count() } columns in the extract ({request.DatasetBundle?.DataSet}) that share the same Order '{ grouping.Key }'", CheckResult.Fail)); } // Warn user if stuff is out of sync with the Catalogue version (changes have happened to the master but not propgated to the copy in this extraction) var outOfSync = selectedcols.OfType <ExtractableColumn>().Where(c => c.IsOutOfSync()).ToArray(); if (outOfSync.Any()) { notifier.OnCheckPerformed(new CheckEventArgs($"'{ds}' columns out of sync with CatalogueItem version(s): { Environment.NewLine + string.Join(',', outOfSync.Select(o => o.ToString() + Environment.NewLine)) }" + $"{ Environment.NewLine } Extraction Configuration: '{config}' ", CheckResult.Warning)); } var nonSelectedCore = cata.GetAllExtractionInformation(ExtractionCategory.Core) .Union(cata.GetAllExtractionInformation(ExtractionCategory.ProjectSpecific)) .Where(ei => !ei.IsExtractionIdentifier && !selectedcols.OfType <ExtractableColumn>().Any(ec => ec.CatalogueExtractionInformation_ID == ei.ID)) .ToArray(); if (nonSelectedCore.Any()) { notifier.OnCheckPerformed(new CheckEventArgs($"'{ds}' Core columns not selected for extractions: { Environment.NewLine + string.Join(',', nonSelectedCore.Select(o => o.ToString() + Environment.NewLine)) }" + $"{ Environment.NewLine } Extraction Configuration: '{config}' ", CheckResult.Warning)); } //Make sure cohort and dataset are on same server before checking (can still get around this at runtime by using ExecuteCrossServerDatasetExtractionSource) if (!cohortServer.Server.Name.Equals(server.Name, StringComparison.CurrentCultureIgnoreCase) || !cohortServer.Server.DatabaseType.Equals(server.DatabaseType)) { notifier.OnCheckPerformed(new CheckEventArgs( $"Cohort is on server '{cohortServer.Server.Name}' ({cohortServer.Server.DatabaseType}) but dataset '{request.DatasetBundle?.DataSet}' is on '{server.Name}' ({server.DatabaseType})" , CheckResult.Warning)); } else { //Try to fetch TOP 1 data try { using (var con = server.BeginNewTransactedConnection()) { //incase user somehow manages to write a filter/transform that nukes data or something DbCommand cmd; try { cmd = server.GetCommand(request.QueryBuilder.SQL, con); cmd.CommandTimeout = timeout; notifier.OnCheckPerformed( new CheckEventArgs( "/*About to send Request SQL :*/" + Environment.NewLine + request.QueryBuilder.SQL, CheckResult.Success)); } catch (QueryBuildingException e) { notifier.OnCheckPerformed(new CheckEventArgs("Failed to assemble query for dataset " + ds, CheckResult.Fail, e)); return; } try { using (var r = cmd.ExecuteReader()) { if (r.Read()) { notifier.OnCheckPerformed(new CheckEventArgs("Read at least 1 row successfully from dataset " + ds, CheckResult.Success)); } else { notifier.OnCheckPerformed(new CheckEventArgs("Dataset " + ds + " is completely empty (when linked with the cohort). " + "Extraction may fail if the Source does not allow empty extractions", CheckResult.Warning)); } } } catch (Exception e) { if (server.GetQuerySyntaxHelper().IsTimeout(e)) { notifier.OnCheckPerformed(new CheckEventArgs(ErrorCodes.ExtractTimeoutChecking, e, timeout)); } else { notifier.OnCheckPerformed(new CheckEventArgs("Failed to execute the query (See below for query)", CheckResult.Fail, e)); } } con.ManagedTransaction.AbandonAndCloseConnection(); } } catch (Exception e) { notifier.OnCheckPerformed(new CheckEventArgs("Failed to execute Top 1 on dataset " + ds, CheckResult.Fail, e)); } } var fetchOptions = _checkGlobals ? FetchOptions.ExtractableGlobalsAndLocals : FetchOptions.ExtractableLocals; foreach (var supportingDocument in cata.GetAllSupportingDocuments(fetchOptions)) { new SupportingDocumentsFetcher(supportingDocument).Check(notifier); } //check catalogue locals foreach (SupportingSQLTable table in cata.GetAllSupportingSQLTablesForCatalogue(fetchOptions)) { new SupportingSQLTableChecker(table).Check(notifier); } if (_alsoCheckPipeline != null) { var engine = new ExtractionPipelineUseCase(_activator, request.Project, request, _alsoCheckPipeline, DataLoadInfo.Empty) .GetEngine(_alsoCheckPipeline, new FromCheckNotifierToDataLoadEventListener(notifier)); engine.Check(notifier); } }
/// <summary> /// Checks the <see cref="SelectedDataSet"/> and reports success/failures to the <paramref name="notifier"/> /// </summary> /// <param name="notifier"></param> public void Check(ICheckNotifier notifier) { var ds = SelectedDataSet.ExtractableDataSet; var config = SelectedDataSet.ExtractionConfiguration; var cohort = config.Cohort; var project = config.Project; const int timeout = 5; notifier.OnCheckPerformed(new CheckEventArgs("Inspecting dataset " + ds, CheckResult.Success)); var selectedcols = new List <IColumn>(config.GetAllExtractableColumnsFor(ds)); if (!selectedcols.Any()) { notifier.OnCheckPerformed( new CheckEventArgs( "Dataset " + ds + " in configuration '" + config + "' has no selected columns", CheckResult.Fail)); return; } var request = new ExtractDatasetCommand(config, cohort, new ExtractableDatasetBundle(ds), selectedcols, new HICProjectSalt(project), new ExtractionDirectory(project.ExtractionDirectory, config)) { TopX = 1 }; try { request.GenerateQueryBuilder(); } catch (Exception e) { notifier.OnCheckPerformed( new CheckEventArgs( "Could not generate valid extraction SQL for dataset " + ds + " in configuration " + config, CheckResult.Fail, e)); return; } var server = request.GetDistinctLiveDatabaseServer(); bool serverExists = server.Exists(); notifier.OnCheckPerformed(new CheckEventArgs("Server " + server + " Exists:" + serverExists, serverExists ? CheckResult.Success : CheckResult.Fail)); var cohortServer = request.ExtractableCohort.ExternalCohortTable.Discover(); if (cohortServer == null || !cohortServer.Exists()) { notifier.OnCheckPerformed(new CheckEventArgs("Cohort server did not exist or was unreachable", CheckResult.Fail)); return; } foreach (IGrouping <string, IColumn> grouping in request.ColumnsToExtract.GroupBy(c => c.GetRuntimeName()).Where(g => g.Count() > 1)) { notifier.OnCheckPerformed(new CheckEventArgs("There are " + grouping.Count() + " columns in the extract called '" + grouping.Key + "'", CheckResult.Fail)); } //Make sure cohort and dataset are on same server before checking (can still get around this at runtime by using ExecuteCrossServerDatasetExtractionSource) if (!cohortServer.Server.Name.Equals(server.Name, StringComparison.CurrentCultureIgnoreCase) || !cohortServer.Server.DatabaseType.Equals(server.DatabaseType)) { notifier.OnCheckPerformed(new CheckEventArgs( string.Format("Cohort is on server '{0}' ({1}) but dataset is on '{2}' ({3})", cohortServer.Server.Name, cohortServer.Server.DatabaseType, server.Name, server.DatabaseType), CheckResult.Warning)); } else { //Try to fetch TOP 1 data try { using (var con = server.BeginNewTransactedConnection()) { //incase user somehow manages to write a filter/transform that nukes data or something DbCommand cmd; try { cmd = server.GetCommand(request.QueryBuilder.SQL, con); cmd.CommandTimeout = timeout; notifier.OnCheckPerformed( new CheckEventArgs( "/*About to send Request SQL :*/" + Environment.NewLine + request.QueryBuilder.SQL, CheckResult.Success)); } catch (QueryBuildingException e) { notifier.OnCheckPerformed(new CheckEventArgs("Failed to assemble query for dataset " + ds, CheckResult.Fail, e)); return; } try { using (var r = cmd.ExecuteReader()) { if (r.Read()) { notifier.OnCheckPerformed(new CheckEventArgs("Read at least 1 row successfully from dataset " + ds, CheckResult.Success)); } else { notifier.OnCheckPerformed(new CheckEventArgs("Dataset " + ds + " is completely empty (when linked with the cohort). " + "Extraction may fail if the Source does not allow empty extractions", CheckResult.Warning)); } } } catch (Exception e) { if (server.GetQuerySyntaxHelper().IsTimeout(e)) { notifier.OnCheckPerformed(new CheckEventArgs("Failed to read rows after " + timeout + "s", CheckResult.Warning, e)); } else { notifier.OnCheckPerformed(new CheckEventArgs("Failed to execute the query (See below for query)", CheckResult.Fail, e)); } } con.ManagedTransaction.AbandonAndCloseConnection(); } } catch (Exception e) { notifier.OnCheckPerformed(new CheckEventArgs("Failed to execute Top 1 on dataset " + ds, CheckResult.Fail, e)); } } var cata = ds.Catalogue; var fetchOptions = _checkGlobals ? FetchOptions.ExtractableGlobalsAndLocals : FetchOptions.ExtractableLocals; foreach (var supportingDocument in cata.GetAllSupportingDocuments(fetchOptions)) { new SupportingDocumentsFetcher(supportingDocument).Check(notifier); } //check catalogue locals foreach (SupportingSQLTable table in cata.GetAllSupportingSQLTablesForCatalogue(fetchOptions)) { new SupportingSQLTableChecker(table).Check(notifier); } if (_alsoCheckPipeline != null) { var engine = new ExtractionPipelineUseCase(request.Project, request, _alsoCheckPipeline, DataLoadInfo.Empty) .GetEngine(_alsoCheckPipeline, new FromCheckNotifierToDataLoadEventListener(notifier)); engine.Check(notifier); } }