コード例 #1
0
        /// <summary>
        /// Checks that all globals pass thier respective checkers (<see cref="SupportingSQLTableChecker"/> and <see cref="SupportingDocumentsFetcher"/>) and that
        /// the <see cref="Pipeline"/> (if any) is capable of extracting the globals.
        /// </summary>
        /// <param name="notifier"></param>
        public void Check(ICheckNotifier notifier)
        {
            foreach (SupportingSQLTable table in _configuration.GetGlobals().OfType <SupportingSQLTable>())
            {
                new SupportingSQLTableChecker(table).Check(notifier);
            }

            foreach (SupportingDocument document in _configuration.GetGlobals().OfType <SupportingDocument>())
            {
                new SupportingDocumentsFetcher(document).Check(notifier);
            }

            if (_alsoCheckPipeline != null && _command != null)
            {
                var engine = new ExtractionPipelineUseCase(_configuration.Project, _command, _alsoCheckPipeline, DataLoadInfo.Empty)
                             .GetEngine(_alsoCheckPipeline, new FromCheckNotifierToDataLoadEventListener(notifier));
                engine.Check(notifier);
            }
        }
コード例 #2
0
        /// <summary>
        /// Checks the <see cref="SelectedDataSet"/> and reports success/failures to the <paramref name="notifier"/>
        /// </summary>
        /// <param name="notifier"></param>
        public void Check(ICheckNotifier notifier)
        {
            var       ds      = SelectedDataSet.ExtractableDataSet;
            var       config  = SelectedDataSet.ExtractionConfiguration;
            var       cohort  = config.Cohort;
            var       project = config.Project;
            const int timeout = 5;

            notifier.OnCheckPerformed(new CheckEventArgs("Inspecting dataset " + ds, CheckResult.Success));

            var selectedcols = new List <IColumn>(config.GetAllExtractableColumnsFor(ds));

            if (!selectedcols.Any())
            {
                notifier.OnCheckPerformed(
                    new CheckEventArgs(
                        "Dataset " + ds + " in configuration '" + config + "' has no selected columns",
                        CheckResult.Fail));

                return;
            }

            ICatalogue cata;

            try
            {
                cata = ds.Catalogue;
            }
            catch (Exception e)
            {
                notifier.OnCheckPerformed(new CheckEventArgs("Unable to find Catalogue for ExtractableDataSet", CheckResult.Fail, e));
                return;
            }

            if (cata.IsInternalDataset)
            {
                notifier.OnCheckPerformed(new CheckEventArgs($"Dataset '{ds}' is marked {nameof(ICatalogue.IsInternalDataset)} so should not be extracted", CheckResult.Fail));
            }

            var request = new ExtractDatasetCommand(config, cohort, new ExtractableDatasetBundle(ds),
                                                    selectedcols, new HICProjectSalt(project), new ExtractionDirectory(project.ExtractionDirectory, config))
            {
                TopX = 1
            };

            try
            {
                request.GenerateQueryBuilder();
            }
            catch (Exception e)
            {
                notifier.OnCheckPerformed(
                    new CheckEventArgs(
                        "Could not generate valid extraction SQL for dataset " + ds +
                        " in configuration " + config, CheckResult.Fail, e));
                return;
            }

            var  server       = request.GetDistinctLiveDatabaseServer();
            bool serverExists = server.Exists();

            notifier.OnCheckPerformed(new CheckEventArgs("Server " + server + " Exists:" + serverExists,
                                                         serverExists ? CheckResult.Success : CheckResult.Fail));

            var cohortServer = request.ExtractableCohort.ExternalCohortTable.Discover();

            if (cohortServer == null || !cohortServer.Exists())
            {
                notifier.OnCheckPerformed(new CheckEventArgs("Cohort server did not exist or was unreachable", CheckResult.Fail));
                return;
            }

            //when 2+ columns have the same Name it's a problem
            foreach (IGrouping <string, IColumn> grouping in request.ColumnsToExtract.GroupBy(c => c.GetRuntimeName()).Where(g => g.Count() > 1))
            {
                notifier.OnCheckPerformed(new CheckEventArgs($"There are { grouping.Count() } columns in the extract ({request.DatasetBundle?.DataSet}) called '{ grouping.Key }'", CheckResult.Fail));
            }

            //when 2+ columns have the same Order it's a problem because
            foreach (IGrouping <int, IColumn> grouping in request.ColumnsToExtract.GroupBy(c => c.Order).Where(g => g.Count() > 1))
            {
                notifier.OnCheckPerformed(new CheckEventArgs($"There are { grouping.Count() } columns in the extract ({request.DatasetBundle?.DataSet}) that share the same Order '{ grouping.Key }'", CheckResult.Fail));
            }

            // Warn user if stuff is out of sync with the Catalogue version (changes have happened to the master but not propgated to the copy in this extraction)
            var outOfSync = selectedcols.OfType <ExtractableColumn>().Where(c => c.IsOutOfSync()).ToArray();

            if (outOfSync.Any())
            {
                notifier.OnCheckPerformed(new CheckEventArgs($"'{ds}' columns out of sync with CatalogueItem version(s): { Environment.NewLine + string.Join(',', outOfSync.Select(o => o.ToString() + Environment.NewLine)) }" +
                                                             $"{ Environment.NewLine } Extraction Configuration: '{config}' ", CheckResult.Warning));
            }

            var nonSelectedCore = cata.GetAllExtractionInformation(ExtractionCategory.Core)
                                  .Union(cata.GetAllExtractionInformation(ExtractionCategory.ProjectSpecific))
                                  .Where(ei => !ei.IsExtractionIdentifier &&
                                         !selectedcols.OfType <ExtractableColumn>().Any(ec => ec.CatalogueExtractionInformation_ID == ei.ID))
                                  .ToArray();

            if (nonSelectedCore.Any())
            {
                notifier.OnCheckPerformed(new CheckEventArgs($"'{ds}' Core columns not selected for extractions: { Environment.NewLine + string.Join(',', nonSelectedCore.Select(o => o.ToString() + Environment.NewLine)) }" +
                                                             $"{ Environment.NewLine } Extraction Configuration: '{config}' ", CheckResult.Warning));
            }

            //Make sure cohort and dataset are on same server before checking (can still get around this at runtime by using ExecuteCrossServerDatasetExtractionSource)
            if (!cohortServer.Server.Name.Equals(server.Name, StringComparison.CurrentCultureIgnoreCase) || !cohortServer.Server.DatabaseType.Equals(server.DatabaseType))
            {
                notifier.OnCheckPerformed(new CheckEventArgs(
                                              $"Cohort is on server '{cohortServer.Server.Name}' ({cohortServer.Server.DatabaseType}) but dataset '{request.DatasetBundle?.DataSet}' is on '{server.Name}' ({server.DatabaseType})"
                                              , CheckResult.Warning));
            }
            else
            {
                //Try to fetch TOP 1 data
                try
                {
                    using (var con = server.BeginNewTransactedConnection())
                    {
                        //incase user somehow manages to write a filter/transform that nukes data or something

                        DbCommand cmd;

                        try
                        {
                            cmd = server.GetCommand(request.QueryBuilder.SQL, con);
                            cmd.CommandTimeout = timeout;
                            notifier.OnCheckPerformed(
                                new CheckEventArgs(
                                    "/*About to send Request SQL :*/" + Environment.NewLine + request.QueryBuilder.SQL,
                                    CheckResult.Success));
                        }
                        catch (QueryBuildingException e)
                        {
                            notifier.OnCheckPerformed(new CheckEventArgs("Failed to assemble query for dataset " + ds,
                                                                         CheckResult.Fail, e));
                            return;
                        }

                        try
                        {
                            using (var r = cmd.ExecuteReader())
                            {
                                if (r.Read())
                                {
                                    notifier.OnCheckPerformed(new CheckEventArgs("Read at least 1 row successfully from dataset " + ds,
                                                                                 CheckResult.Success));
                                }
                                else
                                {
                                    notifier.OnCheckPerformed(new CheckEventArgs("Dataset " + ds + " is completely empty (when linked with the cohort). " +
                                                                                 "Extraction may fail if the Source does not allow empty extractions",
                                                                                 CheckResult.Warning));
                                }
                            }
                        }
                        catch (Exception e)
                        {
                            if (server.GetQuerySyntaxHelper().IsTimeout(e))
                            {
                                notifier.OnCheckPerformed(new CheckEventArgs(ErrorCodes.ExtractTimeoutChecking, e, timeout));
                            }
                            else
                            {
                                notifier.OnCheckPerformed(new CheckEventArgs("Failed to execute the query (See below for query)", CheckResult.Fail, e));
                            }
                        }

                        con.ManagedTransaction.AbandonAndCloseConnection();
                    }
                }
                catch (Exception e)
                {
                    notifier.OnCheckPerformed(new CheckEventArgs("Failed to execute Top 1 on dataset " + ds, CheckResult.Fail, e));
                }
            }

            var fetchOptions = _checkGlobals ? FetchOptions.ExtractableGlobalsAndLocals : FetchOptions.ExtractableLocals;

            foreach (var supportingDocument in cata.GetAllSupportingDocuments(fetchOptions))
            {
                new SupportingDocumentsFetcher(supportingDocument).Check(notifier);
            }

            //check catalogue locals
            foreach (SupportingSQLTable table in cata.GetAllSupportingSQLTablesForCatalogue(fetchOptions))
            {
                new SupportingSQLTableChecker(table).Check(notifier);
            }

            if (_alsoCheckPipeline != null)
            {
                var engine = new ExtractionPipelineUseCase(_activator, request.Project, request, _alsoCheckPipeline, DataLoadInfo.Empty)
                             .GetEngine(_alsoCheckPipeline, new FromCheckNotifierToDataLoadEventListener(notifier));
                engine.Check(notifier);
            }
        }
コード例 #3
0
ファイル: SelectedDataSetsChecker.cs プロジェクト: rkm/RDMP
        /// <summary>
        /// Checks the <see cref="SelectedDataSet"/> and reports success/failures to the <paramref name="notifier"/>
        /// </summary>
        /// <param name="notifier"></param>
        public void Check(ICheckNotifier notifier)
        {
            var       ds      = SelectedDataSet.ExtractableDataSet;
            var       config  = SelectedDataSet.ExtractionConfiguration;
            var       cohort  = config.Cohort;
            var       project = config.Project;
            const int timeout = 5;

            notifier.OnCheckPerformed(new CheckEventArgs("Inspecting dataset " + ds, CheckResult.Success));

            var selectedcols = new List <IColumn>(config.GetAllExtractableColumnsFor(ds));

            if (!selectedcols.Any())
            {
                notifier.OnCheckPerformed(
                    new CheckEventArgs(
                        "Dataset " + ds + " in configuration '" + config + "' has no selected columns",
                        CheckResult.Fail));

                return;
            }

            var request = new ExtractDatasetCommand(config, cohort, new ExtractableDatasetBundle(ds),
                                                    selectedcols, new HICProjectSalt(project), new ExtractionDirectory(project.ExtractionDirectory, config))
            {
                TopX = 1
            };

            try
            {
                request.GenerateQueryBuilder();
            }
            catch (Exception e)
            {
                notifier.OnCheckPerformed(
                    new CheckEventArgs(
                        "Could not generate valid extraction SQL for dataset " + ds +
                        " in configuration " + config, CheckResult.Fail, e));
                return;
            }

            var  server       = request.GetDistinctLiveDatabaseServer();
            bool serverExists = server.Exists();

            notifier.OnCheckPerformed(new CheckEventArgs("Server " + server + " Exists:" + serverExists,
                                                         serverExists ? CheckResult.Success : CheckResult.Fail));

            var cohortServer = request.ExtractableCohort.ExternalCohortTable.Discover();

            if (cohortServer == null || !cohortServer.Exists())
            {
                notifier.OnCheckPerformed(new CheckEventArgs("Cohort server did not exist or was unreachable", CheckResult.Fail));
                return;
            }

            foreach (IGrouping <string, IColumn> grouping in request.ColumnsToExtract.GroupBy(c => c.GetRuntimeName()).Where(g => g.Count() > 1))
            {
                notifier.OnCheckPerformed(new CheckEventArgs("There are " + grouping.Count() + " columns in the extract called '" + grouping.Key + "'", CheckResult.Fail));
            }

            //Make sure cohort and dataset are on same server before checking (can still get around this at runtime by using ExecuteCrossServerDatasetExtractionSource)
            if (!cohortServer.Server.Name.Equals(server.Name, StringComparison.CurrentCultureIgnoreCase) || !cohortServer.Server.DatabaseType.Equals(server.DatabaseType))
            {
                notifier.OnCheckPerformed(new CheckEventArgs(
                                              string.Format("Cohort is on server '{0}' ({1}) but dataset is on '{2}' ({3})",
                                                            cohortServer.Server.Name,
                                                            cohortServer.Server.DatabaseType,
                                                            server.Name, server.DatabaseType), CheckResult.Warning));
            }
            else
            {
                //Try to fetch TOP 1 data
                try
                {
                    using (var con = server.BeginNewTransactedConnection())
                    {
                        //incase user somehow manages to write a filter/transform that nukes data or something

                        DbCommand cmd;

                        try
                        {
                            cmd = server.GetCommand(request.QueryBuilder.SQL, con);
                            cmd.CommandTimeout = timeout;
                            notifier.OnCheckPerformed(
                                new CheckEventArgs(
                                    "/*About to send Request SQL :*/" + Environment.NewLine + request.QueryBuilder.SQL,
                                    CheckResult.Success));
                        }
                        catch (QueryBuildingException e)
                        {
                            notifier.OnCheckPerformed(new CheckEventArgs("Failed to assemble query for dataset " + ds,
                                                                         CheckResult.Fail, e));
                            return;
                        }

                        try
                        {
                            using (var r = cmd.ExecuteReader())
                            {
                                if (r.Read())
                                {
                                    notifier.OnCheckPerformed(new CheckEventArgs("Read at least 1 row successfully from dataset " + ds,
                                                                                 CheckResult.Success));
                                }
                                else
                                {
                                    notifier.OnCheckPerformed(new CheckEventArgs("Dataset " + ds + " is completely empty (when linked with the cohort). " +
                                                                                 "Extraction may fail if the Source does not allow empty extractions",
                                                                                 CheckResult.Warning));
                                }
                            }
                        }
                        catch (Exception e)
                        {
                            if (server.GetQuerySyntaxHelper().IsTimeout(e))
                            {
                                notifier.OnCheckPerformed(new CheckEventArgs("Failed to read rows after " + timeout + "s", CheckResult.Warning, e));
                            }
                            else
                            {
                                notifier.OnCheckPerformed(new CheckEventArgs("Failed to execute the query (See below for query)", CheckResult.Fail, e));
                            }
                        }

                        con.ManagedTransaction.AbandonAndCloseConnection();
                    }
                }
                catch (Exception e)
                {
                    notifier.OnCheckPerformed(new CheckEventArgs("Failed to execute Top 1 on dataset " + ds, CheckResult.Fail, e));
                }
            }

            var cata         = ds.Catalogue;
            var fetchOptions = _checkGlobals ? FetchOptions.ExtractableGlobalsAndLocals : FetchOptions.ExtractableLocals;

            foreach (var supportingDocument in cata.GetAllSupportingDocuments(fetchOptions))
            {
                new SupportingDocumentsFetcher(supportingDocument).Check(notifier);
            }

            //check catalogue locals
            foreach (SupportingSQLTable table in cata.GetAllSupportingSQLTablesForCatalogue(fetchOptions))
            {
                new SupportingSQLTableChecker(table).Check(notifier);
            }

            if (_alsoCheckPipeline != null)
            {
                var engine = new ExtractionPipelineUseCase(request.Project, request, _alsoCheckPipeline, DataLoadInfo.Empty)
                             .GetEngine(_alsoCheckPipeline, new FromCheckNotifierToDataLoadEventListener(notifier));
                engine.Check(notifier);
            }
        }