/// <summary>
/// Checks that all globals pass thier respective checkers (<see cref="SupportingSQLTableChecker"/> and <see cref="SupportingDocumentsFetcher"/>) and that
/// the <see cref="Pipeline"/> (if any) is capable of extracting the globals.
/// </summary>
/// <param name="notifier"></param>
public void Check(ICheckNotifier notifier)
{
foreach (SupportingSQLTable table in _configuration.GetGlobals().OfType <SupportingSQLTable>())
{
new SupportingSQLTableChecker(table).Check(notifier);
}
foreach (SupportingDocument document in _configuration.GetGlobals().OfType <SupportingDocument>())
{
new SupportingDocumentsFetcher(document).Check(notifier);
}
if (_alsoCheckPipeline != null && _command != null)
{
var engine = new ExtractionPipelineUseCase(_configuration.Project, _command, _alsoCheckPipeline, DataLoadInfo.Empty)
.GetEngine(_alsoCheckPipeline, new FromCheckNotifierToDataLoadEventListener(notifier));
engine.Check(notifier);
}
}
/// <summary>
/// Checks the <see cref="SelectedDataSet"/> and reports success/failures to the <paramref name="notifier"/>
/// </summary>
/// <param name="notifier"></param>
public void Check(ICheckNotifier notifier)
{
var ds = SelectedDataSet.ExtractableDataSet;
var config = SelectedDataSet.ExtractionConfiguration;
var cohort = config.Cohort;
var project = config.Project;
const int timeout = 5;
notifier.OnCheckPerformed(new CheckEventArgs("Inspecting dataset " + ds, CheckResult.Success));
var selectedcols = new List <IColumn>(config.GetAllExtractableColumnsFor(ds));
if (!selectedcols.Any())
{
notifier.OnCheckPerformed(
new CheckEventArgs(
"Dataset " + ds + " in configuration '" + config + "' has no selected columns",
CheckResult.Fail));
return;
}
ICatalogue cata;
try
{
cata = ds.Catalogue;
}
catch (Exception e)
{
notifier.OnCheckPerformed(new CheckEventArgs("Unable to find Catalogue for ExtractableDataSet", CheckResult.Fail, e));
return;
}
if (cata.IsInternalDataset)
{
notifier.OnCheckPerformed(new CheckEventArgs($"Dataset '{ds}' is marked {nameof(ICatalogue.IsInternalDataset)} so should not be extracted", CheckResult.Fail));
}
var request = new ExtractDatasetCommand(config, cohort, new ExtractableDatasetBundle(ds),
selectedcols, new HICProjectSalt(project), new ExtractionDirectory(project.ExtractionDirectory, config))
{
TopX = 1
};
try
{
request.GenerateQueryBuilder();
}
catch (Exception e)
{
notifier.OnCheckPerformed(
new CheckEventArgs(
"Could not generate valid extraction SQL for dataset " + ds +
" in configuration " + config, CheckResult.Fail, e));
return;
}
var server = request.GetDistinctLiveDatabaseServer();
bool serverExists = server.Exists();
notifier.OnCheckPerformed(new CheckEventArgs("Server " + server + " Exists:" + serverExists,
serverExists ? CheckResult.Success : CheckResult.Fail));
var cohortServer = request.ExtractableCohort.ExternalCohortTable.Discover();
if (cohortServer == null || !cohortServer.Exists())
{
notifier.OnCheckPerformed(new CheckEventArgs("Cohort server did not exist or was unreachable", CheckResult.Fail));
return;
}
//when 2+ columns have the same Name it's a problem
foreach (IGrouping <string, IColumn> grouping in request.ColumnsToExtract.GroupBy(c => c.GetRuntimeName()).Where(g => g.Count() > 1))
{
notifier.OnCheckPerformed(new CheckEventArgs($"There are { grouping.Count() } columns in the extract ({request.DatasetBundle?.DataSet}) called '{ grouping.Key }'", CheckResult.Fail));
}
//when 2+ columns have the same Order it's a problem because
foreach (IGrouping <int, IColumn> grouping in request.ColumnsToExtract.GroupBy(c => c.Order).Where(g => g.Count() > 1))
{
notifier.OnCheckPerformed(new CheckEventArgs($"There are { grouping.Count() } columns in the extract ({request.DatasetBundle?.DataSet}) that share the same Order '{ grouping.Key }'", CheckResult.Fail));
}
// Warn user if stuff is out of sync with the Catalogue version (changes have happened to the master but not propgated to the copy in this extraction)
var outOfSync = selectedcols.OfType <ExtractableColumn>().Where(c => c.IsOutOfSync()).ToArray();
if (outOfSync.Any())
{
notifier.OnCheckPerformed(new CheckEventArgs($"'{ds}' columns out of sync with CatalogueItem version(s): { Environment.NewLine + string.Join(',', outOfSync.Select(o => o.ToString() + Environment.NewLine)) }" +
$"{ Environment.NewLine } Extraction Configuration: '{config}' ", CheckResult.Warning));
}
var nonSelectedCore = cata.GetAllExtractionInformation(ExtractionCategory.Core)
.Union(cata.GetAllExtractionInformation(ExtractionCategory.ProjectSpecific))
.Where(ei => !ei.IsExtractionIdentifier &&
!selectedcols.OfType <ExtractableColumn>().Any(ec => ec.CatalogueExtractionInformation_ID == ei.ID))
.ToArray();
if (nonSelectedCore.Any())
{
notifier.OnCheckPerformed(new CheckEventArgs($"'{ds}' Core columns not selected for extractions: { Environment.NewLine + string.Join(',', nonSelectedCore.Select(o => o.ToString() + Environment.NewLine)) }" +
$"{ Environment.NewLine } Extraction Configuration: '{config}' ", CheckResult.Warning));
}
//Make sure cohort and dataset are on same server before checking (can still get around this at runtime by using ExecuteCrossServerDatasetExtractionSource)
if (!cohortServer.Server.Name.Equals(server.Name, StringComparison.CurrentCultureIgnoreCase) || !cohortServer.Server.DatabaseType.Equals(server.DatabaseType))
{
notifier.OnCheckPerformed(new CheckEventArgs(
$"Cohort is on server '{cohortServer.Server.Name}' ({cohortServer.Server.DatabaseType}) but dataset '{request.DatasetBundle?.DataSet}' is on '{server.Name}' ({server.DatabaseType})"
, CheckResult.Warning));
}
else
{
//Try to fetch TOP 1 data
try
{
using (var con = server.BeginNewTransactedConnection())
{
//incase user somehow manages to write a filter/transform that nukes data or something
DbCommand cmd;
try
{
cmd = server.GetCommand(request.QueryBuilder.SQL, con);
cmd.CommandTimeout = timeout;
notifier.OnCheckPerformed(
new CheckEventArgs(
"/*About to send Request SQL :*/" + Environment.NewLine + request.QueryBuilder.SQL,
CheckResult.Success));
}
catch (QueryBuildingException e)
{
notifier.OnCheckPerformed(new CheckEventArgs("Failed to assemble query for dataset " + ds,
CheckResult.Fail, e));
return;
}
try
{
using (var r = cmd.ExecuteReader())
{
if (r.Read())
{
notifier.OnCheckPerformed(new CheckEventArgs("Read at least 1 row successfully from dataset " + ds,
CheckResult.Success));
}
else
{
notifier.OnCheckPerformed(new CheckEventArgs("Dataset " + ds + " is completely empty (when linked with the cohort). " +
"Extraction may fail if the Source does not allow empty extractions",
CheckResult.Warning));
}
}
}
catch (Exception e)
{
if (server.GetQuerySyntaxHelper().IsTimeout(e))
{
notifier.OnCheckPerformed(new CheckEventArgs(ErrorCodes.ExtractTimeoutChecking, e, timeout));
}
else
{
notifier.OnCheckPerformed(new CheckEventArgs("Failed to execute the query (See below for query)", CheckResult.Fail, e));
}
}
con.ManagedTransaction.AbandonAndCloseConnection();
}
}
catch (Exception e)
{
notifier.OnCheckPerformed(new CheckEventArgs("Failed to execute Top 1 on dataset " + ds, CheckResult.Fail, e));
}
}
var fetchOptions = _checkGlobals ? FetchOptions.ExtractableGlobalsAndLocals : FetchOptions.ExtractableLocals;
foreach (var supportingDocument in cata.GetAllSupportingDocuments(fetchOptions))
{
new SupportingDocumentsFetcher(supportingDocument).Check(notifier);
}
//check catalogue locals
foreach (SupportingSQLTable table in cata.GetAllSupportingSQLTablesForCatalogue(fetchOptions))
{
new SupportingSQLTableChecker(table).Check(notifier);
}
if (_alsoCheckPipeline != null)
{
var engine = new ExtractionPipelineUseCase(_activator, request.Project, request, _alsoCheckPipeline, DataLoadInfo.Empty)
.GetEngine(_alsoCheckPipeline, new FromCheckNotifierToDataLoadEventListener(notifier));
engine.Check(notifier);
}
}
/// <summary>
/// Checks the <see cref="SelectedDataSet"/> and reports success/failures to the <paramref name="notifier"/>
/// </summary>
/// <param name="notifier"></param>
public void Check(ICheckNotifier notifier)
{
var ds = SelectedDataSet.ExtractableDataSet;
var config = SelectedDataSet.ExtractionConfiguration;
var cohort = config.Cohort;
var project = config.Project;
const int timeout = 5;
notifier.OnCheckPerformed(new CheckEventArgs("Inspecting dataset " + ds, CheckResult.Success));
var selectedcols = new List <IColumn>(config.GetAllExtractableColumnsFor(ds));
if (!selectedcols.Any())
{
notifier.OnCheckPerformed(
new CheckEventArgs(
"Dataset " + ds + " in configuration '" + config + "' has no selected columns",
CheckResult.Fail));
return;
}
var request = new ExtractDatasetCommand(config, cohort, new ExtractableDatasetBundle(ds),
selectedcols, new HICProjectSalt(project), new ExtractionDirectory(project.ExtractionDirectory, config))
{
TopX = 1
};
try
{
request.GenerateQueryBuilder();
}
catch (Exception e)
{
notifier.OnCheckPerformed(
new CheckEventArgs(
"Could not generate valid extraction SQL for dataset " + ds +
" in configuration " + config, CheckResult.Fail, e));
return;
}
var server = request.GetDistinctLiveDatabaseServer();
bool serverExists = server.Exists();
notifier.OnCheckPerformed(new CheckEventArgs("Server " + server + " Exists:" + serverExists,
serverExists ? CheckResult.Success : CheckResult.Fail));
var cohortServer = request.ExtractableCohort.ExternalCohortTable.Discover();
if (cohortServer == null || !cohortServer.Exists())
{
notifier.OnCheckPerformed(new CheckEventArgs("Cohort server did not exist or was unreachable", CheckResult.Fail));
return;
}
foreach (IGrouping <string, IColumn> grouping in request.ColumnsToExtract.GroupBy(c => c.GetRuntimeName()).Where(g => g.Count() > 1))
{
notifier.OnCheckPerformed(new CheckEventArgs("There are " + grouping.Count() + " columns in the extract called '" + grouping.Key + "'", CheckResult.Fail));
}
//Make sure cohort and dataset are on same server before checking (can still get around this at runtime by using ExecuteCrossServerDatasetExtractionSource)
if (!cohortServer.Server.Name.Equals(server.Name, StringComparison.CurrentCultureIgnoreCase) || !cohortServer.Server.DatabaseType.Equals(server.DatabaseType))
{
notifier.OnCheckPerformed(new CheckEventArgs(
string.Format("Cohort is on server '{0}' ({1}) but dataset is on '{2}' ({3})",
cohortServer.Server.Name,
cohortServer.Server.DatabaseType,
server.Name, server.DatabaseType), CheckResult.Warning));
}
else
{
//Try to fetch TOP 1 data
try
{
using (var con = server.BeginNewTransactedConnection())
{
//incase user somehow manages to write a filter/transform that nukes data or something
DbCommand cmd;
try
{
cmd = server.GetCommand(request.QueryBuilder.SQL, con);
cmd.CommandTimeout = timeout;
notifier.OnCheckPerformed(
new CheckEventArgs(
"/*About to send Request SQL :*/" + Environment.NewLine + request.QueryBuilder.SQL,
CheckResult.Success));
}
catch (QueryBuildingException e)
{
notifier.OnCheckPerformed(new CheckEventArgs("Failed to assemble query for dataset " + ds,
CheckResult.Fail, e));
return;
}
try
{
using (var r = cmd.ExecuteReader())
{
if (r.Read())
{
notifier.OnCheckPerformed(new CheckEventArgs("Read at least 1 row successfully from dataset " + ds,
CheckResult.Success));
}
else
{
notifier.OnCheckPerformed(new CheckEventArgs("Dataset " + ds + " is completely empty (when linked with the cohort). " +
"Extraction may fail if the Source does not allow empty extractions",
CheckResult.Warning));
}
}
}
catch (Exception e)
{
if (server.GetQuerySyntaxHelper().IsTimeout(e))
{
notifier.OnCheckPerformed(new CheckEventArgs("Failed to read rows after " + timeout + "s", CheckResult.Warning, e));
}
else
{
notifier.OnCheckPerformed(new CheckEventArgs("Failed to execute the query (See below for query)", CheckResult.Fail, e));
}
}
con.ManagedTransaction.AbandonAndCloseConnection();
}
}
catch (Exception e)
{
notifier.OnCheckPerformed(new CheckEventArgs("Failed to execute Top 1 on dataset " + ds, CheckResult.Fail, e));
}
}
var cata = ds.Catalogue;
var fetchOptions = _checkGlobals ? FetchOptions.ExtractableGlobalsAndLocals : FetchOptions.ExtractableLocals;
foreach (var supportingDocument in cata.GetAllSupportingDocuments(fetchOptions))
{
new SupportingDocumentsFetcher(supportingDocument).Check(notifier);
}
//check catalogue locals
foreach (SupportingSQLTable table in cata.GetAllSupportingSQLTablesForCatalogue(fetchOptions))
{
new SupportingSQLTableChecker(table).Check(notifier);
}
if (_alsoCheckPipeline != null)
{
var engine = new ExtractionPipelineUseCase(request.Project, request, _alsoCheckPipeline, DataLoadInfo.Empty)
.GetEngine(_alsoCheckPipeline, new FromCheckNotifierToDataLoadEventListener(notifier));
engine.Check(notifier);
}
}