public void TestAllowingEmptyDatasets(bool allowEmptyDatasetExtractions)
{
Pipeline p = SetupPipeline();
TruncateDataTable();
var host = new ExtractionPipelineUseCase(_request.Configuration.Project, _request, p, DataLoadInfo.Empty);
var engine = host.GetEngine(p, new ThrowImmediatelyDataLoadEventListener());
host.Source.AllowEmptyExtractions = allowEmptyDatasetExtractions;
var token = new GracefulCancellationToken();
if (allowEmptyDatasetExtractions)
{
var dt = host.Source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), token);
Assert.IsNull(host.Source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), token));
Assert.AreEqual(0, dt.Rows.Count);
Assert.AreEqual(3, dt.Columns.Count);
}
else
{
var exception = Assert.Throws <Exception>(() => host.Source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), token));
Assert.IsTrue(exception.Message.StartsWith("There is no data to load, query returned no rows, query was"));
}
p.DeleteInDatabase();
}
protected override void ExecuteRun(object runnable, OverrideSenderIDataLoadEventListener listener)
{
var dataLoadInfo = StartAudit();
var globalCommand = runnable as ExtractGlobalsCommand;
var datasetCommand = runnable as ExtractDatasetCommand;
var logging = new ToLoggingDatabaseDataLoadEventListener(_logManager, dataLoadInfo);
var fork =
datasetCommand != null ?
new ForkDataLoadEventListener(logging, listener, new ElevateStateListener(datasetCommand)):
new ForkDataLoadEventListener(logging, listener);
if (globalCommand != null)
{
var useCase = new ExtractionPipelineUseCase(_activator, _project, _globalsCommand, _pipeline, dataLoadInfo)
{
Token = Token
};
useCase.Execute(fork);
}
if (datasetCommand != null)
{
var executeUseCase = new ExtractionPipelineUseCase(_activator, _project, datasetCommand, _pipeline, dataLoadInfo)
{
Token = Token
};
executeUseCase.Execute(fork);
}
logging.FinalizeTableLoadInfos();
dataLoadInfo.CloseAndMarkComplete();
}
protected void Execute(out ExtractionPipelineUseCase pipelineUseCase, out IExecuteDatasetExtractionDestination results)
{
DataLoadInfo d = new DataLoadInfo("Internal", _testDatabaseName, "IgnoreMe", "", true, new DiscoveredServer(UnitTestLoggingConnectionString));
Pipeline pipeline = null;
//because extractable columns is likely to include chi column, it will be removed from the collection (for a substitution identifier)
var before = _extractableColumns.ToArray();
try
{
pipeline = SetupPipeline();
pipelineUseCase = new ExtractionPipelineUseCase(_request.Configuration.Project, _request, pipeline, d);
pipelineUseCase.Execute(new ThrowImmediatelyDataLoadEventListener());
Assert.IsNotEmpty(pipelineUseCase.Source.Request.QueryBuilder.SQL);
Assert.IsTrue(pipelineUseCase.ExtractCommand.State == ExtractCommandState.Completed);
}
finally
{
if (pipeline != null)
{
pipeline.DeleteInDatabase();
}
}
results = pipelineUseCase.Destination;
_extractableColumns = new List <IColumn>(before);
}
protected void Execute(out ExtractionPipelineUseCase pipelineUseCase, out IExecuteDatasetExtractionDestination results)
{
DataLoadInfo d = new DataLoadInfo("Internal", _testDatabaseName, "IgnoreMe", "", true, new DiscoveredServer(UnitTestLoggingConnectionString));
Pipeline pipeline = null;
try
{
pipeline = SetupPipeline();
pipelineUseCase = new ExtractionPipelineUseCase(_request.Configuration.Project, _request, pipeline, d);
pipelineUseCase.Execute(new ThrowImmediatelyDataLoadEventListener());
Assert.IsNotEmpty(pipelineUseCase.Source.Request.QueryBuilder.SQL);
Assert.IsTrue(pipelineUseCase.ExtractCommand.State == ExtractCommandState.Completed);
}
finally
{
if (pipeline != null)
{
pipeline.DeleteInDatabase();
}
}
results = pipelineUseCase.Destination;
}
/// <summary>
/// Checks that all globals pass thier respective checkers (<see cref="SupportingSQLTableChecker"/> and <see cref="SupportingDocumentsFetcher"/>) and that
/// the <see cref="Pipeline"/> (if any) is capable of extracting the globals.
/// </summary>
/// <param name="notifier"></param>
public void Check(ICheckNotifier notifier)
{
foreach (SupportingSQLTable table in _configuration.GetGlobals().OfType <SupportingSQLTable>())
{
new SupportingSQLTableChecker(table).Check(notifier);
}
foreach (SupportingDocument document in _configuration.GetGlobals().OfType <SupportingDocument>())
{
new SupportingDocumentsFetcher(document).Check(notifier);
}
if (_alsoCheckPipeline != null && _command != null)
{
var engine = new ExtractionPipelineUseCase(_configuration.Project, _command, _alsoCheckPipeline, DataLoadInfo.Empty)
.GetEngine(_alsoCheckPipeline, new FromCheckNotifierToDataLoadEventListener(notifier));
engine.Check(notifier);
}
}
private void SetupPipelineSelectionExtraction()
{
//already set i tup
if (_extractionPipelineSelectionUI != null)
{
return;
}
//the use case is extracting a dataset
var useCase = ExtractionPipelineUseCase.DesignTime();
//the user is DefaultPipeline_ID field of ExtractionConfiguration
var user = new PipelineUser(typeof(ExtractionConfiguration).GetProperty("DefaultPipeline_ID"), _extractionConfiguration);
//create the UI for this situation
var factory = new PipelineSelectionUIFactory(Activator.RepositoryLocator.CatalogueRepository, user, useCase);
_extractionPipelineSelectionUI = factory.Create(Activator, "Extraction Pipeline", DockStyle.Fill, pChooseExtractionPipeline);
_extractionPipelineSelectionUI.CollapseToSingleLineMode();
}
public WordDataWriter(ExtractionPipelineUseCase executer)
{
if (executer == null)
{
throw new NullReferenceException("Cannot write meta data without the accompanying ExtractionPipelineHost");
}
if (executer.Source.WasCancelled)
{
throw new NullReferenceException("Cannot write meta data since ExtractionPipelineHost reports that it was Cancelled");
}
Executer = executer;
_destination = Executer.Destination;
if (_destination == null)
{
throw new NotSupportedException(GetType().FullName + " only supports destinations which are " + typeof(ExecuteDatasetExtractionFlatFileDestination).FullName);
}
}
protected override void AdjustCommand(ExtractionOptions opts, CommandLineActivity activity)
{
base.AdjustCommand(opts, activity);
var useCase = ExtractionPipelineUseCase.DesignTime();
var compatible = useCase.FilterCompatiblePipelines(BasicActivator.RepositoryLocator.CatalogueRepository.GetAllObjects <Pipeline>()).ToArray();
if (!compatible.Any())
{
throw new Exception("No compatible pipelines");
}
var pipe = BasicActivator.SelectOne("Extraction Pipeline", compatible, null, true);
if (pipe == null)
{
throw new OperationCanceledException();
}
opts.Pipeline = pipe.ID;
}
public override void SetDatabaseObject(IActivateItems activator, ExtractionConfiguration databaseObject)
{
base.SetDatabaseObject(activator, databaseObject);
_extractionConfiguration = databaseObject;
if (!_commonFunctionality.IsSetup)
{
_commonFunctionality.SetUp(RDMPCollection.None, tlvDatasets, activator, olvName, null, new RDMPCollectionCommonFunctionalitySettings()
{
AddFavouriteColumn = false,
AllowPinning = false,
SuppressChildrenAdder = true,
SuppressActivate = true,
AddCheckColumn = false
});
}
var checkedBefore = tlvDatasets.CheckedObjects;
tlvDatasets.ClearObjects();
_globals = _extractionConfiguration.GetGlobals();
_datasets = databaseObject.SelectedDataSets.ToArray();
GetBundledStuff();
//add the folders
tlvDatasets.AddObjects(new object[] { _globalsFolder, _coreDatasetsFolder, _projectSpecificDatasetsFolder });
//enable all to start with
tlvDatasets.EnableObjects(tlvDatasets.Objects);
tlvDatasets.DisableObjects(_globals);
tlvDatasets.DisableObjects(_bundledStuff);
//if there are no globals disable this option
if (!_globals.Any())
{
tlvDatasets.DisableObject(_globalsFolder);
}
//if there are no project specific datasets
if (_datasets.All(sds => sds.ExtractableDataSet.Project_ID == null))
{
tlvDatasets.DisableObject(_projectSpecificDatasetsFolder); //disable this option
}
//if all the datasets are project specific
if (_datasets.All(sds => sds.ExtractableDataSet.Project_ID != null))
{
tlvDatasets.DisableObject(_coreDatasetsFolder);
}
//don't accept refresh while executing
if (checkAndExecuteUI1.IsExecuting)
{
return;
}
if (_pipelineSelectionUI1 == null)
{
//create a new selection UI (pick an extraction pipeliene UI)
var useCase = ExtractionPipelineUseCase.DesignTime();
var factory = new PipelineSelectionUIFactory(Activator.RepositoryLocator.CatalogueRepository, null, useCase);
_pipelineSelectionUI1 = factory.Create("Extraction Pipeline", DockStyle.Fill);
_pipelineSelectionUI1.CollapseToSingleLineMode();
//if the configuration has a default then use that pipeline
if (_extractionConfiguration.DefaultPipeline_ID != null)
{
_pipelineSelectionUI1.Pipeline = _extractionConfiguration.DefaultPipeline;
}
_pipelineSelectionUI1.PipelineChanged += ResetChecksUI;
_pipelinePanel = new ToolStripControlHost((Control)_pipelineSelectionUI1);
helpIcon1.SetHelpText("Extraction", "It is a wise idea to click here if you don't know what this screen can do for you...", BuildHelpFlow());
}
CommonFunctionality.Add(new ToolStripLabel("Extraction Pipeline:"));
CommonFunctionality.Add(_pipelinePanel);
CommonFunctionality.AddHelpStringToToolStrip("Extraction Pipeline", "The sequence of components that will be executed in order to enable the datasets to be extracted. This will start with a source component that performs the linkage against the cohort followed by subsequent components (if any) and then a destination component that writes the final records (e.g. to database / csv file etc).");
CommonFunctionality.AddToMenu(new ExecuteCommandRelease(activator).SetTarget(_extractionConfiguration));
CommonFunctionality.Add(lblMaxConcurrent);
CommonFunctionality.Add(tbMaxConcurrent);
CommonFunctionality.AddHelpStringToToolStrip("Concurrent", "The maximum number of datasets to extract at once. Once this number is reached the remainder will be queued and only started when one of the other extracting datasets completes.");
checkAndExecuteUI1.SetItemActivator(activator);
tlvDatasets.ExpandAll();
if (_isFirstTime)
{
tlvDatasets.CheckAll();
foreach (var disabledObject in tlvDatasets.DisabledObjects.OfType <ArbitraryFolderNode>())
{
tlvDatasets.UncheckObject(disabledObject);
}
}
else if (checkedBefore.Count > 0)
{
tlvDatasets.CheckObjects(checkedBefore);
}
_isFirstTime = false;
}
public DataExportChildProvider(IRDMPPlatformRepositoryServiceLocator repositoryLocator, IChildProvider[] pluginChildProviders, ICheckNotifier errorsCheckNotifier, DataExportChildProvider previousStateIfKnown) : base(repositoryLocator.CatalogueRepository, pluginChildProviders, errorsCheckNotifier, previousStateIfKnown)
{
BlackListedSources = previousStateIfKnown?.BlackListedSources ?? new List <ExternalCohortTable>();
_errorsCheckNotifier = errorsCheckNotifier;
dataExportRepository = repositoryLocator.DataExportRepository;
AllProjectAssociatedCics = GetAllObjects <ProjectCohortIdentificationConfigurationAssociation>(dataExportRepository);
_cicAssociations = new HashSet <int>(AllProjectAssociatedCics.Select(a => a.CohortIdentificationConfiguration_ID));
CohortSources = GetAllObjects <ExternalCohortTable>(dataExportRepository);
ExtractableDataSets = GetAllObjects <ExtractableDataSet>(dataExportRepository);
AddToDictionaries(new HashSet <object>(AllCohortIdentificationConfigurations.Where(cic => _cicAssociations.Contains(cic.ID))), new DescendancyList(AllProjectCohortIdentificationConfigurationsNode));
AddToDictionaries(new HashSet <object>(AllCohortIdentificationConfigurations.Where(cic => !_cicAssociations.Contains(cic.ID))), new DescendancyList(AllFreeCohortIdentificationConfigurationsNode));
_selectedDataSetsWithNoIsExtractionIdentifier = new HashSet <ISelectedDataSets>(dataExportRepository.GetSelectedDatasetsWithNoExtractionIdentifiers());
SelectedDataSets = GetAllObjects <SelectedDataSets>(dataExportRepository);
ReportProgress("Fetching data export objects");
var dsDictionary = ExtractableDataSets.ToDictionary(ds => ds.ID, d => d);
foreach (SelectedDataSets s in SelectedDataSets)
{
s.InjectKnown(dsDictionary[s.ExtractableDataSet_ID]);
}
ReportProgress("Injecting SelectedDataSets");
//This means that the ToString method in ExtractableDataSet doesn't need to go lookup catalogue info
var catalogueIdDict = AllCatalogues.ToDictionary(c => c.ID, c2 => c2);
foreach (ExtractableDataSet ds in ExtractableDataSets)
{
if (catalogueIdDict.TryGetValue(ds.Catalogue_ID, out Catalogue cata))
{
ds.InjectKnown(cata);
}
}
ReportProgress("Injecting ExtractableDataSet");
AllPackages = GetAllObjects <ExtractableDataSetPackage>(dataExportRepository);
Projects = GetAllObjects <Project>(dataExportRepository);
ExtractionConfigurations = GetAllObjects <ExtractionConfiguration>(dataExportRepository);
ReportProgress("Get Projects and Configurations");
ExtractionConfigurationsByProject = ExtractionConfigurations.GroupBy(k => k.Project_ID).ToDictionary(gdc => gdc.Key, gdc => gdc.ToList());
ReportProgress("Grouping Extractions by Project");
AllGlobalExtractionFilterParameters = GetAllObjects <GlobalExtractionFilterParameter>(dataExportRepository);
AllContainers = GetAllObjects <FilterContainer>(dataExportRepository).ToDictionary(o => o.ID, o => o);
AllDeployedExtractionFilters = GetAllObjects <DeployedExtractionFilter>(dataExportRepository);
_allParameters = GetAllObjects <DeployedExtractionFilterParameter>(dataExportRepository);
ReportProgress("Getting Filters");
//if we are using a database repository then we can make use of the caching class DataExportFilterManagerFromChildProvider to speed up
//filter contents
var dbRepo = dataExportRepository as DataExportRepository;
_dataExportFilterManager = dbRepo == null ? dataExportRepository.FilterManager : new DataExportFilterManagerFromChildProvider(dbRepo, this);
ReportProgress("Building FilterManager");
Cohorts = GetAllObjects <ExtractableCohort>(dataExportRepository);
_cohortsByOriginId = new Dictionary <int, HashSet <ExtractableCohort> >();
foreach (ExtractableCohort c in Cohorts)
{
if (!_cohortsByOriginId.ContainsKey(c.OriginID))
{
_cohortsByOriginId.Add(c.OriginID, new HashSet <ExtractableCohort>());
}
_cohortsByOriginId[c.OriginID].Add(c);
}
_configurationToDatasetMapping = new Dictionary <ExtractionConfiguration, List <SelectedDataSets> >();
ReportProgress("Fetching Cohorts");
GetCohortAvailability();
ReportProgress("GetCohortAvailability");
var configToSds = SelectedDataSets.GroupBy(k => k.ExtractionConfiguration_ID).ToDictionary(gdc => gdc.Key, gdc => gdc.ToList());
foreach (ExtractionConfiguration configuration in ExtractionConfigurations)
{
if (configToSds.TryGetValue(configuration.ID, out List <SelectedDataSets> result))
{
_configurationToDatasetMapping.Add(configuration, result);
}
}
ReportProgress("Mapping configurations to datasets");
RootCohortsNode = new AllCohortsNode();
AddChildren(RootCohortsNode, new DescendancyList(RootCohortsNode));
foreach (ExtractableDataSetPackage package in AllPackages)
{
AddChildren(package, new DescendancyList(package));
}
ReportProgress("Packages and Cohorts");
foreach (Project p in Projects)
{
AddChildren(p, new DescendancyList(p));
}
ReportProgress("Projects");
//work out all the Catalogues that are extractable (Catalogues are extractable if there is an ExtractableDataSet with the Catalogue_ID that matches them)
var cataToEds = new Dictionary <int, ExtractableDataSet>(ExtractableDataSets.ToDictionary(k => k.Catalogue_ID));
//inject extractability into Catalogues
foreach (Catalogue catalogue in AllCatalogues)
{
if (cataToEds.TryGetValue(catalogue.ID, out ExtractableDataSet result))
{
catalogue.InjectKnown(result.GetCatalogueExtractabilityStatus());
}
else
{
catalogue.InjectKnown(new CatalogueExtractabilityStatus(false, false));
}
}
ReportProgress("Catalogue extractability injection");
try
{
AddPipelineUseCases(new Dictionary <string, PipelineUseCase>
{
{ "File Import", UploadFileUseCase.DesignTime() },
{ "Extraction", ExtractionPipelineUseCase.DesignTime() },
{ "Release", ReleaseUseCase.DesignTime() },
{ "Cohort Creation", CohortCreationRequest.DesignTime() },
{ "Caching", CachingPipelineUseCase.DesignTime() },
{ "Aggregate Committing", CreateTableFromAggregateUseCase.DesignTime(repositoryLocator.CatalogueRepository) }
});
}
catch (Exception ex)
{
_errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs("Failed to build DesignTime PipelineUseCases", CheckResult.Fail, ex));
}
ReportProgress("Pipeline adding");
}
/// <summary>
/// Checks the <see cref="SelectedDataSet"/> and reports success/failures to the <paramref name="notifier"/>
/// </summary>
/// <param name="notifier"></param>
public void Check(ICheckNotifier notifier)
{
var ds = SelectedDataSet.ExtractableDataSet;
var config = SelectedDataSet.ExtractionConfiguration;
var cohort = config.Cohort;
var project = config.Project;
const int timeout = 5;
notifier.OnCheckPerformed(new CheckEventArgs("Inspecting dataset " + ds, CheckResult.Success));
var selectedcols = new List <IColumn>(config.GetAllExtractableColumnsFor(ds));
if (!selectedcols.Any())
{
notifier.OnCheckPerformed(
new CheckEventArgs(
"Dataset " + ds + " in configuration '" + config + "' has no selected columns",
CheckResult.Fail));
return;
}
ICatalogue cata;
try
{
cata = ds.Catalogue;
}
catch (Exception e)
{
notifier.OnCheckPerformed(new CheckEventArgs("Unable to find Catalogue for ExtractableDataSet", CheckResult.Fail, e));
return;
}
if (cata.IsInternalDataset)
{
notifier.OnCheckPerformed(new CheckEventArgs($"Dataset '{ds}' is marked {nameof(ICatalogue.IsInternalDataset)} so should not be extracted", CheckResult.Fail));
}
var request = new ExtractDatasetCommand(config, cohort, new ExtractableDatasetBundle(ds),
selectedcols, new HICProjectSalt(project), new ExtractionDirectory(project.ExtractionDirectory, config))
{
TopX = 1
};
try
{
request.GenerateQueryBuilder();
}
catch (Exception e)
{
notifier.OnCheckPerformed(
new CheckEventArgs(
"Could not generate valid extraction SQL for dataset " + ds +
" in configuration " + config, CheckResult.Fail, e));
return;
}
var server = request.GetDistinctLiveDatabaseServer();
bool serverExists = server.Exists();
notifier.OnCheckPerformed(new CheckEventArgs("Server " + server + " Exists:" + serverExists,
serverExists ? CheckResult.Success : CheckResult.Fail));
var cohortServer = request.ExtractableCohort.ExternalCohortTable.Discover();
if (cohortServer == null || !cohortServer.Exists())
{
notifier.OnCheckPerformed(new CheckEventArgs("Cohort server did not exist or was unreachable", CheckResult.Fail));
return;
}
//when 2+ columns have the same Name it's a problem
foreach (IGrouping <string, IColumn> grouping in request.ColumnsToExtract.GroupBy(c => c.GetRuntimeName()).Where(g => g.Count() > 1))
{
notifier.OnCheckPerformed(new CheckEventArgs($"There are { grouping.Count() } columns in the extract ({request.DatasetBundle?.DataSet}) called '{ grouping.Key }'", CheckResult.Fail));
}
//when 2+ columns have the same Order it's a problem because
foreach (IGrouping <int, IColumn> grouping in request.ColumnsToExtract.GroupBy(c => c.Order).Where(g => g.Count() > 1))
{
notifier.OnCheckPerformed(new CheckEventArgs($"There are { grouping.Count() } columns in the extract ({request.DatasetBundle?.DataSet}) that share the same Order '{ grouping.Key }'", CheckResult.Fail));
}
// Warn user if stuff is out of sync with the Catalogue version (changes have happened to the master but not propgated to the copy in this extraction)
var outOfSync = selectedcols.OfType <ExtractableColumn>().Where(c => c.IsOutOfSync()).ToArray();
if (outOfSync.Any())
{
notifier.OnCheckPerformed(new CheckEventArgs($"'{ds}' columns out of sync with CatalogueItem version(s): { Environment.NewLine + string.Join(',', outOfSync.Select(o => o.ToString() + Environment.NewLine)) }" +
$"{ Environment.NewLine } Extraction Configuration: '{config}' ", CheckResult.Warning));
}
var nonSelectedCore = cata.GetAllExtractionInformation(ExtractionCategory.Core)
.Union(cata.GetAllExtractionInformation(ExtractionCategory.ProjectSpecific))
.Where(ei => !ei.IsExtractionIdentifier &&
!selectedcols.OfType <ExtractableColumn>().Any(ec => ec.CatalogueExtractionInformation_ID == ei.ID))
.ToArray();
if (nonSelectedCore.Any())
{
notifier.OnCheckPerformed(new CheckEventArgs($"'{ds}' Core columns not selected for extractions: { Environment.NewLine + string.Join(',', nonSelectedCore.Select(o => o.ToString() + Environment.NewLine)) }" +
$"{ Environment.NewLine } Extraction Configuration: '{config}' ", CheckResult.Warning));
}
//Make sure cohort and dataset are on same server before checking (can still get around this at runtime by using ExecuteCrossServerDatasetExtractionSource)
if (!cohortServer.Server.Name.Equals(server.Name, StringComparison.CurrentCultureIgnoreCase) || !cohortServer.Server.DatabaseType.Equals(server.DatabaseType))
{
notifier.OnCheckPerformed(new CheckEventArgs(
$"Cohort is on server '{cohortServer.Server.Name}' ({cohortServer.Server.DatabaseType}) but dataset '{request.DatasetBundle?.DataSet}' is on '{server.Name}' ({server.DatabaseType})"
, CheckResult.Warning));
}
else
{
//Try to fetch TOP 1 data
try
{
using (var con = server.BeginNewTransactedConnection())
{
//incase user somehow manages to write a filter/transform that nukes data or something
DbCommand cmd;
try
{
cmd = server.GetCommand(request.QueryBuilder.SQL, con);
cmd.CommandTimeout = timeout;
notifier.OnCheckPerformed(
new CheckEventArgs(
"/*About to send Request SQL :*/" + Environment.NewLine + request.QueryBuilder.SQL,
CheckResult.Success));
}
catch (QueryBuildingException e)
{
notifier.OnCheckPerformed(new CheckEventArgs("Failed to assemble query for dataset " + ds,
CheckResult.Fail, e));
return;
}
try
{
using (var r = cmd.ExecuteReader())
{
if (r.Read())
{
notifier.OnCheckPerformed(new CheckEventArgs("Read at least 1 row successfully from dataset " + ds,
CheckResult.Success));
}
else
{
notifier.OnCheckPerformed(new CheckEventArgs("Dataset " + ds + " is completely empty (when linked with the cohort). " +
"Extraction may fail if the Source does not allow empty extractions",
CheckResult.Warning));
}
}
}
catch (Exception e)
{
if (server.GetQuerySyntaxHelper().IsTimeout(e))
{
notifier.OnCheckPerformed(new CheckEventArgs(ErrorCodes.ExtractTimeoutChecking, e, timeout));
}
else
{
notifier.OnCheckPerformed(new CheckEventArgs("Failed to execute the query (See below for query)", CheckResult.Fail, e));
}
}
con.ManagedTransaction.AbandonAndCloseConnection();
}
}
catch (Exception e)
{
notifier.OnCheckPerformed(new CheckEventArgs("Failed to execute Top 1 on dataset " + ds, CheckResult.Fail, e));
}
}
var fetchOptions = _checkGlobals ? FetchOptions.ExtractableGlobalsAndLocals : FetchOptions.ExtractableLocals;
foreach (var supportingDocument in cata.GetAllSupportingDocuments(fetchOptions))
{
new SupportingDocumentsFetcher(supportingDocument).Check(notifier);
}
//check catalogue locals
foreach (SupportingSQLTable table in cata.GetAllSupportingSQLTablesForCatalogue(fetchOptions))
{
new SupportingSQLTableChecker(table).Check(notifier);
}
if (_alsoCheckPipeline != null)
{
var engine = new ExtractionPipelineUseCase(_activator, request.Project, request, _alsoCheckPipeline, DataLoadInfo.Empty)
.GetEngine(_alsoCheckPipeline, new FromCheckNotifierToDataLoadEventListener(notifier));
engine.Check(notifier);
}
}
/// <summary>
/// Checks the <see cref="SelectedDataSet"/> and reports success/failures to the <paramref name="notifier"/>
/// </summary>
/// <param name="notifier"></param>
public void Check(ICheckNotifier notifier)
{
var ds = SelectedDataSet.ExtractableDataSet;
var config = SelectedDataSet.ExtractionConfiguration;
var cohort = config.Cohort;
var project = config.Project;
const int timeout = 5;
notifier.OnCheckPerformed(new CheckEventArgs("Inspecting dataset " + ds, CheckResult.Success));
var selectedcols = new List <IColumn>(config.GetAllExtractableColumnsFor(ds));
if (!selectedcols.Any())
{
notifier.OnCheckPerformed(
new CheckEventArgs(
"Dataset " + ds + " in configuration '" + config + "' has no selected columns",
CheckResult.Fail));
return;
}
var request = new ExtractDatasetCommand(config, cohort, new ExtractableDatasetBundle(ds),
selectedcols, new HICProjectSalt(project), new ExtractionDirectory(project.ExtractionDirectory, config))
{
TopX = 1
};
try
{
request.GenerateQueryBuilder();
}
catch (Exception e)
{
notifier.OnCheckPerformed(
new CheckEventArgs(
"Could not generate valid extraction SQL for dataset " + ds +
" in configuration " + config, CheckResult.Fail, e));
return;
}
var server = request.GetDistinctLiveDatabaseServer();
bool serverExists = server.Exists();
notifier.OnCheckPerformed(new CheckEventArgs("Server " + server + " Exists:" + serverExists,
serverExists ? CheckResult.Success : CheckResult.Fail));
var cohortServer = request.ExtractableCohort.ExternalCohortTable.Discover();
if (cohortServer == null || !cohortServer.Exists())
{
notifier.OnCheckPerformed(new CheckEventArgs("Cohort server did not exist or was unreachable", CheckResult.Fail));
return;
}
foreach (IGrouping <string, IColumn> grouping in request.ColumnsToExtract.GroupBy(c => c.GetRuntimeName()).Where(g => g.Count() > 1))
{
notifier.OnCheckPerformed(new CheckEventArgs("There are " + grouping.Count() + " columns in the extract called '" + grouping.Key + "'", CheckResult.Fail));
}
//Make sure cohort and dataset are on same server before checking (can still get around this at runtime by using ExecuteCrossServerDatasetExtractionSource)
if (!cohortServer.Server.Name.Equals(server.Name, StringComparison.CurrentCultureIgnoreCase) || !cohortServer.Server.DatabaseType.Equals(server.DatabaseType))
{
notifier.OnCheckPerformed(new CheckEventArgs(
string.Format("Cohort is on server '{0}' ({1}) but dataset is on '{2}' ({3})",
cohortServer.Server.Name,
cohortServer.Server.DatabaseType,
server.Name, server.DatabaseType), CheckResult.Warning));
}
else
{
//Try to fetch TOP 1 data
try
{
using (var con = server.BeginNewTransactedConnection())
{
//incase user somehow manages to write a filter/transform that nukes data or something
DbCommand cmd;
try
{
cmd = server.GetCommand(request.QueryBuilder.SQL, con);
cmd.CommandTimeout = timeout;
notifier.OnCheckPerformed(
new CheckEventArgs(
"/*About to send Request SQL :*/" + Environment.NewLine + request.QueryBuilder.SQL,
CheckResult.Success));
}
catch (QueryBuildingException e)
{
notifier.OnCheckPerformed(new CheckEventArgs("Failed to assemble query for dataset " + ds,
CheckResult.Fail, e));
return;
}
try
{
using (var r = cmd.ExecuteReader())
{
if (r.Read())
{
notifier.OnCheckPerformed(new CheckEventArgs("Read at least 1 row successfully from dataset " + ds,
CheckResult.Success));
}
else
{
notifier.OnCheckPerformed(new CheckEventArgs("Dataset " + ds + " is completely empty (when linked with the cohort). " +
"Extraction may fail if the Source does not allow empty extractions",
CheckResult.Warning));
}
}
}
catch (Exception e)
{
if (server.GetQuerySyntaxHelper().IsTimeout(e))
{
notifier.OnCheckPerformed(new CheckEventArgs("Failed to read rows after " + timeout + "s", CheckResult.Warning, e));
}
else
{
notifier.OnCheckPerformed(new CheckEventArgs("Failed to execute the query (See below for query)", CheckResult.Fail, e));
}
}
con.ManagedTransaction.AbandonAndCloseConnection();
}
}
catch (Exception e)
{
notifier.OnCheckPerformed(new CheckEventArgs("Failed to execute Top 1 on dataset " + ds, CheckResult.Fail, e));
}
}
var cata = ds.Catalogue;
var fetchOptions = _checkGlobals ? FetchOptions.ExtractableGlobalsAndLocals : FetchOptions.ExtractableLocals;
foreach (var supportingDocument in cata.GetAllSupportingDocuments(fetchOptions))
{
new SupportingDocumentsFetcher(supportingDocument).Check(notifier);
}
//check catalogue locals
foreach (SupportingSQLTable table in cata.GetAllSupportingSQLTablesForCatalogue(fetchOptions))
{
new SupportingSQLTableChecker(table).Check(notifier);
}
if (_alsoCheckPipeline != null)
{
var engine = new ExtractionPipelineUseCase(request.Project, request, _alsoCheckPipeline, DataLoadInfo.Empty)
.GetEngine(_alsoCheckPipeline, new FromCheckNotifierToDataLoadEventListener(notifier));
engine.Check(notifier);
}
}
