private void SetupDataExport()
{
_extractableDataSet = new ExtractableDataSet(DataExportRepository, _catalogue);
_project = new Project(DataExportRepository, _testDatabaseName);
_project.ProjectNumber = 1;
Directory.CreateDirectory(ProjectDirectory);
_project.ExtractionDirectory = ProjectDirectory;
_project.SaveToDatabase();
_configuration = new ExtractionConfiguration(DataExportRepository, _project);
//select the dataset for extraction under this configuration
_selectedDataSet = new SelectedDataSets(RepositoryLocator.DataExportRepository, _configuration, _extractableDataSet, null);
//select all the columns for extraction
foreach (var toSelect in _extractionInformations)
{
var col = new ExtractableColumn(DataExportRepository, _extractableDataSet, _configuration, toSelect, toSelect.Order, toSelect.SelectSQL);
if (col.GetRuntimeName().Equals("PrivateID"))
{
col.IsExtractionIdentifier = true;
}
col.SaveToDatabase();
_extractableColumns.Add(col);
}
}
/// <summary>
/// Returns true if the given <paramref name="modelObject"/> survives filtering based on the supplied inclusion
/// criteria. Anything that isn't in some way related to a <see cref="Catalogue"/> automatically survives filtering
/// </summary>
/// <param name="modelObject"></param>
/// <param name="descendancy"></param>
/// <param name="includeInternal"></param>
/// <param name="includeDeprecated"></param>
/// <param name="includeColdStorage"></param>
/// <param name="includeProjectSpecific"></param>
/// <param name="includeNonExtractable"></param>
/// <returns>True if the item should be shown to the user based on filters</returns>
public static bool Filter(object modelObject, DescendancyList descendancy, bool includeInternal, bool includeDeprecated, bool includeColdStorage, bool includeProjectSpecific, bool includeNonExtractable)
{
var cata = modelObject as ICatalogue;
//doesn't relate to us...
if (cata == null)
{
// or are we one of these things that can be tied to a catalogue
cata = modelObject switch
{
ExtractableDataSet eds => eds.Catalogue,
SelectedDataSets sds => sds.GetCatalogue(),
_ => descendancy?.Parents.OfType <Catalogue>().SingleOrDefault()
};
if (cata == null)
{
return(true);
}
}
bool isProjectSpecific = cata.IsProjectSpecific(null);
bool isExtractable = cata.GetExtractabilityStatus(null) != null && cata.GetExtractabilityStatus(null).IsExtractable;
return((isExtractable && !cata.IsColdStorageDataset && !cata.IsDeprecated && !cata.IsInternalDataset && !isProjectSpecific) ||
((includeColdStorage && cata.IsColdStorageDataset) ||
(includeDeprecated && cata.IsDeprecated) ||
(includeInternal && cata.IsInternalDataset) ||
(includeProjectSpecific && isProjectSpecific) ||
(includeNonExtractable && !isExtractable)));
}
public ExecuteCommandViewThenVsNowSql(IActivateItems activator, SelectedDataSets selectedDataSet) : base(activator)
{
try
{
var rp = new FlatFileReleasePotential(Activator.RepositoryLocator, selectedDataSet);
rp.Check(new IgnoreAllErrorsCheckNotifier());
if (string.IsNullOrWhiteSpace(rp.SqlCurrentConfiguration))
{
SetImpossible("Could not generate Sql for dataset");
}
else
if (string.IsNullOrWhiteSpace(rp.SqlExtracted))
{
SetImpossible("Dataset has never been extracted");
}
else
if (rp.SqlCurrentConfiguration == rp.SqlExtracted)
{
SetImpossible("No differences");
}
_releasePotential = rp;
}
catch (Exception)
{
SetImpossible("Could not make assesment");
}
}
public override void SetDatabaseObject(IActivateItems activator, SelectedDataSets databaseObject)
{
base.SetDatabaseObject(activator, databaseObject);
_extractionConfiguration = databaseObject.ExtractionConfiguration;
_extractableDataSet = databaseObject.ExtractableDataSet;
RegenerateCodeInQueryEditor();
}
public bool IsMissingExtractionIdentifier(SelectedDataSets selectedDataSets)
{
lock (WriteLock)
{
return(_selectedDataSetsWithNoIsExtractionIdentifier.Contains(selectedDataSets));
}
}
private void AddChildren(SelectedDataSets selectedDataSets, DescendancyList descendancy)
{
if (selectedDataSets.RootFilterContainer_ID != null)
{
var rootContainer = AllContainers[selectedDataSets.RootFilterContainer_ID.Value];
AddChildren(rootContainer, descendancy.Add(rootContainer));
AddToDictionaries(new HashSet <object>(new object[] { rootContainer }), descendancy);
}
}
private string DescribeProblem(SelectedDataSets selectedDataSets)
{
if (_exportChildProvider.IsMissingExtractionIdentifier(selectedDataSets))
{
return("There are no IsExtractionIdentifier columns in dataset");
}
return(null);
}
public void ExtractionProgressConstructor_NoTimePeriodicity()
{
var cata = new Catalogue(CatalogueRepository, "MyCata");
var eds = new ExtractableDataSet(DataExportRepository, cata);
var project = new Project(DataExportRepository, "My Proj");
var config = new ExtractionConfiguration(DataExportRepository, project);
var sds = new SelectedDataSets(DataExportRepository, config, eds, null);
var ex = Assert.Throws <ArgumentException>(() => new ExtractionProgress(DataExportRepository, sds));
Assert.AreEqual("Cannot create ExtractionProgress because Catalogue MyCata does not have a time coverage column", ex.Message);
}
private void TestDataExportOfTvf()
{
var config = new ExtractionConfiguration(DataExportRepository, _project);
config.Cohort_ID = DataExportRepository.GetAllObjects <ExtractableCohort>().Single().ID;
config.SaveToDatabase();
var tvfExtractable = new ExtractableDataSet(DataExportRepository, _tvfCatalogue);
var selected = new SelectedDataSets(DataExportRepository, config, tvfExtractable, null);
//make all columns part of the extraction
foreach (ExtractionInformation e in _tvfCatalogue.GetAllExtractionInformation(ExtractionCategory.Any))
{
config.AddColumnToExtraction(tvfExtractable, e);
}
//the default value should be 10
Assert.AreEqual("10", _tvfTableInfo.GetAllParameters().Single().Value);
//configure an extraction specific global of 1 so that only 1 chi number is fetched (which will be in the cohort)
var globalP = new GlobalExtractionFilterParameter(DataExportRepository, config, "DECLARE @numberOfRecords AS int;");
globalP.Value = "1";
globalP.SaveToDatabase();
var extractionCommand = new ExtractDatasetCommand(config, new ExtractableDatasetBundle(tvfExtractable));
var source = new ExecuteDatasetExtractionSource();
source.PreInitialize(extractionCommand, new ThrowImmediatelyDataLoadEventListener());
var dt = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
Assert.AreEqual(1, dt.Rows.Count);
Assert.AreEqual("ReleaseId", dt.Columns[0].ColumnName);
//should be a guid
Assert.IsTrue(dt.Rows[0][0].ToString().Length > 10);
Assert.IsTrue(dt.Rows[0][0].ToString().Contains("-"));
selected.DeleteInDatabase();
globalP.DeleteInDatabase();
config.DeleteInDatabase();
tvfExtractable.DeleteInDatabase();
}
public override void Execute()
{
base.Execute();
if (_selectedDataSet == null && _extractionConfiguration != null)
{
_selectedDataSet = SelectOne(Activator.RepositoryLocator.DataExportRepository.GetAllObjectsWithParent <SelectedDataSets>(_extractionConfiguration));
}
if (_selectedDataSet == null)
{
return;
}
Activator.Activate <ViewExtractionConfigurationSQLUI, SelectedDataSets>(_selectedDataSet);
}
private ExtractionProgress CreateAnExtractionProgress()
{
var cata = new Catalogue(CatalogueRepository, "MyCata");
var cataItem = new CatalogueItem(CatalogueRepository, cata, "MyCol");
var table = new TableInfo(CatalogueRepository, "MyTable");
var col = new ColumnInfo(CatalogueRepository, "mycol", "datetime", table);
var ei = new ExtractionInformation(CatalogueRepository, cataItem, col, "mycol");
cata.TimeCoverage_ExtractionInformation_ID = ei.ID;
cata.SaveToDatabase();
var eds = new ExtractableDataSet(DataExportRepository, cata);
var project = new Project(DataExportRepository, "My Proj");
var config = new ExtractionConfiguration(DataExportRepository, project);
var sds = new SelectedDataSets(DataExportRepository, config, eds, null);
return(new ExtractionProgress(DataExportRepository, sds));
}
public override void SetDatabaseObject(IActivateItems activator, SelectedDataSets databaseObject)
{
base.SetDatabaseObject(activator, databaseObject);
activator.RefreshBus.EstablishLifetimeSubscription(this);
SelectedDataSet = databaseObject;
_dataSet = SelectedDataSet.ExtractableDataSet;
_config = (ExtractionConfiguration)SelectedDataSet.ExtractionConfiguration;
SetupUserInterface();
SortSelectedByOrder();
CommonFunctionality.AddToMenu(new ExecuteCommandShow(activator, databaseObject.ExtractableDataSet.Catalogue, 0, true), "Show Catalogue");
CommonFunctionality.Add(new ExecuteCommandExecuteExtractionConfiguration(activator, databaseObject));
CommonFunctionality.AddChecks(new SelectedDataSetsChecker(SelectedDataSet));
}
public void CreateAndAssociateColumns()
{
var cata = new Catalogue(CatalogueRepository, "MyCat");
var cataItem = new CatalogueItem(CatalogueRepository, cata, "MyCataItem");
var TableInfo = new TableInfo(CatalogueRepository, "Cata");
var ColumnInfo = new ColumnInfo(CatalogueRepository, "Col", "varchar(10)", TableInfo);
var ExtractionInfo = new ExtractionInformation(CatalogueRepository, cataItem, ColumnInfo, "fish");
var ds = new ExtractableDataSet(DataExportRepository, cata);
var proj = new Project(DataExportRepository, "MyProj");
var config = new ExtractionConfiguration(DataExportRepository, proj);
SelectedDataSets selectedDataSets;
var extractableColumn = new ExtractableColumn(DataExportRepository, ds, config, ExtractionInfo, 1, "fish");
try
{
selectedDataSets = new SelectedDataSets(DataExportRepository, config, ds, null);
var cols = config.GetAllExtractableColumnsFor(ds);
Assert.AreEqual(1, cols.Count());
Assert.AreEqual(extractableColumn, cols.Single());
cols = config.GetAllExtractableColumnsFor(ds);
Assert.AreEqual(1, cols.Count());
Assert.AreEqual(extractableColumn, cols.Single());
}
finally
{
extractableColumn.DeleteInDatabase();
config.DeleteInDatabase();
proj.DeleteInDatabase();
ds.DeleteInDatabase();
TableInfo.DeleteInDatabase();
cata.DeleteInDatabase();
}
}
public SelectedDataSetsMenu(RDMPContextMenuStripArgs args, SelectedDataSets selectedDataSet) : base(args, selectedDataSet)
{
_selectedDataSet = selectedDataSet;
_extractionConfiguration = _selectedDataSet.ExtractionConfiguration;
Add(new ExecuteCommandExecuteExtractionConfiguration(_activator, selectedDataSet));
Add(new ExecuteCommandRelease(_activator).SetTarget(selectedDataSet));
/////////////////// Extraction Graphs //////////////////////////////
var graphs = new ToolStripMenuItem("View Extraction Graphs", CatalogueIcons.Graph);
var cata = selectedDataSet.ExtractableDataSet.Catalogue;
// If the Catalogue has been deleted, don't build Catalogue specific menu items
if (cata == null)
{
return;
}
var availableGraphs = cata.AggregateConfigurations.Where(a => !a.IsCohortIdentificationAggregate).ToArray();
foreach (AggregateConfiguration ac in availableGraphs)
{
graphs.DropDownItems.Add(ac.ToString(), CatalogueIcons.Graph, (s, e) => GenerateExtractionGraphs(ac));
}
if (availableGraphs.Length > 1)
{
graphs.DropDownItems.Add("All", null, (s, e) => GenerateExtractionGraphs(availableGraphs));
}
//must have graphs available and must have a cohort
graphs.Enabled = graphs.DropDownItems.Count > 0 && _extractionConfiguration.Cohort_ID != null;
Items.Add(graphs);
////////////////////////////////////////////////////////////////////
Add(new ExecuteCommandViewThenVsNowSql(_activator, selectedDataSet));
Add(new ExecuteCommandOpenExtractionDirectory(_activator, selectedDataSet));
}
public SelectedDataSetsMenu(RDMPContextMenuStripArgs args, SelectedDataSets selectedDataSet) : base(args, selectedDataSet)
{
_selectedDataSet = selectedDataSet;
_extractionConfiguration = _selectedDataSet.ExtractionConfiguration;
var root = selectedDataSet.RootFilterContainer;
AddGoTo <Catalogue>(selectedDataSet.ExtractableDataSet.Catalogue_ID);
Add(new ExecuteCommandExecuteExtractionConfiguration(_activator, selectedDataSet));
Add(new ExecuteCommandRelease(_activator).SetTarget(selectedDataSet));
/////////////////// Extraction Graphs //////////////////////////////
var graphs = new ToolStripMenuItem("View Extraction Graphs", CatalogueIcons.Graph);
var availableGraphs = selectedDataSet.ExtractableDataSet.Catalogue.AggregateConfigurations.Where(a => !a.IsCohortIdentificationAggregate).ToArray();
foreach (AggregateConfiguration ac in availableGraphs)
{
graphs.DropDownItems.Add(ac.ToString(), CatalogueIcons.Graph, (s, e) => GenerateExtractionGraphs(ac));
}
if (availableGraphs.Length > 1)
{
graphs.DropDownItems.Add("All", null, (s, e) => GenerateExtractionGraphs(availableGraphs));
}
//must not be relased and must have graphs available and must have a cohort
graphs.Enabled = !_extractionConfiguration.IsReleased && graphs.DropDownItems.Count > 0 && _extractionConfiguration.Cohort_ID != null;
Items.Add(graphs);
////////////////////////////////////////////////////////////////////
var addRootFilter = new ToolStripMenuItem("Add Filter Container", _activator.CoreIconProvider.GetImage(RDMPConcept.FilterContainer, OverlayKind.Add), (s, e) => AddFilterContainer());
addRootFilter.Enabled = root == null;
Items.Add(addRootFilter);
Add(new ExecuteCommandViewSelectedDataSetsExtractionSql(_activator).SetTarget(selectedDataSet));
Add(new ExecuteCommandViewThenVsNowSql(_activator, selectedDataSet));
}
private void AddChildren(SelectedDataSets selectedDataSets, DescendancyList descendancy)
{
HashSet <object> children = new HashSet <object>();
if (_extractionProgressesBySelectedDataSetID.ContainsKey(selectedDataSets.ID))
{
children.Add(_extractionProgressesBySelectedDataSetID[selectedDataSets.ID]);
}
if (selectedDataSets.RootFilterContainer_ID != null)
{
var rootContainer = AllContainers[selectedDataSets.RootFilterContainer_ID.Value];
children.Add(rootContainer);
AddChildren(rootContainer, descendancy.Add(rootContainer));
}
if (children.Any())
{
AddToDictionaries(children, descendancy);
}
}
public IAtomicCommandWithTarget SetTarget(DatabaseEntity target)
{
if (target is SelectedDataSets)
{
_selectedDataSet = target as SelectedDataSets;
if (_selectedDataSet != null)
{
//must have datasets and have a cohort configured
if (_selectedDataSet.ExtractionConfiguration.Cohort_ID == null)
{
SetImpossible("No cohort has been selected for ExtractionConfiguration");
}
}
}
if (target is ExtractionConfiguration)
{
_extractionConfiguration = target as ExtractionConfiguration;
}
return(this);
}
public void TestNoIsExtractionIdentifierFinding()
{
//nothing in database means no dodgy datasets
Assert.IsEmpty(((DataExportRepository)DataExportRepository).GetSelectedDatasetsWithNoExtractionIdentifiers());
var cata = new Catalogue(CatalogueRepository, "ommn");
var ds = new ExtractableDataSet(DataExportRepository, cata);
var proj = new Project(DataExportRepository, "proj");
var config = new ExtractionConfiguration(DataExportRepository, proj);
var sds = new SelectedDataSets(DataExportRepository, config, ds, null);
//only one selected dataset
var dodgy = ((DataExportRepository)DataExportRepository).GetSelectedDatasetsWithNoExtractionIdentifiers().ToArray();
Assert.AreEqual(1, dodgy.Count());
Assert.AreEqual(sds, dodgy[0]);
//make an extarctable column on that dataset
var col = new ColumnInfo(CatalogueRepository, "ff", "varchar(1)", new TableInfo(CatalogueRepository, "fff"));
var ci = new CatalogueItem(CatalogueRepository, cata, "A");
var ei = new ExtractionInformation(CatalogueRepository, ci, col, col.Name);
var ec = new ExtractableColumn(DataExportRepository, ds, config, ei, 0, col.Name);
//still shouldn't be dodgy
dodgy = ((DataExportRepository)DataExportRepository).GetSelectedDatasetsWithNoExtractionIdentifiers().ToArray();
Assert.AreEqual(1, dodgy.Count());
Assert.AreEqual(sds, dodgy[0]);
//now make it non dodgy by being IsExtractionIdentifier
ec.IsExtractionIdentifier = true;
ec.SaveToDatabase();
//no longer dodgy because there is an extraction identifier
dodgy = ((DataExportRepository)DataExportRepository).GetSelectedDatasetsWithNoExtractionIdentifiers().ToArray();
Assert.AreEqual(0, dodgy.Count());
}
public DataExportChildProvider(IRDMPPlatformRepositoryServiceLocator repositoryLocator, IChildProvider[] pluginChildProviders, ICheckNotifier errorsCheckNotifier, DataExportChildProvider previousStateIfKnown) : base(repositoryLocator.CatalogueRepository, pluginChildProviders, errorsCheckNotifier, previousStateIfKnown)
{
BlackListedSources = previousStateIfKnown?.BlackListedSources ?? new List <ExternalCohortTable>();
_errorsCheckNotifier = errorsCheckNotifier;
dataExportRepository = repositoryLocator.DataExportRepository;
AllProjectAssociatedCics = GetAllObjects <ProjectCohortIdentificationConfigurationAssociation>(dataExportRepository);
_cicAssociations = new HashSet <int>(AllProjectAssociatedCics.Select(a => a.CohortIdentificationConfiguration_ID));
CohortSources = GetAllObjects <ExternalCohortTable>(dataExportRepository);
ExtractableDataSets = GetAllObjects <ExtractableDataSet>(dataExportRepository);
AddToDictionaries(new HashSet <object>(AllCohortIdentificationConfigurations.Where(cic => _cicAssociations.Contains(cic.ID))), new DescendancyList(AllProjectCohortIdentificationConfigurationsNode));
AddToDictionaries(new HashSet <object>(AllCohortIdentificationConfigurations.Where(cic => !_cicAssociations.Contains(cic.ID))), new DescendancyList(AllFreeCohortIdentificationConfigurationsNode));
_selectedDataSetsWithNoIsExtractionIdentifier = new HashSet <ISelectedDataSets>(dataExportRepository.GetSelectedDatasetsWithNoExtractionIdentifiers());
SelectedDataSets = GetAllObjects <SelectedDataSets>(dataExportRepository);
ReportProgress("Fetching data export objects");
var dsDictionary = ExtractableDataSets.ToDictionary(ds => ds.ID, d => d);
foreach (SelectedDataSets s in SelectedDataSets)
{
s.InjectKnown(dsDictionary[s.ExtractableDataSet_ID]);
}
ReportProgress("Injecting SelectedDataSets");
//This means that the ToString method in ExtractableDataSet doesn't need to go lookup catalogue info
var catalogueIdDict = AllCatalogues.ToDictionary(c => c.ID, c2 => c2);
foreach (ExtractableDataSet ds in ExtractableDataSets)
{
if (catalogueIdDict.TryGetValue(ds.Catalogue_ID, out Catalogue cata))
{
ds.InjectKnown(cata);
}
}
ReportProgress("Injecting ExtractableDataSet");
AllPackages = GetAllObjects <ExtractableDataSetPackage>(dataExportRepository);
Projects = GetAllObjects <Project>(dataExportRepository);
ExtractionConfigurations = GetAllObjects <ExtractionConfiguration>(dataExportRepository);
ReportProgress("Get Projects and Configurations");
ExtractionConfigurationsByProject = ExtractionConfigurations.GroupBy(k => k.Project_ID).ToDictionary(gdc => gdc.Key, gdc => gdc.ToList());
ReportProgress("Grouping Extractions by Project");
AllGlobalExtractionFilterParameters = GetAllObjects <GlobalExtractionFilterParameter>(dataExportRepository);
AllContainers = GetAllObjects <FilterContainer>(dataExportRepository).ToDictionary(o => o.ID, o => o);
AllDeployedExtractionFilters = GetAllObjects <DeployedExtractionFilter>(dataExportRepository);
_allParameters = GetAllObjects <DeployedExtractionFilterParameter>(dataExportRepository);
ReportProgress("Getting Filters");
//if we are using a database repository then we can make use of the caching class DataExportFilterManagerFromChildProvider to speed up
//filter contents
var dbRepo = dataExportRepository as DataExportRepository;
_dataExportFilterManager = dbRepo == null ? dataExportRepository.FilterManager : new DataExportFilterManagerFromChildProvider(dbRepo, this);
ReportProgress("Building FilterManager");
Cohorts = GetAllObjects <ExtractableCohort>(dataExportRepository);
_cohortsByOriginId = new Dictionary <int, HashSet <ExtractableCohort> >();
foreach (ExtractableCohort c in Cohorts)
{
if (!_cohortsByOriginId.ContainsKey(c.OriginID))
{
_cohortsByOriginId.Add(c.OriginID, new HashSet <ExtractableCohort>());
}
_cohortsByOriginId[c.OriginID].Add(c);
}
_configurationToDatasetMapping = new Dictionary <ExtractionConfiguration, List <SelectedDataSets> >();
ReportProgress("Fetching Cohorts");
GetCohortAvailability();
ReportProgress("GetCohortAvailability");
var configToSds = SelectedDataSets.GroupBy(k => k.ExtractionConfiguration_ID).ToDictionary(gdc => gdc.Key, gdc => gdc.ToList());
foreach (ExtractionConfiguration configuration in ExtractionConfigurations)
{
if (configToSds.TryGetValue(configuration.ID, out List <SelectedDataSets> result))
{
_configurationToDatasetMapping.Add(configuration, result);
}
}
ReportProgress("Mapping configurations to datasets");
RootCohortsNode = new AllCohortsNode();
AddChildren(RootCohortsNode, new DescendancyList(RootCohortsNode));
foreach (ExtractableDataSetPackage package in AllPackages)
{
AddChildren(package, new DescendancyList(package));
}
ReportProgress("Packages and Cohorts");
foreach (Project p in Projects)
{
AddChildren(p, new DescendancyList(p));
}
ReportProgress("Projects");
//work out all the Catalogues that are extractable (Catalogues are extractable if there is an ExtractableDataSet with the Catalogue_ID that matches them)
var cataToEds = new Dictionary <int, ExtractableDataSet>(ExtractableDataSets.ToDictionary(k => k.Catalogue_ID));
//inject extractability into Catalogues
foreach (Catalogue catalogue in AllCatalogues)
{
if (cataToEds.TryGetValue(catalogue.ID, out ExtractableDataSet result))
{
catalogue.InjectKnown(result.GetCatalogueExtractabilityStatus());
}
else
{
catalogue.InjectKnown(new CatalogueExtractabilityStatus(false, false));
}
}
ReportProgress("Catalogue extractability injection");
try
{
AddPipelineUseCases(new Dictionary <string, PipelineUseCase>
{
{ "File Import", UploadFileUseCase.DesignTime() },
{ "Extraction", ExtractionPipelineUseCase.DesignTime() },
{ "Release", ReleaseUseCase.DesignTime() },
{ "Cohort Creation", CohortCreationRequest.DesignTime() },
{ "Caching", CachingPipelineUseCase.DesignTime() },
{ "Aggregate Committing", CreateTableFromAggregateUseCase.DesignTime(repositoryLocator.CatalogueRepository) }
});
}
catch (Exception ex)
{
_errorsCheckNotifier.OnCheckPerformed(new CheckEventArgs("Failed to build DesignTime PipelineUseCases", CheckResult.Fail, ex));
}
ReportProgress("Pipeline adding");
}
public ExecuteCommandExecuteExtractionConfiguration(IActivateItems activator, SelectedDataSets selectedDataSet) : this(activator)
{
_extractionConfiguration = (ExtractionConfiguration)selectedDataSet.ExtractionConfiguration;
_selectedDataSet = selectedDataSet;
}
/// <summary>
/// Creates a minimum viable object of Type T. This includes the object and any dependencies e.g. a
/// <see cref="ColumnInfo"/> cannot exist without a <see cref="TableInfo"/>.
/// </summary>
/// <typeparam name="T">Type of object you want to create</typeparam>
/// <returns></returns>
/// <exception cref="NotSupportedException">If there is not yet an implementation for the given T. Feel free to write one.</exception>
protected T WhenIHaveA <T>() where T : DatabaseEntity
{
if (typeof(T) == typeof(Catalogue))
{
return((T)(object)Save(new Catalogue(Repository, "Mycata")));
}
if (typeof(T) == typeof(ExtendedProperty))
{
return((T)(object)new ExtendedProperty(Repository, Save(new Catalogue(Repository, "Mycata")), "TestProp", 0));
}
if (typeof(T) == typeof(CatalogueItem))
{
var cata = new Catalogue(Repository, "Mycata");
return((T)(object)new CatalogueItem(Repository, cata, "MyCataItem"));
}
if (typeof(T) == typeof(ExtractionInformation))
{
var col = WhenIHaveA <ColumnInfo>();
var cata = new Catalogue(Repository, "Mycata");
var ci = new CatalogueItem(Repository, cata, "MyCataItem");
var ei = new ExtractionInformation(Repository, ci, col, "MyCataItem");
return((T)(object)Save(ei));
}
if (typeof(T) == typeof(TableInfo))
{
var table = new TableInfo(Repository, "My_Table")
{
DatabaseType = DatabaseType.MicrosoftSQLServer
};
return((T)(object)table);
}
if (typeof(T) == typeof(ColumnInfo))
{
var ti = WhenIHaveA <TableInfo>();
var col = new ColumnInfo(Repository, "My_Col", "varchar(10)", ti);
return((T)(object)col);
}
if (typeof(T) == typeof(AggregateConfiguration))
{
ExtractionInformation dateEi;
ExtractionInformation otherEi;
return((T)(object)WhenIHaveA <AggregateConfiguration>(out dateEi, out otherEi));
}
if (typeof(T) == typeof(ExternalDatabaseServer))
{
return((T)(object)Save(new ExternalDatabaseServer(Repository, "My Server", null)));
}
if (typeof(T) == typeof(ANOTable))
{
ExternalDatabaseServer server;
return((T)(object)WhenIHaveA <ANOTable>(out server));
}
if (typeof(T) == typeof(LoadMetadata))
{
//creates the table, column, catalogue, catalogue item and extraction information
var ei = WhenIHaveA <ExtractionInformation>();
var cata = ei.CatalogueItem.Catalogue;
var ti = ei.ColumnInfo.TableInfo;
ti.Server = "localhost";
ti.Database = "mydb";
ti.SaveToDatabase();
var lmd = new LoadMetadata(Repository, "MyLoad");
cata.LoadMetadata_ID = lmd.ID;
cata.SaveToDatabase();
return((T)(object)lmd);
}
if (typeof(T) == typeof(AggregateTopX))
{
var agg = WhenIHaveA <AggregateConfiguration>();
return((T)(object)new AggregateTopX(Repository, agg, 10));
}
if (typeof(T) == typeof(ConnectionStringKeyword))
{
return((T)(object)new ConnectionStringKeyword(Repository, DatabaseType.MicrosoftSQLServer, "MultipleActiveResultSets", "true"));
}
if (typeof(T) == typeof(DashboardLayout))
{
return((T)(object)new DashboardLayout(Repository, "My Layout"));
}
if (typeof(T) == typeof(DashboardControl))
{
var layout = WhenIHaveA <DashboardLayout>();
return((T)(object)Save(new DashboardControl(Repository, layout, typeof(int), 0, 0, 100, 100, "")
{
ControlType = "GoodBadCataloguePieChart"
}));
}
if (typeof(T) == typeof(DashboardObjectUse))
{
var layout = WhenIHaveA <DashboardLayout>();
var control = Save(new DashboardControl(Repository, layout, typeof(int), 0, 0, 100, 100, "")
{
ControlType = "GoodBadCataloguePieChart"
});
var use = new DashboardObjectUse(Repository, control, WhenIHaveA <Catalogue>());
return((T)(object)use);
}
if (typeof(T) == typeof(ExtractionFilter))
{
var ei = WhenIHaveA <ExtractionInformation>();
return((T)(object)new ExtractionFilter(Repository, "My Filter", ei));
}
if (typeof(T) == typeof(ExtractionFilterParameter))
{
var filter = WhenIHaveA <ExtractionFilter>();
filter.WhereSQL = "@myParam = 'T'";
return((T)(object)new ExtractionFilterParameter(Repository, "DECLARE @myParam varchar(10)", filter));
}
if (typeof(T) == typeof(ExtractionFilterParameterSetValue))
{
var parameter = WhenIHaveA <ExtractionFilterParameter>();
var set = new ExtractionFilterParameterSet(Repository, parameter.ExtractionFilter, "Parameter Set");
return((T)(object)new ExtractionFilterParameterSetValue(Repository, set, parameter));
}
if (typeof(T) == typeof(ExtractionFilterParameterSet))
{
return((T)(object)WhenIHaveA <ExtractionFilterParameterSetValue>().ExtractionFilterParameterSet);
}
if (typeof(T) == typeof(Favourite))
{
return((T)(object)new Favourite(Repository, WhenIHaveA <Catalogue>()));
}
if (typeof(T) == typeof(ObjectExport))
{
ShareManager sm;
return((T)(object)WhenIHaveA <ObjectExport>(out sm));
}
if (typeof(T) == typeof(ObjectImport))
{
ShareManager sm;
ObjectExport export = WhenIHaveA <ObjectExport>(out sm);
return((T)(object)sm.GetImportAs(export.SharingUID, WhenIHaveA <Catalogue>()));
}
if (typeof(T) == typeof(WindowLayout))
{
return((T)(object)new WindowLayout(Repository, "My window arrangement", "<html><body>ignore this</body></html>"));
}
if (typeof(T) == typeof(RemoteRDMP))
{
return((T)(object)new RemoteRDMP(Repository));
}
if (typeof(T) == typeof(CohortIdentificationConfiguration))
{
return((T)(object)new CohortIdentificationConfiguration(Repository, "My cic"));
}
if (typeof(T) == typeof(JoinableCohortAggregateConfiguration))
{
var config = WhenIHaveCohortAggregateConfiguration("PatientIndexTable");
var cic = WhenIHaveA <CohortIdentificationConfiguration>();
cic.EnsureNamingConvention(config);
return((T)(object)new JoinableCohortAggregateConfiguration(Repository, cic, config));
}
if (typeof(T) == typeof(JoinableCohortAggregateConfigurationUse))
{
var joinable = WhenIHaveA <JoinableCohortAggregateConfiguration>();
var config = WhenIHaveCohortAggregateConfiguration("Aggregate");
return((T)(object)joinable.AddUser(config));
}
if (typeof(T) == typeof(Rdmp.Core.Curation.Data.Plugin))
{
return((T)(object)new Rdmp.Core.Curation.Data.Plugin(Repository, new FileInfo("bob.nupkg"), new Version(1, 1, 1), new Version(1, 1, 1)));
}
if (typeof(T) == typeof(LoadModuleAssembly))
{
var dll = Path.Combine(TestContext.CurrentContext.TestDirectory, "a.nupkg");
File.WriteAllBytes(dll, new byte[] { 0x11 });
return((T)(object)new LoadModuleAssembly(Repository, new FileInfo(dll), WhenIHaveA <Rdmp.Core.Curation.Data.Plugin>()));
}
if (typeof(T) == typeof(AggregateContinuousDateAxis))
{
ExtractionInformation dateEi;
ExtractionInformation otherEi;
var config = WhenIHaveA <AggregateConfiguration>(out dateEi, out otherEi);
//remove the other Ei
config.AggregateDimensions[0].DeleteInDatabase();
//add the date one
var dim = new AggregateDimension(Repository, dateEi, config);
return((T)(object)new AggregateContinuousDateAxis(Repository, dim));
}
if (typeof(T) == typeof(AggregateDimension))
{
return((T)(object)WhenIHaveA <AggregateConfiguration>().AggregateDimensions[0]);
}
if (typeof(T) == typeof(AggregateFilterContainer))
{
var config = WhenIHaveA <AggregateConfiguration>();
var container = new AggregateFilterContainer(Repository, FilterContainerOperation.AND);
config.RootFilterContainer_ID = container.ID;
config.SaveToDatabase();
return((T)(object)container);
}
if (typeof(T) == typeof(AggregateFilter))
{
var container = WhenIHaveA <AggregateFilterContainer>();
return((T)(object)new AggregateFilter(Repository, "My Filter", container));
}
if (typeof(T) == typeof(AggregateFilterParameter))
{
var filter = WhenIHaveA <AggregateFilter>();
filter.WhereSQL = "@MyP = 'mnnn apples'";
filter.SaveToDatabase();
return((T)filter.GetFilterFactory().CreateNewParameter(filter, "DECLARE @MyP as varchar(10)"));
}
if (typeof(T) == typeof(LoadProgress))
{
return((T)(object)new LoadProgress(Repository, WhenIHaveA <LoadMetadata>()));
}
if (typeof(T) == typeof(CacheProgress))
{
return((T)(object)new CacheProgress(Repository, WhenIHaveA <LoadProgress>()));
}
if (typeof(T) == typeof(CacheFetchFailure))
{
return((T)(object)new CacheFetchFailure(Repository, WhenIHaveA <CacheProgress>(), DateTime.Now.Subtract(new TimeSpan(1, 0, 0, 0)), DateTime.Now, new Exception("It didn't work")));
}
if (typeof(T) == typeof(CohortAggregateContainer))
{
var cic = WhenIHaveA <CohortIdentificationConfiguration>();
cic.CreateRootContainerIfNotExists();
return((T)(object)cic.RootCohortAggregateContainer);
}
if (typeof(T) == typeof(AnyTableSqlParameter))
{
var cic = WhenIHaveA <CohortIdentificationConfiguration>();
return((T)(object)new AnyTableSqlParameter(Repository, cic, "DECLARE @myGlobal as varchar(10)"));
}
if (typeof(T) == typeof(DataAccessCredentials))
{
return((T)(object)new DataAccessCredentials(Repository, "My credentials"));
}
if (typeof(T) == typeof(GovernancePeriod))
{
return((T)(object)new GovernancePeriod(Repository));
}
if (typeof(T) == typeof(GovernanceDocument))
{
var fi = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, "myfile.txt"));
return((T)(object)new GovernanceDocument(Repository, WhenIHaveA <GovernancePeriod>(), fi));
}
if (typeof(T) == typeof(PermissionWindow))
{
return((T)(object)new PermissionWindow(Repository));
}
if (typeof(T) == typeof(JoinInfo))
{
ColumnInfo col1;
ColumnInfo col2;
ColumnInfo col3;
WhenIHaveTwoTables(out col1, out col2, out col3);
return((T)(object)new JoinInfo(Repository, col1, col2, ExtractionJoinType.Left, null));
}
if (typeof(T) == typeof(Lookup))
{
ColumnInfo col1;
ColumnInfo col2;
ColumnInfo col3;
WhenIHaveTwoTables(out col1, out col2, out col3);
return((T)(object)new Lookup(Repository, col3, col1, col2, ExtractionJoinType.Left, null));
}
if (typeof(T) == typeof(LookupCompositeJoinInfo))
{
var lookup = WhenIHaveA <Lookup>();
var otherJoinFk = new ColumnInfo(Repository, "otherJoinKeyForeign", "int", lookup.ForeignKey.TableInfo);
var otherJoinPk = new ColumnInfo(Repository, "otherJoinKeyPrimary", "int", lookup.PrimaryKey.TableInfo);
return((T)(object)new LookupCompositeJoinInfo(Repository, lookup, otherJoinFk, otherJoinPk));
}
if (typeof(T) == typeof(Pipeline))
{
return((T)(object)new Pipeline(Repository, "My Pipeline"));
}
if (typeof(T) == typeof(PipelineComponent))
{
return((T)(object)new PipelineComponent(Repository, WhenIHaveA <Pipeline>(), typeof(ColumnForbidder), 0, "My Component"));
}
if (typeof(T) == typeof(PipelineComponentArgument))
{
var comp = WhenIHaveA <PipelineComponent>();
return((T)comp.CreateArgumentsForClassIfNotExists <ColumnForbidder>().First());
}
if (typeof(T) == typeof(PreLoadDiscardedColumn))
{
return((T)(object)new PreLoadDiscardedColumn(Repository, WhenIHaveA <TableInfo>(), "MyDiscardedColum"));
}
if (typeof(T) == typeof(ProcessTask))
{
return((T)(object)new ProcessTask(Repository, WhenIHaveA <LoadMetadata>(), LoadStage.AdjustRaw));
}
if (typeof(T) == typeof(ProcessTaskArgument))
{
return((T)(object)new ProcessTaskArgument(Repository, WhenIHaveA <ProcessTask>()));
}
if (typeof(T) == typeof(StandardRegex))
{
return((T)(object)new StandardRegex(Repository));
}
if (typeof(T) == typeof(SupportingSQLTable))
{
return((T)(object)new SupportingSQLTable(Repository, WhenIHaveA <Catalogue>(), "Some Handy Query"));
}
if (typeof(T) == typeof(TicketingSystemConfiguration))
{
return((T)(object)new TicketingSystemConfiguration(Repository, "My Ticketing System"));
}
if (typeof(T) == typeof(SupportingDocument))
{
return((T)(object)new SupportingDocument(Repository, WhenIHaveA <Catalogue>(), "HelpFile.docx"));
}
if (typeof(T) == typeof(Project))
{
return((T)(object)new Project(Repository, "My Project"));
}
if (typeof(T) == typeof(ExtractionConfiguration))
{
return((T)(object)new ExtractionConfiguration(Repository, WhenIHaveA <Project>()));
}
if (typeof(T) == typeof(ExtractableDataSet))
{
//To make an extractable dataset we need an extraction identifier (e.g. chi) that will be linked in the cohort
var ei = WhenIHaveA <ExtractionInformation>();
ei.IsExtractionIdentifier = true;
ei.SaveToDatabase();
//And we need another column too just for sanity sakes (in the same table)
var ci2 = new CatalogueItem(Repository, ei.CatalogueItem.Catalogue, "ci2");
var col2 = new ColumnInfo(Repository, "My_Col2", "varchar(10)", ei.ColumnInfo.TableInfo);
var ei2 = new ExtractionInformation(Repository, ci2, col2, col2.GetFullyQualifiedName());
return((T)(object)new ExtractableDataSet(Repository, ei.CatalogueItem.Catalogue));
}
if (typeof(T) == typeof(CumulativeExtractionResults))
{
return((T)(object)new CumulativeExtractionResults(Repository, WhenIHaveA <ExtractionConfiguration>(), WhenIHaveA <ExtractableDataSet>(), "SELECT * FROM Anywhere"));
}
if (typeof(T) == typeof(SelectedDataSets))
{
var eds = WhenIHaveA <ExtractableDataSet>();
var config = WhenIHaveA <ExtractionConfiguration>();
foreach (var ei in eds.Catalogue.GetAllExtractionInformation(ExtractionCategory.Any))
{
var ec = new ExtractableColumn(Repository, eds, config, ei, ei.Order, ei.SelectSQL);
}
return((T)(object)new SelectedDataSets(Repository, config, eds, null));
}
if (typeof(T) == typeof(ReleaseLog))
{
var file = Path.Combine(TestContext.CurrentContext.TestDirectory, "myDataset.csv");
File.WriteAllText(file, "omg rows");
var sds = WhenIHaveA <SelectedDataSets>();
new CumulativeExtractionResults(Repository, sds.ExtractionConfiguration, sds.ExtractableDataSet, "SELECT * FROM ANYWHERE");
var potential = new FlatFileReleasePotential(new RepositoryProvider(Repository), sds);
return((T)(object)new ReleaseLog(Repository,
potential,
new ReleaseEnvironmentPotential(sds.ExtractionConfiguration),
false,
new DirectoryInfo(TestContext.CurrentContext.TestDirectory),
new FileInfo(file)));
}
if (typeof(T) == typeof(ExtractableDataSetPackage))
{
return((T)(object)new ExtractableDataSetPackage(Repository, "My Cool Package"));
}
if (typeof(T) == typeof(SupplementalExtractionResults))
{
return((T)(object)new SupplementalExtractionResults(Repository, WhenIHaveA <CumulativeExtractionResults>(), "Select * from Lookup", WhenIHaveA <SupportingSQLTable>()));
}
if (typeof(T) == typeof(SelectedDataSetsForcedJoin))
{
return((T)(object)new SelectedDataSetsForcedJoin(Repository, WhenIHaveA <SelectedDataSets>(), WhenIHaveA <TableInfo>()));
}
if (typeof(T) == typeof(ProjectCohortIdentificationConfigurationAssociation))
{
return((T)(object)new ProjectCohortIdentificationConfigurationAssociation(Repository, WhenIHaveA <Project>(), WhenIHaveA <CohortIdentificationConfiguration>()));
}
if (typeof(T) == typeof(ExternalCohortTable))
{
return(Save((T)(object)new ExternalCohortTable(Repository, "My cohorts", DatabaseType.MicrosoftSQLServer)
{
Database = "MyCohortsDb",
DefinitionTableForeignKeyField = "c_id",
PrivateIdentifierField = "priv",
ReleaseIdentifierField = "rel",
TableName = "Cohorts",
DefinitionTableName = "InventoryTable",
Server = "localhost\\sqlexpress"
}));
}
if (typeof(T) == typeof(ExtractableCohort))
{
throw new NotSupportedException("You should inherit from TestsRequiringACohort instead, cohorts have to exist to be constructed");
}
if (typeof(T) == typeof(GlobalExtractionFilterParameter))
{
return((T)(object)new GlobalExtractionFilterParameter(Repository, WhenIHaveA <ExtractionConfiguration>(), "DECLARE @ExtractionGlobal as varchar(100)"));
}
if (typeof(T) == typeof(ExtractableColumn))
{
var ei = WhenIHaveA <ExtractionInformation>();
var eds = new ExtractableDataSet(Repository, ei.CatalogueItem.Catalogue);
var config = WhenIHaveA <ExtractionConfiguration>();
config.AddDatasetToConfiguration(eds);
return((T)(object)config.GetAllExtractableColumnsFor(eds).Single());
}
if (typeof(T) == typeof(FilterContainer))
{
var sds = WhenIHaveA <SelectedDataSets>();
var container = new FilterContainer(Repository, FilterContainerOperation.AND);
sds.RootFilterContainer_ID = container.ID;
sds.SaveToDatabase();
return((T)(object)container);
}
if (typeof(T) == typeof(DeployedExtractionFilter))
{
var container = WhenIHaveA <FilterContainer>();
return((T)(object)new DeployedExtractionFilter(Repository, "Fish = 'haddock'", container));
}
if (typeof(T) == typeof(DeployedExtractionFilterParameter))
{
var filter = WhenIHaveA <DeployedExtractionFilter>();
filter.WhereSQL = "@had = 'enough'";
return((T)(object)filter.GetFilterFactory().CreateNewParameter(filter, "DECLARE @had as varchar(100)"));
}
if (typeof(T) == typeof(ExtractionProgress))
{
var cata = new Catalogue(Repository, "MyCata");
var cataItem = new CatalogueItem(Repository, cata, "MyCol");
var table = new TableInfo(Repository, "MyTable");
var col = new ColumnInfo(Repository, "mycol", "datetime", table);
var ei = new ExtractionInformation(Repository, cataItem, col, "mycol");
cata.TimeCoverage_ExtractionInformation_ID = ei.ID;
cata.SaveToDatabase();
var eds = new ExtractableDataSet(Repository, cata);
var project = new Project(Repository, "My Proj");
var config = new ExtractionConfiguration(Repository, project);
var sds = new SelectedDataSets(Repository, config, eds, null);
return((T)(object)new ExtractionProgress(Repository, sds));
}
throw new TestCaseNotWrittenYetException(typeof(T));
}
public void TickAllFor(SelectedDataSets selectedDataSet)
{
tlvDatasets.UncheckAll();
tlvDatasets.CheckObject(_globalsFolder);
tlvDatasets.CheckObject(selectedDataSet);
}
public ExecuteCommandViewExtractionSql(IBasicActivateItems activator, SelectedDataSets sds)
: base(activator)
{
_extractionConfiguration = sds.ExtractionConfiguration;
_selectedDataSet = sds;
}
public SelectedDataSetsMenu(RDMPContextMenuStripArgs args, SelectedDataSets selectedDataSet) : base(args, selectedDataSet)
{
_selectedDataSet = selectedDataSet;
_extractionConfiguration = _selectedDataSet.ExtractionConfiguration;
var root = selectedDataSet.RootFilterContainer;
Add(new ExecuteCommandExecuteExtractionConfiguration(_activator, selectedDataSet));
Add(new ExecuteCommandRelease(_activator).SetTarget(selectedDataSet));
/////////////////// Extraction Graphs //////////////////////////////
var graphs = new ToolStripMenuItem("View Extraction Graphs", CatalogueIcons.Graph);
var cata = selectedDataSet.ExtractableDataSet.Catalogue;
// If the Catalogue has been deleted, don't build Catalogue specific menu items
if (cata == null)
{
return;
}
var availableGraphs = cata.AggregateConfigurations.Where(a => !a.IsCohortIdentificationAggregate).ToArray();
foreach (AggregateConfiguration ac in availableGraphs)
{
graphs.DropDownItems.Add(ac.ToString(), CatalogueIcons.Graph, (s, e) => GenerateExtractionGraphs(ac));
}
if (availableGraphs.Length > 1)
{
graphs.DropDownItems.Add("All", null, (s, e) => GenerateExtractionGraphs(availableGraphs));
}
//must have graphs available and must have a cohort
graphs.Enabled = graphs.DropDownItems.Count > 0 && _extractionConfiguration.Cohort_ID != null;
Items.Add(graphs);
////////////////////////////////////////////////////////////////////
Items.Add(new ToolStripSeparator());
var addRootFilter = new ToolStripMenuItem("Add Filter Container", _activator.CoreIconProvider.GetImage(RDMPConcept.FilterContainer, OverlayKind.Add), (s, e) => AddFilterContainer());
addRootFilter.Enabled = root == null;
Items.Add(addRootFilter);
Add(new ExecuteCommandImportFilterContainerTree(_activator, selectedDataSet));
Add(new ExecuteCommandCreateNewFilter(_activator,
new DeployedExtractionFilterFactory(_activator.RepositoryLocator.DataExportRepository),
() => {
selectedDataSet.CreateRootContainerIfNotExists();
return(selectedDataSet.RootFilterContainer);
}));
Add(new ExecuteCommandCreateNewFilterFromCatalogue(_activator,
selectedDataSet.ExtractableDataSet.Catalogue,
() =>
{
selectedDataSet.CreateRootContainerIfNotExists();
return(selectedDataSet.RootFilterContainer);
}));
Items.Add(new ToolStripSeparator());
Add(new ExecuteCommandViewSelectedDataSetsExtractionSql(_activator).SetTarget(selectedDataSet));
Add(new ExecuteCommandViewThenVsNowSql(_activator, selectedDataSet));
Add(new ExecuteCommandOpenExtractionDirectory(_activator, selectedDataSet));
}
/// <summary>
/// Use this constructor at runtime
/// </summary>
/// <param name="selectedDataSet"></param>
/// <param name="graph"></param>
public ExtractionAggregateGraphObjectCollection(SelectedDataSets selectedDataSet, AggregateConfiguration graph) : this()
{
DatabaseObjects.Add(selectedDataSet);
DatabaseObjects.Add(graph);
}
public ExecuteCommandViewThenVsNowSql(IActivateItems activator, SelectedDataSets selectedDataSet) : base(activator)
{
_selectedDataSet = selectedDataSet;
}
public ExecuteCommandResetExtractionProgress(IBasicActivateItems activator, SelectedDataSets sds) : base(activator)
{
AddClearTarget(sds);
EvaluateIsImpossible();
}
