private void RefreshPipelineList()
{
var before = ddPipelines.SelectedItem as Pipeline;
ddPipelines.Items.Clear();
var context = _useCase.GetContext();
//add pipelines
var allPipelines = _repository.GetAllObjects <Pipeline>();
ddPipelines.Items.AddRange(context == null || cbOnlyShowCompatiblePipelines.Checked == false
? allPipelines.ToArray() //no context/show incompatible enabled so add all pipelines
: allPipelines.Where(context.IsAllowable).ToArray()); //only compatible components
ddPipelines.Items.Add("<<None>>");
//reselect if it is still there
if (before != null)
{
var toReselect = ddPipelines.Items.OfType <Pipeline>().SingleOrDefault(p => p.ID == before.ID);
//if we can reselect the users previously selected one
if (toReselect != null)
{
ddPipelines.SelectedItem = toReselect;
return;
}
}
//if there is only one pipeline select it
ddPipelines.SelectedItem = ddPipelines.Items.OfType <Pipeline>().Count() == 1
? (object)ddPipelines.Items.OfType <Pipeline>().Single()
: "<<None>>";
}
public PrivateIdentifierPrototype[] GetPrivateIdentifierCandidates()
{
//get the extraction identifiers
var extractionInformations = _catalogueRepository.GetAllObjects <ExtractionInformation>().Where(ei => ei.IsExtractionIdentifier);
//name + datatype, ideally we want to find 30 fields called 'PatientIndex' in 30 datasets all as char(10) fields but more likely we will get a slew of different spellings and dodgy datatypes (varchar(max) etc)
var toReturn = new List <PrivateIdentifierPrototype>();
//for each extraction identifier get the name of the column and give the associated data type
foreach (var extractionInformation in extractionInformations)
{
//do not process ExtractionInformations when the ColumnInfo is COLUMNINFO_MISSING
if (extractionInformation.ColumnInfo == null || !extractionInformation.ColumnInfo.Exists())
{
continue;
}
var match = toReturn.SingleOrDefault(prototype => prototype.IsCompatible(extractionInformation));
if (match != null)
{
match.MatchingExtractionInformations.Add(extractionInformation);
}
else
{
toReturn.Add(new PrivateIdentifierPrototype(extractionInformation));
}
}
return(toReturn.ToArray());
}
/// <summary>
/// Returns an existing export definition for the object o or generates a new one. This will give you a SharingUID and
/// enable the object for sharing with other users who have RDMP.
/// </summary>
/// <param name="o"></param>
/// <returns></returns>
public ObjectExport GetNewOrExistingExportFor(IMapsDirectlyToDatabaseTable o)
{
var existingExport = _catalogueRepository.GetAllObjects <ObjectExport>().SingleOrDefault(e => e.IsReferenceTo(o));
if (existingExport != null)
{
return(existingExport);
}
var existingImport = _catalogueRepository.GetAllObjects <ObjectImport>().SingleOrDefault(e => e.IsReferenceTo(o));
if (existingImport != null)
{
return(new ObjectExport(_catalogueRepository, o, existingImport.SharingUIDAsGuid));
}
return(new ObjectExport(_catalogueRepository, o, Guid.NewGuid()));
}
private IEnumerable <PreLoadDiscardedColumn> GetAllPreloadDiscardedColumnsInScope(ICatalogueRepository repository, IArgumentHost parent)
{
if (parent is ProcessTask || parent is LoadMetadata)
{
return(GetTableInfosInScope(repository, parent).SelectMany(t => t.PreLoadDiscardedColumns));
}
return(repository.GetAllObjects <PreLoadDiscardedColumn>());
}
private IEnumerable <ColumnInfo> GetColumnInfosInScope(ICatalogueRepository repository, IArgumentHost parent)
{
if (parent is ProcessTask || parent is LoadMetadata)
{
return(GetTableInfosInScope(repository, parent).SelectMany(ti => ti.ColumnInfos));
}
return(repository.GetAllObjects <ColumnInfo>());
}
/// <summary>
/// Returns all <see cref="TableInfo"/> which have all the <paramref name="fields"/> listed
/// </summary>
/// <param name="fields"></param>
/// <returns></returns>
public virtual IEnumerable <TableInfo> GetAllTables(string[] fields)
{
//the tables we should consider
var tables = TableInfosToUpdate != null && TableInfosToUpdate.Any() ?
_repository.GetAllObjectsInIDList <TableInfo>(TableInfosToUpdate):
_repository.GetAllObjects <TableInfo>();
// get only those that have all the WHERE/SET columns in them
return(tables.Where(t => fields.All(f => t.ColumnInfos.Select(c => c.GetRuntimeName()).Contains(f))));
}
public JoinInfo[] GetAllJoinInfosBetweenColumnInfoSets(ColumnInfo[] set1, ColumnInfo[] set2)
{
//assemble the IN SQL arrays
if (set1.Length == 0)
{
throw new NullReferenceException("Cannot find joins because column set 1 was empty");
}
if (set2.Length == 0)
{
throw new NullReferenceException("Cannot find joins because column set 2 was empty");
}
var idSet1 = new HashSet <int>(set1.Select(o => o.ID));
var idSet2 = new HashSet <int>(set2.Select(o => o.ID));
return
(_repository.GetAllObjects <JoinInfo>()
.Where(j =>
(idSet1.Contains(j.ForeignKey_ID) && idSet2.Contains(j.PrimaryKey_ID))
||
(idSet1.Contains(j.PrimaryKey_ID) && idSet2.Contains(j.ForeignKey_ID)))
.ToArray());
}
private IEnumerable <TableInfo> GetTableInfosInScope(ICatalogueRepository repository, IArgumentHost parent)
{
if (parent is ProcessTask pt)
{
return(pt.GetTableInfos());
}
if (parent is LoadMetadata lmd)
{
return(lmd.GetDistinctTableInfoList(true));
}
return(repository.GetAllObjects <TableInfo>());
}
/// <summary>
/// Returns the latest version of each plugin which is compatible with the running RDMP software version (as determined
/// by the listed <see cref="Curation.Data.Plugin.RdmpVersion"/>)
/// </summary>
/// <returns></returns>
public Curation.Data.Plugin[] GetCompatiblePlugins()
{
var location = Assembly.GetExecutingAssembly().Location;
if (location == null)
{
throw new Exception("Assembly had no listed Location");
}
var fileVersion = FileVersionInfo.GetVersionInfo(location).FileVersion;
var runningSoftwareVersion = new Version(fileVersion);
//nupkg that are compatible with the running software
var plugins = _repository.GetAllObjects <Curation.Data.Plugin>().Where(a => a.RdmpVersion.IsCompatibleWith(runningSoftwareVersion, 2));
//latest versions
var latestVersionsOfPlugins = from p in plugins
group p by p.GetShortName() into grp
select grp.OrderByDescending(p => p.PluginVersion).FirstOrDefault();
return(latestVersionsOfPlugins.ToArray());
}
/// <summary>
/// Refresh the list of pipeline components
/// </summary>
private void RefreshPipelineList()
{
var before = ddPipelines.SelectedItem as Pipeline;
ddPipelines.Items.Clear();
var context = _useCase.GetContext();
//add pipelines sorted alphabetically
var allPipelines = _repository.GetAllObjects <Pipeline>().OrderBy(p => p.Name).ToArray();
ddPipelines.Items.Add("<<None>>");
ddPipelines.Items.AddRange(allPipelines.Where(_useCase.IsAllowable).ToArray());
ddPipelines.Items.Add(ShowAll);
if (showAll)
{
ddPipelines.Items.AddRange(allPipelines.Where(o => !_useCase.IsAllowable(o)).ToArray());
}
//reselect if it is still there
if (before != null)
{
var toReselect = ddPipelines.Items.OfType <Pipeline>().SingleOrDefault(p => p.ID == before.ID);
//if we can reselect the users previously selected one
if (toReselect != null)
{
ddPipelines.SelectedItem = toReselect;
return;
}
}
//if there is only one pipeline select it
ddPipelines.SelectedItem = ddPipelines.Items.OfType <Pipeline>().Count() == 1
? (object)ddPipelines.Items.OfType <Pipeline>().Single()
: "<<None>>";
}
public void GenerateReport()
{
var f = GetUniqueFilenameInWorkArea("GovernanceReport", ".csv");
using (var s = new StreamWriter(f.FullName))
{
using (CsvWriter writer = new CsvWriter(s, CultureInfo.CurrentCulture))
{
writer.Configuration.Delimiter = ",";
writer.WriteField("Extractable Datasets");
writer.NextRecord();
writer.WriteField("Folder");
writer.WriteField("Catalogue");
writer.WriteField("Current Governance");
writer.WriteField("Dataset Period");
writer.WriteField("Description");
writer.NextRecord();
Dictionary <GovernancePeriod, ICatalogue[]> govs = _repository.GetAllObjects <GovernancePeriod>().ToDictionary(period => period, period => period.GovernedCatalogues.ToArray());
foreach (Catalogue catalogue in _repository.GetAllObjects <Catalogue>()
.Where(c => c.CatalogueItems.Any(ci => ci.ExtractionInformation != null))
.OrderBy(c => c.Name))
{
if (catalogue.IsDeprecated || catalogue.IsColdStorageDataset || catalogue.IsInternalDataset)
{
continue;
}
//get active governances
var activeGovs = govs.Where(kvp => kvp.Value.Contains(catalogue) && !kvp.Key.IsExpired()).Select(g => g.Key).ToArray();
var expiredGovs = govs.Where(kvp => kvp.Value.Contains(catalogue) && kvp.Key.IsExpired()).Select(g => g.Key).ToArray();
string relevantGovernance = null;
if (activeGovs.Any())
{
relevantGovernance = string.Join(",", activeGovs.Select(gov => gov.Name));
}
else if (expiredGovs.Any())
{
relevantGovernance = "No Current Governance (Expired Governances: " + string.Join(",", expiredGovs.Select(gov => gov.Name)) + ")";
}
else
{
relevantGovernance = "No Governance Required";
}
//write the results out to Excel
writer.WriteField(catalogue.Folder);
writer.WriteField(catalogue.Name);
writer.WriteField(relevantGovernance);
writer.WriteField(_timespanCalculator.GetHumanReadableTimespanIfKnownOf(catalogue, true, out _));
writer.WriteField(ShortenDescription(catalogue.Description));
writer.NextRecord();
}
writer.NextRecord();
// next section header
writer.WriteField("Active Governance");
OutputGovernanceList(govs, writer, false);
writer.NextRecord();
// next section header
writer.WriteField("Expired Governance");
OutputGovernanceList(govs, writer, true);
}
}
ShowFile(f);
}
public SecondaryConstraintUI(ICatalogueRepository repository, SecondaryConstraint secondaryConstriant, string[] otherColumns)
{
const int rowHeight = 30;
//the amount of additional space required to accomodate description labels
int inflation = 0;
_repository = repository;
this.SecondaryConstriant = secondaryConstriant;
_otherColumns = otherColumns;
InitializeComponent();
if (repository == null)
{
return;
}
cbxConsequence.DataSource = Enum.GetValues(typeof(Consequence));
cbxConsequence.SelectedItem = secondaryConstriant.Consequence;
//put the name of the secondary constraint into the header
this.lblType.Text = SecondaryConstriant.GetType().Name;
lblConsequence.Left = lblType.Right + 5;
cbxConsequence.Left = lblConsequence.Right + 5;
//work out what properties can be set on this constraint and create the relevant controls using reflection
_requiredProperties = secondaryConstriant.GetType().GetProperties().Where(p =>
p.CanRead && p.CanWrite && p.GetSetMethod(true).IsPublic
&& p.Name != "Name" &&//skip this one, it is Writeable in order to support XMLSerialization...
p.Name != "Consequence" &&//skip this one because it is dealt with explicitly
!p.IsDefined(typeof(HideOnValidationUI), true)
).ToArray();
for (int i = 0; i < _requiredProperties.Length; i++)
{
var currentRowPanel = new Panel();
currentRowPanel.Bounds = new Rectangle(0, 0, tableLayoutPanel1.Width, rowHeight);
currentRowPanel.Anchor = AnchorStyles.Top | AnchorStyles.Left | AnchorStyles.Right;
currentRowPanel.Margin = Padding.Empty;
tableLayoutPanel1.CellBorderStyle = TableLayoutPanelCellBorderStyle.Inset;
tableLayoutPanel1.Controls.Add(currentRowPanel, 0, i + 1);
Label lblName = new Label();
lblName.Text = _requiredProperties[i].Name;
lblName.AutoSize = true;
object currentValue = _requiredProperties[i].GetValue(SecondaryConstriant, null);
//Hard Typed properties - Bool
if (_requiredProperties[i].PropertyType == typeof(bool))
{
CheckBox boolControl = new CheckBox();
boolControl.Text = _requiredProperties[i].Name;
boolControl.Tag = i;
boolControl.Checked = (bool)currentValue;
boolControl.CheckStateChanged += (s, e) => _requiredProperties[(int)boolControl.Tag].SetValue(SecondaryConstriant, boolControl.Checked, null);
currentRowPanel.Controls.Add(boolControl);
}
else
if (_requiredProperties[i].PropertyType == typeof(PredictionRule)) //Hard Typed property PredictionRule
{
//for prediction rules fields
ComboBox cbx = new ComboBox();
cbx.Items.AddRange(Validator.GetPredictionExtraTypes());
cbx.Items.Add("");
cbx.DropDownStyle = ComboBoxStyle.DropDownList;
cbx.DisplayMember = "Name";
cbx.Tag = i;
cbx.SelectedIndexChanged += (s, e) => _requiredProperties[(int)cbx.Tag].SetValue(SecondaryConstriant, cbx.SelectedItem is Type ? Activator.CreateInstance((Type)cbx.SelectedItem) : null);
cbx.Width = 200;
//The dropdown box is a list of Types but we are actually instantiating a value when user selects it (for XML Serialization). Consequently we must now get the Type for selection purposes
if (currentValue != null)
{
cbx.SelectedItem = currentValue.GetType();
}
currentRowPanel.Controls.Add(lblName);
cbx.Left = lblName.Right + 5;
currentRowPanel.Controls.Add(cbx);
}
else
{
//it's a value control (basically anything that can be represented by text (i.e. not a boolean))
Control valueControl;
//if it is expects a column then create a dropdown box
if (_requiredProperties[i].IsDefined(typeof(ExpectsColumnNameAsInput), true))
{
//for column fields
ComboBox cbx = new ComboBox();
cbx.Items.AddRange(_otherColumns);
cbx.Items.Add("");
cbx.DropDownStyle = ComboBoxStyle.DropDownList;
cbx.Tag = i;
cbx.SelectedIndexChanged += (s, e) => _requiredProperties[(int)cbx.Tag].SetValue(SecondaryConstriant, Convert.ChangeType(cbx.SelectedItem, _requiredProperties[(int)cbx.Tag].PropertyType), null);
cbx.Width = 350;
valueControl = cbx;
}
else
if (typeof(IMapsDirectlyToDatabaseTable).IsAssignableFrom(_requiredProperties[i].PropertyType))//it is a Catalogue type
{
ComboBox dd = new ComboBox();
dd.Tag = i;
dd.Width = 350;
dd.AutoCompleteSource = AutoCompleteSource.ListItems;
dd.AutoCompleteMode = AutoCompleteMode.Suggest;
var entities = _repository.GetAllObjects(_requiredProperties[i].PropertyType).ToArray();
if (!entities.Any())
{
if (_requiredProperties[i].PropertyType == typeof(StandardRegex))
{
MessageBox.Show("You currently do not have any standard regex concepts in your database. These can be created from the Table(Advanced) collection.",
"No StandardRegex configured in your Catalogue");
}
else
{
MessageBox.Show("You currently do not have any " + _requiredProperties[i].PropertyType + " in your database", "Catalogue Entity Collection Empty");
}
}
else
{
dd.Items.Add("<<Clear>>");
dd.Items.AddRange(entities);
dd.SelectedIndexChanged += (s, e) => _requiredProperties[(int)dd.Tag].SetValue(SecondaryConstriant, dd.SelectedItem as IMapsDirectlyToDatabaseTable, null);
}
//See if it has a value
IRevertable v = _requiredProperties[i].GetValue(SecondaryConstriant, null) as IRevertable;
//It has a value, this is a dropdown control right here though so if the revertable state out of date then it means someone else made a change to the database while we were picking columns
if (v != null)
{
v.RevertToDatabaseState();
}
valueControl = dd;
}
else //otherwise create a textbox
{
//for string fields
valueControl = new TextBox();
//if they edit this then write it to the SecondaryConstraint... we can't put i in directly because it goes out of scope so instead we stuff it into Tag and then
//get it back at delegate execution time when they change the text.
valueControl.Tag = i;
valueControl.TextChanged += setTextOrParseableProperty;
if (_requiredProperties[i].IsDefined(typeof(ExpectsLotsOfText), true))
{
valueControl.Width = 300;
}
}
if (currentValue != null)
{
valueControl.Text = _requiredProperties[i].GetValue(SecondaryConstriant, null).ToString().Replace("00:00:00", "");
}
currentRowPanel.Controls.Add(lblName);
valueControl.Left = lblName.Right + 5;
currentRowPanel.Controls.Add(valueControl);
}
var desc = _requiredProperties[i].GetCustomAttribute <DescriptionAttribute>();
if (desc != null)
{
var lbl = new Label();
lbl.AutoSize = true;
lbl.Text = desc.Description;
lbl.Font = new Font(lbl.Font, FontStyle.Italic);
//make some space for it
inflation += lbl.Height - 7;
lbl.Top = rowHeight - 7;
currentRowPanel.Controls.Add(lbl);
currentRowPanel.Height = rowHeight + lbl.Height;
}
}
//first row
tableLayoutPanel1.RowStyles[0].SizeType = SizeType.AutoSize;
Height = (_requiredProperties.Length * rowHeight) + 35 + inflation;
loadingComplete = true;
}
public static StandardRegex GetGlobalIgnorePatternIfAny(ICatalogueRepository repository)
{
return(repository.GetAllObjects <StandardRegex>().OrderBy(r => r.ID).FirstOrDefault(r => r.ConceptName == StandardRegex.DataLoadEngineGlobalIgnorePattern));
}
/// <summary>
/// Generates the dita files and logs progress / errors to the <paramref name="listener"/>
/// </summary>
/// <param name="listener"></param>
public void Extract(IDataLoadEventListener listener)
{
string xml = "";
xml += @"<?xml version=""1.0"" encoding=""UTF-8""?>
<!DOCTYPE map PUBLIC ""-//OASIS//DTD DITA Map//EN""
""map.dtd"">" + Environment.NewLine;
xml += "<map>" + Environment.NewLine;
xml += "<title>HIC Data Catalogue</title>" + Environment.NewLine;
xml += @"<topicmeta product=""hicdc"" rev=""1"">" + Environment.NewLine;
xml += "<author>Wilfred Bonney; Thomas Nind; Mikhail Ghattas</author>" + Environment.NewLine;
xml += "<publisher>Health Informatics Centre (HIC), University of Dundee</publisher>" + Environment.NewLine;
xml += "</topicmeta>" + Environment.NewLine;
xml += @"<topicref href=""introduction.dita""/>" + Environment.NewLine;
GenerateIntroductionFile("introduction.dita");
xml += @"<topicref href=""dataset.dita"">" + Environment.NewLine;
GenerateDataSetFile("dataset.dita");
xml += Environment.NewLine;
//get all the catalogues then sort them alphabetically
List <Catalogue> catas = new List <Catalogue>(_repository.GetAllObjects <Catalogue>().Where(c => !(c.IsDeprecated || c.IsInternalDataset || c.IsColdStorageDataset)));
catas.Sort();
Stopwatch sw = Stopwatch.StartNew();
int cataloguesCompleted = 0;
foreach (Catalogue c in catas)
{
listener.OnProgress(this, new ProgressEventArgs("Extracting", new ProgressMeasurement(cataloguesCompleted++, ProgressType.Records, catas.Count), sw.Elapsed));
//ensure that it has an acryonym
if (string.IsNullOrWhiteSpace(c.Acronym))
{
throw new Exception("Dita Extraction requires that each catalogue have a unique Acronym, the catalogue " + c.Name + " is missing an Acronym");
}
if (c.Name.Contains("\\") || c.Name.Contains("/"))
{
throw new Exception("Dita Extractor does not support catalogues with backslashes or forward slashs in their name");
}
//catalogue main file
xml += "<topicref href=\"" + GetFileNameForCatalogue(c) + "\">" + Environment.NewLine;
CreateCatalogueFile(c);
//catalogue items
List <CatalogueItem> cataItems = c.CatalogueItems.ToList();
cataItems.Sort();
foreach (CatalogueItem ci in cataItems)
{
xml += "<topicref href=\"" + GetFileNameForCatalogueItem(c, ci) + "\"/>" + Environment.NewLine;
CreateCatalogueItemFile(c, ci);
}
xml += "</topicref>" + Environment.NewLine;
//completed - mostly for end of loop tbh
listener.OnProgress(this, new ProgressEventArgs("Extracting", new ProgressMeasurement(cataloguesCompleted, ProgressType.Records, catas.Count), sw.Elapsed));
}
xml += Environment.NewLine;
xml += @"</topicref>" + Environment.NewLine;
xml += "</map>";
File.WriteAllText(Path.Combine(_folderToCreateIn.FullName, "hic_data_catalogue.ditamap"), xml);
}