/// <summary> /// The user has selected an extractable thing in the catalogue and opted to include it in the extraction /// So we have to convert it to an <see cref="ExtractableColumn"/> (which has configuration specific stuff - and lets /// data analyst override stuff for this extraction only) /// /// <para>Then add it to the right hand list</para> /// </summary> /// <param name="columns"></param> private void Include(params IColumn[] columns) { olvSelected.BeginUpdate(); try { //for each column we are adding foreach (IColumn c in columns) { //make sure it is up to date with database IRevertable r = c as IRevertable; //if the column is out of date if (r != null && r.HasLocalChanges().Evaluation == ChangeDescription.DatabaseCopyDifferent) { r.RevertToDatabaseState();//get a fresh copy } //add to the config ExtractableColumn addMe = _config.AddColumnToExtraction(_dataSet, c); //update on the UI olvSelected.AddObject(addMe); } } finally { olvSelected.EndUpdate(); } RefreshDisabledObjectStatus(); SortSelectedByOrder(); Publish(_config); }
private void SetupDataExport() { _extractableDataSet = new ExtractableDataSet(DataExportRepository, _catalogue); _project = new Project(DataExportRepository, _testDatabaseName); _project.ProjectNumber = 1; Directory.CreateDirectory(ProjectDirectory); _project.ExtractionDirectory = ProjectDirectory; _project.SaveToDatabase(); _configuration = new ExtractionConfiguration(DataExportRepository, _project); //select the dataset for extraction under this configuration _selectedDataSet = new SelectedDataSets(RepositoryLocator.DataExportRepository, _configuration, _extractableDataSet, null); //select all the columns for extraction foreach (var toSelect in _extractionInformations) { var col = new ExtractableColumn(DataExportRepository, _extractableDataSet, _configuration, toSelect, toSelect.Order, toSelect.SelectSQL); if (col.GetRuntimeName().Equals("PrivateID")) { col.IsExtractionIdentifier = true; } col.SaveToDatabase(); _extractableColumns.Add(col); } }
private void HandleReorder(ExtractableColumn sourceColumn, IOrderable targetOrderable, DropTargetLocation location) { if (targetOrderable == null) { targetOrderable = olvSelected.Objects.Cast <IOrderable>().OrderByDescending(o => o.Order).FirstOrDefault(); } if (targetOrderable == null) { return; } int destinationOrder = targetOrderable.Order; switch (location) { case DropTargetLocation.AboveItem: //bump down the other columns foreach (ConcreteColumn c in olvSelected.Objects.OfType <ConcreteColumn>().ToArray()) { if (c.Order >= destinationOrder && !Equals(c, sourceColumn)) { c.Order++; c.SaveToDatabase(); } } //should now be space at the destination order position sourceColumn.Order = destinationOrder; break; case DropTargetLocation.None: case DropTargetLocation.BelowItem: //bump up other columns foreach (ConcreteColumn c in olvSelected.Objects.OfType <ConcreteColumn>().ToArray()) { if (c.Order <= destinationOrder && !Equals(c, sourceColumn)) { c.Order--; c.SaveToDatabase(); } } sourceColumn.Order = destinationOrder; break; default: throw new ArgumentOutOfRangeException(); } sourceColumn.SaveToDatabase(); olvSelected.RefreshObjects(olvSelected.Objects.OfType <object>().ToArray()); SortSelectedByOrder(); }
private void HandleReorder(ModelDropEventArgs e) { if (e.SourceModels == null || e.SourceModels.Count != 1) { return; } ExtractableColumn sourceColumn = (ExtractableColumn)e.SourceModels[0]; HandleReorder(sourceColumn, (IOrderable)e.TargetModel, e.DropTargetLocation); }
/// <summary> /// The user has selected an extractable thing in the catalogue and opted to include it in the extraction /// So we have to convert it to an ExtractableColumn (which has configuration specific stuff - and lets /// data analyst override stuff for this extraction only) /// /// <para>Then add it to the right hand list</para> /// </summary> /// <param name="item"></param> private ExtractableColumn AddColumnToExtraction(IColumn item) { IRevertable r = item as IRevertable; //if the column is out of date if (r != null && r.HasLocalChanges().Evaluation == ChangeDescription.DatabaseCopyDifferent) { r.RevertToDatabaseState();//get a fresh copy } ExtractableColumn addMe = _config.AddColumnToExtraction(_dataSet, item); olvSelected.AddObject(addMe); RefreshDisabledObjectStatus(); SortSelectedByOrder(); return(addMe); }
public void CreateAndAssociateColumns() { var cata = new Catalogue(CatalogueRepository, "MyCat"); var cataItem = new CatalogueItem(CatalogueRepository, cata, "MyCataItem"); var TableInfo = new TableInfo(CatalogueRepository, "Cata"); var ColumnInfo = new ColumnInfo(CatalogueRepository, "Col", "varchar(10)", TableInfo); var ExtractionInfo = new ExtractionInformation(CatalogueRepository, cataItem, ColumnInfo, "fish"); var ds = new ExtractableDataSet(DataExportRepository, cata); var proj = new Project(DataExportRepository, "MyProj"); var config = new ExtractionConfiguration(DataExportRepository, proj); SelectedDataSets selectedDataSets; var extractableColumn = new ExtractableColumn(DataExportRepository, ds, config, ExtractionInfo, 1, "fish"); try { selectedDataSets = new SelectedDataSets(DataExportRepository, config, ds, null); var cols = config.GetAllExtractableColumnsFor(ds); Assert.AreEqual(1, cols.Count()); Assert.AreEqual(extractableColumn, cols.Single()); cols = config.GetAllExtractableColumnsFor(ds); Assert.AreEqual(1, cols.Count()); Assert.AreEqual(extractableColumn, cols.Single()); } finally { extractableColumn.DeleteInDatabase(); config.DeleteInDatabase(); proj.DeleteInDatabase(); ds.DeleteInDatabase(); TableInfo.DeleteInDatabase(); cata.DeleteInDatabase(); } }
public void TestNoIsExtractionIdentifierFinding() { //nothing in database means no dodgy datasets Assert.IsEmpty(((DataExportRepository)DataExportRepository).GetSelectedDatasetsWithNoExtractionIdentifiers()); var cata = new Catalogue(CatalogueRepository, "ommn"); var ds = new ExtractableDataSet(DataExportRepository, cata); var proj = new Project(DataExportRepository, "proj"); var config = new ExtractionConfiguration(DataExportRepository, proj); var sds = new SelectedDataSets(DataExportRepository, config, ds, null); //only one selected dataset var dodgy = ((DataExportRepository)DataExportRepository).GetSelectedDatasetsWithNoExtractionIdentifiers().ToArray(); Assert.AreEqual(1, dodgy.Count()); Assert.AreEqual(sds, dodgy[0]); //make an extarctable column on that dataset var col = new ColumnInfo(CatalogueRepository, "ff", "varchar(1)", new TableInfo(CatalogueRepository, "fff")); var ci = new CatalogueItem(CatalogueRepository, cata, "A"); var ei = new ExtractionInformation(CatalogueRepository, ci, col, col.Name); var ec = new ExtractableColumn(DataExportRepository, ds, config, ei, 0, col.Name); //still shouldn't be dodgy dodgy = ((DataExportRepository)DataExportRepository).GetSelectedDatasetsWithNoExtractionIdentifiers().ToArray(); Assert.AreEqual(1, dodgy.Count()); Assert.AreEqual(sds, dodgy[0]); //now make it non dodgy by being IsExtractionIdentifier ec.IsExtractionIdentifier = true; ec.SaveToDatabase(); //no longer dodgy because there is an extraction identifier dodgy = ((DataExportRepository)DataExportRepository).GetSelectedDatasetsWithNoExtractionIdentifiers().ToArray(); Assert.AreEqual(0, dodgy.Count()); }
/// <summary> /// Creates a minimum viable object of Type T. This includes the object and any dependencies e.g. a /// <see cref="ColumnInfo"/> cannot exist without a <see cref="TableInfo"/>. /// </summary> /// <typeparam name="T">Type of object you want to create</typeparam> /// <returns></returns> /// <exception cref="NotSupportedException">If there is not yet an implementation for the given T. Feel free to write one.</exception> protected T WhenIHaveA <T>() where T : DatabaseEntity { if (typeof(T) == typeof(Catalogue)) { return((T)(object)Save(new Catalogue(Repository, "Mycata"))); } if (typeof(T) == typeof(ExtendedProperty)) { return((T)(object)new ExtendedProperty(Repository, Save(new Catalogue(Repository, "Mycata")), "TestProp", 0)); } if (typeof(T) == typeof(CatalogueItem)) { var cata = new Catalogue(Repository, "Mycata"); return((T)(object)new CatalogueItem(Repository, cata, "MyCataItem")); } if (typeof(T) == typeof(ExtractionInformation)) { var col = WhenIHaveA <ColumnInfo>(); var cata = new Catalogue(Repository, "Mycata"); var ci = new CatalogueItem(Repository, cata, "MyCataItem"); var ei = new ExtractionInformation(Repository, ci, col, "MyCataItem"); return((T)(object)Save(ei)); } if (typeof(T) == typeof(TableInfo)) { var table = new TableInfo(Repository, "My_Table") { DatabaseType = DatabaseType.MicrosoftSQLServer }; return((T)(object)table); } if (typeof(T) == typeof(ColumnInfo)) { var ti = WhenIHaveA <TableInfo>(); var col = new ColumnInfo(Repository, "My_Col", "varchar(10)", ti); return((T)(object)col); } if (typeof(T) == typeof(AggregateConfiguration)) { ExtractionInformation dateEi; ExtractionInformation otherEi; return((T)(object)WhenIHaveA <AggregateConfiguration>(out dateEi, out otherEi)); } if (typeof(T) == typeof(ExternalDatabaseServer)) { return((T)(object)Save(new ExternalDatabaseServer(Repository, "My Server", null))); } if (typeof(T) == typeof(ANOTable)) { ExternalDatabaseServer server; return((T)(object)WhenIHaveA <ANOTable>(out server)); } if (typeof(T) == typeof(LoadMetadata)) { //creates the table, column, catalogue, catalogue item and extraction information var ei = WhenIHaveA <ExtractionInformation>(); var cata = ei.CatalogueItem.Catalogue; var ti = ei.ColumnInfo.TableInfo; ti.Server = "localhost"; ti.Database = "mydb"; ti.SaveToDatabase(); var lmd = new LoadMetadata(Repository, "MyLoad"); cata.LoadMetadata_ID = lmd.ID; cata.SaveToDatabase(); return((T)(object)lmd); } if (typeof(T) == typeof(AggregateTopX)) { var agg = WhenIHaveA <AggregateConfiguration>(); return((T)(object)new AggregateTopX(Repository, agg, 10)); } if (typeof(T) == typeof(ConnectionStringKeyword)) { return((T)(object)new ConnectionStringKeyword(Repository, DatabaseType.MicrosoftSQLServer, "MultipleActiveResultSets", "true")); } if (typeof(T) == typeof(DashboardLayout)) { return((T)(object)new DashboardLayout(Repository, "My Layout")); } if (typeof(T) == typeof(DashboardControl)) { var layout = WhenIHaveA <DashboardLayout>(); return((T)(object)Save(new DashboardControl(Repository, layout, typeof(int), 0, 0, 100, 100, "") { ControlType = "GoodBadCataloguePieChart" })); } if (typeof(T) == typeof(DashboardObjectUse)) { var layout = WhenIHaveA <DashboardLayout>(); var control = Save(new DashboardControl(Repository, layout, typeof(int), 0, 0, 100, 100, "") { ControlType = "GoodBadCataloguePieChart" }); var use = new DashboardObjectUse(Repository, control, WhenIHaveA <Catalogue>()); return((T)(object)use); } if (typeof(T) == typeof(ExtractionFilter)) { var ei = WhenIHaveA <ExtractionInformation>(); return((T)(object)new ExtractionFilter(Repository, "My Filter", ei)); } if (typeof(T) == typeof(ExtractionFilterParameter)) { var filter = WhenIHaveA <ExtractionFilter>(); filter.WhereSQL = "@myParam = 'T'"; return((T)(object)new ExtractionFilterParameter(Repository, "DECLARE @myParam varchar(10)", filter)); } if (typeof(T) == typeof(ExtractionFilterParameterSetValue)) { var parameter = WhenIHaveA <ExtractionFilterParameter>(); var set = new ExtractionFilterParameterSet(Repository, parameter.ExtractionFilter, "Parameter Set"); return((T)(object)new ExtractionFilterParameterSetValue(Repository, set, parameter)); } if (typeof(T) == typeof(ExtractionFilterParameterSet)) { return((T)(object)WhenIHaveA <ExtractionFilterParameterSetValue>().ExtractionFilterParameterSet); } if (typeof(T) == typeof(Favourite)) { return((T)(object)new Favourite(Repository, WhenIHaveA <Catalogue>())); } if (typeof(T) == typeof(ObjectExport)) { ShareManager sm; return((T)(object)WhenIHaveA <ObjectExport>(out sm)); } if (typeof(T) == typeof(ObjectImport)) { ShareManager sm; ObjectExport export = WhenIHaveA <ObjectExport>(out sm); return((T)(object)sm.GetImportAs(export.SharingUID, WhenIHaveA <Catalogue>())); } if (typeof(T) == typeof(WindowLayout)) { return((T)(object)new WindowLayout(Repository, "My window arrangement", "<html><body>ignore this</body></html>")); } if (typeof(T) == typeof(RemoteRDMP)) { return((T)(object)new RemoteRDMP(Repository)); } if (typeof(T) == typeof(CohortIdentificationConfiguration)) { return((T)(object)new CohortIdentificationConfiguration(Repository, "My cic")); } if (typeof(T) == typeof(JoinableCohortAggregateConfiguration)) { var config = WhenIHaveCohortAggregateConfiguration("PatientIndexTable"); var cic = WhenIHaveA <CohortIdentificationConfiguration>(); cic.EnsureNamingConvention(config); return((T)(object)new JoinableCohortAggregateConfiguration(Repository, cic, config)); } if (typeof(T) == typeof(JoinableCohortAggregateConfigurationUse)) { var joinable = WhenIHaveA <JoinableCohortAggregateConfiguration>(); var config = WhenIHaveCohortAggregateConfiguration("Aggregate"); return((T)(object)joinable.AddUser(config)); } if (typeof(T) == typeof(Rdmp.Core.Curation.Data.Plugin)) { return((T)(object)new Rdmp.Core.Curation.Data.Plugin(Repository, new FileInfo("bob.nupkg"), new Version(1, 1, 1), new Version(1, 1, 1))); } if (typeof(T) == typeof(LoadModuleAssembly)) { var dll = Path.Combine(TestContext.CurrentContext.TestDirectory, "a.nupkg"); File.WriteAllBytes(dll, new byte[] { 0x11 }); return((T)(object)new LoadModuleAssembly(Repository, new FileInfo(dll), WhenIHaveA <Rdmp.Core.Curation.Data.Plugin>())); } if (typeof(T) == typeof(AggregateContinuousDateAxis)) { ExtractionInformation dateEi; ExtractionInformation otherEi; var config = WhenIHaveA <AggregateConfiguration>(out dateEi, out otherEi); //remove the other Ei config.AggregateDimensions[0].DeleteInDatabase(); //add the date one var dim = new AggregateDimension(Repository, dateEi, config); return((T)(object)new AggregateContinuousDateAxis(Repository, dim)); } if (typeof(T) == typeof(AggregateDimension)) { return((T)(object)WhenIHaveA <AggregateConfiguration>().AggregateDimensions[0]); } if (typeof(T) == typeof(AggregateFilterContainer)) { var config = WhenIHaveA <AggregateConfiguration>(); var container = new AggregateFilterContainer(Repository, FilterContainerOperation.AND); config.RootFilterContainer_ID = container.ID; config.SaveToDatabase(); return((T)(object)container); } if (typeof(T) == typeof(AggregateFilter)) { var container = WhenIHaveA <AggregateFilterContainer>(); return((T)(object)new AggregateFilter(Repository, "My Filter", container)); } if (typeof(T) == typeof(AggregateFilterParameter)) { var filter = WhenIHaveA <AggregateFilter>(); filter.WhereSQL = "@MyP = 'mnnn apples'"; filter.SaveToDatabase(); return((T)filter.GetFilterFactory().CreateNewParameter(filter, "DECLARE @MyP as varchar(10)")); } if (typeof(T) == typeof(LoadProgress)) { return((T)(object)new LoadProgress(Repository, WhenIHaveA <LoadMetadata>())); } if (typeof(T) == typeof(CacheProgress)) { return((T)(object)new CacheProgress(Repository, WhenIHaveA <LoadProgress>())); } if (typeof(T) == typeof(CacheFetchFailure)) { return((T)(object)new CacheFetchFailure(Repository, WhenIHaveA <CacheProgress>(), DateTime.Now.Subtract(new TimeSpan(1, 0, 0, 0)), DateTime.Now, new Exception("It didn't work"))); } if (typeof(T) == typeof(CohortAggregateContainer)) { var cic = WhenIHaveA <CohortIdentificationConfiguration>(); cic.CreateRootContainerIfNotExists(); return((T)(object)cic.RootCohortAggregateContainer); } if (typeof(T) == typeof(AnyTableSqlParameter)) { var cic = WhenIHaveA <CohortIdentificationConfiguration>(); return((T)(object)new AnyTableSqlParameter(Repository, cic, "DECLARE @myGlobal as varchar(10)")); } if (typeof(T) == typeof(DataAccessCredentials)) { return((T)(object)new DataAccessCredentials(Repository, "My credentials")); } if (typeof(T) == typeof(GovernancePeriod)) { return((T)(object)new GovernancePeriod(Repository)); } if (typeof(T) == typeof(GovernanceDocument)) { var fi = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, "myfile.txt")); return((T)(object)new GovernanceDocument(Repository, WhenIHaveA <GovernancePeriod>(), fi)); } if (typeof(T) == typeof(PermissionWindow)) { return((T)(object)new PermissionWindow(Repository)); } if (typeof(T) == typeof(JoinInfo)) { ColumnInfo col1; ColumnInfo col2; ColumnInfo col3; WhenIHaveTwoTables(out col1, out col2, out col3); return((T)(object)new JoinInfo(Repository, col1, col2, ExtractionJoinType.Left, null)); } if (typeof(T) == typeof(Lookup)) { ColumnInfo col1; ColumnInfo col2; ColumnInfo col3; WhenIHaveTwoTables(out col1, out col2, out col3); return((T)(object)new Lookup(Repository, col3, col1, col2, ExtractionJoinType.Left, null)); } if (typeof(T) == typeof(LookupCompositeJoinInfo)) { var lookup = WhenIHaveA <Lookup>(); var otherJoinFk = new ColumnInfo(Repository, "otherJoinKeyForeign", "int", lookup.ForeignKey.TableInfo); var otherJoinPk = new ColumnInfo(Repository, "otherJoinKeyPrimary", "int", lookup.PrimaryKey.TableInfo); return((T)(object)new LookupCompositeJoinInfo(Repository, lookup, otherJoinFk, otherJoinPk)); } if (typeof(T) == typeof(Pipeline)) { return((T)(object)new Pipeline(Repository, "My Pipeline")); } if (typeof(T) == typeof(PipelineComponent)) { return((T)(object)new PipelineComponent(Repository, WhenIHaveA <Pipeline>(), typeof(ColumnForbidder), 0, "My Component")); } if (typeof(T) == typeof(PipelineComponentArgument)) { var comp = WhenIHaveA <PipelineComponent>(); return((T)comp.CreateArgumentsForClassIfNotExists <ColumnForbidder>().First()); } if (typeof(T) == typeof(PreLoadDiscardedColumn)) { return((T)(object)new PreLoadDiscardedColumn(Repository, WhenIHaveA <TableInfo>(), "MyDiscardedColum")); } if (typeof(T) == typeof(ProcessTask)) { return((T)(object)new ProcessTask(Repository, WhenIHaveA <LoadMetadata>(), LoadStage.AdjustRaw)); } if (typeof(T) == typeof(ProcessTaskArgument)) { return((T)(object)new ProcessTaskArgument(Repository, WhenIHaveA <ProcessTask>())); } if (typeof(T) == typeof(StandardRegex)) { return((T)(object)new StandardRegex(Repository)); } if (typeof(T) == typeof(SupportingSQLTable)) { return((T)(object)new SupportingSQLTable(Repository, WhenIHaveA <Catalogue>(), "Some Handy Query")); } if (typeof(T) == typeof(TicketingSystemConfiguration)) { return((T)(object)new TicketingSystemConfiguration(Repository, "My Ticketing System")); } if (typeof(T) == typeof(SupportingDocument)) { return((T)(object)new SupportingDocument(Repository, WhenIHaveA <Catalogue>(), "HelpFile.docx")); } if (typeof(T) == typeof(Project)) { return((T)(object)new Project(Repository, "My Project")); } if (typeof(T) == typeof(ExtractionConfiguration)) { return((T)(object)new ExtractionConfiguration(Repository, WhenIHaveA <Project>())); } if (typeof(T) == typeof(ExtractableDataSet)) { //To make an extractable dataset we need an extraction identifier (e.g. chi) that will be linked in the cohort var ei = WhenIHaveA <ExtractionInformation>(); ei.IsExtractionIdentifier = true; ei.SaveToDatabase(); //And we need another column too just for sanity sakes (in the same table) var ci2 = new CatalogueItem(Repository, ei.CatalogueItem.Catalogue, "ci2"); var col2 = new ColumnInfo(Repository, "My_Col2", "varchar(10)", ei.ColumnInfo.TableInfo); var ei2 = new ExtractionInformation(Repository, ci2, col2, col2.GetFullyQualifiedName()); return((T)(object)new ExtractableDataSet(Repository, ei.CatalogueItem.Catalogue)); } if (typeof(T) == typeof(CumulativeExtractionResults)) { return((T)(object)new CumulativeExtractionResults(Repository, WhenIHaveA <ExtractionConfiguration>(), WhenIHaveA <ExtractableDataSet>(), "SELECT * FROM Anywhere")); } if (typeof(T) == typeof(SelectedDataSets)) { var eds = WhenIHaveA <ExtractableDataSet>(); var config = WhenIHaveA <ExtractionConfiguration>(); foreach (var ei in eds.Catalogue.GetAllExtractionInformation(ExtractionCategory.Any)) { var ec = new ExtractableColumn(Repository, eds, config, ei, ei.Order, ei.SelectSQL); } return((T)(object)new SelectedDataSets(Repository, config, eds, null)); } if (typeof(T) == typeof(ReleaseLog)) { var file = Path.Combine(TestContext.CurrentContext.TestDirectory, "myDataset.csv"); File.WriteAllText(file, "omg rows"); var sds = WhenIHaveA <SelectedDataSets>(); new CumulativeExtractionResults(Repository, sds.ExtractionConfiguration, sds.ExtractableDataSet, "SELECT * FROM ANYWHERE"); var potential = new FlatFileReleasePotential(new RepositoryProvider(Repository), sds); return((T)(object)new ReleaseLog(Repository, potential, new ReleaseEnvironmentPotential(sds.ExtractionConfiguration), false, new DirectoryInfo(TestContext.CurrentContext.TestDirectory), new FileInfo(file))); } if (typeof(T) == typeof(ExtractableDataSetPackage)) { return((T)(object)new ExtractableDataSetPackage(Repository, "My Cool Package")); } if (typeof(T) == typeof(SupplementalExtractionResults)) { return((T)(object)new SupplementalExtractionResults(Repository, WhenIHaveA <CumulativeExtractionResults>(), "Select * from Lookup", WhenIHaveA <SupportingSQLTable>())); } if (typeof(T) == typeof(SelectedDataSetsForcedJoin)) { return((T)(object)new SelectedDataSetsForcedJoin(Repository, WhenIHaveA <SelectedDataSets>(), WhenIHaveA <TableInfo>())); } if (typeof(T) == typeof(ProjectCohortIdentificationConfigurationAssociation)) { return((T)(object)new ProjectCohortIdentificationConfigurationAssociation(Repository, WhenIHaveA <Project>(), WhenIHaveA <CohortIdentificationConfiguration>())); } if (typeof(T) == typeof(ExternalCohortTable)) { return(Save((T)(object)new ExternalCohortTable(Repository, "My cohorts", DatabaseType.MicrosoftSQLServer) { Database = "MyCohortsDb", DefinitionTableForeignKeyField = "c_id", PrivateIdentifierField = "priv", ReleaseIdentifierField = "rel", TableName = "Cohorts", DefinitionTableName = "InventoryTable", Server = "localhost\\sqlexpress" })); } if (typeof(T) == typeof(ExtractableCohort)) { throw new NotSupportedException("You should inherit from TestsRequiringACohort instead, cohorts have to exist to be constructed"); } if (typeof(T) == typeof(GlobalExtractionFilterParameter)) { return((T)(object)new GlobalExtractionFilterParameter(Repository, WhenIHaveA <ExtractionConfiguration>(), "DECLARE @ExtractionGlobal as varchar(100)")); } if (typeof(T) == typeof(ExtractableColumn)) { var ei = WhenIHaveA <ExtractionInformation>(); var eds = new ExtractableDataSet(Repository, ei.CatalogueItem.Catalogue); var config = WhenIHaveA <ExtractionConfiguration>(); config.AddDatasetToConfiguration(eds); return((T)(object)config.GetAllExtractableColumnsFor(eds).Single()); } if (typeof(T) == typeof(FilterContainer)) { var sds = WhenIHaveA <SelectedDataSets>(); var container = new FilterContainer(Repository, FilterContainerOperation.AND); sds.RootFilterContainer_ID = container.ID; sds.SaveToDatabase(); return((T)(object)container); } if (typeof(T) == typeof(DeployedExtractionFilter)) { var container = WhenIHaveA <FilterContainer>(); return((T)(object)new DeployedExtractionFilter(Repository, "Fish = 'haddock'", container)); } if (typeof(T) == typeof(DeployedExtractionFilterParameter)) { var filter = WhenIHaveA <DeployedExtractionFilter>(); filter.WhereSQL = "@had = 'enough'"; return((T)(object)filter.GetFilterFactory().CreateNewParameter(filter, "DECLARE @had as varchar(100)")); } if (typeof(T) == typeof(ExtractionProgress)) { var cata = new Catalogue(Repository, "MyCata"); var cataItem = new CatalogueItem(Repository, cata, "MyCol"); var table = new TableInfo(Repository, "MyTable"); var col = new ColumnInfo(Repository, "mycol", "datetime", table); var ei = new ExtractionInformation(Repository, cataItem, col, "mycol"); cata.TimeCoverage_ExtractionInformation_ID = ei.ID; cata.SaveToDatabase(); var eds = new ExtractableDataSet(Repository, cata); var project = new Project(Repository, "My Proj"); var config = new ExtractionConfiguration(Repository, project); var sds = new SelectedDataSets(Repository, config, eds, null); return((T)(object)new ExtractionProgress(Repository, sds)); } throw new TestCaseNotWrittenYetException(typeof(T)); }
public void SQLServerDestination() { DiscoveredDatabase dbToExtractTo = null; var ci = new CatalogueItem(CatalogueRepository, _catalogue, "YearOfBirth"); var columnToTransform = _columnInfos.Single(c => c.GetRuntimeName().Equals("DateOfBirth", StringComparison.CurrentCultureIgnoreCase)); string transform = "YEAR(" + columnToTransform.Name + ")"; var ei = new ExtractionInformation(CatalogueRepository, ci, columnToTransform, transform); ei.Alias = "YearOfBirth"; ei.ExtractionCategory = ExtractionCategory.Core; ei.SaveToDatabase(); //make it part of the ExtractionConfiguration var newColumn = new ExtractableColumn(DataExportRepository, _selectedDataSet.ExtractableDataSet, (ExtractionConfiguration)_selectedDataSet.ExtractionConfiguration, ei, 0, ei.SelectSQL); newColumn.Alias = ei.Alias; newColumn.SaveToDatabase(); _extractableColumns.Add(newColumn); //recreate the extraction command so it gets updated with the new column too. _request = new ExtractDatasetCommand(_configuration, _extractableCohort, new ExtractableDatasetBundle(_extractableDataSet), _extractableColumns, new HICProjectSalt(_project), new ExtractionDirectory(@"C:\temp\", _configuration)); try { _configuration.Name = "ExecuteFullExtractionToDatabaseMSSqlDestinationTest"; _configuration.SaveToDatabase(); ExtractionPipelineUseCase execute; IExecuteDatasetExtractionDestination result; var dbname = TestDatabaseNames.GetConsistentName(_project.Name + "_" + _project.ProjectNumber); dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); if (dbToExtractTo.Exists()) { dbToExtractTo.Drop(); } base.Execute(out execute, out result); var destinationTable = dbToExtractTo.ExpectTable(_expectedTableName); Assert.IsTrue(destinationTable.Exists()); var dt = destinationTable.GetDataTable(); Assert.AreEqual(1, dt.Rows.Count); Assert.AreEqual(_cohortKeysGenerated[_cohortKeysGenerated.Keys.First()].Trim(), dt.Rows[0]["ReleaseID"]); Assert.AreEqual(new DateTime(2001, 1, 1), dt.Rows[0]["DateOfBirth"]); Assert.AreEqual(2001, dt.Rows[0]["YearOfBirth"]); Assert.AreEqual(columnToTransform.Data_type, destinationTable.DiscoverColumn("DateOfBirth").DataType.SQLType); Assert.AreEqual("int", destinationTable.DiscoverColumn("YearOfBirth").DataType.SQLType); } finally { if (_extractionServer != null) { _extractionServer.DeleteInDatabase(); } if (dbToExtractTo != null) { dbToExtractTo.Drop(); } } }
public void SQLServerDestination(bool lookupsEtc) { DiscoveredDatabase dbToExtractTo = null; var ci = new CatalogueItem(CatalogueRepository, _catalogue, "YearOfBirth"); _columnToTransform = _columnInfos.Single(c => c.GetRuntimeName().Equals("DateOfBirth", StringComparison.CurrentCultureIgnoreCase)); string transform = "YEAR(" + _columnToTransform.Name + ")"; if (_catalogue.GetAllExtractionInformation(ExtractionCategory.Any).All(ei => ei.GetRuntimeName() != "YearOfBirth")) { var ei = new ExtractionInformation(CatalogueRepository, ci, _columnToTransform, transform); ei.Alias = "YearOfBirth"; ei.ExtractionCategory = ExtractionCategory.Core; ei.SaveToDatabase(); //make it part of the ExtractionConfiguration var newColumn = new ExtractableColumn(DataExportRepository, _selectedDataSet.ExtractableDataSet, (ExtractionConfiguration)_selectedDataSet.ExtractionConfiguration, ei, 0, ei.SelectSQL); newColumn.Alias = ei.Alias; newColumn.SaveToDatabase(); _extractableColumns.Add(newColumn); } if (lookupsEtc) { CreateLookupsEtc(); } try { _configuration.Name = "ExecuteFullExtractionToDatabaseMSSqlDestinationTest"; _configuration.SaveToDatabase(); var dbname = TestDatabaseNames.GetConsistentName(_project.Name + "_" + _project.ProjectNumber); dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); if (dbToExtractTo.Exists()) { dbToExtractTo.Drop(); } base.ExecuteRunner(); var destinationTable = dbToExtractTo.ExpectTable(_expectedTableName); Assert.IsTrue(destinationTable.Exists()); var dt = destinationTable.GetDataTable(); Assert.AreEqual(1, dt.Rows.Count); Assert.AreEqual(_cohortKeysGenerated[_cohortKeysGenerated.Keys.First()].Trim(), dt.Rows[0]["ReleaseID"]); Assert.AreEqual(new DateTime(2001, 1, 1), dt.Rows[0]["DateOfBirth"]); Assert.AreEqual(2001, dt.Rows[0]["YearOfBirth"]); Assert.AreEqual(_columnToTransform.Data_type, destinationTable.DiscoverColumn("DateOfBirth").DataType.SQLType); Assert.AreEqual("int", destinationTable.DiscoverColumn("YearOfBirth").DataType.SQLType); if (lookupsEtc) { AssertLookupsEtcExist(dbToExtractTo); } } finally { if (dbToExtractTo != null && dbToExtractTo.Exists()) { dbToExtractTo.Drop(); } _pipeline?.DeleteInDatabase(); } }
public void Extract_ProjectSpecificCatalogue_AppendedColumn() { //make the catalogue a custom catalogue for this project CustomExtractableDataSet.Project_ID = _project.ID; CustomExtractableDataSet.SaveToDatabase(); var pipe = SetupPipeline(); pipe.Name = "Extract_ProjectSpecificCatalogue_AppendedColumn Pipe"; pipe.SaveToDatabase(); var extraColumn = CustomCatalogue.GetAllExtractionInformation(ExtractionCategory.ProjectSpecific).Single(e => e.GetRuntimeName().Equals("SuperSecretThing")); var asExtractable = new ExtractableColumn(DataExportRepository, _extractableDataSet, _configuration, extraColumn, 10, extraColumn.SelectSQL); //get rid of any lingering joins foreach (JoinInfo j in CatalogueRepository.GetAllObjects <JoinInfo>()) { j.DeleteInDatabase(); } //add the ability to join the two tables in the query var idCol = _extractableDataSet.Catalogue.GetAllExtractionInformation(ExtractionCategory.Core).Single(c => c.IsExtractionIdentifier).ColumnInfo; var otherIdCol = CustomCatalogue.GetAllExtractionInformation(ExtractionCategory.ProjectSpecific).Single(e => e.GetRuntimeName().Equals("PrivateID")).ColumnInfo; new JoinInfo(CatalogueRepository, idCol, otherIdCol, ExtractionJoinType.Left, null); //generate a new request (this will include the newly created column) _request = new ExtractDatasetCommand(_configuration, new ExtractableDatasetBundle(_extractableDataSet)); var tbl = Database.ExpectTable("TestTable"); tbl.Truncate(); using (var blk = tbl.BeginBulkInsert()) { var dt = new DataTable(); dt.Columns.Add("PrivateID"); dt.Columns.Add("Name"); dt.Columns.Add("DateOfBirth"); dt.Rows.Add(new object[] { "Priv_12345", "Bob", "2001-01-01" }); dt.Rows.Add(new object[] { "Priv_wtf11", "Frank", "2001-10-29" }); blk.Upload(dt); } ExtractionPipelineUseCase useCase; IExecuteDatasetExtractionDestination results; Execute(out useCase, out results); var mainDataTableCsv = results.DirectoryPopulated.GetFiles().Single(f => f.Name.Equals("TestTable.csv")); Assert.IsNotNull(mainDataTableCsv); Assert.AreEqual("TestTable.csv", mainDataTableCsv.Name); var lines = File.ReadAllLines(mainDataTableCsv.FullName); Assert.AreEqual("ReleaseID,Name,DateOfBirth,SuperSecretThing", lines[0]); var bobLine = lines.Single(l => l.StartsWith("Pub_54321,Bob")); var frankLine = lines.Single(l => l.StartsWith("Pub_11ftw,Frank")); Assert.AreEqual("Pub_54321,Bob,2001-01-01,monkeys can all secretly fly", bobLine); Assert.AreEqual("Pub_11ftw,Frank,2001-10-29,the wizard of OZ was a man behind a machine", frankLine); asExtractable.DeleteInDatabase(); }
public void ExtractableColumnTest() { ExtractableDataSet dataSet = null; ExtractionConfiguration configuration = null; Project project = null; Catalogue cata = null; CatalogueItem cataItem = null; ColumnInfo column = null; TableInfo table = null; ExtractionInformation extractionInformation = null; ExtractableColumn extractableColumn = null; try { //setup catalogue side of things cata = new Catalogue(CatalogueRepository, "unit_test_ExtractableColumnTest_Cata"); cataItem = new CatalogueItem(CatalogueRepository, cata, "unit_test_ExtractableColumnTest_CataItem"); table = new TableInfo(CatalogueRepository, "DaveTable"); column = new ColumnInfo(CatalogueRepository, "Name", "string", table); cataItem.SetColumnInfo(column); extractionInformation = new ExtractionInformation(CatalogueRepository, cataItem, column, "Hashme(Name)"); //setup extractor side of things dataSet = new ExtractableDataSet(DataExportRepository, cata); project = new Project(DataExportRepository, "unit_test_ExtractableColumnTest_Proj"); configuration = new ExtractionConfiguration(DataExportRepository, project); extractableColumn = new ExtractableColumn(DataExportRepository, dataSet, configuration, extractionInformation, 0, "Hashme2(Name)"); Assert.AreEqual(configuration.GetAllExtractableColumnsFor(dataSet).Length, 1); } finally { if (extractionInformation != null) { extractionInformation.DeleteInDatabase(); } if (column != null) { column.DeleteInDatabase(); } if (table != null) { table.DeleteInDatabase(); } if (cataItem != null) { cataItem.DeleteInDatabase(); } if (configuration != null) { configuration.DeleteInDatabase(); } if (project != null) { project.DeleteInDatabase(); } if (dataSet != null) { dataSet.DeleteInDatabase(); } if (cata != null) { cata.DeleteInDatabase(); } } }