public override void Execute() { base.Execute(); var existing = HICDatabaseConfiguration.GetGlobalIgnorePatternIfAny(BasicActivator.RepositoryLocator.CatalogueRepository); if (existing == null) { existing = new StandardRegex(BasicActivator.RepositoryLocator.CatalogueRepository) { ConceptName = StandardRegex.DataLoadEngineGlobalIgnorePattern, Description = "Regex that will be applied as an ignore when running the data load engine", Regex = "^ignore_.*" }; existing.SaveToDatabase(); } if (_explicitPatternProvided) { existing.Regex = _pattern; existing.SaveToDatabase(); } else { Publish(existing); Activate(existing); } }
public CheckEntireDataLoadProcess(ILoadMetadata loadMetadata, HICDatabaseConfiguration databaseConfiguration, HICLoadConfigurationFlags loadConfigurationFlags, MEF mef) { _databaseConfiguration = databaseConfiguration; _loadConfigurationFlags = loadConfigurationFlags; _mef = mef; LoadMetadata = loadMetadata; }
public void Test_IsolateSingleTableWithSchema_Duplication(string schemaExpression, bool includeQualifiers) { var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); //Create a table in 'RAW' (has no constraints) var dt = new DataTable(); dt.Columns.Add("A"); dt.Columns.Add("B"); dt.Rows.Add("Fish", 1); dt.Rows.Add("Fish", 2); dt.Rows.Add("Fish", 3); dt.Rows.Add("Frank", 2); dt.Rows.Add("Candy", 2); var tbl = db.CreateTable("MyCoolTable2", dt); //import the table and make A look like a primary key to the metadata layer (and A would be pk in LIVE but not in RAW ofc) Import(tbl, out var tableInfoCreated, out var columnInfosCreated); var syntax = db.Server.GetQuerySyntaxHelper(); tableInfoCreated.Name = (includeQualifiers ? syntax.EnsureWrapped(db.GetRuntimeName()) : db.GetRuntimeName()) + schemaExpression + (includeQualifiers ? syntax.EnsureWrapped(tbl.GetRuntimeName()) : tbl.GetRuntimeName()); tableInfoCreated.SaveToDatabase(); foreach (ColumnInfo column in columnInfosCreated) { column.Name = tableInfoCreated.Name + "." + (includeQualifiers ? syntax.EnsureWrapped(column.GetRuntimeName()) : column.GetRuntimeName()); column.SaveToDatabase(); } //lie about the primary key status var a = columnInfosCreated.Single(c => c.GetRuntimeName().Equals("A")); a.IsPrimaryKey = true; a.SaveToDatabase(); var mutilator = GetMutilator(db, tableInfoCreated); mutilator.Check(new AcceptAllCheckNotifier()); var config = new HICDatabaseConfiguration(db.Server, RdmpMockFactory.Mock_INameDatabasesAndTablesDuringLoads(db, "MyCoolTable2")); var job = new ThrowImmediatelyDataLoadJob(config, tableInfoCreated); mutilator.Initialize(db, LoadStage.AdjustRaw); mutilator.Mutilate(job); dt = tbl.GetDataTable(); Assert.AreEqual(2, dt.Rows.Count); var dtIsolation = tbl.Database.ExpectTable("MyCoolTable2_Isolation").GetDataTable(); Assert.AreEqual(3, dtIsolation.Rows.Count); }
public MigrationHost(DiscoveredDatabase sourceDbInfo, DiscoveredDatabase destinationDbInfo, MigrationConfiguration migrationConfig, HICDatabaseConfiguration databaseConfiguration) { _sourceDbInfo = sourceDbInfo; _destinationDbInfo = destinationDbInfo; _migrationConfig = migrationConfig; _databaseConfiguration = databaseConfiguration; }
public LoadDiagramServerNode(LoadBubble bubble, DiscoveredDatabase database, TableInfo[] loadTables, HICDatabaseConfiguration config) : base(database.Server.Name, database.Server.DatabaseType) { _bubble = bubble; _database = database; _loadTables = loadTables; _config = config; string serverName = database.Server.Name; switch (bubble) { case LoadBubble.Raw: _description = "RAW Server:" + serverName; break; case LoadBubble.Staging: _description = "STAGING Server:" + serverName; break; case LoadBubble.Live: _description = "LIVE Server:" + serverName; break; default: throw new ArgumentOutOfRangeException("bubble"); } //Live can have multiple databases (for lookups) if (_bubble == LoadBubble.Live) { var servers = loadTables.Select(t => t.Server).Distinct().ToArray(); if (servers.Length > 1) { _description = "Ambiguous LIVE Servers:" + string.Join(",", servers); ErrorDescription = "The TableInfo collection that underly the Catalogues in this data load configuration are on different servers. The servers they believe they live on are:" + string.Join(",", servers) + ". All TableInfos in a load must belong on the same server or the load will not work."; } string[] databases = _loadTables.Select(t => t.GetDatabaseRuntimeName()).Distinct().ToArray(); _liveDatabaseDictionary = new Dictionary <DiscoveredDatabase, TableInfo[]>(); foreach (string dbname in databases) { _liveDatabaseDictionary.Add(_database.Server.ExpectDatabase(dbname), _loadTables.Where(t => t.GetDatabaseRuntimeName().Equals(dbname, StringComparison.CurrentCultureIgnoreCase)).ToArray()); } } //if it is live yield all the lookups if (_bubble == LoadBubble.Live) { foreach (var kvp in _liveDatabaseDictionary) { Children.Add(new LoadDiagramDatabaseNode(_bubble, kvp.Key, kvp.Value, _config)); } } else { Children.Add(new LoadDiagramDatabaseNode(_bubble, _database, _loadTables, _config)); } }
public MigrateRAWToStaging(HICDatabaseConfiguration databaseConfiguration, HICLoadConfigurationFlags loadConfigurationFlags) { _databaseConfiguration = databaseConfiguration; _loadConfigurationFlags = loadConfigurationFlags; Description = "Migrate RAW to Staging"; SkipComponent = !_loadConfigurationFlags.DoLoadToStaging; }
public TableInfoCloneOperation(HICDatabaseConfiguration hicDatabaseConfiguration, TableInfo tableInfo, LoadBubble copyToBubble, IDataLoadEventListener listener) { _hicDatabaseConfiguration = hicDatabaseConfiguration; _tableInfo = tableInfo; _copyToBubble = copyToBubble; this._listener = listener; DropIdentityColumns = true; }
public TableInfoCloneOperation(HICDatabaseConfiguration hicDatabaseConfiguration, TableInfo tableInfo, LoadBubble copyToBubble) { _hicDatabaseConfiguration = hicDatabaseConfiguration; _tableInfo = tableInfo; _copyToBubble = copyToBubble; DropIdentityColumns = true; }
public MigrateStagingToLive(HICDatabaseConfiguration databaseConfiguration, HICLoadConfigurationFlags loadConfigurationFlags) { _databaseConfiguration = databaseConfiguration; _loadConfigurationFlags = loadConfigurationFlags; Description = "Migrate Staging to Live"; SkipComponent = !_loadConfigurationFlags.DoMigrateFromStagingToLive; }
/// <summary> /// Runs the DLE using a custom names for RAW/STAGING. Pass in the load to execute and the files/directories to process /// in the batch. /// </summary> /// <param name="lmd"></param> /// <param name="payload"></param> /// <returns>The exit code of the data load after it completes</returns> private ExitCodeType RunDLE(LoadMetadata lmd, object payload) { var catalogueRepository = (CatalogueRepository)lmd.Repository; //ensures that RAW/STAGING always have unique names _configuration = new HICDatabaseConfiguration(lmd, _namer); _configuration.UpdateButDoNotDiff = new Regex("^MessageGuid"); var logManager = catalogueRepository.GetDefaultLogManager(); logManager.CreateNewLoggingTaskIfNotExists(lmd.GetDistinctLoggingTask()); // Create the pipeline to pass into the DataLoadProcess object var dataLoadFactory = new HICDataLoadFactory(lmd, _configuration, new HICLoadConfigurationFlags(), catalogueRepository, logManager); var stagingCreator = _namer as ICreateAndDestroyStagingDuringLoads; if (stagingCreator != null) { stagingCreator.CreateStaging(lmd.GetDistinctLiveDatabaseServer()); } var listener = new NLogThrowerDataLoadEventListener(NLog.LogManager.GetCurrentClassLogger()); IDataLoadExecution execution = dataLoadFactory.Create(listener); IExternalDatabaseServer raw = catalogueRepository.GetServerDefaults().GetDefaultFor(PermissableDefaults.RAWDataLoadServer); DiscoveredServer liveDb = lmd.GetDistinctLiveDatabaseServer(); //do we want to try to cut down the time it takes to do RAW=>STAGING by using INSERT INTO instead of running anonymisation/migration pipeline if (_useInsertIntoForRawMigration) { //if it is on the same server swap out the migration engine for INSERT INTO if (raw == null || (raw.Server != null && raw.Server.Equals(liveDb.Name) && raw.DatabaseType == liveDb.DatabaseType)) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "SWAPPING RAW=>STAGING migration strategy to INSERT INTO")); SwapMigrateRAWToStagingComponent(execution.Components); } else { //Cannot use because different servers / DatabaseTypes. listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "CANNOT SWAP RAW=>STAGING migration strategy to INSERT INTO because RAW is on '" + raw.Server + "' (" + raw.DatabaseType + ") and STAGING is on '" + liveDb.Name + "' (" + liveDb.DatabaseType + ")")); } } else { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Flag is false for SWAP RAW=>STAGING migration strategy to INSERT INTO So won't do it")); } var procedure = new DataLoadProcess(_repositoryLocator, lmd, null, logManager, listener, execution, _configuration); ExitCodeType exitCode = procedure.Run(new GracefulCancellationToken(), payload); return(exitCode); }
public void ExecuteSqlRuntimeTask_InvalidID(DatabaseType dbType) { var dt = new DataTable(); dt.Columns.Add("Lawl"); dt.Rows.Add(new object[] { 2 }); var db = GetCleanedServer(dbType, true); var tbl = db.CreateTable("Fish", dt); TableInfo ti; ColumnInfo[] cols; Import(tbl, out ti, out cols); string sql = @"UPDATE {T:0} Set {C:0} = 1"; IRuntimeTask task; IProcessTask pt; var dir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "ExecuteSqlFileRuntimeTaskTests", true); var sqlArg = new IArgument[] { Mock.Of <IArgument>(x => x.Name == "Sql" && x.Value == sql && x.GetValueAsSystemType() == sql) }; var args = new RuntimeArgumentCollection(sqlArg, new StageArgs(LoadStage.AdjustRaw, db, dir)); pt = Mock.Of <IProcessTask>(x => x.Path == typeof(ExecuteSqlMutilation).FullName && x.GetAllArguments() == sqlArg ); task = new MutilateDataTablesRuntimeTask(pt, args, CatalogueRepository.MEF); task.Check(new ThrowImmediatelyCheckNotifier()); HICDatabaseConfiguration configuration = new HICDatabaseConfiguration(db.Server); var job = new ThrowImmediatelyDataLoadJob(); job.RegularTablesToLoad = new List <ITableInfo> { ti }; job.LookupTablesToLoad = new List <ITableInfo>(); job.Configuration = configuration; var ex = Assert.Throws <Exception>(() => task.Run(job, new GracefulCancellationToken())); StringAssert.Contains("Mutilate failed", ex.Message); StringAssert.Contains("Failed to find a TableInfo in the load with ID 0", ex.InnerException.Message); task.LoadCompletedSoDispose(Core.DataLoad.ExitCodeType.Success, new ThrowImmediatelyDataLoadEventListener()); }
public LoadDiagramDatabaseNode(LoadBubble bubble, DiscoveredDatabase database, TableInfo[] loadTables, HICDatabaseConfiguration config) { _bubble = bubble; Database = database; _loadTables = loadTables; _config = config; DatabaseName = Database.GetRuntimeName(); _anticipatedChildren.AddRange(_loadTables.Select(t => new LoadDiagramTableNode(this, t, _bubble, _config))); }
public void ExecuteSqlFileRuntimeTask_ValidID_CustomNamer(DatabaseType dbType) { var dt = new DataTable(); dt.Columns.Add("Lawl"); dt.Rows.Add(new object[] { 2 }); var db = GetCleanedServer(dbType, true); var tbl = db.CreateTable("Fish", dt); var tableName = "AAAAAAA"; TableInfo ti; ColumnInfo[] cols; Import(tbl, out ti, out cols); FileInfo f = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, "Bob.sql")); File.WriteAllText(f.FullName, @"UPDATE {T:" + ti.ID + "} Set {C:" + cols[0].ID + "} = 1"); tbl.Rename(tableName); //we renamed the table to simulate RAW, confirm TableInfo doesn't think it exists Assert.IsFalse(ti.Discover(DataAccessContext.InternalDataProcessing).Exists()); var pt = Mock.Of <IProcessTask>(x => x.Path == f.FullName); var dir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "ExecuteSqlFileRuntimeTaskTests", true); var task = new ExecuteSqlFileRuntimeTask(pt, new RuntimeArgumentCollection(new IArgument[0], new StageArgs(LoadStage.AdjustRaw, db, dir))); task.Check(new ThrowImmediatelyCheckNotifier()); //create a namer that tells the user var namer = RdmpMockFactory.Mock_INameDatabasesAndTablesDuringLoads(db, tableName); HICDatabaseConfiguration configuration = new HICDatabaseConfiguration(db.Server, namer); IDataLoadJob job = Mock.Of <IDataLoadJob>(x => x.RegularTablesToLoad == new List <ITableInfo> { ti } && x.LookupTablesToLoad == new List <ITableInfo>() && x.Configuration == configuration); task.Run(job, new GracefulCancellationToken()); Assert.AreEqual(1, tbl.GetDataTable().Rows[0][0]); tbl.Drop(); }
public DataLoadProcess(IRDMPPlatformRepositoryServiceLocator repositoryLocator, ILoadMetadata loadMetadata, ICheckable preExecutionChecker, ILogManager logManager, IDataLoadEventListener dataLoadEventListener, IDataLoadExecution loadExecution, HICDatabaseConfiguration configuration) { _repositoryLocator = repositoryLocator; LoadMetadata = loadMetadata; DataLoadEventListener = dataLoadEventListener; _configuration = configuration; LoadExecution = loadExecution; _preExecutionChecker = preExecutionChecker; LogManager = logManager; ExitCode = ExitCodeType.Success; JobProvider = new JobFactory(loadMetadata, logManager); }
public HICDataLoadFactory(ILoadMetadata loadMetadata, HICDatabaseConfiguration databaseConfiguration, HICLoadConfigurationFlags loadConfigurationFlags, ICatalogueRepository repository, ILogManager logManager) { _databaseConfiguration = databaseConfiguration; _loadConfigurationFlags = loadConfigurationFlags; _repository = repository; _logManager = logManager; LoadMetadata = loadMetadata; // If we are not supplied any catalogues to load, it is expected that we will load all catalogues associated with the provided ILoadMetadata _cataloguesToLoad = LoadMetadata.GetAllCatalogues().ToList(); if (!_cataloguesToLoad.Any()) { throw new InvalidOperationException("LoadMetadata " + LoadMetadata.ID + " is not related to any Catalogues, there is nothing to load"); } }
public void Test_IsolateSingleTable_Dupliction(DatabaseType dbType) { var db = GetCleanedServer(dbType); //Create a table in 'RAW' (has no constraints) var dt = new DataTable(); dt.Columns.Add("A"); dt.Columns.Add("B"); dt.Rows.Add("Fish", 1); dt.Rows.Add("Fish", 2); dt.Rows.Add("Fish", 3); dt.Rows.Add("Frank", 2); dt.Rows.Add("Candy", 2); var tbl = db.CreateTable("MyCoolTable2", dt); //import the table and make A look like a primary key to the metadata layer (and A would be pk in LIVE but not in RAW ofc) TableInfo tableInfoCreated; ColumnInfo[] columnInfosCreated; Import(tbl, out tableInfoCreated, out columnInfosCreated); //lie abot the primary key status var a = columnInfosCreated.Single(c => c.GetRuntimeName().Equals("A")); a.IsPrimaryKey = true; a.SaveToDatabase(); var mutilator = GetMutilator(db, tableInfoCreated); mutilator.Check(new AcceptAllCheckNotifier()); var config = new HICDatabaseConfiguration(db.Server, RdmpMockFactory.Mock_INameDatabasesAndTablesDuringLoads(db, "MyCoolTable2")); var job = new ThrowImmediatelyDataLoadJob(config, tableInfoCreated); mutilator.Initialize(db, LoadStage.AdjustRaw); mutilator.Mutilate(job); dt = tbl.GetDataTable(); Assert.AreEqual(2, dt.Rows.Count); var dtIsolation = tbl.Database.ExpectTable("MyCoolTable2_Isolation").GetDataTable(); Assert.AreEqual(3, dtIsolation.Rows.Count); }
public void RefreshUIFromDatabase() { tlvLoadedTables.ClearObjects(); if (_loadMetadata == null) { return; } TableInfo[] allTables; HICDatabaseConfiguration config; try { if (!_loadMetadata.GetAllCatalogues().Any()) { throw new Exception("There are no Catalogues (Datasets) associated with this LoadMetadata, choose one or more Catalogues by clicking 'Edit..' in LoadMetadataUI "); } allTables = _loadMetadata.GetDistinctTableInfoList(true).ToArray(); config = new HICDatabaseConfiguration(_loadMetadata); } catch (Exception e) { CommonFunctionality.Fatal("Could not fetch data", e); tlvLoadedTables.Visible = false; return; } tlvLoadedTables.Visible = true; _raw = new LoadDiagramServerNode(LoadBubble.Raw, config.DeployInfo[LoadBubble.Raw], allTables, config); var staging = new LoadDiagramServerNode(LoadBubble.Staging, config.DeployInfo[LoadBubble.Staging], allTables, config); var live = new LoadDiagramServerNode(LoadBubble.Live, config.DeployInfo[LoadBubble.Live], allTables, config); tlvLoadedTables.AddObject(_raw); tlvLoadedTables.AddObject(staging); tlvLoadedTables.AddObject(live); //expand the servers foreach (var rootObject in tlvLoadedTables.Objects) { ExpandToDepth(2, rootObject); } loadStateUI1.SetStatus(LoadStateUI.LoadState.Unknown); }
public void ExecuteSqlFileRuntimeTask_InvalidID(DatabaseType dbType) { var dt = new DataTable(); dt.Columns.Add("Lawl"); dt.Rows.Add(new object[] { 2 }); var db = GetCleanedServer(dbType, true); var tbl = db.CreateTable("Fish", dt); TableInfo ti; ColumnInfo[] cols; Import(tbl, out ti, out cols); FileInfo f = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, "Bob.sql")); File.WriteAllText(f.FullName, @"UPDATE {T:0} Set {C:0} = 1"); var pt = Mock.Of <IProcessTask>(x => x.Path == f.FullName); var dir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "ExecuteSqlFileRuntimeTaskTests", true); var task = new ExecuteSqlFileRuntimeTask(pt, new RuntimeArgumentCollection(new IArgument[0], new StageArgs(LoadStage.AdjustRaw, db, dir))); task.Check(new ThrowImmediatelyCheckNotifier()); HICDatabaseConfiguration configuration = new HICDatabaseConfiguration(db.Server); IDataLoadJob job = Mock.Of <IDataLoadJob>(x => x.RegularTablesToLoad == new List <ITableInfo> { ti } && x.LookupTablesToLoad == new List <ITableInfo>() && x.Configuration == configuration); var ex = Assert.Throws <ExecuteSqlFileRuntimeTaskException>(() => task.Run(job, new GracefulCancellationToken())); StringAssert.Contains("Failed to find a TableInfo in the load with ID 0", ex.Message); task.LoadCompletedSoDispose(Core.DataLoad.ExitCodeType.Success, new ThrowImmediatelyDataLoadEventListener()); }
public DataLoadJob(IRDMPPlatformRepositoryServiceLocator repositoryLocator, string description, ILogManager logManager, ILoadMetadata loadMetadata, ILoadDirectory directory, IDataLoadEventListener listener, HICDatabaseConfiguration configuration) { _logManager = logManager; RepositoryLocator = repositoryLocator; LoadMetadata = loadMetadata; LoadDirectory = directory; Configuration = configuration; _listener = listener; Description = description; List <ICatalogue> catalogues = LoadMetadata.GetAllCatalogues().ToList(); if (LoadMetadata != null) { _loggingTask = GetLoggingTask(catalogues); } RegularTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetTableInfoList(false)).Distinct().ToList(); LookupTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetLookupTableInfoList()).Distinct().ToList(); }
public void TestHICDatabaseConfiguration_ExpectTables(bool testLookup) { var conf = new HICDatabaseConfiguration(new DiscoveredServer("localhost", "mydb", DatabaseType.MicrosoftSQLServer, null, null), new FixedStagingDatabaseNamer("mydb")); var ti = WhenIHaveA <TableInfo>(); var lookup = WhenIHaveA <TableInfo>(); lookup.Name = "MyHeartyLookup"; lookup.Database = "LookupsDb"; lookup.SaveToDatabase(); var job = Mock.Of <IDataLoadJob>(m => m.RegularTablesToLoad == new List <ITableInfo>(new [] { ti }) && m.LookupTablesToLoad == new List <ITableInfo>(new [] { lookup })); var result = conf.ExpectTables(job, LoadBubble.Raw, testLookup).ToArray(); Assert.AreEqual(testLookup ? 2 : 1, result.Length); StringAssert.AreEqualIgnoringCase("mydb_RAW", result[0].Database.GetRuntimeName()); StringAssert.AreEqualIgnoringCase("My_Table", result[0].GetRuntimeName()); if (testLookup) { StringAssert.AreEqualIgnoringCase("mydb_RAW", result[1].Database.GetRuntimeName()); StringAssert.AreEqualIgnoringCase("MyHeartyLookup", result[1].GetRuntimeName()); } result = conf.ExpectTables(job, LoadBubble.Staging, testLookup).ToArray(); Assert.AreEqual(testLookup ? 2 : 1, result.Length); StringAssert.AreEqualIgnoringCase("DLE_STAGING", result[0].Database.GetRuntimeName()); StringAssert.AreEqualIgnoringCase("mydb_My_Table_STAGING", result[0].GetRuntimeName()); if (testLookup) { StringAssert.AreEqualIgnoringCase("DLE_STAGING", result[1].Database.GetRuntimeName()); StringAssert.AreEqualIgnoringCase("mydb_MyHeartyLookup_STAGING", result[1].GetRuntimeName()); } }
public void Test_CloneTable() { var dt = new DataTable(); dt.Columns.Add("FF"); var db = GetCleanedServer(FAnsi.DatabaseType.MicrosoftSQLServer); var tbl = db.CreateTable("MyTable", dt); Import(tbl, out var ti, out _); var config = new HICDatabaseConfiguration(tbl.Database.Server); //create a RAW table schema called TableName_Isolation var cloner = new TableInfoCloneOperation(config, (TableInfo)ti, LoadBubble.Live, new ThrowImmediatelyDataLoadEventListener()); cloner.CloneTable(tbl.Database, tbl.Database, tbl, tbl.GetRuntimeName() + "_copy", true, true, true, ti.PreLoadDiscardedColumns); var tbl2 = tbl.Database.ExpectTable(tbl.GetRuntimeName() + "_copy"); Assert.IsTrue(tbl2.Exists()); }
public void TestPayloadInjection() { BulkTestsData b = new BulkTestsData(CatalogueRepository, DiscoveredDatabaseICanCreateRandomTablesIn, 10); b.SetupTestData(); b.ImportAsCatalogue(); var lmd = new LoadMetadata(CatalogueRepository, "Loading"); lmd.LocationOfFlatFiles = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "delme", true).RootPath.FullName; lmd.SaveToDatabase(); CatalogueRepository.MEF.AddTypeToCatalogForTesting(typeof(TestPayloadAttacher)); b.catalogue.LoadMetadata_ID = lmd.ID; b.catalogue.LoggingDataTask = "TestPayloadInjection"; b.catalogue.SaveToDatabase(); var lm = new LogManager(new ServerDefaults(CatalogueRepository).GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID)); lm.CreateNewLoggingTaskIfNotExists("TestPayloadInjection"); var pt = new ProcessTask(CatalogueRepository, lmd, LoadStage.Mounting); pt.Path = typeof(TestPayloadAttacher).FullName; pt.ProcessTaskType = ProcessTaskType.Attacher; pt.SaveToDatabase(); var config = new HICDatabaseConfiguration(DiscoveredDatabaseICanCreateRandomTablesIn.Server); var factory = new HICDataLoadFactory(lmd, config, new HICLoadConfigurationFlags(), CatalogueRepository, lm); IDataLoadExecution execution = factory.Create(new ThrowImmediatelyDataLoadEventListener()); var proceedure = new DataLoadProcess(RepositoryLocator, lmd, null, lm, new ThrowImmediatelyDataLoadEventListener(), execution, config); proceedure.Run(new GracefulCancellationToken(), payload); Assert.IsTrue(PayloadTest.Success, "Expected IAttacher to detect Payload and set this property to true"); }
public LoadDiagramTableNode(LoadDiagramDatabaseNode databaseNode, TableInfo tableInfo, LoadBubble bubble, HICDatabaseConfiguration config) { _databaseNode = databaseNode; TableInfo = tableInfo; Bubble = bubble; _config = config; State = LoadDiagramState.Anticipated; TableName = TableInfo.GetRuntimeName(Bubble); //only reference schema if it is LIVE string schema = bubble >= LoadBubble.Live ? tableInfo.Schema: null; Table = databaseNode.Database.ExpectTable(TableName, schema); var cols = TableInfo.GetColumnsAtStage(Bubble.ToLoadStage()) .Select(c => new LoadDiagramColumnNode(this, c, Bubble)); _anticipatedChildren.AddRange(cols); }
private void RunAttachStageWithNormalJob(RemoteTableAttacher attacher, DiscoveredDatabase db) { //the table to get data from attacher.RemoteTableName = "table1"; attacher.RAWTableName = "table2"; attacher.Check(new ThrowImmediatelyCheckNotifier()); attacher.Initialize(null, db); using (var dt = new DataTable()) { dt.Columns.Add("Col1"); dt.Rows.Add("fff"); var tbl1 = db.CreateTable("table1", dt); var tbl2 = db.CreateTable("table2", new [] { new DatabaseColumnRequest("Col1", new DatabaseTypeRequest(typeof(string), 5)) }); Assert.AreEqual(1, tbl1.GetRowCount()); Assert.AreEqual(0, tbl2.GetRowCount()); var logManager = new LogManager(new DiscoveredServer(UnitTestLoggingConnectionString)); var lmd = RdmpMockFactory.Mock_LoadMetadataLoadingTable(tbl2); Mock.Get(lmd).Setup(p => p.CatalogueRepository).Returns(CatalogueRepository); logManager.CreateNewLoggingTaskIfNotExists(lmd.GetDistinctLoggingTask()); var dbConfiguration = new HICDatabaseConfiguration(lmd, RdmpMockFactory.Mock_INameDatabasesAndTablesDuringLoads(db, "table2")); var job = new DataLoadJob(RepositoryLocator, "test job", logManager, lmd, new TestLoadDirectory(), new ThrowImmediatelyDataLoadEventListener(), dbConfiguration); job.StartLogging(); attacher.Attach(job, new GracefulCancellationToken()); Assert.AreEqual(1, tbl1.GetRowCount()); Assert.AreEqual(1, tbl2.GetRowCount()); } }
// todo: refactor to cut down on ctor params public IterativeScheduledDataLoadProcess(IRDMPPlatformRepositoryServiceLocator repositoryLocator, ILoadMetadata loadMetadata, ICheckable preExecutionChecker, IDataLoadExecution loadExecution, JobDateGenerationStrategyFactory jobDateGenerationStrategyFactory, ILoadProgressSelectionStrategy loadProgressSelectionStrategy, int?overrideNumberOfDaysToLoad, ILogManager logManager, IDataLoadEventListener dataLoadEventsreceiver, HICDatabaseConfiguration configuration) : base(repositoryLocator, loadMetadata, preExecutionChecker, loadExecution, jobDateGenerationStrategyFactory, loadProgressSelectionStrategy, overrideNumberOfDaysToLoad, logManager, dataLoadEventsreceiver, configuration) { }
public DatabaseCloner(HICDatabaseConfiguration hicDatabaseConfiguration) { _hicDatabaseConfiguration = hicDatabaseConfiguration; _tablesCreated = new List <TableInfoCloneOperation>(); _databasesCreated = new List <DiscoveredDatabase>(); }
public void TestTemporalTable(bool ignoreWithGlobalPattern) { var dbtype = FAnsi.DatabaseType.MicrosoftSQLServer; var db = GetCleanedServer(dbtype); using (var con = db.Server.GetConnection()) { con.Open(); db.Server.GetCommand(sql, con).ExecuteNonQuery(); } var tbl = db.ExpectTable("Employee"); var defaults = new ServerDefaults(CatalogueRepository); var logServer = defaults.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID); var logManager = new LogManager(logServer); var raw = db.Server.ExpectDatabase(db.GetRuntimeName() + "_RAW"); if (raw.Exists()) { raw.Drop(); } //define a new load configuration var lmd = new LoadMetadata(CatalogueRepository, "MyLoad"); lmd.IgnoreTrigger = true; lmd.SaveToDatabase(); ITableInfo ti = Import(tbl, lmd, logManager); var projectDirectory = SetupLoadDirectory(lmd); CreateCSVProcessTask(lmd, ti, "*.csv"); //create a text file to load where we update Frank's favourite colour (it's a pk field) and we insert a new record (MrMurder) File.WriteAllText( Path.Combine(projectDirectory.ForLoading.FullName, "LoadMe.csv"), @"EmployeeID,Name,Position,Department,Address,AnnualSalary 1,Frank,Boss,Department of F'Tang, 22 Innsmouth Way, 55000.5 2,Herbert,Super Boss,Department of F'Tang, 22 Innsmouth Way, 155000.5"); //the checks will probably need to be run as ddl admin because it involves creating _Archive table and trigger the first time //clean SetUp RAW / STAGING etc and generally accept proposed cleanup operations var checker = new CheckEntireDataLoadProcess(lmd, new HICDatabaseConfiguration(lmd), new HICLoadConfigurationFlags(), CatalogueRepository.MEF); checker.Check(new AcceptAllCheckNotifier()); if (ignoreWithGlobalPattern) { var regex = new StandardRegex(RepositoryLocator.CatalogueRepository) { ConceptName = StandardRegex.DataLoadEngineGlobalIgnorePattern, Regex = "^Valid((From)|(To))$" }; regex.SaveToDatabase(); } else { var col = ti.ColumnInfos.Single(c => c.GetRuntimeName().Equals("ValidFrom")); col.IgnoreInLoads = true; col.SaveToDatabase(); col = ti.ColumnInfos.Single(c => c.GetRuntimeName().Equals("ValidTo")); col.IgnoreInLoads = true; col.SaveToDatabase(); } var dbConfig = new HICDatabaseConfiguration(lmd, null); var loadFactory = new HICDataLoadFactory( lmd, dbConfig, new HICLoadConfigurationFlags(), CatalogueRepository, logManager ); var exe = loadFactory.Create(new ThrowImmediatelyDataLoadEventListener()); var exitCode = exe.Run( new DataLoadJob(RepositoryLocator, "Go go go!", logManager, lmd, projectDirectory, new ThrowImmediatelyDataLoadEventListener(), dbConfig), new GracefulCancellationToken()); Assert.AreEqual(ExitCodeType.Success, exitCode); //frank should be updated to his new departement and role Assert.AreEqual(2, tbl.GetRowCount()); var result = tbl.GetDataTable(); var frank = result.Rows.Cast <DataRow>().Single(r => (string)r["Name"] == "Frank"); Assert.AreEqual("Department of F'Tang", frank["Department"]); Assert.AreEqual("Boss", frank["Position"]); //post test cleanup foreach (var regex in RepositoryLocator.CatalogueRepository.GetAllObjects <StandardRegex>()) { regex.DeleteInDatabase(); } }
public abstract IDataLoadJob Create(IRDMPPlatformRepositoryServiceLocator repositoryLocator, IDataLoadEventListener listener, HICDatabaseConfiguration configuration);
// This no longer copies between servers, but the original test didn't guarantee that would happen anyway public void CloneDatabaseAndTable() { string testLiveDatabaseName = TestDatabaseNames.GetConsistentName("TEST"); var testDb = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(testLiveDatabaseName); var raw = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(testLiveDatabaseName + "_RAW"); foreach (DiscoveredDatabase db in new[] { raw, testDb }) { if (db.Exists()) { foreach (DiscoveredTable table in db.DiscoverTables(true)) { table.Drop(); } db.Drop(); } } DiscoveredServerICanCreateRandomDatabasesAndTablesOn.CreateDatabase(testLiveDatabaseName); Assert.IsTrue(testDb.Exists()); testDb.CreateTable("Table_1", new[] { new DatabaseColumnRequest("Id", "int") }); //clone the builder var builder = new SqlConnectionStringBuilder(DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Builder.ConnectionString) { InitialCatalog = testLiveDatabaseName }; var dbConfiguration = new HICDatabaseConfiguration(new DiscoveredServer(builder), null, new ServerDefaults(CatalogueRepository)); var cloner = new DatabaseCloner(dbConfiguration); try { var cloneDb = cloner.CreateDatabaseForStage(LoadBubble.Raw); //confirm database appeared Assert.IsTrue(DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(testLiveDatabaseName + "_RAW").Exists()); //now create a catalogue and wire it SetUp to the table TEST on the test database server Catalogue cata = SetupATestCatalogue(builder, testLiveDatabaseName, "Table_1"); //now clone the catalogue data structures to MachineName foreach (TableInfo tableInfo in cata.GetTableInfoList(false)) { cloner.CreateTablesInDatabaseFromCatalogueInfo(new ThrowImmediatelyDataLoadEventListener(), tableInfo, LoadBubble.Raw); } Assert.IsTrue(raw.Exists()); Assert.IsTrue(raw.ExpectTable("Table_1").Exists()); } finally { cloner.LoadCompletedSoDispose(ExitCodeType.Success, new ThrowImmediatelyDataLoadEventListener()); while (toCleanUp.Count > 0) { try { toCleanUp.Pop().DeleteInDatabase(); } catch (Exception e) { //always clean SetUp everything Console.WriteLine(e); } } } }
public PopulateRAW(List <IRuntimeTask> collection, HICDatabaseConfiguration databaseConfiguration) : base(collection.Cast <IDataLoadComponent>().ToList()) { _databaseConfiguration = databaseConfiguration; Description = "Populate RAW"; }