public override void Execute()
{
base.Execute();
var existing = HICDatabaseConfiguration.GetGlobalIgnorePatternIfAny(BasicActivator.RepositoryLocator.CatalogueRepository);
if (existing == null)
{
existing = new StandardRegex(BasicActivator.RepositoryLocator.CatalogueRepository)
{
ConceptName = StandardRegex.DataLoadEngineGlobalIgnorePattern,
Description = "Regex that will be applied as an ignore when running the data load engine",
Regex = "^ignore_.*"
};
existing.SaveToDatabase();
}
if (_explicitPatternProvided)
{
existing.Regex = _pattern;
existing.SaveToDatabase();
}
else
{
Publish(existing);
Activate(existing);
}
}
public CheckEntireDataLoadProcess(ILoadMetadata loadMetadata, HICDatabaseConfiguration databaseConfiguration, HICLoadConfigurationFlags loadConfigurationFlags, MEF mef)
{
_databaseConfiguration = databaseConfiguration;
_loadConfigurationFlags = loadConfigurationFlags;
_mef = mef;
LoadMetadata = loadMetadata;
}
public void Test_IsolateSingleTableWithSchema_Duplication(string schemaExpression, bool includeQualifiers)
{
var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer);
//Create a table in 'RAW' (has no constraints)
var dt = new DataTable();
dt.Columns.Add("A");
dt.Columns.Add("B");
dt.Rows.Add("Fish", 1);
dt.Rows.Add("Fish", 2);
dt.Rows.Add("Fish", 3);
dt.Rows.Add("Frank", 2);
dt.Rows.Add("Candy", 2);
var tbl = db.CreateTable("MyCoolTable2", dt);
//import the table and make A look like a primary key to the metadata layer (and A would be pk in LIVE but not in RAW ofc)
Import(tbl, out var tableInfoCreated, out var columnInfosCreated);
var syntax = db.Server.GetQuerySyntaxHelper();
tableInfoCreated.Name =
(includeQualifiers ? syntax.EnsureWrapped(db.GetRuntimeName()) : db.GetRuntimeName())
+ schemaExpression +
(includeQualifiers ? syntax.EnsureWrapped(tbl.GetRuntimeName()) : tbl.GetRuntimeName());
tableInfoCreated.SaveToDatabase();
foreach (ColumnInfo column in columnInfosCreated)
{
column.Name = tableInfoCreated.Name + "." +
(includeQualifiers ? syntax.EnsureWrapped(column.GetRuntimeName()) : column.GetRuntimeName());
column.SaveToDatabase();
}
//lie about the primary key status
var a = columnInfosCreated.Single(c => c.GetRuntimeName().Equals("A"));
a.IsPrimaryKey = true;
a.SaveToDatabase();
var mutilator = GetMutilator(db, tableInfoCreated);
mutilator.Check(new AcceptAllCheckNotifier());
var config = new HICDatabaseConfiguration(db.Server, RdmpMockFactory.Mock_INameDatabasesAndTablesDuringLoads(db, "MyCoolTable2"));
var job = new ThrowImmediatelyDataLoadJob(config, tableInfoCreated);
mutilator.Initialize(db, LoadStage.AdjustRaw);
mutilator.Mutilate(job);
dt = tbl.GetDataTable();
Assert.AreEqual(2, dt.Rows.Count);
var dtIsolation = tbl.Database.ExpectTable("MyCoolTable2_Isolation").GetDataTable();
Assert.AreEqual(3, dtIsolation.Rows.Count);
}
public MigrationHost(DiscoveredDatabase sourceDbInfo, DiscoveredDatabase destinationDbInfo, MigrationConfiguration migrationConfig, HICDatabaseConfiguration databaseConfiguration)
{
_sourceDbInfo = sourceDbInfo;
_destinationDbInfo = destinationDbInfo;
_migrationConfig = migrationConfig;
_databaseConfiguration = databaseConfiguration;
}
public LoadDiagramServerNode(LoadBubble bubble, DiscoveredDatabase database, TableInfo[] loadTables, HICDatabaseConfiguration config) : base(database.Server.Name, database.Server.DatabaseType)
{
_bubble = bubble;
_database = database;
_loadTables = loadTables;
_config = config;
string serverName = database.Server.Name;
switch (bubble)
{
case LoadBubble.Raw:
_description = "RAW Server:" + serverName;
break;
case LoadBubble.Staging:
_description = "STAGING Server:" + serverName;
break;
case LoadBubble.Live:
_description = "LIVE Server:" + serverName;
break;
default:
throw new ArgumentOutOfRangeException("bubble");
}
//Live can have multiple databases (for lookups)
if (_bubble == LoadBubble.Live)
{
var servers = loadTables.Select(t => t.Server).Distinct().ToArray();
if (servers.Length > 1)
{
_description = "Ambiguous LIVE Servers:" + string.Join(",", servers);
ErrorDescription = "The TableInfo collection that underly the Catalogues in this data load configuration are on different servers. The servers they believe they live on are:" + string.Join(",", servers) + ". All TableInfos in a load must belong on the same server or the load will not work.";
}
string[] databases = _loadTables.Select(t => t.GetDatabaseRuntimeName()).Distinct().ToArray();
_liveDatabaseDictionary = new Dictionary <DiscoveredDatabase, TableInfo[]>();
foreach (string dbname in databases)
{
_liveDatabaseDictionary.Add(_database.Server.ExpectDatabase(dbname), _loadTables.Where(t => t.GetDatabaseRuntimeName().Equals(dbname, StringComparison.CurrentCultureIgnoreCase)).ToArray());
}
}
//if it is live yield all the lookups
if (_bubble == LoadBubble.Live)
{
foreach (var kvp in _liveDatabaseDictionary)
{
Children.Add(new LoadDiagramDatabaseNode(_bubble, kvp.Key, kvp.Value, _config));
}
}
else
{
Children.Add(new LoadDiagramDatabaseNode(_bubble, _database, _loadTables, _config));
}
}
public MigrateRAWToStaging(HICDatabaseConfiguration databaseConfiguration, HICLoadConfigurationFlags loadConfigurationFlags)
{
_databaseConfiguration = databaseConfiguration;
_loadConfigurationFlags = loadConfigurationFlags;
Description = "Migrate RAW to Staging";
SkipComponent = !_loadConfigurationFlags.DoLoadToStaging;
}
public TableInfoCloneOperation(HICDatabaseConfiguration hicDatabaseConfiguration, TableInfo tableInfo, LoadBubble copyToBubble, IDataLoadEventListener listener)
{
_hicDatabaseConfiguration = hicDatabaseConfiguration;
_tableInfo = tableInfo;
_copyToBubble = copyToBubble;
this._listener = listener;
DropIdentityColumns = true;
}
public TableInfoCloneOperation(HICDatabaseConfiguration hicDatabaseConfiguration, TableInfo tableInfo, LoadBubble copyToBubble)
{
_hicDatabaseConfiguration = hicDatabaseConfiguration;
_tableInfo = tableInfo;
_copyToBubble = copyToBubble;
DropIdentityColumns = true;
}
public MigrateStagingToLive(HICDatabaseConfiguration databaseConfiguration, HICLoadConfigurationFlags loadConfigurationFlags)
{
_databaseConfiguration = databaseConfiguration;
_loadConfigurationFlags = loadConfigurationFlags;
Description = "Migrate Staging to Live";
SkipComponent = !_loadConfigurationFlags.DoMigrateFromStagingToLive;
}
/// <summary>
/// Runs the DLE using a custom names for RAW/STAGING. Pass in the load to execute and the files/directories to process
/// in the batch.
/// </summary>
/// <param name="lmd"></param>
/// <param name="payload"></param>
/// <returns>The exit code of the data load after it completes</returns>
private ExitCodeType RunDLE(LoadMetadata lmd, object payload)
{
var catalogueRepository = (CatalogueRepository)lmd.Repository;
//ensures that RAW/STAGING always have unique names
_configuration = new HICDatabaseConfiguration(lmd, _namer);
_configuration.UpdateButDoNotDiff = new Regex("^MessageGuid");
var logManager = catalogueRepository.GetDefaultLogManager();
logManager.CreateNewLoggingTaskIfNotExists(lmd.GetDistinctLoggingTask());
// Create the pipeline to pass into the DataLoadProcess object
var dataLoadFactory = new HICDataLoadFactory(lmd, _configuration, new HICLoadConfigurationFlags(),
catalogueRepository, logManager);
var stagingCreator = _namer as ICreateAndDestroyStagingDuringLoads;
if (stagingCreator != null)
{
stagingCreator.CreateStaging(lmd.GetDistinctLiveDatabaseServer());
}
var listener = new NLogThrowerDataLoadEventListener(NLog.LogManager.GetCurrentClassLogger());
IDataLoadExecution execution = dataLoadFactory.Create(listener);
IExternalDatabaseServer raw = catalogueRepository.GetServerDefaults().GetDefaultFor(PermissableDefaults.RAWDataLoadServer);
DiscoveredServer liveDb = lmd.GetDistinctLiveDatabaseServer();
//do we want to try to cut down the time it takes to do RAW=>STAGING by using INSERT INTO instead of running anonymisation/migration pipeline
if (_useInsertIntoForRawMigration)
{
//if it is on the same server swap out the migration engine for INSERT INTO
if (raw == null || (raw.Server != null && raw.Server.Equals(liveDb.Name) && raw.DatabaseType == liveDb.DatabaseType))
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "SWAPPING RAW=>STAGING migration strategy to INSERT INTO"));
SwapMigrateRAWToStagingComponent(execution.Components);
}
else
{
//Cannot use because different servers / DatabaseTypes.
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "CANNOT SWAP RAW=>STAGING migration strategy to INSERT INTO because RAW is on '" + raw.Server + "' (" + raw.DatabaseType + ") and STAGING is on '" + liveDb.Name + "' (" + liveDb.DatabaseType + ")"));
}
}
else
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Flag is false for SWAP RAW=>STAGING migration strategy to INSERT INTO So won't do it"));
}
var procedure = new DataLoadProcess(_repositoryLocator, lmd, null, logManager, listener, execution, _configuration);
ExitCodeType exitCode = procedure.Run(new GracefulCancellationToken(), payload);
return(exitCode);
}
public void ExecuteSqlRuntimeTask_InvalidID(DatabaseType dbType)
{
var dt = new DataTable();
dt.Columns.Add("Lawl");
dt.Rows.Add(new object[] { 2 });
var db = GetCleanedServer(dbType, true);
var tbl = db.CreateTable("Fish", dt);
TableInfo ti;
ColumnInfo[] cols;
Import(tbl, out ti, out cols);
string sql = @"UPDATE {T:0} Set {C:0} = 1";
IRuntimeTask task;
IProcessTask pt;
var dir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "ExecuteSqlFileRuntimeTaskTests", true);
var sqlArg = new IArgument[] { Mock.Of <IArgument>(x =>
x.Name == "Sql" &&
x.Value == sql &&
x.GetValueAsSystemType() == sql) };
var args = new RuntimeArgumentCollection(sqlArg, new StageArgs(LoadStage.AdjustRaw, db, dir));
pt = Mock.Of <IProcessTask>(x =>
x.Path == typeof(ExecuteSqlMutilation).FullName &&
x.GetAllArguments() == sqlArg
);
task = new MutilateDataTablesRuntimeTask(pt, args, CatalogueRepository.MEF);
task.Check(new ThrowImmediatelyCheckNotifier());
HICDatabaseConfiguration configuration = new HICDatabaseConfiguration(db.Server);
var job = new ThrowImmediatelyDataLoadJob();
job.RegularTablesToLoad = new List <ITableInfo> {
ti
};
job.LookupTablesToLoad = new List <ITableInfo>();
job.Configuration = configuration;
var ex = Assert.Throws <Exception>(() => task.Run(job, new GracefulCancellationToken()));
StringAssert.Contains("Mutilate failed", ex.Message);
StringAssert.Contains("Failed to find a TableInfo in the load with ID 0", ex.InnerException.Message);
task.LoadCompletedSoDispose(Core.DataLoad.ExitCodeType.Success, new ThrowImmediatelyDataLoadEventListener());
}
public LoadDiagramDatabaseNode(LoadBubble bubble, DiscoveredDatabase database, TableInfo[] loadTables, HICDatabaseConfiguration config)
{
_bubble = bubble;
Database = database;
_loadTables = loadTables;
_config = config;
DatabaseName = Database.GetRuntimeName();
_anticipatedChildren.AddRange(_loadTables.Select(t => new LoadDiagramTableNode(this, t, _bubble, _config)));
}
public void ExecuteSqlFileRuntimeTask_ValidID_CustomNamer(DatabaseType dbType)
{
var dt = new DataTable();
dt.Columns.Add("Lawl");
dt.Rows.Add(new object[] { 2 });
var db = GetCleanedServer(dbType, true);
var tbl = db.CreateTable("Fish", dt);
var tableName = "AAAAAAA";
TableInfo ti;
ColumnInfo[] cols;
Import(tbl, out ti, out cols);
FileInfo f = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, "Bob.sql"));
File.WriteAllText(f.FullName, @"UPDATE {T:" + ti.ID + "} Set {C:" + cols[0].ID + "} = 1");
tbl.Rename(tableName);
//we renamed the table to simulate RAW, confirm TableInfo doesn't think it exists
Assert.IsFalse(ti.Discover(DataAccessContext.InternalDataProcessing).Exists());
var pt = Mock.Of <IProcessTask>(x => x.Path == f.FullName);
var dir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "ExecuteSqlFileRuntimeTaskTests", true);
var task = new ExecuteSqlFileRuntimeTask(pt, new RuntimeArgumentCollection(new IArgument[0], new StageArgs(LoadStage.AdjustRaw, db, dir)));
task.Check(new ThrowImmediatelyCheckNotifier());
//create a namer that tells the user
var namer = RdmpMockFactory.Mock_INameDatabasesAndTablesDuringLoads(db, tableName);
HICDatabaseConfiguration configuration = new HICDatabaseConfiguration(db.Server, namer);
IDataLoadJob job = Mock.Of <IDataLoadJob>(x =>
x.RegularTablesToLoad == new List <ITableInfo> {
ti
} &&
x.LookupTablesToLoad == new List <ITableInfo>() &&
x.Configuration == configuration);
task.Run(job, new GracefulCancellationToken());
Assert.AreEqual(1, tbl.GetDataTable().Rows[0][0]);
tbl.Drop();
}
public DataLoadProcess(IRDMPPlatformRepositoryServiceLocator repositoryLocator, ILoadMetadata loadMetadata, ICheckable preExecutionChecker, ILogManager logManager, IDataLoadEventListener dataLoadEventListener, IDataLoadExecution loadExecution, HICDatabaseConfiguration configuration)
{
_repositoryLocator = repositoryLocator;
LoadMetadata = loadMetadata;
DataLoadEventListener = dataLoadEventListener;
_configuration = configuration;
LoadExecution = loadExecution;
_preExecutionChecker = preExecutionChecker;
LogManager = logManager;
ExitCode = ExitCodeType.Success;
JobProvider = new JobFactory(loadMetadata, logManager);
}
public HICDataLoadFactory(ILoadMetadata loadMetadata, HICDatabaseConfiguration databaseConfiguration, HICLoadConfigurationFlags loadConfigurationFlags, ICatalogueRepository repository, ILogManager logManager)
{
_databaseConfiguration = databaseConfiguration;
_loadConfigurationFlags = loadConfigurationFlags;
_repository = repository;
_logManager = logManager;
LoadMetadata = loadMetadata;
// If we are not supplied any catalogues to load, it is expected that we will load all catalogues associated with the provided ILoadMetadata
_cataloguesToLoad = LoadMetadata.GetAllCatalogues().ToList();
if (!_cataloguesToLoad.Any())
{
throw new InvalidOperationException("LoadMetadata " + LoadMetadata.ID + " is not related to any Catalogues, there is nothing to load");
}
}
public void Test_IsolateSingleTable_Dupliction(DatabaseType dbType)
{
var db = GetCleanedServer(dbType);
//Create a table in 'RAW' (has no constraints)
var dt = new DataTable();
dt.Columns.Add("A");
dt.Columns.Add("B");
dt.Rows.Add("Fish", 1);
dt.Rows.Add("Fish", 2);
dt.Rows.Add("Fish", 3);
dt.Rows.Add("Frank", 2);
dt.Rows.Add("Candy", 2);
var tbl = db.CreateTable("MyCoolTable2", dt);
//import the table and make A look like a primary key to the metadata layer (and A would be pk in LIVE but not in RAW ofc)
TableInfo tableInfoCreated;
ColumnInfo[] columnInfosCreated;
Import(tbl, out tableInfoCreated, out columnInfosCreated);
//lie abot the primary key status
var a = columnInfosCreated.Single(c => c.GetRuntimeName().Equals("A"));
a.IsPrimaryKey = true;
a.SaveToDatabase();
var mutilator = GetMutilator(db, tableInfoCreated);
mutilator.Check(new AcceptAllCheckNotifier());
var config = new HICDatabaseConfiguration(db.Server, RdmpMockFactory.Mock_INameDatabasesAndTablesDuringLoads(db, "MyCoolTable2"));
var job = new ThrowImmediatelyDataLoadJob(config, tableInfoCreated);
mutilator.Initialize(db, LoadStage.AdjustRaw);
mutilator.Mutilate(job);
dt = tbl.GetDataTable();
Assert.AreEqual(2, dt.Rows.Count);
var dtIsolation = tbl.Database.ExpectTable("MyCoolTable2_Isolation").GetDataTable();
Assert.AreEqual(3, dtIsolation.Rows.Count);
}
public void RefreshUIFromDatabase()
{
tlvLoadedTables.ClearObjects();
if (_loadMetadata == null)
{
return;
}
TableInfo[] allTables;
HICDatabaseConfiguration config;
try
{
if (!_loadMetadata.GetAllCatalogues().Any())
{
throw new Exception("There are no Catalogues (Datasets) associated with this LoadMetadata, choose one or more Catalogues by clicking 'Edit..' in LoadMetadataUI ");
}
allTables = _loadMetadata.GetDistinctTableInfoList(true).ToArray();
config = new HICDatabaseConfiguration(_loadMetadata);
}
catch (Exception e)
{
CommonFunctionality.Fatal("Could not fetch data", e);
tlvLoadedTables.Visible = false;
return;
}
tlvLoadedTables.Visible = true;
_raw = new LoadDiagramServerNode(LoadBubble.Raw, config.DeployInfo[LoadBubble.Raw], allTables, config);
var staging = new LoadDiagramServerNode(LoadBubble.Staging, config.DeployInfo[LoadBubble.Staging], allTables, config);
var live = new LoadDiagramServerNode(LoadBubble.Live, config.DeployInfo[LoadBubble.Live], allTables, config);
tlvLoadedTables.AddObject(_raw);
tlvLoadedTables.AddObject(staging);
tlvLoadedTables.AddObject(live);
//expand the servers
foreach (var rootObject in tlvLoadedTables.Objects)
{
ExpandToDepth(2, rootObject);
}
loadStateUI1.SetStatus(LoadStateUI.LoadState.Unknown);
}
public void ExecuteSqlFileRuntimeTask_InvalidID(DatabaseType dbType)
{
var dt = new DataTable();
dt.Columns.Add("Lawl");
dt.Rows.Add(new object[] { 2 });
var db = GetCleanedServer(dbType, true);
var tbl = db.CreateTable("Fish", dt);
TableInfo ti;
ColumnInfo[] cols;
Import(tbl, out ti, out cols);
FileInfo f = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, "Bob.sql"));
File.WriteAllText(f.FullName, @"UPDATE {T:0} Set {C:0} = 1");
var pt = Mock.Of <IProcessTask>(x => x.Path == f.FullName);
var dir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "ExecuteSqlFileRuntimeTaskTests", true);
var task = new ExecuteSqlFileRuntimeTask(pt, new RuntimeArgumentCollection(new IArgument[0], new StageArgs(LoadStage.AdjustRaw, db, dir)));
task.Check(new ThrowImmediatelyCheckNotifier());
HICDatabaseConfiguration configuration = new HICDatabaseConfiguration(db.Server);
IDataLoadJob job = Mock.Of <IDataLoadJob>(x =>
x.RegularTablesToLoad == new List <ITableInfo> {
ti
} &&
x.LookupTablesToLoad == new List <ITableInfo>() &&
x.Configuration == configuration);
var ex = Assert.Throws <ExecuteSqlFileRuntimeTaskException>(() => task.Run(job, new GracefulCancellationToken()));
StringAssert.Contains("Failed to find a TableInfo in the load with ID 0", ex.Message);
task.LoadCompletedSoDispose(Core.DataLoad.ExitCodeType.Success, new ThrowImmediatelyDataLoadEventListener());
}
public DataLoadJob(IRDMPPlatformRepositoryServiceLocator repositoryLocator, string description, ILogManager logManager, ILoadMetadata loadMetadata, ILoadDirectory directory, IDataLoadEventListener listener, HICDatabaseConfiguration configuration)
{
_logManager = logManager;
RepositoryLocator = repositoryLocator;
LoadMetadata = loadMetadata;
LoadDirectory = directory;
Configuration = configuration;
_listener = listener;
Description = description;
List <ICatalogue> catalogues = LoadMetadata.GetAllCatalogues().ToList();
if (LoadMetadata != null)
{
_loggingTask = GetLoggingTask(catalogues);
}
RegularTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetTableInfoList(false)).Distinct().ToList();
LookupTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetLookupTableInfoList()).Distinct().ToList();
}
public void TestHICDatabaseConfiguration_ExpectTables(bool testLookup)
{
var conf = new HICDatabaseConfiguration(new DiscoveredServer("localhost", "mydb",
DatabaseType.MicrosoftSQLServer, null, null), new FixedStagingDatabaseNamer("mydb"));
var ti = WhenIHaveA <TableInfo>();
var lookup = WhenIHaveA <TableInfo>();
lookup.Name = "MyHeartyLookup";
lookup.Database = "LookupsDb";
lookup.SaveToDatabase();
var job = Mock.Of <IDataLoadJob>(m =>
m.RegularTablesToLoad == new List <ITableInfo>(new [] { ti }) &&
m.LookupTablesToLoad == new List <ITableInfo>(new [] { lookup }));
var result = conf.ExpectTables(job, LoadBubble.Raw, testLookup).ToArray();
Assert.AreEqual(testLookup ? 2 : 1, result.Length);
StringAssert.AreEqualIgnoringCase("mydb_RAW", result[0].Database.GetRuntimeName());
StringAssert.AreEqualIgnoringCase("My_Table", result[0].GetRuntimeName());
if (testLookup)
{
StringAssert.AreEqualIgnoringCase("mydb_RAW", result[1].Database.GetRuntimeName());
StringAssert.AreEqualIgnoringCase("MyHeartyLookup", result[1].GetRuntimeName());
}
result = conf.ExpectTables(job, LoadBubble.Staging, testLookup).ToArray();
Assert.AreEqual(testLookup ? 2 : 1, result.Length);
StringAssert.AreEqualIgnoringCase("DLE_STAGING", result[0].Database.GetRuntimeName());
StringAssert.AreEqualIgnoringCase("mydb_My_Table_STAGING", result[0].GetRuntimeName());
if (testLookup)
{
StringAssert.AreEqualIgnoringCase("DLE_STAGING", result[1].Database.GetRuntimeName());
StringAssert.AreEqualIgnoringCase("mydb_MyHeartyLookup_STAGING", result[1].GetRuntimeName());
}
}
public void Test_CloneTable()
{
var dt = new DataTable();
dt.Columns.Add("FF");
var db = GetCleanedServer(FAnsi.DatabaseType.MicrosoftSQLServer);
var tbl = db.CreateTable("MyTable", dt);
Import(tbl, out var ti, out _);
var config = new HICDatabaseConfiguration(tbl.Database.Server);
//create a RAW table schema called TableName_Isolation
var cloner = new TableInfoCloneOperation(config, (TableInfo)ti, LoadBubble.Live, new ThrowImmediatelyDataLoadEventListener());
cloner.CloneTable(tbl.Database, tbl.Database, tbl, tbl.GetRuntimeName() + "_copy", true, true, true, ti.PreLoadDiscardedColumns);
var tbl2 = tbl.Database.ExpectTable(tbl.GetRuntimeName() + "_copy");
Assert.IsTrue(tbl2.Exists());
}
public void TestPayloadInjection()
{
BulkTestsData b = new BulkTestsData(CatalogueRepository, DiscoveredDatabaseICanCreateRandomTablesIn, 10);
b.SetupTestData();
b.ImportAsCatalogue();
var lmd = new LoadMetadata(CatalogueRepository, "Loading");
lmd.LocationOfFlatFiles = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "delme", true).RootPath.FullName;
lmd.SaveToDatabase();
CatalogueRepository.MEF.AddTypeToCatalogForTesting(typeof(TestPayloadAttacher));
b.catalogue.LoadMetadata_ID = lmd.ID;
b.catalogue.LoggingDataTask = "TestPayloadInjection";
b.catalogue.SaveToDatabase();
var lm = new LogManager(new ServerDefaults(CatalogueRepository).GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID));
lm.CreateNewLoggingTaskIfNotExists("TestPayloadInjection");
var pt = new ProcessTask(CatalogueRepository, lmd, LoadStage.Mounting);
pt.Path = typeof(TestPayloadAttacher).FullName;
pt.ProcessTaskType = ProcessTaskType.Attacher;
pt.SaveToDatabase();
var config = new HICDatabaseConfiguration(DiscoveredDatabaseICanCreateRandomTablesIn.Server);
var factory = new HICDataLoadFactory(lmd, config, new HICLoadConfigurationFlags(), CatalogueRepository, lm);
IDataLoadExecution execution = factory.Create(new ThrowImmediatelyDataLoadEventListener());
var proceedure = new DataLoadProcess(RepositoryLocator, lmd, null, lm, new ThrowImmediatelyDataLoadEventListener(), execution, config);
proceedure.Run(new GracefulCancellationToken(), payload);
Assert.IsTrue(PayloadTest.Success, "Expected IAttacher to detect Payload and set this property to true");
}
public LoadDiagramTableNode(LoadDiagramDatabaseNode databaseNode, TableInfo tableInfo, LoadBubble bubble, HICDatabaseConfiguration config)
{
_databaseNode = databaseNode;
TableInfo = tableInfo;
Bubble = bubble;
_config = config;
State = LoadDiagramState.Anticipated;
TableName = TableInfo.GetRuntimeName(Bubble);
//only reference schema if it is LIVE
string schema = bubble >= LoadBubble.Live ? tableInfo.Schema: null;
Table = databaseNode.Database.ExpectTable(TableName, schema);
var cols =
TableInfo.GetColumnsAtStage(Bubble.ToLoadStage())
.Select(c => new LoadDiagramColumnNode(this, c, Bubble));
_anticipatedChildren.AddRange(cols);
}
private void RunAttachStageWithNormalJob(RemoteTableAttacher attacher, DiscoveredDatabase db)
{
//the table to get data from
attacher.RemoteTableName = "table1";
attacher.RAWTableName = "table2";
attacher.Check(new ThrowImmediatelyCheckNotifier());
attacher.Initialize(null, db);
using (var dt = new DataTable())
{
dt.Columns.Add("Col1");
dt.Rows.Add("fff");
var tbl1 = db.CreateTable("table1", dt);
var tbl2 = db.CreateTable("table2", new [] { new DatabaseColumnRequest("Col1", new DatabaseTypeRequest(typeof(string), 5)) });
Assert.AreEqual(1, tbl1.GetRowCount());
Assert.AreEqual(0, tbl2.GetRowCount());
var logManager = new LogManager(new DiscoveredServer(UnitTestLoggingConnectionString));
var lmd = RdmpMockFactory.Mock_LoadMetadataLoadingTable(tbl2);
Mock.Get(lmd).Setup(p => p.CatalogueRepository).Returns(CatalogueRepository);
logManager.CreateNewLoggingTaskIfNotExists(lmd.GetDistinctLoggingTask());
var dbConfiguration = new HICDatabaseConfiguration(lmd, RdmpMockFactory.Mock_INameDatabasesAndTablesDuringLoads(db, "table2"));
var job = new DataLoadJob(RepositoryLocator, "test job", logManager, lmd, new TestLoadDirectory(), new ThrowImmediatelyDataLoadEventListener(), dbConfiguration);
job.StartLogging();
attacher.Attach(job, new GracefulCancellationToken());
Assert.AreEqual(1, tbl1.GetRowCount());
Assert.AreEqual(1, tbl2.GetRowCount());
}
}
// todo: refactor to cut down on ctor params
public IterativeScheduledDataLoadProcess(IRDMPPlatformRepositoryServiceLocator repositoryLocator, ILoadMetadata loadMetadata, ICheckable preExecutionChecker, IDataLoadExecution loadExecution, JobDateGenerationStrategyFactory jobDateGenerationStrategyFactory, ILoadProgressSelectionStrategy loadProgressSelectionStrategy, int?overrideNumberOfDaysToLoad, ILogManager logManager, IDataLoadEventListener dataLoadEventsreceiver, HICDatabaseConfiguration configuration)
: base(repositoryLocator, loadMetadata, preExecutionChecker, loadExecution, jobDateGenerationStrategyFactory, loadProgressSelectionStrategy, overrideNumberOfDaysToLoad, logManager, dataLoadEventsreceiver, configuration)
{
}
public DatabaseCloner(HICDatabaseConfiguration hicDatabaseConfiguration)
{
_hicDatabaseConfiguration = hicDatabaseConfiguration;
_tablesCreated = new List <TableInfoCloneOperation>();
_databasesCreated = new List <DiscoveredDatabase>();
}
public void TestTemporalTable(bool ignoreWithGlobalPattern)
{
var dbtype = FAnsi.DatabaseType.MicrosoftSQLServer;
var db = GetCleanedServer(dbtype);
using (var con = db.Server.GetConnection())
{
con.Open();
db.Server.GetCommand(sql, con).ExecuteNonQuery();
}
var tbl = db.ExpectTable("Employee");
var defaults = new ServerDefaults(CatalogueRepository);
var logServer = defaults.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID);
var logManager = new LogManager(logServer);
var raw = db.Server.ExpectDatabase(db.GetRuntimeName() + "_RAW");
if (raw.Exists())
{
raw.Drop();
}
//define a new load configuration
var lmd = new LoadMetadata(CatalogueRepository, "MyLoad");
lmd.IgnoreTrigger = true;
lmd.SaveToDatabase();
ITableInfo ti = Import(tbl, lmd, logManager);
var projectDirectory = SetupLoadDirectory(lmd);
CreateCSVProcessTask(lmd, ti, "*.csv");
//create a text file to load where we update Frank's favourite colour (it's a pk field) and we insert a new record (MrMurder)
File.WriteAllText(
Path.Combine(projectDirectory.ForLoading.FullName, "LoadMe.csv"),
@"EmployeeID,Name,Position,Department,Address,AnnualSalary
1,Frank,Boss,Department of F'Tang, 22 Innsmouth Way, 55000.5
2,Herbert,Super Boss,Department of F'Tang, 22 Innsmouth Way, 155000.5");
//the checks will probably need to be run as ddl admin because it involves creating _Archive table and trigger the first time
//clean SetUp RAW / STAGING etc and generally accept proposed cleanup operations
var checker = new CheckEntireDataLoadProcess(lmd, new HICDatabaseConfiguration(lmd), new HICLoadConfigurationFlags(), CatalogueRepository.MEF);
checker.Check(new AcceptAllCheckNotifier());
if (ignoreWithGlobalPattern)
{
var regex = new StandardRegex(RepositoryLocator.CatalogueRepository)
{
ConceptName = StandardRegex.DataLoadEngineGlobalIgnorePattern,
Regex = "^Valid((From)|(To))$"
};
regex.SaveToDatabase();
}
else
{
var col = ti.ColumnInfos.Single(c => c.GetRuntimeName().Equals("ValidFrom"));
col.IgnoreInLoads = true;
col.SaveToDatabase();
col = ti.ColumnInfos.Single(c => c.GetRuntimeName().Equals("ValidTo"));
col.IgnoreInLoads = true;
col.SaveToDatabase();
}
var dbConfig = new HICDatabaseConfiguration(lmd, null);
var loadFactory = new HICDataLoadFactory(
lmd,
dbConfig,
new HICLoadConfigurationFlags(),
CatalogueRepository,
logManager
);
var exe = loadFactory.Create(new ThrowImmediatelyDataLoadEventListener());
var exitCode = exe.Run(
new DataLoadJob(RepositoryLocator, "Go go go!", logManager, lmd, projectDirectory, new ThrowImmediatelyDataLoadEventListener(), dbConfig),
new GracefulCancellationToken());
Assert.AreEqual(ExitCodeType.Success, exitCode);
//frank should be updated to his new departement and role
Assert.AreEqual(2, tbl.GetRowCount());
var result = tbl.GetDataTable();
var frank = result.Rows.Cast <DataRow>().Single(r => (string)r["Name"] == "Frank");
Assert.AreEqual("Department of F'Tang", frank["Department"]);
Assert.AreEqual("Boss", frank["Position"]);
//post test cleanup
foreach (var regex in RepositoryLocator.CatalogueRepository.GetAllObjects <StandardRegex>())
{
regex.DeleteInDatabase();
}
}
public abstract IDataLoadJob Create(IRDMPPlatformRepositoryServiceLocator repositoryLocator, IDataLoadEventListener listener, HICDatabaseConfiguration configuration);
// This no longer copies between servers, but the original test didn't guarantee that would happen anyway
public void CloneDatabaseAndTable()
{
string testLiveDatabaseName = TestDatabaseNames.GetConsistentName("TEST");
var testDb = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(testLiveDatabaseName);
var raw = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(testLiveDatabaseName + "_RAW");
foreach (DiscoveredDatabase db in new[] { raw, testDb })
{
if (db.Exists())
{
foreach (DiscoveredTable table in db.DiscoverTables(true))
{
table.Drop();
}
db.Drop();
}
}
DiscoveredServerICanCreateRandomDatabasesAndTablesOn.CreateDatabase(testLiveDatabaseName);
Assert.IsTrue(testDb.Exists());
testDb.CreateTable("Table_1", new[] { new DatabaseColumnRequest("Id", "int") });
//clone the builder
var builder = new SqlConnectionStringBuilder(DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Builder.ConnectionString)
{
InitialCatalog = testLiveDatabaseName
};
var dbConfiguration = new HICDatabaseConfiguration(new DiscoveredServer(builder), null, new ServerDefaults(CatalogueRepository));
var cloner = new DatabaseCloner(dbConfiguration);
try
{
var cloneDb = cloner.CreateDatabaseForStage(LoadBubble.Raw);
//confirm database appeared
Assert.IsTrue(DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(testLiveDatabaseName + "_RAW").Exists());
//now create a catalogue and wire it SetUp to the table TEST on the test database server
Catalogue cata = SetupATestCatalogue(builder, testLiveDatabaseName, "Table_1");
//now clone the catalogue data structures to MachineName
foreach (TableInfo tableInfo in cata.GetTableInfoList(false))
{
cloner.CreateTablesInDatabaseFromCatalogueInfo(new ThrowImmediatelyDataLoadEventListener(), tableInfo, LoadBubble.Raw);
}
Assert.IsTrue(raw.Exists());
Assert.IsTrue(raw.ExpectTable("Table_1").Exists());
}
finally
{
cloner.LoadCompletedSoDispose(ExitCodeType.Success, new ThrowImmediatelyDataLoadEventListener());
while (toCleanUp.Count > 0)
{
try
{
toCleanUp.Pop().DeleteInDatabase();
}
catch (Exception e)
{
//always clean SetUp everything
Console.WriteLine(e);
}
}
}
}
public PopulateRAW(List <IRuntimeTask> collection, HICDatabaseConfiguration databaseConfiguration) : base(collection.Cast <IDataLoadComponent>().ToList())
{
_databaseConfiguration = databaseConfiguration;
Description = "Populate RAW";
}
