public MigrationConfiguration(DiscoveredDatabase fromDatabaseInfo, LoadBubble fromBubble, LoadBubble toBubble, INameDatabasesAndTablesDuringLoads namer)
{
_fromDatabaseInfo = fromDatabaseInfo;
_fromBubble = fromBubble;
_toBubble = toBubble;
_namer = namer;
}
// Indexer declaration.
// If index is out of range, the temps array will throw the exception.
public DiscoveredDatabase this[LoadBubble index]
{
get
{
return(DatabaseInfoList[index]);
}
}
public LoadDiagramServerNode(LoadBubble bubble, DiscoveredDatabase database, TableInfo[] loadTables, HICDatabaseConfiguration config) : base(database.Server.Name, database.Server.DatabaseType)
{
_bubble = bubble;
_database = database;
_loadTables = loadTables;
_config = config;
string serverName = database.Server.Name;
switch (bubble)
{
case LoadBubble.Raw:
_description = "RAW Server:" + serverName;
break;
case LoadBubble.Staging:
_description = "STAGING Server:" + serverName;
break;
case LoadBubble.Live:
_description = "LIVE Server:" + serverName;
break;
default:
throw new ArgumentOutOfRangeException("bubble");
}
//Live can have multiple databases (for lookups)
if (_bubble == LoadBubble.Live)
{
var servers = loadTables.Select(t => t.Server).Distinct().ToArray();
if (servers.Length > 1)
{
_description = "Ambiguous LIVE Servers:" + string.Join(",", servers);
ErrorDescription = "The TableInfo collection that underly the Catalogues in this data load configuration are on different servers. The servers they believe they live on are:" + string.Join(",", servers) + ". All TableInfos in a load must belong on the same server or the load will not work.";
}
string[] databases = _loadTables.Select(t => t.GetDatabaseRuntimeName()).Distinct().ToArray();
_liveDatabaseDictionary = new Dictionary <DiscoveredDatabase, TableInfo[]>();
foreach (string dbname in databases)
{
_liveDatabaseDictionary.Add(_database.Server.ExpectDatabase(dbname), _loadTables.Where(t => t.GetDatabaseRuntimeName().Equals(dbname, StringComparison.CurrentCultureIgnoreCase)).ToArray());
}
}
//if it is live yield all the lookups
if (_bubble == LoadBubble.Live)
{
foreach (var kvp in _liveDatabaseDictionary)
{
Children.Add(new LoadDiagramDatabaseNode(_bubble, kvp.Key, kvp.Value, _config));
}
}
else
{
Children.Add(new LoadDiagramDatabaseNode(_bubble, _database, _loadTables, _config));
}
}
public TableInfoCloneOperation(HICDatabaseConfiguration hicDatabaseConfiguration, TableInfo tableInfo, LoadBubble copyToBubble)
{
_hicDatabaseConfiguration = hicDatabaseConfiguration;
_tableInfo = tableInfo;
_copyToBubble = copyToBubble;
DropIdentityColumns = true;
}
public TableInfoCloneOperation(HICDatabaseConfiguration hicDatabaseConfiguration, TableInfo tableInfo, LoadBubble copyToBubble, IDataLoadEventListener listener)
{
_hicDatabaseConfiguration = hicDatabaseConfiguration;
_tableInfo = tableInfo;
_copyToBubble = copyToBubble;
this._listener = listener;
DropIdentityColumns = true;
}
/// <inheritdoc/>
public virtual string GetName(string tableName, LoadBubble convention)
{
if (!Suffixes.ContainsKey(convention))
{
throw new ArgumentException("Do not have a suffix for convention: " + convention);
}
return(tableName + Suffixes[convention]);
}
/// <inheritdoc/>
public override string GetDatabaseName(string rootDatabaseName, LoadBubble stage)
{
if (stage == LoadBubble.Staging)
{
return(_stagingDatabaseName);
}
return(base.GetDatabaseName(rootDatabaseName, stage));
}
/// <inheritdoc/>
public override string GetName(string tableName, LoadBubble convention)
{
if (convention == LoadBubble.Staging)
{
return(_databaseName + "_" + tableName + Suffixes[convention]);
}
return(base.GetName(tableName, convention));
}
public string GetName(string tableName, LoadBubble convention)
{
//all tables get called CC
if (convention < LoadBubble.Live)
{
return("CC");
}
return(tableName);
}
public override string GetName(string tableName, LoadBubble convention)
{
var basic = base.GetName(tableName, convention);
if (convention == LoadBubble.Live || convention == LoadBubble.Archive)
{
return(basic);
}
return("t" + _guid.Replace("-", "") + basic);
}
/// <summary>
/// Prefixes STAGING and RAW databases with the guid
/// </summary>
/// <param name="rootDatabaseName"></param>
/// <param name="stage"></param>
/// <returns></returns>
public override string GetDatabaseName(string rootDatabaseName, LoadBubble stage)
{
var basic = base.GetDatabaseName(rootDatabaseName, stage);
if (stage == LoadBubble.Live || stage == LoadBubble.Archive)
{
return(basic);
}
return("t" + _guid.Replace("-", "") + basic);
}
public LoadDiagramDatabaseNode(LoadBubble bubble, DiscoveredDatabase database, TableInfo[] loadTables, HICDatabaseConfiguration config)
{
_bubble = bubble;
Database = database;
_loadTables = loadTables;
_config = config;
DatabaseName = Database.GetRuntimeName();
_anticipatedChildren.AddRange(_loadTables.Select(t => new LoadDiagramTableNode(this, t, _bubble, _config)));
}
/// <inheritdoc/>
public string GetRuntimeName(LoadBubble bubble, INameDatabasesAndTablesDuringLoads tableNamingScheme = null)
{
// If no naming scheme is specified, the default 'FixedStaging...' prepends the database name and appends '_STAGING'
if (tableNamingScheme == null)
{
tableNamingScheme = new FixedStagingDatabaseNamer(Database);
}
string baseName = GetQuerySyntaxHelper().GetRuntimeName(Name);
return(tableNamingScheme.GetName(baseName, bubble));
}
public void CreateTablesInStage(DatabaseCloner cloner, LoadBubble stage)
{
foreach (TableInfo regularTableInfo in RegularTablesToLoad)
{
cloner.CreateTablesInDatabaseFromCatalogueInfo(_listener, regularTableInfo, stage);
}
foreach (TableInfo lookupTableInfo in LookupTablesToLoad)
{
cloner.CreateTablesInDatabaseFromCatalogueInfo(_listener, lookupTableInfo, stage);
}
PushForDisposal(cloner);
}
public DiscoveredDatabase CreateDatabaseForStage(LoadBubble stageToCreateDatabase)
{
if (stageToCreateDatabase == LoadBubble.Live)
{
throw new Exception("Please don't try to create databases on the live server");
}
var dbInfo = _hicDatabaseConfiguration.DeployInfo[stageToCreateDatabase];
dbInfo.Server.CreateDatabase(dbInfo.GetRuntimeName());
_databasesCreated.Add(dbInfo);
return(dbInfo);
}
/// <summary>
/// Returns all tables in the load as they would be named in the given <paramref name="stage"/>
/// </summary>
/// <param name="job">DLE job</param>
/// <param name="stage"></param>
/// <param name="includeLookups"></param>
/// <returns></returns>
public IEnumerable <DiscoveredTable> ExpectTables(IDataLoadJob job, LoadBubble stage, bool includeLookups)
{
var db = DeployInfo.DatabaseInfoList[stage];
foreach (ITableInfo t in job.RegularTablesToLoad)
{
yield return(db.ExpectTable(t.GetRuntimeName(stage, DatabaseNamer)));
}
if (includeLookups)
{
foreach (ITableInfo t in job.LookupTablesToLoad)
{
yield return(db.ExpectTable(t.GetRuntimeName(stage, DatabaseNamer)));
}
}
}
/// <inheritdoc/>
public virtual string GetDatabaseName(string rootDatabaseName, LoadBubble stage)
{
switch (stage)
{
case LoadBubble.Raw:
return(rootDatabaseName + "_RAW");
case LoadBubble.Staging:
return(rootDatabaseName + "_STAGING");
case LoadBubble.Live:
return(rootDatabaseName);
default:
throw new ArgumentOutOfRangeException("stage");
}
}
public string GetDatabaseName(string rootDatabaseName, LoadBubble convention)
{
//RAW is AA, Staging is BB
switch (convention)
{
case LoadBubble.Raw:
return("AA_RAW");
case LoadBubble.Staging:
return("BB_STAGING");
case LoadBubble.Live:
case LoadBubble.Archive:
return(rootDatabaseName);
default:
throw new ArgumentOutOfRangeException("convention");
}
}
private void CheckDatabaseExistsForStage(LoadBubble deploymentStage, string successMessage, string failureMessage)
{
var dbInfo = _databaseConfiguration.DeployInfo[deploymentStage];
if (!dbInfo.Exists())
{
var createDatabase = _notifier.OnCheckPerformed(new CheckEventArgs(failureMessage + ": " + dbInfo, CheckResult.Fail, null, "Create " + dbInfo.GetRuntimeName() + " on " + dbInfo.Server.Name));
if (createDatabase)
{
dbInfo.Server.CreateDatabase(dbInfo.GetRuntimeName());
}
}
else
{
_notifier.OnCheckPerformed(new CheckEventArgs(successMessage + ": " + dbInfo, CheckResult.Success, null));
}
}
/// <summary>
/// Converts a <see cref="LoadBubble"/> into a <see cref="LoadStage"/>
/// </summary>
/// <param name="bubble"></param>
/// <returns></returns>
public static LoadStage ToLoadStage(this LoadBubble bubble)
{
switch (bubble)
{
case LoadBubble.Raw:
return(LoadStage.AdjustRaw);
case LoadBubble.Staging:
return(LoadStage.AdjustStaging);
case LoadBubble.Live:
return(LoadStage.PostLoad);
case LoadBubble.Archive:
throw new Exception("LoadBubble.Archive refers to _Archive tables, therefore it cannot be translated into a LoadStage");
default:
throw new ArgumentOutOfRangeException("bubble");
}
}
public LoadDiagramTableNode(LoadDiagramDatabaseNode databaseNode, TableInfo tableInfo, LoadBubble bubble, HICDatabaseConfiguration config)
{
_databaseNode = databaseNode;
TableInfo = tableInfo;
Bubble = bubble;
_config = config;
State = LoadDiagramState.Anticipated;
TableName = TableInfo.GetRuntimeName(Bubble);
//only reference schema if it is LIVE
string schema = bubble >= LoadBubble.Live ? tableInfo.Schema: null;
Table = databaseNode.Database.ExpectTable(TableName, schema);
var cols =
TableInfo.GetColumnsAtStage(Bubble.ToLoadStage())
.Select(c => new LoadDiagramColumnNode(this, c, Bubble));
_anticipatedChildren.AddRange(cols);
}
public LoadDiagramColumnNode(LoadDiagramTableNode tableNode, IHasStageSpecificRuntimeName column, LoadBubble bubble)
{
_tableNode = tableNode;
_column = column;
_bubble = bubble;
ColumnName = _column.GetRuntimeName(_bubble.ToLoadStage());
var colInfo = _column as ColumnInfo;
var preLoadDiscarded = _column as PreLoadDiscardedColumn;
if (preLoadDiscarded != null)
{
_expectedDataType = preLoadDiscarded.SqlDataType;
}
else
if (colInfo != null)
{
_expectedDataType = colInfo.GetRuntimeDataType(_bubble.ToLoadStage());
}
else
{
throw new Exception("Expected _column to be ColumnInfo or PreLoadDiscardedColumn but it was:" + _column.GetType().Name);
}
}
public void CreateTablesInDatabaseFromCatalogueInfo(IDataLoadEventListener listener, TableInfo tableInfo, LoadBubble copyToStage)
{
if (copyToStage == LoadBubble.Live)
{
throw new Exception("Please don't try to create tables in the live database");
}
var destDbInfo = _hicDatabaseConfiguration.DeployInfo[copyToStage];
var cloneOperation = new TableInfoCloneOperation(_hicDatabaseConfiguration, tableInfo, copyToStage)
{
DropHICColumns = copyToStage == LoadBubble.Raw,//don't drop columns like hic_sourceID, these are optional for population (and don't get Diff'ed) but should still be there
AllowNulls = copyToStage == LoadBubble.Raw
};
cloneOperation.Execute();
_tablesCreated.Add(cloneOperation);
if (copyToStage == LoadBubble.Raw)
{
var tableName = tableInfo.GetRuntimeName(copyToStage, _hicDatabaseConfiguration.DatabaseNamer);
var table = destDbInfo.ExpectTable(tableName);
string[] existingColumns = tableInfo.ColumnInfos.Select(c => c.GetRuntimeName(LoadStage.AdjustRaw)).ToArray();
foreach (PreLoadDiscardedColumn preLoadDiscardedColumn in tableInfo.PreLoadDiscardedColumns)
{
//this column does not get dropped so will be in live TableInfo
if (preLoadDiscardedColumn.Destination == DiscardedColumnDestination.Dilute)
{
continue;
}
if (existingColumns.Any(e => e.Equals(preLoadDiscardedColumn.GetRuntimeName(LoadStage.AdjustRaw))))
{
throw new Exception("There is a column called " + preLoadDiscardedColumn.GetRuntimeName(LoadStage.AdjustRaw) + " as both a PreLoadDiscardedColumn and in the TableInfo (live table), you should either drop the column from the live table or remove it as a PreLoadDiscarded column");
}
//add all the preload discarded columns because they could be routed to ANO store or sent to oblivion
AddColumnToTable(table, preLoadDiscardedColumn.RuntimeColumnName, preLoadDiscardedColumn.SqlDataType, listener);
}
//deal with anonymisation transforms e.g. ANOCHI of datatype varchar(12) would have to become a column called CHI of datatype varchar(10) on creation in RAW
var columnInfosWithANOTransforms = tableInfo.ColumnInfos.Where(c => c.ANOTable_ID != null).ToArray();
if (columnInfosWithANOTransforms.Any())
{
foreach (ColumnInfo col in columnInfosWithANOTransforms)
{
var liveName = col.GetRuntimeName(LoadStage.PostLoad);
var rawName = col.GetRuntimeName(LoadStage.AdjustRaw);
var rawDataType = col.GetRuntimeDataType(LoadStage.AdjustRaw);
DropColumnFromTable(table, liveName, listener);
AddColumnToTable(table, rawName, rawDataType, listener);
}
}
}
}
public void CreateTablesInStage(DatabaseCloner cloner, LoadBubble stage)
{
}
// Check that the column infos from the catalogue match up with what is actually in the staging databases
private void CheckColumnInfosMatchWithWhatIsInDatabaseAtStage(IEnumerable <ITableInfo> allTableInfos, LoadBubble deploymentStage)
{
var dbInfo = _databaseConfiguration.DeployInfo[deploymentStage];
foreach (var tableInfo in allTableInfos)
{
var columnNames = tableInfo.ColumnInfos.Select(info => info.GetRuntimeName()).ToList();
if (!columnNames.Any())
{
_notifier.OnCheckPerformed(new CheckEventArgs("Table '" + tableInfo.GetRuntimeName() + "' has no ColumnInfos", CheckResult.Fail, null));
}
string tableName = tableInfo.GetRuntimeName(deploymentStage, _databaseConfiguration.DatabaseNamer);
var table = dbInfo.ExpectTable(tableName);
if (!table.Exists())
{
throw new Exception("PreExecutionChecker spotted that table does not exist:" + table + " it was about to check whether the TableInfo matched the columns or not");
}
}
}
public string GetDatabaseName(string rootDatabaseName, LoadBubble convention)
{
return(rootDatabaseName);
}
public string GetName(string tableName, LoadBubble convention)
{
return(tableName);
}
