public MigrationConfiguration(DiscoveredDatabase fromDatabaseInfo, LoadBubble fromBubble, LoadBubble toBubble, INameDatabasesAndTablesDuringLoads namer) { _fromDatabaseInfo = fromDatabaseInfo; _fromBubble = fromBubble; _toBubble = toBubble; _namer = namer; }
// Indexer declaration. // If index is out of range, the temps array will throw the exception. public DiscoveredDatabase this[LoadBubble index] { get { return(DatabaseInfoList[index]); } }
public LoadDiagramServerNode(LoadBubble bubble, DiscoveredDatabase database, TableInfo[] loadTables, HICDatabaseConfiguration config) : base(database.Server.Name, database.Server.DatabaseType) { _bubble = bubble; _database = database; _loadTables = loadTables; _config = config; string serverName = database.Server.Name; switch (bubble) { case LoadBubble.Raw: _description = "RAW Server:" + serverName; break; case LoadBubble.Staging: _description = "STAGING Server:" + serverName; break; case LoadBubble.Live: _description = "LIVE Server:" + serverName; break; default: throw new ArgumentOutOfRangeException("bubble"); } //Live can have multiple databases (for lookups) if (_bubble == LoadBubble.Live) { var servers = loadTables.Select(t => t.Server).Distinct().ToArray(); if (servers.Length > 1) { _description = "Ambiguous LIVE Servers:" + string.Join(",", servers); ErrorDescription = "The TableInfo collection that underly the Catalogues in this data load configuration are on different servers. The servers they believe they live on are:" + string.Join(",", servers) + ". All TableInfos in a load must belong on the same server or the load will not work."; } string[] databases = _loadTables.Select(t => t.GetDatabaseRuntimeName()).Distinct().ToArray(); _liveDatabaseDictionary = new Dictionary <DiscoveredDatabase, TableInfo[]>(); foreach (string dbname in databases) { _liveDatabaseDictionary.Add(_database.Server.ExpectDatabase(dbname), _loadTables.Where(t => t.GetDatabaseRuntimeName().Equals(dbname, StringComparison.CurrentCultureIgnoreCase)).ToArray()); } } //if it is live yield all the lookups if (_bubble == LoadBubble.Live) { foreach (var kvp in _liveDatabaseDictionary) { Children.Add(new LoadDiagramDatabaseNode(_bubble, kvp.Key, kvp.Value, _config)); } } else { Children.Add(new LoadDiagramDatabaseNode(_bubble, _database, _loadTables, _config)); } }
public TableInfoCloneOperation(HICDatabaseConfiguration hicDatabaseConfiguration, TableInfo tableInfo, LoadBubble copyToBubble) { _hicDatabaseConfiguration = hicDatabaseConfiguration; _tableInfo = tableInfo; _copyToBubble = copyToBubble; DropIdentityColumns = true; }
public TableInfoCloneOperation(HICDatabaseConfiguration hicDatabaseConfiguration, TableInfo tableInfo, LoadBubble copyToBubble, IDataLoadEventListener listener) { _hicDatabaseConfiguration = hicDatabaseConfiguration; _tableInfo = tableInfo; _copyToBubble = copyToBubble; this._listener = listener; DropIdentityColumns = true; }
/// <inheritdoc/> public virtual string GetName(string tableName, LoadBubble convention) { if (!Suffixes.ContainsKey(convention)) { throw new ArgumentException("Do not have a suffix for convention: " + convention); } return(tableName + Suffixes[convention]); }
/// <inheritdoc/> public override string GetDatabaseName(string rootDatabaseName, LoadBubble stage) { if (stage == LoadBubble.Staging) { return(_stagingDatabaseName); } return(base.GetDatabaseName(rootDatabaseName, stage)); }
/// <inheritdoc/> public override string GetName(string tableName, LoadBubble convention) { if (convention == LoadBubble.Staging) { return(_databaseName + "_" + tableName + Suffixes[convention]); } return(base.GetName(tableName, convention)); }
public string GetName(string tableName, LoadBubble convention) { //all tables get called CC if (convention < LoadBubble.Live) { return("CC"); } return(tableName); }
public override string GetName(string tableName, LoadBubble convention) { var basic = base.GetName(tableName, convention); if (convention == LoadBubble.Live || convention == LoadBubble.Archive) { return(basic); } return("t" + _guid.Replace("-", "") + basic); }
/// <summary> /// Prefixes STAGING and RAW databases with the guid /// </summary> /// <param name="rootDatabaseName"></param> /// <param name="stage"></param> /// <returns></returns> public override string GetDatabaseName(string rootDatabaseName, LoadBubble stage) { var basic = base.GetDatabaseName(rootDatabaseName, stage); if (stage == LoadBubble.Live || stage == LoadBubble.Archive) { return(basic); } return("t" + _guid.Replace("-", "") + basic); }
public LoadDiagramDatabaseNode(LoadBubble bubble, DiscoveredDatabase database, TableInfo[] loadTables, HICDatabaseConfiguration config) { _bubble = bubble; Database = database; _loadTables = loadTables; _config = config; DatabaseName = Database.GetRuntimeName(); _anticipatedChildren.AddRange(_loadTables.Select(t => new LoadDiagramTableNode(this, t, _bubble, _config))); }
/// <inheritdoc/> public string GetRuntimeName(LoadBubble bubble, INameDatabasesAndTablesDuringLoads tableNamingScheme = null) { // If no naming scheme is specified, the default 'FixedStaging...' prepends the database name and appends '_STAGING' if (tableNamingScheme == null) { tableNamingScheme = new FixedStagingDatabaseNamer(Database); } string baseName = GetQuerySyntaxHelper().GetRuntimeName(Name); return(tableNamingScheme.GetName(baseName, bubble)); }
public void CreateTablesInStage(DatabaseCloner cloner, LoadBubble stage) { foreach (TableInfo regularTableInfo in RegularTablesToLoad) { cloner.CreateTablesInDatabaseFromCatalogueInfo(_listener, regularTableInfo, stage); } foreach (TableInfo lookupTableInfo in LookupTablesToLoad) { cloner.CreateTablesInDatabaseFromCatalogueInfo(_listener, lookupTableInfo, stage); } PushForDisposal(cloner); }
public DiscoveredDatabase CreateDatabaseForStage(LoadBubble stageToCreateDatabase) { if (stageToCreateDatabase == LoadBubble.Live) { throw new Exception("Please don't try to create databases on the live server"); } var dbInfo = _hicDatabaseConfiguration.DeployInfo[stageToCreateDatabase]; dbInfo.Server.CreateDatabase(dbInfo.GetRuntimeName()); _databasesCreated.Add(dbInfo); return(dbInfo); }
/// <summary> /// Returns all tables in the load as they would be named in the given <paramref name="stage"/> /// </summary> /// <param name="job">DLE job</param> /// <param name="stage"></param> /// <param name="includeLookups"></param> /// <returns></returns> public IEnumerable <DiscoveredTable> ExpectTables(IDataLoadJob job, LoadBubble stage, bool includeLookups) { var db = DeployInfo.DatabaseInfoList[stage]; foreach (ITableInfo t in job.RegularTablesToLoad) { yield return(db.ExpectTable(t.GetRuntimeName(stage, DatabaseNamer))); } if (includeLookups) { foreach (ITableInfo t in job.LookupTablesToLoad) { yield return(db.ExpectTable(t.GetRuntimeName(stage, DatabaseNamer))); } } }
/// <inheritdoc/> public virtual string GetDatabaseName(string rootDatabaseName, LoadBubble stage) { switch (stage) { case LoadBubble.Raw: return(rootDatabaseName + "_RAW"); case LoadBubble.Staging: return(rootDatabaseName + "_STAGING"); case LoadBubble.Live: return(rootDatabaseName); default: throw new ArgumentOutOfRangeException("stage"); } }
public string GetDatabaseName(string rootDatabaseName, LoadBubble convention) { //RAW is AA, Staging is BB switch (convention) { case LoadBubble.Raw: return("AA_RAW"); case LoadBubble.Staging: return("BB_STAGING"); case LoadBubble.Live: case LoadBubble.Archive: return(rootDatabaseName); default: throw new ArgumentOutOfRangeException("convention"); } }
private void CheckDatabaseExistsForStage(LoadBubble deploymentStage, string successMessage, string failureMessage) { var dbInfo = _databaseConfiguration.DeployInfo[deploymentStage]; if (!dbInfo.Exists()) { var createDatabase = _notifier.OnCheckPerformed(new CheckEventArgs(failureMessage + ": " + dbInfo, CheckResult.Fail, null, "Create " + dbInfo.GetRuntimeName() + " on " + dbInfo.Server.Name)); if (createDatabase) { dbInfo.Server.CreateDatabase(dbInfo.GetRuntimeName()); } } else { _notifier.OnCheckPerformed(new CheckEventArgs(successMessage + ": " + dbInfo, CheckResult.Success, null)); } }
/// <summary> /// Converts a <see cref="LoadBubble"/> into a <see cref="LoadStage"/> /// </summary> /// <param name="bubble"></param> /// <returns></returns> public static LoadStage ToLoadStage(this LoadBubble bubble) { switch (bubble) { case LoadBubble.Raw: return(LoadStage.AdjustRaw); case LoadBubble.Staging: return(LoadStage.AdjustStaging); case LoadBubble.Live: return(LoadStage.PostLoad); case LoadBubble.Archive: throw new Exception("LoadBubble.Archive refers to _Archive tables, therefore it cannot be translated into a LoadStage"); default: throw new ArgumentOutOfRangeException("bubble"); } }
public LoadDiagramTableNode(LoadDiagramDatabaseNode databaseNode, TableInfo tableInfo, LoadBubble bubble, HICDatabaseConfiguration config) { _databaseNode = databaseNode; TableInfo = tableInfo; Bubble = bubble; _config = config; State = LoadDiagramState.Anticipated; TableName = TableInfo.GetRuntimeName(Bubble); //only reference schema if it is LIVE string schema = bubble >= LoadBubble.Live ? tableInfo.Schema: null; Table = databaseNode.Database.ExpectTable(TableName, schema); var cols = TableInfo.GetColumnsAtStage(Bubble.ToLoadStage()) .Select(c => new LoadDiagramColumnNode(this, c, Bubble)); _anticipatedChildren.AddRange(cols); }
public LoadDiagramColumnNode(LoadDiagramTableNode tableNode, IHasStageSpecificRuntimeName column, LoadBubble bubble) { _tableNode = tableNode; _column = column; _bubble = bubble; ColumnName = _column.GetRuntimeName(_bubble.ToLoadStage()); var colInfo = _column as ColumnInfo; var preLoadDiscarded = _column as PreLoadDiscardedColumn; if (preLoadDiscarded != null) { _expectedDataType = preLoadDiscarded.SqlDataType; } else if (colInfo != null) { _expectedDataType = colInfo.GetRuntimeDataType(_bubble.ToLoadStage()); } else { throw new Exception("Expected _column to be ColumnInfo or PreLoadDiscardedColumn but it was:" + _column.GetType().Name); } }
public void CreateTablesInDatabaseFromCatalogueInfo(IDataLoadEventListener listener, TableInfo tableInfo, LoadBubble copyToStage) { if (copyToStage == LoadBubble.Live) { throw new Exception("Please don't try to create tables in the live database"); } var destDbInfo = _hicDatabaseConfiguration.DeployInfo[copyToStage]; var cloneOperation = new TableInfoCloneOperation(_hicDatabaseConfiguration, tableInfo, copyToStage) { DropHICColumns = copyToStage == LoadBubble.Raw,//don't drop columns like hic_sourceID, these are optional for population (and don't get Diff'ed) but should still be there AllowNulls = copyToStage == LoadBubble.Raw }; cloneOperation.Execute(); _tablesCreated.Add(cloneOperation); if (copyToStage == LoadBubble.Raw) { var tableName = tableInfo.GetRuntimeName(copyToStage, _hicDatabaseConfiguration.DatabaseNamer); var table = destDbInfo.ExpectTable(tableName); string[] existingColumns = tableInfo.ColumnInfos.Select(c => c.GetRuntimeName(LoadStage.AdjustRaw)).ToArray(); foreach (PreLoadDiscardedColumn preLoadDiscardedColumn in tableInfo.PreLoadDiscardedColumns) { //this column does not get dropped so will be in live TableInfo if (preLoadDiscardedColumn.Destination == DiscardedColumnDestination.Dilute) { continue; } if (existingColumns.Any(e => e.Equals(preLoadDiscardedColumn.GetRuntimeName(LoadStage.AdjustRaw)))) { throw new Exception("There is a column called " + preLoadDiscardedColumn.GetRuntimeName(LoadStage.AdjustRaw) + " as both a PreLoadDiscardedColumn and in the TableInfo (live table), you should either drop the column from the live table or remove it as a PreLoadDiscarded column"); } //add all the preload discarded columns because they could be routed to ANO store or sent to oblivion AddColumnToTable(table, preLoadDiscardedColumn.RuntimeColumnName, preLoadDiscardedColumn.SqlDataType, listener); } //deal with anonymisation transforms e.g. ANOCHI of datatype varchar(12) would have to become a column called CHI of datatype varchar(10) on creation in RAW var columnInfosWithANOTransforms = tableInfo.ColumnInfos.Where(c => c.ANOTable_ID != null).ToArray(); if (columnInfosWithANOTransforms.Any()) { foreach (ColumnInfo col in columnInfosWithANOTransforms) { var liveName = col.GetRuntimeName(LoadStage.PostLoad); var rawName = col.GetRuntimeName(LoadStage.AdjustRaw); var rawDataType = col.GetRuntimeDataType(LoadStage.AdjustRaw); DropColumnFromTable(table, liveName, listener); AddColumnToTable(table, rawName, rawDataType, listener); } } } }
public void CreateTablesInStage(DatabaseCloner cloner, LoadBubble stage) { }
// Check that the column infos from the catalogue match up with what is actually in the staging databases private void CheckColumnInfosMatchWithWhatIsInDatabaseAtStage(IEnumerable <ITableInfo> allTableInfos, LoadBubble deploymentStage) { var dbInfo = _databaseConfiguration.DeployInfo[deploymentStage]; foreach (var tableInfo in allTableInfos) { var columnNames = tableInfo.ColumnInfos.Select(info => info.GetRuntimeName()).ToList(); if (!columnNames.Any()) { _notifier.OnCheckPerformed(new CheckEventArgs("Table '" + tableInfo.GetRuntimeName() + "' has no ColumnInfos", CheckResult.Fail, null)); } string tableName = tableInfo.GetRuntimeName(deploymentStage, _databaseConfiguration.DatabaseNamer); var table = dbInfo.ExpectTable(tableName); if (!table.Exists()) { throw new Exception("PreExecutionChecker spotted that table does not exist:" + table + " it was about to check whether the TableInfo matched the columns or not"); } } }
public string GetDatabaseName(string rootDatabaseName, LoadBubble convention) { return(rootDatabaseName); }
public string GetName(string tableName, LoadBubble convention) { return(tableName); }