/// <summary> /// Returns an appropriate class that implements IDataCopy based on the passed in source and dest Sql types /// </summary> public static IDataCopy GetInstance(SqlFlavor sourceSqlFlavor, SqlFlavor destSqlFlavor, IDataUtils sourceDataUtils, IDataUtils destDataUtils, Logger logger) { if (sourceDataUtils.GetType() == typeof(TestDataUtils) && destDataUtils.GetType() == typeof(TestDataUtils)) { return new TestDataCopy((TestDataUtils)sourceDataUtils, (TestDataUtils)destDataUtils); } switch (sourceSqlFlavor) { case SqlFlavor.MSSQL: if (destSqlFlavor == SqlFlavor.MSSQL) { return new MSSQLToMSSQLDataCopy((MSSQLDataUtils)sourceDataUtils, (MSSQLDataUtils)destDataUtils, logger); } else if (destSqlFlavor == SqlFlavor.Netezza) { return new MSSQLToNetezzaDataCopy((MSSQLDataUtils)sourceDataUtils, (NetezzaDataUtils)destDataUtils, logger, Config.Slave, Config.NetezzaUser, Config.NetezzaPrivateKeyPath); } else if (destSqlFlavor == SqlFlavor.Vertica) { return new MSSQLToVerticaDataCopy((MSSQLDataUtils)sourceDataUtils, (VerticaDataUtils)destDataUtils, logger); } else if (destSqlFlavor == SqlFlavor.MySQL) { return new MSSQLToMySQLDataCopy((MSSQLDataUtils)sourceDataUtils, (MySQLDataUtils)destDataUtils, logger); } break; case SqlFlavor.MySQL: if (destSqlFlavor == SqlFlavor.MSSQL) { return new MySQLToMSSQLDataCopy((MySQLDataUtils)sourceDataUtils, (MSSQLDataUtils)destDataUtils, logger); } break; } //if we get here without returning it means something was passed in that isn't supported throw new NotImplementedException("Specified SQL types not supported for data copying!"); }
/// <summary> /// Drops all tables in the list of tables given that have names ending in _{CTID} where CTID is one of the numbers in ctids /// </summary> /// <param name="dataUtils">Data utils for the server on which to drop the tables</param> public static void DeleteOldTables(IEnumerable<long> ctids, IEnumerable<TTable> tables, IDataUtils dataUtils, string dbName ) { foreach (var table in tables) { int lastUnderscore = table.name.LastIndexOf('_'); if (lastUnderscore == -1) { continue; } string end = table.name.Substring(lastUnderscore + 1); int tableCtid; if (!int.TryParse(end, out tableCtid)) { continue; } if (ctids.Contains(tableCtid)) { dataUtils.DropTableIfExists(dbName, table.name, table.schema); } } }
//base keyword invokes the base class's constructor public SlaveMaintenance(IDataUtils relayDataUtils, IDataUtils slaveDataUtils, Logger logger) : base(relayDataUtils, slaveDataUtils, logger) { }
//base keyword invokes the base class's constructor public MasterMaintenance(IDataUtils dataUtils, IDataUtils destDataUtils, Logger logger) : base(dataUtils, destDataUtils, logger) { }
//base keyword invokes the base class's constructor public Notifier(IDataUtils dataUtils, IEmailClient emailClient, Logger logger) : base(dataUtils, null, logger) { this.emailClient = emailClient; }
public ShardCoordinator(IDataUtils dataUtils, Logger logger) : base(dataUtils, dataUtils, logger) { shardDatabases = Config.ShardDatabases; tablesWithChanges = new List<TableConf>(); }
public Slave(IDataUtils sourceDataUtils, IDataUtils destDataUtils, Logger logger) : base(sourceDataUtils, destDataUtils, logger) { }
/// <summary> /// Set field list values for each table in the config /// </summary> /// <param name="Database">Database name to run on</param> /// <param name="tableConfArray">Array of tableconf objects to loop through and set field lists on</param> public virtual void SetFieldLists(string database, IEnumerable<TableConf> tableConfArray, IDataUtils dataUtils) { foreach (TableConf t in tableConfArray) { try { List<TColumn> columns = dataUtils.GetFieldList(database, t.Name, t.SchemaName); SetFieldList(t, columns); } catch (Exception e) { HandleException(e, t, "Error setting field lists for table " + t.SchemaName + "." + t.Name + ": " + e.Message + " - Stack Trace:" + e.StackTrace); } } }
protected Agent(IDataUtils sourceDataUtils, IDataUtils destDataUtils, Logger logger) { this.sourceDataUtils = sourceDataUtils; this.destDataUtils = destDataUtils; this.logger = logger; }
public ShardCoordinator(IDataUtils dataUtils, Logger logger) : base(dataUtils, dataUtils, logger) { shardDatabases = Config.ShardDatabases; tablesWithChanges = new List <TableConf>(); }
/// <summary> /// Parse XML EVENTDATA and create zero or more SchemaChange objects from that /// </summary> /// <param name="tables">Array of table configuration objects</param> /// <param name="dbName">Database for retrieving data type info</param> public List<SchemaChange> Parse(IEnumerable<TableConf> tables, IDataUtils dataUtils, string dbName) { var schemaChanges = new List<SchemaChange>(); string columnName; string tableName; SchemaChangeType changeType; DataType dataType; SchemaChange sc; string newColumnName; XmlNode node; var xml = new XmlDocument(); xml.LoadXml(EventData); if (xml == null) { Console.WriteLine("XML failed to load"); Console.WriteLine(EventData); } string eventType = xml.SelectSingleNode("EVENT_INSTANCE/EventType").InnerText; if (eventType == "ALTER_TABLE") { node = xml.SelectSingleNode("EVENT_INSTANCE/AlterTableActionList"); } else if (eventType == "RENAME") { node = xml.SelectSingleNode("EVENT_INSTANCE/Parameters"); } else { //this is a DDL event type that we don't care about publishing, so ignore it return schemaChanges; } if (node == null) { //we'll get here on events that do an ALTER_TABLE but don't change any columns, //such as "alter table enable change_tracking" return schemaChanges; } if (node.FirstChild.Name == "Param") { tableName = xml.SelectSingleNode("/EVENT_INSTANCE/TargetObjectName").InnerText; } else { tableName = xml.SelectSingleNode("/EVENT_INSTANCE/ObjectName").InnerText; } string schemaName = xml.SelectSingleNode("/EVENT_INSTANCE/SchemaName").InnerText; //String.Compare method returns 0 if the strings are equal, the third "true" flag is for a case insensitive comparison //Get table config object TableConf t = tables.SingleOrDefault(item => String.Compare(item.Name, tableName, ignoreCase: true) == 0); if (t == null) { //the DDL event applies to a table not in our so we just ignore it return schemaChanges; } switch (node.FirstChild.Name) { case "Param": changeType = SchemaChangeType.Rename; columnName = xml.SelectSingleNode("/EVENT_INSTANCE/ObjectName").InnerText; newColumnName = xml.SelectSingleNode("/EVENT_INSTANCE/NewObjectName").InnerText; if (t.ColumnList == null || t.ColumnList.Contains(columnName, StringComparer.OrdinalIgnoreCase)) { sc = new SchemaChange(DdeID, changeType, schemaName, tableName, columnName, newColumnName); schemaChanges.Add(sc); } break; case "Alter": changeType = SchemaChangeType.Modify; foreach (XmlNode xColumn in xml.SelectNodes("/EVENT_INSTANCE/AlterTableActionList/Alter/Columns/Name")) { columnName = xColumn.InnerText; if (t.ColumnList == null || t.ColumnList.Contains(columnName, StringComparer.OrdinalIgnoreCase)) { try { dataType = ParseDataType(dataUtils.GetDataType(dbName, tableName, schemaName, columnName)); sc = new SchemaChange(DdeID, changeType, schemaName, tableName, columnName, null, dataType); schemaChanges.Add(sc); } catch (DoesNotExistException) { //if we get a does not exist exception, it generally means the column was created and then dropped, //which is ok. break; } } } break; case "Create": changeType = SchemaChangeType.Add; tableName = xml.SelectSingleNode("/EVENT_INSTANCE/ObjectName").InnerText; foreach (XmlNode xColumn in xml.SelectNodes("/EVENT_INSTANCE/AlterTableActionList/Create/Columns/Name")) { columnName = xColumn.InnerText; //if column list is specified, only publish schema changes if the column is already in the list. we don't want //slaves adding a new column that we don't plan to publish changes for. //if column list is null, we want changes associated with all columns. if (t.ColumnList == null || t.ColumnList.Contains(columnName, StringComparer.OrdinalIgnoreCase)) { try { var type = dataUtils.GetDataType(dbName, tableName, schemaName, columnName); dataType = ParseDataType(type); sc = new SchemaChange(DdeID, changeType, schemaName, tableName, columnName, null, dataType); schemaChanges.Add(sc); } catch (DoesNotExistException) { //if we get a does not exist exception, it generally means the column was created and then dropped, //which is ok. break; } } } break; case "Drop": changeType = SchemaChangeType.Drop; tableName = xml.SelectSingleNode("/EVENT_INSTANCE/ObjectName").InnerText; foreach (XmlNode xColumn in xml.SelectNodes("/EVENT_INSTANCE/AlterTableActionList/Drop/Columns/Name")) { columnName = xColumn.InnerText; if (t.ColumnList == null || t.ColumnList.Contains(columnName, StringComparer.OrdinalIgnoreCase)) { sc = new SchemaChange(DdeID, changeType, schemaName, tableName, columnName, null, null); schemaChanges.Add(sc); } } break; } return schemaChanges; }
public Logger(string statsdHost, string statsdPort, string errorLogDB, string logFile, IDataUtils dataUtils) : this(statsdHost, statsdPort, errorLogDB, logFile) { this.dataUtils = dataUtils; }
/// <summary> /// Returns an appropriate class that implements IDataCopy based on the passed in source and dest Sql types /// </summary> public static IDataCopy GetInstance(SqlFlavor sourceSqlFlavor, SqlFlavor destSqlFlavor, IDataUtils sourceDataUtils, IDataUtils destDataUtils, Logger logger) { if (sourceDataUtils.GetType() == typeof(TestDataUtils) && destDataUtils.GetType() == typeof(TestDataUtils)) { return(new TestDataCopy((TestDataUtils)sourceDataUtils, (TestDataUtils)destDataUtils)); } switch (sourceSqlFlavor) { case SqlFlavor.MSSQL: if (destSqlFlavor == SqlFlavor.MSSQL) { return(new MSSQLToMSSQLDataCopy((MSSQLDataUtils)sourceDataUtils, (MSSQLDataUtils)destDataUtils, logger)); } else if (destSqlFlavor == SqlFlavor.Netezza) { return(new MSSQLToNetezzaDataCopy((MSSQLDataUtils)sourceDataUtils, (NetezzaDataUtils)destDataUtils, logger, Config.Slave, Config.NetezzaUser, Config.NetezzaPrivateKeyPath)); } else if (destSqlFlavor == SqlFlavor.Vertica) { return(new MSSQLToVerticaDataCopy((MSSQLDataUtils)sourceDataUtils, (VerticaDataUtils)destDataUtils, logger)); } else if (destSqlFlavor == SqlFlavor.MySQL) { return(new MSSQLToMySQLDataCopy((MSSQLDataUtils)sourceDataUtils, (MySQLDataUtils)destDataUtils, logger)); } break; case SqlFlavor.MySQL: if (destSqlFlavor == SqlFlavor.MSSQL) { return(new MySQLToMSSQLDataCopy((MySQLDataUtils)sourceDataUtils, (MSSQLDataUtils)destDataUtils, logger)); } break; } //if we get here without returning it means something was passed in that isn't supported throw new NotImplementedException("Specified SQL types not supported for data copying!"); }
/// <summary> /// Drops all tables in the list of tables given that have names ending in _{CTID} where CTID is one of the numbers in ctids /// </summary> /// <param name="dataUtils">Data utils for the server on which to drop the tables</param> public static void DeleteOldTables(IEnumerable <long> ctids, IEnumerable <TTable> tables, IDataUtils dataUtils, string dbName) { foreach (var table in tables) { int lastUnderscore = table.name.LastIndexOf('_'); if (lastUnderscore == -1) { continue; } string end = table.name.Substring(lastUnderscore + 1); int tableCtid; if (!int.TryParse(end, out tableCtid)) { continue; } if (ctids.Contains(tableCtid)) { dataUtils.DropTableIfExists(dbName, table.name, table.schema); } } }
/// <summary> /// Parse XML EVENTDATA and create zero or more SchemaChange objects from that /// </summary> /// <param name="tables">Array of table configuration objects</param> /// <param name="dbName">Database for retrieving data type info</param> public List <SchemaChange> Parse(IEnumerable <TableConf> tables, IDataUtils dataUtils, string dbName) { var schemaChanges = new List <SchemaChange>(); string columnName; string tableName; SchemaChangeType changeType; DataType dataType; SchemaChange sc; string newColumnName; XmlNode node; var xml = new XmlDocument(); xml.LoadXml(EventData); if (xml == null) { Console.WriteLine("XML failed to load"); Console.WriteLine(EventData); } string eventType = xml.SelectSingleNode("EVENT_INSTANCE/EventType").InnerText; if (eventType == "ALTER_TABLE") { node = xml.SelectSingleNode("EVENT_INSTANCE/AlterTableActionList"); } else if (eventType == "RENAME") { node = xml.SelectSingleNode("EVENT_INSTANCE/Parameters"); } else { //this is a DDL event type that we don't care about publishing, so ignore it return(schemaChanges); } if (node == null) { //we'll get here on events that do an ALTER_TABLE but don't change any columns, //such as "alter table enable change_tracking" return(schemaChanges); } if (node.FirstChild.Name == "Param") { tableName = xml.SelectSingleNode("/EVENT_INSTANCE/TargetObjectName").InnerText; } else { tableName = xml.SelectSingleNode("/EVENT_INSTANCE/ObjectName").InnerText; } string schemaName = xml.SelectSingleNode("/EVENT_INSTANCE/SchemaName").InnerText; //String.Compare method returns 0 if the strings are equal, the third "true" flag is for a case insensitive comparison //Get table config object TableConf t = tables.SingleOrDefault(item => String.Compare(item.Name, tableName, ignoreCase: true) == 0); if (t == null) { //the DDL event applies to a table not in our so we just ignore it return(schemaChanges); } switch (node.FirstChild.Name) { case "Param": changeType = SchemaChangeType.Rename; columnName = xml.SelectSingleNode("/EVENT_INSTANCE/ObjectName").InnerText; newColumnName = xml.SelectSingleNode("/EVENT_INSTANCE/NewObjectName").InnerText; if (t.ColumnList == null || t.ColumnList.Contains(columnName, StringComparer.OrdinalIgnoreCase)) { sc = new SchemaChange(DdeID, changeType, schemaName, tableName, columnName, newColumnName); schemaChanges.Add(sc); } break; case "Alter": changeType = SchemaChangeType.Modify; foreach (XmlNode xColumn in xml.SelectNodes("/EVENT_INSTANCE/AlterTableActionList/Alter/Columns/Name")) { columnName = xColumn.InnerText; if (t.ColumnList == null || t.ColumnList.Contains(columnName, StringComparer.OrdinalIgnoreCase)) { try { dataType = DataType.ParseDataType(dataUtils.GetDataType(dbName, tableName, schemaName, columnName)); sc = new SchemaChange(DdeID, changeType, schemaName, tableName, columnName, null, dataType); schemaChanges.Add(sc); } catch (DoesNotExistException) { //if we get a does not exist exception, it generally means the column was created and then dropped, //which is ok. break; } } } break; case "Create": changeType = SchemaChangeType.Add; tableName = xml.SelectSingleNode("/EVENT_INSTANCE/ObjectName").InnerText; foreach (XmlNode xColumn in xml.SelectNodes("/EVENT_INSTANCE/AlterTableActionList/Create/Columns/Name")) { columnName = xColumn.InnerText; //if column list is specified, only publish schema changes if the column is already in the list. we don't want //slaves adding a new column that we don't plan to publish changes for. //if column list is null, we want changes associated with all columns. if (t.ColumnList == null || t.ColumnList.Contains(columnName, StringComparer.OrdinalIgnoreCase)) { try { var type = dataUtils.GetDataType(dbName, tableName, schemaName, columnName); dataType = DataType.ParseDataType(type); sc = new SchemaChange(DdeID, changeType, schemaName, tableName, columnName, null, dataType); schemaChanges.Add(sc); } catch (DoesNotExistException) { //if we get a does not exist exception, it generally means the column was created and then dropped, //which is ok. break; } } } break; case "Drop": changeType = SchemaChangeType.Drop; tableName = xml.SelectSingleNode("/EVENT_INSTANCE/ObjectName").InnerText; foreach (XmlNode xColumn in xml.SelectNodes("/EVENT_INSTANCE/AlterTableActionList/Drop/Columns/Name")) { columnName = xColumn.InnerText; if (t.ColumnList == null || t.ColumnList.Contains(columnName, StringComparer.OrdinalIgnoreCase)) { sc = new SchemaChange(DdeID, changeType, schemaName, tableName, columnName, null, null); schemaChanges.Add(sc); } } break; } return(schemaChanges); }
public override void SetFieldLists(string database, IEnumerable <TableConf> tableConfs, IDataUtils dataUtils) { var allFieldLists = dataUtils.GetAllFields(database, tableConfs.ToDictionary(t => t, t => t.Name)); foreach (var table in tableConfs) { SetFieldList(table, allFieldLists[table]); } }
public Master(IDataUtils sourceDataUtils, IDataUtils destDataUtils, Logger logger) : base(sourceDataUtils, destDataUtils, logger) { }
public RelayMaintenance(IDataUtils dataUtils, Logger logger) : base(dataUtils, null, logger) { }
public override void SetFieldLists(string database, IEnumerable<TableConf> tableConfs, IDataUtils dataUtils) { var allFieldLists = dataUtils.GetAllFields(database, tableConfs.ToDictionary(t => t, t => t.Name)); foreach (var table in tableConfs) { SetFieldList(table, allFieldLists[table]); } }
/// <summary> /// Set field list values for each table in the config /// </summary> /// <param name="Database">Database name to run on</param> /// <param name="tableConfArray">Array of tableconf objects to loop through and set field lists on</param> public virtual void SetFieldLists(string database, IEnumerable <TableConf> tableConfArray, IDataUtils dataUtils) { foreach (TableConf t in tableConfArray) { try { List <TColumn> columns = dataUtils.GetFieldList(database, t.Name, t.SchemaName); SetFieldList(t, columns); } catch (Exception e) { HandleException(e, t, "Error setting field lists for table " + t.SchemaName + "." + t.Name + ": " + e.Message + " - Stack Trace:" + e.StackTrace); } } }