/// <summary>
/// Returns an appropriate class that implements IDataCopy based on the passed in source and dest Sql types
/// </summary>
public static IDataCopy GetInstance(SqlFlavor sourceSqlFlavor, SqlFlavor destSqlFlavor, IDataUtils sourceDataUtils, IDataUtils destDataUtils, Logger logger)
{
if (sourceDataUtils.GetType() == typeof(TestDataUtils) && destDataUtils.GetType() == typeof(TestDataUtils)) {
return new TestDataCopy((TestDataUtils)sourceDataUtils, (TestDataUtils)destDataUtils);
}
switch (sourceSqlFlavor) {
case SqlFlavor.MSSQL:
if (destSqlFlavor == SqlFlavor.MSSQL) {
return new MSSQLToMSSQLDataCopy((MSSQLDataUtils)sourceDataUtils, (MSSQLDataUtils)destDataUtils, logger);
} else if (destSqlFlavor == SqlFlavor.Netezza) {
return new MSSQLToNetezzaDataCopy((MSSQLDataUtils)sourceDataUtils, (NetezzaDataUtils)destDataUtils, logger, Config.Slave, Config.NetezzaUser, Config.NetezzaPrivateKeyPath);
} else if (destSqlFlavor == SqlFlavor.Vertica) {
return new MSSQLToVerticaDataCopy((MSSQLDataUtils)sourceDataUtils, (VerticaDataUtils)destDataUtils, logger);
} else if (destSqlFlavor == SqlFlavor.MySQL) {
return new MSSQLToMySQLDataCopy((MSSQLDataUtils)sourceDataUtils, (MySQLDataUtils)destDataUtils, logger);
}
break;
case SqlFlavor.MySQL:
if (destSqlFlavor == SqlFlavor.MSSQL) {
return new MySQLToMSSQLDataCopy((MySQLDataUtils)sourceDataUtils, (MSSQLDataUtils)destDataUtils, logger);
}
break;
}
//if we get here without returning it means something was passed in that isn't supported
throw new NotImplementedException("Specified SQL types not supported for data copying!");
}
/// <summary>
/// Drops all tables in the list of tables given that have names ending in _{CTID} where CTID is one of the numbers in ctids
/// </summary>
/// <param name="dataUtils">Data utils for the server on which to drop the tables</param>
public static void DeleteOldTables(IEnumerable<long> ctids, IEnumerable<TTable> tables, IDataUtils dataUtils, string dbName )
{
foreach (var table in tables) {
int lastUnderscore = table.name.LastIndexOf('_');
if (lastUnderscore == -1) {
continue;
}
string end = table.name.Substring(lastUnderscore + 1);
int tableCtid;
if (!int.TryParse(end, out tableCtid)) {
continue;
}
if (ctids.Contains(tableCtid)) {
dataUtils.DropTableIfExists(dbName, table.name, table.schema);
}
}
}
//base keyword invokes the base class's constructor
public SlaveMaintenance(IDataUtils relayDataUtils, IDataUtils slaveDataUtils, Logger logger)
: base(relayDataUtils, slaveDataUtils, logger)
{
}
//base keyword invokes the base class's constructor
public MasterMaintenance(IDataUtils dataUtils, IDataUtils destDataUtils, Logger logger)
: base(dataUtils, destDataUtils, logger)
{
}
//base keyword invokes the base class's constructor
public Notifier(IDataUtils dataUtils, IEmailClient emailClient, Logger logger)
: base(dataUtils, null, logger)
{
this.emailClient = emailClient;
}
public ShardCoordinator(IDataUtils dataUtils, Logger logger)
: base(dataUtils, dataUtils, logger)
{
shardDatabases = Config.ShardDatabases;
tablesWithChanges = new List<TableConf>();
}
public Slave(IDataUtils sourceDataUtils, IDataUtils destDataUtils, Logger logger)
: base(sourceDataUtils, destDataUtils, logger)
{
}
/// <summary>
/// Set field list values for each table in the config
/// </summary>
/// <param name="Database">Database name to run on</param>
/// <param name="tableConfArray">Array of tableconf objects to loop through and set field lists on</param>
public virtual void SetFieldLists(string database, IEnumerable<TableConf> tableConfArray, IDataUtils dataUtils)
{
foreach (TableConf t in tableConfArray) {
try {
List<TColumn> columns = dataUtils.GetFieldList(database, t.Name, t.SchemaName);
SetFieldList(t, columns);
} catch (Exception e) {
HandleException(e, t, "Error setting field lists for table " + t.SchemaName + "." + t.Name + ": " + e.Message + " - Stack Trace:" + e.StackTrace);
}
}
}
protected Agent(IDataUtils sourceDataUtils, IDataUtils destDataUtils, Logger logger)
{
this.sourceDataUtils = sourceDataUtils;
this.destDataUtils = destDataUtils;
this.logger = logger;
}
public ShardCoordinator(IDataUtils dataUtils, Logger logger)
: base(dataUtils, dataUtils, logger)
{
shardDatabases = Config.ShardDatabases;
tablesWithChanges = new List <TableConf>();
}
/// <summary>
/// Parse XML EVENTDATA and create zero or more SchemaChange objects from that
/// </summary>
/// <param name="tables">Array of table configuration objects</param>
/// <param name="dbName">Database for retrieving data type info</param>
public List<SchemaChange> Parse(IEnumerable<TableConf> tables, IDataUtils dataUtils, string dbName)
{
var schemaChanges = new List<SchemaChange>();
string columnName;
string tableName;
SchemaChangeType changeType;
DataType dataType;
SchemaChange sc;
string newColumnName;
XmlNode node;
var xml = new XmlDocument();
xml.LoadXml(EventData);
if (xml == null) {
Console.WriteLine("XML failed to load");
Console.WriteLine(EventData);
}
string eventType = xml.SelectSingleNode("EVENT_INSTANCE/EventType").InnerText;
if (eventType == "ALTER_TABLE") {
node = xml.SelectSingleNode("EVENT_INSTANCE/AlterTableActionList");
} else if (eventType == "RENAME") {
node = xml.SelectSingleNode("EVENT_INSTANCE/Parameters");
} else {
//this is a DDL event type that we don't care about publishing, so ignore it
return schemaChanges;
}
if (node == null) {
//we'll get here on events that do an ALTER_TABLE but don't change any columns,
//such as "alter table enable change_tracking"
return schemaChanges;
}
if (node.FirstChild.Name == "Param") {
tableName = xml.SelectSingleNode("/EVENT_INSTANCE/TargetObjectName").InnerText;
} else {
tableName = xml.SelectSingleNode("/EVENT_INSTANCE/ObjectName").InnerText;
}
string schemaName = xml.SelectSingleNode("/EVENT_INSTANCE/SchemaName").InnerText;
//String.Compare method returns 0 if the strings are equal, the third "true" flag is for a case insensitive comparison
//Get table config object
TableConf t = tables.SingleOrDefault(item => String.Compare(item.Name, tableName, ignoreCase: true) == 0);
if (t == null) {
//the DDL event applies to a table not in our so we just ignore it
return schemaChanges;
}
switch (node.FirstChild.Name) {
case "Param":
changeType = SchemaChangeType.Rename;
columnName = xml.SelectSingleNode("/EVENT_INSTANCE/ObjectName").InnerText;
newColumnName = xml.SelectSingleNode("/EVENT_INSTANCE/NewObjectName").InnerText;
if (t.ColumnList == null || t.ColumnList.Contains(columnName, StringComparer.OrdinalIgnoreCase)) {
sc = new SchemaChange(DdeID, changeType, schemaName, tableName, columnName, newColumnName);
schemaChanges.Add(sc);
}
break;
case "Alter":
changeType = SchemaChangeType.Modify;
foreach (XmlNode xColumn in xml.SelectNodes("/EVENT_INSTANCE/AlterTableActionList/Alter/Columns/Name")) {
columnName = xColumn.InnerText;
if (t.ColumnList == null || t.ColumnList.Contains(columnName, StringComparer.OrdinalIgnoreCase)) {
try {
dataType = ParseDataType(dataUtils.GetDataType(dbName, tableName, schemaName, columnName));
sc = new SchemaChange(DdeID, changeType, schemaName, tableName, columnName, null, dataType);
schemaChanges.Add(sc);
} catch (DoesNotExistException) {
//if we get a does not exist exception, it generally means the column was created and then dropped,
//which is ok.
break;
}
}
}
break;
case "Create":
changeType = SchemaChangeType.Add;
tableName = xml.SelectSingleNode("/EVENT_INSTANCE/ObjectName").InnerText;
foreach (XmlNode xColumn in xml.SelectNodes("/EVENT_INSTANCE/AlterTableActionList/Create/Columns/Name")) {
columnName = xColumn.InnerText;
//if column list is specified, only publish schema changes if the column is already in the list. we don't want
//slaves adding a new column that we don't plan to publish changes for.
//if column list is null, we want changes associated with all columns.
if (t.ColumnList == null || t.ColumnList.Contains(columnName, StringComparer.OrdinalIgnoreCase)) {
try {
var type = dataUtils.GetDataType(dbName, tableName, schemaName, columnName);
dataType = ParseDataType(type);
sc = new SchemaChange(DdeID, changeType, schemaName, tableName, columnName, null, dataType);
schemaChanges.Add(sc);
} catch (DoesNotExistException) {
//if we get a does not exist exception, it generally means the column was created and then dropped,
//which is ok.
break;
}
}
}
break;
case "Drop":
changeType = SchemaChangeType.Drop;
tableName = xml.SelectSingleNode("/EVENT_INSTANCE/ObjectName").InnerText;
foreach (XmlNode xColumn in xml.SelectNodes("/EVENT_INSTANCE/AlterTableActionList/Drop/Columns/Name")) {
columnName = xColumn.InnerText;
if (t.ColumnList == null || t.ColumnList.Contains(columnName, StringComparer.OrdinalIgnoreCase)) {
sc = new SchemaChange(DdeID, changeType, schemaName, tableName, columnName, null, null);
schemaChanges.Add(sc);
}
}
break;
}
return schemaChanges;
}
public Logger(string statsdHost, string statsdPort, string errorLogDB, string logFile, IDataUtils dataUtils)
: this(statsdHost, statsdPort, errorLogDB, logFile)
{
this.dataUtils = dataUtils;
}
/// <summary>
/// Returns an appropriate class that implements IDataCopy based on the passed in source and dest Sql types
/// </summary>
public static IDataCopy GetInstance(SqlFlavor sourceSqlFlavor, SqlFlavor destSqlFlavor, IDataUtils sourceDataUtils, IDataUtils destDataUtils, Logger logger)
{
if (sourceDataUtils.GetType() == typeof(TestDataUtils) && destDataUtils.GetType() == typeof(TestDataUtils))
{
return(new TestDataCopy((TestDataUtils)sourceDataUtils, (TestDataUtils)destDataUtils));
}
switch (sourceSqlFlavor)
{
case SqlFlavor.MSSQL:
if (destSqlFlavor == SqlFlavor.MSSQL)
{
return(new MSSQLToMSSQLDataCopy((MSSQLDataUtils)sourceDataUtils, (MSSQLDataUtils)destDataUtils, logger));
}
else if (destSqlFlavor == SqlFlavor.Netezza)
{
return(new MSSQLToNetezzaDataCopy((MSSQLDataUtils)sourceDataUtils, (NetezzaDataUtils)destDataUtils, logger, Config.Slave, Config.NetezzaUser, Config.NetezzaPrivateKeyPath));
}
else if (destSqlFlavor == SqlFlavor.Vertica)
{
return(new MSSQLToVerticaDataCopy((MSSQLDataUtils)sourceDataUtils, (VerticaDataUtils)destDataUtils, logger));
}
else if (destSqlFlavor == SqlFlavor.MySQL)
{
return(new MSSQLToMySQLDataCopy((MSSQLDataUtils)sourceDataUtils, (MySQLDataUtils)destDataUtils, logger));
}
break;
case SqlFlavor.MySQL:
if (destSqlFlavor == SqlFlavor.MSSQL)
{
return(new MySQLToMSSQLDataCopy((MySQLDataUtils)sourceDataUtils, (MSSQLDataUtils)destDataUtils, logger));
}
break;
}
//if we get here without returning it means something was passed in that isn't supported
throw new NotImplementedException("Specified SQL types not supported for data copying!");
}
/// <summary>
/// Drops all tables in the list of tables given that have names ending in _{CTID} where CTID is one of the numbers in ctids
/// </summary>
/// <param name="dataUtils">Data utils for the server on which to drop the tables</param>
public static void DeleteOldTables(IEnumerable <long> ctids, IEnumerable <TTable> tables, IDataUtils dataUtils, string dbName)
{
foreach (var table in tables)
{
int lastUnderscore = table.name.LastIndexOf('_');
if (lastUnderscore == -1)
{
continue;
}
string end = table.name.Substring(lastUnderscore + 1);
int tableCtid;
if (!int.TryParse(end, out tableCtid))
{
continue;
}
if (ctids.Contains(tableCtid))
{
dataUtils.DropTableIfExists(dbName, table.name, table.schema);
}
}
}
/// <summary>
/// Parse XML EVENTDATA and create zero or more SchemaChange objects from that
/// </summary>
/// <param name="tables">Array of table configuration objects</param>
/// <param name="dbName">Database for retrieving data type info</param>
public List <SchemaChange> Parse(IEnumerable <TableConf> tables, IDataUtils dataUtils, string dbName)
{
var schemaChanges = new List <SchemaChange>();
string columnName;
string tableName;
SchemaChangeType changeType;
DataType dataType;
SchemaChange sc;
string newColumnName;
XmlNode node;
var xml = new XmlDocument();
xml.LoadXml(EventData);
if (xml == null)
{
Console.WriteLine("XML failed to load");
Console.WriteLine(EventData);
}
string eventType = xml.SelectSingleNode("EVENT_INSTANCE/EventType").InnerText;
if (eventType == "ALTER_TABLE")
{
node = xml.SelectSingleNode("EVENT_INSTANCE/AlterTableActionList");
}
else if (eventType == "RENAME")
{
node = xml.SelectSingleNode("EVENT_INSTANCE/Parameters");
}
else
{
//this is a DDL event type that we don't care about publishing, so ignore it
return(schemaChanges);
}
if (node == null)
{
//we'll get here on events that do an ALTER_TABLE but don't change any columns,
//such as "alter table enable change_tracking"
return(schemaChanges);
}
if (node.FirstChild.Name == "Param")
{
tableName = xml.SelectSingleNode("/EVENT_INSTANCE/TargetObjectName").InnerText;
}
else
{
tableName = xml.SelectSingleNode("/EVENT_INSTANCE/ObjectName").InnerText;
}
string schemaName = xml.SelectSingleNode("/EVENT_INSTANCE/SchemaName").InnerText;
//String.Compare method returns 0 if the strings are equal, the third "true" flag is for a case insensitive comparison
//Get table config object
TableConf t = tables.SingleOrDefault(item => String.Compare(item.Name, tableName, ignoreCase: true) == 0);
if (t == null)
{
//the DDL event applies to a table not in our so we just ignore it
return(schemaChanges);
}
switch (node.FirstChild.Name)
{
case "Param":
changeType = SchemaChangeType.Rename;
columnName = xml.SelectSingleNode("/EVENT_INSTANCE/ObjectName").InnerText;
newColumnName = xml.SelectSingleNode("/EVENT_INSTANCE/NewObjectName").InnerText;
if (t.ColumnList == null || t.ColumnList.Contains(columnName, StringComparer.OrdinalIgnoreCase))
{
sc = new SchemaChange(DdeID, changeType, schemaName, tableName, columnName, newColumnName);
schemaChanges.Add(sc);
}
break;
case "Alter":
changeType = SchemaChangeType.Modify;
foreach (XmlNode xColumn in xml.SelectNodes("/EVENT_INSTANCE/AlterTableActionList/Alter/Columns/Name"))
{
columnName = xColumn.InnerText;
if (t.ColumnList == null || t.ColumnList.Contains(columnName, StringComparer.OrdinalIgnoreCase))
{
try {
dataType = DataType.ParseDataType(dataUtils.GetDataType(dbName, tableName, schemaName, columnName));
sc = new SchemaChange(DdeID, changeType, schemaName, tableName, columnName, null, dataType);
schemaChanges.Add(sc);
} catch (DoesNotExistException) {
//if we get a does not exist exception, it generally means the column was created and then dropped,
//which is ok.
break;
}
}
}
break;
case "Create":
changeType = SchemaChangeType.Add;
tableName = xml.SelectSingleNode("/EVENT_INSTANCE/ObjectName").InnerText;
foreach (XmlNode xColumn in xml.SelectNodes("/EVENT_INSTANCE/AlterTableActionList/Create/Columns/Name"))
{
columnName = xColumn.InnerText;
//if column list is specified, only publish schema changes if the column is already in the list. we don't want
//slaves adding a new column that we don't plan to publish changes for.
//if column list is null, we want changes associated with all columns.
if (t.ColumnList == null || t.ColumnList.Contains(columnName, StringComparer.OrdinalIgnoreCase))
{
try {
var type = dataUtils.GetDataType(dbName, tableName, schemaName, columnName);
dataType = DataType.ParseDataType(type);
sc = new SchemaChange(DdeID, changeType, schemaName, tableName, columnName, null, dataType);
schemaChanges.Add(sc);
} catch (DoesNotExistException) {
//if we get a does not exist exception, it generally means the column was created and then dropped,
//which is ok.
break;
}
}
}
break;
case "Drop":
changeType = SchemaChangeType.Drop;
tableName = xml.SelectSingleNode("/EVENT_INSTANCE/ObjectName").InnerText;
foreach (XmlNode xColumn in xml.SelectNodes("/EVENT_INSTANCE/AlterTableActionList/Drop/Columns/Name"))
{
columnName = xColumn.InnerText;
if (t.ColumnList == null || t.ColumnList.Contains(columnName, StringComparer.OrdinalIgnoreCase))
{
sc = new SchemaChange(DdeID, changeType, schemaName, tableName, columnName, null, null);
schemaChanges.Add(sc);
}
}
break;
}
return(schemaChanges);
}
//base keyword invokes the base class's constructor
public MasterMaintenance(IDataUtils dataUtils, IDataUtils destDataUtils, Logger logger)
: base(dataUtils, destDataUtils, logger)
{
}
public override void SetFieldLists(string database, IEnumerable <TableConf> tableConfs, IDataUtils dataUtils)
{
var allFieldLists = dataUtils.GetAllFields(database, tableConfs.ToDictionary(t => t, t => t.Name));
foreach (var table in tableConfs)
{
SetFieldList(table, allFieldLists[table]);
}
}
public Master(IDataUtils sourceDataUtils, IDataUtils destDataUtils, Logger logger)
: base(sourceDataUtils, destDataUtils, logger)
{
}
public Slave(IDataUtils sourceDataUtils, IDataUtils destDataUtils, Logger logger)
: base(sourceDataUtils, destDataUtils, logger)
{
}
public RelayMaintenance(IDataUtils dataUtils, Logger logger)
: base(dataUtils, null, logger)
{
}
protected Agent(IDataUtils sourceDataUtils, IDataUtils destDataUtils, Logger logger)
{
this.sourceDataUtils = sourceDataUtils;
this.destDataUtils = destDataUtils;
this.logger = logger;
}
public RelayMaintenance(IDataUtils dataUtils, Logger logger)
: base(dataUtils, null, logger)
{
}
//base keyword invokes the base class's constructor
public Notifier(IDataUtils dataUtils, IEmailClient emailClient, Logger logger)
: base(dataUtils, null, logger)
{
this.emailClient = emailClient;
}
public override void SetFieldLists(string database, IEnumerable<TableConf> tableConfs, IDataUtils dataUtils)
{
var allFieldLists = dataUtils.GetAllFields(database, tableConfs.ToDictionary(t => t, t => t.Name));
foreach (var table in tableConfs) {
SetFieldList(table, allFieldLists[table]);
}
}
/// <summary>
/// Set field list values for each table in the config
/// </summary>
/// <param name="Database">Database name to run on</param>
/// <param name="tableConfArray">Array of tableconf objects to loop through and set field lists on</param>
public virtual void SetFieldLists(string database, IEnumerable <TableConf> tableConfArray, IDataUtils dataUtils)
{
foreach (TableConf t in tableConfArray)
{
try {
List <TColumn> columns = dataUtils.GetFieldList(database, t.Name, t.SchemaName);
SetFieldList(t, columns);
} catch (Exception e) {
HandleException(e, t, "Error setting field lists for table " + t.SchemaName + "." + t.Name + ": " + e.Message + " - Stack Trace:" + e.StackTrace);
}
}
}
public Master(IDataUtils sourceDataUtils, IDataUtils destDataUtils, Logger logger)
: base(sourceDataUtils, destDataUtils, logger)
{
}
public Logger(string statsdHost, string statsdPort, string errorLogDB, string logFile, IDataUtils dataUtils)
: this(statsdHost, statsdPort, errorLogDB, logFile)
{
this.dataUtils = dataUtils;
}
//base keyword invokes the base class's constructor
public SlaveMaintenance(IDataUtils relayDataUtils, IDataUtils slaveDataUtils, Logger logger)
: base(relayDataUtils, slaveDataUtils, logger)
{
}
