public override void Run()
{
DateTime start = DateTime.Now;
logger.Log("Getting CHANGE_TRACKING_CURRENT_VERSION from master", LogLevel.Trace);
Int64 currentVersion = sourceDataUtils.GetCurrentCTVersion(Config.MasterDB);
logger.Log("Initializing CT batch", LogLevel.Info);
//set up the variables and CT version info for this run
ctb = InitializeBatch(currentVersion);
if (Config.Sharding && ctb == null)
{
logger.Log("Last batch completed and there is no new batch to work on.", LogLevel.Info);
return;
}
Logger.SetProperty("CTID", ctb.CTID);
logger.Log(ctb, LogLevel.Debug);
logger.Log("Working on CTID " + ctb.CTID, LogLevel.Info);
IDictionary <string, Int64> changesCaptured;
logger.Log("Calculating field lists for configured tables", LogLevel.Info);
SetFieldLists(Config.MasterDB, Config.Tables, sourceDataUtils);
//capture changes/publish schema changes is now one unit, so we can just check either.
if ((ctb.SyncBitWise & Convert.ToInt32(SyncBitWise.CaptureChanges)) == 0)
{
changesCaptured = CaptureChanges(currentVersion);
PublishSchemaChanges();
destDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.PublishSchemaChanges), AgentType.Master);
destDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.CaptureChanges), AgentType.Master);
}
else
{
logger.Log("CreateChangeTables succeeded on the previous run, running GetRowCounts instead to populate changesCaptured object", LogLevel.Info);
changesCaptured = GetRowCounts(Config.Tables, Config.MasterCTDB, ctb.CTID);
logger.Log("Successfully populated changesCaptured with a list of rowcounts for each changetable", LogLevel.Debug);
}
var sw = Stopwatch.StartNew();
//copy change tables from master to relay server
logger.Log("Beginning publish changetables step, copying CT tables to the relay server", LogLevel.Info);
PublishChangeTables(Config.MasterCTDB, Config.RelayDB, ctb.CTID, changesCaptured);
logger.Log("Publishing info table", LogLevel.Info);
PublishTableInfo(Config.Tables, Config.RelayDB, changesCaptured, ctb.CTID);
logger.Log("Successfully published changetables, persisting bitwise now", LogLevel.Debug);
//this signifies the end of the master's responsibility for this batch
destDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.UploadChanges), AgentType.Master);
logger.Log("Wrote bitwise value of " + Convert.ToInt32(SyncBitWise.UploadChanges) + " to tblCTVersion", LogLevel.Trace);
logger.Timing(StepTimingKey("UploadChanges"), (int)sw.ElapsedMilliseconds);
logger.Log("Master agent work complete", LogLevel.Info);
var elapsed = DateTime.Now - start;
logger.Timing(TimingKey, (int)elapsed.TotalMinutes);
sourceDataUtils.CleanUpInitializeTable(Config.MasterCTDB, ctb.SyncStartTime.Value);
}
private void ConsolidateTables(ChangeTrackingBatch batch)
{
logger.Log("Consolidating tables", LogLevel.Info);
var actions = new List <Action>();
foreach (var tableDb in tableDBFieldLists)
{
var table = tableDb.Key;
var dbColumns = tableDb.Value;
var firstDB = tableDb.Value.FirstOrDefault(t => t.Value.Count > 0).Key;
if (firstDB == null)
{
logger.Log("No shard has CT changes for table " + table.Name, LogLevel.Debug);
continue;
}
tablesWithChanges.Add(table);
SetFieldList(table, firstDB, batch);
Action act = () => MergeTable(batch, dbColumns, table, firstDB);
actions.Add(act);
}
logger.Log("Parallel invocation of " + actions.Count + " table merges", LogLevel.Info);
//interestingly, Parallel.Invoke does in fact bubble up exceptions, but not until after all threads have completed.
//actually it looks like what it does is wrap its exceptions in an AggregateException. We don't ever catch those
//though because if any exceptions happen inside of MergeTable it would generally be due to things like the server
//being down or a query timeout.
var options = new ParallelOptions();
options.MaxDegreeOfParallelism = Config.MaxThreads;
Parallel.Invoke(options, actions.ToArray());
}
public void TestSchemasOutOfSync()
{
var ctb = new ChangeTrackingBatch(0, 0, 0, 0);
var dbFieldLists = new List <Dictionary <string, List <TColumn> > >();
var db1 = new List <TColumn> {
new TColumn("a", false, null, true)
};
var db2 = new List <TColumn> {
new TColumn("a", false, null, true)
};
var dict1 = new Dictionary <string, List <TColumn> > {
{ "db1", db1 },
{ "db2", db2 }
};
dbFieldLists.Add(dict1);
shardDatabases = new List <string> {
"db1", "db2"
};
Assert.False(SchemasOutOfSync(dbFieldLists));
db1[0] = new TColumn("a", true, null, true);
Assert.True(SchemasOutOfSync(dbFieldLists));
db1.Clear();
db1.Add(new TColumn("a", false, null, true));
db1.Add(new TColumn("b", false, null, true));
db2.Clear();
db2.Add(new TColumn("b", false, null, true));
db2.Add(new TColumn("a", false, null, true));
Assert.False(SchemasOutOfSync(dbFieldLists));
}
public void TestSchemasOutOfSync()
{
var ctb = new ChangeTrackingBatch(0, 0, 0, 0);
var dbFieldLists = new List<Dictionary<string, List<TColumn>>>();
var db1 = new List<TColumn>{
new TColumn("a",false, null, true)
};
var db2 = new List<TColumn>{
new TColumn("a",false, null, true)
};
var dict1 = new Dictionary<string, List<TColumn>>{
{"db1", db1},
{"db2",db2}
};
dbFieldLists.Add(dict1);
shardDatabases = new List<string> { "db1", "db2" };
Assert.False(SchemasOutOfSync(dbFieldLists));
db1[0] = new TColumn("a", true, null, true);
Assert.True(SchemasOutOfSync(dbFieldLists));
db1.Clear();
db1.Add(new TColumn("a", false, null, true));
db1.Add(new TColumn("b", false, null, true));
db2.Clear();
db2.Add(new TColumn("b", false, null, true));
db2.Add(new TColumn("a", false, null, true));
Assert.False(SchemasOutOfSync(dbFieldLists));
}
public override void Run()
{
var batch = new ChangeTrackingBatch(sourceDataUtils.GetLastCTBatch(Config.RelayDB, AgentType.ShardCoordinator));
Logger.SetProperty("CTID", batch.CTID);
if ((batch.SyncBitWise & Convert.ToInt32(SyncBitWise.UploadChanges)) > 0) {
CreateNewVersionsForShards(batch);
return;
}
logger.Log("Working on CTID " + batch.CTID, LogLevel.Info);
if (AllShardMastersDone(batch)) {
logger.Log("All shard masters are done, checking field lists", LogLevel.Info);
tableDBFieldLists = GetFieldListsByDB(batch.CTID);
if (SchemasOutOfSync(tableDBFieldLists.Values)) {
foreach (var sd in shardDatabases) {
sourceDataUtils.RevertCTBatch(sd, batch.CTID);
}
logger.Log("Schemas out of sync, quitting", LogLevel.Info);
return;
}
logger.Log("Field lists in sync, consolidating", LogLevel.Info);
Consolidate(batch);
sourceDataUtils.WriteBitWise(Config.RelayDB, batch.CTID,
Convert.ToInt32(SyncBitWise.CaptureChanges) | Convert.ToInt32(SyncBitWise.UploadChanges), AgentType.ShardCoordinator);
//now that the current batch is done, create a new one for the masters to work on
CreateNewVersionsForShards(batch);
} else {
logger.Log("Not all shards are done yet, waiting until they catch up", LogLevel.Info);
}
}
private void RecordRowCounts(RowCounts actual, ChangeTrackingBatch ctb)
{
var expected = sourceDataUtils.GetExpectedRowCounts(Config.RelayDB, ctb.CTID);
logger.Log("Expected row counts: " + expected + " | actual: " + actual, LogLevel.Info);
double diff = expected - actual.Inserted;
double mismatch;
if (expected == 0)
{
if (actual.Inserted == 0)
{
mismatch = 0.0;
}
else
{
logger.Log("Expected 0 rows, got " + actual.Inserted + " rows inserted on slave.", LogLevel.Error);
return;
}
}
else
{
mismatch = diff / expected;
}
int percentDiff = (int)(mismatch * 100);
string key = string.Format("db.mssql_changetracking_counters.RecordCountMismatchProd{0}.{1}", Config.Slave.Replace('.', '_'), Config.SlaveDB);
logger.Increment(key, percentDiff);
}
public void TestOverrideZeroStartVersion()
{
var table = new TableConf();
table.Name = "tableName";
table.SchemaName = "schema";
string db = "db";
string ctdb = "ctdb";
long minValidVersion = 5;
var batch = new ChangeTrackingBatch(1, 0, 10, 0);
var sourceUtils = new Mock <IDataUtils>();
this.sourceDataUtils = sourceUtils.Object;
sourceUtils.Setup((ut) => ut.GetMinValidVersion(db, table.Name, table.SchemaName))
.Returns(minValidVersion).Verifiable();
sourceUtils.Setup((ut) => ut.CheckTableExists(db, table.Name, It.IsAny <string>()))
.Returns(true);
sourceUtils.Setup((ut) => ut.IsChangeTrackingEnabled(db, table.Name, table.SchemaName))
.Returns(true);
CreateChangeTable(table, db, ctdb, batch);
sourceUtils.Verify(
(ut) => ut.SelectIntoCTTable(ctdb, It.IsAny <TableConf>(), db, It.IsAny <ChangeTrackingBatch>(), It.IsAny <int>(), minValidVersion)
);
}
public void TestNoOverrideNonZeroStartVersion()
{
var table = new TableConf();
table.Name = "tableName";
table.SchemaName = "schmea";
string db = "db";
string ctdb = "ctdb";
var batch = new ChangeTrackingBatch(1, 1, 10, 0);
var sourceUtils = new Mock <IDataUtils>();
this.sourceDataUtils = sourceUtils.Object;
sourceUtils.Setup((ut) => ut.GetMinValidVersion(db, table.Name, table.SchemaName))
.Returns(5);
sourceUtils.Setup((ut) => ut.CheckTableExists(db, table.Name, It.IsAny <string>()))
.Returns(true);
sourceUtils.Setup((ut) => ut.IsChangeTrackingEnabled(db, table.Name, table.SchemaName))
.Returns(true);
sourceUtils.Setup((ut) => ut.IsBeingInitialized(ctdb, It.IsAny <TableConf>()))
.Returns(false);
sourceUtils.Setup((ut) => ut.GetInitializeStartVersion(ctdb, It.IsAny <TableConf>()))
.Returns(new long?());
CreateChangeTable(table, db, ctdb, batch);
//sourceUtils.Verify(
// (ut) => ut.SelectIntoCTTable(It.IsAny<string>(), It.IsAny<TableConf>(), It.IsAny<string>(
sourceUtils.Verify(
(ut) => ut.SelectIntoCTTable(ctdb, It.IsAny <TableConf>(), db, It.IsAny <ChangeTrackingBatch>(), It.IsAny <int>(), batch.SyncStartVersion),
Times.Never(),
"Should not select into CT table when the start version is less than minValidVersion");
}
private void SetFieldList(TableConf table, string database, ChangeTrackingBatch batch)
{
var cols = sourceDataUtils.GetFieldList(database, table.ToCTName(batch.CTID), table.SchemaName);
var pks = sourceDataUtils.GetPrimaryKeysFromInfoTable(table, batch.CTID, database);
foreach (var pk in pks)
{
cols.First((c => c.name == pk)).isPk = true;
}
SetFieldList(table, cols);
}
/// <summary>
/// Runs a single change tracking batch
/// </summary>
/// <param name="ctb">Change tracking batch object to work on</param>
private void RunSingleBatch(ChangeTrackingBatch ctb)
{
Stopwatch sw;
logger.Log("Applying schema changes ", LogLevel.Info);
ApplySchemaChangesAndWrite(ctb);
//marking this field so that all completed slave batches will have the same values
sourceDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.ConsolidateBatches), AgentType.Slave);
logger.Log("Populating table list", LogLevel.Info);
List <ChangeTable> existingCTTables = PopulateTableList(Config.Tables, Config.RelayDB, new List <ChangeTrackingBatch>()
{
ctb
});
logger.Log("Capturing field lists", LogLevel.Info);
SetFieldListsSlave(Config.RelayDB, Config.Tables, ctb, existingCTTables);
if ((ctb.SyncBitWise & Convert.ToInt32(SyncBitWise.DownloadChanges)) == 0)
{
logger.Log("Downloading changes", LogLevel.Info);
sw = Stopwatch.StartNew();
CopyChangeTables(Config.Tables, Config.RelayDB, Config.SlaveCTDB, ctb.CTID, existingCTTables);
logger.Log("CopyChangeTables: " + sw.Elapsed, LogLevel.Trace);
sourceDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.DownloadChanges), AgentType.Slave);
logger.Timing(StepTimingKey("DownloadChanges"), (int)sw.ElapsedMilliseconds);
}
if ((ctb.SyncBitWise & Convert.ToInt32(SyncBitWise.ApplyChanges)) == 0)
{
logger.Log("Applying changes", LogLevel.Info);
sw = Stopwatch.StartNew();
RowCounts total = ApplyChanges(existingCTTables, ctb.CTID, isConsolidated: false);
RecordRowCounts(total, ctb);
logger.Log("ApplyChanges: " + sw.Elapsed, LogLevel.Trace);
sourceDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.ApplyChanges), AgentType.Slave);
logger.Timing(StepTimingKey("ApplyChanges"), (int)sw.ElapsedMilliseconds);
}
logger.Log("Syncing history tables", LogLevel.Info);
sw = Stopwatch.StartNew();
SyncHistoryTables(Config.SlaveCTDB, existingCTTables, isConsolidated: false);
logger.Log("SyncHistoryTables: " + sw.Elapsed, LogLevel.Trace);
var syncStopTime = DateTime.Now;
sourceDataUtils.MarkBatchComplete(Config.RelayDB, ctb.CTID, syncStopTime, Config.Slave);
string key = String.Format(
"db.mssql_changetracking_counters.DataDurationToSync{0}.{1}",
Config.Slave.Replace('.', '_'),
Config.SlaveDB);
logger.Increment(key, (int)(syncStopTime - ctb.SyncStartTime.Value).TotalMinutes);
logger.Timing(StepTimingKey("SyncHistoryTables"), (int)sw.ElapsedMilliseconds);
}
private void Consolidate(ChangeTrackingBatch batch)
{
if ((batch.SyncBitWise & Convert.ToInt32(SyncBitWise.PublishSchemaChanges)) == 0)
{
logger.Log("Publishing schema changes", LogLevel.Debug);
PublishSchemaChanges(batch);
sourceDataUtils.WriteBitWise(Config.RelayDB, batch.CTID, Convert.ToInt32(SyncBitWise.PublishSchemaChanges), AgentType.ShardCoordinator);
}
ConsolidateTables(batch);
ConsolidateInfoTables(batch);
}
private void ApplySchemaChangesAndWrite(ChangeTrackingBatch ctb)
{
if ((ctb.SyncBitWise & Convert.ToInt32(SyncBitWise.ApplySchemaChanges)) == 0)
{
logger.Log("Applying schema changes", LogLevel.Debug);
var sw = Stopwatch.StartNew();
ApplySchemaChanges(Config.RelayDB, Config.SlaveDB, ctb.CTID);
sourceDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.ApplySchemaChanges), AgentType.Slave);
ctb.SyncBitWise += Convert.ToInt32(SyncBitWise.ApplySchemaChanges);
logger.Timing(StepTimingKey("ApplySchemaChanges"), (int)sw.ElapsedMilliseconds);
}
}
private void ConsolidateInfoTables(ChangeTrackingBatch batch)
{
logger.Log("Consolidating info tables", LogLevel.Debug);
var rowCounts = GetRowCounts(Config.Tables, Config.RelayDB, batch.CTID);
//publish table info with actual rowcounts for the tables that had changes
PublishTableInfo(tablesWithChanges, Config.RelayDB, rowCounts, batch.CTID);
//pull in the table info for tables that didn't have changes from the shard databases
//start with the master shard database first because it is assumed to have the
//most up to date schema information
foreach (var sd in shardDatabases.OrderBy(d => d == Config.MasterShard ? 0 : 1))
{
sourceDataUtils.MergeInfoTable(sd, Config.RelayDB, batch.CTID);
}
}
private ChangeTrackingBatch CreateNewVersionsForShards(ChangeTrackingBatch batch)
{
logger.Log("Creating new CT versions for slaves", LogLevel.Info);
Int64 CTID = sourceDataUtils.CreateCTVersion(Config.RelayDB, 0, 0).CTID;
Logger.SetProperty("CTID", CTID);
foreach (var db in shardDatabases)
{
var b = new ChangeTrackingBatch(sourceDataUtils.GetLastCTBatch(db, AgentType.ShardCoordinator));
sourceDataUtils.CreateShardCTVersion(db, CTID, b.SyncStopVersion);
}
logger.Log("Created new CT Version " + CTID + " on " + string.Join(",", shardDatabases), LogLevel.Info);
batch = new ChangeTrackingBatch(CTID, 0, 0, 0);
return(batch);
}
public void TestInitializeBatch_LastBatchSuccessful()
{
DataTable tblCTVersion = ((TestDataUtils)destDataUtils).testData.Tables["dbo.tblCTVersion", "RELAY.CT_testdb"];
DataRow row = tblCTVersion.Rows[tblCTVersion.Rows.Count - 1];
Int64 CTID = row.Field <Int64>("CTID");
row["syncBitWise"] = SyncBitWise.UploadChanges;
row["syncStartVersion"] = 1000;
row["syncStopVersion"] = 2000;
ChangeTrackingBatch expected = new ChangeTrackingBatch(CTID + 1, 2000, 2500, 0);
ChangeTrackingBatch actual = InitializeBatch(2500);
Assert.True(actual.Equals(expected));
//undo any writes
((TestDataUtils)destDataUtils).ReloadData("test1");
}
public void TestInitializeBatch_LastBatchPartial()
{
DataTable tblCTVersion = ((TestDataUtils)destDataUtils).testData.Tables["dbo.tblCTVersion", "RELAY.CT_testdb"];
DataRow row = tblCTVersion.Rows[tblCTVersion.Rows.Count - 1];
Int64 CTID = row.Field <Int64>("CTID");
Int32 syncBitWise = Convert.ToInt32(SyncBitWise.CaptureChanges) + Convert.ToInt32(SyncBitWise.PublishSchemaChanges);
row["syncBitWise"] = syncBitWise;
row["syncStartVersion"] = 1000;
row["syncStopVersion"] = 2000;
//in this case we expect to just continue working on the same batch with no changes
ChangeTrackingBatch expected = new ChangeTrackingBatch(CTID, 1000, 2000, syncBitWise);
ChangeTrackingBatch actual = InitializeBatch(2500);
Assert.True(actual.Equals(expected));
}
private void MergeTable(ChangeTrackingBatch batch, Dictionary <string, List <TColumn> > dbColumns, TableConf table, string firstDB)
{
logger.Log(new { message = "Merging table", Table = table.Name }, LogLevel.Debug);
var dc = DataCopyFactory.GetInstance(Config.RelayType, Config.RelayType, sourceDataUtils, sourceDataUtils, logger);
dc.CopyTableDefinition(firstDB, table.ToCTName(batch.CTID), table.SchemaName, Config.RelayDB, table.ToCTName(batch.CTID));
foreach (var dbNameFields in dbColumns)
{
var dbName = dbNameFields.Key;
var columns = dbNameFields.Value;
if (columns.Count == 0)
{
//no changes in this DB for this table
continue;
}
sourceDataUtils.MergeCTTable(table, Config.RelayDB, dbName, batch.CTID);
}
}
public void TestPublishTableInfo()
{
//undo changes
((TestDataUtils)destDataUtils).ReloadData("test1");
ctb = new ChangeTrackingBatch(101, 1000, 2000, 0);
PublishTableInfo(Config.Tables, "CT_testdb", changesCaptured, ctb.CTID);
DataRow actual = ((TestDataUtils)destDataUtils).testData.Tables["dbo.tblCTTableInfo_101", "RELAY.CT_testdb"].Rows[0];
Assert.True(actual.Field <string>("CtiTableName") == "test1" &&
actual.Field <string>("CtiSchemaName") == "dbo" &&
actual.Field <string>("CtiPKList") == "column1" &&
actual.Field <int>("CtiExpectedRows") == 1);
actual = ((TestDataUtils)destDataUtils).testData.Tables["dbo.tblCTTableInfo_101", "RELAY.CT_testdb"].Rows[1];
Assert.True(actual.Field <string>("CtiTableName") == "test2" &&
actual.Field <string>("CtiSchemaName") == "dbo" &&
actual.Field <string>("CtiPKList") == "column1" &&
actual.Field <int>("CtiExpectedRows") == 0);
}
public void TestInitializeBatch_LastBatchFailed()
{
DataTable tblCTVersion = ((TestDataUtils)destDataUtils).testData.Tables["dbo.tblCTVersion", "RELAY.CT_testdb"];
DataRow row = tblCTVersion.Rows[tblCTVersion.Rows.Count - 1];
Int64 CTID = row.Field <Int64>("CTID");
row["syncBitWise"] = 0;
row["syncStartVersion"] = 1000;
row["syncStopVersion"] = 2000;
//in this case we expect syncStopVersion to get updated from 2000 to 2500 but the rest of the batch to stay the same
ChangeTrackingBatch expected = new ChangeTrackingBatch(CTID, 1000, 2500, 0);
ChangeTrackingBatch actual = InitializeBatch(2500);
Assert.True(actual.Equals(expected));
//undo any writes
((TestDataUtils)destDataUtils).ReloadData("test1");
}
/// <summary>
/// Initializes version/batch info for a run
/// </summary>
/// <returns>List of change tracking batches to work on</returns>
private IList <ChangeTrackingBatch> InitializeBatch(int bitwise)
{
ChangeTrackingBatch ctb;
IList <ChangeTrackingBatch> batches = new List <ChangeTrackingBatch>();
DataRow lastBatch = sourceDataUtils.GetLastCTBatch(Config.RelayDB, AgentType.Slave, Config.Slave);
//apparently we shouldn't hit the code block below except for unit tests?
//in the db init we're supposed to write the first row, so it (in theory) shouldn't return null
if (lastBatch == null)
{
ctb = new ChangeTrackingBatch(1, 0, 0, 0);
batches.Add(ctb);
logger.Log("Couldn't find any records for the last change tracking batch! Returning the default new CTB.", LogLevel.Warn);
return(batches);
}
if ((lastBatch.Field <Int32>("syncBitWise") & bitwise) == bitwise)
{
logger.Log("Last batch was successful, checking for new batches.", LogLevel.Info);
DataTable pendingVersions = sourceDataUtils.GetPendingCTVersions(Config.RelayDB, lastBatch.Field <Int64>("CTID"), Convert.ToInt32(SyncBitWise.UploadChanges));
logger.Log("Retrieved " + pendingVersions.Rows.Count + " pending CT version(s) to work on.", LogLevel.Info);
foreach (DataRow row in pendingVersions.Rows)
{
ctb = new ChangeTrackingBatch(row);
batches.Add(ctb);
sourceDataUtils.CreateSlaveCTVersion(Config.RelayDB, ctb, Config.Slave);
}
return(batches);
}
ctb = new ChangeTrackingBatch(lastBatch);
logger.Log(new { message = "Last batch failed, retrying", CTID = ctb.CTID }, LogLevel.Warn);
batches.Add(ctb);
return(batches);
}
public override void Run()
{
var batch = new ChangeTrackingBatch(sourceDataUtils.GetLastCTBatch(Config.RelayDB, AgentType.ShardCoordinator));
Logger.SetProperty("CTID", batch.CTID);
if ((batch.SyncBitWise & Convert.ToInt32(SyncBitWise.UploadChanges)) > 0)
{
CreateNewVersionsForShards(batch);
return;
}
logger.Log("Working on CTID " + batch.CTID, LogLevel.Info);
if (AllShardMastersDone(batch))
{
logger.Log("All shard masters are done, checking field lists", LogLevel.Info);
tableDBFieldLists = GetFieldListsByDB(batch.CTID);
if (SchemasOutOfSync(tableDBFieldLists.Values))
{
foreach (var sd in shardDatabases)
{
sourceDataUtils.RevertCTBatch(sd, batch.CTID);
}
logger.Log("Schemas out of sync, quitting", LogLevel.Info);
return;
}
logger.Log("Field lists in sync, consolidating", LogLevel.Info);
Consolidate(batch);
sourceDataUtils.WriteBitWise(Config.RelayDB, batch.CTID,
Convert.ToInt32(SyncBitWise.CaptureChanges) | Convert.ToInt32(SyncBitWise.UploadChanges), AgentType.ShardCoordinator);
//now that the current batch is done, create a new one for the masters to work on
CreateNewVersionsForShards(batch);
}
else
{
logger.Log("Not all shards are done yet, waiting until they catch up", LogLevel.Info);
}
}
/// <summary>
/// Consolidates multiple batches and runs them as a group
/// </summary>
/// <param name="ctidTable">DataTable object listing all the batches</param>
private void RunMultiBatch(IList <ChangeTrackingBatch> batches)
{
ChangeTrackingBatch endBatch = batches.OrderBy(item => item.CTID).Last();
Logger.SetProperty("CTID", endBatch.CTID);
logger.Log(string.Format("Running consolidated batches from CTID {0} to {1}",
batches.OrderBy(item => item.CTID).First().CTID, endBatch.CTID), LogLevel.Info);
//from here forward all operations will use the bitwise value for the last CTID since they are operating on this whole set of batches
logger.Log("Populating changetable list for all CTIDs", LogLevel.Info);
List <ChangeTable> existingCTTables = PopulateTableList(Config.Tables, Config.RelayDB, batches);
logger.Log("Capturing field lists", LogLevel.Info);
SetFieldListsSlave(Config.RelayDB, Config.Tables, endBatch, existingCTTables);
foreach (ChangeTrackingBatch batch in batches)
{
if ((batch.SyncBitWise & Convert.ToInt32(SyncBitWise.ApplySchemaChanges)) == 0)
{
logger.Log("Applying schema changes for batch " + batch.CTID, LogLevel.Debug);
ApplySchemaChanges(Config.RelayDB, Config.SlaveDB, batch.CTID);
sourceDataUtils.WriteBitWise(Config.RelayDB, batch.CTID, Convert.ToInt32(SyncBitWise.ApplySchemaChanges), AgentType.Slave);
}
}
if ((endBatch.SyncBitWise & Convert.ToInt32(SyncBitWise.ConsolidateBatches)) == 0)
{
var sw = Stopwatch.StartNew();
logger.Log("Consolidating batches", LogLevel.Info);
ConsolidateBatches(existingCTTables);
sourceDataUtils.WriteBitWise(Config.RelayDB, endBatch.CTID, Convert.ToInt32(SyncBitWise.ConsolidateBatches), AgentType.Slave);
logger.Timing(StepTimingKey("ConsolidateBatches"), (int)sw.ElapsedMilliseconds);
}
if ((endBatch.SyncBitWise & Convert.ToInt32(SyncBitWise.DownloadChanges)) == 0)
{
var sw = Stopwatch.StartNew();
logger.Log("Downloading consolidated changetables", LogLevel.Info);
CopyChangeTables(Config.Tables, Config.RelayDB, Config.SlaveCTDB, endBatch.CTID, existingCTTables, isConsolidated: true);
logger.Log("Changes downloaded successfully", LogLevel.Debug);
sourceDataUtils.WriteBitWise(Config.RelayDB, endBatch.CTID, Convert.ToInt32(SyncBitWise.DownloadChanges), AgentType.Slave);
logger.Timing(StepTimingKey("DownloadChanges"), (int)sw.ElapsedMilliseconds);
}
RowCounts total;
if ((endBatch.SyncBitWise & Convert.ToInt32(SyncBitWise.ApplyChanges)) == 0)
{
var sw = Stopwatch.StartNew();
logger.Log("Applying changes", LogLevel.Info);
total = ApplyChanges(existingCTTables, endBatch.CTID, isConsolidated: true);
sourceDataUtils.WriteBitWise(Config.RelayDB, endBatch.CTID, Convert.ToInt32(SyncBitWise.ApplyChanges), AgentType.Slave);
//Expected rowcounts across multiple batches are not currently calculated, it's unclear
//how we would actually want to calculate them since by the nature of consolidation duplicate inserts/updates are eliminated.
//Commenting this out for now.
//RecordRowCounts(total, endBatch);
logger.Timing(StepTimingKey("ApplyChanges"), (int)sw.ElapsedMilliseconds);
}
logger.Log("Figuring out the last batch for each changetable", LogLevel.Debug);
var lastChangedTables = new List <ChangeTable>();
foreach (var group in existingCTTables.GroupBy(c => c.name))
{
var table = group.First();
lastChangedTables.Add(new ChangeTable(table.name, endBatch.CTID, table.schemaName, table.slaveName));
}
if ((endBatch.SyncBitWise & Convert.ToInt32(SyncBitWise.SyncHistoryTables)) == 0)
{
var sw = Stopwatch.StartNew();
logger.Log("Syncing history tables", LogLevel.Info);
SyncHistoryTables(Config.SlaveCTDB, lastChangedTables, isConsolidated: true);
sourceDataUtils.WriteBitWise(Config.RelayDB, endBatch.CTID, Convert.ToInt32(SyncBitWise.SyncHistoryTables), AgentType.Slave);
logger.Timing(StepTimingKey("SyncHistoryTables"), (int)sw.ElapsedMilliseconds);
}
//success! go through and mark all the batches as complete in the db
sourceDataUtils.MarkBatchesComplete(Config.RelayDB, batches.Select(b => b.CTID), DateTime.Now, Config.Slave);
logger.Timing("db.mssql_changetracking_counters.DataAppliedAsOf." + Config.SlaveDB, DateTime.Now.Hour + DateTime.Now.Minute / 60);
}
private void PublishSchemaChanges(ChangeTrackingBatch batch)
{
var dc = DataCopyFactory.GetInstance(Config.RelayType, Config.RelayType, sourceDataUtils, sourceDataUtils, logger);
dc.CopyTable(Config.MasterShard, batch.schemaChangeTable, "dbo", Config.RelayDB, Config.DataCopyTimeout);
}
private ChangeTrackingBatch CreateNewVersionsForShards(ChangeTrackingBatch batch)
{
logger.Log("Creating new CT versions for slaves", LogLevel.Info);
Int64 CTID = sourceDataUtils.CreateCTVersion(Config.RelayDB, 0, 0).CTID;
Logger.SetProperty("CTID", CTID);
foreach (var db in shardDatabases) {
var b = new ChangeTrackingBatch(sourceDataUtils.GetLastCTBatch(db, AgentType.ShardCoordinator));
sourceDataUtils.CreateShardCTVersion(db, CTID, b.SyncStopVersion);
}
logger.Log("Created new CT Version " + CTID + " on " + string.Join(",", shardDatabases), LogLevel.Info);
batch = new ChangeTrackingBatch(CTID, 0, 0, 0);
return batch;
}
private void ConsolidateInfoTables(ChangeTrackingBatch batch)
{
logger.Log("Consolidating info tables", LogLevel.Debug);
var rowCounts = GetRowCounts(Config.Tables, Config.RelayDB, batch.CTID);
//publish table info with actual rowcounts for the tables that had changes
PublishTableInfo(tablesWithChanges, Config.RelayDB, rowCounts, batch.CTID);
//pull in the table info for tables that didn't have changes from the shard databases
//start with the master shard database first because it is assumed to have the
//most up to date schema information
foreach (var sd in shardDatabases.OrderBy(d => d == Config.MasterShard ? 0 : 1)) {
sourceDataUtils.MergeInfoTable(sd, Config.RelayDB, batch.CTID);
}
}
private bool AllShardMastersDone(ChangeTrackingBatch batch)
{
return shardDatabases.All(dbName => (sourceDataUtils.GetCTBatch(dbName, batch.CTID).SyncBitWise & Convert.ToInt32(SyncBitWise.UploadChanges)) > 0);
}
/// <summary>
/// Runs a single change tracking batch
/// </summary>
/// <param name="CTID">Change tracking batch object to work on</param>
private void RunSingleBatch(ChangeTrackingBatch ctb)
{
Stopwatch sw;
logger.Log("Applying schema changes ", LogLevel.Info);
ApplySchemaChangesAndWrite(ctb);
//marking this field so that all completed slave batches will have the same values
sourceDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.ConsolidateBatches), AgentType.Slave);
if ((ctb.SyncBitWise & Convert.ToInt32(SyncBitWise.DownloadChanges)) == 0) {
logger.Log("Downloading changes", LogLevel.Info);
sw = Stopwatch.StartNew();
CopyChangeTables(Config.Tables, Config.RelayDB, Config.SlaveCTDB, ctb.CTID);
logger.Log("CopyChangeTables: " + sw.Elapsed, LogLevel.Trace);
sourceDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.DownloadChanges), AgentType.Slave);
logger.Timing(StepTimingKey("DownloadChanges"), (int)sw.ElapsedMilliseconds);
}
logger.Log("Populating table list", LogLevel.Info);
List<ChangeTable> existingCTTables = PopulateTableList(Config.Tables, Config.RelayDB, new List<ChangeTrackingBatch>() { ctb });
if ((ctb.SyncBitWise & Convert.ToInt32(SyncBitWise.ApplyChanges)) == 0) {
logger.Log("Applying changes", LogLevel.Info);
sw = Stopwatch.StartNew();
SetFieldListsSlave(Config.RelayDB, Config.Tables, ctb, existingCTTables);
RowCounts total = ApplyChanges(existingCTTables, ctb.CTID, isConsolidated: false);
RecordRowCounts(total, ctb);
logger.Log("ApplyChanges: " + sw.Elapsed, LogLevel.Trace);
sourceDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.ApplyChanges), AgentType.Slave);
logger.Timing(StepTimingKey("ApplyChanges"), (int)sw.ElapsedMilliseconds);
}
logger.Log("Syncing history tables", LogLevel.Info);
sw = Stopwatch.StartNew();
SyncHistoryTables(Config.SlaveCTDB, existingCTTables, isConsolidated: false);
logger.Log("SyncHistoryTables: " + sw.Elapsed, LogLevel.Trace);
var syncStopTime = DateTime.Now;
sourceDataUtils.MarkBatchComplete(Config.RelayDB, ctb.CTID, syncStopTime, Config.Slave);
string key = String.Format(
"db.mssql_changetracking_counters.DataDurationToSync{0}.{1}",
Config.Slave.Replace('.', '_'),
Config.SlaveDB);
logger.Increment(key, (int)(syncStopTime - ctb.SyncStartTime.Value).TotalMinutes);
logger.Timing(StepTimingKey("SyncHistoryTables"), (int)sw.ElapsedMilliseconds);
}
/// <summary>
/// Initializes version/batch info for a run
/// </summary>
/// <returns>List of change tracking batches to work on</returns>
private IList<ChangeTrackingBatch> InitializeBatch(int bitwise)
{
ChangeTrackingBatch ctb;
IList<ChangeTrackingBatch> batches = new List<ChangeTrackingBatch>();
DataRow lastBatch = sourceDataUtils.GetLastCTBatch(Config.RelayDB, AgentType.Slave, Config.Slave);
if (lastBatch == null) {
ctb = new ChangeTrackingBatch(1, 0, 0, 0);
batches.Add(ctb);
return batches;
}
if ((lastBatch.Field<Int32>("syncBitWise") & bitwise) == bitwise) {
logger.Log("Last batch was successful, checking for new batches.", LogLevel.Info);
DataTable pendingVersions = sourceDataUtils.GetPendingCTVersions(Config.RelayDB, lastBatch.Field<Int64>("CTID"), Convert.ToInt32(SyncBitWise.UploadChanges));
logger.Log("Retrieved " + pendingVersions.Rows.Count + " pending CT version(s) to work on.", LogLevel.Info);
foreach (DataRow row in pendingVersions.Rows) {
ctb = new ChangeTrackingBatch(row);
batches.Add(ctb);
sourceDataUtils.CreateSlaveCTVersion(Config.RelayDB, ctb, Config.Slave);
}
return batches;
}
ctb = new ChangeTrackingBatch(lastBatch);
logger.Log(new { message = "Last batch failed, retrying", CTID = ctb.CTID }, LogLevel.Warn);
batches.Add(ctb);
return batches;
}
private void PublishSchemaChanges(ChangeTrackingBatch batch)
{
var dc = DataCopyFactory.GetInstance(Config.RelayType, Config.RelayType, sourceDataUtils, sourceDataUtils, logger);
dc.CopyTable(Config.MasterShard, batch.schemaChangeTable, "dbo", Config.RelayDB, Config.DataCopyTimeout);
}
/// <summary>
/// Loops through passed in array of table objects, creates changedata tables for each one on the CT DB
/// </summary>
/// <param name="tables">Array of table config objects to create CT tables for</param>
/// <param name="sourceDB">Database the source data lives in</param>
/// <param name="sourceCTDB">Database the changetables should go to</param>
/// <param name="startVersion">Change tracking version to start with</param>
/// <param name="stopVersion">Change tracking version to stop at</param>
/// <param name="CTID">CT batch ID this is being run for</param>
protected IDictionary <string, Int64> CreateChangeTables(string sourceDB, string sourceCTDB, ChangeTrackingBatch batch)
{
var changesCaptured = new ConcurrentDictionary <string, Int64>();
var actions = new List <Action>();
foreach (TableConf t in Config.Tables)
{
//local variables inside the loop required for the action to bind properly
TableConf table = t;
long rowsAffected;
Action act = () => {
logger.Log("Creating changetable for " + table.SchemaName + "." + table.Name, LogLevel.Debug);
rowsAffected = CreateChangeTable(table, sourceDB, sourceCTDB, batch);
changesCaptured.TryAdd(table.SchemaName + "." + table.Name, rowsAffected);
logger.Log(rowsAffected + " changes captured for table " + table.SchemaName + "." + table.Name, LogLevel.Trace);
};
actions.Add(act);
}
var options = new ParallelOptions();
options.MaxDegreeOfParallelism = Config.MaxThreads;
logger.Log("Parallel invocation of " + actions.Count + " change captures", LogLevel.Trace);
Parallel.Invoke(options, actions.ToArray());
return(changesCaptured);
}
/// <summary>
/// Creates changetable for an individual table
/// </summary>
/// <param name="table">Config table object to create changes for</param>
/// <param name="sourceDB">Database the source data lives in</param>
/// <param name="sourceCTDB">Database the changetables should go to</param>
/// <param name="batch">Batch to work on</param>
protected long CreateChangeTable(TableConf table, string sourceDB, string sourceCTDB, ChangeTrackingBatch batch)
{
string ctTableName = table.ToCTName(batch.CTID);
string reason;
long tableStartVersion = batch.SyncStartVersion;
long minValidVersion = sourceDataUtils.GetMinValidVersion(sourceDB, table.Name, table.SchemaName);
if (batch.SyncStartVersion == 0)
{
tableStartVersion = minValidVersion;
}
if (sourceDataUtils.IsBeingInitialized(sourceCTDB, table))
{
return(0);
}
long?initializeVersion = sourceDataUtils.GetInitializeStartVersion(sourceCTDB, table);
if (initializeVersion.HasValue)
{
tableStartVersion = initializeVersion.Value;
}
if (!ValidateSourceTable(sourceDB, table.Name, table.SchemaName, tableStartVersion, minValidVersion, out reason))
{
string message = "Change table creation impossible because : " + reason;
if (table.StopOnError)
{
throw new Exception(message);
}
else
{
logger.Log(message, LogLevel.Error);
return(0);
}
}
logger.Log("Dropping table " + ctTableName + " if it exists", LogLevel.Trace);
sourceDataUtils.DropTableIfExists(sourceCTDB, ctTableName, table.SchemaName);
logger.Log("Calling SelectIntoCTTable to create CT table", LogLevel.Trace);
Int64 rowsAffected = sourceDataUtils.SelectIntoCTTable(sourceCTDB, table, sourceDB, batch, Config.QueryTimeout, tableStartVersion);
logger.Log("Rows affected for table " + table.SchemaName + "." + table.Name + ": " + Convert.ToString(rowsAffected), LogLevel.Debug);
return(rowsAffected);
}
public void CreateSlaveCTVersion(string dbName, ChangeTrackingBatch ctb, string slaveIdentifier)
{
throw new NotImplementedException("Netezza is only supported as a slave!");
}
public Dictionary<TableConf, IList<string>> GetAllPrimaryKeys(string dbName, IEnumerable<TableConf> tables, ChangeTrackingBatch batch)
{
throw new NotImplementedException();
}
public int SelectIntoCTTable(string sourceCTDB, TableConf table, string sourceDB, ChangeTrackingBatch batch, int timeout, long? startVersionOverride)
{
throw new NotImplementedException("Netezza is only supported as a slave!");
}
/// <summary>
/// Creates changetable for an individual table
/// </summary>
/// <param name="table">Config table object to create changes for</param>
/// <param name="sourceDB">Database the source data lives in</param>
/// <param name="sourceCTDB">Database the changetables should go to</param>
/// <param name="batch">Batch to work on</param>
protected long CreateChangeTable(TableConf table, string sourceDB, string sourceCTDB, ChangeTrackingBatch batch)
{
string ctTableName = table.ToCTName(batch.CTID);
string reason;
long tableStartVersion = batch.SyncStartVersion;
long minValidVersion = sourceDataUtils.GetMinValidVersion(sourceDB, table.Name, table.SchemaName);
if (batch.SyncStartVersion == 0) {
tableStartVersion = minValidVersion;
}
if (sourceDataUtils.IsBeingInitialized(sourceCTDB, table)) {
return 0;
}
long? initializeVersion = sourceDataUtils.GetInitializeStartVersion(sourceCTDB, table);
if (initializeVersion.HasValue) {
tableStartVersion = initializeVersion.Value;
}
if (!ValidateSourceTable(sourceDB, table.Name, table.SchemaName, tableStartVersion, minValidVersion, out reason)) {
string message = "Change table creation impossible because : " + reason;
if (table.StopOnError) {
throw new Exception(message);
} else {
logger.Log(message, LogLevel.Error);
return 0;
}
}
logger.Log("Dropping table " + ctTableName + " if it exists", LogLevel.Trace);
sourceDataUtils.DropTableIfExists(sourceCTDB, ctTableName, table.SchemaName);
logger.Log("Calling SelectIntoCTTable to create CT table", LogLevel.Trace);
Int64 rowsAffected = sourceDataUtils.SelectIntoCTTable(sourceCTDB, table, sourceDB, batch, Config.QueryTimeout, tableStartVersion);
logger.Log("Rows affected for table " + table.SchemaName + "." + table.Name + ": " + Convert.ToString(rowsAffected), LogLevel.Debug);
return rowsAffected;
}
public void CreateSlaveCTVersion(string dbName, ChangeTrackingBatch ctb, string slaveIdentifier)
{
throw new NotImplementedException();
}
/// <summary>
/// Loops through passed in array of table objects, creates changedata tables for each one on the CT DB
/// </summary>
/// <param name="tables">Array of table config objects to create CT tables for</param>
/// <param name="sourceDB">Database the source data lives in</param>
/// <param name="sourceCTDB">Database the changetables should go to</param>
/// <param name="startVersion">Change tracking version to start with</param>
/// <param name="stopVersion">Change tracking version to stop at</param>
/// <param name="CTID">CT batch ID this is being run for</param>
protected IDictionary<string, Int64> CreateChangeTables(string sourceDB, string sourceCTDB, ChangeTrackingBatch batch)
{
var changesCaptured = new ConcurrentDictionary<string, Int64>();
var actions = new List<Action>();
foreach (TableConf t in Config.Tables) {
//local variables inside the loop required for the action to bind properly
TableConf table = t;
long rowsAffected;
Action act = () => {
logger.Log("Creating changetable for " + table.SchemaName + "." + table.Name, LogLevel.Debug);
rowsAffected = CreateChangeTable(table, sourceDB, sourceCTDB, batch);
changesCaptured.TryAdd(table.SchemaName + "." + table.Name, rowsAffected);
logger.Log(rowsAffected + " changes captured for table " + table.SchemaName + "." + table.Name, LogLevel.Trace);
};
actions.Add(act);
}
var options = new ParallelOptions();
options.MaxDegreeOfParallelism = Config.MaxThreads;
logger.Log("Parallel invocation of " + actions.Count + " change captures", LogLevel.Trace);
Parallel.Invoke(options, actions.ToArray());
return changesCaptured;
}
public int SelectIntoCTTable(string sourceCTDB, TableConf table, string sourceDB, ChangeTrackingBatch ctb, int timeout, long? overrideStartVersion)
{
//no good way to fake this with DataTables so just return and make sure we are also unit testing the
//methods that generate these sfield lists
return testData.Tables[table.SchemaName + "." + table.ToCTName(ctb.CTID), GetTableSpace(sourceCTDB)].Rows.Count;
}
private void RecordRowCounts(RowCounts actual, ChangeTrackingBatch ctb)
{
var expected = sourceDataUtils.GetExpectedRowCounts(Config.RelayDB, ctb.CTID);
logger.Log("Expected row counts: " + expected + " | actual: " + actual, LogLevel.Info);
double diff = expected - actual.Inserted;
double mismatch;
if (expected == 0) {
if (actual.Inserted == 0) {
mismatch = 0.0;
} else {
logger.Log("Expected 0 rows, got " + actual.Inserted + " rows inserted on slave.", LogLevel.Error);
return;
}
} else {
mismatch = diff / expected;
}
int percentDiff = (int)(mismatch * 100);
string key = string.Format("db.mssql_changetracking_counters.RecordCountMismatchProd{0}.{1}", Config.Slave.Replace('.', '_'), Config.SlaveDB);
logger.Increment(key, percentDiff);
}
public void CreateSlaveCTVersion(string dbName, ChangeTrackingBatch ctb, string slaveIdentifier)
{
throw new NotImplementedException();
}
private void SetFieldListsSlave(string dbName, IEnumerable<TableConf> tables, ChangeTrackingBatch batch, List<ChangeTable> existingCTTables)
{
//map each table to the last appropriate CT table, ditching tableconfs with no corresponding CT tables
var tableCTName = new Dictionary<TableConf, string>();
foreach (var table in tables) {
ChangeTable changeTable = existingCTTables.Where(ct => ct.name == table.Name).OrderBy(ct => ct.CTID).LastOrDefault();
if (changeTable == null) {
continue;
}
long lastCTIDWithChanges = changeTable.CTID.Value;
tableCTName[table] = table.ToCTName(lastCTIDWithChanges);
}
Dictionary<TableConf, IList<TColumn>> allColumnsByTable = sourceDataUtils.GetAllFields(dbName, tableCTName);
//even though GetAllFields returns whether it's part of the PK, the PK info will always say false on the relay since the relay
//doesn't necessarily define primary keys
Dictionary<TableConf, IList<string>> primaryKeysByTable = sourceDataUtils.GetAllPrimaryKeys(dbName, tableCTName.Keys, batch);
//tableCTName.Keys instead of tables because we've already filtered this for tables that don't have change tables
//note: allColumnsByTable.Keys or primaryKeysByTable.Keys should work just as well
foreach (var table in tableCTName.Keys) {
IEnumerable<TColumn> columns;
try {
//this is a hacky solution but we will have these columns in CT tables but actually are not interested in them here.
columns = allColumnsByTable[table].Where(c => (c.name != "SYS_CHANGE_VERSION" && c.name != "SYS_CHANGE_OPERATION"));
} catch (KeyNotFoundException) {
var e = new Exception("Column list for table " + tableCTName[table] + " not found in " + dbName);
HandleException(e, table);
//if we handled the exception by just logging an error, this table is still broken so we need to continue
continue;
}
IList<string> pks;
try {
pks = primaryKeysByTable[table];
} catch (KeyNotFoundException) {
var e = new Exception("Primary keys for table " + table.FullName + " not found in " + dbName + ".dbo.tblCTTableInfo_" + batch.CTID);
HandleException(e, table);
//if we handled the exception by just logging an error, this table is still broken so we need to continue
continue;
}
foreach (var pk in pks) {
columns.First((c => c.name == pk)).isPk = true;
}
SetFieldList(table, columns);
}
}
/// <summary>
/// Initializes version/batch info for a run
/// </summary>
/// <returns>List of change tracking batches to work on</returns>
private IList<ChangeTrackingBatch> InitializeBatch(int bitwise)
{
ChangeTrackingBatch ctb;
IList<ChangeTrackingBatch> batches = new List<ChangeTrackingBatch>();
DataRow lastBatch = sourceDataUtils.GetLastCTBatch(Config.RelayDB, AgentType.Slave, Config.Slave);
//apparently we shouldn't hit the code block below except for unit tests?
//in the db init we're supposed to write the first row, so it (in theory) shouldn't return null
if (lastBatch == null) {
ctb = new ChangeTrackingBatch(1, 0, 0, 0);
batches.Add(ctb);
logger.Log("Couldn't find any records for the last change tracking batch! Returning the default new CTB.", LogLevel.Warn);
return batches;
}
if ((lastBatch.Field<Int32>("syncBitWise") & bitwise) == bitwise) {
logger.Log("Last batch was successful, checking for new batches.", LogLevel.Info);
DataTable pendingVersions = sourceDataUtils.GetPendingCTVersions(Config.RelayDB, lastBatch.Field<Int64>("CTID"), Convert.ToInt32(SyncBitWise.UploadChanges));
logger.Log("Retrieved " + pendingVersions.Rows.Count + " pending CT version(s) to work on.", LogLevel.Info);
foreach (DataRow row in pendingVersions.Rows) {
ctb = new ChangeTrackingBatch(row);
batches.Add(ctb);
sourceDataUtils.CreateSlaveCTVersion(Config.RelayDB, ctb, Config.Slave);
}
return batches;
}
ctb = new ChangeTrackingBatch(lastBatch);
logger.Log(new { message = "Last batch failed, retrying", CTID = ctb.CTID }, LogLevel.Warn);
batches.Add(ctb);
return batches;
}
private void Consolidate(ChangeTrackingBatch batch)
{
if ((batch.SyncBitWise & Convert.ToInt32(SyncBitWise.PublishSchemaChanges)) == 0) {
logger.Log("Publishing schema changes", LogLevel.Debug);
PublishSchemaChanges(batch);
sourceDataUtils.WriteBitWise(Config.RelayDB, batch.CTID, Convert.ToInt32(SyncBitWise.PublishSchemaChanges), AgentType.ShardCoordinator);
}
ConsolidateTables(batch);
ConsolidateInfoTables(batch);
}
private void SetFieldListsSlave(string dbName, IEnumerable <TableConf> tables, ChangeTrackingBatch batch, List <ChangeTable> existingCTTables)
{
//map each table to the last appropriate CT table, ditching tableconfs with no corresponding CT tables
var tableCTName = new Dictionary <TableConf, string>();
foreach (var table in tables)
{
ChangeTable changeTable = existingCTTables.Where(ct => String.Compare(ct.name, table.Name, StringComparison.OrdinalIgnoreCase) == 0).OrderBy(ct => ct.CTID).LastOrDefault();
if (changeTable == null)
{
continue;
}
long lastCTIDWithChanges = changeTable.CTID.Value;
tableCTName[table] = table.ToCTName(lastCTIDWithChanges);
}
Dictionary <TableConf, IList <TColumn> > allColumnsByTable = sourceDataUtils.GetAllFields(dbName, tableCTName);
//even though GetAllFields returns whether it's part of the PK, the PK info will always say false on the relay since the relay
//doesn't necessarily define primary keys
Dictionary <TableConf, IList <string> > primaryKeysByTable = sourceDataUtils.GetAllPrimaryKeys(dbName, tableCTName.Keys, batch);
//tableCTName.Keys instead of tables because we've already filtered this for tables that don't have change tables
//note: allColumnsByTable.Keys or primaryKeysByTable.Keys should work just as well
foreach (var table in tableCTName.Keys)
{
IEnumerable <TColumn> columns;
try {
//this is a hacky solution but we will have these columns in CT tables but actually are not interested in them here.
columns = allColumnsByTable[table].Where(c => (c.name != "SYS_CHANGE_VERSION" && c.name != "SYS_CHANGE_OPERATION"));
} catch (KeyNotFoundException) {
var e = new Exception("Column list for table " + tableCTName[table] + " not found in " + dbName);
HandleException(e, table);
//if we handled the exception by just logging an error, this table is still broken so we need to continue
continue;
}
IList <string> pks;
try {
pks = primaryKeysByTable[table];
} catch (KeyNotFoundException) {
var e = new Exception("Primary keys for table " + table.FullName + " not found in " + dbName + ".dbo.tblCTTableInfo_" + batch.CTID);
HandleException(e, table);
//if we handled the exception by just logging an error, this table is still broken so we need to continue
continue;
}
foreach (var pk in pks)
{
columns.First((c => c.name == pk)).isPk = true;
}
SetFieldList(table, columns);
}
}
private void ConsolidateTables(ChangeTrackingBatch batch)
{
logger.Log("Consolidating tables", LogLevel.Info);
var actions = new List<Action>();
foreach (var tableDb in tableDBFieldLists) {
var table = tableDb.Key;
var dbColumns = tableDb.Value;
var firstDB = tableDb.Value.FirstOrDefault(t => t.Value.Count > 0).Key;
if (firstDB == null) {
logger.Log("No shard has CT changes for table " + table.Name, LogLevel.Debug);
continue;
}
tablesWithChanges.Add(table);
SetFieldList(table, firstDB, batch);
Action act = () => MergeTable(batch, dbColumns, table, firstDB);
actions.Add(act);
}
logger.Log("Parallel invocation of " + actions.Count + " table merges", LogLevel.Info);
//interestingly, Parallel.Invoke does in fact bubble up exceptions, but not until after all threads have completed.
//actually it looks like what it does is wrap its exceptions in an AggregateException. We don't ever catch those
//though because if any exceptions happen inside of MergeTable it would generally be due to things like the server
//being down or a query timeout.
var options = new ParallelOptions();
options.MaxDegreeOfParallelism = Config.MaxThreads;
Parallel.Invoke(options, actions.ToArray());
}
private bool AllShardMastersDone(ChangeTrackingBatch batch)
{
return(shardDatabases.All(dbName => (sourceDataUtils.GetCTBatch(dbName, batch.CTID).SyncBitWise & Convert.ToInt32(SyncBitWise.UploadChanges)) > 0));
}
private void MergeTable(ChangeTrackingBatch batch, Dictionary<string, List<TColumn>> dbColumns, TableConf table, string firstDB)
{
logger.Log(new { message = "Merging table", Table = table.Name }, LogLevel.Debug);
var dc = DataCopyFactory.GetInstance(Config.RelayType, Config.RelayType, sourceDataUtils, sourceDataUtils, logger);
dc.CopyTableDefinition(firstDB, table.ToCTName(batch.CTID), table.SchemaName, Config.RelayDB, table.ToCTName(batch.CTID));
foreach (var dbNameFields in dbColumns) {
var dbName = dbNameFields.Key;
var columns = dbNameFields.Value;
if (columns.Count == 0) {
//no changes in this DB for this table
continue;
}
sourceDataUtils.MergeCTTable(table, Config.RelayDB, dbName, batch.CTID);
}
}
public void CreateSlaveCTVersion(string dbName, ChangeTrackingBatch ctb, string slaveIdentifier)
{
throw new NotImplementedException("Netezza is only supported as a slave!");
}
private void SetFieldList(TableConf table, string database, ChangeTrackingBatch batch)
{
var cols = sourceDataUtils.GetFieldList(database, table.ToCTName(batch.CTID), table.SchemaName);
var pks = sourceDataUtils.GetPrimaryKeysFromInfoTable(table, batch.CTID, database);
foreach (var pk in pks) {
cols.First((c => c.name == pk)).isPk = true;
}
SetFieldList(table, cols);
}
public int SelectIntoCTTable(string sourceCTDB, TableConf table, string sourceDB, ChangeTrackingBatch batch, int timeout, long?startVersionOverride)
{
throw new NotImplementedException("Netezza is only supported as a slave!");
}
public int SelectIntoCTTable(string sourceCTDB, TableConf table, string sourceDB, ChangeTrackingBatch ctb, int timeout, long?overrideStartVersion)
{
//no good way to fake this with DataTables so just return and make sure we are also unit testing the
//methods that generate these sfield lists
return(testData.Tables[table.SchemaName + "." + table.ToCTName(ctb.CTID), GetTableSpace(sourceCTDB)].Rows.Count);
}
private void ApplySchemaChangesAndWrite(ChangeTrackingBatch ctb)
{
if ((ctb.SyncBitWise & Convert.ToInt32(SyncBitWise.ApplySchemaChanges)) == 0) {
logger.Log("Applying schema changes", LogLevel.Debug);
var sw = Stopwatch.StartNew();
ApplySchemaChanges(Config.RelayDB, Config.SlaveDB, ctb.CTID);
sourceDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.ApplySchemaChanges), AgentType.Slave);
ctb.SyncBitWise += Convert.ToInt32(SyncBitWise.ApplySchemaChanges);
logger.Timing(StepTimingKey("ApplySchemaChanges"), (int)sw.ElapsedMilliseconds);
}
}
public Dictionary <TableConf, IList <string> > GetAllPrimaryKeys(string dbName, IEnumerable <TableConf> tables, ChangeTrackingBatch batch)
{
throw new NotImplementedException();
}
public override void Run()
{
DateTime start = DateTime.Now;
logger.Log("Getting CHANGE_TRACKING_CURRENT_VERSION from master", LogLevel.Trace);
Int64 currentVersion = sourceDataUtils.GetCurrentCTVersion(Config.MasterDB);
logger.Log("Initializing CT batch", LogLevel.Info);
//set up the variables and CT version info for this run
ctb = InitializeBatch(currentVersion);
if (Config.Sharding && ctb == null) {
logger.Log("Last batch completed and there is no new batch to work on.", LogLevel.Info);
return;
}
Logger.SetProperty("CTID", ctb.CTID);
logger.Log(ctb, LogLevel.Debug);
logger.Log("Working on CTID " + ctb.CTID, LogLevel.Info);
IDictionary<string, Int64> changesCaptured;
logger.Log("Calculating field lists for configured tables", LogLevel.Info);
SetFieldLists(Config.MasterDB, Config.Tables, sourceDataUtils);
//capture changes/publish schema changes is now one unit, so we can just check either.
if ((ctb.SyncBitWise & Convert.ToInt32(SyncBitWise.CaptureChanges)) == 0) {
changesCaptured = CaptureChanges(currentVersion);
PublishSchemaChanges();
destDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.PublishSchemaChanges), AgentType.Master);
destDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.CaptureChanges), AgentType.Master);
} else {
logger.Log("CreateChangeTables succeeded on the previous run, running GetRowCounts instead to populate changesCaptured object", LogLevel.Info);
changesCaptured = GetRowCounts(Config.Tables, Config.MasterCTDB, ctb.CTID);
logger.Log("Successfully populated changesCaptured with a list of rowcounts for each changetable", LogLevel.Debug);
}
var sw = Stopwatch.StartNew();
//copy change tables from master to relay server
logger.Log("Beginning publish changetables step, copying CT tables to the relay server", LogLevel.Info);
PublishChangeTables(Config.MasterCTDB, Config.RelayDB, ctb.CTID, changesCaptured);
logger.Log("Publishing info table", LogLevel.Info);
PublishTableInfo(Config.Tables, Config.RelayDB, changesCaptured, ctb.CTID);
logger.Log("Successfully published changetables, persisting bitwise now", LogLevel.Debug);
//this signifies the end of the master's responsibility for this batch
destDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.UploadChanges), AgentType.Master);
logger.Log("Wrote bitwise value of " + Convert.ToInt32(SyncBitWise.UploadChanges) + " to tblCTVersion", LogLevel.Trace);
logger.Timing(StepTimingKey("UploadChanges"), (int)sw.ElapsedMilliseconds);
logger.Log("Master agent work complete", LogLevel.Info);
var elapsed = DateTime.Now - start;
logger.Timing(TimingKey, (int)elapsed.TotalMinutes);
sourceDataUtils.CleanUpInitializeTable(Config.MasterCTDB, ctb.SyncStartTime.Value);
}
