Пример #1
0
        public override void Run()
        {
            DateTime start = DateTime.Now;

            logger.Log("Getting CHANGE_TRACKING_CURRENT_VERSION from master", LogLevel.Trace);
            Int64 currentVersion = sourceDataUtils.GetCurrentCTVersion(Config.MasterDB);

            logger.Log("Initializing CT batch", LogLevel.Info);
            //set up the variables and CT version info for this run
            ctb = InitializeBatch(currentVersion);
            if (Config.Sharding && ctb == null)
            {
                logger.Log("Last batch completed and there is no new batch to work on.", LogLevel.Info);
                return;
            }
            Logger.SetProperty("CTID", ctb.CTID);
            logger.Log(ctb, LogLevel.Debug);

            logger.Log("Working on CTID " + ctb.CTID, LogLevel.Info);
            IDictionary <string, Int64> changesCaptured;

            logger.Log("Calculating field lists for configured tables", LogLevel.Info);
            SetFieldLists(Config.MasterDB, Config.Tables, sourceDataUtils);

            //capture changes/publish schema changes is now one unit, so we can just check either.
            if ((ctb.SyncBitWise & Convert.ToInt32(SyncBitWise.CaptureChanges)) == 0)
            {
                changesCaptured = CaptureChanges(currentVersion);
                PublishSchemaChanges();
                destDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.PublishSchemaChanges), AgentType.Master);
                destDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.CaptureChanges), AgentType.Master);
            }
            else
            {
                logger.Log("CreateChangeTables succeeded on the previous run, running GetRowCounts instead to populate changesCaptured object", LogLevel.Info);
                changesCaptured = GetRowCounts(Config.Tables, Config.MasterCTDB, ctb.CTID);
                logger.Log("Successfully populated changesCaptured with a list of rowcounts for each changetable", LogLevel.Debug);
            }

            var sw = Stopwatch.StartNew();

            //copy change tables from master to relay server
            logger.Log("Beginning publish changetables step, copying CT tables to the relay server", LogLevel.Info);
            PublishChangeTables(Config.MasterCTDB, Config.RelayDB, ctb.CTID, changesCaptured);
            logger.Log("Publishing info table", LogLevel.Info);
            PublishTableInfo(Config.Tables, Config.RelayDB, changesCaptured, ctb.CTID);
            logger.Log("Successfully published changetables, persisting bitwise now", LogLevel.Debug);

            //this signifies the end of the master's responsibility for this batch
            destDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.UploadChanges), AgentType.Master);
            logger.Log("Wrote bitwise value of " + Convert.ToInt32(SyncBitWise.UploadChanges) + " to tblCTVersion", LogLevel.Trace);
            logger.Timing(StepTimingKey("UploadChanges"), (int)sw.ElapsedMilliseconds);

            logger.Log("Master agent work complete", LogLevel.Info);
            var elapsed = DateTime.Now - start;

            logger.Timing(TimingKey, (int)elapsed.TotalMinutes);

            sourceDataUtils.CleanUpInitializeTable(Config.MasterCTDB, ctb.SyncStartTime.Value);
        }
Пример #2
0
        private void ConsolidateTables(ChangeTrackingBatch batch)
        {
            logger.Log("Consolidating tables", LogLevel.Info);
            var actions = new List <Action>();

            foreach (var tableDb in tableDBFieldLists)
            {
                var table     = tableDb.Key;
                var dbColumns = tableDb.Value;
                var firstDB   = tableDb.Value.FirstOrDefault(t => t.Value.Count > 0).Key;
                if (firstDB == null)
                {
                    logger.Log("No shard has CT changes for table " + table.Name, LogLevel.Debug);
                    continue;
                }
                tablesWithChanges.Add(table);
                SetFieldList(table, firstDB, batch);

                Action act = () => MergeTable(batch, dbColumns, table, firstDB);
                actions.Add(act);
            }
            logger.Log("Parallel invocation of " + actions.Count + " table merges", LogLevel.Info);
            //interestingly, Parallel.Invoke does in fact bubble up exceptions, but not until after all threads have completed.
            //actually it looks like what it does is wrap its exceptions in an AggregateException. We don't ever catch those
            //though because if any exceptions happen inside of MergeTable it would generally be due to things like the server
            //being down or a query timeout.
            var options = new ParallelOptions();

            options.MaxDegreeOfParallelism = Config.MaxThreads;
            Parallel.Invoke(options, actions.ToArray());
        }
Пример #3
0
        public void TestSchemasOutOfSync()
        {
            var ctb          = new ChangeTrackingBatch(0, 0, 0, 0);
            var dbFieldLists = new List <Dictionary <string, List <TColumn> > >();
            var db1          = new List <TColumn> {
                new TColumn("a", false, null, true)
            };
            var db2 = new List <TColumn> {
                new TColumn("a", false, null, true)
            };
            var dict1 = new Dictionary <string, List <TColumn> > {
                { "db1", db1 },
                { "db2", db2 }
            };

            dbFieldLists.Add(dict1);
            shardDatabases = new List <string> {
                "db1", "db2"
            };
            Assert.False(SchemasOutOfSync(dbFieldLists));
            db1[0] = new TColumn("a", true, null, true);
            Assert.True(SchemasOutOfSync(dbFieldLists));
            db1.Clear();
            db1.Add(new TColumn("a", false, null, true));
            db1.Add(new TColumn("b", false, null, true));
            db2.Clear();
            db2.Add(new TColumn("b", false, null, true));
            db2.Add(new TColumn("a", false, null, true));
            Assert.False(SchemasOutOfSync(dbFieldLists));
        }
Пример #4
0
 public void TestSchemasOutOfSync()
 {
     var ctb = new ChangeTrackingBatch(0, 0, 0, 0);
     var dbFieldLists = new List<Dictionary<string, List<TColumn>>>();
     var db1 = new List<TColumn>{
         new TColumn("a",false, null, true)
     };
     var db2 = new List<TColumn>{
         new TColumn("a",false, null, true)
     };
     var dict1 = new Dictionary<string, List<TColumn>>{
         {"db1", db1},
         {"db2",db2}
     };
     dbFieldLists.Add(dict1);
     shardDatabases = new List<string> { "db1", "db2" };
     Assert.False(SchemasOutOfSync(dbFieldLists));
     db1[0] = new TColumn("a", true, null, true);
     Assert.True(SchemasOutOfSync(dbFieldLists));
     db1.Clear();
     db1.Add(new TColumn("a", false, null, true));
     db1.Add(new TColumn("b", false, null, true));
     db2.Clear();
     db2.Add(new TColumn("b", false, null, true));
     db2.Add(new TColumn("a", false, null, true));
     Assert.False(SchemasOutOfSync(dbFieldLists));
 }
Пример #5
0
        public override void Run()
        {
            var batch = new ChangeTrackingBatch(sourceDataUtils.GetLastCTBatch(Config.RelayDB, AgentType.ShardCoordinator));
            Logger.SetProperty("CTID", batch.CTID);
            if ((batch.SyncBitWise & Convert.ToInt32(SyncBitWise.UploadChanges)) > 0) {
                CreateNewVersionsForShards(batch);
                return;
            }
            logger.Log("Working on CTID " + batch.CTID, LogLevel.Info);

            if (AllShardMastersDone(batch)) {
                logger.Log("All shard masters are done, checking field lists", LogLevel.Info);
                tableDBFieldLists = GetFieldListsByDB(batch.CTID);
                if (SchemasOutOfSync(tableDBFieldLists.Values)) {
                    foreach (var sd in shardDatabases) {
                        sourceDataUtils.RevertCTBatch(sd, batch.CTID);
                    }
                    logger.Log("Schemas out of sync, quitting", LogLevel.Info);
                    return;
                }
                logger.Log("Field lists in sync, consolidating", LogLevel.Info);
                Consolidate(batch);
                sourceDataUtils.WriteBitWise(Config.RelayDB, batch.CTID,
                    Convert.ToInt32(SyncBitWise.CaptureChanges) | Convert.ToInt32(SyncBitWise.UploadChanges), AgentType.ShardCoordinator);
                //now that the current batch is done, create a new one for the masters to work on
                CreateNewVersionsForShards(batch);
            } else {
                logger.Log("Not all shards are done yet, waiting until they catch up", LogLevel.Info);
            }
        }
Пример #6
0
        private void RecordRowCounts(RowCounts actual, ChangeTrackingBatch ctb)
        {
            var expected = sourceDataUtils.GetExpectedRowCounts(Config.RelayDB, ctb.CTID);

            logger.Log("Expected row counts: " + expected + " | actual: " + actual, LogLevel.Info);
            double diff = expected - actual.Inserted;
            double mismatch;

            if (expected == 0)
            {
                if (actual.Inserted == 0)
                {
                    mismatch = 0.0;
                }
                else
                {
                    logger.Log("Expected 0 rows, got " + actual.Inserted + " rows inserted on slave.", LogLevel.Error);
                    return;
                }
            }
            else
            {
                mismatch = diff / expected;
            }
            int    percentDiff = (int)(mismatch * 100);
            string key         = string.Format("db.mssql_changetracking_counters.RecordCountMismatchProd{0}.{1}", Config.Slave.Replace('.', '_'), Config.SlaveDB);

            logger.Increment(key, percentDiff);
        }
Пример #7
0
        public void TestOverrideZeroStartVersion()
        {
            var table = new TableConf();

            table.Name       = "tableName";
            table.SchemaName = "schema";
            string db              = "db";
            string ctdb            = "ctdb";
            long   minValidVersion = 5;
            var    batch           = new ChangeTrackingBatch(1, 0, 10, 0);

            var sourceUtils = new Mock <IDataUtils>();

            this.sourceDataUtils = sourceUtils.Object;
            sourceUtils.Setup((ut) => ut.GetMinValidVersion(db, table.Name, table.SchemaName))
            .Returns(minValidVersion).Verifiable();
            sourceUtils.Setup((ut) => ut.CheckTableExists(db, table.Name, It.IsAny <string>()))
            .Returns(true);
            sourceUtils.Setup((ut) => ut.IsChangeTrackingEnabled(db, table.Name, table.SchemaName))
            .Returns(true);

            CreateChangeTable(table, db, ctdb, batch);
            sourceUtils.Verify(
                (ut) => ut.SelectIntoCTTable(ctdb, It.IsAny <TableConf>(), db, It.IsAny <ChangeTrackingBatch>(), It.IsAny <int>(), minValidVersion)
                );
        }
Пример #8
0
        public void TestNoOverrideNonZeroStartVersion()
        {
            var table = new TableConf();

            table.Name       = "tableName";
            table.SchemaName = "schmea";
            string db    = "db";
            string ctdb  = "ctdb";
            var    batch = new ChangeTrackingBatch(1, 1, 10, 0);

            var sourceUtils = new Mock <IDataUtils>();

            this.sourceDataUtils = sourceUtils.Object;
            sourceUtils.Setup((ut) => ut.GetMinValidVersion(db, table.Name, table.SchemaName))
            .Returns(5);
            sourceUtils.Setup((ut) => ut.CheckTableExists(db, table.Name, It.IsAny <string>()))
            .Returns(true);
            sourceUtils.Setup((ut) => ut.IsChangeTrackingEnabled(db, table.Name, table.SchemaName))
            .Returns(true);
            sourceUtils.Setup((ut) => ut.IsBeingInitialized(ctdb, It.IsAny <TableConf>()))
            .Returns(false);
            sourceUtils.Setup((ut) => ut.GetInitializeStartVersion(ctdb, It.IsAny <TableConf>()))
            .Returns(new long?());

            CreateChangeTable(table, db, ctdb, batch);
            //sourceUtils.Verify(
            //    (ut) => ut.SelectIntoCTTable(It.IsAny<string>(), It.IsAny<TableConf>(), It.IsAny<string>(
            sourceUtils.Verify(
                (ut) => ut.SelectIntoCTTable(ctdb, It.IsAny <TableConf>(), db, It.IsAny <ChangeTrackingBatch>(), It.IsAny <int>(), batch.SyncStartVersion),
                Times.Never(),
                "Should not select into CT table when the start version is less than minValidVersion");
        }
Пример #9
0
        private void SetFieldList(TableConf table, string database, ChangeTrackingBatch batch)
        {
            var cols = sourceDataUtils.GetFieldList(database, table.ToCTName(batch.CTID), table.SchemaName);
            var pks  = sourceDataUtils.GetPrimaryKeysFromInfoTable(table, batch.CTID, database);

            foreach (var pk in pks)
            {
                cols.First((c => c.name == pk)).isPk = true;
            }
            SetFieldList(table, cols);
        }
Пример #10
0
        /// <summary>
        /// Runs a single change tracking batch
        /// </summary>
        /// <param name="ctb">Change tracking batch object to work on</param>
        private void RunSingleBatch(ChangeTrackingBatch ctb)
        {
            Stopwatch sw;

            logger.Log("Applying schema changes ", LogLevel.Info);
            ApplySchemaChangesAndWrite(ctb);
            //marking this field so that all completed slave batches will have the same values
            sourceDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.ConsolidateBatches), AgentType.Slave);

            logger.Log("Populating table list", LogLevel.Info);

            List <ChangeTable> existingCTTables = PopulateTableList(Config.Tables, Config.RelayDB, new List <ChangeTrackingBatch>()
            {
                ctb
            });

            logger.Log("Capturing field lists", LogLevel.Info);
            SetFieldListsSlave(Config.RelayDB, Config.Tables, ctb, existingCTTables);

            if ((ctb.SyncBitWise & Convert.ToInt32(SyncBitWise.DownloadChanges)) == 0)
            {
                logger.Log("Downloading changes", LogLevel.Info);
                sw = Stopwatch.StartNew();
                CopyChangeTables(Config.Tables, Config.RelayDB, Config.SlaveCTDB, ctb.CTID, existingCTTables);
                logger.Log("CopyChangeTables: " + sw.Elapsed, LogLevel.Trace);
                sourceDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.DownloadChanges), AgentType.Slave);
                logger.Timing(StepTimingKey("DownloadChanges"), (int)sw.ElapsedMilliseconds);
            }

            if ((ctb.SyncBitWise & Convert.ToInt32(SyncBitWise.ApplyChanges)) == 0)
            {
                logger.Log("Applying changes", LogLevel.Info);
                sw = Stopwatch.StartNew();
                RowCounts total = ApplyChanges(existingCTTables, ctb.CTID, isConsolidated: false);
                RecordRowCounts(total, ctb);
                logger.Log("ApplyChanges: " + sw.Elapsed, LogLevel.Trace);
                sourceDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.ApplyChanges), AgentType.Slave);
                logger.Timing(StepTimingKey("ApplyChanges"), (int)sw.ElapsedMilliseconds);
            }
            logger.Log("Syncing history tables", LogLevel.Info);
            sw = Stopwatch.StartNew();
            SyncHistoryTables(Config.SlaveCTDB, existingCTTables, isConsolidated: false);
            logger.Log("SyncHistoryTables: " + sw.Elapsed, LogLevel.Trace);
            var syncStopTime = DateTime.Now;

            sourceDataUtils.MarkBatchComplete(Config.RelayDB, ctb.CTID, syncStopTime, Config.Slave);
            string key = String.Format(
                "db.mssql_changetracking_counters.DataDurationToSync{0}.{1}",
                Config.Slave.Replace('.', '_'),
                Config.SlaveDB);

            logger.Increment(key, (int)(syncStopTime - ctb.SyncStartTime.Value).TotalMinutes);
            logger.Timing(StepTimingKey("SyncHistoryTables"), (int)sw.ElapsedMilliseconds);
        }
Пример #11
0
 private void Consolidate(ChangeTrackingBatch batch)
 {
     if ((batch.SyncBitWise & Convert.ToInt32(SyncBitWise.PublishSchemaChanges)) == 0)
     {
         logger.Log("Publishing schema changes", LogLevel.Debug);
         PublishSchemaChanges(batch);
         sourceDataUtils.WriteBitWise(Config.RelayDB, batch.CTID, Convert.ToInt32(SyncBitWise.PublishSchemaChanges), AgentType.ShardCoordinator);
     }
     ConsolidateTables(batch);
     ConsolidateInfoTables(batch);
 }
Пример #12
0
        private void ApplySchemaChangesAndWrite(ChangeTrackingBatch ctb)
        {
            if ((ctb.SyncBitWise & Convert.ToInt32(SyncBitWise.ApplySchemaChanges)) == 0)
            {
                logger.Log("Applying schema changes", LogLevel.Debug);
                var sw = Stopwatch.StartNew();

                ApplySchemaChanges(Config.RelayDB, Config.SlaveDB, ctb.CTID);

                sourceDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.ApplySchemaChanges), AgentType.Slave);
                ctb.SyncBitWise += Convert.ToInt32(SyncBitWise.ApplySchemaChanges);

                logger.Timing(StepTimingKey("ApplySchemaChanges"), (int)sw.ElapsedMilliseconds);
            }
        }
Пример #13
0
        private void ConsolidateInfoTables(ChangeTrackingBatch batch)
        {
            logger.Log("Consolidating info tables", LogLevel.Debug);
            var rowCounts = GetRowCounts(Config.Tables, Config.RelayDB, batch.CTID);

            //publish table info with actual rowcounts for the tables that had changes
            PublishTableInfo(tablesWithChanges, Config.RelayDB, rowCounts, batch.CTID);
            //pull in the table info for tables that didn't have changes from the shard databases
            //start with the master shard database first because it is assumed to have the
            //most up to date schema information
            foreach (var sd in shardDatabases.OrderBy(d => d == Config.MasterShard ? 0 : 1))
            {
                sourceDataUtils.MergeInfoTable(sd, Config.RelayDB, batch.CTID);
            }
        }
Пример #14
0
        private ChangeTrackingBatch CreateNewVersionsForShards(ChangeTrackingBatch batch)
        {
            logger.Log("Creating new CT versions for slaves", LogLevel.Info);
            Int64 CTID = sourceDataUtils.CreateCTVersion(Config.RelayDB, 0, 0).CTID;

            Logger.SetProperty("CTID", CTID);
            foreach (var db in shardDatabases)
            {
                var b = new ChangeTrackingBatch(sourceDataUtils.GetLastCTBatch(db, AgentType.ShardCoordinator));
                sourceDataUtils.CreateShardCTVersion(db, CTID, b.SyncStopVersion);
            }
            logger.Log("Created new CT Version " + CTID + " on " + string.Join(",", shardDatabases), LogLevel.Info);
            batch = new ChangeTrackingBatch(CTID, 0, 0, 0);
            return(batch);
        }
Пример #15
0
        public void TestInitializeBatch_LastBatchSuccessful()
        {
            DataTable tblCTVersion = ((TestDataUtils)destDataUtils).testData.Tables["dbo.tblCTVersion", "RELAY.CT_testdb"];
            DataRow   row          = tblCTVersion.Rows[tblCTVersion.Rows.Count - 1];

            Int64 CTID = row.Field <Int64>("CTID");

            row["syncBitWise"]      = SyncBitWise.UploadChanges;
            row["syncStartVersion"] = 1000;
            row["syncStopVersion"]  = 2000;

            ChangeTrackingBatch expected = new ChangeTrackingBatch(CTID + 1, 2000, 2500, 0);
            ChangeTrackingBatch actual   = InitializeBatch(2500);

            Assert.True(actual.Equals(expected));
            //undo any writes
            ((TestDataUtils)destDataUtils).ReloadData("test1");
        }
Пример #16
0
        public void TestInitializeBatch_LastBatchPartial()
        {
            DataTable tblCTVersion = ((TestDataUtils)destDataUtils).testData.Tables["dbo.tblCTVersion", "RELAY.CT_testdb"];
            DataRow   row          = tblCTVersion.Rows[tblCTVersion.Rows.Count - 1];

            Int64 CTID        = row.Field <Int64>("CTID");
            Int32 syncBitWise = Convert.ToInt32(SyncBitWise.CaptureChanges) + Convert.ToInt32(SyncBitWise.PublishSchemaChanges);

            row["syncBitWise"]      = syncBitWise;
            row["syncStartVersion"] = 1000;
            row["syncStopVersion"]  = 2000;

            //in this case we expect to just continue working on the same batch with no changes
            ChangeTrackingBatch expected = new ChangeTrackingBatch(CTID, 1000, 2000, syncBitWise);
            ChangeTrackingBatch actual   = InitializeBatch(2500);

            Assert.True(actual.Equals(expected));
        }
Пример #17
0
        private void MergeTable(ChangeTrackingBatch batch, Dictionary <string, List <TColumn> > dbColumns, TableConf table, string firstDB)
        {
            logger.Log(new { message = "Merging table", Table = table.Name }, LogLevel.Debug);
            var dc = DataCopyFactory.GetInstance(Config.RelayType, Config.RelayType, sourceDataUtils, sourceDataUtils, logger);

            dc.CopyTableDefinition(firstDB, table.ToCTName(batch.CTID), table.SchemaName, Config.RelayDB, table.ToCTName(batch.CTID));
            foreach (var dbNameFields in dbColumns)
            {
                var dbName  = dbNameFields.Key;
                var columns = dbNameFields.Value;
                if (columns.Count == 0)
                {
                    //no changes in this DB for this table
                    continue;
                }
                sourceDataUtils.MergeCTTable(table, Config.RelayDB, dbName, batch.CTID);
            }
        }
Пример #18
0
        public void TestPublishTableInfo()
        {
            //undo changes
            ((TestDataUtils)destDataUtils).ReloadData("test1");
            ctb = new ChangeTrackingBatch(101, 1000, 2000, 0);
            PublishTableInfo(Config.Tables, "CT_testdb", changesCaptured, ctb.CTID);
            DataRow actual = ((TestDataUtils)destDataUtils).testData.Tables["dbo.tblCTTableInfo_101", "RELAY.CT_testdb"].Rows[0];

            Assert.True(actual.Field <string>("CtiTableName") == "test1" &&
                        actual.Field <string>("CtiSchemaName") == "dbo" &&
                        actual.Field <string>("CtiPKList") == "column1" &&
                        actual.Field <int>("CtiExpectedRows") == 1);

            actual = ((TestDataUtils)destDataUtils).testData.Tables["dbo.tblCTTableInfo_101", "RELAY.CT_testdb"].Rows[1];
            Assert.True(actual.Field <string>("CtiTableName") == "test2" &&
                        actual.Field <string>("CtiSchemaName") == "dbo" &&
                        actual.Field <string>("CtiPKList") == "column1" &&
                        actual.Field <int>("CtiExpectedRows") == 0);
        }
Пример #19
0
        public void TestInitializeBatch_LastBatchFailed()
        {
            DataTable tblCTVersion = ((TestDataUtils)destDataUtils).testData.Tables["dbo.tblCTVersion", "RELAY.CT_testdb"];
            DataRow   row          = tblCTVersion.Rows[tblCTVersion.Rows.Count - 1];

            Int64 CTID = row.Field <Int64>("CTID");

            row["syncBitWise"]      = 0;
            row["syncStartVersion"] = 1000;
            row["syncStopVersion"]  = 2000;

            //in this case we expect syncStopVersion to get updated from 2000 to 2500 but the rest of the batch to stay the same
            ChangeTrackingBatch expected = new ChangeTrackingBatch(CTID, 1000, 2500, 0);
            ChangeTrackingBatch actual   = InitializeBatch(2500);

            Assert.True(actual.Equals(expected));
            //undo any writes
            ((TestDataUtils)destDataUtils).ReloadData("test1");
        }
Пример #20
0
        /// <summary>
        /// Initializes version/batch info for a run
        /// </summary>
        /// <returns>List of change tracking batches to work on</returns>
        private IList <ChangeTrackingBatch> InitializeBatch(int bitwise)
        {
            ChangeTrackingBatch         ctb;
            IList <ChangeTrackingBatch> batches = new List <ChangeTrackingBatch>();

            DataRow lastBatch = sourceDataUtils.GetLastCTBatch(Config.RelayDB, AgentType.Slave, Config.Slave);

            //apparently we shouldn't hit the code block below except for unit tests?
            //in the db init we're supposed to write the first row, so it (in theory) shouldn't return null
            if (lastBatch == null)
            {
                ctb = new ChangeTrackingBatch(1, 0, 0, 0);
                batches.Add(ctb);
                logger.Log("Couldn't find any records for the last change tracking batch! Returning the default new CTB.", LogLevel.Warn);
                return(batches);
            }

            if ((lastBatch.Field <Int32>("syncBitWise") & bitwise) == bitwise)
            {
                logger.Log("Last batch was successful, checking for new batches.", LogLevel.Info);

                DataTable pendingVersions = sourceDataUtils.GetPendingCTVersions(Config.RelayDB, lastBatch.Field <Int64>("CTID"), Convert.ToInt32(SyncBitWise.UploadChanges));
                logger.Log("Retrieved " + pendingVersions.Rows.Count + " pending CT version(s) to work on.", LogLevel.Info);

                foreach (DataRow row in pendingVersions.Rows)
                {
                    ctb = new ChangeTrackingBatch(row);
                    batches.Add(ctb);
                    sourceDataUtils.CreateSlaveCTVersion(Config.RelayDB, ctb, Config.Slave);
                }
                return(batches);
            }
            ctb = new ChangeTrackingBatch(lastBatch);
            logger.Log(new { message = "Last batch failed, retrying", CTID = ctb.CTID }, LogLevel.Warn);
            batches.Add(ctb);
            return(batches);
        }
Пример #21
0
        public override void Run()
        {
            var batch = new ChangeTrackingBatch(sourceDataUtils.GetLastCTBatch(Config.RelayDB, AgentType.ShardCoordinator));

            Logger.SetProperty("CTID", batch.CTID);
            if ((batch.SyncBitWise & Convert.ToInt32(SyncBitWise.UploadChanges)) > 0)
            {
                CreateNewVersionsForShards(batch);
                return;
            }
            logger.Log("Working on CTID " + batch.CTID, LogLevel.Info);

            if (AllShardMastersDone(batch))
            {
                logger.Log("All shard masters are done, checking field lists", LogLevel.Info);
                tableDBFieldLists = GetFieldListsByDB(batch.CTID);
                if (SchemasOutOfSync(tableDBFieldLists.Values))
                {
                    foreach (var sd in shardDatabases)
                    {
                        sourceDataUtils.RevertCTBatch(sd, batch.CTID);
                    }
                    logger.Log("Schemas out of sync, quitting", LogLevel.Info);
                    return;
                }
                logger.Log("Field lists in sync, consolidating", LogLevel.Info);
                Consolidate(batch);
                sourceDataUtils.WriteBitWise(Config.RelayDB, batch.CTID,
                                             Convert.ToInt32(SyncBitWise.CaptureChanges) | Convert.ToInt32(SyncBitWise.UploadChanges), AgentType.ShardCoordinator);
                //now that the current batch is done, create a new one for the masters to work on
                CreateNewVersionsForShards(batch);
            }
            else
            {
                logger.Log("Not all shards are done yet, waiting until they catch up", LogLevel.Info);
            }
        }
Пример #22
0
        /// <summary>
        /// Consolidates multiple batches and runs them as a group
        /// </summary>
        /// <param name="ctidTable">DataTable object listing all the batches</param>
        private void RunMultiBatch(IList <ChangeTrackingBatch> batches)
        {
            ChangeTrackingBatch endBatch = batches.OrderBy(item => item.CTID).Last();

            Logger.SetProperty("CTID", endBatch.CTID);
            logger.Log(string.Format("Running consolidated batches from CTID {0} to {1}",
                                     batches.OrderBy(item => item.CTID).First().CTID, endBatch.CTID), LogLevel.Info);

            //from here forward all operations will use the bitwise value for the last CTID since they are operating on this whole set of batches
            logger.Log("Populating changetable list for all CTIDs", LogLevel.Info);
            List <ChangeTable> existingCTTables = PopulateTableList(Config.Tables, Config.RelayDB, batches);

            logger.Log("Capturing field lists", LogLevel.Info);
            SetFieldListsSlave(Config.RelayDB, Config.Tables, endBatch, existingCTTables);

            foreach (ChangeTrackingBatch batch in batches)
            {
                if ((batch.SyncBitWise & Convert.ToInt32(SyncBitWise.ApplySchemaChanges)) == 0)
                {
                    logger.Log("Applying schema changes for batch " + batch.CTID, LogLevel.Debug);
                    ApplySchemaChanges(Config.RelayDB, Config.SlaveDB, batch.CTID);
                    sourceDataUtils.WriteBitWise(Config.RelayDB, batch.CTID, Convert.ToInt32(SyncBitWise.ApplySchemaChanges), AgentType.Slave);
                }
            }

            if ((endBatch.SyncBitWise & Convert.ToInt32(SyncBitWise.ConsolidateBatches)) == 0)
            {
                var sw = Stopwatch.StartNew();
                logger.Log("Consolidating batches", LogLevel.Info);
                ConsolidateBatches(existingCTTables);
                sourceDataUtils.WriteBitWise(Config.RelayDB, endBatch.CTID, Convert.ToInt32(SyncBitWise.ConsolidateBatches), AgentType.Slave);
                logger.Timing(StepTimingKey("ConsolidateBatches"), (int)sw.ElapsedMilliseconds);
            }

            if ((endBatch.SyncBitWise & Convert.ToInt32(SyncBitWise.DownloadChanges)) == 0)
            {
                var sw = Stopwatch.StartNew();
                logger.Log("Downloading consolidated changetables", LogLevel.Info);
                CopyChangeTables(Config.Tables, Config.RelayDB, Config.SlaveCTDB, endBatch.CTID, existingCTTables, isConsolidated: true);
                logger.Log("Changes downloaded successfully", LogLevel.Debug);
                sourceDataUtils.WriteBitWise(Config.RelayDB, endBatch.CTID, Convert.ToInt32(SyncBitWise.DownloadChanges), AgentType.Slave);
                logger.Timing(StepTimingKey("DownloadChanges"), (int)sw.ElapsedMilliseconds);
            }

            RowCounts total;

            if ((endBatch.SyncBitWise & Convert.ToInt32(SyncBitWise.ApplyChanges)) == 0)
            {
                var sw = Stopwatch.StartNew();
                logger.Log("Applying changes", LogLevel.Info);
                total = ApplyChanges(existingCTTables, endBatch.CTID, isConsolidated: true);
                sourceDataUtils.WriteBitWise(Config.RelayDB, endBatch.CTID, Convert.ToInt32(SyncBitWise.ApplyChanges), AgentType.Slave);
                //Expected rowcounts across multiple batches are not currently calculated, it's unclear
                //how we would actually want to calculate them since by the nature of consolidation duplicate inserts/updates are eliminated.
                //Commenting this out for now.
                //RecordRowCounts(total, endBatch);
                logger.Timing(StepTimingKey("ApplyChanges"), (int)sw.ElapsedMilliseconds);
            }

            logger.Log("Figuring out the last batch for each changetable", LogLevel.Debug);
            var lastChangedTables = new List <ChangeTable>();

            foreach (var group in existingCTTables.GroupBy(c => c.name))
            {
                var table = group.First();
                lastChangedTables.Add(new ChangeTable(table.name, endBatch.CTID, table.schemaName, table.slaveName));
            }

            if ((endBatch.SyncBitWise & Convert.ToInt32(SyncBitWise.SyncHistoryTables)) == 0)
            {
                var sw = Stopwatch.StartNew();
                logger.Log("Syncing history tables", LogLevel.Info);
                SyncHistoryTables(Config.SlaveCTDB, lastChangedTables, isConsolidated: true);
                sourceDataUtils.WriteBitWise(Config.RelayDB, endBatch.CTID, Convert.ToInt32(SyncBitWise.SyncHistoryTables), AgentType.Slave);
                logger.Timing(StepTimingKey("SyncHistoryTables"), (int)sw.ElapsedMilliseconds);
            }
            //success! go through and mark all the batches as complete in the db
            sourceDataUtils.MarkBatchesComplete(Config.RelayDB, batches.Select(b => b.CTID), DateTime.Now, Config.Slave);
            logger.Timing("db.mssql_changetracking_counters.DataAppliedAsOf." + Config.SlaveDB, DateTime.Now.Hour + DateTime.Now.Minute / 60);
        }
Пример #23
0
 private void PublishSchemaChanges(ChangeTrackingBatch batch)
 {
     var dc = DataCopyFactory.GetInstance(Config.RelayType, Config.RelayType, sourceDataUtils, sourceDataUtils, logger);
     dc.CopyTable(Config.MasterShard, batch.schemaChangeTable, "dbo", Config.RelayDB, Config.DataCopyTimeout);
 }
Пример #24
0
 private ChangeTrackingBatch CreateNewVersionsForShards(ChangeTrackingBatch batch)
 {
     logger.Log("Creating new CT versions for slaves", LogLevel.Info);
     Int64 CTID = sourceDataUtils.CreateCTVersion(Config.RelayDB, 0, 0).CTID;
     Logger.SetProperty("CTID", CTID);
     foreach (var db in shardDatabases) {
         var b = new ChangeTrackingBatch(sourceDataUtils.GetLastCTBatch(db, AgentType.ShardCoordinator));
         sourceDataUtils.CreateShardCTVersion(db, CTID, b.SyncStopVersion);
     }
     logger.Log("Created new CT Version " + CTID + " on " + string.Join(",", shardDatabases), LogLevel.Info);
     batch = new ChangeTrackingBatch(CTID, 0, 0, 0);
     return batch;
 }
Пример #25
0
 private void ConsolidateInfoTables(ChangeTrackingBatch batch)
 {
     logger.Log("Consolidating info tables", LogLevel.Debug);
     var rowCounts = GetRowCounts(Config.Tables, Config.RelayDB, batch.CTID);
     //publish table info with actual rowcounts for the tables that had changes
     PublishTableInfo(tablesWithChanges, Config.RelayDB, rowCounts, batch.CTID);
     //pull in the table info for tables that didn't have changes from the shard databases
     //start with the master shard database first because it is assumed to have the
     //most up to date schema information
     foreach (var sd in shardDatabases.OrderBy(d => d == Config.MasterShard ? 0 : 1)) {
         sourceDataUtils.MergeInfoTable(sd, Config.RelayDB, batch.CTID);
     }
 }
Пример #26
0
 private bool AllShardMastersDone(ChangeTrackingBatch batch)
 {
     return shardDatabases.All(dbName => (sourceDataUtils.GetCTBatch(dbName, batch.CTID).SyncBitWise & Convert.ToInt32(SyncBitWise.UploadChanges)) > 0);
 }
Пример #27
0
        /// <summary>
        /// Runs a single change tracking batch
        /// </summary>
        /// <param name="CTID">Change tracking batch object to work on</param>
        private void RunSingleBatch(ChangeTrackingBatch ctb)
        {
            Stopwatch sw;
            logger.Log("Applying schema changes ", LogLevel.Info);
            ApplySchemaChangesAndWrite(ctb);
            //marking this field so that all completed slave batches will have the same values
            sourceDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.ConsolidateBatches), AgentType.Slave);
            if ((ctb.SyncBitWise & Convert.ToInt32(SyncBitWise.DownloadChanges)) == 0) {
                logger.Log("Downloading changes", LogLevel.Info);
                sw = Stopwatch.StartNew();
                CopyChangeTables(Config.Tables, Config.RelayDB, Config.SlaveCTDB, ctb.CTID);
                logger.Log("CopyChangeTables: " + sw.Elapsed, LogLevel.Trace);
                sourceDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.DownloadChanges), AgentType.Slave);
                logger.Timing(StepTimingKey("DownloadChanges"), (int)sw.ElapsedMilliseconds);
            }

            logger.Log("Populating table list", LogLevel.Info);
            List<ChangeTable> existingCTTables = PopulateTableList(Config.Tables, Config.RelayDB, new List<ChangeTrackingBatch>() { ctb });

            if ((ctb.SyncBitWise & Convert.ToInt32(SyncBitWise.ApplyChanges)) == 0) {
                logger.Log("Applying changes", LogLevel.Info);
                sw = Stopwatch.StartNew();
                SetFieldListsSlave(Config.RelayDB, Config.Tables, ctb, existingCTTables);
                RowCounts total = ApplyChanges(existingCTTables, ctb.CTID, isConsolidated: false);
                RecordRowCounts(total, ctb);
                logger.Log("ApplyChanges: " + sw.Elapsed, LogLevel.Trace);
                sourceDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.ApplyChanges), AgentType.Slave);
                logger.Timing(StepTimingKey("ApplyChanges"), (int)sw.ElapsedMilliseconds);
            }
            logger.Log("Syncing history tables", LogLevel.Info);
            sw = Stopwatch.StartNew();
            SyncHistoryTables(Config.SlaveCTDB, existingCTTables, isConsolidated: false);
            logger.Log("SyncHistoryTables: " + sw.Elapsed, LogLevel.Trace);
            var syncStopTime = DateTime.Now;
            sourceDataUtils.MarkBatchComplete(Config.RelayDB, ctb.CTID, syncStopTime, Config.Slave);
            string key = String.Format(
                "db.mssql_changetracking_counters.DataDurationToSync{0}.{1}",
                Config.Slave.Replace('.', '_'),
                Config.SlaveDB);
            logger.Increment(key, (int)(syncStopTime - ctb.SyncStartTime.Value).TotalMinutes);
            logger.Timing(StepTimingKey("SyncHistoryTables"), (int)sw.ElapsedMilliseconds);
        }
Пример #28
0
        /// <summary>
        /// Initializes version/batch info for a run
        /// </summary>
        /// <returns>List of change tracking batches to work on</returns>
        private IList<ChangeTrackingBatch> InitializeBatch(int bitwise)
        {
            ChangeTrackingBatch ctb;
            IList<ChangeTrackingBatch> batches = new List<ChangeTrackingBatch>();

            DataRow lastBatch = sourceDataUtils.GetLastCTBatch(Config.RelayDB, AgentType.Slave, Config.Slave);
            if (lastBatch == null) {
                ctb = new ChangeTrackingBatch(1, 0, 0, 0);
                batches.Add(ctb);
                return batches;
            }

            if ((lastBatch.Field<Int32>("syncBitWise") & bitwise) == bitwise) {
                logger.Log("Last batch was successful, checking for new batches.", LogLevel.Info);

                DataTable pendingVersions = sourceDataUtils.GetPendingCTVersions(Config.RelayDB, lastBatch.Field<Int64>("CTID"), Convert.ToInt32(SyncBitWise.UploadChanges));
                logger.Log("Retrieved " + pendingVersions.Rows.Count + " pending CT version(s) to work on.", LogLevel.Info);

                foreach (DataRow row in pendingVersions.Rows) {
                    ctb = new ChangeTrackingBatch(row);
                    batches.Add(ctb);
                    sourceDataUtils.CreateSlaveCTVersion(Config.RelayDB, ctb, Config.Slave);
                }
                return batches;
            }
            ctb = new ChangeTrackingBatch(lastBatch);
            logger.Log(new { message = "Last batch failed, retrying", CTID = ctb.CTID }, LogLevel.Warn);
            batches.Add(ctb);
            return batches;
        }
Пример #29
0
        private void PublishSchemaChanges(ChangeTrackingBatch batch)
        {
            var dc = DataCopyFactory.GetInstance(Config.RelayType, Config.RelayType, sourceDataUtils, sourceDataUtils, logger);

            dc.CopyTable(Config.MasterShard, batch.schemaChangeTable, "dbo", Config.RelayDB, Config.DataCopyTimeout);
        }
Пример #30
0
        /// <summary>
        /// Loops through passed in array of table objects, creates changedata tables for each one on the CT DB
        /// </summary>
        /// <param name="tables">Array of table config objects to create CT tables for</param>
        /// <param name="sourceDB">Database the source data lives in</param>
        /// <param name="sourceCTDB">Database the changetables should go to</param>
        /// <param name="startVersion">Change tracking version to start with</param>
        /// <param name="stopVersion">Change tracking version to stop at</param>
        /// <param name="CTID">CT batch ID this is being run for</param>
        protected IDictionary <string, Int64> CreateChangeTables(string sourceDB, string sourceCTDB, ChangeTrackingBatch batch)
        {
            var changesCaptured = new ConcurrentDictionary <string, Int64>();
            var actions         = new List <Action>();

            foreach (TableConf t in Config.Tables)
            {
                //local variables inside the loop required for the action to bind properly
                TableConf table = t;
                long      rowsAffected;
                Action    act = () => {
                    logger.Log("Creating changetable for " + table.SchemaName + "." + table.Name, LogLevel.Debug);
                    rowsAffected = CreateChangeTable(table, sourceDB, sourceCTDB, batch);
                    changesCaptured.TryAdd(table.SchemaName + "." + table.Name, rowsAffected);
                    logger.Log(rowsAffected + " changes captured for table " + table.SchemaName + "." + table.Name, LogLevel.Trace);
                };
                actions.Add(act);
            }
            var options = new ParallelOptions();

            options.MaxDegreeOfParallelism = Config.MaxThreads;
            logger.Log("Parallel invocation of " + actions.Count + " change captures", LogLevel.Trace);
            Parallel.Invoke(options, actions.ToArray());
            return(changesCaptured);
        }
Пример #31
0
        /// <summary>
        /// Creates changetable for an individual table
        /// </summary>
        /// <param name="table">Config table object to create changes for</param>
        /// <param name="sourceDB">Database the source data lives in</param>
        /// <param name="sourceCTDB">Database the changetables should go to</param>
        /// <param name="batch">Batch to work on</param>
        protected long CreateChangeTable(TableConf table, string sourceDB, string sourceCTDB, ChangeTrackingBatch batch)
        {
            string ctTableName = table.ToCTName(batch.CTID);
            string reason;

            long tableStartVersion = batch.SyncStartVersion;
            long minValidVersion   = sourceDataUtils.GetMinValidVersion(sourceDB, table.Name, table.SchemaName);

            if (batch.SyncStartVersion == 0)
            {
                tableStartVersion = minValidVersion;
            }

            if (sourceDataUtils.IsBeingInitialized(sourceCTDB, table))
            {
                return(0);
            }

            long?initializeVersion = sourceDataUtils.GetInitializeStartVersion(sourceCTDB, table);

            if (initializeVersion.HasValue)
            {
                tableStartVersion = initializeVersion.Value;
            }

            if (!ValidateSourceTable(sourceDB, table.Name, table.SchemaName, tableStartVersion, minValidVersion, out reason))
            {
                string message = "Change table creation impossible because : " + reason;
                if (table.StopOnError)
                {
                    throw new Exception(message);
                }
                else
                {
                    logger.Log(message, LogLevel.Error);
                    return(0);
                }
            }

            logger.Log("Dropping table " + ctTableName + " if it exists", LogLevel.Trace);
            sourceDataUtils.DropTableIfExists(sourceCTDB, ctTableName, table.SchemaName);

            logger.Log("Calling SelectIntoCTTable to create CT table", LogLevel.Trace);
            Int64 rowsAffected = sourceDataUtils.SelectIntoCTTable(sourceCTDB, table, sourceDB, batch, Config.QueryTimeout, tableStartVersion);

            logger.Log("Rows affected for table " + table.SchemaName + "." + table.Name + ": " + Convert.ToString(rowsAffected), LogLevel.Debug);
            return(rowsAffected);
        }
Пример #32
0
 public void CreateSlaveCTVersion(string dbName, ChangeTrackingBatch ctb, string slaveIdentifier)
 {
     throw new NotImplementedException("Netezza is only supported as a slave!");
 }
Пример #33
0
 public Dictionary<TableConf, IList<string>> GetAllPrimaryKeys(string dbName, IEnumerable<TableConf> tables, ChangeTrackingBatch batch)
 {
     throw new NotImplementedException();
 }
Пример #34
0
 public int SelectIntoCTTable(string sourceCTDB, TableConf table, string sourceDB, ChangeTrackingBatch batch, int timeout, long? startVersionOverride)
 {
     throw new NotImplementedException("Netezza is only supported as a slave!");
 }
Пример #35
0
        /// <summary>
        /// Creates changetable for an individual table
        /// </summary>
        /// <param name="table">Config table object to create changes for</param>
        /// <param name="sourceDB">Database the source data lives in</param>
        /// <param name="sourceCTDB">Database the changetables should go to</param>
        /// <param name="batch">Batch to work on</param>
        protected long CreateChangeTable(TableConf table, string sourceDB, string sourceCTDB, ChangeTrackingBatch batch)
        {
            string ctTableName = table.ToCTName(batch.CTID);
            string reason;

            long tableStartVersion = batch.SyncStartVersion;
            long minValidVersion = sourceDataUtils.GetMinValidVersion(sourceDB, table.Name, table.SchemaName);
            if (batch.SyncStartVersion == 0) {
                tableStartVersion = minValidVersion;
            }

            if (sourceDataUtils.IsBeingInitialized(sourceCTDB, table)) {
                return 0;
            }

            long? initializeVersion = sourceDataUtils.GetInitializeStartVersion(sourceCTDB, table);
            if (initializeVersion.HasValue) {
                tableStartVersion = initializeVersion.Value;
            }

            if (!ValidateSourceTable(sourceDB, table.Name, table.SchemaName, tableStartVersion, minValidVersion, out reason)) {
                string message = "Change table creation impossible because : " + reason;
                if (table.StopOnError) {
                    throw new Exception(message);
                } else {
                    logger.Log(message, LogLevel.Error);
                    return 0;
                }
            }

            logger.Log("Dropping table " + ctTableName + " if it exists", LogLevel.Trace);
            sourceDataUtils.DropTableIfExists(sourceCTDB, ctTableName, table.SchemaName);

            logger.Log("Calling SelectIntoCTTable to create CT table", LogLevel.Trace);
            Int64 rowsAffected = sourceDataUtils.SelectIntoCTTable(sourceCTDB, table, sourceDB, batch, Config.QueryTimeout, tableStartVersion);

            logger.Log("Rows affected for table " + table.SchemaName + "." + table.Name + ": " + Convert.ToString(rowsAffected), LogLevel.Debug);
            return rowsAffected;
        }
Пример #36
0
 public void CreateSlaveCTVersion(string dbName, ChangeTrackingBatch ctb, string slaveIdentifier)
 {
     throw new NotImplementedException();
 }
Пример #37
0
 /// <summary>
 /// Loops through passed in array of table objects, creates changedata tables for each one on the CT DB
 /// </summary>
 /// <param name="tables">Array of table config objects to create CT tables for</param>
 /// <param name="sourceDB">Database the source data lives in</param>
 /// <param name="sourceCTDB">Database the changetables should go to</param>
 /// <param name="startVersion">Change tracking version to start with</param>
 /// <param name="stopVersion">Change tracking version to stop at</param>
 /// <param name="CTID">CT batch ID this is being run for</param>
 protected IDictionary<string, Int64> CreateChangeTables(string sourceDB, string sourceCTDB, ChangeTrackingBatch batch)
 {
     var changesCaptured = new ConcurrentDictionary<string, Int64>();
     var actions = new List<Action>();
     foreach (TableConf t in Config.Tables) {
         //local variables inside the loop required for the action to bind properly
         TableConf table = t;
         long rowsAffected;
         Action act = () => {
             logger.Log("Creating changetable for " + table.SchemaName + "." + table.Name, LogLevel.Debug);
             rowsAffected = CreateChangeTable(table, sourceDB, sourceCTDB, batch);
             changesCaptured.TryAdd(table.SchemaName + "." + table.Name, rowsAffected);
             logger.Log(rowsAffected + " changes captured for table " + table.SchemaName + "." + table.Name, LogLevel.Trace);
         };
         actions.Add(act);
     }
     var options = new ParallelOptions();
     options.MaxDegreeOfParallelism = Config.MaxThreads;
     logger.Log("Parallel invocation of " + actions.Count + " change captures", LogLevel.Trace);
     Parallel.Invoke(options, actions.ToArray());
     return changesCaptured;
 }
Пример #38
0
 public int SelectIntoCTTable(string sourceCTDB, TableConf table, string sourceDB, ChangeTrackingBatch ctb, int timeout, long? overrideStartVersion)
 {
     //no good way to fake this with DataTables so just return and make sure we are also unit testing the
     //methods that generate these sfield lists
     return testData.Tables[table.SchemaName + "." + table.ToCTName(ctb.CTID), GetTableSpace(sourceCTDB)].Rows.Count;
 }
Пример #39
0
 private void RecordRowCounts(RowCounts actual, ChangeTrackingBatch ctb)
 {
     var expected = sourceDataUtils.GetExpectedRowCounts(Config.RelayDB, ctb.CTID);
     logger.Log("Expected row counts: " + expected + " | actual: " + actual, LogLevel.Info);
     double diff = expected - actual.Inserted;
     double mismatch;
     if (expected == 0) {
         if (actual.Inserted == 0) {
             mismatch = 0.0;
         } else {
             logger.Log("Expected 0 rows, got " + actual.Inserted + " rows inserted on slave.", LogLevel.Error);
             return;
         }
     } else {
         mismatch = diff / expected;
     }
     int percentDiff = (int)(mismatch * 100);
     string key = string.Format("db.mssql_changetracking_counters.RecordCountMismatchProd{0}.{1}", Config.Slave.Replace('.', '_'), Config.SlaveDB);
     logger.Increment(key, percentDiff);
 }
Пример #40
0
 public void CreateSlaveCTVersion(string dbName, ChangeTrackingBatch ctb, string slaveIdentifier)
 {
     throw new NotImplementedException();
 }
Пример #41
0
        private void SetFieldListsSlave(string dbName, IEnumerable<TableConf> tables, ChangeTrackingBatch batch, List<ChangeTable> existingCTTables)
        {
            //map each table to the last appropriate CT table, ditching tableconfs with no corresponding CT tables
            var tableCTName = new Dictionary<TableConf, string>();
            foreach (var table in tables) {
                ChangeTable changeTable = existingCTTables.Where(ct => ct.name == table.Name).OrderBy(ct => ct.CTID).LastOrDefault();
                if (changeTable == null) {
                    continue;
                }
                long lastCTIDWithChanges = changeTable.CTID.Value;
                tableCTName[table] = table.ToCTName(lastCTIDWithChanges);
            }
            Dictionary<TableConf, IList<TColumn>> allColumnsByTable = sourceDataUtils.GetAllFields(dbName, tableCTName);
            //even though GetAllFields returns whether it's part of the PK, the PK info will always say false on the relay since the relay
            //doesn't necessarily define primary keys
            Dictionary<TableConf, IList<string>> primaryKeysByTable = sourceDataUtils.GetAllPrimaryKeys(dbName, tableCTName.Keys, batch);

            //tableCTName.Keys instead of tables because we've already filtered this for tables that don't have change tables
            //note: allColumnsByTable.Keys or primaryKeysByTable.Keys should work just as well
            foreach (var table in tableCTName.Keys) {
                IEnumerable<TColumn> columns;
                try {
                    //this is a hacky solution but we will have these columns in CT tables but actually are not interested in them here.
                    columns = allColumnsByTable[table].Where(c => (c.name != "SYS_CHANGE_VERSION" && c.name != "SYS_CHANGE_OPERATION"));
                } catch (KeyNotFoundException) {
                    var e = new Exception("Column list for table " + tableCTName[table] + " not found in " + dbName);
                    HandleException(e, table);
                    //if we handled the exception by just logging an error, this table is still broken so we need to continue
                    continue;
                }
                IList<string> pks;
                try {
                    pks = primaryKeysByTable[table];
                } catch (KeyNotFoundException) {
                    var e = new Exception("Primary keys for table " + table.FullName + " not found in " + dbName + ".dbo.tblCTTableInfo_" + batch.CTID);
                    HandleException(e, table);
                    //if we handled the exception by just logging an error, this table is still broken so we need to continue
                    continue;
                }
                foreach (var pk in pks) {
                    columns.First((c => c.name == pk)).isPk = true;
                }
                SetFieldList(table, columns);
            }
        }
Пример #42
0
        /// <summary>
        /// Initializes version/batch info for a run
        /// </summary>
        /// <returns>List of change tracking batches to work on</returns>
        private IList<ChangeTrackingBatch> InitializeBatch(int bitwise)
        {
            ChangeTrackingBatch ctb;
            IList<ChangeTrackingBatch> batches = new List<ChangeTrackingBatch>();

            DataRow lastBatch = sourceDataUtils.GetLastCTBatch(Config.RelayDB, AgentType.Slave, Config.Slave);
            //apparently we shouldn't hit the code block below except for unit tests?
            //in the db init we're supposed to write the first row, so it (in theory) shouldn't return null
            if (lastBatch == null) {
                ctb = new ChangeTrackingBatch(1, 0, 0, 0);
                batches.Add(ctb);
                logger.Log("Couldn't find any records for the last change tracking batch! Returning the default new CTB.", LogLevel.Warn);
                return batches;
            }

            if ((lastBatch.Field<Int32>("syncBitWise") & bitwise) == bitwise) {
                logger.Log("Last batch was successful, checking for new batches.", LogLevel.Info);

                DataTable pendingVersions = sourceDataUtils.GetPendingCTVersions(Config.RelayDB, lastBatch.Field<Int64>("CTID"), Convert.ToInt32(SyncBitWise.UploadChanges));
                logger.Log("Retrieved " + pendingVersions.Rows.Count + " pending CT version(s) to work on.", LogLevel.Info);

                foreach (DataRow row in pendingVersions.Rows) {
                    ctb = new ChangeTrackingBatch(row);
                    batches.Add(ctb);
                    sourceDataUtils.CreateSlaveCTVersion(Config.RelayDB, ctb, Config.Slave);
                }
                return batches;
            }
            ctb = new ChangeTrackingBatch(lastBatch);
            logger.Log(new { message = "Last batch failed, retrying", CTID = ctb.CTID }, LogLevel.Warn);
            batches.Add(ctb);
            return batches;
        }
Пример #43
0
 private void Consolidate(ChangeTrackingBatch batch)
 {
     if ((batch.SyncBitWise & Convert.ToInt32(SyncBitWise.PublishSchemaChanges)) == 0) {
         logger.Log("Publishing schema changes", LogLevel.Debug);
         PublishSchemaChanges(batch);
         sourceDataUtils.WriteBitWise(Config.RelayDB, batch.CTID, Convert.ToInt32(SyncBitWise.PublishSchemaChanges), AgentType.ShardCoordinator);
     }
     ConsolidateTables(batch);
     ConsolidateInfoTables(batch);
 }
Пример #44
0
        private void SetFieldListsSlave(string dbName, IEnumerable <TableConf> tables, ChangeTrackingBatch batch, List <ChangeTable> existingCTTables)
        {
            //map each table to the last appropriate CT table, ditching tableconfs with no corresponding CT tables
            var tableCTName = new Dictionary <TableConf, string>();

            foreach (var table in tables)
            {
                ChangeTable changeTable = existingCTTables.Where(ct => String.Compare(ct.name, table.Name, StringComparison.OrdinalIgnoreCase) == 0).OrderBy(ct => ct.CTID).LastOrDefault();

                if (changeTable == null)
                {
                    continue;
                }
                long lastCTIDWithChanges = changeTable.CTID.Value;
                tableCTName[table] = table.ToCTName(lastCTIDWithChanges);
            }
            Dictionary <TableConf, IList <TColumn> > allColumnsByTable = sourceDataUtils.GetAllFields(dbName, tableCTName);
            //even though GetAllFields returns whether it's part of the PK, the PK info will always say false on the relay since the relay
            //doesn't necessarily define primary keys
            Dictionary <TableConf, IList <string> > primaryKeysByTable = sourceDataUtils.GetAllPrimaryKeys(dbName, tableCTName.Keys, batch);

            //tableCTName.Keys instead of tables because we've already filtered this for tables that don't have change tables
            //note: allColumnsByTable.Keys or primaryKeysByTable.Keys should work just as well
            foreach (var table in tableCTName.Keys)
            {
                IEnumerable <TColumn> columns;
                try {
                    //this is a hacky solution but we will have these columns in CT tables but actually are not interested in them here.
                    columns = allColumnsByTable[table].Where(c => (c.name != "SYS_CHANGE_VERSION" && c.name != "SYS_CHANGE_OPERATION"));
                } catch (KeyNotFoundException) {
                    var e = new Exception("Column list for table " + tableCTName[table] + " not found in " + dbName);
                    HandleException(e, table);
                    //if we handled the exception by just logging an error, this table is still broken so we need to continue
                    continue;
                }
                IList <string> pks;
                try {
                    pks = primaryKeysByTable[table];
                } catch (KeyNotFoundException) {
                    var e = new Exception("Primary keys for table " + table.FullName + " not found in " + dbName + ".dbo.tblCTTableInfo_" + batch.CTID);
                    HandleException(e, table);
                    //if we handled the exception by just logging an error, this table is still broken so we need to continue
                    continue;
                }
                foreach (var pk in pks)
                {
                    columns.First((c => c.name == pk)).isPk = true;
                }
                SetFieldList(table, columns);
            }
        }
Пример #45
0
        private void ConsolidateTables(ChangeTrackingBatch batch)
        {
            logger.Log("Consolidating tables", LogLevel.Info);
            var actions = new List<Action>();
            foreach (var tableDb in tableDBFieldLists) {
                var table = tableDb.Key;
                var dbColumns = tableDb.Value;
                var firstDB = tableDb.Value.FirstOrDefault(t => t.Value.Count > 0).Key;
                if (firstDB == null) {
                    logger.Log("No shard has CT changes for table " + table.Name, LogLevel.Debug);
                    continue;
                }
                tablesWithChanges.Add(table);
                SetFieldList(table, firstDB, batch);

                Action act = () => MergeTable(batch, dbColumns, table, firstDB);
                actions.Add(act);
            }
            logger.Log("Parallel invocation of " + actions.Count + " table merges", LogLevel.Info);
            //interestingly, Parallel.Invoke does in fact bubble up exceptions, but not until after all threads have completed.
            //actually it looks like what it does is wrap its exceptions in an AggregateException. We don't ever catch those
            //though because if any exceptions happen inside of MergeTable it would generally be due to things like the server
            //being down or a query timeout.
            var options = new ParallelOptions();
            options.MaxDegreeOfParallelism = Config.MaxThreads;
            Parallel.Invoke(options, actions.ToArray());
        }
Пример #46
0
 private bool AllShardMastersDone(ChangeTrackingBatch batch)
 {
     return(shardDatabases.All(dbName => (sourceDataUtils.GetCTBatch(dbName, batch.CTID).SyncBitWise & Convert.ToInt32(SyncBitWise.UploadChanges)) > 0));
 }
Пример #47
0
 private void MergeTable(ChangeTrackingBatch batch, Dictionary<string, List<TColumn>> dbColumns, TableConf table, string firstDB)
 {
     logger.Log(new { message = "Merging table", Table = table.Name }, LogLevel.Debug);
     var dc = DataCopyFactory.GetInstance(Config.RelayType, Config.RelayType, sourceDataUtils, sourceDataUtils, logger);
     dc.CopyTableDefinition(firstDB, table.ToCTName(batch.CTID), table.SchemaName, Config.RelayDB, table.ToCTName(batch.CTID));
     foreach (var dbNameFields in dbColumns) {
         var dbName = dbNameFields.Key;
         var columns = dbNameFields.Value;
         if (columns.Count == 0) {
             //no changes in this DB for this table
             continue;
         }
         sourceDataUtils.MergeCTTable(table, Config.RelayDB, dbName, batch.CTID);
     }
 }
Пример #48
0
 public void CreateSlaveCTVersion(string dbName, ChangeTrackingBatch ctb, string slaveIdentifier)
 {
     throw new NotImplementedException("Netezza is only supported as a slave!");
 }
Пример #49
0
 private void SetFieldList(TableConf table, string database, ChangeTrackingBatch batch)
 {
     var cols = sourceDataUtils.GetFieldList(database, table.ToCTName(batch.CTID), table.SchemaName);
     var pks = sourceDataUtils.GetPrimaryKeysFromInfoTable(table, batch.CTID, database);
     foreach (var pk in pks) {
         cols.First((c => c.name == pk)).isPk = true;
     }
     SetFieldList(table, cols);
 }
Пример #50
0
 public int SelectIntoCTTable(string sourceCTDB, TableConf table, string sourceDB, ChangeTrackingBatch batch, int timeout, long?startVersionOverride)
 {
     throw new NotImplementedException("Netezza is only supported as a slave!");
 }
Пример #51
0
 public int SelectIntoCTTable(string sourceCTDB, TableConf table, string sourceDB, ChangeTrackingBatch ctb, int timeout, long?overrideStartVersion)
 {
     //no good way to fake this with DataTables so just return and make sure we are also unit testing the
     //methods that generate these sfield lists
     return(testData.Tables[table.SchemaName + "." + table.ToCTName(ctb.CTID), GetTableSpace(sourceCTDB)].Rows.Count);
 }
Пример #52
0
        private void ApplySchemaChangesAndWrite(ChangeTrackingBatch ctb)
        {
            if ((ctb.SyncBitWise & Convert.ToInt32(SyncBitWise.ApplySchemaChanges)) == 0) {
                logger.Log("Applying schema changes", LogLevel.Debug);
                var sw = Stopwatch.StartNew();

                ApplySchemaChanges(Config.RelayDB, Config.SlaveDB, ctb.CTID);

                sourceDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.ApplySchemaChanges), AgentType.Slave);
                ctb.SyncBitWise += Convert.ToInt32(SyncBitWise.ApplySchemaChanges);

                logger.Timing(StepTimingKey("ApplySchemaChanges"), (int)sw.ElapsedMilliseconds);
            }
        }
Пример #53
0
 public Dictionary <TableConf, IList <string> > GetAllPrimaryKeys(string dbName, IEnumerable <TableConf> tables, ChangeTrackingBatch batch)
 {
     throw new NotImplementedException();
 }
Пример #54
0
        public override void Run()
        {
            DateTime start = DateTime.Now;
            logger.Log("Getting CHANGE_TRACKING_CURRENT_VERSION from master", LogLevel.Trace);
            Int64 currentVersion = sourceDataUtils.GetCurrentCTVersion(Config.MasterDB);

            logger.Log("Initializing CT batch", LogLevel.Info);
            //set up the variables and CT version info for this run
            ctb = InitializeBatch(currentVersion);
            if (Config.Sharding && ctb == null) {
                logger.Log("Last batch completed and there is no new batch to work on.", LogLevel.Info);
                return;
            }
            Logger.SetProperty("CTID", ctb.CTID);
            logger.Log(ctb, LogLevel.Debug);

            logger.Log("Working on CTID " + ctb.CTID, LogLevel.Info);
            IDictionary<string, Int64> changesCaptured;

            logger.Log("Calculating field lists for configured tables", LogLevel.Info);
            SetFieldLists(Config.MasterDB, Config.Tables, sourceDataUtils);

            //capture changes/publish schema changes is now one unit, so we can just check either.
            if ((ctb.SyncBitWise & Convert.ToInt32(SyncBitWise.CaptureChanges)) == 0) {
                changesCaptured = CaptureChanges(currentVersion);
                PublishSchemaChanges();
                destDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.PublishSchemaChanges), AgentType.Master);
                destDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.CaptureChanges), AgentType.Master);
            } else {
                logger.Log("CreateChangeTables succeeded on the previous run, running GetRowCounts instead to populate changesCaptured object", LogLevel.Info);
                changesCaptured = GetRowCounts(Config.Tables, Config.MasterCTDB, ctb.CTID);
                logger.Log("Successfully populated changesCaptured with a list of rowcounts for each changetable", LogLevel.Debug);
            }

            var sw = Stopwatch.StartNew();
            //copy change tables from master to relay server
            logger.Log("Beginning publish changetables step, copying CT tables to the relay server", LogLevel.Info);
            PublishChangeTables(Config.MasterCTDB, Config.RelayDB, ctb.CTID, changesCaptured);
            logger.Log("Publishing info table", LogLevel.Info);
            PublishTableInfo(Config.Tables, Config.RelayDB, changesCaptured, ctb.CTID);
            logger.Log("Successfully published changetables, persisting bitwise now", LogLevel.Debug);

            //this signifies the end of the master's responsibility for this batch
            destDataUtils.WriteBitWise(Config.RelayDB, ctb.CTID, Convert.ToInt32(SyncBitWise.UploadChanges), AgentType.Master);
            logger.Log("Wrote bitwise value of " + Convert.ToInt32(SyncBitWise.UploadChanges) + " to tblCTVersion", LogLevel.Trace);
            logger.Timing(StepTimingKey("UploadChanges"), (int)sw.ElapsedMilliseconds);

            logger.Log("Master agent work complete", LogLevel.Info);
            var elapsed = DateTime.Now - start;
            logger.Timing(TimingKey, (int)elapsed.TotalMinutes);

            sourceDataUtils.CleanUpInitializeTable(Config.MasterCTDB, ctb.SyncStartTime.Value);
        }