public void Publish_Shards_Asycn() { var currentConfig = ShardSetConfig.LoadCurrent(TestShardSetName); currentConfig.Servers.Clear(); var serverLocation = ConfigurationManager.AppSettings["TestSQLServer"] ?? "(localdb)\v11.0"; var server = Server.Load(serverLocation) ?? new Server { ServerInstanceName = serverLocation, Location = "Test Server Location", MaxShardsAllowed = -1, }; server.MaxShardsAllowed = -1; server = server.Save(); currentConfig.Servers.Add(server); currentConfig.Save(); currentConfig.UpdateShardMap(); currentConfig.Save(); currentConfig.DeployShardMap(true); currentConfig.PublishShardMap(true); var queue = ShardSetActionQueue.GetQueue(); queue.QueueProcessingEvent += Queue_QueueProcessingEvent; queue.CheckAndProcessQueue(Guid.NewGuid()); }
public void Elastic_Scale_Up_Test() { SaveTestSettings(); SaveTestShardSetConfig(); DeployCurrentShardSetConfigAsync(); // process the queues var queue = ShardSetActionQueue.GetQueue(); queue.QueueProcessingEvent += Queue_QueueProcessingEvent; queue.CheckAndProcessQueue(Guid.NewGuid()); // create some test data across the shards //CreateTestData(); // Reread the test shard set configuration to get the Current Shard Map Id value // of prior deployment var shardSetConfig = ShardSetConfig.LoadCurrent(TestShardSetName); // create a new configuration with more shards shardSetConfig.TargetShardCount = 8; shardSetConfig.MaxShardCount = 8; shardSetConfig.UpdateShardMap(); shardSetConfig.Save(); // redeploy and publish async shardSetConfig.DeployShardMap(true); shardSetConfig.PublishShardMap(true); // execute from queues until clear queue.CheckAndProcessQueue(Guid.NewGuid()); }
public void T4_Pin_Data_To_Shard() { var shardSetConfig = ShardSetConfig.LoadCurrent(TestShardSetName); var pointerShard = shardSetConfig.Shards.First(); var shardlet = Shardlet.Load(TestShardSetName, "1"); shardlet.MoveToShard(pointerShard, true, true, new Guid(), _queueAndUseWorkerRoles); }
/// <summary> /// Executes the query. /// </summary> /// <param name="commandText">The command text.</param> /// <returns>DataTable.</returns> public DataTable ExecuteQuery(string commandText) { //todo: needs another look on how the tasks are being run var shardSetConfig = ShardSetConfig.LoadCurrent(_shardSetName); var shardMap = shardSetConfig.ShardMap; var settings = Settings.Load(); var finalResult = new DataTable(); var totalShards = shardMap.Shards.Count; var tasks = new Task <DataTable> [totalShards]; var i = 0; foreach (var shard in shardMap.Shards.ToList()) { var connectionString = "Server=" + shard.ServerInstanceName + ";Database=" + shard.Catalog + ";User Id=" + settings.ShardUser + ";Password="******";"; tasks[i] = Task.Run(() => ExecuteQuery(commandText, new ReliableSqlConnection(connectionString).Current)); i++; } var allComplete = false; while (!allComplete) { foreach (Task task in tasks) { if (!task.Status.Equals(TaskStatus.RanToCompletion)) { if (task.Status.Equals(TaskStatus.Faulted)) { Debug.WriteLine("Faulted: " + task.Exception.InnerException.Message); return(null); } Thread.Sleep(25); break; } allComplete = true; } } foreach (var dt in tasks) { finalResult.Merge(dt.Result); } return(finalResult); }
public void Load_Edit_Save_Table_Group_Config() { var r = new Random(); var dbSize = r.Next(250); var currentConfig = ShardSetConfig.LoadCurrent(TestShardSetName); currentConfig.MaxShardSizeMb = dbSize; currentConfig.Save(); var newCurrentConfig = ShardSetConfig.LoadCurrent(TestShardSetName); Assert.AreEqual(dbSize, newCurrentConfig.MaxShardSizeMb); }
public void T2_Update_Reference_Data_And_Propagate() { // Scenario - nightly job updates the CurrencyRate table in AwMain // Run data update // Queue a Shard Sync for Range Shards // Queue a Shard Sync for Pointer Shards // update the CurrencyRate table in AwMain LoadCurrencyRatesInAwMain(500); // Get the shard set configuration var shardSetConfig = ShardSetConfig.LoadCurrent(TestShardSetName); // range shard synchronization shardSetConfig.SyncShards(_queueAndUseWorkerRoles); // pointer shard synchronization shardSetConfig.SyncPointerShards(_queueAndUseWorkerRoles); }
public void T6_Scale_In() { // read the test shard set configuration var shardSetConfig = ShardSetConfig.LoadCurrent(TestShardSetName); // create a new configuration with more shards shardSetConfig.TargetShardCount = 3; shardSetConfig.MaxShardCount = 3; // recalculate the shard map and save a new configuration shardSetConfig.UpdateShardMap(); shardSetConfig.Save(); // shard deployment shardSetConfig.DeployShardMap(true); // update of online shard map shardSetConfig.PublishShardMap(true); }
/// <summary> /// Executes the non query on all shards in the shard map. /// </summary> /// <param name="commandText">The command text.</param> /// <remarks> /// NOTE: This only works properly if the current configuration is in sync with the /// actual shards. /// </remarks> public void ExecuteNonQuery(string commandText) { var shardSetConfig = ShardSetConfig.LoadCurrent(_shardSetName); var shardMap = shardSetConfig.ShardMap; var settings = Settings.Load(); var totalShards = shardMap.Shards.Count; var tasks = new Task[totalShards]; var i = 0; foreach (var shard in shardMap.Shards.ToList()) { var connString = "Server=" + shard.ServerInstanceName + ";Database=" + shard.Catalog + ";User Id=" + settings.ShardUser + ";Password="******";"; tasks[i] = Task.Run(() => ExecuteNonQuery(commandText, new ReliableSqlConnection(connString).Current)); i++; } var allComplete = false; while (!allComplete) { foreach (var task in tasks) { if (!task.Status.Equals(TaskStatus.RanToCompletion)) { if (task.Status.Equals(TaskStatus.Faulted)) { Debug.WriteLine("Faulted: " + task.Exception.InnerException.Message); } Thread.Sleep(25); break; } allComplete = true; } } }
protected static ShardSetConfig SetupTestShardSetConfig() { // Get the test shard set configuration var shardSetConfig = ShardSetConfig.LoadCurrent(TestShardSetName); // Get the test server var serverLocation = ConfigurationManager.AppSettings["TestSQLServer"] ?? @"(localdb)\v11.0"; var server = GetTestServer(serverLocation); // For testing don;t worry about how many shards are in the server server.MaxShardsAllowed = -1; server = server.Save(); // Clear out the servers and just add the test server shardSetConfig.Servers.Clear(); shardSetConfig.Servers.Add(server); // Update the shard map to point to the databases to the new server shardSetConfig.UpdateShardMap(); shardSetConfig.Save(); return(shardSetConfig); }
public void Add_ShardMap_To_Azure() { var currentConfig = ShardSetConfig.LoadCurrent(TestShardSetName); currentConfig.PublishShardMap(); }