public void TestRandomCrashServer() { var client = Utils.GetClient(); Random random = new Random(); while (true) { var configState = Utils.GetFullConfigState(client); var shardServers = configState.shardServers; var quorum = configState.quorums[0]; var numShardServers = shardServers.Count; if (quorum.inactiveNodes.Count > numAllowedInactiveNodes) { var inactiveSleepTime = random.Next(minInactiveSleepTime, minInactiveSleepTime + random.Next(randomInactiveSleepTime)); Utils.Log("Inactive found, sleeping {0} until {1}...", inactiveSleepTime, DateTime.Now.AddSeconds(inactiveSleepTime).ToString("u")); Thread.Sleep(inactiveSleepTime * 1000); continue; } var httpURI = ""; if (random.Next(2) == 0) { // select controller var victimController = configState.controllers[random.Next(configState.controllers.Count)]; httpURI = ConfigStateHelpers.GetControllerURL(victimController); Utils.Log("Killing controller {0}", victimController.nodeID); } else { // select shard server var victimNodeID = quorum.activeNodes[random.Next(quorum.activeNodes.Count)]; foreach (var shardServer in shardServers) { if (shardServer.nodeID == victimNodeID) { httpURI = ConfigStateHelpers.GetShardServerURL(shardServer); Utils.Log("Killing shard server {0}", shardServer.nodeID); break; } } } if (httpURI != "") { var httpCrashURI = Utils.HTTP.BuildUri(httpURI, "debug?randomcrash&key=" + debugKey + "&interval=" + crashInterval * 1000); Utils.Log(httpCrashURI); Utils.HTTP.GET(httpCrashURI); } var sleepTime = random.Next(minCrashSleepTime, minCrashSleepTime + random.Next(randomCrashSleepTime)); Utils.Log("Sleeping {0} until {1}...", sleepTime, DateTime.Now.AddSeconds(sleepTime).ToString("u")); Thread.Sleep(sleepTime * 1000); } }
public void TestRandomCrashShardServer() { var client = new Client(Utils.GetConfigNodes()); Random random = new Random(); while (true) { var configState = Utils.GetFullConfigState(client); if (configState == null) { var sleepTime = random.Next(minInactiveSleepTime, minInactiveSleepTime + random.Next(randomInactiveSleepTime)); Console.WriteLine("No controller could serve configState, sleeping {0}...", sleepTime); Thread.Sleep(sleepTime * 1000); continue; } var shardServers = configState.shardServers; var quorum = configState.quorums[0]; var numShardServers = shardServers.Count; if (quorum.inactiveNodes.Count > numAllowedInactiveNodes) { var sleepTime = random.Next(minInactiveSleepTime, minInactiveSleepTime + random.Next(randomInactiveSleepTime)); Console.WriteLine("Inactive found, sleeping {0}...", sleepTime); Thread.Sleep(sleepTime * 1000); continue; } var victimNodeID = quorum.activeNodes[random.Next(quorum.activeNodes.Count)]; foreach (var shardServer in shardServers) { if (shardServer.nodeID == victimNodeID) { var shardHttpURI = ConfigStateHelpers.GetShardServerURL(shardServer); Console.WriteLine("Killing {0}", shardHttpURI); Utils.HTTP.GET(Utils.HTTP.BuildUri(shardHttpURI, "debug?randomcrash&key=" + debugKey + "&interval=300")); var sleepTime = random.Next(minCrashSleepTime, minCrashSleepTime + random.Next(randomCrashSleepTime)); Console.WriteLine("Sleeping {0}...", sleepTime); Thread.Sleep(sleepTime * 1000); } } } }
public void TestChaosMonkey() { var client = Utils.GetClient(); Random random = new Random(); bool allowInactive = false; var sleepInterval = ConfigFile.Config.GetIntValue("sleepInterval", 4); while (true) { var configState = Utils.GetFullConfigState(client); var randomFunction = random.Next(0, 100); var shardServer = GetRandomShardServer(configState, random, allowInactive); if (shardServer == null) { RandomSleep(random, sleepInterval); continue; } var shardHttpURI = ConfigStateHelpers.GetShardServerURL(shardServer); // 90% sleep // 5% async infinite loop // 4% yield infinite loop // 1% crash if (randomFunction >= 0 && randomFunction < 90) { SleepServer(shardHttpURI, sleepInterval); } else if (randomFunction >= 90 && randomFunction < 95) { RunInfiniteLoopOnServer(shardHttpURI, crashInterval, true); } else if (randomFunction >= 95 && randomFunction < 99) { RunInfiniteLoopOnServer(shardHttpURI, crashInterval, false); } else if (randomFunction >= 99) { CrashServer(shardHttpURI, crashInterval); } } }
public void TestRandomSleepPrimaryShardServer() { var client = Utils.GetClient(); var sleepInterval = 4; Random random = new Random(); while (true) { var configState = Utils.GetFullConfigState(client); var shardServers = configState.shardServers; var quorum = configState.quorums[0]; var numShardServers = shardServers.Count; if (quorum.inactiveNodes.Count > 0) { var sleepTime = random.Next(sleepInterval * 2, sleepInterval * 2 + random.Next(sleepInterval * 2)); Console.WriteLine("Inactive found, sleeping {0}...", sleepTime); Thread.Sleep(sleepTime * 1000); continue; } var victimNodeID = quorum.activeNodes.Where(nodeID => nodeID == quorum.primaryID).First(); foreach (var shardServer in shardServers) { if (shardServer.nodeID == victimNodeID) { var shardHttpURI = ConfigStateHelpers.GetShardServerURL(shardServer); Console.WriteLine("Sleeping {0} for {1} secs", shardHttpURI, sleepInterval); Utils.HTTP.GET(Utils.HTTP.BuildUri(shardHttpURI, "debug?sleep=" + sleepInterval + "&key=" + debugKey)); var sleepTime = random.Next(sleepInterval * 2, sleepInterval * 2 + random.Next(sleepInterval * 2)); Console.WriteLine("Sleeping {0}...", sleepTime); Thread.Sleep(sleepTime * 1000); } } } }
public void TestRandomSleepShardServer() { var client = Utils.GetClient(); var sleepInterval = ConfigFile.Config.GetIntValue("sleepInterval", 4); Random random = new Random(); while (true) { var configState = Utils.GetFullConfigState(client); var quorum = configState == null ? null : configState.quorums[0]; if (configState == null || quorum.inactiveNodes.Count > 0) { var sleepTime = random.Next(sleepInterval * 2, sleepInterval * 2 + random.Next(sleepInterval * 2)); Console.WriteLine("Inactive found, sleeping {0}...", sleepTime); Thread.Sleep(sleepTime * 1000); continue; } var shardServers = configState.shardServers; var victimNodeID = quorum.activeNodes[random.Next(quorum.activeNodes.Count)]; foreach (var shardServer in shardServers) { if (shardServer.nodeID == victimNodeID) { bool isPrimary = (quorum.hasPrimary && quorum.primaryID == victimNodeID); var shardHttpURI = ConfigStateHelpers.GetShardServerURL(shardServer); Console.WriteLine("Sleeping {0} for {1} secs {2}", shardHttpURI, sleepInterval, isPrimary ? "(Primary)" : ""); Utils.HTTP.GET(Utils.HTTP.BuildUri(shardHttpURI, "debug?sleep=" + sleepInterval + "&key=" + debugKey)); var sleepTime = random.Next(sleepInterval * 2, sleepInterval * 2 + random.Next(sleepInterval * 2)); Console.WriteLine("Sleeping {0}...", sleepTime); Thread.Sleep(sleepTime * 1000); } } } }
public static void ParallelTableCountHTTP(List <ConfigState.ShardServer> shardServers, Int64 tableID, string startKey, string endKey) { var threads = new CounterThreadState[shardServers.Count]; var i = 0; foreach (var shardServer in shardServers) { var url = ConfigStateHelpers.GetShardServerURL(shardServer) + "count?tableID=" + tableID + "&startKey=" + startKey + "&endKey=" + endKey; System.Console.WriteLine(" Getting count on nodeID: " + shardServer.nodeID + ", tableID: " + tableID); threads[i] = new CounterThreadState(); threads[i].thread = new Thread(new ThreadStart(threads[i].CounterThreadFunc)); threads[i].shardServer = shardServer; threads[i].count = -1; threads[i].url = url; threads[i].thread.Start(); i += 1; } Int64 prevCount = -1; foreach (var thread in threads) { thread.thread.Join(); if (prevCount == -1) { prevCount = thread.count; } System.Console.WriteLine(" Result on " + thread.shardServer.nodeID + ": " + thread.count + ", elapsed: " + thread.elapsed.Seconds + "s, cps: " + (Int64)(thread.count / thread.elapsed.TotalMilliseconds) * 1000); Assert.IsTrue(thread.count != -1 && thread.count == prevCount); prevCount = thread.count; } }