public void TestRandomCrashServer()
{
var client = Utils.GetClient();
Random random = new Random();
while (true)
{
var configState = Utils.GetFullConfigState(client);
var shardServers = configState.shardServers;
var quorum = configState.quorums[0];
var numShardServers = shardServers.Count;
if (quorum.inactiveNodes.Count > numAllowedInactiveNodes)
{
var inactiveSleepTime = random.Next(minInactiveSleepTime, minInactiveSleepTime + random.Next(randomInactiveSleepTime));
Utils.Log("Inactive found, sleeping {0} until {1}...", inactiveSleepTime, DateTime.Now.AddSeconds(inactiveSleepTime).ToString("u"));
Thread.Sleep(inactiveSleepTime * 1000);
continue;
}
var httpURI = "";
if (random.Next(2) == 0)
{
// select controller
var victimController = configState.controllers[random.Next(configState.controllers.Count)];
httpURI = ConfigStateHelpers.GetControllerURL(victimController);
Utils.Log("Killing controller {0}", victimController.nodeID);
}
else
{
// select shard server
var victimNodeID = quorum.activeNodes[random.Next(quorum.activeNodes.Count)];
foreach (var shardServer in shardServers)
{
if (shardServer.nodeID == victimNodeID)
{
httpURI = ConfigStateHelpers.GetShardServerURL(shardServer);
Utils.Log("Killing shard server {0}", shardServer.nodeID);
break;
}
}
}
if (httpURI != "")
{
var httpCrashURI = Utils.HTTP.BuildUri(httpURI, "debug?randomcrash&key=" + debugKey + "&interval=" + crashInterval * 1000);
Utils.Log(httpCrashURI);
Utils.HTTP.GET(httpCrashURI);
}
var sleepTime = random.Next(minCrashSleepTime, minCrashSleepTime + random.Next(randomCrashSleepTime));
Utils.Log("Sleeping {0} until {1}...", sleepTime, DateTime.Now.AddSeconds(sleepTime).ToString("u"));
Thread.Sleep(sleepTime * 1000);
}
}
public void TestRandomCrashShardServer()
{
var client = new Client(Utils.GetConfigNodes());
Random random = new Random();
while (true)
{
var configState = Utils.GetFullConfigState(client);
if (configState == null)
{
var sleepTime = random.Next(minInactiveSleepTime, minInactiveSleepTime + random.Next(randomInactiveSleepTime));
Console.WriteLine("No controller could serve configState, sleeping {0}...", sleepTime);
Thread.Sleep(sleepTime * 1000);
continue;
}
var shardServers = configState.shardServers;
var quorum = configState.quorums[0];
var numShardServers = shardServers.Count;
if (quorum.inactiveNodes.Count > numAllowedInactiveNodes)
{
var sleepTime = random.Next(minInactiveSleepTime, minInactiveSleepTime + random.Next(randomInactiveSleepTime));
Console.WriteLine("Inactive found, sleeping {0}...", sleepTime);
Thread.Sleep(sleepTime * 1000);
continue;
}
var victimNodeID = quorum.activeNodes[random.Next(quorum.activeNodes.Count)];
foreach (var shardServer in shardServers)
{
if (shardServer.nodeID == victimNodeID)
{
var shardHttpURI = ConfigStateHelpers.GetShardServerURL(shardServer);
Console.WriteLine("Killing {0}", shardHttpURI);
Utils.HTTP.GET(Utils.HTTP.BuildUri(shardHttpURI, "debug?randomcrash&key=" + debugKey + "&interval=300"));
var sleepTime = random.Next(minCrashSleepTime, minCrashSleepTime + random.Next(randomCrashSleepTime));
Console.WriteLine("Sleeping {0}...", sleepTime);
Thread.Sleep(sleepTime * 1000);
}
}
}
}
public void TestChaosMonkey()
{
var client = Utils.GetClient();
Random random = new Random();
bool allowInactive = false;
var sleepInterval = ConfigFile.Config.GetIntValue("sleepInterval", 4);
while (true)
{
var configState = Utils.GetFullConfigState(client);
var randomFunction = random.Next(0, 100);
var shardServer = GetRandomShardServer(configState, random, allowInactive);
if (shardServer == null)
{
RandomSleep(random, sleepInterval);
continue;
}
var shardHttpURI = ConfigStateHelpers.GetShardServerURL(shardServer);
// 90% sleep
// 5% async infinite loop
// 4% yield infinite loop
// 1% crash
if (randomFunction >= 0 && randomFunction < 90)
{
SleepServer(shardHttpURI, sleepInterval);
}
else if (randomFunction >= 90 && randomFunction < 95)
{
RunInfiniteLoopOnServer(shardHttpURI, crashInterval, true);
}
else if (randomFunction >= 95 && randomFunction < 99)
{
RunInfiniteLoopOnServer(shardHttpURI, crashInterval, false);
}
else if (randomFunction >= 99)
{
CrashServer(shardHttpURI, crashInterval);
}
}
}
public void TestRandomSleepPrimaryShardServer()
{
var client = Utils.GetClient();
var sleepInterval = 4;
Random random = new Random();
while (true)
{
var configState = Utils.GetFullConfigState(client);
var shardServers = configState.shardServers;
var quorum = configState.quorums[0];
var numShardServers = shardServers.Count;
if (quorum.inactiveNodes.Count > 0)
{
var sleepTime = random.Next(sleepInterval * 2, sleepInterval * 2 + random.Next(sleepInterval * 2));
Console.WriteLine("Inactive found, sleeping {0}...", sleepTime);
Thread.Sleep(sleepTime * 1000);
continue;
}
var victimNodeID = quorum.activeNodes.Where(nodeID => nodeID == quorum.primaryID).First();
foreach (var shardServer in shardServers)
{
if (shardServer.nodeID == victimNodeID)
{
var shardHttpURI = ConfigStateHelpers.GetShardServerURL(shardServer);
Console.WriteLine("Sleeping {0} for {1} secs", shardHttpURI, sleepInterval);
Utils.HTTP.GET(Utils.HTTP.BuildUri(shardHttpURI, "debug?sleep=" + sleepInterval + "&key=" + debugKey));
var sleepTime = random.Next(sleepInterval * 2, sleepInterval * 2 + random.Next(sleepInterval * 2));
Console.WriteLine("Sleeping {0}...", sleepTime);
Thread.Sleep(sleepTime * 1000);
}
}
}
}
public void TestRandomSleepShardServer()
{
var client = Utils.GetClient();
var sleepInterval = ConfigFile.Config.GetIntValue("sleepInterval", 4);
Random random = new Random();
while (true)
{
var configState = Utils.GetFullConfigState(client);
var quorum = configState == null ? null : configState.quorums[0];
if (configState == null || quorum.inactiveNodes.Count > 0)
{
var sleepTime = random.Next(sleepInterval * 2, sleepInterval * 2 + random.Next(sleepInterval * 2));
Console.WriteLine("Inactive found, sleeping {0}...", sleepTime);
Thread.Sleep(sleepTime * 1000);
continue;
}
var shardServers = configState.shardServers;
var victimNodeID = quorum.activeNodes[random.Next(quorum.activeNodes.Count)];
foreach (var shardServer in shardServers)
{
if (shardServer.nodeID == victimNodeID)
{
bool isPrimary = (quorum.hasPrimary && quorum.primaryID == victimNodeID);
var shardHttpURI = ConfigStateHelpers.GetShardServerURL(shardServer);
Console.WriteLine("Sleeping {0} for {1} secs {2}", shardHttpURI, sleepInterval, isPrimary ? "(Primary)" : "");
Utils.HTTP.GET(Utils.HTTP.BuildUri(shardHttpURI, "debug?sleep=" + sleepInterval + "&key=" + debugKey));
var sleepTime = random.Next(sleepInterval * 2, sleepInterval * 2 + random.Next(sleepInterval * 2));
Console.WriteLine("Sleeping {0}...", sleepTime);
Thread.Sleep(sleepTime * 1000);
}
}
}
}
public static void ParallelTableCountHTTP(List <ConfigState.ShardServer> shardServers, Int64 tableID, string startKey, string endKey)
{
var threads = new CounterThreadState[shardServers.Count];
var i = 0;
foreach (var shardServer in shardServers)
{
var url = ConfigStateHelpers.GetShardServerURL(shardServer) + "count?tableID=" + tableID + "&startKey=" + startKey + "&endKey=" + endKey;
System.Console.WriteLine(" Getting count on nodeID: " + shardServer.nodeID + ", tableID: " + tableID);
threads[i] = new CounterThreadState();
threads[i].thread = new Thread(new ThreadStart(threads[i].CounterThreadFunc));
threads[i].shardServer = shardServer;
threads[i].count = -1;
threads[i].url = url;
threads[i].thread.Start();
i += 1;
}
Int64 prevCount = -1;
foreach (var thread in threads)
{
thread.thread.Join();
if (prevCount == -1)
{
prevCount = thread.count;
}
System.Console.WriteLine(" Result on " + thread.shardServer.nodeID + ": " + thread.count + ", elapsed: " + thread.elapsed.Seconds + "s, cps: " + (Int64)(thread.count / thread.elapsed.TotalMilliseconds) * 1000);
Assert.IsTrue(thread.count != -1 && thread.count == prevCount);
prevCount = thread.count;
}
}
