コード例 #1
0
        public void TestRandomCrashServer()
        {
            var client = Utils.GetClient();

            Random random = new Random();

            while (true)
            {
                var configState     = Utils.GetFullConfigState(client);
                var shardServers    = configState.shardServers;
                var quorum          = configState.quorums[0];
                var numShardServers = shardServers.Count;

                if (quorum.inactiveNodes.Count > numAllowedInactiveNodes)
                {
                    var inactiveSleepTime = random.Next(minInactiveSleepTime, minInactiveSleepTime + random.Next(randomInactiveSleepTime));
                    Utils.Log("Inactive found, sleeping {0} until {1}...", inactiveSleepTime, DateTime.Now.AddSeconds(inactiveSleepTime).ToString("u"));
                    Thread.Sleep(inactiveSleepTime * 1000);
                    continue;
                }

                var httpURI = "";
                if (random.Next(2) == 0)
                {
                    // select controller
                    var victimController = configState.controllers[random.Next(configState.controllers.Count)];
                    httpURI = ConfigStateHelpers.GetControllerURL(victimController);
                    Utils.Log("Killing controller {0}", victimController.nodeID);
                }
                else
                {
                    // select shard server
                    var victimNodeID = quorum.activeNodes[random.Next(quorum.activeNodes.Count)];
                    foreach (var shardServer in shardServers)
                    {
                        if (shardServer.nodeID == victimNodeID)
                        {
                            httpURI = ConfigStateHelpers.GetShardServerURL(shardServer);
                            Utils.Log("Killing shard server {0}", shardServer.nodeID);
                            break;
                        }
                    }
                }


                if (httpURI != "")
                {
                    var httpCrashURI = Utils.HTTP.BuildUri(httpURI, "debug?randomcrash&key=" + debugKey + "&interval=" + crashInterval * 1000);
                    Utils.Log(httpCrashURI);
                    Utils.HTTP.GET(httpCrashURI);
                }

                var sleepTime = random.Next(minCrashSleepTime, minCrashSleepTime + random.Next(randomCrashSleepTime));
                Utils.Log("Sleeping {0} until {1}...", sleepTime, DateTime.Now.AddSeconds(sleepTime).ToString("u"));
                Thread.Sleep(sleepTime * 1000);
            }
        }
コード例 #2
0
        public void TestRandomCrashShardServer()
        {
            var client = new Client(Utils.GetConfigNodes());

            Random random = new Random();

            while (true)
            {
                var configState = Utils.GetFullConfigState(client);
                if (configState == null)
                {
                    var sleepTime = random.Next(minInactiveSleepTime, minInactiveSleepTime + random.Next(randomInactiveSleepTime));
                    Console.WriteLine("No controller could serve configState, sleeping {0}...", sleepTime);
                    Thread.Sleep(sleepTime * 1000);
                    continue;
                }

                var shardServers    = configState.shardServers;
                var quorum          = configState.quorums[0];
                var numShardServers = shardServers.Count;

                if (quorum.inactiveNodes.Count > numAllowedInactiveNodes)
                {
                    var sleepTime = random.Next(minInactiveSleepTime, minInactiveSleepTime + random.Next(randomInactiveSleepTime));
                    Console.WriteLine("Inactive found, sleeping {0}...", sleepTime);
                    Thread.Sleep(sleepTime * 1000);
                    continue;
                }

                var victimNodeID = quorum.activeNodes[random.Next(quorum.activeNodes.Count)];
                foreach (var shardServer in shardServers)
                {
                    if (shardServer.nodeID == victimNodeID)
                    {
                        var shardHttpURI = ConfigStateHelpers.GetShardServerURL(shardServer);
                        Console.WriteLine("Killing {0}", shardHttpURI);
                        Utils.HTTP.GET(Utils.HTTP.BuildUri(shardHttpURI, "debug?randomcrash&key=" + debugKey + "&interval=300"));
                        var sleepTime = random.Next(minCrashSleepTime, minCrashSleepTime + random.Next(randomCrashSleepTime));
                        Console.WriteLine("Sleeping {0}...", sleepTime);
                        Thread.Sleep(sleepTime * 1000);
                    }
                }
            }
        }
コード例 #3
0
        public void TestChaosMonkey()
        {
            var    client        = Utils.GetClient();
            Random random        = new Random();
            bool   allowInactive = false;
            var    sleepInterval = ConfigFile.Config.GetIntValue("sleepInterval", 4);

            while (true)
            {
                var configState    = Utils.GetFullConfigState(client);
                var randomFunction = random.Next(0, 100);

                var shardServer = GetRandomShardServer(configState, random, allowInactive);
                if (shardServer == null)
                {
                    RandomSleep(random, sleepInterval);
                    continue;
                }

                var shardHttpURI = ConfigStateHelpers.GetShardServerURL(shardServer);

                // 90% sleep
                // 5% async infinite loop
                // 4% yield infinite loop
                // 1% crash
                if (randomFunction >= 0 && randomFunction < 90)
                {
                    SleepServer(shardHttpURI, sleepInterval);
                }
                else if (randomFunction >= 90 && randomFunction < 95)
                {
                    RunInfiniteLoopOnServer(shardHttpURI, crashInterval, true);
                }
                else if (randomFunction >= 95 && randomFunction < 99)
                {
                    RunInfiniteLoopOnServer(shardHttpURI, crashInterval, false);
                }
                else if (randomFunction >= 99)
                {
                    CrashServer(shardHttpURI, crashInterval);
                }
            }
        }
コード例 #4
0
        public void TestRandomSleepPrimaryShardServer()
        {
            var client        = Utils.GetClient();
            var sleepInterval = 4;

            Random random = new Random();

            while (true)
            {
                var configState     = Utils.GetFullConfigState(client);
                var shardServers    = configState.shardServers;
                var quorum          = configState.quorums[0];
                var numShardServers = shardServers.Count;

                if (quorum.inactiveNodes.Count > 0)
                {
                    var sleepTime = random.Next(sleepInterval * 2, sleepInterval * 2 + random.Next(sleepInterval * 2));
                    Console.WriteLine("Inactive found, sleeping {0}...", sleepTime);
                    Thread.Sleep(sleepTime * 1000);
                    continue;
                }

                var victimNodeID = quorum.activeNodes.Where(nodeID => nodeID == quorum.primaryID).First();
                foreach (var shardServer in shardServers)
                {
                    if (shardServer.nodeID == victimNodeID)
                    {
                        var shardHttpURI = ConfigStateHelpers.GetShardServerURL(shardServer);
                        Console.WriteLine("Sleeping {0} for {1} secs", shardHttpURI, sleepInterval);
                        Utils.HTTP.GET(Utils.HTTP.BuildUri(shardHttpURI, "debug?sleep=" + sleepInterval + "&key=" + debugKey));
                        var sleepTime = random.Next(sleepInterval * 2, sleepInterval * 2 + random.Next(sleepInterval * 2));
                        Console.WriteLine("Sleeping {0}...", sleepTime);
                        Thread.Sleep(sleepTime * 1000);
                    }
                }
            }
        }
コード例 #5
0
        public void TestRandomSleepShardServer()
        {
            var client        = Utils.GetClient();
            var sleepInterval = ConfigFile.Config.GetIntValue("sleepInterval", 4);

            Random random = new Random();

            while (true)
            {
                var configState = Utils.GetFullConfigState(client);
                var quorum      = configState == null ? null : configState.quorums[0];

                if (configState == null || quorum.inactiveNodes.Count > 0)
                {
                    var sleepTime = random.Next(sleepInterval * 2, sleepInterval * 2 + random.Next(sleepInterval * 2));
                    Console.WriteLine("Inactive found, sleeping {0}...", sleepTime);
                    Thread.Sleep(sleepTime * 1000);
                    continue;
                }

                var shardServers = configState.shardServers;
                var victimNodeID = quorum.activeNodes[random.Next(quorum.activeNodes.Count)];
                foreach (var shardServer in shardServers)
                {
                    if (shardServer.nodeID == victimNodeID)
                    {
                        bool isPrimary    = (quorum.hasPrimary && quorum.primaryID == victimNodeID);
                        var  shardHttpURI = ConfigStateHelpers.GetShardServerURL(shardServer);
                        Console.WriteLine("Sleeping {0} for {1} secs {2}", shardHttpURI, sleepInterval, isPrimary ? "(Primary)" : "");
                        Utils.HTTP.GET(Utils.HTTP.BuildUri(shardHttpURI, "debug?sleep=" + sleepInterval + "&key=" + debugKey));
                        var sleepTime = random.Next(sleepInterval * 2, sleepInterval * 2 + random.Next(sleepInterval * 2));
                        Console.WriteLine("Sleeping {0}...", sleepTime);
                        Thread.Sleep(sleepTime * 1000);
                    }
                }
            }
        }
コード例 #6
0
        public static void ParallelTableCountHTTP(List <ConfigState.ShardServer> shardServers, Int64 tableID, string startKey, string endKey)
        {
            var threads = new CounterThreadState[shardServers.Count];

            var i = 0;

            foreach (var shardServer in shardServers)
            {
                var url = ConfigStateHelpers.GetShardServerURL(shardServer) + "count?tableID=" + tableID + "&startKey=" + startKey + "&endKey=" + endKey;
                System.Console.WriteLine("  Getting count on nodeID: " + shardServer.nodeID + ", tableID: " + tableID);

                threads[i]             = new CounterThreadState();
                threads[i].thread      = new Thread(new ThreadStart(threads[i].CounterThreadFunc));
                threads[i].shardServer = shardServer;
                threads[i].count       = -1;
                threads[i].url         = url;

                threads[i].thread.Start();

                i += 1;
            }

            Int64 prevCount = -1;

            foreach (var thread in threads)
            {
                thread.thread.Join();
                if (prevCount == -1)
                {
                    prevCount = thread.count;
                }
                System.Console.WriteLine("  Result on " + thread.shardServer.nodeID + ": " + thread.count + ", elapsed: " + thread.elapsed.Seconds + "s, cps: " + (Int64)(thread.count / thread.elapsed.TotalMilliseconds) * 1000);
                Assert.IsTrue(thread.count != -1 && thread.count == prevCount);
                prevCount = thread.count;
            }
        }