Ejemplo n.º 1
0
        public static void CheckDatabaseWithNumericKeys(string databaseName, string tableName)
        {
            var   client          = new Client(Utils.GetConfigNodes());
            var   jsonConfigState = client.GetJSONConfigState();
            var   configState     = Utils.JsonDeserialize <ConfigState>(System.Text.Encoding.UTF8.GetBytes(jsonConfigState));
            var   shardServers    = new List <ConfigState.ShardServer>();
            Int64 tableID         = 0;

            foreach (var database in configState.databases)
            {
                if (database.name == databaseName)
                {
                    foreach (var table in configState.tables)
                    {
                        if (table.databaseID == database.databaseID && table.name == tableName)
                        {
                            shardServers = ConfigStateHelpers.GetShardServersByTable(table, configState);
                            tableID      = table.tableID;
                            break;
                        }
                    }
                }
                if (tableID != 0)
                {
                    break;
                }
            }

            Assert.IsTrue(tableID != 0);

            ulong num = 50 * 1000 * 1000;

            CompareNumericTableKeysHTTP(shardServers, tableID, num);
            CompareNumericTableKeysBackwardsHTTP(shardServers, tableID, num);
        }
Ejemplo n.º 2
0
        public void CheckClusterConsistencyByCount()
        {
            var client          = new Client(Utils.GetConfigNodes());
            var jsonConfigState = client.GetJSONConfigState();
            var configState     = Utils.JsonDeserialize <ConfigState>(System.Text.Encoding.UTF8.GetBytes(jsonConfigState));

            foreach (var table in configState.tables)
            {
                var shards = ConfigStateHelpers.GetTableShards(table, configState.shards);
                foreach (var shard in shards)
                {
                    System.Console.WriteLine("\nCounting shard " + shard.shardID);

                    var tableID      = shard.tableID;
                    var startKey     = shard.firstKey;
                    var endKey       = shard.lastKey;
                    var quorum       = ConfigStateHelpers.GetQuorum(configState, shard.quorumID);
                    var shardServers = ConfigStateHelpers.GetQuorumActiveShardServers(configState, quorum);

                    ParallelTableCountHTTP(shardServers, tableID, startKey, endKey);

                    System.Console.WriteLine("Shard " + shard.shardID + " is consistent.\n");
                }
            }
        }
Ejemplo n.º 3
0
        public static void FixDiffs(Client client, List <ConfigState.ShardServer> shardServers, Int64 tableID, List <string> diffs)
        {
            var i = 0;

            foreach (var key in diffs)
            {
                i += 1;
                byte[] startKey        = Utils.StringToByteArray(key);
                byte[] endKey          = Utils.NextKey(startKey);
                var    serverKeyValues = ConfigStateHelpers.ParallelFetchTableKeyValuesHTTP(shardServers, tableID, startKey, endKey, true);

                if (Array.TrueForAll(serverKeyValues, val => (val.Count == 1 && Utils.ByteArraysEqual(val.First().Key, serverKeyValues[0].First().Key))))
                {
                    continue;
                }

                foreach (var keyValue in serverKeyValues)
                {
                    if (keyValue == null || keyValue.Count == 0)
                    {
                        continue;
                    }

                    if (keyValue.First().Value.Length > 0)
                    {
                        Assert.IsTrue(Utils.ByteArraysEqual(Utils.StringToByteArray(key), keyValue.First().Key));

                        Console.WriteLine("Setting key {0}", key);
                        Table table = new Table(client, null, (ulong)tableID, "");
                        table.Set(startKey, keyValue.First().Value);
                        client.Submit();
                    }
                }
            }
        }
Ejemplo n.º 4
0
        public static void FillDatabaseWithNumericKeys(string databaseName, string tableName)
        {
            var client = new Client(Utils.GetConfigNodes());

            Assert.IsTrue(ConfigStateHelpers.TryDeleteDatabase(client, databaseName));
            var database = client.CreateDatabase(databaseName);

            Assert.IsNotNull(database, "Cannot create database " + databaseName);
            var table = ConfigStateHelpers.TryCreateTable(database, tableName);

            Assert.IsNotNull(table, "Cannot create table " + tableName);

            System.Console.WriteLine("Filling the database...");

            DateTime last = DateTime.Now;

            for (ulong i = 0; i < 100 * 1000 * 1000; i++)
            {
                var key = Utils.Id(i);
                table.Set(key, key);
                TimeSpan timeSpan = DateTime.Now - last;
                if (timeSpan.TotalSeconds >= 60)
                {
                    System.Console.WriteLine("i: " + i);
                    last = DateTime.Now;
                }
            }

            client.Submit();

            System.Console.WriteLine("Database filling is done.");
        }
Ejemplo n.º 5
0
        public static void CompareTableKeyValuesHTTP(List <ConfigState.ShardServer> shardServers, Int64 tableID, byte[] startKey, byte[] endKey)
        {
            if (shardServers.Count <= 1)
            {
                return;
            }

            var i = 0;

            while (true)
            {
                var serverKeys = ConfigStateHelpers.ParallelFetchTableKeyValuesHTTP(shardServers, tableID, startKey, endKey, true);
                int numEmpty   = serverKeys.Count(keys => keys.Count == 0);
                if (numEmpty == serverKeys.Length)
                {
                    break;
                }

                System.Console.WriteLine("StartKey: " + Utils.ByteArrayToString(startKey));

                for (i = 1; i < serverKeys.Length; i++)
                {
                    if (serverKeys[i].Count != serverKeys[0].Count)
                    {
                        System.Console.WriteLine("Inconsistency at tableID: " + tableID);
                        System.Console.WriteLine("NodeID: " + shardServers.ElementAt(i).nodeID + ", count: " + serverKeys[i].Count);
                        System.Console.WriteLine("NodeID: " + shardServers.ElementAt(0).nodeID + ", count: " + serverKeys[i].Count);
                        Assert.Fail("Inconsistency at tableID: " + tableID);
                        return;
                    }


                    for (var j = 0; j < serverKeys[0].Count; j++)
                    {
                        var a = serverKeys[i][j];
                        var b = serverKeys[0][j];
                        if (Utils.ByteArraysEqual(a.Key, b.Key) && Utils.ByteArraysEqual(a.Value, b.Value))
                        {
                            continue;
                        }

                        System.Console.WriteLine("Inconsistency at tableID: " + tableID);
                        System.Console.WriteLine("NodeID: " + shardServers.ElementAt(i).nodeID + ", key: " + Utils.ByteArrayToString(a.Key));
                        System.Console.WriteLine("NodeID: " + shardServers.ElementAt(0).nodeID + ", key: " + Utils.ByteArrayToString(b.Key));
                        Assert.Fail("Inconsistency at tableID: " + tableID);
                        //return;
                        goto Out;
                    }
                }

Out:
                if (serverKeys[0].Count == 0)
                {
                    break;
                }

                startKey = Utils.NextKey(serverKeys[0].Last().Key);
            }
        }
Ejemplo n.º 6
0
        public void TestRandomCrashServer()
        {
            var client = Utils.GetClient();

            Random random = new Random();

            while (true)
            {
                var configState     = Utils.GetFullConfigState(client);
                var shardServers    = configState.shardServers;
                var quorum          = configState.quorums[0];
                var numShardServers = shardServers.Count;

                if (quorum.inactiveNodes.Count > numAllowedInactiveNodes)
                {
                    var inactiveSleepTime = random.Next(minInactiveSleepTime, minInactiveSleepTime + random.Next(randomInactiveSleepTime));
                    Utils.Log("Inactive found, sleeping {0} until {1}...", inactiveSleepTime, DateTime.Now.AddSeconds(inactiveSleepTime).ToString("u"));
                    Thread.Sleep(inactiveSleepTime * 1000);
                    continue;
                }

                var httpURI = "";
                if (random.Next(2) == 0)
                {
                    // select controller
                    var victimController = configState.controllers[random.Next(configState.controllers.Count)];
                    httpURI = ConfigStateHelpers.GetControllerURL(victimController);
                    Utils.Log("Killing controller {0}", victimController.nodeID);
                }
                else
                {
                    // select shard server
                    var victimNodeID = quorum.activeNodes[random.Next(quorum.activeNodes.Count)];
                    foreach (var shardServer in shardServers)
                    {
                        if (shardServer.nodeID == victimNodeID)
                        {
                            httpURI = ConfigStateHelpers.GetShardServerURL(shardServer);
                            Utils.Log("Killing shard server {0}", shardServer.nodeID);
                            break;
                        }
                    }
                }


                if (httpURI != "")
                {
                    var httpCrashURI = Utils.HTTP.BuildUri(httpURI, "debug?randomcrash&key=" + debugKey + "&interval=" + crashInterval * 1000);
                    Utils.Log(httpCrashURI);
                    Utils.HTTP.GET(httpCrashURI);
                }

                var sleepTime = random.Next(minCrashSleepTime, minCrashSleepTime + random.Next(randomCrashSleepTime));
                Utils.Log("Sleeping {0} until {1}...", sleepTime, DateTime.Now.AddSeconds(sleepTime).ToString("u"));
                Thread.Sleep(sleepTime * 1000);
            }
        }
Ejemplo n.º 7
0
        public static void FixConsistency(Client client, List <ConfigState.ShardServer> shardServers, Int64 tableID, string startKey, string endKey)
        {
            if (shardServers.Count <= 1)
            {
                return;
            }

            var keyDiffs   = new HashSet <string>();
            var serverKeys = new string[shardServers.Count][];
            var i          = 0;

            while (true)
            {
                System.Console.WriteLine("StartKey: " + startKey);
                serverKeys = ConfigStateHelpers.ParallelFetchTableKeysHTTP(shardServers, tableID, startKey, endKey, true);

                for (i = 1; i < serverKeys.Length; i++)
                {
                    if (serverKeys[0].Length > 0 && serverKeys[i].Length > 1 &&
                        serverKeys[0].First().CompareTo(serverKeys[i].Last()) < 0)
                    {
                        foreach (var diff in serverKeys[i].Except(serverKeys[0]))
                        {
                            keyDiffs.Add(diff);
                        }
                    }

                    if (serverKeys[0].Length > 1 && serverKeys[i].Length > 0 &&
                        serverKeys[i].First().CompareTo(serverKeys[0].Last()) < 0)
                    {
                        foreach (var diff in serverKeys[0].Except(serverKeys[i]))
                        {
                            keyDiffs.Add(diff);
                        }
                    }
                }

                if (keyDiffs.Count != 0)
                {
                    FixDiffs(client, shardServers, tableID, keyDiffs.ToList());
                    startKey = keyDiffs.Last();
                    keyDiffs = new HashSet <string>();
                    continue;
                }

                if (serverKeys[0].Length <= 1)
                {
                    break;
                }

                startKey = serverKeys[0][serverKeys[0].Length - 1];
            }
        }
Ejemplo n.º 8
0
        public static void CompareTableKeysHTTP(List <ConfigState.ShardServer> shardServers, Int64 tableID, string startKey, string endKey)
        {
            if (shardServers.Count <= 1)
            {
                return;
            }

            var serverKeys = new string[shardServers.Count][];
            var i          = 0;

            while (true)
            {
                serverKeys = ConfigStateHelpers.ParallelFetchTableKeysHTTP(shardServers, tableID, startKey, endKey, true);

                for (i = 1; i < serverKeys.Length; i++)
                {
                    if (!serverKeys[i].SequenceEqual(serverKeys[0]))
                    {
                        for (var j = 0; j < Math.Max(serverKeys[i].Length, serverKeys[0].Length); j++)
                        {
                            var a = serverKeys[i][j];
                            var b = serverKeys[0][j];
                            if (a == b)
                            {
                                continue;
                            }

                            System.Console.WriteLine("Inconsistency at tableID: " + tableID);
                            System.Console.WriteLine("NodeID: " + shardServers.ElementAt(i).nodeID + ", key: " + a);
                            System.Console.WriteLine("NodeID: " + shardServers.ElementAt(0).nodeID + ", key: " + b);
                            //Assert.IsTrue(a == b);
                            //return;
                            goto Out;
                        }
                    }
                }

Out:
                if (serverKeys[0].Length <= 1)
                {
                    break;
                }

                startKey = serverKeys[0][serverKeys[0].Length - 1];
                System.Console.WriteLine("StartKey: " + startKey);
            }
        }
Ejemplo n.º 9
0
        public void TestRandomCrashShardServer()
        {
            var client = new Client(Utils.GetConfigNodes());

            Random random = new Random();

            while (true)
            {
                var configState = Utils.GetFullConfigState(client);
                if (configState == null)
                {
                    var sleepTime = random.Next(minInactiveSleepTime, minInactiveSleepTime + random.Next(randomInactiveSleepTime));
                    Console.WriteLine("No controller could serve configState, sleeping {0}...", sleepTime);
                    Thread.Sleep(sleepTime * 1000);
                    continue;
                }

                var shardServers    = configState.shardServers;
                var quorum          = configState.quorums[0];
                var numShardServers = shardServers.Count;

                if (quorum.inactiveNodes.Count > numAllowedInactiveNodes)
                {
                    var sleepTime = random.Next(minInactiveSleepTime, minInactiveSleepTime + random.Next(randomInactiveSleepTime));
                    Console.WriteLine("Inactive found, sleeping {0}...", sleepTime);
                    Thread.Sleep(sleepTime * 1000);
                    continue;
                }

                var victimNodeID = quorum.activeNodes[random.Next(quorum.activeNodes.Count)];
                foreach (var shardServer in shardServers)
                {
                    if (shardServer.nodeID == victimNodeID)
                    {
                        var shardHttpURI = ConfigStateHelpers.GetShardServerURL(shardServer);
                        Console.WriteLine("Killing {0}", shardHttpURI);
                        Utils.HTTP.GET(Utils.HTTP.BuildUri(shardHttpURI, "debug?randomcrash&key=" + debugKey + "&interval=300"));
                        var sleepTime = random.Next(minCrashSleepTime, minCrashSleepTime + random.Next(randomCrashSleepTime));
                        Console.WriteLine("Sleeping {0}...", sleepTime);
                        Thread.Sleep(sleepTime * 1000);
                    }
                }
            }
        }
Ejemplo n.º 10
0
        public static void CompareNumericTableKeysBackwardsHTTP(List <ConfigState.ShardServer> shardServers, Int64 tableID, ulong num)
        {
            var   serverKeys = new string[shardServers.Count][];
            var   startKey   = Utils.Id(num);
            var   endKey     = "";
            ulong counter    = num;

            while (true)
            {
                serverKeys = ConfigStateHelpers.ParallelFetchTableKeysHTTP(shardServers, tableID, startKey, endKey, false);
                var referenceKeys = ConfigStateHelpers.GenerateNumericKeysBackwards(counter, (ulong)serverKeys[0].Length);

                for (var i = 0; i < serverKeys.Length; i++)
                {
                    if (!serverKeys[i].SequenceEqual(referenceKeys))
                    {
                        for (var j = 0; j < Math.Max(serverKeys[i].Length, referenceKeys.Length); j++)
                        {
                            var a = serverKeys[i][j];
                            var b = referenceKeys[j];
                            if (a == b)
                            {
                                continue;
                            }
                            //Assert.IsTrue(a == b);
                            System.Console.WriteLine("Inconsistency at tableID: " + tableID);
                            System.Console.WriteLine("NodeID: " + shardServers.ElementAt(i).nodeID + ", key: " + a);
                            Assert.IsTrue(a == b);
                            return;
                        }
                    }
                }

                if (serverKeys[0].Length <= 1)
                {
                    break;
                }

                startKey = serverKeys[0][serverKeys[0].Length - 1];
                counter -= (ulong)referenceKeys.Length - 1;
                System.Console.WriteLine("StartKey: " + startKey);
            }
        }
Ejemplo n.º 11
0
        public void TestChaosMonkey()
        {
            var    client        = Utils.GetClient();
            Random random        = new Random();
            bool   allowInactive = false;
            var    sleepInterval = ConfigFile.Config.GetIntValue("sleepInterval", 4);

            while (true)
            {
                var configState    = Utils.GetFullConfigState(client);
                var randomFunction = random.Next(0, 100);

                var shardServer = GetRandomShardServer(configState, random, allowInactive);
                if (shardServer == null)
                {
                    RandomSleep(random, sleepInterval);
                    continue;
                }

                var shardHttpURI = ConfigStateHelpers.GetShardServerURL(shardServer);

                // 90% sleep
                // 5% async infinite loop
                // 4% yield infinite loop
                // 1% crash
                if (randomFunction >= 0 && randomFunction < 90)
                {
                    SleepServer(shardHttpURI, sleepInterval);
                }
                else if (randomFunction >= 90 && randomFunction < 95)
                {
                    RunInfiniteLoopOnServer(shardHttpURI, crashInterval, true);
                }
                else if (randomFunction >= 95 && randomFunction < 99)
                {
                    RunInfiniteLoopOnServer(shardHttpURI, crashInterval, false);
                }
                else if (randomFunction >= 99)
                {
                    CrashServer(shardHttpURI, crashInterval);
                }
            }
        }
Ejemplo n.º 12
0
        public void CheckConfigStateShardConsistency()
        {
            Console.WriteLine("\nChecking config state shard consistency...\n");

            var client          = new Client(Utils.GetConfigNodes());
            var jsonConfigState = client.GetJSONConfigState();
            var configState     = Utils.JsonDeserialize <ConfigState>(System.Text.Encoding.UTF8.GetBytes(jsonConfigState));

            foreach (var table in configState.tables)
            {
                System.Console.WriteLine("\nChecking table {0}", table.tableID);

                var shards   = ConfigStateHelpers.GetTableShards(table, configState.shards);
                var shardMap = new Dictionary <string, ConfigState.Shard>();

                // build a map of shards by firstkey
                foreach (var shard in shards)
                {
                    ConfigState.Shard testShard;
                    Assert.IsFalse(shardMap.TryGetValue(shard.firstKey, out testShard));
                    shardMap.Add(shard.firstKey, shard);
                }

                // walk the map of shards by the last key of the shard
                var firstKey = "";
                int count    = 0;
                ConfigState.Shard tmpShard;
                while (shardMap.TryGetValue(firstKey, out tmpShard))
                {
                    count += 1;
                    if (tmpShard.lastKey == "")
                    {
                        break;
                    }
                    firstKey = tmpShard.lastKey;
                }

                Assert.IsTrue(count == shards.Count);
            }
        }
Ejemplo n.º 13
0
        public void TestRandomSleepPrimaryShardServer()
        {
            var client        = Utils.GetClient();
            var sleepInterval = 4;

            Random random = new Random();

            while (true)
            {
                var configState     = Utils.GetFullConfigState(client);
                var shardServers    = configState.shardServers;
                var quorum          = configState.quorums[0];
                var numShardServers = shardServers.Count;

                if (quorum.inactiveNodes.Count > 0)
                {
                    var sleepTime = random.Next(sleepInterval * 2, sleepInterval * 2 + random.Next(sleepInterval * 2));
                    Console.WriteLine("Inactive found, sleeping {0}...", sleepTime);
                    Thread.Sleep(sleepTime * 1000);
                    continue;
                }

                var victimNodeID = quorum.activeNodes.Where(nodeID => nodeID == quorum.primaryID).First();
                foreach (var shardServer in shardServers)
                {
                    if (shardServer.nodeID == victimNodeID)
                    {
                        var shardHttpURI = ConfigStateHelpers.GetShardServerURL(shardServer);
                        Console.WriteLine("Sleeping {0} for {1} secs", shardHttpURI, sleepInterval);
                        Utils.HTTP.GET(Utils.HTTP.BuildUri(shardHttpURI, "debug?sleep=" + sleepInterval + "&key=" + debugKey));
                        var sleepTime = random.Next(sleepInterval * 2, sleepInterval * 2 + random.Next(sleepInterval * 2));
                        Console.WriteLine("Sleeping {0}...", sleepTime);
                        Thread.Sleep(sleepTime * 1000);
                    }
                }
            }
        }
Ejemplo n.º 14
0
        public void TestRandomSleepShardServer()
        {
            var client        = Utils.GetClient();
            var sleepInterval = ConfigFile.Config.GetIntValue("sleepInterval", 4);

            Random random = new Random();

            while (true)
            {
                var configState = Utils.GetFullConfigState(client);
                var quorum      = configState == null ? null : configState.quorums[0];

                if (configState == null || quorum.inactiveNodes.Count > 0)
                {
                    var sleepTime = random.Next(sleepInterval * 2, sleepInterval * 2 + random.Next(sleepInterval * 2));
                    Console.WriteLine("Inactive found, sleeping {0}...", sleepTime);
                    Thread.Sleep(sleepTime * 1000);
                    continue;
                }

                var shardServers = configState.shardServers;
                var victimNodeID = quorum.activeNodes[random.Next(quorum.activeNodes.Count)];
                foreach (var shardServer in shardServers)
                {
                    if (shardServer.nodeID == victimNodeID)
                    {
                        bool isPrimary    = (quorum.hasPrimary && quorum.primaryID == victimNodeID);
                        var  shardHttpURI = ConfigStateHelpers.GetShardServerURL(shardServer);
                        Console.WriteLine("Sleeping {0} for {1} secs {2}", shardHttpURI, sleepInterval, isPrimary ? "(Primary)" : "");
                        Utils.HTTP.GET(Utils.HTTP.BuildUri(shardHttpURI, "debug?sleep=" + sleepInterval + "&key=" + debugKey));
                        var sleepTime = random.Next(sleepInterval * 2, sleepInterval * 2 + random.Next(sleepInterval * 2));
                        Console.WriteLine("Sleeping {0}...", sleepTime);
                        Thread.Sleep(sleepTime * 1000);
                    }
                }
            }
        }
Ejemplo n.º 15
0
        public static void ParallelTableCountHTTP(List <ConfigState.ShardServer> shardServers, Int64 tableID, string startKey, string endKey)
        {
            var threads = new CounterThreadState[shardServers.Count];

            var i = 0;

            foreach (var shardServer in shardServers)
            {
                var url = ConfigStateHelpers.GetShardServerURL(shardServer) + "count?tableID=" + tableID + "&startKey=" + startKey + "&endKey=" + endKey;
                System.Console.WriteLine("  Getting count on nodeID: " + shardServer.nodeID + ", tableID: " + tableID);

                threads[i]             = new CounterThreadState();
                threads[i].thread      = new Thread(new ThreadStart(threads[i].CounterThreadFunc));
                threads[i].shardServer = shardServer;
                threads[i].count       = -1;
                threads[i].url         = url;

                threads[i].thread.Start();

                i += 1;
            }

            Int64 prevCount = -1;

            foreach (var thread in threads)
            {
                thread.thread.Join();
                if (prevCount == -1)
                {
                    prevCount = thread.count;
                }
                System.Console.WriteLine("  Result on " + thread.shardServer.nodeID + ": " + thread.count + ", elapsed: " + thread.elapsed.Seconds + "s, cps: " + (Int64)(thread.count / thread.elapsed.TotalMilliseconds) * 1000);
                Assert.IsTrue(thread.count != -1 && thread.count == prevCount);
                prevCount = thread.count;
            }
        }