public static void CheckDatabaseWithNumericKeys(string databaseName, string tableName) { var client = new Client(Utils.GetConfigNodes()); var jsonConfigState = client.GetJSONConfigState(); var configState = Utils.JsonDeserialize <ConfigState>(System.Text.Encoding.UTF8.GetBytes(jsonConfigState)); var shardServers = new List <ConfigState.ShardServer>(); Int64 tableID = 0; foreach (var database in configState.databases) { if (database.name == databaseName) { foreach (var table in configState.tables) { if (table.databaseID == database.databaseID && table.name == tableName) { shardServers = ConfigStateHelpers.GetShardServersByTable(table, configState); tableID = table.tableID; break; } } } if (tableID != 0) { break; } } Assert.IsTrue(tableID != 0); ulong num = 50 * 1000 * 1000; CompareNumericTableKeysHTTP(shardServers, tableID, num); CompareNumericTableKeysBackwardsHTTP(shardServers, tableID, num); }
public void CheckClusterConsistencyByCount() { var client = new Client(Utils.GetConfigNodes()); var jsonConfigState = client.GetJSONConfigState(); var configState = Utils.JsonDeserialize <ConfigState>(System.Text.Encoding.UTF8.GetBytes(jsonConfigState)); foreach (var table in configState.tables) { var shards = ConfigStateHelpers.GetTableShards(table, configState.shards); foreach (var shard in shards) { System.Console.WriteLine("\nCounting shard " + shard.shardID); var tableID = shard.tableID; var startKey = shard.firstKey; var endKey = shard.lastKey; var quorum = ConfigStateHelpers.GetQuorum(configState, shard.quorumID); var shardServers = ConfigStateHelpers.GetQuorumActiveShardServers(configState, quorum); ParallelTableCountHTTP(shardServers, tableID, startKey, endKey); System.Console.WriteLine("Shard " + shard.shardID + " is consistent.\n"); } } }
public static void FixDiffs(Client client, List <ConfigState.ShardServer> shardServers, Int64 tableID, List <string> diffs) { var i = 0; foreach (var key in diffs) { i += 1; byte[] startKey = Utils.StringToByteArray(key); byte[] endKey = Utils.NextKey(startKey); var serverKeyValues = ConfigStateHelpers.ParallelFetchTableKeyValuesHTTP(shardServers, tableID, startKey, endKey, true); if (Array.TrueForAll(serverKeyValues, val => (val.Count == 1 && Utils.ByteArraysEqual(val.First().Key, serverKeyValues[0].First().Key)))) { continue; } foreach (var keyValue in serverKeyValues) { if (keyValue == null || keyValue.Count == 0) { continue; } if (keyValue.First().Value.Length > 0) { Assert.IsTrue(Utils.ByteArraysEqual(Utils.StringToByteArray(key), keyValue.First().Key)); Console.WriteLine("Setting key {0}", key); Table table = new Table(client, null, (ulong)tableID, ""); table.Set(startKey, keyValue.First().Value); client.Submit(); } } } }
public static void FillDatabaseWithNumericKeys(string databaseName, string tableName) { var client = new Client(Utils.GetConfigNodes()); Assert.IsTrue(ConfigStateHelpers.TryDeleteDatabase(client, databaseName)); var database = client.CreateDatabase(databaseName); Assert.IsNotNull(database, "Cannot create database " + databaseName); var table = ConfigStateHelpers.TryCreateTable(database, tableName); Assert.IsNotNull(table, "Cannot create table " + tableName); System.Console.WriteLine("Filling the database..."); DateTime last = DateTime.Now; for (ulong i = 0; i < 100 * 1000 * 1000; i++) { var key = Utils.Id(i); table.Set(key, key); TimeSpan timeSpan = DateTime.Now - last; if (timeSpan.TotalSeconds >= 60) { System.Console.WriteLine("i: " + i); last = DateTime.Now; } } client.Submit(); System.Console.WriteLine("Database filling is done."); }
public static void CompareTableKeyValuesHTTP(List <ConfigState.ShardServer> shardServers, Int64 tableID, byte[] startKey, byte[] endKey) { if (shardServers.Count <= 1) { return; } var i = 0; while (true) { var serverKeys = ConfigStateHelpers.ParallelFetchTableKeyValuesHTTP(shardServers, tableID, startKey, endKey, true); int numEmpty = serverKeys.Count(keys => keys.Count == 0); if (numEmpty == serverKeys.Length) { break; } System.Console.WriteLine("StartKey: " + Utils.ByteArrayToString(startKey)); for (i = 1; i < serverKeys.Length; i++) { if (serverKeys[i].Count != serverKeys[0].Count) { System.Console.WriteLine("Inconsistency at tableID: " + tableID); System.Console.WriteLine("NodeID: " + shardServers.ElementAt(i).nodeID + ", count: " + serverKeys[i].Count); System.Console.WriteLine("NodeID: " + shardServers.ElementAt(0).nodeID + ", count: " + serverKeys[i].Count); Assert.Fail("Inconsistency at tableID: " + tableID); return; } for (var j = 0; j < serverKeys[0].Count; j++) { var a = serverKeys[i][j]; var b = serverKeys[0][j]; if (Utils.ByteArraysEqual(a.Key, b.Key) && Utils.ByteArraysEqual(a.Value, b.Value)) { continue; } System.Console.WriteLine("Inconsistency at tableID: " + tableID); System.Console.WriteLine("NodeID: " + shardServers.ElementAt(i).nodeID + ", key: " + Utils.ByteArrayToString(a.Key)); System.Console.WriteLine("NodeID: " + shardServers.ElementAt(0).nodeID + ", key: " + Utils.ByteArrayToString(b.Key)); Assert.Fail("Inconsistency at tableID: " + tableID); //return; goto Out; } } Out: if (serverKeys[0].Count == 0) { break; } startKey = Utils.NextKey(serverKeys[0].Last().Key); } }
public void TestRandomCrashServer() { var client = Utils.GetClient(); Random random = new Random(); while (true) { var configState = Utils.GetFullConfigState(client); var shardServers = configState.shardServers; var quorum = configState.quorums[0]; var numShardServers = shardServers.Count; if (quorum.inactiveNodes.Count > numAllowedInactiveNodes) { var inactiveSleepTime = random.Next(minInactiveSleepTime, minInactiveSleepTime + random.Next(randomInactiveSleepTime)); Utils.Log("Inactive found, sleeping {0} until {1}...", inactiveSleepTime, DateTime.Now.AddSeconds(inactiveSleepTime).ToString("u")); Thread.Sleep(inactiveSleepTime * 1000); continue; } var httpURI = ""; if (random.Next(2) == 0) { // select controller var victimController = configState.controllers[random.Next(configState.controllers.Count)]; httpURI = ConfigStateHelpers.GetControllerURL(victimController); Utils.Log("Killing controller {0}", victimController.nodeID); } else { // select shard server var victimNodeID = quorum.activeNodes[random.Next(quorum.activeNodes.Count)]; foreach (var shardServer in shardServers) { if (shardServer.nodeID == victimNodeID) { httpURI = ConfigStateHelpers.GetShardServerURL(shardServer); Utils.Log("Killing shard server {0}", shardServer.nodeID); break; } } } if (httpURI != "") { var httpCrashURI = Utils.HTTP.BuildUri(httpURI, "debug?randomcrash&key=" + debugKey + "&interval=" + crashInterval * 1000); Utils.Log(httpCrashURI); Utils.HTTP.GET(httpCrashURI); } var sleepTime = random.Next(minCrashSleepTime, minCrashSleepTime + random.Next(randomCrashSleepTime)); Utils.Log("Sleeping {0} until {1}...", sleepTime, DateTime.Now.AddSeconds(sleepTime).ToString("u")); Thread.Sleep(sleepTime * 1000); } }
public static void FixConsistency(Client client, List <ConfigState.ShardServer> shardServers, Int64 tableID, string startKey, string endKey) { if (shardServers.Count <= 1) { return; } var keyDiffs = new HashSet <string>(); var serverKeys = new string[shardServers.Count][]; var i = 0; while (true) { System.Console.WriteLine("StartKey: " + startKey); serverKeys = ConfigStateHelpers.ParallelFetchTableKeysHTTP(shardServers, tableID, startKey, endKey, true); for (i = 1; i < serverKeys.Length; i++) { if (serverKeys[0].Length > 0 && serverKeys[i].Length > 1 && serverKeys[0].First().CompareTo(serverKeys[i].Last()) < 0) { foreach (var diff in serverKeys[i].Except(serverKeys[0])) { keyDiffs.Add(diff); } } if (serverKeys[0].Length > 1 && serverKeys[i].Length > 0 && serverKeys[i].First().CompareTo(serverKeys[0].Last()) < 0) { foreach (var diff in serverKeys[0].Except(serverKeys[i])) { keyDiffs.Add(diff); } } } if (keyDiffs.Count != 0) { FixDiffs(client, shardServers, tableID, keyDiffs.ToList()); startKey = keyDiffs.Last(); keyDiffs = new HashSet <string>(); continue; } if (serverKeys[0].Length <= 1) { break; } startKey = serverKeys[0][serverKeys[0].Length - 1]; } }
public static void CompareTableKeysHTTP(List <ConfigState.ShardServer> shardServers, Int64 tableID, string startKey, string endKey) { if (shardServers.Count <= 1) { return; } var serverKeys = new string[shardServers.Count][]; var i = 0; while (true) { serverKeys = ConfigStateHelpers.ParallelFetchTableKeysHTTP(shardServers, tableID, startKey, endKey, true); for (i = 1; i < serverKeys.Length; i++) { if (!serverKeys[i].SequenceEqual(serverKeys[0])) { for (var j = 0; j < Math.Max(serverKeys[i].Length, serverKeys[0].Length); j++) { var a = serverKeys[i][j]; var b = serverKeys[0][j]; if (a == b) { continue; } System.Console.WriteLine("Inconsistency at tableID: " + tableID); System.Console.WriteLine("NodeID: " + shardServers.ElementAt(i).nodeID + ", key: " + a); System.Console.WriteLine("NodeID: " + shardServers.ElementAt(0).nodeID + ", key: " + b); //Assert.IsTrue(a == b); //return; goto Out; } } } Out: if (serverKeys[0].Length <= 1) { break; } startKey = serverKeys[0][serverKeys[0].Length - 1]; System.Console.WriteLine("StartKey: " + startKey); } }
public void TestRandomCrashShardServer() { var client = new Client(Utils.GetConfigNodes()); Random random = new Random(); while (true) { var configState = Utils.GetFullConfigState(client); if (configState == null) { var sleepTime = random.Next(minInactiveSleepTime, minInactiveSleepTime + random.Next(randomInactiveSleepTime)); Console.WriteLine("No controller could serve configState, sleeping {0}...", sleepTime); Thread.Sleep(sleepTime * 1000); continue; } var shardServers = configState.shardServers; var quorum = configState.quorums[0]; var numShardServers = shardServers.Count; if (quorum.inactiveNodes.Count > numAllowedInactiveNodes) { var sleepTime = random.Next(minInactiveSleepTime, minInactiveSleepTime + random.Next(randomInactiveSleepTime)); Console.WriteLine("Inactive found, sleeping {0}...", sleepTime); Thread.Sleep(sleepTime * 1000); continue; } var victimNodeID = quorum.activeNodes[random.Next(quorum.activeNodes.Count)]; foreach (var shardServer in shardServers) { if (shardServer.nodeID == victimNodeID) { var shardHttpURI = ConfigStateHelpers.GetShardServerURL(shardServer); Console.WriteLine("Killing {0}", shardHttpURI); Utils.HTTP.GET(Utils.HTTP.BuildUri(shardHttpURI, "debug?randomcrash&key=" + debugKey + "&interval=300")); var sleepTime = random.Next(minCrashSleepTime, minCrashSleepTime + random.Next(randomCrashSleepTime)); Console.WriteLine("Sleeping {0}...", sleepTime); Thread.Sleep(sleepTime * 1000); } } } }
public static void CompareNumericTableKeysBackwardsHTTP(List <ConfigState.ShardServer> shardServers, Int64 tableID, ulong num) { var serverKeys = new string[shardServers.Count][]; var startKey = Utils.Id(num); var endKey = ""; ulong counter = num; while (true) { serverKeys = ConfigStateHelpers.ParallelFetchTableKeysHTTP(shardServers, tableID, startKey, endKey, false); var referenceKeys = ConfigStateHelpers.GenerateNumericKeysBackwards(counter, (ulong)serverKeys[0].Length); for (var i = 0; i < serverKeys.Length; i++) { if (!serverKeys[i].SequenceEqual(referenceKeys)) { for (var j = 0; j < Math.Max(serverKeys[i].Length, referenceKeys.Length); j++) { var a = serverKeys[i][j]; var b = referenceKeys[j]; if (a == b) { continue; } //Assert.IsTrue(a == b); System.Console.WriteLine("Inconsistency at tableID: " + tableID); System.Console.WriteLine("NodeID: " + shardServers.ElementAt(i).nodeID + ", key: " + a); Assert.IsTrue(a == b); return; } } } if (serverKeys[0].Length <= 1) { break; } startKey = serverKeys[0][serverKeys[0].Length - 1]; counter -= (ulong)referenceKeys.Length - 1; System.Console.WriteLine("StartKey: " + startKey); } }
public void TestChaosMonkey() { var client = Utils.GetClient(); Random random = new Random(); bool allowInactive = false; var sleepInterval = ConfigFile.Config.GetIntValue("sleepInterval", 4); while (true) { var configState = Utils.GetFullConfigState(client); var randomFunction = random.Next(0, 100); var shardServer = GetRandomShardServer(configState, random, allowInactive); if (shardServer == null) { RandomSleep(random, sleepInterval); continue; } var shardHttpURI = ConfigStateHelpers.GetShardServerURL(shardServer); // 90% sleep // 5% async infinite loop // 4% yield infinite loop // 1% crash if (randomFunction >= 0 && randomFunction < 90) { SleepServer(shardHttpURI, sleepInterval); } else if (randomFunction >= 90 && randomFunction < 95) { RunInfiniteLoopOnServer(shardHttpURI, crashInterval, true); } else if (randomFunction >= 95 && randomFunction < 99) { RunInfiniteLoopOnServer(shardHttpURI, crashInterval, false); } else if (randomFunction >= 99) { CrashServer(shardHttpURI, crashInterval); } } }
public void CheckConfigStateShardConsistency() { Console.WriteLine("\nChecking config state shard consistency...\n"); var client = new Client(Utils.GetConfigNodes()); var jsonConfigState = client.GetJSONConfigState(); var configState = Utils.JsonDeserialize <ConfigState>(System.Text.Encoding.UTF8.GetBytes(jsonConfigState)); foreach (var table in configState.tables) { System.Console.WriteLine("\nChecking table {0}", table.tableID); var shards = ConfigStateHelpers.GetTableShards(table, configState.shards); var shardMap = new Dictionary <string, ConfigState.Shard>(); // build a map of shards by firstkey foreach (var shard in shards) { ConfigState.Shard testShard; Assert.IsFalse(shardMap.TryGetValue(shard.firstKey, out testShard)); shardMap.Add(shard.firstKey, shard); } // walk the map of shards by the last key of the shard var firstKey = ""; int count = 0; ConfigState.Shard tmpShard; while (shardMap.TryGetValue(firstKey, out tmpShard)) { count += 1; if (tmpShard.lastKey == "") { break; } firstKey = tmpShard.lastKey; } Assert.IsTrue(count == shards.Count); } }
public void TestRandomSleepPrimaryShardServer() { var client = Utils.GetClient(); var sleepInterval = 4; Random random = new Random(); while (true) { var configState = Utils.GetFullConfigState(client); var shardServers = configState.shardServers; var quorum = configState.quorums[0]; var numShardServers = shardServers.Count; if (quorum.inactiveNodes.Count > 0) { var sleepTime = random.Next(sleepInterval * 2, sleepInterval * 2 + random.Next(sleepInterval * 2)); Console.WriteLine("Inactive found, sleeping {0}...", sleepTime); Thread.Sleep(sleepTime * 1000); continue; } var victimNodeID = quorum.activeNodes.Where(nodeID => nodeID == quorum.primaryID).First(); foreach (var shardServer in shardServers) { if (shardServer.nodeID == victimNodeID) { var shardHttpURI = ConfigStateHelpers.GetShardServerURL(shardServer); Console.WriteLine("Sleeping {0} for {1} secs", shardHttpURI, sleepInterval); Utils.HTTP.GET(Utils.HTTP.BuildUri(shardHttpURI, "debug?sleep=" + sleepInterval + "&key=" + debugKey)); var sleepTime = random.Next(sleepInterval * 2, sleepInterval * 2 + random.Next(sleepInterval * 2)); Console.WriteLine("Sleeping {0}...", sleepTime); Thread.Sleep(sleepTime * 1000); } } } }
public void TestRandomSleepShardServer() { var client = Utils.GetClient(); var sleepInterval = ConfigFile.Config.GetIntValue("sleepInterval", 4); Random random = new Random(); while (true) { var configState = Utils.GetFullConfigState(client); var quorum = configState == null ? null : configState.quorums[0]; if (configState == null || quorum.inactiveNodes.Count > 0) { var sleepTime = random.Next(sleepInterval * 2, sleepInterval * 2 + random.Next(sleepInterval * 2)); Console.WriteLine("Inactive found, sleeping {0}...", sleepTime); Thread.Sleep(sleepTime * 1000); continue; } var shardServers = configState.shardServers; var victimNodeID = quorum.activeNodes[random.Next(quorum.activeNodes.Count)]; foreach (var shardServer in shardServers) { if (shardServer.nodeID == victimNodeID) { bool isPrimary = (quorum.hasPrimary && quorum.primaryID == victimNodeID); var shardHttpURI = ConfigStateHelpers.GetShardServerURL(shardServer); Console.WriteLine("Sleeping {0} for {1} secs {2}", shardHttpURI, sleepInterval, isPrimary ? "(Primary)" : ""); Utils.HTTP.GET(Utils.HTTP.BuildUri(shardHttpURI, "debug?sleep=" + sleepInterval + "&key=" + debugKey)); var sleepTime = random.Next(sleepInterval * 2, sleepInterval * 2 + random.Next(sleepInterval * 2)); Console.WriteLine("Sleeping {0}...", sleepTime); Thread.Sleep(sleepTime * 1000); } } } }
public static void ParallelTableCountHTTP(List <ConfigState.ShardServer> shardServers, Int64 tableID, string startKey, string endKey) { var threads = new CounterThreadState[shardServers.Count]; var i = 0; foreach (var shardServer in shardServers) { var url = ConfigStateHelpers.GetShardServerURL(shardServer) + "count?tableID=" + tableID + "&startKey=" + startKey + "&endKey=" + endKey; System.Console.WriteLine(" Getting count on nodeID: " + shardServer.nodeID + ", tableID: " + tableID); threads[i] = new CounterThreadState(); threads[i].thread = new Thread(new ThreadStart(threads[i].CounterThreadFunc)); threads[i].shardServer = shardServer; threads[i].count = -1; threads[i].url = url; threads[i].thread.Start(); i += 1; } Int64 prevCount = -1; foreach (var thread in threads) { thread.thread.Join(); if (prevCount == -1) { prevCount = thread.count; } System.Console.WriteLine(" Result on " + thread.shardServer.nodeID + ": " + thread.count + ", elapsed: " + thread.elapsed.Seconds + "s, cps: " + (Int64)(thread.count / thread.elapsed.TotalMilliseconds) * 1000); Assert.IsTrue(thread.count != -1 && thread.count == prevCount); prevCount = thread.count; } }