private void RetryPolicyTest(ITestCluster testCluster) { PolicyTestTools policyTestTools = new PolicyTestTools(); policyTestTools.TableName = TestUtils.GetUniqueTableName(); policyTestTools.CreateSchema(testCluster.Session, 2); // Test before state policyTestTools.InitPreparedStatement(testCluster, 12); policyTestTools.Query(testCluster, 12); int clusterPosQueried = 1; int clusterPosNotQueried = 2; if (!policyTestTools.Coordinators.ContainsKey(testCluster.ClusterIpPrefix + clusterPosQueried)) { clusterPosQueried = 2; clusterPosNotQueried = 1; } policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + clusterPosQueried + ":" + DefaultCassandraPort, 1); // Stop one of the nodes and test policyTestTools.ResetCoordinators(); testCluster.Stop(clusterPosQueried); TestUtils.WaitForDown(testCluster.ClusterIpPrefix + clusterPosQueried, testCluster.Cluster, 30); policyTestTools.Query(testCluster, 120); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + clusterPosNotQueried + ":" + DefaultCassandraPort, 120); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + clusterPosQueried + ":" + DefaultCassandraPort, 0); // Start the node that was just down, then down the node that was just up policyTestTools.ResetCoordinators(); testCluster.Start(clusterPosQueried); TestUtils.WaitForUp(testCluster.ClusterIpPrefix + clusterPosQueried, DefaultCassandraPort, 30); // Test successful reads DateTime futureDateTime = DateTime.Now.AddSeconds(120); while (policyTestTools.Coordinators.Count < 2 && DateTime.Now < futureDateTime) { policyTestTools.Query(testCluster, 120); Thread.Sleep(75); } // Ensure that the nodes were contacted policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + clusterPosQueried + ":" + DefaultCassandraPort, 1); policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + clusterPosNotQueried + ":" + DefaultCassandraPort, 1); }
public void RoundRobin_OneDc_OneNodeAdded_OneNodeDecommissioned() { // Setup PolicyTestTools policyTestTools = new PolicyTestTools(); ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1); testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new RoundRobinPolicy()); testCluster.InitClient(); policyTestTools.CreateSchema(testCluster.Session); policyTestTools.InitPreparedStatement(testCluster, 12); policyTestTools.Query(testCluster, 12); // Validate that all host were queried equally policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 12); // Add new node to the end of second cluster, remove node from beginning of first cluster policyTestTools.ResetCoordinators(); // Bootstrap step int bootStrapPos = 2; testCluster.BootstrapNode(bootStrapPos); string newlyBootstrappedIp = testCluster.ClusterIpPrefix + bootStrapPos; TestUtils.WaitForUp(newlyBootstrappedIp, DefaultCassandraPort, 30); // Validate expected nodes where queried policyTestTools.WaitForPolicyToolsQueryToHitBootstrappedIp(testCluster, newlyBootstrappedIp); policyTestTools.Query(testCluster, 12); policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 6); policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6); // decommission old node policyTestTools.ResetCoordinators(); testCluster.DecommissionNode(1); TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, testCluster.Cluster, 60); policyTestTools.Query(testCluster, 12); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 12); }
public void RoundRobin_TwoDCs_EachDcHasOneNodeAddedAndDecommissioned() { // Setup PolicyTestTools policyTestTools = new PolicyTestTools(); ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1, 1, DefaultMaxClusterCreateRetries, true); testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new RoundRobinPolicy()); testCluster.InitClient(); policyTestTools.CreateSchema(testCluster.Session); policyTestTools.InitPreparedStatement(testCluster, 12); policyTestTools.Query(testCluster, 12); // Validate that all host were queried equally policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 6); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6); // Add new node to the end of first cluster, remove node from beginning of first cluster policyTestTools.ResetCoordinators(); // Bootstrap step testCluster.BootstrapNode(3, "dc1"); string newlyBootstrappedIp = testCluster.ClusterIpPrefix + "3"; TestUtils.WaitForUp(newlyBootstrappedIp, DefaultCassandraPort, 30); // Validate expected nodes where queried policyTestTools.WaitForPolicyToolsQueryToHitBootstrappedIp(testCluster, newlyBootstrappedIp); policyTestTools.Query(testCluster, 12); policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 4); policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 4); policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 4); // Remove node from beginning of first cluster policyTestTools.ResetCoordinators(); testCluster.DecommissionNode(1); TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, testCluster.Cluster, 20); // Validate expected nodes where queried policyTestTools.Query(testCluster, 12); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 6); // Add new node to the end of second cluster, remove node from beginning of first cluster policyTestTools.ResetCoordinators(); testCluster.BootstrapNode(4, "dc2"); newlyBootstrappedIp = testCluster.ClusterIpPrefix + "4"; TestUtils.WaitForUp(newlyBootstrappedIp, DefaultCassandraPort, 30); policyTestTools.ResetCoordinators(); policyTestTools.Query(testCluster, 12); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 4); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 4); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, 4); // Remove node from beginning of second cluster policyTestTools.ResetCoordinators(); testCluster.DecommissionNode(2); TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 20); policyTestTools.Query(testCluster, 12); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 0); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 6); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, 6); }