/// <summary>
/// Tests DowngradingConsistencyRetryPolicy
///
/// @test_category connection:retry_policy
/// </summary>
public void DowngradingConsistencyRetryPolicyTest(Builder builder)
{
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(3);
testCluster.Builder = builder;
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session, 3);
// FIXME: Race condition where the nodes are not fully up yet and assertQueried reports slightly different numbers
TestUtils.WaitForSchemaAgreement(testCluster.Cluster);
policyTestTools.InitPreparedStatement(testCluster, 12, ConsistencyLevel.All);
policyTestTools.Query(testCluster, 12, ConsistencyLevel.All);
policyTestTools.AssertAchievedConsistencyLevel(ConsistencyLevel.All);
//Kill one node: 2 nodes alive
testCluster.Stop(2);
TestUtils.WaitForDown(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 20);
//After killing one node, the achieved consistency level should be downgraded
policyTestTools.ResetCoordinators();
policyTestTools.Query(testCluster, 12, ConsistencyLevel.All);
policyTestTools.AssertAchievedConsistencyLevel(ConsistencyLevel.Two);
}
public void TryNextHostRetryPolicyTest()
{
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(2);
var socketOptions = new SocketOptions().SetReadTimeoutMillis(2000);
testCluster.Builder = Cluster.Builder()
.WithRetryPolicy(new LoggingRetryPolicy(TryNextHostRetryPolicy.Instance))
.AddContactPoint(testCluster.ClusterIpPrefix + "1")
.AddContactPoint(testCluster.ClusterIpPrefix + "2")
.WithSocketOptions(socketOptions);
testCluster.InitClient();
// Setup cluster
PolicyTestTools policyTestTools = new PolicyTestTools();
policyTestTools.CreateSchema(testCluster.Session, 2);
policyTestTools.InitPreparedStatement(testCluster, 12);
// Try with both hosts
policyTestTools.Query(testCluster, 10);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + 1 + ":" + DefaultCassandraPort, 5);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + 2 + ":" + DefaultCassandraPort, 5);
// Try with host 1
policyTestTools.ResetCoordinators();
testCluster.PauseNode(2);
policyTestTools.Query(testCluster, 10);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 1 + ":" + DefaultCassandraPort, 10);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 2 + ":" + DefaultCassandraPort, 0);
testCluster.ResumeNode(2);
// Try with host 2
policyTestTools.ResetCoordinators();
testCluster.PauseNode(1);
policyTestTools.Query(testCluster, 10);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 1 + ":" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 2 + ":" + DefaultCassandraPort, 10);
// Try with 0 hosts
policyTestTools.ResetCoordinators();
testCluster.PauseNode(2);
Assert.Throws <NoHostAvailableException>(() => policyTestTools.Query(testCluster, 10));
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 1 + ":" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 2 + ":" + DefaultCassandraPort, 0);
testCluster.ResumeNode(1);
testCluster.ResumeNode(2);
}
public void RoundRobin_TwoDCs_DcAware()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1, 1, DefaultMaxClusterCreateRetries, true);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new DCAwareRoundRobinPolicy("dc2"));
testCluster.InitClient();
policyTestTools.CreateMultiDcSchema(testCluster.Session);
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 12);
}
private void RetryPolicyTest(ITestCluster testCluster)
{
PolicyTestTools policyTestTools = new PolicyTestTools();
policyTestTools.TableName = TestUtils.GetUniqueTableName();
policyTestTools.CreateSchema(testCluster.Session, 2);
// Test before state
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
int clusterPosQueried = 1;
int clusterPosNotQueried = 2;
if (!policyTestTools.Coordinators.ContainsKey(testCluster.ClusterIpPrefix + clusterPosQueried))
{
clusterPosQueried = 2;
clusterPosNotQueried = 1;
}
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + clusterPosQueried + ":" + DefaultCassandraPort, 1);
// Stop one of the nodes and test
policyTestTools.ResetCoordinators();
testCluster.Stop(clusterPosQueried);
TestUtils.WaitForDown(testCluster.ClusterIpPrefix + clusterPosQueried, testCluster.Cluster, 30);
policyTestTools.Query(testCluster, 120);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + clusterPosNotQueried + ":" + DefaultCassandraPort, 120);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + clusterPosQueried + ":" + DefaultCassandraPort, 0);
// Start the node that was just down, then down the node that was just up
policyTestTools.ResetCoordinators();
testCluster.Start(clusterPosQueried);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + clusterPosQueried, DefaultCassandraPort, 30);
// Test successful reads
DateTime futureDateTime = DateTime.Now.AddSeconds(120);
while (policyTestTools.Coordinators.Count < 2 && DateTime.Now < futureDateTime)
{
policyTestTools.Query(testCluster, 120);
Thread.Sleep(75);
}
// Ensure that the nodes were contacted
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + clusterPosQueried + ":" + DefaultCassandraPort, 1);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + clusterPosNotQueried + ":" + DefaultCassandraPort, 1);
}
public void RoundRobin_TokenAware_TwoDCsWithOneNodeEach_ReplicationFactorTwo()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1, 1, DefaultMaxClusterCreateRetries, true);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()));
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session, 2);
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6);
}
public void RoundRobin_OneDc_OneNodeAdded_OneNodeDecommissioned()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new RoundRobinPolicy());
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session);
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
// Validate that all host were queried equally
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 12);
// Add new node to the end of second cluster, remove node from beginning of first cluster
policyTestTools.ResetCoordinators();
// Bootstrap step
int bootStrapPos = 2;
testCluster.BootstrapNode(bootStrapPos);
string newlyBootstrappedIp = testCluster.ClusterIpPrefix + bootStrapPos;
TestUtils.WaitForUp(newlyBootstrappedIp, DefaultCassandraPort, 30);
// Validate expected nodes where queried
policyTestTools.WaitForPolicyToolsQueryToHitBootstrappedIp(testCluster, newlyBootstrappedIp);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6);
// decommission old node
policyTestTools.ResetCoordinators();
testCluster.DecommissionNode(1);
TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, testCluster.Cluster, 60);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 12);
}
public void RoundRobin_OneDc_AllNodesForceStoppedOneAtATime()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(2);
testCluster.Builder = Cluster.Builder()
.WithLoadBalancingPolicy(new RoundRobinPolicy())
.WithQueryTimeout(10000);
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session);
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6);
policyTestTools.ResetCoordinators();
testCluster.StopForce(1);
TestUtils.WaitForDown(testCluster.ClusterIpPrefix + "1", testCluster.Cluster, 20);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 12);
testCluster.StopForce(2);
TestUtils.WaitForDown(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 20);
try
{
policyTestTools.Query(testCluster, 3);
Assert.Fail("Exception should have been thrown, but wasn't!");
}
catch (NoHostAvailableException)
{
Trace.TraceInformation("Expected NoHostAvailableException exception was thrown.");
}
}
public void TokenMap_Rebuild_With_Decommissioned_DC_Existing_RF()
{
// Create a 2dc:1node each cluster
var clusterOptions = new TestClusterOptions();
clusterOptions.Dc2NodeLength = 1;
var testCluster = TestClusterManager.CreateNew(1, clusterOptions);
testCluster.Builder = Cluster.Builder().AddContactPoint("127.0.0.1").WithLoadBalancingPolicy(new DCAwareRoundRobinPolicy("dc1"));
testCluster.Cluster = testCluster.Builder.Build();
testCluster.Session = testCluster.Cluster.Connect();
PolicyTestTools policyTestTools = new PolicyTestTools();
// Create a ks with RF = dc1:1, dc2:1
policyTestTools.CreateMultiDcSchema(testCluster.Session, 1, 1);
policyTestTools.InitPreparedStatement(testCluster, 12, false, ConsistencyLevel.All);
// Replicas now in each node in each dc
policyTestTools.Query(testCluster, 12, false);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 0);
testCluster.Cluster.Shutdown();
testCluster.DecommissionNode(1);
// dc1 no longer has any hosts
testCluster.Builder = Cluster.Builder().AddContactPoint("127.0.0.2").WithLoadBalancingPolicy(new DCAwareRoundRobinPolicy("dc2"));
testCluster.Cluster = testCluster.Builder.Build();
// Should be able to connect and rebuild token map
testCluster.Session = testCluster.Cluster.Connect(policyTestTools.DefaultKeyspace);
policyTestTools.ResetCoordinators();
policyTestTools.Query(testCluster, 12, false);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 12);
testCluster.Cluster.Shutdown();
}
public void TokenAwareTest(bool usePrepared)
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1, 1, DefaultMaxClusterCreateRetries, true);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()));
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session);
//clusterInfo.Cluster.RefreshSchema();
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
// Not the best test ever, we should use OPP and check we do it the
// right nodes. But since M3P is hard-coded for now, let just check
// we just hit only one node.
int nodePosToDecommission = 2;
int nodePositionToNotDecommission = 1;
if (policyTestTools.Coordinators.ContainsKey(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort))
{
nodePosToDecommission = 1;
nodePositionToNotDecommission = 2;
}
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePosToDecommission + ":" + DefaultCassandraPort, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePositionToNotDecommission + ":" + DefaultCassandraPort, 0);
// now try again having stopped the node that was just queried
policyTestTools.ResetCoordinators();
testCluster.DecommissionNode(nodePosToDecommission);
TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + nodePosToDecommission + ":" + DefaultCassandraPort, testCluster.Cluster, 40);
policyTestTools.Query(testCluster, 12, usePrepared);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePosToDecommission + ":" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePositionToNotDecommission + ":" + DefaultCassandraPort, 12);
}
public void ReconnectionPolicyTest(Builder builder, long restartTime, long retryTime, long breakTime)
{
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1);
_policyTestTools.CreateSchema(testCluster.Session, 1);
_policyTestTools.InitPreparedStatement(testCluster, 12);
_policyTestTools.Query(testCluster, 12);
// Ensure a basic test works
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 12);
_policyTestTools.ResetCoordinators();
testCluster.StopForce(1);
// Start timing and ensure that the node is down
//long startTime = 0;
var startTime = Stopwatch.StartNew(); // = 0;
try
{
//startTime = System.nanoTime() / 1000000000;
_policyTestTools.Query(testCluster, 12);
Assert.Fail("Test race condition where node has not shut off quickly enough.");
}
catch (NoHostAvailableException) {}
long elapsedSeconds;
bool restarted = false;
while (true)
{
//thisTime = System.nanoTime() / 1000000000;
elapsedSeconds = startTime.ElapsedMilliseconds / 1000;
// Restart node at restartTime
if (!restarted && elapsedSeconds > restartTime)
{
testCluster.Start(1);
restarted = true;
}
// Continue testing queries each second
try
{
_policyTestTools.Query(testCluster, 12);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 12);
_policyTestTools.ResetCoordinators();
// Ensure the time when the query completes successfully is what was expected
Assert.True(retryTime - 6 < elapsedSeconds && elapsedSeconds < retryTime + 6, string.Format("Waited {0} seconds instead an expected {1} seconds wait", elapsedSeconds, retryTime));
}
catch (NoHostAvailableException)
{
Thread.Sleep(1000);
continue;
}
Thread.Sleep((int)(breakTime * 1000));
// The same query once more, just to be sure
_policyTestTools.Query(testCluster, 12);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 12);
_policyTestTools.ResetCoordinators();
// Ensure the reconnection times reset
testCluster.StopForce(1);
// Start timing and ensure that the node is down
//startTime = 0;
startTime.Reset();
try
{
//startTime = System.nanoTime() / 1000000000;
startTime.Start();
_policyTestTools.Query(testCluster, 12);
Assert.Fail("Test race condition where node has not shut off quickly enough.");
}
catch (NoHostAvailableException)
{
}
restarted = false;
while (true)
{
//elapsedSeconds = System.nanoTime() / 1000000000;
elapsedSeconds = startTime.ElapsedMilliseconds / 1000;
// Restart node at restartTime
if (!restarted && elapsedSeconds > restartTime)
{
testCluster.Start(1);
restarted = true;
}
// Continue testing queries each second
try
{
_policyTestTools.Query(testCluster, 12);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 12);
_policyTestTools.ResetCoordinators();
// Ensure the time when the query completes successfully is what was expected
Assert.True(retryTime - 3 < elapsedSeconds && elapsedSeconds < retryTime + 3, string.Format("Waited {0} seconds instead an expected {1} seconds wait", elapsedSeconds, retryTime));
}
catch (NoHostAvailableException)
{
Thread.Sleep(1000);
continue;
}
break;
}
break;
}
}
public void ReplicationFactorOne_TokenAware()
{
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(3);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()));
testCluster.InitClient();
_policyTestTools.CreateSchema(testCluster.Session, 1);
_policyTestTools.InitPreparedStatement(testCluster, 12, ConsistencyLevel.One);
_policyTestTools.Query(testCluster, 12, ConsistencyLevel.One);
string coordinatorHostQueried = _policyTestTools.Coordinators.First().Key.Split(':').First();
int awareCoord = int.Parse(coordinatorHostQueried.Split('.').Last());
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + awareCoord + ":" + DefaultCassandraPort, 12);
_policyTestTools.ResetCoordinators();
testCluster.StopForce(awareCoord);
TestUtils.WaitForDownWithWait(testCluster.ClusterIpPrefix + awareCoord + ":" + DefaultCassandraPort, testCluster.Cluster, 30);
var acceptedList = new List <ConsistencyLevel>
{
ConsistencyLevel.Any
};
var failList = new List <ConsistencyLevel>
{
ConsistencyLevel.One,
ConsistencyLevel.Two,
ConsistencyLevel.Three,
ConsistencyLevel.Quorum,
ConsistencyLevel.All
};
// Test successful writes
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
}
catch (Exception e)
{
Assert.Fail(String.Format("Test failed at CL.{0} with message: {1}", consistencyLevel, e.Message));
}
}
// Test successful reads
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List <string>
{
"ANY ConsistencyLevel is only supported for writes"
};
Assert.True(acceptableErrorMessages.Contains(e.Message));
}
}
// Test writes which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
Assert.Fail(String.Format("It must not pass at consistency level {0}.", consistencyLevel));
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List <string>
{
"consistency level LOCAL_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)",
"consistency level EACH_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)"
};
Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message));
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
catch (WriteTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
}
// Test reads which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel));
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List <string>
{
"consistency level LOCAL_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)",
"EACH_QUORUM ConsistencyLevel is only supported for writes"
};
Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message));
}
catch (ReadTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
}
}
public void RoundRobin_TwoDCs_EachDcHasOneNodeAddedAndDecommissioned()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1, 1, DefaultMaxClusterCreateRetries, true);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new RoundRobinPolicy());
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session);
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
// Validate that all host were queried equally
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6);
// Add new node to the end of first cluster, remove node from beginning of first cluster
policyTestTools.ResetCoordinators();
// Bootstrap step
testCluster.BootstrapNode(3, "dc1");
string newlyBootstrappedIp = testCluster.ClusterIpPrefix + "3";
TestUtils.WaitForUp(newlyBootstrappedIp, DefaultCassandraPort, 30);
// Validate expected nodes where queried
policyTestTools.WaitForPolicyToolsQueryToHitBootstrappedIp(testCluster, newlyBootstrappedIp);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 4);
// Remove node from beginning of first cluster
policyTestTools.ResetCoordinators();
testCluster.DecommissionNode(1);
TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, testCluster.Cluster, 20);
// Validate expected nodes where queried
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 6);
// Add new node to the end of second cluster, remove node from beginning of first cluster
policyTestTools.ResetCoordinators();
testCluster.BootstrapNode(4, "dc2");
newlyBootstrappedIp = testCluster.ClusterIpPrefix + "4";
TestUtils.WaitForUp(newlyBootstrappedIp, DefaultCassandraPort, 30);
policyTestTools.ResetCoordinators();
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, 4);
// Remove node from beginning of second cluster
policyTestTools.ResetCoordinators();
testCluster.DecommissionNode(2);
TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 20);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, 6);
}