/// <summary>
/// Tests DowngradingConsistencyRetryPolicy
///
/// @test_category connection:retry_policy
/// </summary>
public void DowngradingConsistencyRetryPolicyTest(Builder builder)
{
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(3);
testCluster.Builder = builder;
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session, 3);
// FIXME: Race condition where the nodes are not fully up yet and assertQueried reports slightly different numbers
TestUtils.WaitForSchemaAgreement(testCluster.Cluster);
policyTestTools.InitPreparedStatement(testCluster, 12, ConsistencyLevel.All);
policyTestTools.Query(testCluster, 12, ConsistencyLevel.All);
policyTestTools.AssertAchievedConsistencyLevel(ConsistencyLevel.All);
//Kill one node: 2 nodes alive
testCluster.Stop(2);
TestUtils.WaitForDown(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 20);
//After killing one node, the achieved consistency level should be downgraded
policyTestTools.ResetCoordinators();
policyTestTools.Query(testCluster, 12, ConsistencyLevel.All);
policyTestTools.AssertAchievedConsistencyLevel(ConsistencyLevel.Two);
}
public void RoundRobin_OneDc_OneNodeAdded_OneNodeDecommissioned()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new RoundRobinPolicy());
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session);
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
// Validate that all host were queried equally
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 12);
// Add new node to the end of second cluster, remove node from beginning of first cluster
policyTestTools.ResetCoordinators();
// Bootstrap step
int bootStrapPos = 2;
testCluster.BootstrapNode(bootStrapPos);
string newlyBootstrappedIp = testCluster.ClusterIpPrefix + bootStrapPos;
TestUtils.WaitForUp(newlyBootstrappedIp, DefaultCassandraPort, 30);
// Validate expected nodes where queried
policyTestTools.WaitForPolicyToolsQueryToHitBootstrappedIp(testCluster, newlyBootstrappedIp);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6);
// decommission old node
policyTestTools.ResetCoordinators();
testCluster.DecommissionNode(1);
TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, testCluster.Cluster, 60);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 12);
}
public void RoundRobin_TwoDCs_EachDcHasOneNodeAddedAndDecommissioned()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1, 1, DefaultMaxClusterCreateRetries, true);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new RoundRobinPolicy());
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session);
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
// Validate that all host were queried equally
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6);
// Add new node to the end of first cluster, remove node from beginning of first cluster
policyTestTools.ResetCoordinators();
// Bootstrap step
testCluster.BootstrapNode(3, "dc1");
string newlyBootstrappedIp = testCluster.ClusterIpPrefix + "3";
TestUtils.WaitForUp(newlyBootstrappedIp, DefaultCassandraPort, 30);
// Validate expected nodes where queried
policyTestTools.WaitForPolicyToolsQueryToHitBootstrappedIp(testCluster, newlyBootstrappedIp);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 4);
// Remove node from beginning of first cluster
policyTestTools.ResetCoordinators();
testCluster.DecommissionNode(1);
TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, testCluster.Cluster, 20);
// Validate expected nodes where queried
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 6);
// Add new node to the end of second cluster, remove node from beginning of first cluster
policyTestTools.ResetCoordinators();
testCluster.BootstrapNode(4, "dc2");
newlyBootstrappedIp = testCluster.ClusterIpPrefix + "4";
TestUtils.WaitForUp(newlyBootstrappedIp, DefaultCassandraPort, 30);
policyTestTools.ResetCoordinators();
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, 4);
// Remove node from beginning of second cluster
policyTestTools.ResetCoordinators();
testCluster.DecommissionNode(2);
TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 20);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, 6);
}
public void TokenMap_Rebuild_With_Decommissioned_DC_Existing_RF()
{
// Create a 2dc:1node each cluster
var clusterOptions = new TestClusterOptions();
clusterOptions.Dc2NodeLength = 1;
var testCluster = TestClusterManager.CreateNew(1, clusterOptions);
testCluster.Builder = Cluster.Builder().AddContactPoint("127.0.0.1").WithLoadBalancingPolicy(new DCAwareRoundRobinPolicy("dc1"));
testCluster.Cluster = testCluster.Builder.Build();
testCluster.Session = testCluster.Cluster.Connect();
PolicyTestTools policyTestTools = new PolicyTestTools();
// Create a ks with RF = dc1:1, dc2:1
policyTestTools.CreateMultiDcSchema(testCluster.Session, 1, 1);
policyTestTools.InitPreparedStatement(testCluster, 12, false, ConsistencyLevel.All);
// Replicas now in each node in each dc
policyTestTools.Query(testCluster, 12, false);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 0);
testCluster.Cluster.Shutdown();
testCluster.DecommissionNode(1);
// dc1 no longer has any hosts
testCluster.Builder = Cluster.Builder().AddContactPoint("127.0.0.2").WithLoadBalancingPolicy(new DCAwareRoundRobinPolicy("dc2"));
testCluster.Cluster = testCluster.Builder.Build();
// Should be able to connect and rebuild token map
testCluster.Session = testCluster.Cluster.Connect(policyTestTools.DefaultKeyspace);
policyTestTools.ResetCoordinators();
policyTestTools.Query(testCluster, 12, false);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 12);
testCluster.Cluster.Shutdown();
}
public void TokenAwareTest(bool usePrepared)
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1, 1, DefaultMaxClusterCreateRetries, true);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()));
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session);
//clusterInfo.Cluster.RefreshSchema();
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
// Not the best test ever, we should use OPP and check we do it the
// right nodes. But since M3P is hard-coded for now, let just check
// we just hit only one node.
int nodePosToDecommission = 2;
int nodePositionToNotDecommission = 1;
if (policyTestTools.Coordinators.ContainsKey(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort))
{
nodePosToDecommission = 1;
nodePositionToNotDecommission = 2;
}
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePosToDecommission + ":" + DefaultCassandraPort, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePositionToNotDecommission + ":" + DefaultCassandraPort, 0);
// now try again having stopped the node that was just queried
policyTestTools.ResetCoordinators();
testCluster.DecommissionNode(nodePosToDecommission);
TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + nodePosToDecommission + ":" + DefaultCassandraPort, testCluster.Cluster, 40);
policyTestTools.Query(testCluster, 12, usePrepared);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePosToDecommission + ":" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePositionToNotDecommission + ":" + DefaultCassandraPort, 12);
}
public void RoundRobin_OneDc_AllNodesForceStoppedOneAtATime()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(2);
testCluster.Builder = Cluster.Builder()
.WithLoadBalancingPolicy(new RoundRobinPolicy())
.WithQueryTimeout(10000);
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session);
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6);
policyTestTools.ResetCoordinators();
testCluster.StopForce(1);
TestUtils.WaitForDown(testCluster.ClusterIpPrefix + "1", testCluster.Cluster, 20);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 12);
testCluster.StopForce(2);
TestUtils.WaitForDown(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 20);
try
{
policyTestTools.Query(testCluster, 3);
Assert.Fail("Exception should have been thrown, but wasn't!");
}
catch (NoHostAvailableException)
{
Trace.TraceInformation("Expected NoHostAvailableException exception was thrown.");
}
}
private void RetryPolicyTest(ITestCluster testCluster)
{
PolicyTestTools policyTestTools = new PolicyTestTools();
policyTestTools.TableName = TestUtils.GetUniqueTableName();
policyTestTools.CreateSchema(testCluster.Session, 2);
// Test before state
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
int clusterPosQueried = 1;
int clusterPosNotQueried = 2;
if (!policyTestTools.Coordinators.ContainsKey(testCluster.ClusterIpPrefix + clusterPosQueried))
{
clusterPosQueried = 2;
clusterPosNotQueried = 1;
}
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + clusterPosQueried + ":" + DefaultCassandraPort, 1);
// Stop one of the nodes and test
policyTestTools.ResetCoordinators();
testCluster.Stop(clusterPosQueried);
TestUtils.WaitForDown(testCluster.ClusterIpPrefix + clusterPosQueried, testCluster.Cluster, 30);
policyTestTools.Query(testCluster, 120);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + clusterPosNotQueried + ":" + DefaultCassandraPort, 120);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + clusterPosQueried + ":" + DefaultCassandraPort, 0);
// Start the node that was just down, then down the node that was just up
policyTestTools.ResetCoordinators();
testCluster.Start(clusterPosQueried);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + clusterPosQueried, DefaultCassandraPort, 30);
// Test successful reads
DateTime futureDateTime = DateTime.Now.AddSeconds(120);
while (policyTestTools.Coordinators.Count < 2 && DateTime.Now < futureDateTime)
{
policyTestTools.Query(testCluster, 120);
Thread.Sleep(75);
}
// Ensure that the nodes were contacted
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + clusterPosQueried + ":" + DefaultCassandraPort, 1);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + clusterPosNotQueried + ":" + DefaultCassandraPort, 1);
}
public void TryNextHostRetryPolicyTest()
{
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(2);
var socketOptions = new SocketOptions().SetReadTimeoutMillis(2000);
testCluster.Builder = Cluster.Builder()
.WithRetryPolicy(new LoggingRetryPolicy(TryNextHostRetryPolicy.Instance))
.AddContactPoint(testCluster.ClusterIpPrefix + "1")
.AddContactPoint(testCluster.ClusterIpPrefix + "2")
.WithSocketOptions(socketOptions);
testCluster.InitClient();
// Setup cluster
PolicyTestTools policyTestTools = new PolicyTestTools();
policyTestTools.CreateSchema(testCluster.Session, 2);
policyTestTools.InitPreparedStatement(testCluster, 12);
// Try with both hosts
policyTestTools.Query(testCluster, 10);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + 1 + ":" + DefaultCassandraPort, 5);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + 2 + ":" + DefaultCassandraPort, 5);
// Try with host 1
policyTestTools.ResetCoordinators();
testCluster.PauseNode(2);
policyTestTools.Query(testCluster, 10);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 1 + ":" + DefaultCassandraPort, 10);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 2 + ":" + DefaultCassandraPort, 0);
testCluster.ResumeNode(2);
// Try with host 2
policyTestTools.ResetCoordinators();
testCluster.PauseNode(1);
policyTestTools.Query(testCluster, 10);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 1 + ":" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 2 + ":" + DefaultCassandraPort, 10);
// Try with 0 hosts
policyTestTools.ResetCoordinators();
testCluster.PauseNode(2);
Assert.Throws<NoHostAvailableException>( () => policyTestTools.Query(testCluster, 10));
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 1 + ":" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 2 + ":" + DefaultCassandraPort, 0);
testCluster.ResumeNode(1);
testCluster.ResumeNode(2);
}
