/// <summary>
/// Tests DowngradingConsistencyRetryPolicy
///
/// @test_category connection:retry_policy
/// </summary>
public void DowngradingConsistencyRetryPolicyTest(Builder builder)
{
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(3);
testCluster.Builder = builder;
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session, 3);
// FIXME: Race condition where the nodes are not fully up yet and assertQueried reports slightly different numbers
TestUtils.WaitForSchemaAgreement(testCluster.Cluster);
policyTestTools.InitPreparedStatement(testCluster, 12, ConsistencyLevel.All);
policyTestTools.Query(testCluster, 12, ConsistencyLevel.All);
policyTestTools.AssertAchievedConsistencyLevel(ConsistencyLevel.All);
//Kill one node: 2 nodes alive
testCluster.Stop(2);
TestUtils.WaitForDown(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 20);
//After killing one node, the achieved consistency level should be downgraded
policyTestTools.ResetCoordinators();
policyTestTools.Query(testCluster, 12, ConsistencyLevel.All);
policyTestTools.AssertAchievedConsistencyLevel(ConsistencyLevel.Two);
}
public void ClusterWaitsForSchemaChangesUntilMaxWaitTimeIsReached()
{
ITestCluster nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(2, 0, true, false);
nonShareableTestCluster.Stop(2);
using (var cluster = Cluster.Builder()
.AddContactPoint(nonShareableTestCluster.InitialContactPoint)
.Build())
{
var session = cluster.Connect();
//Will wait for all the nodes to have the same schema
session.Execute("CREATE KEYSPACE ks1 WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 3}");
}
}
private void ReprepareTest(bool useKeyspace)
{
string keyspace = DefaultKeyspaceName;
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1);
testCluster.InitClient(); // make sure client session was just created
var nonShareableSession = testCluster.Session;
string fqKeyspaceName = "";
if (useKeyspace)
{
nonShareableSession.ChangeKeyspace(keyspace);
}
else
{
fqKeyspaceName = keyspace + ".";
}
try
{
nonShareableSession.Execute("CREATE TABLE " + fqKeyspaceName + "test(k text PRIMARY KEY, i int)");
}
catch (AlreadyExistsException)
{
}
nonShareableSession.Execute("INSERT INTO " + fqKeyspaceName + "test (k, i) VALUES ('123', 17)");
nonShareableSession.Execute("INSERT INTO " + fqKeyspaceName + "test (k, i) VALUES ('124', 18)");
PreparedStatement ps = nonShareableSession.Prepare("SELECT * FROM " + fqKeyspaceName + "test WHERE k = ?");
var rs = nonShareableSession.Execute(ps.Bind("123"));
Assert.AreEqual(rs.First().GetValue <int>("i"), 17);
testCluster.Stop(1);
TestUtils.WaitForDown(testCluster.ClusterIpPrefix + "1", testCluster.Cluster, 40);
testCluster.Start(1);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "1", DefaultCassandraPort, 60);
Assert.True(nonShareableSession.Cluster.AllHosts().Select(h => h.IsUp).Any(), "There should be one node up");
for (var i = 0; i < 10; i++)
{
var rowset = nonShareableSession.Execute(ps.Bind("124"));
Assert.AreEqual(rowset.First().GetValue <int>("i"), 18);
}
}
private void RetryPolicyTest(ITestCluster testCluster)
{
PolicyTestTools policyTestTools = new PolicyTestTools();
policyTestTools.TableName = TestUtils.GetUniqueTableName();
policyTestTools.CreateSchema(testCluster.Session, 2);
// Test before state
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
int clusterPosQueried = 1;
int clusterPosNotQueried = 2;
if (!policyTestTools.Coordinators.ContainsKey(testCluster.ClusterIpPrefix + clusterPosQueried))
{
clusterPosQueried = 2;
clusterPosNotQueried = 1;
}
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + clusterPosQueried + ":" + DefaultCassandraPort, 1);
// Stop one of the nodes and test
policyTestTools.ResetCoordinators();
testCluster.Stop(clusterPosQueried);
TestUtils.WaitForDown(testCluster.ClusterIpPrefix + clusterPosQueried, testCluster.Cluster, 30);
policyTestTools.Query(testCluster, 120);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + clusterPosNotQueried + ":" + DefaultCassandraPort, 120);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + clusterPosQueried + ":" + DefaultCassandraPort, 0);
// Start the node that was just down, then down the node that was just up
policyTestTools.ResetCoordinators();
testCluster.Start(clusterPosQueried);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + clusterPosQueried, DefaultCassandraPort, 30);
// Test successful reads
DateTime futureDateTime = DateTime.Now.AddSeconds(120);
while (policyTestTools.Coordinators.Count < 2 && DateTime.Now < futureDateTime)
{
policyTestTools.Query(testCluster, 120);
Thread.Sleep(75);
}
// Ensure that the nodes were contacted
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + clusterPosQueried + ":" + DefaultCassandraPort, 1);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + clusterPosNotQueried + ":" + DefaultCassandraPort, 1);
}
public async Task Should_Remove_Decommissioned_Node()
{
const int numberOfNodes = 2;
_realCluster = TestClusterManager.CreateNew(numberOfNodes);
using (var cluster = Cluster.Builder().AddContactPoint(_realCluster.InitialContactPoint).Build())
{
await Connect(cluster, false, session =>
{
Assert.AreEqual(numberOfNodes, cluster.AllHosts().Count);
if (TestClusterManager.SupportsDecommissionForcefully())
{
_realCluster.DecommissionNodeForcefully(numberOfNodes);
}
else
{
_realCluster.DecommissionNode(numberOfNodes);
}
_realCluster.Stop(numberOfNodes);
Trace.TraceInformation("Node decommissioned");
string decommisionedNode = null;
TestHelper.RetryAssert(() =>
{
decommisionedNode = _realCluster.ClusterIpPrefix + 2;
Assert.False(TestUtils.IsNodeReachable(IPAddress.Parse(decommisionedNode)));
//New node should be part of the metadata
Assert.AreEqual(1, cluster.AllHosts().Count);
}, 100, 100);
var queried = false;
for (var i = 0; i < 10; i++)
{
var rs = session.Execute("SELECT key FROM system.local");
if (rs.Info.QueriedHost.Address.ToString() == decommisionedNode)
{
queried = true;
break;
}
}
Assert.False(queried, "Removed node should be queried");
}).ConfigureAwait(false);
}
}
public void ClusterWaitsForSchemaChangesUntilMaxWaitTimeIsReachedMultiple()
{
var index = 0;
ITestCluster nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(2, 0, true, false);
nonShareableTestCluster.Stop(2);
using (var cluster = Cluster.Builder()
.AddContactPoint(nonShareableTestCluster.InitialContactPoint)
.Build())
{
var session = cluster.Connect();
//Will wait for all the nodes to have the same schema
session.Execute("CREATE KEYSPACE ks1 WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 3}");
session.ChangeKeyspace("ks1");
TestHelper.ParallelInvoke(() =>
{
session.Execute("CREATE TABLE tbl1" + Interlocked.Increment(ref index) + " (id uuid primary key)");
}, 10);
}
}
public void FailoverThenReconnect()
{
var parallelOptions = new ParallelOptions
{
TaskScheduler = new ThreadPerTaskScheduler(),
MaxDegreeOfParallelism = 100
};
var policy = new ConstantReconnectionPolicy(500);
ITestCluster nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(4);
nonShareableTestCluster.Builder = new Builder().WithReconnectionPolicy(policy);
nonShareableTestCluster.InitClient(); // this will replace the existing session using the newly assigned Builder instance
var session = nonShareableTestCluster.Session;
// Check query to host distribution before killing nodes
var queriedHosts = new List <string>();
DateTime futureDateTime = DateTime.Now.AddSeconds(120);
while ((from singleHost in queriedHosts select singleHost).Distinct().Count() < 4 && DateTime.Now < futureDateTime)
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
queriedHosts.Add(rs.Info.QueriedHost.ToString());
Thread.Sleep(50);
}
Assert.AreEqual(4, (from singleHost in queriedHosts select singleHost).Distinct().Count(), "All hosts should have been queried!");
// Create list of actions
Action selectAction = () =>
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
Assert.Greater(rs.Count(), 0);
};
var actions = new List <Action>();
for (var i = 0; i < 100; i++)
{
actions.Add(selectAction);
}
//Check that the control connection is using first host
StringAssert.StartsWith(nonShareableTestCluster.ClusterIpPrefix + "1", nonShareableTestCluster.Cluster.Metadata.ControlConnection.BindAddress.ToString());
//Kill some nodes
//Including the one used by the control connection
actions.Insert(20, () => nonShareableTestCluster.Stop(1));
actions.Insert(20, () => nonShareableTestCluster.Stop(2));
actions.Insert(80, () => nonShareableTestCluster.Stop(3));
//Execute in parallel more than 100 actions
Parallel.Invoke(parallelOptions, actions.ToArray());
//Wait for the nodes to be killed
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "1", nonShareableTestCluster.Cluster, 20);
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "2", nonShareableTestCluster.Cluster, 20);
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "3", nonShareableTestCluster.Cluster, 20);
actions = new List <Action>();
for (var i = 0; i < 100; i++)
{
actions.Add(selectAction);
}
//Check that the control connection is using first host
//bring back some nodes
actions.Insert(3, () => nonShareableTestCluster.Start(3));
actions.Insert(50, () => nonShareableTestCluster.Start(2));
actions.Insert(50, () => nonShareableTestCluster.Start(1));
//Execute in parallel more than 100 actions
Trace.TraceInformation("Start invoking with restart nodes");
Parallel.Invoke(parallelOptions, actions.ToArray());
//Wait for the nodes to be restarted
TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "1", DefaultCassandraPort, 30);
TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "2", DefaultCassandraPort, 30);
TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "3", DefaultCassandraPort, 30);
queriedHosts.Clear();
// keep querying hosts until they are all queried, or time runs out
futureDateTime = DateTime.Now.AddSeconds(120);
while ((from singleHost in queriedHosts select singleHost).Distinct().Count() < 4 && DateTime.Now < futureDateTime)
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
queriedHosts.Add(rs.Info.QueriedHost.ToString());
Thread.Sleep(50);
}
//Check that one of the restarted nodes were queried
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, queriedHosts);
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, queriedHosts);
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, queriedHosts);
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, queriedHosts);
//Check that the control connection is still using last host
StringAssert.StartsWith(nonShareableTestCluster.ClusterIpPrefix + "4", nonShareableTestCluster.Cluster.Metadata.ControlConnection.BindAddress.ToString());
}
private void RetryPolicyTest(ITestCluster testCluster)
{
PolicyTestTools policyTestTools = new PolicyTestTools();
policyTestTools.TableName = TestUtils.GetUniqueTableName();
policyTestTools.CreateSchema(testCluster.Session, 2);
// Test before state
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
int clusterPosQueried = 1;
int clusterPosNotQueried = 2;
if (!policyTestTools.Coordinators.ContainsKey(testCluster.ClusterIpPrefix + clusterPosQueried))
{
clusterPosQueried = 2;
clusterPosNotQueried = 1;
}
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + clusterPosQueried + ":" + DefaultCassandraPort, 1);
// Stop one of the nodes and test
policyTestTools.ResetCoordinators();
testCluster.Stop(clusterPosQueried);
TestUtils.WaitForDown(testCluster.ClusterIpPrefix + clusterPosQueried, testCluster.Cluster, 30);
policyTestTools.Query(testCluster, 120);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + clusterPosNotQueried + ":" + DefaultCassandraPort, 120);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + clusterPosQueried + ":" + DefaultCassandraPort, 0);
// Start the node that was just down, then down the node that was just up
policyTestTools.ResetCoordinators();
testCluster.Start(clusterPosQueried);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + clusterPosQueried, DefaultCassandraPort, 30);
// Test successful reads
DateTime futureDateTime = DateTime.Now.AddSeconds(120);
while (policyTestTools.Coordinators.Count < 2 && DateTime.Now < futureDateTime)
{
policyTestTools.Query(testCluster, 120);
Thread.Sleep(75);
}
// Ensure that the nodes were contacted
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + clusterPosQueried + ":" + DefaultCassandraPort, 1);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + clusterPosNotQueried + ":" + DefaultCassandraPort, 1);
}