public void Should_Remove_Decommissioned_Node()
{
_testCluster = TestClusterManager.CreateNew(2);
using (var cluster = Cluster.Builder().AddContactPoint(_testCluster.InitialContactPoint).Build())
{
var session = cluster.Connect();
Assert.AreEqual(2, cluster.AllHosts().Count);
_testCluster.DecommissionNode(2);
Trace.TraceInformation("Node decommissioned");
Thread.Sleep(10000);
var decommisionedNode = _testCluster.ClusterIpPrefix + 2;
Assert.False(TestHelper.TryConnect(decommisionedNode), "Removed node should not accept connections");
//New node should be part of the metadata
Assert.AreEqual(1, cluster.AllHosts().Count);
var queried = false;
for (var i = 0; i < 10; i++)
{
var rs = session.Execute("SELECT key FROM system.local");
if (rs.Info.QueriedHost.Address.ToString() == decommisionedNode)
{
queried = true;
break;
}
}
Assert.False(queried, "Removed node should be queried");
}
}
public async Task Should_Remove_Decommissioned_Node(bool asyncConnect)
{
_testCluster = TestClusterManager.CreateNew(2);
var cluster = Cluster.Builder().AddContactPoint(_testCluster.InitialContactPoint).Build();
await Connect(cluster, asyncConnect, session =>
{
Assert.AreEqual(2, cluster.AllHosts().Count);
_testCluster.DecommissionNode(2);
Trace.TraceInformation("Node decommissioned");
Thread.Sleep(10000);
var decommisionedNode = _testCluster.ClusterIpPrefix + 2;
Assert.False(TestUtils.IsNodeReachable(IPAddress.Parse(decommisionedNode)));
//New node should be part of the metadata
Assert.AreEqual(1, cluster.AllHosts().Count);
var queried = false;
for (var i = 0; i < 10; i++)
{
var rs = session.Execute("SELECT key FROM system.local");
if (rs.Info.QueriedHost.Address.ToString() == decommisionedNode)
{
queried = true;
break;
}
}
Assert.False(queried, "Removed node should be queried");
}).ConfigureAwait(false);
}
public async Task Should_Remove_Decommissioned_Node()
{
const int numberOfNodes = 2;
_realCluster = TestClusterManager.CreateNew(numberOfNodes);
using (var cluster = Cluster.Builder().AddContactPoint(_realCluster.InitialContactPoint).Build())
{
await Connect(cluster, false, session =>
{
Assert.AreEqual(numberOfNodes, cluster.AllHosts().Count);
if (TestClusterManager.SupportsDecommissionForcefully())
{
_realCluster.DecommissionNodeForcefully(numberOfNodes);
}
else
{
_realCluster.DecommissionNode(numberOfNodes);
}
_realCluster.Stop(numberOfNodes);
Trace.TraceInformation("Node decommissioned");
string decommisionedNode = null;
TestHelper.RetryAssert(() =>
{
decommisionedNode = _realCluster.ClusterIpPrefix + 2;
Assert.False(TestUtils.IsNodeReachable(IPAddress.Parse(decommisionedNode)));
//New node should be part of the metadata
Assert.AreEqual(1, cluster.AllHosts().Count);
}, 100, 100);
var queried = false;
for (var i = 0; i < 10; i++)
{
var rs = session.Execute("SELECT key FROM system.local");
if (rs.Info.QueriedHost.Address.ToString() == decommisionedNode)
{
queried = true;
break;
}
}
Assert.False(queried, "Removed node should be queried");
}).ConfigureAwait(false);
}
}
public void RoundRobin_OneDc_OneNodeAdded_OneNodeDecommissioned()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new RoundRobinPolicy());
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session);
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
// Validate that all host were queried equally
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 12);
// Add new node to the end of second cluster, remove node from beginning of first cluster
policyTestTools.ResetCoordinators();
// Bootstrap step
int bootStrapPos = 2;
testCluster.BootstrapNode(bootStrapPos);
string newlyBootstrappedIp = testCluster.ClusterIpPrefix + bootStrapPos;
TestUtils.WaitForUp(newlyBootstrappedIp, DefaultCassandraPort, 30);
// Validate expected nodes where queried
policyTestTools.WaitForPolicyToolsQueryToHitBootstrappedIp(testCluster, newlyBootstrappedIp);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6);
// decommission old node
policyTestTools.ResetCoordinators();
testCluster.DecommissionNode(1);
TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, testCluster.Cluster, 60);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 12);
}
public async Task Should_Remove_Decommissioned_Node(bool asyncConnect)
{
const int numberOfNodes = 2;
_testCluster = TestClusterManager.CreateNew(numberOfNodes);
var cluster = Cluster.Builder()
.AddContactPoints(_testCluster.InitialContactPoint, _testCluster.ClusterIpPrefix + numberOfNodes)
.Build();
await Run(cluster, asyncConnect, session =>
{
Assert.AreEqual(numberOfNodes, cluster.AllHosts().Count);
if (TestClusterManager.DseVersion >= Version.Parse("5.1.0"))
{
_testCluster.DecommissionNodeForcefully(numberOfNodes);
}
else
{
_testCluster.DecommissionNode(numberOfNodes);
}
Trace.TraceInformation("Node decommissioned");
Thread.Sleep(10000);
var decommisionedNode = _testCluster.ClusterIpPrefix + numberOfNodes;
Assert.False(TestUtils.IsNodeReachable(IPAddress.Parse(decommisionedNode)));
//New node should be part of the metadata
Assert.AreEqual(numberOfNodes - 1, cluster.AllHosts().Count);
var queried = false;
for (var i = 0; i < 10; i++)
{
var rs = session.Execute("SELECT key FROM system.local");
if (rs.Info.QueriedHost.Address.ToString() == decommisionedNode)
{
queried = true;
break;
}
}
Assert.False(queried, "Removed node should be queried");
});
}
public void TokenAwareTest(bool usePrepared)
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1, 1, DefaultMaxClusterCreateRetries, true);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()));
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session);
//clusterInfo.Cluster.RefreshSchema();
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
// Not the best test ever, we should use OPP and check we do it the
// right nodes. But since M3P is hard-coded for now, let just check
// we just hit only one node.
int nodePosToDecommission = 2;
int nodePositionToNotDecommission = 1;
if (policyTestTools.Coordinators.ContainsKey(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort))
{
nodePosToDecommission = 1;
nodePositionToNotDecommission = 2;
}
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePosToDecommission + ":" + DefaultCassandraPort, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePositionToNotDecommission + ":" + DefaultCassandraPort, 0);
// now try again having stopped the node that was just queried
policyTestTools.ResetCoordinators();
testCluster.DecommissionNode(nodePosToDecommission);
TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + nodePosToDecommission + ":" + DefaultCassandraPort, testCluster.Cluster, 40);
policyTestTools.Query(testCluster, 12, usePrepared);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePosToDecommission + ":" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePositionToNotDecommission + ":" + DefaultCassandraPort, 12);
}
public void RoundRobin_TwoDCs_EachDcHasOneNodeAddedAndDecommissioned()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1, 1, DefaultMaxClusterCreateRetries, true);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new RoundRobinPolicy());
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session);
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
// Validate that all host were queried equally
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6);
// Add new node to the end of first cluster, remove node from beginning of first cluster
policyTestTools.ResetCoordinators();
// Bootstrap step
testCluster.BootstrapNode(3, "dc1");
string newlyBootstrappedIp = testCluster.ClusterIpPrefix + "3";
TestUtils.WaitForUp(newlyBootstrappedIp, DefaultCassandraPort, 30);
// Validate expected nodes where queried
policyTestTools.WaitForPolicyToolsQueryToHitBootstrappedIp(testCluster, newlyBootstrappedIp);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 4);
// Remove node from beginning of first cluster
policyTestTools.ResetCoordinators();
testCluster.DecommissionNode(1);
TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, testCluster.Cluster, 20);
// Validate expected nodes where queried
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 6);
// Add new node to the end of second cluster, remove node from beginning of first cluster
policyTestTools.ResetCoordinators();
testCluster.BootstrapNode(4, "dc2");
newlyBootstrappedIp = testCluster.ClusterIpPrefix + "4";
TestUtils.WaitForUp(newlyBootstrappedIp, DefaultCassandraPort, 30);
policyTestTools.ResetCoordinators();
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, 4);
// Remove node from beginning of second cluster
policyTestTools.ResetCoordinators();
testCluster.DecommissionNode(2);
TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 20);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, 6);
}
public void Should_Remove_Decommissioned_Node()
{
_testCluster = TestClusterManager.CreateNew(2);
using (var cluster = Cluster.Builder().AddContactPoint(_testCluster.InitialContactPoint).Build())
{
var session = cluster.Connect();
Assert.AreEqual(2, cluster.AllHosts().Count);
_testCluster.DecommissionNode(2);
Trace.TraceInformation("Node decommissioned");
Thread.Sleep(10000);
var decommisionedNode = _testCluster.ClusterIpPrefix + 2;
Assert.False(TestHelper.TryConnect(decommisionedNode), "Removed node should not accept connections");
//New node should be part of the metadata
Assert.AreEqual(1, cluster.AllHosts().Count);
var queried = false;
for (var i = 0; i < 10; i++)
{
var rs = session.Execute("SELECT key FROM system.local");
if (rs.Info.QueriedHost.Address.ToString() == decommisionedNode)
{
queried = true;
break;
}
}
Assert.False(queried, "Removed node should be queried");
}
}