public async Task Should_Remove_Decommissioned_Node()
{
const int numberOfNodes = 2;
_realCluster = TestClusterManager.CreateNew(numberOfNodes);
using (var cluster = Cluster.Builder().AddContactPoint(_realCluster.InitialContactPoint).Build())
{
await Connect(cluster, false, session =>
{
Assert.AreEqual(numberOfNodes, cluster.AllHosts().Count);
if (TestClusterManager.SupportsDecommissionForcefully())
{
_realCluster.DecommissionNodeForcefully(numberOfNodes);
}
else
{
_realCluster.DecommissionNode(numberOfNodes);
}
_realCluster.Stop(numberOfNodes);
Trace.TraceInformation("Node decommissioned");
string decommisionedNode = null;
TestHelper.RetryAssert(() =>
{
decommisionedNode = _realCluster.ClusterIpPrefix + 2;
Assert.False(TestUtils.IsNodeReachable(IPAddress.Parse(decommisionedNode)));
//New node should be part of the metadata
Assert.AreEqual(1, cluster.AllHosts().Count);
}, 100, 100);
var queried = false;
for (var i = 0; i < 10; i++)
{
var rs = session.Execute("SELECT key FROM system.local");
if (rs.Info.QueriedHost.Address.ToString() == decommisionedNode)
{
queried = true;
break;
}
}
Assert.False(queried, "Removed node should be queried");
}).ConfigureAwait(false);
}
}
public void Should_UseNewHostInQueryPlans_When_HostIsDecommissionedAndJoinsAgain()
{
var testCluster = _realCluster.Value;
using (var cluster =
Cluster.Builder()
.AddContactPoint(testCluster.InitialContactPoint)
.WithSocketOptions(new SocketOptions().SetReadTimeoutMillis(22000).SetConnectTimeoutMillis(60000))
.WithPoolingOptions(
new PoolingOptions()
.SetCoreConnectionsPerHost(HostDistance.Local, 2)
.SetMaxConnectionsPerHost(HostDistance.Local, 2))
.Build())
{
var session = (IInternalSession)cluster.Connect();
session.CreateKeyspaceIfNotExists("testks");
session.ChangeKeyspace("testks");
session.Execute("CREATE TABLE test_table (id text, PRIMARY KEY (id))");
// Assert that there are 2 pools, one for each host
var hosts = session.Cluster.AllHosts().ToList();
var pool1 = session.GetExistingPool(hosts[0].Address);
Assert.AreEqual(2, pool1.OpenConnections);
var pool2 = session.GetExistingPool(hosts[1].Address);
Assert.AreEqual(2, pool2.OpenConnections);
// Assert that both hosts are used in queries
var set = new HashSet <IPEndPoint>();
foreach (var i in Enumerable.Range(1, 100))
{
var rs = session.Execute($"INSERT INTO test_table(id) VALUES ('{i}')");
set.Add(rs.Info.QueriedHost);
}
Assert.AreEqual(2, set.Count);
// Decommission node
if (!TestClusterManager.SupportsDecommissionForcefully())
{
testCluster.DecommissionNode(1);
}
else
{
testCluster.DecommissionNodeForcefully(1);
}
testCluster.Stop(1);
// Assert that only one host is used in queries
set.Clear();
foreach (var i in Enumerable.Range(1, 100))
{
var rs = session.Execute($"INSERT INTO test_table(id) VALUES ('{i}')");
set.Add(rs.Info.QueriedHost);
}
Assert.AreEqual(1, set.Count);
var removedHost = hosts.Single(h => !h.Address.Equals(set.First()));
// Bring back the decommissioned node
testCluster.Start(1, "--jvm_arg=\"-Dcassandra.override_decommission=true\"");
// Assert that there are 2 hosts
TestHelper.RetryAssert(() =>
{
Assert.AreEqual(2, cluster.AllHosts().Count);
}, 1000, 180);
// Assert that queries use both hosts again
set.Clear();
var idx = 1;
TestHelper.RetryAssert(() =>
{
var rs = session.Execute($"INSERT INTO test_table(id) VALUES ('{idx++}')");
set.Add(rs.Info.QueriedHost);
Assert.AreEqual(2, set.Count);
}, 500, 240);
TestHelper.RetryAssert(() =>
{
pool2 = session.GetExistingPool(removedHost.Address);
Assert.IsNotNull(pool2);
Assert.AreEqual(2, pool2.OpenConnections);
}, 500, 60);
}
}
public void TokenMap_Should_RebuildTokenMap_When_NodeIsDecommissioned()
{
var listener = new TestTraceListener();
var level = Diagnostics.CassandraTraceSwitch.Level;
Diagnostics.CassandraTraceSwitch.Level = TraceLevel.Verbose;
Trace.Listeners.Add(listener);
try
{
TestCluster = TestClusterManager.CreateNew(3, new TestClusterOptions {
UseVNodes = true
});
var keyspaceName = TestUtils.GetUniqueKeyspaceName().ToLower();
ClusterObjSync = Cluster.Builder()
.AddContactPoint(TestCluster.InitialContactPoint)
.WithMetadataSyncOptions(new MetadataSyncOptions().SetMetadataSyncEnabled(true))
.WithReconnectionPolicy(new ConstantReconnectionPolicy(5000))
.Build();
ClusterObjNotSync = Cluster.Builder()
.AddContactPoint(TestCluster.InitialContactPoint)
.WithMetadataSyncOptions(new MetadataSyncOptions().SetMetadataSyncEnabled(false))
.WithReconnectionPolicy(new ConstantReconnectionPolicy(5000))
.Build();
var sessionNotSync = ClusterObjNotSync.Connect();
var sessionSync = ClusterObjSync.Connect();
var createKeyspaceCql = $"CREATE KEYSPACE {keyspaceName} WITH replication = {{'class': 'SimpleStrategy', 'replication_factor' : 3}}";
sessionNotSync.Execute(createKeyspaceCql);
TestUtils.WaitForSchemaAgreement(ClusterObjNotSync);
TestUtils.WaitForSchemaAgreement(ClusterObjSync);
sessionNotSync.ChangeKeyspace(keyspaceName);
sessionSync.ChangeKeyspace(keyspaceName);
ICollection <Host> replicasSync = null;
ICollection <Host> replicasNotSync = null;
TestHelper.RetryAssert(() =>
{
Assert.AreEqual(3, ClusterObjSync.Metadata.Hosts.Count);
Assert.AreEqual(3, ClusterObjNotSync.Metadata.Hosts.Count);
replicasSync = ClusterObjSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123"));
replicasNotSync = ClusterObjNotSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123"));
Assert.AreEqual(3, replicasSync.Count);
Assert.AreEqual(1, replicasNotSync.Count);
}, 100, 150);
var oldTokenMapNotSync = ClusterObjNotSync.Metadata.TokenToReplicasMap;
var oldTokenMapSync = ClusterObjSync.Metadata.TokenToReplicasMap;
if (TestClusterManager.SupportsDecommissionForcefully())
{
this.TestCluster.DecommissionNodeForcefully(1);
}
else
{
this.TestCluster.DecommissionNode(1);
}
this.TestCluster.Stop(1);
TestHelper.RetryAssert(() =>
{
Assert.AreEqual(2, ClusterObjSync.Metadata.Hosts.Count, "ClusterObjSync.Metadata.Hosts.Count");
Assert.AreEqual(2, ClusterObjNotSync.Metadata.Hosts.Count, "ClusterObjNotSync.Metadata.Hosts.Count");
replicasSync = ClusterObjSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123"));
replicasNotSync = ClusterObjNotSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123"));
Assert.AreEqual(2, replicasSync.Count, "replicasSync.Count");
Assert.AreEqual(1, replicasNotSync.Count, "replicasNotSync.Count");
Assert.IsFalse(object.ReferenceEquals(ClusterObjNotSync.Metadata.TokenToReplicasMap, oldTokenMapNotSync));
Assert.IsFalse(object.ReferenceEquals(ClusterObjSync.Metadata.TokenToReplicasMap, oldTokenMapSync));
}, 1000, 360);
oldTokenMapNotSync = ClusterObjNotSync.Metadata.TokenToReplicasMap;
oldTokenMapSync = ClusterObjSync.Metadata.TokenToReplicasMap;
this.TestCluster.BootstrapNode(4);
TestHelper.RetryAssert(() =>
{
Assert.AreEqual(3, ClusterObjSync.Metadata.Hosts.Count);
Assert.AreEqual(3, ClusterObjNotSync.Metadata.Hosts.Count);
replicasSync = ClusterObjSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123"));
replicasNotSync = ClusterObjNotSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123"));
Assert.AreEqual(3, replicasSync.Count);
Assert.AreEqual(1, replicasNotSync.Count);
Assert.IsFalse(object.ReferenceEquals(ClusterObjNotSync.Metadata.TokenToReplicasMap, oldTokenMapNotSync));
Assert.IsFalse(object.ReferenceEquals(ClusterObjSync.Metadata.TokenToReplicasMap, oldTokenMapSync));
}, 1000, 360);
}
catch (Exception ex)
{
Trace.Flush();
Assert.Fail("Exception: " + ex.ToString() + Environment.NewLine + string.Join(Environment.NewLine, listener.Queue.ToArray()));
}
finally
{
Trace.Listeners.Remove(listener);
Diagnostics.CassandraTraceSwitch.Level = level;
}
}
public void TokenMap_Should_RebuildTokenMap_When_NodeIsDecommissioned()
{
TestCluster = TestClusterManager.CreateNew(3, new TestClusterOptions {
UseVNodes = true
});
var keyspaceName = TestUtils.GetUniqueKeyspaceName().ToLower();
ClusterObjSync = Cluster.Builder()
.AddContactPoint(TestCluster.InitialContactPoint)
.WithMetadataSyncOptions(new MetadataSyncOptions().SetMetadataSyncEnabled(true))
.Build();
ClusterObjNotSync = Cluster.Builder()
.AddContactPoint(TestCluster.InitialContactPoint)
.WithMetadataSyncOptions(new MetadataSyncOptions().SetMetadataSyncEnabled(false))
.Build();
var sessionNotSync = ClusterObjNotSync.Connect();
var sessionSync = ClusterObjSync.Connect();
var createKeyspaceCql = $"CREATE KEYSPACE {keyspaceName} WITH replication = {{'class': 'SimpleStrategy', 'replication_factor' : 3}}";
sessionNotSync.Execute(createKeyspaceCql);
TestUtils.WaitForSchemaAgreement(ClusterObjNotSync);
TestUtils.WaitForSchemaAgreement(ClusterObjSync);
sessionNotSync.ChangeKeyspace(keyspaceName);
sessionSync.ChangeKeyspace(keyspaceName);
ICollection <Host> replicasSync = null;
ICollection <Host> replicasNotSync = null;
TestHelper.RetryAssert(() =>
{
Assert.AreEqual(3, ClusterObjSync.Metadata.Hosts.Count);
Assert.AreEqual(3, ClusterObjNotSync.Metadata.Hosts.Count);
replicasSync = ClusterObjSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123"));
replicasNotSync = ClusterObjNotSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123"));
Assert.AreEqual(3, replicasSync.Count);
Assert.AreEqual(1, replicasNotSync.Count);
}, 100, 150);
var oldTokenMapNotSync = ClusterObjNotSync.Metadata.TokenToReplicasMap;
var oldTokenMapSync = ClusterObjSync.Metadata.TokenToReplicasMap;
if (TestClusterManager.SupportsDecommissionForcefully())
{
this.TestCluster.DecommissionNodeForcefully(1);
}
else
{
this.TestCluster.DecommissionNode(1);
}
this.TestCluster.Remove(1);
TestHelper.RetryAssert(() =>
{
Assert.AreEqual(2, ClusterObjSync.Metadata.Hosts.Count);
Assert.AreEqual(2, ClusterObjNotSync.Metadata.Hosts.Count);
replicasSync = ClusterObjSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123"));
replicasNotSync = ClusterObjNotSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123"));
Assert.AreEqual(2, replicasSync.Count);
Assert.AreEqual(1, replicasNotSync.Count);
Assert.IsFalse(object.ReferenceEquals(ClusterObjNotSync.Metadata.TokenToReplicasMap, oldTokenMapNotSync));
Assert.IsFalse(object.ReferenceEquals(ClusterObjSync.Metadata.TokenToReplicasMap, oldTokenMapSync));
}, 1000, 360);
oldTokenMapNotSync = ClusterObjNotSync.Metadata.TokenToReplicasMap;
oldTokenMapSync = ClusterObjSync.Metadata.TokenToReplicasMap;
this.TestCluster.BootstrapNode(4);
TestHelper.RetryAssert(() =>
{
Assert.AreEqual(3, ClusterObjSync.Metadata.Hosts.Count);
Assert.AreEqual(3, ClusterObjNotSync.Metadata.Hosts.Count);
replicasSync = ClusterObjSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123"));
replicasNotSync = ClusterObjNotSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123"));
Assert.AreEqual(3, replicasSync.Count);
Assert.AreEqual(1, replicasNotSync.Count);
Assert.IsFalse(object.ReferenceEquals(ClusterObjNotSync.Metadata.TokenToReplicasMap, oldTokenMapNotSync));
Assert.IsFalse(object.ReferenceEquals(ClusterObjSync.Metadata.TokenToReplicasMap, oldTokenMapSync));
}, 1000, 360);
}
public void Should_RemoveNodeMetricsAndDisposeMetricsContext_When_HostIsRemoved()
{
_metricsRoot = new MetricsBuilder().Build();
var cluster = GetNewTemporaryCluster(b => b.WithMetrics(_metricsRoot.CreateDriverMetricsProvider()));
var session = cluster.Connect();
var metrics = session.GetMetrics();
Assert.AreEqual(3, cluster.Metadata.Hosts.Count);
Assert.AreEqual(3, metrics.NodeMetrics.Count);
// get address for host that will be removed from the cluster in this test
var address = TestCluster.ClusterIpPrefix + "2";
var hostToBeRemoved = cluster.Metadata.Hosts.First(h => h.Address.Address.Equals(IPAddress.Parse(address)));
// check node metrics are valid
var gauge = metrics.GetNodeGauge(hostToBeRemoved, NodeMetric.Gauges.OpenConnections);
var appMetricsGaugeValue = _metricsRoot.Snapshot.GetGaugeValue(gauge.Context, gauge.Name);
Assert.Greater(gauge.GetValue().Value, 0);
Assert.AreEqual(gauge.GetValue().Value, appMetricsGaugeValue);
// check node metrics context in app metrics is valid
var context = _metricsRoot.Snapshot.GetForContext(gauge.Context);
Assert.True(context.IsNotEmpty());
Assert.AreEqual(2, context.Gauges.Count());
// remove host from cluster
if (!TestClusterManager.SupportsDecommissionForcefully())
{
TestCluster.DecommissionNode(2);
}
else
{
TestCluster.DecommissionNodeForcefully(2);
}
TestCluster.Stop(2);
try
{
TestHelper.RetryAssert(() => { Assert.AreEqual(2, cluster.Metadata.Hosts.Count, "metadata hosts count failed"); }, 200, 50);
TestHelper.RetryAssert(() => { Assert.AreEqual(2, metrics.NodeMetrics.Count, "Node metrics count failed"); }, 10, 500);
}
catch
{
TestCluster.Start(2, "--jvm_arg=\"-Dcassandra.override_decommission=true\"");
throw;
}
// Check node's metrics were removed from app metrics registry
context = _metricsRoot.Snapshot.GetForContext(gauge.Context);
Assert.False(context.IsNotEmpty());
Assert.AreEqual(0, context.Gauges.Count());
TestCluster.Start(2, "--jvm_arg=\"-Dcassandra.override_decommission=true\"");
TestHelper.RetryAssert(() => { Assert.AreEqual(3, cluster.Metadata.Hosts.Count, "metadata hosts count after bootstrap failed"); },
200, 50);
// when new host is chosen by LBP, connection pool is created
foreach (var _ in Enumerable.Range(0, 5))
{
session.Execute("SELECT * FROM system.local");
}
TestHelper.RetryAssert(() => { Assert.AreEqual(3, metrics.NodeMetrics.Count, "Node metrics count after bootstrap failed"); }, 10,
500);
// Check node's metrics were added again
context = _metricsRoot.Snapshot.GetForContext(gauge.Context);
Assert.True(context.IsNotEmpty());
Assert.AreEqual(2, context.Gauges.Count());
}