public async Task Should_Remove_Decommissioned_Node() { const int numberOfNodes = 2; _realCluster = TestClusterManager.CreateNew(numberOfNodes); using (var cluster = Cluster.Builder().AddContactPoint(_realCluster.InitialContactPoint).Build()) { await Connect(cluster, false, session => { Assert.AreEqual(numberOfNodes, cluster.AllHosts().Count); if (TestClusterManager.SupportsDecommissionForcefully()) { _realCluster.DecommissionNodeForcefully(numberOfNodes); } else { _realCluster.DecommissionNode(numberOfNodes); } _realCluster.Stop(numberOfNodes); Trace.TraceInformation("Node decommissioned"); string decommisionedNode = null; TestHelper.RetryAssert(() => { decommisionedNode = _realCluster.ClusterIpPrefix + 2; Assert.False(TestUtils.IsNodeReachable(IPAddress.Parse(decommisionedNode))); //New node should be part of the metadata Assert.AreEqual(1, cluster.AllHosts().Count); }, 100, 100); var queried = false; for (var i = 0; i < 10; i++) { var rs = session.Execute("SELECT key FROM system.local"); if (rs.Info.QueriedHost.Address.ToString() == decommisionedNode) { queried = true; break; } } Assert.False(queried, "Removed node should be queried"); }).ConfigureAwait(false); } }
public void Should_UseNewHostInQueryPlans_When_HostIsDecommissionedAndJoinsAgain() { var testCluster = _realCluster.Value; using (var cluster = Cluster.Builder() .AddContactPoint(testCluster.InitialContactPoint) .WithSocketOptions(new SocketOptions().SetReadTimeoutMillis(22000).SetConnectTimeoutMillis(60000)) .WithPoolingOptions( new PoolingOptions() .SetCoreConnectionsPerHost(HostDistance.Local, 2) .SetMaxConnectionsPerHost(HostDistance.Local, 2)) .Build()) { var session = (IInternalSession)cluster.Connect(); session.CreateKeyspaceIfNotExists("testks"); session.ChangeKeyspace("testks"); session.Execute("CREATE TABLE test_table (id text, PRIMARY KEY (id))"); // Assert that there are 2 pools, one for each host var hosts = session.Cluster.AllHosts().ToList(); var pool1 = session.GetExistingPool(hosts[0].Address); Assert.AreEqual(2, pool1.OpenConnections); var pool2 = session.GetExistingPool(hosts[1].Address); Assert.AreEqual(2, pool2.OpenConnections); // Assert that both hosts are used in queries var set = new HashSet <IPEndPoint>(); foreach (var i in Enumerable.Range(1, 100)) { var rs = session.Execute($"INSERT INTO test_table(id) VALUES ('{i}')"); set.Add(rs.Info.QueriedHost); } Assert.AreEqual(2, set.Count); // Decommission node if (!TestClusterManager.SupportsDecommissionForcefully()) { testCluster.DecommissionNode(1); } else { testCluster.DecommissionNodeForcefully(1); } testCluster.Stop(1); // Assert that only one host is used in queries set.Clear(); foreach (var i in Enumerable.Range(1, 100)) { var rs = session.Execute($"INSERT INTO test_table(id) VALUES ('{i}')"); set.Add(rs.Info.QueriedHost); } Assert.AreEqual(1, set.Count); var removedHost = hosts.Single(h => !h.Address.Equals(set.First())); // Bring back the decommissioned node testCluster.Start(1, "--jvm_arg=\"-Dcassandra.override_decommission=true\""); // Assert that there are 2 hosts TestHelper.RetryAssert(() => { Assert.AreEqual(2, cluster.AllHosts().Count); }, 1000, 180); // Assert that queries use both hosts again set.Clear(); var idx = 1; TestHelper.RetryAssert(() => { var rs = session.Execute($"INSERT INTO test_table(id) VALUES ('{idx++}')"); set.Add(rs.Info.QueriedHost); Assert.AreEqual(2, set.Count); }, 500, 240); TestHelper.RetryAssert(() => { pool2 = session.GetExistingPool(removedHost.Address); Assert.IsNotNull(pool2); Assert.AreEqual(2, pool2.OpenConnections); }, 500, 60); } }
public void TokenMap_Should_RebuildTokenMap_When_NodeIsDecommissioned() { var listener = new TestTraceListener(); var level = Diagnostics.CassandraTraceSwitch.Level; Diagnostics.CassandraTraceSwitch.Level = TraceLevel.Verbose; Trace.Listeners.Add(listener); try { TestCluster = TestClusterManager.CreateNew(3, new TestClusterOptions { UseVNodes = true }); var keyspaceName = TestUtils.GetUniqueKeyspaceName().ToLower(); ClusterObjSync = Cluster.Builder() .AddContactPoint(TestCluster.InitialContactPoint) .WithMetadataSyncOptions(new MetadataSyncOptions().SetMetadataSyncEnabled(true)) .WithReconnectionPolicy(new ConstantReconnectionPolicy(5000)) .Build(); ClusterObjNotSync = Cluster.Builder() .AddContactPoint(TestCluster.InitialContactPoint) .WithMetadataSyncOptions(new MetadataSyncOptions().SetMetadataSyncEnabled(false)) .WithReconnectionPolicy(new ConstantReconnectionPolicy(5000)) .Build(); var sessionNotSync = ClusterObjNotSync.Connect(); var sessionSync = ClusterObjSync.Connect(); var createKeyspaceCql = $"CREATE KEYSPACE {keyspaceName} WITH replication = {{'class': 'SimpleStrategy', 'replication_factor' : 3}}"; sessionNotSync.Execute(createKeyspaceCql); TestUtils.WaitForSchemaAgreement(ClusterObjNotSync); TestUtils.WaitForSchemaAgreement(ClusterObjSync); sessionNotSync.ChangeKeyspace(keyspaceName); sessionSync.ChangeKeyspace(keyspaceName); ICollection <Host> replicasSync = null; ICollection <Host> replicasNotSync = null; TestHelper.RetryAssert(() => { Assert.AreEqual(3, ClusterObjSync.Metadata.Hosts.Count); Assert.AreEqual(3, ClusterObjNotSync.Metadata.Hosts.Count); replicasSync = ClusterObjSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123")); replicasNotSync = ClusterObjNotSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123")); Assert.AreEqual(3, replicasSync.Count); Assert.AreEqual(1, replicasNotSync.Count); }, 100, 150); var oldTokenMapNotSync = ClusterObjNotSync.Metadata.TokenToReplicasMap; var oldTokenMapSync = ClusterObjSync.Metadata.TokenToReplicasMap; if (TestClusterManager.SupportsDecommissionForcefully()) { this.TestCluster.DecommissionNodeForcefully(1); } else { this.TestCluster.DecommissionNode(1); } this.TestCluster.Stop(1); TestHelper.RetryAssert(() => { Assert.AreEqual(2, ClusterObjSync.Metadata.Hosts.Count, "ClusterObjSync.Metadata.Hosts.Count"); Assert.AreEqual(2, ClusterObjNotSync.Metadata.Hosts.Count, "ClusterObjNotSync.Metadata.Hosts.Count"); replicasSync = ClusterObjSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123")); replicasNotSync = ClusterObjNotSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123")); Assert.AreEqual(2, replicasSync.Count, "replicasSync.Count"); Assert.AreEqual(1, replicasNotSync.Count, "replicasNotSync.Count"); Assert.IsFalse(object.ReferenceEquals(ClusterObjNotSync.Metadata.TokenToReplicasMap, oldTokenMapNotSync)); Assert.IsFalse(object.ReferenceEquals(ClusterObjSync.Metadata.TokenToReplicasMap, oldTokenMapSync)); }, 1000, 360); oldTokenMapNotSync = ClusterObjNotSync.Metadata.TokenToReplicasMap; oldTokenMapSync = ClusterObjSync.Metadata.TokenToReplicasMap; this.TestCluster.BootstrapNode(4); TestHelper.RetryAssert(() => { Assert.AreEqual(3, ClusterObjSync.Metadata.Hosts.Count); Assert.AreEqual(3, ClusterObjNotSync.Metadata.Hosts.Count); replicasSync = ClusterObjSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123")); replicasNotSync = ClusterObjNotSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123")); Assert.AreEqual(3, replicasSync.Count); Assert.AreEqual(1, replicasNotSync.Count); Assert.IsFalse(object.ReferenceEquals(ClusterObjNotSync.Metadata.TokenToReplicasMap, oldTokenMapNotSync)); Assert.IsFalse(object.ReferenceEquals(ClusterObjSync.Metadata.TokenToReplicasMap, oldTokenMapSync)); }, 1000, 360); } catch (Exception ex) { Trace.Flush(); Assert.Fail("Exception: " + ex.ToString() + Environment.NewLine + string.Join(Environment.NewLine, listener.Queue.ToArray())); } finally { Trace.Listeners.Remove(listener); Diagnostics.CassandraTraceSwitch.Level = level; } }
public void TokenMap_Should_RebuildTokenMap_When_NodeIsDecommissioned() { TestCluster = TestClusterManager.CreateNew(3, new TestClusterOptions { UseVNodes = true }); var keyspaceName = TestUtils.GetUniqueKeyspaceName().ToLower(); ClusterObjSync = Cluster.Builder() .AddContactPoint(TestCluster.InitialContactPoint) .WithMetadataSyncOptions(new MetadataSyncOptions().SetMetadataSyncEnabled(true)) .Build(); ClusterObjNotSync = Cluster.Builder() .AddContactPoint(TestCluster.InitialContactPoint) .WithMetadataSyncOptions(new MetadataSyncOptions().SetMetadataSyncEnabled(false)) .Build(); var sessionNotSync = ClusterObjNotSync.Connect(); var sessionSync = ClusterObjSync.Connect(); var createKeyspaceCql = $"CREATE KEYSPACE {keyspaceName} WITH replication = {{'class': 'SimpleStrategy', 'replication_factor' : 3}}"; sessionNotSync.Execute(createKeyspaceCql); TestUtils.WaitForSchemaAgreement(ClusterObjNotSync); TestUtils.WaitForSchemaAgreement(ClusterObjSync); sessionNotSync.ChangeKeyspace(keyspaceName); sessionSync.ChangeKeyspace(keyspaceName); ICollection <Host> replicasSync = null; ICollection <Host> replicasNotSync = null; TestHelper.RetryAssert(() => { Assert.AreEqual(3, ClusterObjSync.Metadata.Hosts.Count); Assert.AreEqual(3, ClusterObjNotSync.Metadata.Hosts.Count); replicasSync = ClusterObjSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123")); replicasNotSync = ClusterObjNotSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123")); Assert.AreEqual(3, replicasSync.Count); Assert.AreEqual(1, replicasNotSync.Count); }, 100, 150); var oldTokenMapNotSync = ClusterObjNotSync.Metadata.TokenToReplicasMap; var oldTokenMapSync = ClusterObjSync.Metadata.TokenToReplicasMap; if (TestClusterManager.SupportsDecommissionForcefully()) { this.TestCluster.DecommissionNodeForcefully(1); } else { this.TestCluster.DecommissionNode(1); } this.TestCluster.Remove(1); TestHelper.RetryAssert(() => { Assert.AreEqual(2, ClusterObjSync.Metadata.Hosts.Count); Assert.AreEqual(2, ClusterObjNotSync.Metadata.Hosts.Count); replicasSync = ClusterObjSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123")); replicasNotSync = ClusterObjNotSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123")); Assert.AreEqual(2, replicasSync.Count); Assert.AreEqual(1, replicasNotSync.Count); Assert.IsFalse(object.ReferenceEquals(ClusterObjNotSync.Metadata.TokenToReplicasMap, oldTokenMapNotSync)); Assert.IsFalse(object.ReferenceEquals(ClusterObjSync.Metadata.TokenToReplicasMap, oldTokenMapSync)); }, 1000, 360); oldTokenMapNotSync = ClusterObjNotSync.Metadata.TokenToReplicasMap; oldTokenMapSync = ClusterObjSync.Metadata.TokenToReplicasMap; this.TestCluster.BootstrapNode(4); TestHelper.RetryAssert(() => { Assert.AreEqual(3, ClusterObjSync.Metadata.Hosts.Count); Assert.AreEqual(3, ClusterObjNotSync.Metadata.Hosts.Count); replicasSync = ClusterObjSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123")); replicasNotSync = ClusterObjNotSync.Metadata.GetReplicas(keyspaceName, Encoding.UTF8.GetBytes("123")); Assert.AreEqual(3, replicasSync.Count); Assert.AreEqual(1, replicasNotSync.Count); Assert.IsFalse(object.ReferenceEquals(ClusterObjNotSync.Metadata.TokenToReplicasMap, oldTokenMapNotSync)); Assert.IsFalse(object.ReferenceEquals(ClusterObjSync.Metadata.TokenToReplicasMap, oldTokenMapSync)); }, 1000, 360); }
public void Should_RemoveNodeMetricsAndDisposeMetricsContext_When_HostIsRemoved() { _metricsRoot = new MetricsBuilder().Build(); var cluster = GetNewTemporaryCluster(b => b.WithMetrics(_metricsRoot.CreateDriverMetricsProvider())); var session = cluster.Connect(); var metrics = session.GetMetrics(); Assert.AreEqual(3, cluster.Metadata.Hosts.Count); Assert.AreEqual(3, metrics.NodeMetrics.Count); // get address for host that will be removed from the cluster in this test var address = TestCluster.ClusterIpPrefix + "2"; var hostToBeRemoved = cluster.Metadata.Hosts.First(h => h.Address.Address.Equals(IPAddress.Parse(address))); // check node metrics are valid var gauge = metrics.GetNodeGauge(hostToBeRemoved, NodeMetric.Gauges.OpenConnections); var appMetricsGaugeValue = _metricsRoot.Snapshot.GetGaugeValue(gauge.Context, gauge.Name); Assert.Greater(gauge.GetValue().Value, 0); Assert.AreEqual(gauge.GetValue().Value, appMetricsGaugeValue); // check node metrics context in app metrics is valid var context = _metricsRoot.Snapshot.GetForContext(gauge.Context); Assert.True(context.IsNotEmpty()); Assert.AreEqual(2, context.Gauges.Count()); // remove host from cluster if (!TestClusterManager.SupportsDecommissionForcefully()) { TestCluster.DecommissionNode(2); } else { TestCluster.DecommissionNodeForcefully(2); } TestCluster.Stop(2); try { TestHelper.RetryAssert(() => { Assert.AreEqual(2, cluster.Metadata.Hosts.Count, "metadata hosts count failed"); }, 200, 50); TestHelper.RetryAssert(() => { Assert.AreEqual(2, metrics.NodeMetrics.Count, "Node metrics count failed"); }, 10, 500); } catch { TestCluster.Start(2, "--jvm_arg=\"-Dcassandra.override_decommission=true\""); throw; } // Check node's metrics were removed from app metrics registry context = _metricsRoot.Snapshot.GetForContext(gauge.Context); Assert.False(context.IsNotEmpty()); Assert.AreEqual(0, context.Gauges.Count()); TestCluster.Start(2, "--jvm_arg=\"-Dcassandra.override_decommission=true\""); TestHelper.RetryAssert(() => { Assert.AreEqual(3, cluster.Metadata.Hosts.Count, "metadata hosts count after bootstrap failed"); }, 200, 50); // when new host is chosen by LBP, connection pool is created foreach (var _ in Enumerable.Range(0, 5)) { session.Execute("SELECT * FROM system.local"); } TestHelper.RetryAssert(() => { Assert.AreEqual(3, metrics.NodeMetrics.Count, "Node metrics count after bootstrap failed"); }, 10, 500); // Check node's metrics were added again context = _metricsRoot.Snapshot.GetForContext(gauge.Context); Assert.True(context.IsNotEmpty()); Assert.AreEqual(2, context.Gauges.Count()); }