public async Task Should_Add_And_Query_Newly_Bootstrapped_Node(bool asyncConnect)
{
_testCluster = TestClusterManager.CreateNew();
var cluster = Cluster.Builder().AddContactPoint(_testCluster.InitialContactPoint).Build();
await Connect(cluster, asyncConnect, session =>
{
Assert.AreEqual(1, cluster.AllHosts().Count);
_testCluster.BootstrapNode(2);
var queried = false;
Trace.TraceInformation("Node bootstrapped");
Thread.Sleep(10000);
var newNodeAddress = _testCluster.ClusterIpPrefix + 2;
Assert.True(TestUtils.IsNodeReachable(IPAddress.Parse(newNodeAddress)));
//New node should be part of the metadata
Assert.AreEqual(2, cluster.AllHosts().Count);
for (var i = 0; i < 10; i++)
{
var rs = session.Execute("SELECT key FROM system.local");
if (rs.Info.QueriedHost.Address.ToString() == newNodeAddress)
{
queried = true;
break;
}
}
Assert.True(queried, "Newly bootstrapped node should be queried");
}).ConfigureAwait(false);
}
public void Should_Add_And_Query_Newly_Bootstrapped_Node()
{
_testCluster = TestClusterManager.CreateNew();
using (var cluster = Cluster.Builder().AddContactPoint(_testCluster.InitialContactPoint).Build())
{
var session = cluster.Connect();
Assert.AreEqual(1, cluster.AllHosts().Count);
_testCluster.BootstrapNode(2);
var queried = false;
Trace.TraceInformation("Node bootstrapped");
Thread.Sleep(10000);
var newNodeAddress = _testCluster.ClusterIpPrefix + 2;
Assert.True(TestHelper.TryConnect(newNodeAddress), "New node does not accept connections");
//New node should be part of the metadata
Assert.AreEqual(2, cluster.AllHosts().Count);
for (var i = 0; i < 10; i++)
{
var rs = session.Execute("SELECT key FROM system.local");
if (rs.Info.QueriedHost.Address.ToString() == newNodeAddress)
{
queried = true;
break;
}
}
Assert.True(queried, "Newly bootstrapped node should be queried");
}
}
public void BootstrappedNode()
{
var policy = new ConstantReconnectionPolicy(500);
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1);
testCluster.Builder = new Builder().WithReconnectionPolicy(policy);
testCluster.InitClient(); // this will replace the existing session using the newly assigned Builder instance
testCluster.BootstrapNode(2);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "2", DefaultCassandraPort, 60);
//Wait for the join to be online
string newlyBootstrappedHost = testCluster.ClusterIpPrefix + 2;
TestUtils.ValidateBootStrappedNodeIsQueried(testCluster, 2, newlyBootstrappedHost);
}
private void PrepareForSparkTest(ITestCluster testCluster)
{
const string replicationConfigStr = "{'class' : 'SimpleStrategy', 'replication_factor' : 2}";
using (var cluster = Cluster.Builder().AddContactPoint(TestClusterManager.InitialContactPoint).Build())
{
WaitForWorkers(1);
var session = cluster.Connect();
Trace.TraceInformation("GraphMultiNodeTests: Altering keyspace for dse_leases");
session.Execute(
"ALTER KEYSPACE dse_leases WITH REPLICATION = {'class': 'NetworkTopologyStrategy', 'GraphAnalytics': '2'}");
Trace.TraceInformation("GraphMultiNodeTests: Bootstrapping node 2");
testCluster.BootstrapNode(2, false);
Trace.TraceInformation("GraphMultiNodeTests: Setting workload");
testCluster.SetNodeWorkloads(2, new[] { "graph", "spark" });
Trace.TraceInformation("GraphMultiNodeTests: Starting node 2");
testCluster.Start(2);
Trace.TraceInformation("Waiting additional time for new node to be ready");
Thread.Sleep(15000);
WaitForWorkers(2);
Trace.TraceInformation("GraphMultiNodeTests: Creating graph");
session.ExecuteGraph(new SimpleGraphStatement(
"system.graph(name)" +
".option('graph.replication_config').set(replicationConfig)" +
".option('graph.system_replication_config').set(replicationConfig)" +
".ifNotExists()" +
(!TestClusterManager.SupportsNextGenGraph() ? string.Empty : ".engine(Classic)") +
".create()",
new { name = GraphMultiNodeTests.GraphName, replicationConfig = replicationConfigStr }));
Trace.TraceInformation("GraphMultiNodeTests: Created graph");
var graphStatements = new StringBuilder();
graphStatements.Append(BaseIntegrationTest.MakeStrict + "\n");
graphStatements.Append(BaseIntegrationTest.AllowScans + "\n");
graphStatements.Append(BaseIntegrationTest.ClassicSchemaGremlinQuery + "\n");
graphStatements.Append(BaseIntegrationTest.ClassicLoadGremlinQuery);
session.ExecuteGraph(new SimpleGraphStatement(graphStatements.ToString()).SetGraphName(GraphMultiNodeTests.GraphName));
}
}
public void RoundRobin_OneDc_OneNodeAdded_OneNodeDecommissioned()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new RoundRobinPolicy());
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session);
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
// Validate that all host were queried equally
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 12);
// Add new node to the end of second cluster, remove node from beginning of first cluster
policyTestTools.ResetCoordinators();
// Bootstrap step
int bootStrapPos = 2;
testCluster.BootstrapNode(bootStrapPos);
string newlyBootstrappedIp = testCluster.ClusterIpPrefix + bootStrapPos;
TestUtils.WaitForUp(newlyBootstrappedIp, DefaultCassandraPort, 30);
// Validate expected nodes where queried
policyTestTools.WaitForPolicyToolsQueryToHitBootstrappedIp(testCluster, newlyBootstrappedIp);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6);
// decommission old node
policyTestTools.ResetCoordinators();
testCluster.DecommissionNode(1);
TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, testCluster.Cluster, 60);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 12);
}
public async Task Should_Add_And_Query_Newly_Bootstrapped_Node(bool asyncConnect)
{
_testCluster = TestClusterManager.CreateNew();
var cluster = Cluster.Builder().AddContactPoint(_testCluster.InitialContactPoint).Build();
await Connect(cluster, asyncConnect, session =>
{
Assert.AreEqual(1, cluster.AllHosts().Count);
_testCluster.BootstrapNode(2);
Trace.TraceInformation("Node bootstrapped");
var newNodeAddress = _testCluster.ClusterIpPrefix + 2;
var newNodeIpAddress = IPAddress.Parse(newNodeAddress);
TestHelper.RetryAssert(() =>
{
Assert.True(TestUtils.IsNodeReachable(newNodeIpAddress));
//New node should be part of the metadata
Assert.AreEqual(2, cluster.AllHosts().Count);
},
2000,
30);
TestHelper.RetryAssert(() =>
{
var host = cluster.AllHosts().FirstOrDefault(h => h.Address.Address.Equals(newNodeIpAddress));
Assert.IsNotNull(host);
var count = host.Tokens?.Count();
Assert.IsTrue(count.HasValue);
Assert.IsTrue(count.Value > 0, "Tokens Count: " + count);
});
TestHelper.RetryAssert(() =>
{
var rs = session.Execute("SELECT key FROM system.local");
Assert.True(rs.Info.QueriedHost.Address.ToString() == newNodeAddress, "Newly bootstrapped node should be queried");
},
1,
100);
}).ConfigureAwait(false);
}
public async Task Should_Add_And_Query_Newly_Bootstrapped_Node()
{
_realCluster = TestClusterManager.CreateNew();
using (var cluster = Cluster.Builder()
.WithSocketOptions(new SocketOptions().SetReadTimeoutMillis(22000).SetConnectTimeoutMillis(60000))
.AddContactPoint(_realCluster.InitialContactPoint)
.Build())
{
await Connect(cluster, false, session =>
{
Assert.AreEqual(1, cluster.AllHosts().Count);
_realCluster.BootstrapNode(2);
Trace.TraceInformation("Node bootstrapped");
var newNodeAddress = _realCluster.ClusterIpPrefix + 2;
var newNodeIpAddress = IPAddress.Parse(newNodeAddress);
TestHelper.RetryAssert(() =>
{
Assert.True(TestUtils.IsNodeReachable(newNodeIpAddress));
//New node should be part of the metadata
Assert.AreEqual(2, cluster.AllHosts().Count);
var host = cluster.AllHosts().FirstOrDefault(h => h.Address.Address.Equals(newNodeIpAddress));
Assert.IsNotNull(host);
},
2000,
90);
TestHelper.RetryAssert(() =>
{
var rs = session.Execute("SELECT key FROM system.local");
Assert.True(rs.Info.QueriedHost.Address.ToString() == newNodeAddress, "Newly bootstrapped node should be queried");
},
1,
100);
}).ConfigureAwait(false);
}
}
public void RoundRobin_TwoDCs_EachDcHasOneNodeAddedAndDecommissioned()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1, 1, DefaultMaxClusterCreateRetries, true);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new RoundRobinPolicy());
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session);
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
// Validate that all host were queried equally
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6);
// Add new node to the end of first cluster, remove node from beginning of first cluster
policyTestTools.ResetCoordinators();
// Bootstrap step
testCluster.BootstrapNode(3, "dc1");
string newlyBootstrappedIp = testCluster.ClusterIpPrefix + "3";
TestUtils.WaitForUp(newlyBootstrappedIp, DefaultCassandraPort, 30);
// Validate expected nodes where queried
policyTestTools.WaitForPolicyToolsQueryToHitBootstrappedIp(testCluster, newlyBootstrappedIp);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 4);
// Remove node from beginning of first cluster
policyTestTools.ResetCoordinators();
testCluster.DecommissionNode(1);
TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, testCluster.Cluster, 20);
// Validate expected nodes where queried
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 6);
// Add new node to the end of second cluster, remove node from beginning of first cluster
policyTestTools.ResetCoordinators();
testCluster.BootstrapNode(4, "dc2");
newlyBootstrappedIp = testCluster.ClusterIpPrefix + "4";
TestUtils.WaitForUp(newlyBootstrappedIp, DefaultCassandraPort, 30);
policyTestTools.ResetCoordinators();
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, 4);
// Remove node from beginning of second cluster
policyTestTools.ResetCoordinators();
testCluster.DecommissionNode(2);
TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 20);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, 6);
}
public void Should_Add_And_Query_Newly_Bootstrapped_Node()
{
_testCluster = TestClusterManager.CreateNew();
using (var cluster = Cluster.Builder().AddContactPoint(_testCluster.InitialContactPoint).Build())
{
var session = cluster.Connect();
Assert.AreEqual(1, cluster.AllHosts().Count);
_testCluster.BootstrapNode(2);
var queried = false;
Trace.TraceInformation("Node bootstrapped");
Thread.Sleep(10000);
var newNodeAddress = _testCluster.ClusterIpPrefix + 2;
Assert.True(TestHelper.TryConnect(newNodeAddress), "New node does not accept connections");
//New node should be part of the metadata
Assert.AreEqual(2, cluster.AllHosts().Count);
for (var i = 0; i < 10; i++)
{
var rs = session.Execute("SELECT key FROM system.local");
if (rs.Info.QueriedHost.Address.ToString() == newNodeAddress)
{
queried = true;
break;
}
}
Assert.True(queried, "Newly bootstrapped node should be queried");
}
}