public void BootstrapNodeTest()
{
var policy = new ConstantReconnectionPolicy(500);
var builder = Cluster.Builder().WithReconnectionPolicy(policy);
var clusterInfo = TestUtils.CcmSetup(1, builder);
try
{
var session = clusterInfo.Session;
for (var i = 0; i < 100; i++)
{
if (i == 50)
{
TestUtils.CcmBootstrapNode(clusterInfo, 2);
TestUtils.CcmStart(clusterInfo, 2);
}
session.Execute("SELECT * FROM system.schema_columnfamilies").Count();
}
//Wait for the join to be online
Thread.Sleep(120000);
var list = new List <IPEndPoint>();
for (var i = 0; i < 100; i++)
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
rs.Count();
list.Add(rs.Info.QueriedHost);
}
Assert.True(list.Any(ip => ip.ToString().StartsWith(IpPrefix + "2")), "The new node should be queried");
}
finally
{
TestUtils.CcmRemove(clusterInfo);
}
}
public void BootstrapNodeTest()
{
var policy = new ConstantReconnectionPolicy(500);
var builder = Cluster.Builder().WithReconnectionPolicy(policy);
var clusterInfo = TestUtils.CcmSetup(1, builder);
try
{
var session = clusterInfo.Session;
for (var i = 0; i < 100; i++)
{
if (i == 50)
{
TestUtils.CcmBootstrapNode(clusterInfo, 2);
TestUtils.CcmStart(clusterInfo, 2);
}
session.Execute("SELECT * FROM system.schema_columnfamilies").Count();
}
//Wait for the join to be online
Thread.Sleep(120000);
var list = new List<IPAddress>();
for (var i = 0; i < 100; i++)
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
rs.Count();
list.Add(rs.Info.QueriedHost);
}
Assert.True(list.Any(ip => ip.ToString() == IpPrefix + "2"), "The new node should be queried");
}
finally
{
TestUtils.CcmRemove(clusterInfo);
}
}
public void ReconnectionRecyclesPool()
{
var policy = new ConstantReconnectionPolicy(5000);
var nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(2, 1, true, false);
nonShareableTestCluster.Builder = new Builder().WithReconnectionPolicy(policy);
nonShareableTestCluster.InitClient(); // this will replace the existing session using the newly assigned Builder instance
var session = (Session)nonShareableTestCluster.Session;
var hosts = new List<IPEndPoint>();
for (var i = 0; i < 50; i++)
{
var rs = session.Execute("SELECT * FROM system.local");
if (i == 20)
{
nonShareableTestCluster.StopForce(2);
}
else if (i == 30)
{
nonShareableTestCluster.Start(2);
Thread.Sleep(5000);
}
hosts.Add(rs.Info.QueriedHost);
}
var pool = session.GetOrCreateConnectionPool(TestHelper.CreateHost(nonShareableTestCluster.InitialContactPoint), HostDistance.Local);
var connections = pool.OpenConnections.ToArray();
var expectedCoreConnections = nonShareableTestCluster.Cluster.Configuration
.GetPoolingOptions((byte)session.BinaryProtocolVersion)
.GetCoreConnectionsPerHost(HostDistance.Local);
Assert.AreEqual(expectedCoreConnections, connections.Length);
Assert.True(connections.All(c => !c.IsClosed));
}
public void FailoverTest()
{
var parallelOptions = new ParallelOptions();
parallelOptions.TaskScheduler = new ThreadPerTaskScheduler();
parallelOptions.MaxDegreeOfParallelism = 100;
var policy = new ConstantReconnectionPolicy(Int32.MaxValue);
ITestCluster nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(4);
nonShareableTestCluster.Builder = new Builder().WithReconnectionPolicy(policy);
nonShareableTestCluster.InitClient(); // this will replace the existing session using the newly assigned Builder instance
var session = nonShareableTestCluster.Session;
// Check query to host distribution before killing nodes
var queriedHosts = new List <string>();
DateTime futureDateTime = DateTime.Now.AddSeconds(120);
while ((from singleHost in queriedHosts select singleHost).Distinct().Count() < 4 && DateTime.Now < futureDateTime)
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
queriedHosts.Add(rs.Info.QueriedHost.ToString());
Thread.Sleep(50);
}
Assert.AreEqual(4, (from singleHost in queriedHosts select singleHost).Distinct().Count(), "All hosts should have been queried!");
// Create List of actions
Action selectAction = () =>
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
Assert.Greater(rs.Count(), 0);
};
var actions = new List <Action>();
for (var i = 0; i < 100; i++)
{
actions.Add(selectAction);
}
//kill some nodes.
actions.Insert(20, () => nonShareableTestCluster.StopForce(1));
actions.Insert(20, () => nonShareableTestCluster.StopForce(2));
actions.Insert(80, () => nonShareableTestCluster.StopForce(3));
//Execute in parallel more than 100 actions
Parallel.Invoke(parallelOptions, actions.ToArray());
// Wait for the nodes to be killed
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "1", nonShareableTestCluster.Cluster, 20);
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "2", nonShareableTestCluster.Cluster, 20);
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "3", nonShareableTestCluster.Cluster, 20);
// Execute some more SELECTs
for (var i = 0; i < 250; i++)
{
var rowSet2 = session.Execute("SELECT * FROM system.schema_columnfamilies");
Assert.Greater(rowSet2.Count(), 0);
StringAssert.StartsWith(nonShareableTestCluster.ClusterIpPrefix + "4", rowSet2.Info.QueriedHost.ToString());
}
}
public void TokenMapSimpleStrategyWithKeyspaceTest()
{
var rp = new ConstantReconnectionPolicy(1);
var hosts = new List <Host>
{
{ TestHelper.CreateHost("192.168.0.0", "dc1", "rack", new HashSet <string> {
"0"
}) },
{ TestHelper.CreateHost("192.168.0.1", "dc1", "rack", new HashSet <string> {
"10"
}) },
{ TestHelper.CreateHost("192.168.0.2", "dc1", "rack", new HashSet <string> {
"20"
}) }
};
const string strategy = ReplicationStrategies.SimpleStrategy;
var keyspaces = new List <KeyspaceMetadata>
{
new KeyspaceMetadata(null, "ks1", true, strategy, new Dictionary <string, int> {
{ "replication_factor", 2 }
}),
new KeyspaceMetadata(null, "ks2", true, strategy, new Dictionary <string, int> {
{ "replication_factor", 10 }
})
};
var tokenMap = TokenMap.Build("Murmur3Partitioner", hosts, keyspaces);
//the primary replica and the next
var replicas = tokenMap.GetReplicas("ks1", new M3PToken(0));
Assert.AreEqual("0,1", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));
replicas = tokenMap.GetReplicas("ks1", new M3PToken(-100));
Assert.AreEqual("0,1", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));
//Greater than the greatest token
replicas = tokenMap.GetReplicas("ks1", new M3PToken(500000));
Assert.AreEqual("0,1", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));
//The next replica should be the first
replicas = tokenMap.GetReplicas("ks1", new M3PToken(20));
Assert.AreEqual("2,0", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));
//The closest replica and the next
replicas = tokenMap.GetReplicas("ks1", new M3PToken(19));
Assert.AreEqual("2,0", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));
//Even if the replication factor is greater than the ring, it should return only ring size
replicas = tokenMap.GetReplicas("ks2", new M3PToken(5));
Assert.AreEqual("1,2,0", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));
//The primary replica only as the keyspace was not found
replicas = tokenMap.GetReplicas(null, new M3PToken(0));
Assert.AreEqual("0", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));
replicas = tokenMap.GetReplicas(null, new M3PToken(10));
Assert.AreEqual("1", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));
replicas = tokenMap.GetReplicas("ks_does_not_exist", new M3PToken(20));
Assert.AreEqual("2", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));
replicas = tokenMap.GetReplicas(null, new M3PToken(19));
Assert.AreEqual("2", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));
}
public void FailoverReconnectTest()
{
var parallelOptions = new ParallelOptions
{
TaskScheduler = new ThreadPerTaskScheduler(),
MaxDegreeOfParallelism = 1000
};
var policy = new ConstantReconnectionPolicy(1000);
var builder = Cluster.Builder().WithReconnectionPolicy(policy);
var clusterInfo = TestUtils.CcmSetup(4, builder);
try
{
var session = clusterInfo.Session;
Action selectAction = () =>
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
Assert.Greater(rs.Count(), 0);
};
var actions = new List <Action>();
for (var i = 0; i < 100; i++)
{
actions.Add(selectAction);
}
//kill some nodes.
actions.Insert(20, () => TestUtils.CcmStopForce(clusterInfo, 1));
actions.Insert(20, () => TestUtils.CcmStopForce(clusterInfo, 2));
actions.Insert(80, () => TestUtils.CcmStopForce(clusterInfo, 3));
//Execute in parallel more than 100 actions
Parallel.Invoke(parallelOptions, actions.ToArray());
actions = new List <Action>();
for (var i = 0; i < 100; i++)
{
actions.Add(selectAction);
}
//bring back some nodes
actions.Insert(3, () => TestUtils.CcmStart(clusterInfo, 3));
actions.Insert(50, () => TestUtils.CcmStart(clusterInfo, 2));
actions.Insert(50, () => TestUtils.CcmStart(clusterInfo, 1));
//Execute in parallel more than 100 actions
Parallel.Invoke(parallelOptions, actions.ToArray());
}
finally
{
TestUtils.CcmRemove(clusterInfo);
}
}
public void FailoverTest()
{
var parallelOptions = new ParallelOptions();
parallelOptions.TaskScheduler = new ThreadPerTaskScheduler();
parallelOptions.MaxDegreeOfParallelism = 1000;
var policy = new ConstantReconnectionPolicy(Int32.MaxValue);
var builder = Cluster.Builder().WithReconnectionPolicy(policy);
var clusterInfo = TestUtils.CcmSetup(4, builder);
try
{
var session = clusterInfo.Session;
Action selectAction = () =>
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
Assert.Greater(rs.Count(), 0);
};
var actions = new List <Action>();
for (var i = 0; i < 100; i++)
{
actions.Add(selectAction);
}
//kill some nodes.
actions.Insert(20, () => TestUtils.CcmStopForce(clusterInfo, 1));
actions.Insert(20, () => TestUtils.CcmStopForce(clusterInfo, 2));
actions.Insert(80, () => TestUtils.CcmStopForce(clusterInfo, 3));
//Execute in parallel more than 100 actions
Parallel.Invoke(parallelOptions, actions.ToArray());
//Wait for it to be killed
Thread.Sleep(30000);
//Execute serially selects
for (var i = 0; i < 100; i++)
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
Assert.Greater(rs.Count(), 0);
Assert.AreEqual(IpPrefix + "4", rs.Info.QueriedHost.ToString());
}
//The control connection should be using the available node
StringAssert.StartsWith(IpPrefix + "4", clusterInfo.Cluster.Metadata.ControlConnection.BindAddress.ToString());
}
finally
{
TestUtils.CcmRemove(clusterInfo);
}
}
public void BootstrappedNode()
{
var policy = new ConstantReconnectionPolicy(500);
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1);
testCluster.Builder = new Builder().WithReconnectionPolicy(policy);
testCluster.InitClient(); // this will replace the existing session using the newly assigned Builder instance
testCluster.BootstrapNode(2);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "2", DefaultCassandraPort, 60);
//Wait for the join to be online
string newlyBootstrappedHost = testCluster.ClusterIpPrefix + 2;
TestUtils.ValidateBootStrappedNodeIsQueried(testCluster, 2, newlyBootstrappedHost);
}
public void UpdateLocalNodeInfoModifiesHost()
{
var rp = new ConstantReconnectionPolicy(1000);
var metadata = new Metadata(rp);
var config = new Configuration();
var cc = new ControlConnection(GetCluster(), metadata);
cc.Host = TestHelper.CreateHost("127.0.0.1");
var row = TestHelper.CreateRow(new Dictionary <string, object>
{
{ "cluster_name", "ut-cluster" }, { "data_center", "ut-dc" }, { "rack", "ut-rack" }, { "tokens", null }
});
cc.UpdateLocalInfo(row);
Assert.AreEqual("ut-cluster", metadata.ClusterName);
Assert.AreEqual("ut-dc", cc.Host.Datacenter);
Assert.AreEqual("ut-rack", cc.Host.Rack);
}
public void UpdatePeersInfoWithNullRpcIgnores()
{
var rp = new ConstantReconnectionPolicy(1000);
var metadata = new Metadata(rp);
var cc = new ControlConnection(GetCluster(), metadata);
cc.Host = TestHelper.CreateHost("127.0.0.1");
metadata.AddHost(cc.Host.Address);
var rows = TestHelper.CreateRows(new List <Dictionary <string, object> >
{
new Dictionary <string, object> {
{ "rpc_address", null }, { "peer", null }, { "data_center", "ut-dc2" }, { "rack", "ut-rack" }, { "tokens", null }
}
});
cc.UpdatePeersInfo(rows);
//Only local host present
Assert.AreEqual(1, metadata.AllHosts().Count);
}
public void ConstantReconnectionPolicyTest()
{
Builder builder = Cluster.Builder().WithReconnectionPolicy(new ConstantReconnectionPolicy(25 * 1000));
// Ensure that ConstantReconnectionPolicy is what we should be testing
if (!(builder.GetConfiguration().Policies.ReconnectionPolicy is ConstantReconnectionPolicy))
{
Assert.Fail("Set policy does not match retrieved policy.");
}
// Test basic getters
ConstantReconnectionPolicy reconnectionPolicy = (ConstantReconnectionPolicy)builder.GetConfiguration().Policies.ReconnectionPolicy;
Assert.True(reconnectionPolicy.ConstantDelayMs == 25 * 1000);
// Test erroneous instantiations
try
{
new ConstantReconnectionPolicy(-1);
Assert.Fail();
}
catch (ArgumentException)
{
}
// Test nextDelays()
var schedule = new ConstantReconnectionPolicy(10 * 1000).NewSchedule();
Assert.True(schedule.NextDelayMs() == 10000);
Assert.True(schedule.NextDelayMs() == 10000);
Assert.True(schedule.NextDelayMs() == 10000);
Assert.True(schedule.NextDelayMs() == 10000);
Assert.True(schedule.NextDelayMs() == 10000);
//// Run integration test
//int restartTime = 32;
//int retryTime = 50;
//int breakTime = 10; // time until next reconnection attempt
//reconnectionPolicyTest(builder, restartTime, retryTime, breakTime);
}
public void ReconnectionRecyclesPool()
{
var policy = new ConstantReconnectionPolicy(5000);
var builder = Cluster.Builder().WithReconnectionPolicy(policy);
var clusterInfo = TestUtils.CcmSetup(2, builder);
try
{
var session = (Session)clusterInfo.Session;
var hosts = new List <IPEndPoint>();
for (var i = 0; i < 50; i++)
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
if (i == 20)
{
TestUtils.CcmStopForce(clusterInfo, 2);
}
else if (i == 30)
{
TestUtils.CcmStart(clusterInfo, 2);
Thread.Sleep(5000);
}
hosts.Add(rs.Info.QueriedHost);
}
var endpoint = new IPEndPoint(IPAddress.Parse(IpPrefix + "1"), ProtocolOptions.DefaultPort);
var pool = session.GetConnectionPool(new Host(endpoint, policy), HostDistance.Local);
var connections = pool.OpenConnections.ToArray();
var expectedCoreConnections = clusterInfo.Cluster.Configuration
.GetPoolingOptions(connections.First().ProtocolVersion)
.GetCoreConnectionsPerHost(HostDistance.Local);
Assert.AreEqual(expectedCoreConnections, connections.Length);
Assert.True(connections.All(c => !c.IsClosed));
}
finally
{
TestUtils.CcmRemove(clusterInfo);
}
}
public void ReconnectionRecyclesPool()
{
var policy = new ConstantReconnectionPolicy(5000);
var nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(2, 1, true, false);
nonShareableTestCluster.Builder = new Builder().WithReconnectionPolicy(policy);
nonShareableTestCluster.InitClient(); // this will replace the existing session using the newly assigned Builder instance
var session = (Session)nonShareableTestCluster.Session;
var hosts = new List <IPEndPoint>();
for (var i = 0; i < 50; i++)
{
var rs = session.Execute("SELECT * FROM system.local");
if (i == 20)
{
nonShareableTestCluster.StopForce(2);
}
else if (i == 30)
{
nonShareableTestCluster.Start(2);
Thread.Sleep(5000);
}
hosts.Add(rs.Info.QueriedHost);
}
var pool = session.GetOrCreateConnectionPool(TestHelper.CreateHost(nonShareableTestCluster.InitialContactPoint), HostDistance.Local);
var connections = pool.OpenConnections.ToArray();
var expectedCoreConnections = nonShareableTestCluster.Cluster.Configuration
.GetPoolingOptions((byte)session.BinaryProtocolVersion)
.GetCoreConnectionsPerHost(HostDistance.Local);
Assert.AreEqual(expectedCoreConnections, connections.Length);
Assert.True(connections.All(c => !c.IsClosed));
}
public void UpdatePeersInfoModifiesPool()
{
var rp = new ConstantReconnectionPolicy(1000);
var metadata = new Metadata(rp);
var config = new Configuration();
var cc = new ControlConnection(GetCluster(), metadata);
cc.Host = TestHelper.CreateHost("127.0.0.1");
metadata.AddHost(cc.Host.Address);
var hostAddress2 = IPAddress.Parse("127.0.0.2");
var hostAddress3 = IPAddress.Parse("127.0.0.3");
var rows = TestHelper.CreateRows(new List <Dictionary <string, object> >
{
new Dictionary <string, object> {
{ "rpc_address", hostAddress2 }, { "peer", null }, { "data_center", "ut-dc2" }, { "rack", "ut-rack2" }, { "tokens", null }
},
new Dictionary <string, object> {
{ "rpc_address", IPAddress.Parse("0.0.0.0") }, { "peer", hostAddress3 }, { "data_center", "ut-dc3" }, { "rack", "ut-rack3" }, { "tokens", null }
}
});
cc.UpdatePeersInfo(rows);
Assert.AreEqual(3, metadata.AllHosts().Count);
//using rpc_address
var host2 = metadata.GetHost(new IPEndPoint(hostAddress2, ProtocolOptions.DefaultPort));
Assert.NotNull(host2);
Assert.AreEqual("ut-dc2", host2.Datacenter);
Assert.AreEqual("ut-rack2", host2.Rack);
//with rpc_address = 0.0.0.0, use peer
var host3 = metadata.GetHost(new IPEndPoint(hostAddress3, ProtocolOptions.DefaultPort));
Assert.NotNull(host3);
Assert.AreEqual("ut-dc3", host3.Datacenter);
Assert.AreEqual("ut-rack3", host3.Rack);
}
public void constantReconnectionPolicyTest()
{
Builder builder = Cluster.Builder().WithReconnectionPolicy(new ConstantReconnectionPolicy(25 * 1000));
// Ensure that ConstantReconnectionPolicy is what we should be testing
if (!(builder.GetConfiguration().Policies.ReconnectionPolicy is ConstantReconnectionPolicy))
{
Assert.Fail("Set policy does not match retrieved policy.");
}
// Test basic getters
ConstantReconnectionPolicy reconnectionPolicy = (ConstantReconnectionPolicy)builder.GetConfiguration().Policies.ReconnectionPolicy;
Assert.True(reconnectionPolicy.ConstantDelayMs == 25 * 1000);
// Test erroneous instantiations
try
{
new ConstantReconnectionPolicy(-1);
Assert.Fail();
}
catch (ArgumentException e) { }
// Test nextDelays()
var schedule = new ConstantReconnectionPolicy(10 * 1000).NewSchedule();
Assert.True(schedule.NextDelayMs() == 10000);
Assert.True(schedule.NextDelayMs() == 10000);
Assert.True(schedule.NextDelayMs() == 10000);
Assert.True(schedule.NextDelayMs() == 10000);
Assert.True(schedule.NextDelayMs() == 10000);
// Run integration test
int restartTime = 32;
int retryTime = 50;
int breakTime = 10; // time until next reconnection attempt
reconnectionPolicyTest(builder, restartTime, retryTime, breakTime);
}
public void FailoverThenReconnect()
{
var parallelOptions = new ParallelOptions
{
TaskScheduler = new ThreadPerTaskScheduler(),
MaxDegreeOfParallelism = 100
};
var policy = new ConstantReconnectionPolicy(500);
var nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(4, 1, true, false);
using (var cluster = Cluster.Builder().AddContactPoint(nonShareableTestCluster.InitialContactPoint).WithReconnectionPolicy(policy).Build())
{
var session = cluster.Connect();
// Check query to host distribution before killing nodes
var queriedHosts = new List <string>();
DateTime futureDateTime = DateTime.Now.AddSeconds(120);
while ((from singleHost in queriedHosts select singleHost).Distinct().Count() < 4 && DateTime.Now < futureDateTime)
{
var rs = session.Execute("SELECT * FROM system.local");
queriedHosts.Add(rs.Info.QueriedHost.ToString());
Thread.Sleep(50);
}
Assert.AreEqual(4, (from singleHost in queriedHosts select singleHost).Distinct().Count(), "All hosts should have been queried!");
// Create list of actions
Action selectAction = () =>
{
var rs = session.Execute("SELECT * FROM system.local");
Assert.Greater(rs.Count(), 0);
};
var actions = new List <Action>();
for (var i = 0; i < 100; i++)
{
actions.Add(selectAction);
//Check that the control connection is using first host
StringAssert.StartsWith(nonShareableTestCluster.ClusterIpPrefix + "1", nonShareableTestCluster.Cluster.Metadata.ControlConnection.Address.ToString());
//Kill some nodes
//Including the one used by the control connection
actions.Insert(20, () => nonShareableTestCluster.Stop(1));
actions.Insert(20, () => nonShareableTestCluster.Stop(2));
actions.Insert(80, () => nonShareableTestCluster.Stop(3));
//Execute in parallel more than 100 actions
Parallel.Invoke(parallelOptions, actions.ToArray());
//Wait for the nodes to be killed
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "1", nonShareableTestCluster.Cluster, 20);
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "2", nonShareableTestCluster.Cluster, 20);
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "3", nonShareableTestCluster.Cluster, 20);
actions = new List <Action>();
for (var j = 0; j < 100; j++)
{
actions.Add(selectAction);
}
//Check that the control connection is using first host
//bring back some nodes
actions.Insert(3, () => nonShareableTestCluster.Start(3));
actions.Insert(50, () => nonShareableTestCluster.Start(2));
actions.Insert(50, () => nonShareableTestCluster.Start(1));
//Execute in parallel more than 100 actions
Trace.TraceInformation("Start invoking with restart nodes");
Parallel.Invoke(parallelOptions, actions.ToArray());
//Wait for the nodes to be restarted
TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "1", DefaultCassandraPort, 30);
TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "2", DefaultCassandraPort, 30);
TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "3", DefaultCassandraPort, 30);
queriedHosts.Clear();
// keep querying hosts until they are all queried, or time runs out
futureDateTime = DateTime.Now.AddSeconds(120);
while ((from singleHost in queriedHosts select singleHost).Distinct().Count() < 4 && DateTime.Now < futureDateTime)
{
var rs = session.Execute("SELECT * FROM system.local");
queriedHosts.Add(rs.Info.QueriedHost.ToString());
Thread.Sleep(50);
}
//Check that one of the restarted nodes were queried
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, queriedHosts);
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, queriedHosts);
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, queriedHosts);
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, queriedHosts);
//Check that the control connection is still using last host
StringAssert.StartsWith(nonShareableTestCluster.ClusterIpPrefix + "4", nonShareableTestCluster.Cluster.Metadata.ControlConnection.Address.ToString());
}
}
}
public void FailoverTest()
{
var parallelOptions = new ParallelOptions();
parallelOptions.TaskScheduler = new ThreadPerTaskScheduler();
parallelOptions.MaxDegreeOfParallelism = 100;
var policy = new ConstantReconnectionPolicy(int.MaxValue);
var nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(4, 1, true, false);
using (var cluster = Cluster.Builder().AddContactPoint(nonShareableTestCluster.InitialContactPoint).WithReconnectionPolicy(policy).Build())
{
var session = cluster.Connect();
// Check query to host distribution before killing nodes
var queriedHosts = new List<string>();
DateTime futureDateTime = DateTime.Now.AddSeconds(120);
while ((from singleHost in queriedHosts select singleHost).Distinct().Count() < 4 && DateTime.Now < futureDateTime)
{
var rs = session.Execute("SELECT * FROM system.local");
queriedHosts.Add(rs.Info.QueriedHost.ToString());
Thread.Sleep(50);
}
Assert.AreEqual(4, (from singleHost in queriedHosts select singleHost).Distinct().Count(), "All hosts should have been queried!");
// Create List of actions
Action selectAction = () =>
{
var rs = session.Execute("SELECT * FROM system.local");
Assert.Greater(rs.Count(), 0);
};
var actions = new List<Action>();
for (var i = 0; i < 100; i++)
{
actions.Add(selectAction);
}
//kill some nodes.
actions.Insert(20, () => nonShareableTestCluster.StopForce(1));
actions.Insert(20, () => nonShareableTestCluster.StopForce(2));
actions.Insert(80, () => nonShareableTestCluster.StopForce(3));
//Execute in parallel more than 100 actions
Parallel.Invoke(parallelOptions, actions.ToArray());
// Wait for the nodes to be killed
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "1", nonShareableTestCluster.Cluster, 20);
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "2", nonShareableTestCluster.Cluster, 20);
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "3", nonShareableTestCluster.Cluster, 20);
// Execute some more SELECTs
for (var i = 0; i < 250; i++)
{
var rowSet2 = session.Execute("SELECT * FROM system.local");
Assert.Greater(rowSet2.Count(), 0);
StringAssert.StartsWith(nonShareableTestCluster.ClusterIpPrefix + "4", rowSet2.Info.QueriedHost.ToString());
}
}
}
public void FailoverTest()
{
var parallelOptions = new ParallelOptions();
parallelOptions.TaskScheduler = new ThreadPerTaskScheduler();
parallelOptions.MaxDegreeOfParallelism = 1000;
var policy = new ConstantReconnectionPolicy(300);
var builder = Cluster.Builder().WithReconnectionPolicy(policy);
var clusterInfo = TestUtils.CcmSetup(4, builder);
try
{
var session = clusterInfo.Session;
Action selectAction = () =>
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
Assert.Greater(rs.Count(), 0);
};
var actions = new List<Action>();
for (var i = 0; i < 100; i++ )
{
actions.Add(selectAction);
}
//kill some nodes.
actions.Insert(20, () => TestUtils.CcmStopForce(clusterInfo, 1));
actions.Insert(20, () => TestUtils.CcmStopForce(clusterInfo, 2));
actions.Insert(80, () => TestUtils.CcmStopForce(clusterInfo, 3));
//Execute in parallel more than 100 actions
Parallel.Invoke(parallelOptions, actions.ToArray());
actions = new List<Action>();
}
finally
{
TestUtils.CcmRemove(clusterInfo);
}
}
public void FailoverThenReconnect()
{
var parallelOptions = new ParallelOptions
{
TaskScheduler = new ThreadPerTaskScheduler(),
MaxDegreeOfParallelism = 100
};
var policy = new ConstantReconnectionPolicy(500);
var nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(4, 1, true, false);
using (var cluster = Cluster.Builder().AddContactPoint(nonShareableTestCluster.InitialContactPoint).WithReconnectionPolicy(policy).Build())
{
var session = cluster.Connect();
// Check query to host distribution before killing nodes
var queriedHosts = new List<string>();
DateTime futureDateTime = DateTime.Now.AddSeconds(120);
while ((from singleHost in queriedHosts select singleHost).Distinct().Count() < 4 && DateTime.Now < futureDateTime)
{
var rs = session.Execute("SELECT * FROM system.local");
queriedHosts.Add(rs.Info.QueriedHost.ToString());
Thread.Sleep(50);
}
Assert.AreEqual(4, (from singleHost in queriedHosts select singleHost).Distinct().Count(), "All hosts should have been queried!");
// Create list of actions
Action selectAction = () =>
{
var rs = session.Execute("SELECT * FROM system.local");
Assert.Greater(rs.Count(), 0);
};
var actions = new List<Action>();
for (var i = 0; i < 100; i++)
{
actions.Add(selectAction);
//Check that the control connection is using first host
StringAssert.StartsWith(nonShareableTestCluster.ClusterIpPrefix + "1", nonShareableTestCluster.Cluster.Metadata.ControlConnection.Address.ToString());
//Kill some nodes
//Including the one used by the control connection
actions.Insert(20, () => nonShareableTestCluster.Stop(1));
actions.Insert(20, () => nonShareableTestCluster.Stop(2));
actions.Insert(80, () => nonShareableTestCluster.Stop(3));
//Execute in parallel more than 100 actions
Parallel.Invoke(parallelOptions, actions.ToArray());
//Wait for the nodes to be killed
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "1", nonShareableTestCluster.Cluster, 20);
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "2", nonShareableTestCluster.Cluster, 20);
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "3", nonShareableTestCluster.Cluster, 20);
actions = new List<Action>();
for (var j = 0; j < 100; j++)
{
actions.Add(selectAction);
}
//Check that the control connection is using first host
//bring back some nodes
actions.Insert(3, () => nonShareableTestCluster.Start(3));
actions.Insert(50, () => nonShareableTestCluster.Start(2));
actions.Insert(50, () => nonShareableTestCluster.Start(1));
//Execute in parallel more than 100 actions
Trace.TraceInformation("Start invoking with restart nodes");
Parallel.Invoke(parallelOptions, actions.ToArray());
//Wait for the nodes to be restarted
TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "1", DefaultCassandraPort, 30);
TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "2", DefaultCassandraPort, 30);
TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "3", DefaultCassandraPort, 30);
queriedHosts.Clear();
// keep querying hosts until they are all queried, or time runs out
futureDateTime = DateTime.Now.AddSeconds(120);
while ((from singleHost in queriedHosts select singleHost).Distinct().Count() < 4 && DateTime.Now < futureDateTime)
{
var rs = session.Execute("SELECT * FROM system.local");
queriedHosts.Add(rs.Info.QueriedHost.ToString());
Thread.Sleep(50);
}
//Check that one of the restarted nodes were queried
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, queriedHosts);
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, queriedHosts);
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, queriedHosts);
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, queriedHosts);
//Check that the control connection is still using last host
StringAssert.StartsWith(nonShareableTestCluster.ClusterIpPrefix + "4", nonShareableTestCluster.Cluster.Metadata.ControlConnection.Address.ToString());
}
}
}
public void ReconnectionRecyclesPool()
{
var policy = new ConstantReconnectionPolicy(5000);
var builder = Cluster.Builder().WithReconnectionPolicy(policy);
var clusterInfo = TestUtils.CcmSetup(2, builder);
try
{
var session = (Session)clusterInfo.Session;
var hosts = new List<IPAddress>();
for (var i = 0; i < 50; i++)
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
if (i == 20)
{
TestUtils.CcmStopForce(clusterInfo, 2);
}
else if (i == 30)
{
TestUtils.CcmStart(clusterInfo, 2);
Thread.Sleep(5000);
}
hosts.Add(rs.Info.QueriedHost);
}
var pool = session.GetConnectionPool(new Host(IPAddress.Parse(IpPrefix + "1"), policy), HostDistance.Local);
var connections = pool.OpenConnections.ToArray();
var expectedCoreConnections = clusterInfo.Cluster.Configuration
.GetPoolingOptions(connections.First().ProtocolVersion)
.GetCoreConnectionsPerHost(HostDistance.Local);
Assert.AreEqual(expectedCoreConnections, connections.Length);
Assert.True(connections.All(c => !c.IsClosed));
}
finally
{
TestUtils.CcmRemove(clusterInfo);
}
}
public void BootstrappedNode()
{
var policy = new ConstantReconnectionPolicy(500);
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1);
testCluster.Builder = new Builder().WithReconnectionPolicy(policy);
testCluster.InitClient(); // this will replace the existing session using the newly assigned Builder instance
testCluster.BootstrapNode(2);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "2", DefaultCassandraPort, 60);
//Wait for the join to be online
string newlyBootstrappedHost = testCluster.ClusterIpPrefix + 2;
TestUtils.ValidateBootStrappedNodeIsQueried(testCluster, 2, newlyBootstrappedHost);
}
internal ConstantSchedule(ConstantReconnectionPolicy owner)
{
this._owner = owner;
}
public void TokenMapNetworkTopologyStrategyWithKeyspaceTest()
{
var rp = new ConstantReconnectionPolicy(1);
var hosts = new List <Host>
{
{ TestHelper.CreateHost("192.168.0.0", "dc1", "rack1", new HashSet <string> {
"0"
}) },
{ TestHelper.CreateHost("192.168.0.1", "dc1", "rack1", new HashSet <string> {
"100"
}) },
{ TestHelper.CreateHost("192.168.0.2", "dc1", "rack1", new HashSet <string> {
"200"
}) },
{ TestHelper.CreateHost("192.168.0.100", "dc2", "rack1", new HashSet <string> {
"1"
}) },
{ TestHelper.CreateHost("192.168.0.101", "dc2", "rack1", new HashSet <string> {
"101"
}) },
{ TestHelper.CreateHost("192.168.0.102", "dc2", "rack1", new HashSet <string> {
"201"
}) }
};
const string strategy = ReplicationStrategies.NetworkTopologyStrategy;
var keyspaces = new List <KeyspaceMetadata>
{
//network strategy with rf 2 per dc
new KeyspaceMetadata(null, "ks1", true, strategy, new Dictionary <string, int> {
{ "dc1", 2 }, { "dc2", 2 }
}),
//Testing simple (even it is not supposed to be)
new KeyspaceMetadata(null, "ks2", true, ReplicationStrategies.SimpleStrategy, new Dictionary <string, int> {
{ "replication_factor", 3 }
}),
//network strategy with rf 3 dc1 and 1 dc2
new KeyspaceMetadata(null, "ks3", true, strategy, new Dictionary <string, int> {
{ "dc1", 3 }, { "dc2", 1 }, { "dc3", 5 }
}),
//network strategy with rf 4 dc1
new KeyspaceMetadata(null, "ks4", true, strategy, new Dictionary <string, int> {
{ "dc1", 5 }
})
};
var tokenMap = TokenMap.Build("Murmur3Partitioner", hosts, keyspaces);
//KS1
//the primary replica and the next
var replicas = tokenMap.GetReplicas("ks1", new M3PToken(0));
Assert.AreEqual("0,100,1,101", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));
//The next replica should be the first
replicas = tokenMap.GetReplicas("ks1", new M3PToken(200));
Assert.AreEqual("2,102,0,100", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));
//The closest replica and the next
replicas = tokenMap.GetReplicas("ks1", new M3PToken(190));
Assert.AreEqual("2,102,0,100", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));
//KS2
//Simple strategy: 3 tokens no matter which dc
replicas = tokenMap.GetReplicas("ks2", new M3PToken(5000));
Assert.AreEqual("0,100,1", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));
//KS3
replicas = tokenMap.GetReplicas("ks3", new M3PToken(0));
Assert.AreEqual("0,100,1,2", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));
replicas = tokenMap.GetReplicas("ks3", new M3PToken(201));
Assert.AreEqual("102,0,1,2", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));
//KS4
replicas = tokenMap.GetReplicas("ks4", new M3PToken(0));
Assert.AreEqual("0,1,2", String.Join(",", replicas.Select(TestHelper.GetLastAddressByte)));
}
public void FailoverReconnectTest()
{
var parallelOptions = new ParallelOptions
{
TaskScheduler = new ThreadPerTaskScheduler(),
MaxDegreeOfParallelism = 1000
};
var policy = new ConstantReconnectionPolicy(5000);
var builder = Cluster.Builder().WithReconnectionPolicy(policy);
var clusterInfo = TestUtils.CcmSetup(4, builder);
try
{
var session = clusterInfo.Session;
Action selectAction = () =>
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
Assert.Greater(rs.Count(), 0);
};
var actions = new List <Action>();
for (var i = 0; i < 100; i++)
{
actions.Add(selectAction);
}
//Check that the control connection is using first host
StringAssert.StartsWith(IpPrefix + "1", clusterInfo.Cluster.Metadata.ControlConnection.BindAddress.ToString());
//Kill some nodes
//Including the one used by the control connection
actions.Insert(20, () => TestUtils.CcmStopForce(clusterInfo, 1));
actions.Insert(20, () => TestUtils.CcmStopForce(clusterInfo, 2));
actions.Insert(80, () => TestUtils.CcmStopForce(clusterInfo, 3));
//Execute in parallel more than 100 actions
Trace.TraceInformation("Start invoking with kill nodes");
Parallel.Invoke(parallelOptions, actions.ToArray());
//Wait for the nodes to be killed
Thread.Sleep(30000);
actions = new List <Action>();
for (var i = 0; i < 100; i++)
{
actions.Add(selectAction);
}
//Check that the control connection is using first host
//bring back some nodes
actions.Insert(3, () => TestUtils.CcmStart(clusterInfo, 3));
actions.Insert(50, () => TestUtils.CcmStart(clusterInfo, 2));
actions.Insert(50, () => TestUtils.CcmStart(clusterInfo, 1));
//Execute in parallel more than 100 actions
Trace.TraceInformation("Start invoking with restart nodes");
Parallel.Invoke(parallelOptions, actions.ToArray());
//Wait for the nodes to be restarted
Thread.Sleep(30000);
var queriedHosts = new List <IPEndPoint>();
for (var i = 0; i < 100; i++)
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
queriedHosts.Add(rs.Info.QueriedHost);
}
//Check that one of the restarted nodes were queried
Assert.True(queriedHosts.Any(address => address.ToString().StartsWith(IpPrefix + "3")));
//Check that the control connection is still using last host
StringAssert.StartsWith(IpPrefix + "4", clusterInfo.Cluster.Metadata.ControlConnection.BindAddress.ToString());
}
finally
{
TestUtils.CcmRemove(clusterInfo);
}
}
