public void TokenAware_BindInt_NoHops()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.CreateNew(3);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()));
testCluster.InitClient();
// Test
var session = testCluster.Session;
policyTestTools.TableName = TestUtils.GetUniqueTableName();
policyTestTools.CreateSchema(session, 1);
var traces = new List <QueryTrace>();
var pstmt = session.Prepare("INSERT INTO " + policyTestTools.TableName + " (k, i) VALUES (?, ?)");
for (var i = (int)short.MinValue; i < short.MinValue + 40; i++)
{
var statement = pstmt
.Bind(i, i)
.EnableTracing();
var rs = session.Execute(statement);
traces.Add(rs.Info.QueryTrace);
}
//Check that there weren't any hops
foreach (var t in traces)
{
//The coordinator must be the only one executing the query
Assert.True(t.Events.All(e => e.Source.ToString() == t.Coordinator.ToString()), "There were trace events from another host for coordinator " + t.Coordinator);
}
}
public void SetupFixture()
{
if (TestClusterManager.CheckCassandraVersion(false, Version.Parse("2.2"), Comparison.LessThan))
{
Assert.Ignore("Requires Cassandra version >= 2.2");
return;
}
Diagnostics.CassandraTraceSwitch.Level = TraceLevel.Info;
_testCluster = TestClusterManager.CreateNew(1, new TestClusterOptions
{
CassandraYaml = new[]
{
"batch_size_warn_threshold_in_kb:5",
"batch_size_fail_threshold_in_kb:50"
},
//Using a mirroring handler, the server will reply providing the same payload that was sent
JvmArgs = new[] { "-Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler" }
});
_testCluster.InitClient();
Session = _testCluster.Session;
Session.Execute(string.Format(TestUtils.CreateKeyspaceSimpleFormat, Keyspace, 1));
Session.Execute(string.Format(TestUtils.CreateTableSimpleFormat, Table));
}
public void TokenAware_Prepared_Composite_NoHops()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetTestCluster(3);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()));
testCluster.InitClient();
// Test
var session = testCluster.Session;
policyTestTools.CreateSchema(session);
policyTestTools.TableName = TestUtils.GetUniqueTableName();
session.Execute(String.Format("CREATE TABLE {0} (k1 text, k2 int, i int, PRIMARY KEY ((k1, k2)))", policyTestTools.TableName));
Thread.Sleep(1000);
var ps = session.Prepare(String.Format("INSERT INTO {0} (k1, k2, i) VALUES (?, ?, ?)", policyTestTools.TableName));
var traces = new List <QueryTrace>();
for (var i = 0; i < 10; i++)
{
var statement = ps.Bind(i.ToString(), i, i).EnableTracing();
//Routing key is calculated by the driver
Assert.NotNull(statement.RoutingKey);
var rs = session.Execute(statement);
traces.Add(rs.Info.QueryTrace);
}
//Check that there weren't any hops
foreach (var t in traces)
{
//The coordinator must be the only one executing the query
Assert.True(t.Events.All(e => e.Source.ToString() == t.Coordinator.ToString()), "There were trace events from another host for coordinator " + t.Coordinator);
}
}
public void FailoverTest()
{
var parallelOptions = new ParallelOptions();
parallelOptions.TaskScheduler = new ThreadPerTaskScheduler();
parallelOptions.MaxDegreeOfParallelism = 100;
var policy = new ConstantReconnectionPolicy(Int32.MaxValue);
ITestCluster nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(4);
nonShareableTestCluster.Builder = new Builder().WithReconnectionPolicy(policy);
nonShareableTestCluster.InitClient(); // this will replace the existing session using the newly assigned Builder instance
var session = nonShareableTestCluster.Session;
// Check query to host distribution before killing nodes
var queriedHosts = new List <string>();
DateTime futureDateTime = DateTime.Now.AddSeconds(120);
while ((from singleHost in queriedHosts select singleHost).Distinct().Count() < 4 && DateTime.Now < futureDateTime)
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
queriedHosts.Add(rs.Info.QueriedHost.ToString());
Thread.Sleep(50);
}
Assert.AreEqual(4, (from singleHost in queriedHosts select singleHost).Distinct().Count(), "All hosts should have been queried!");
// Create List of actions
Action selectAction = () =>
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
Assert.Greater(rs.Count(), 0);
};
var actions = new List <Action>();
for (var i = 0; i < 100; i++)
{
actions.Add(selectAction);
}
//kill some nodes.
actions.Insert(20, () => nonShareableTestCluster.StopForce(1));
actions.Insert(20, () => nonShareableTestCluster.StopForce(2));
actions.Insert(80, () => nonShareableTestCluster.StopForce(3));
//Execute in parallel more than 100 actions
Parallel.Invoke(parallelOptions, actions.ToArray());
// Wait for the nodes to be killed
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "1", nonShareableTestCluster.Cluster, 20);
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "2", nonShareableTestCluster.Cluster, 20);
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "3", nonShareableTestCluster.Cluster, 20);
// Execute some more SELECTs
for (var i = 0; i < 250; i++)
{
var rowSet2 = session.Execute("SELECT * FROM system.schema_columnfamilies");
Assert.Greater(rowSet2.Count(), 0);
StringAssert.StartsWith(nonShareableTestCluster.ClusterIpPrefix + "4", rowSet2.Info.QueriedHost.ToString());
}
}
/// <summary>
/// Tests DowngradingConsistencyRetryPolicy
///
/// @test_category connection:retry_policy
/// </summary>
public void DowngradingConsistencyRetryPolicyTest(Builder builder)
{
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(3);
testCluster.Builder = builder;
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session, 3);
// FIXME: Race condition where the nodes are not fully up yet and assertQueried reports slightly different numbers
TestUtils.WaitForSchemaAgreement(testCluster.Cluster);
policyTestTools.InitPreparedStatement(testCluster, 12, ConsistencyLevel.All);
policyTestTools.Query(testCluster, 12, ConsistencyLevel.All);
policyTestTools.AssertAchievedConsistencyLevel(ConsistencyLevel.All);
//Kill one node: 2 nodes alive
testCluster.Stop(2);
TestUtils.WaitForDown(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 20);
//After killing one node, the achieved consistency level should be downgraded
policyTestTools.ResetCoordinators();
policyTestTools.Query(testCluster, 12, ConsistencyLevel.All);
policyTestTools.AssertAchievedConsistencyLevel(ConsistencyLevel.Two);
}
public void TokenAware_TargetPartition_NoHops()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetTestCluster(3);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()));
testCluster.InitClient();
// Test
policyTestTools.TableName = TestUtils.GetUniqueTableName();
policyTestTools.CreateSchema(testCluster.Session, 1);
var traces = new List <QueryTrace>();
for (var i = -10; i < 10; i++)
{
var partitionKey = BitConverter.GetBytes(i).Reverse().ToArray();
var statement = new SimpleStatement(String.Format("INSERT INTO " + policyTestTools.TableName + " (k, i) VALUES ({0}, {0})", i))
.SetRoutingKey(new RoutingKey()
{
RawRoutingKey = partitionKey
})
.EnableTracing();
var rs = testCluster.Session.Execute(statement);
traces.Add(rs.Info.QueryTrace);
}
//Check that there weren't any hops
foreach (var t in traces)
{
//The coordinator must be the only one executing the query
Assert.True(t.Events.All(e => e.Source.ToString() == t.Coordinator.ToString()), "There were trace events from another host for coordinator " + t.Coordinator);
}
}
public void TokenAware_TargetWrongPartition_HopsOccur()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetTestCluster(3);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()));
testCluster.InitClient();
var session = testCluster.Session;
policyTestTools.TableName = TestUtils.GetUniqueTableName();
policyTestTools.CreateSchema(session, 1);
var traces = new List <QueryTrace>();
for (var i = 1; i < 10; i++)
{
//The partition key is wrongly calculated
var statement = new SimpleStatement(String.Format("INSERT INTO " + policyTestTools.TableName + " (k, i) VALUES ({0}, {0})", i))
.SetRoutingKey(new RoutingKey()
{
RawRoutingKey = new byte[] { 0, 0, 0, 0 }
})
.EnableTracing();
var rs = session.Execute(statement);
traces.Add(rs.Info.QueryTrace);
}
//Check that there were hops
var hopsPerQuery = traces.Select(t => t.Events.Any(e => e.Source.ToString() == t.Coordinator.ToString()));
Assert.True(hopsPerQuery.Any(v => v));
}
/// <summary>
/// Validate client behavior with replication three, only one DC
/// with load balancing policy TokenAware, DCAwareRoundRobin,
/// with retry policy DowngradingConsistencyRetryPolicy
/// after a node is taken down
///
/// @test_category consistency
/// @test_category connection:outage,retry_policy
/// @test_category load_balancing:round_robin
/// </summary>
public void ReplicationFactorThree_RoundRobin_DowngradingConsistencyRetryPolicy()
{
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(3);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new RoundRobinPolicy()).WithRetryPolicy(DowngradingConsistencyRetryPolicy.Instance);
testCluster.InitClient();
TestReplicationFactorThree(testCluster);
}
public void AlwaysIgnoreRetryPolicyTest()
{
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(2);
testCluster.Builder = Cluster.Builder()
.WithRetryPolicy(new LoggingRetryPolicy(AlwaysIgnoreRetryPolicy.Instance))
.AddContactPoint(testCluster.ClusterIpPrefix + "1")
.AddContactPoint(testCluster.ClusterIpPrefix + "2");
testCluster.InitClient();
RetryPolicyTest(testCluster);
}
public void BootstrappedNode()
{
var policy = new ConstantReconnectionPolicy(500);
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1);
testCluster.Builder = new Builder().WithReconnectionPolicy(policy);
testCluster.InitClient(); // this will replace the existing session using the newly assigned Builder instance
testCluster.BootstrapNode(2);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "2", DefaultCassandraPort, 60);
//Wait for the join to be online
string newlyBootstrappedHost = testCluster.ClusterIpPrefix + 2;
TestUtils.ValidateBootStrappedNodeIsQueried(testCluster, 2, newlyBootstrappedHost);
}
public void TryNextHostRetryPolicyTest()
{
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(2);
var socketOptions = new SocketOptions().SetReadTimeoutMillis(2000);
testCluster.Builder = Cluster.Builder()
.WithRetryPolicy(new LoggingRetryPolicy(TryNextHostRetryPolicy.Instance))
.AddContactPoint(testCluster.ClusterIpPrefix + "1")
.AddContactPoint(testCluster.ClusterIpPrefix + "2")
.WithSocketOptions(socketOptions);
testCluster.InitClient();
// Setup cluster
PolicyTestTools policyTestTools = new PolicyTestTools();
policyTestTools.CreateSchema(testCluster.Session, 2);
policyTestTools.InitPreparedStatement(testCluster, 12);
// Try with both hosts
policyTestTools.Query(testCluster, 10);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + 1 + ":" + DefaultCassandraPort, 5);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + 2 + ":" + DefaultCassandraPort, 5);
// Try with host 1
policyTestTools.ResetCoordinators();
testCluster.PauseNode(2);
policyTestTools.Query(testCluster, 10);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 1 + ":" + DefaultCassandraPort, 10);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 2 + ":" + DefaultCassandraPort, 0);
testCluster.ResumeNode(2);
// Try with host 2
policyTestTools.ResetCoordinators();
testCluster.PauseNode(1);
policyTestTools.Query(testCluster, 10);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 1 + ":" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 2 + ":" + DefaultCassandraPort, 10);
// Try with 0 hosts
policyTestTools.ResetCoordinators();
testCluster.PauseNode(2);
Assert.Throws <NoHostAvailableException>(() => policyTestTools.Query(testCluster, 10));
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 1 + ":" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 2 + ":" + DefaultCassandraPort, 0);
testCluster.ResumeNode(1);
testCluster.ResumeNode(2);
}
private void ReprepareTest(bool useKeyspace)
{
string keyspace = DefaultKeyspaceName;
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1);
testCluster.InitClient(); // make sure client session was just created
var nonShareableSession = testCluster.Session;
string fqKeyspaceName = "";
if (useKeyspace)
{
nonShareableSession.ChangeKeyspace(keyspace);
}
else
{
fqKeyspaceName = keyspace + ".";
}
try
{
nonShareableSession.Execute("CREATE TABLE " + fqKeyspaceName + "test(k text PRIMARY KEY, i int)");
}
catch (AlreadyExistsException)
{
}
nonShareableSession.Execute("INSERT INTO " + fqKeyspaceName + "test (k, i) VALUES ('123', 17)");
nonShareableSession.Execute("INSERT INTO " + fqKeyspaceName + "test (k, i) VALUES ('124', 18)");
PreparedStatement ps = nonShareableSession.Prepare("SELECT * FROM " + fqKeyspaceName + "test WHERE k = ?");
var rs = nonShareableSession.Execute(ps.Bind("123"));
Assert.AreEqual(rs.First().GetValue <int>("i"), 17);
testCluster.Stop(1);
TestUtils.WaitForDown(testCluster.ClusterIpPrefix + "1", testCluster.Cluster, 40);
testCluster.Start(1);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "1", DefaultCassandraPort, 60);
Assert.True(nonShareableSession.Cluster.AllHosts().Select(h => h.IsUp).Any(), "There should be one node up");
for (var i = 0; i < 10; i++)
{
var rowset = nonShareableSession.Execute(ps.Bind("124"));
Assert.AreEqual(rowset.First().GetValue <int>("i"), 18);
}
}
public void RoundRobin_TwoDCs_DcAware()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1, 1, DefaultMaxClusterCreateRetries, true);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new DCAwareRoundRobinPolicy("dc2"));
testCluster.InitClient();
policyTestTools.CreateMultiDcSchema(testCluster.Session);
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 12);
}
public void RoundRobin_DcAware_BuildClusterWithNonExistentDc()
{
ITestCluster testCluster = TestClusterManager.GetTestCluster(1);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new DCAwareRoundRobinPolicy("dc2"));
try
{
testCluster.InitClient();
Assert.Fail("Expected exception was not thrown!");
}
catch (ArgumentException e)
{
string expectedErrMsg = "Datacenter dc2 does not match any of the nodes, available datacenters: datacenter1.";
Assert.IsTrue(e.Message.Contains(expectedErrMsg));
}
}
public void RoundRobin_TokenAware_TwoDCsWithOneNodeEach_ReplicationFactorTwo()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1, 1, DefaultMaxClusterCreateRetries, true);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()));
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session, 2);
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6);
}
public void RoundRobin_OneDc_OneNodeAdded_OneNodeDecommissioned()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new RoundRobinPolicy());
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session);
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
// Validate that all host were queried equally
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 12);
// Add new node to the end of second cluster, remove node from beginning of first cluster
policyTestTools.ResetCoordinators();
// Bootstrap step
int bootStrapPos = 2;
testCluster.BootstrapNode(bootStrapPos);
string newlyBootstrappedIp = testCluster.ClusterIpPrefix + bootStrapPos;
TestUtils.WaitForUp(newlyBootstrappedIp, DefaultCassandraPort, 30);
// Validate expected nodes where queried
policyTestTools.WaitForPolicyToolsQueryToHitBootstrappedIp(testCluster, newlyBootstrappedIp);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6);
// decommission old node
policyTestTools.ResetCoordinators();
testCluster.DecommissionNode(1);
TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, testCluster.Cluster, 60);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 12);
}
public void TokenAware_Composite_NoHops()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetTestCluster(3);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()));
testCluster.InitClient();
// Test
var session = testCluster.Session;
policyTestTools.CreateSchema(session);
policyTestTools.TableName = TestUtils.GetUniqueTableName();
session.Execute(String.Format("CREATE TABLE {0} (k1 text, k2 int, i int, PRIMARY KEY ((k1, k2)))", policyTestTools.TableName));
var traces = new List <QueryTrace>();
for (var i = 0; i < 10; i++)
{
var statement = new SimpleStatement(String.Format("INSERT INTO " + policyTestTools.TableName + " (k1, k2, i) VALUES ('{0}', {0}, {0})", i))
.SetRoutingKey(
new RoutingKey()
{
RawRoutingKey = Encoding.UTF8.GetBytes(i.ToString())
},
new RoutingKey()
{
RawRoutingKey = BitConverter.GetBytes(i).Reverse().ToArray()
})
.EnableTracing();
var rs = session.Execute(statement);
traces.Add(rs.Info.QueryTrace);
}
//Check that there weren't any hops
foreach (var t in traces)
{
//The coordinator must be the only one executing the query
Assert.True(t.Events.All(e => e.Source.ToString() == t.Coordinator.ToString()), "There were trace events from another host for coordinator " + t.Coordinator);
}
}
private bool TryToInititalizeClusterClient(ITestCluster testCluster)
{
try
{
foreach (string host in testCluster.ExpectedInitialHosts)
{
TestUtils.WaitForUp(host, DefaultCassandraPort, 30);
}
// at this point we expect all the nodes to be up
testCluster.InitClient();
return(true);
}
catch (Exception e)
{
Trace.TraceError("Unexpected Error occurred when trying to get shared test cluster with InitialContactPoint: " + testCluster.InitialContactPoint + ", name: " + testCluster.Name);
Trace.TraceError("Error Message: " + e.Message);
Trace.TraceError("Error Stack Trace: " + e.StackTrace);
Trace.TraceError("Removing this cluster, and looping back to create a new one ... ");
}
return(false);
}
public void RoundRobin_OneDc_AllNodesForceStoppedOneAtATime()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(2);
testCluster.Builder = Cluster.Builder()
.WithLoadBalancingPolicy(new RoundRobinPolicy())
.WithQueryTimeout(10000);
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session);
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6);
policyTestTools.ResetCoordinators();
testCluster.StopForce(1);
TestUtils.WaitForDown(testCluster.ClusterIpPrefix + "1", testCluster.Cluster, 20);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 12);
testCluster.StopForce(2);
TestUtils.WaitForDown(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 20);
try
{
policyTestTools.Query(testCluster, 3);
Assert.Fail("Exception should have been thrown, but wasn't!");
}
catch (NoHostAvailableException)
{
Trace.TraceInformation("Expected NoHostAvailableException exception was thrown.");
}
}
public void TokenAware_BindString_NoHops()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetTestCluster(3);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()));
testCluster.InitClient();
// Test
var session = testCluster.Session;
policyTestTools.CreateSchema(session);
policyTestTools.TableName = TestUtils.GetUniqueTableName();
session.Execute(String.Format("CREATE TABLE {0} (k text PRIMARY KEY, i int)", policyTestTools.TableName));
var traces = new List <QueryTrace>();
string key = "value";
for (var i = 100; i < 140; i++)
{
key += (char)i;
var partitionKey = Encoding.UTF8.GetBytes(key);
var statement = new SimpleStatement("INSERT INTO " + policyTestTools.TableName + " (k, i) VALUES (?, ?)", key, i)
.SetRoutingKey(new RoutingKey()
{
RawRoutingKey = partitionKey
})
.EnableTracing();
var rs = session.Execute(statement);
traces.Add(rs.Info.QueryTrace);
}
//Check that there weren't any hops
foreach (var t in traces)
{
//The coordinator must be the only one executing the query
Assert.True(t.Events.All(e => e.Source.ToString() == t.Coordinator.ToString()), "There were trace events from another host for coordinator " + t.Coordinator);
}
}
public void TokenAware_Guid_NoHops()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetTestCluster(3);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()));
testCluster.InitClient();
// Test
var session = testCluster.Session;
string uniqueTableName = TestUtils.GetUniqueTableName();
policyTestTools.CreateSchema(session);
session.Execute(String.Format("CREATE TABLE {0} (k uuid PRIMARY KEY, i int)", uniqueTableName));
var traces = new List <QueryTrace>();
for (var i = 0; i < 10; i++)
{
var key = Guid.NewGuid();
var statement = new SimpleStatement(String.Format("INSERT INTO " + uniqueTableName + " (k, i) VALUES ({0}, {1})", key, i))
.SetRoutingKey(
new RoutingKey()
{
RawRoutingKey = TypeCodec.GuidShuffle(key.ToByteArray())
})
.EnableTracing();
var rs = session.Execute(statement);
traces.Add(rs.Info.QueryTrace);
}
//Check that there weren't any hops
foreach (var t in traces)
{
//The coordinator must be the only one executing the query
Assert.True(t.Events.All(e => e.Source.ToString() == t.Coordinator.ToString()), "There were trace events from another host for coordinator " + t.Coordinator);
}
}
public void ReconnectionRecyclesPool()
{
var policy = new ConstantReconnectionPolicy(5000);
ITestCluster nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(2);
nonShareableTestCluster.Builder = new Builder().WithReconnectionPolicy(policy);
nonShareableTestCluster.InitClient(); // this will replace the existing session using the newly assigned Builder instance
var session = (Session)nonShareableTestCluster.Session;
var hosts = new List <IPEndPoint>();
for (var i = 0; i < 50; i++)
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
if (i == 20)
{
nonShareableTestCluster.StopForce(2);
}
else if (i == 30)
{
nonShareableTestCluster.Start(2);
Thread.Sleep(5000);
}
hosts.Add(rs.Info.QueriedHost);
}
var pool = session.GetOrCreateConnectionPool(TestHelper.CreateHost(nonShareableTestCluster.InitialContactPoint), HostDistance.Local);
var connections = pool.OpenConnections.ToArray();
var expectedCoreConnections = nonShareableTestCluster.Cluster.Configuration
.GetPoolingOptions(connections.First().ProtocolVersion)
.GetCoreConnectionsPerHost(HostDistance.Local);
Assert.AreEqual(expectedCoreConnections, connections.Length);
Assert.True(connections.All(c => !c.IsClosed));
}
public void TokenAwareTest(bool usePrepared)
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1, 1, DefaultMaxClusterCreateRetries, true);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()));
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session);
//clusterInfo.Cluster.RefreshSchema();
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
// Not the best test ever, we should use OPP and check we do it the
// right nodes. But since M3P is hard-coded for now, let just check
// we just hit only one node.
int nodePosToDecommission = 2;
int nodePositionToNotDecommission = 1;
if (policyTestTools.Coordinators.ContainsKey(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort))
{
nodePosToDecommission = 1;
nodePositionToNotDecommission = 2;
}
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePosToDecommission + ":" + DefaultCassandraPort, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePositionToNotDecommission + ":" + DefaultCassandraPort, 0);
// now try again having stopped the node that was just queried
policyTestTools.ResetCoordinators();
testCluster.DecommissionNode(nodePosToDecommission);
TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + nodePosToDecommission + ":" + DefaultCassandraPort, testCluster.Cluster, 40);
policyTestTools.Query(testCluster, 12, usePrepared);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePosToDecommission + ":" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePositionToNotDecommission + ":" + DefaultCassandraPort, 12);
}
public void Parallel_Insert_And_Select_Sync()
{
var originalTraceLevel = Diagnostics.CassandraTraceSwitch.Level;
ITestCluster testCluster = TestClusterManager.GetTestCluster(3);
Diagnostics.CassandraTraceSwitch.Level = TraceLevel.Warning;
testCluster.Builder = Cluster.Builder().WithRetryPolicy(DowngradingConsistencyRetryPolicy.Instance);
testCluster.InitClient();
ISession session = testCluster.Session;
string uniqueKsName = "keyspace_" + Randomm.RandomAlphaNum(10);
session.Execute(@"CREATE KEYSPACE " + uniqueKsName +
" WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 3};");
TestUtils.WaitForSchemaAgreement(testCluster.Cluster);
session.ChangeKeyspace(uniqueKsName);
string tableName = "table_" + Guid.NewGuid().ToString("N").ToLower();
session.Execute(String.Format(TestUtils.CREATE_TABLE_TIME_SERIES, tableName));
TestUtils.WaitForSchemaAgreement(testCluster.Cluster);
var insertQuery = String.Format("INSERT INTO {0} (id, event_time, text_sample) VALUES (?, ?, ?)", tableName);
var insertQueryPrepared = session.Prepare(insertQuery);
var selectQuery = String.Format("SELECT * FROM {0} LIMIT 10000", tableName);
const int rowsPerId = 1000;
object insertQueryStatement = new SimpleStatement(insertQuery);
if (CassandraVersion.Major < 2)
{
//Use prepared statements all the way as it is not possible to bind on a simple statement with C* 1.2
insertQueryStatement = session.Prepare(insertQuery);
}
var actionInsert = GetInsertAction(session, insertQueryStatement, ConsistencyLevel.Quorum, rowsPerId);
var actionInsertPrepared = GetInsertAction(session, insertQueryPrepared, ConsistencyLevel.Quorum, rowsPerId);
var actionSelect = GetSelectAction(session, selectQuery, ConsistencyLevel.Quorum, 10);
//Execute insert sync to have some records
actionInsert();
//Execute select sync to assert that everything is going OK
actionSelect();
var actions = new List <Action>();
for (var i = 0; i < 10; i++)
{
//Add 10 actions to execute
actions.AddRange(new[] { actionInsert, actionSelect, actionInsertPrepared });
actions.AddRange(new[] { actionSelect, actionInsert, actionInsertPrepared, actionInsert });
actions.AddRange(new[] { actionInsertPrepared, actionInsertPrepared, actionSelect });
}
//Execute in parallel the 100 actions
var parallelOptions = new ParallelOptions();
parallelOptions.TaskScheduler = new ThreadPerTaskScheduler();
parallelOptions.MaxDegreeOfParallelism = 300;
Parallel.Invoke(parallelOptions, actions.ToArray());
Parallel.Invoke(actions.ToArray());
Diagnostics.CassandraTraceSwitch.Level = originalTraceLevel;
}
/// <summary>
/// Validate client behavior with replication three, multiple DCs
/// with load balancing policy TokenAware, DCAwareRoundRobin,
/// with retry policy DowngradingConsistencyRetryPolicy
/// after a node is taken down
///
/// @test_category consistency
/// @test_category connection:outage,retry_policy
/// @test_category load_balancing:round_robin,token_aware,dc_aware
/// </summary>
public void ReplicationFactorThree_TwoDcs_DcAware_DowngradingConsistencyRetryPolicy()
{
// Seetup
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(3, 3, DefaultMaxClusterCreateRetries, true);
testCluster.Builder = Cluster.Builder()
.WithLoadBalancingPolicy(new TokenAwarePolicy(new DCAwareRoundRobinPolicy("dc2")))
.WithRetryPolicy(DowngradingConsistencyRetryPolicy.Instance);
testCluster.InitClient();
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "1", DefaultCassandraPort, 30);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "2", DefaultCassandraPort, 30);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "3", DefaultCassandraPort, 30);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "4", DefaultCassandraPort, 30);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "5", DefaultCassandraPort, 30);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "6", DefaultCassandraPort, 30);
// Test
_policyTestTools.CreateMultiDcSchema(testCluster.Session, 3, 3);
_policyTestTools.InitPreparedStatement(testCluster, 12, ConsistencyLevel.Two);
_policyTestTools.Query(testCluster, 12, ConsistencyLevel.Two);
// Validate expected number of host / query counts
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 0);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 0);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, 4);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "5:" + DefaultCassandraPort, 4);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "6:" + DefaultCassandraPort, 4);
_policyTestTools.ResetCoordinators();
testCluster.StopForce(2);
// FIXME: This sleep is needed to allow the waitFor() to work
TestUtils.WaitForDownWithWait(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 5);
var acceptedList = new List <ConsistencyLevel>
{
ConsistencyLevel.Any,
ConsistencyLevel.One,
ConsistencyLevel.Two,
ConsistencyLevel.Quorum,
ConsistencyLevel.Three,
ConsistencyLevel.All,
ConsistencyLevel.LocalQuorum,
ConsistencyLevel.EachQuorum
};
var failList = new List <ConsistencyLevel>();
// Test successful writes
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
}
catch (Exception e)
{
Assert.Fail(String.Format("Test failed at CL.{0} with message: {1}", consistencyLevel, e.Message));
}
}
// Test successful reads
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List <string>
{
"EACH_QUORUM ConsistencyLevel is only supported for writes",
"ANY ConsistencyLevel is only supported for writes"
};
Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message));
}
}
// Test writes which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel));
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
catch (WriteTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
}
// Test reads which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel));
}
catch (ReadTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
}
}
public void ReplicationFactorThree_TwoDCs_DowngradingConsistencyRetryPolicy()
{
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(3, 3, DefaultMaxClusterCreateRetries, true);
testCluster.Builder = Cluster.Builder()
.WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()))
.WithRetryPolicy(DowngradingConsistencyRetryPolicy.Instance);
testCluster.InitClient();
_policyTestTools.CreateMultiDcSchema(testCluster.Session, 3, 3);
_policyTestTools.InitPreparedStatement(testCluster, 12, ConsistencyLevel.Two);
// a maximum of 4 IPs should have returned values for the query -- two copies per each of the two DCs
int queriesPerIteration = 4;
int queriesCompleted = 0;
int actualTries = 0;
int maxTries = 20;
while (_policyTestTools.Coordinators.Count() < 4 && actualTries < maxTries)
{
_policyTestTools.Query(testCluster, queriesPerIteration, ConsistencyLevel.Two);
queriesCompleted += queriesPerIteration;
}
Assert.IsTrue(_policyTestTools.Coordinators.Count() >= 4, "The minimum number of hosts queried was not met!");
int totalQueriesForAllHosts = _policyTestTools.Coordinators.Sum(c => c.Value);
Assert.AreEqual(queriesCompleted, totalQueriesForAllHosts,
"The sum of queries for all hosts should equal the number of queries recorded by the calling test!");
_policyTestTools.ResetCoordinators();
testCluster.StopForce(2);
// FIXME: This sleep is needed to allow the waitFor() to work
TestUtils.WaitForDownWithWait(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 5);
var acceptedList = new List <ConsistencyLevel>
{
ConsistencyLevel.Any,
ConsistencyLevel.One,
ConsistencyLevel.Two,
ConsistencyLevel.Quorum,
ConsistencyLevel.Three,
ConsistencyLevel.All,
ConsistencyLevel.LocalQuorum,
ConsistencyLevel.EachQuorum
};
var failList = new List <ConsistencyLevel>();
// Test successful writes
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
}
catch (Exception e)
{
Assert.Fail(String.Format("Test failed at CL.{0} with message: {1}", consistencyLevel, e.Message));
}
}
// Test successful reads
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List <string>
{
"EACH_QUORUM ConsistencyLevel is only supported for writes",
"ANY ConsistencyLevel is only supported for writes"
};
Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message));
}
}
// Test writes which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
Assert.Fail("Expected Exception was not thrown for ConsistencyLevel :" + consistencyLevel);
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
catch (WriteTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
}
// Test reads which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel));
}
catch (ReadTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
}
}
public void ReplicationFactorTwo_DowngradingConsistencyRetryPolicy()
{
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(3);
testCluster.Builder = Cluster.Builder()
.WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()))
.WithRetryPolicy(DowngradingConsistencyRetryPolicy.Instance);
testCluster.InitClient();
_policyTestTools.CreateSchema(testCluster.Session, 2);
_policyTestTools.InitPreparedStatement(testCluster, 12, ConsistencyLevel.Two);
_policyTestTools.Query(testCluster, 12, ConsistencyLevel.Two);
string coordinatorHostQueried = _policyTestTools.Coordinators.First().Key.Split(':').First();;
int awareCoord = int.Parse(coordinatorHostQueried.Split('.').Last());
int coordinatorsWithMoreThanZeroQueries = 0;
foreach (var coordinator in _policyTestTools.Coordinators)
{
coordinatorsWithMoreThanZeroQueries++;
_policyTestTools.AssertQueried(coordinator.Key.ToString(), 6);
}
Assert.AreEqual(2, coordinatorsWithMoreThanZeroQueries);
_policyTestTools.ResetCoordinators();
testCluster.StopForce(awareCoord);
TestUtils.WaitForDownWithWait(testCluster.ClusterIpPrefix + awareCoord, testCluster.Cluster, 30);
var acceptedList = new List <ConsistencyLevel>
{
ConsistencyLevel.Any,
ConsistencyLevel.One,
ConsistencyLevel.Two,
ConsistencyLevel.Quorum,
ConsistencyLevel.Three,
ConsistencyLevel.All
};
var failList = new List <ConsistencyLevel>();
// Test successful writes
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
}
catch (Exception e)
{
Assert.Fail(String.Format("Test failed at CL.{0} with message: {1}", consistencyLevel, e.Message));
}
}
// Test successful reads
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List <string>
{
"ANY ConsistencyLevel is only supported for writes"
};
Assert.True(acceptableErrorMessages.Contains(e.Message));
}
}
// Test writes which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel));
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List <string>
{
"consistency level LOCAL_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)",
"consistency level EACH_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)"
};
Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message));
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
catch (WriteTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
}
// Test reads which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel));
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List <string>
{
"consistency level LOCAL_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)",
"EACH_QUORUM ConsistencyLevel is only supported for writes"
};
Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message));
}
catch (ReadTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
}
}
public void RoundRobin_TwoDCs_EachDcHasOneNodeAddedAndDecommissioned()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1, 1, DefaultMaxClusterCreateRetries, true);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new RoundRobinPolicy());
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session);
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
// Validate that all host were queried equally
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6);
// Add new node to the end of first cluster, remove node from beginning of first cluster
policyTestTools.ResetCoordinators();
// Bootstrap step
testCluster.BootstrapNode(3, "dc1");
string newlyBootstrappedIp = testCluster.ClusterIpPrefix + "3";
TestUtils.WaitForUp(newlyBootstrappedIp, DefaultCassandraPort, 30);
// Validate expected nodes where queried
policyTestTools.WaitForPolicyToolsQueryToHitBootstrappedIp(testCluster, newlyBootstrappedIp);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 4);
// Remove node from beginning of first cluster
policyTestTools.ResetCoordinators();
testCluster.DecommissionNode(1);
TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, testCluster.Cluster, 20);
// Validate expected nodes where queried
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 6);
// Add new node to the end of second cluster, remove node from beginning of first cluster
policyTestTools.ResetCoordinators();
testCluster.BootstrapNode(4, "dc2");
newlyBootstrappedIp = testCluster.ClusterIpPrefix + "4";
TestUtils.WaitForUp(newlyBootstrappedIp, DefaultCassandraPort, 30);
policyTestTools.ResetCoordinators();
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, 4);
// Remove node from beginning of second cluster
policyTestTools.ResetCoordinators();
testCluster.DecommissionNode(2);
TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 20);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, 6);
}
private bool TryToInititalizeClusterClient(ITestCluster testCluster)
{
try
{
foreach (string host in testCluster.ExpectedInitialHosts)
TestUtils.WaitForUp(host, DefaultCassandraPort, 30);
// at this point we expect all the nodes to be up
testCluster.InitClient();
return true;
}
catch (Exception e)
{
Trace.TraceError("Unexpected Error occurred when trying to get shared test cluster with InitialContactPoint: " + testCluster.InitialContactPoint + ", name: " + testCluster.Name);
Trace.TraceError("Error Message: " + e.Message);
Trace.TraceError("Error Stack Trace: " + e.StackTrace);
Trace.TraceError("Removing this cluster, and looping back to create a new one ... ");
}
return false;
}
public void FailoverThenReconnect()
{
var parallelOptions = new ParallelOptions
{
TaskScheduler = new ThreadPerTaskScheduler(),
MaxDegreeOfParallelism = 100
};
var policy = new ConstantReconnectionPolicy(500);
ITestCluster nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(4);
nonShareableTestCluster.Builder = new Builder().WithReconnectionPolicy(policy);
nonShareableTestCluster.InitClient(); // this will replace the existing session using the newly assigned Builder instance
var session = nonShareableTestCluster.Session;
// Check query to host distribution before killing nodes
var queriedHosts = new List <string>();
DateTime futureDateTime = DateTime.Now.AddSeconds(120);
while ((from singleHost in queriedHosts select singleHost).Distinct().Count() < 4 && DateTime.Now < futureDateTime)
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
queriedHosts.Add(rs.Info.QueriedHost.ToString());
Thread.Sleep(50);
}
Assert.AreEqual(4, (from singleHost in queriedHosts select singleHost).Distinct().Count(), "All hosts should have been queried!");
// Create list of actions
Action selectAction = () =>
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
Assert.Greater(rs.Count(), 0);
};
var actions = new List <Action>();
for (var i = 0; i < 100; i++)
{
actions.Add(selectAction);
}
//Check that the control connection is using first host
StringAssert.StartsWith(nonShareableTestCluster.ClusterIpPrefix + "1", nonShareableTestCluster.Cluster.Metadata.ControlConnection.BindAddress.ToString());
//Kill some nodes
//Including the one used by the control connection
actions.Insert(20, () => nonShareableTestCluster.Stop(1));
actions.Insert(20, () => nonShareableTestCluster.Stop(2));
actions.Insert(80, () => nonShareableTestCluster.Stop(3));
//Execute in parallel more than 100 actions
Parallel.Invoke(parallelOptions, actions.ToArray());
//Wait for the nodes to be killed
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "1", nonShareableTestCluster.Cluster, 20);
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "2", nonShareableTestCluster.Cluster, 20);
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "3", nonShareableTestCluster.Cluster, 20);
actions = new List <Action>();
for (var i = 0; i < 100; i++)
{
actions.Add(selectAction);
}
//Check that the control connection is using first host
//bring back some nodes
actions.Insert(3, () => nonShareableTestCluster.Start(3));
actions.Insert(50, () => nonShareableTestCluster.Start(2));
actions.Insert(50, () => nonShareableTestCluster.Start(1));
//Execute in parallel more than 100 actions
Trace.TraceInformation("Start invoking with restart nodes");
Parallel.Invoke(parallelOptions, actions.ToArray());
//Wait for the nodes to be restarted
TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "1", DefaultCassandraPort, 30);
TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "2", DefaultCassandraPort, 30);
TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "3", DefaultCassandraPort, 30);
queriedHosts.Clear();
// keep querying hosts until they are all queried, or time runs out
futureDateTime = DateTime.Now.AddSeconds(120);
while ((from singleHost in queriedHosts select singleHost).Distinct().Count() < 4 && DateTime.Now < futureDateTime)
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
queriedHosts.Add(rs.Info.QueriedHost.ToString());
Thread.Sleep(50);
}
//Check that one of the restarted nodes were queried
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, queriedHosts);
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, queriedHosts);
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, queriedHosts);
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, queriedHosts);
//Check that the control connection is still using last host
StringAssert.StartsWith(nonShareableTestCluster.ClusterIpPrefix + "4", nonShareableTestCluster.Cluster.Metadata.ControlConnection.BindAddress.ToString());
}