public void TokenAware_Guid_NoHops()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.CreateNew(3);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()));
testCluster.InitClient();
// Test
var session = testCluster.Session;
string uniqueTableName = TestUtils.GetUniqueTableName();
policyTestTools.CreateSchema(session);
session.Execute(String.Format("CREATE TABLE {0} (k uuid PRIMARY KEY, i int)", uniqueTableName));
var traces = new List <QueryTrace>();
for (var i = 0; i < 10; i++)
{
var key = Guid.NewGuid();
var statement = new SimpleStatement(String.Format("INSERT INTO " + uniqueTableName + " (k, i) VALUES ({0}, {1})", key, i))
.SetRoutingKey(
new RoutingKey()
{
RawRoutingKey = TypeSerializer.GuidShuffle(key.ToByteArray())
})
.EnableTracing();
var rs = session.Execute(statement);
traces.Add(rs.Info.QueryTrace);
}
//Check that there weren't any hops
foreach (var t in traces)
{
//The coordinator must be the only one executing the query
Assert.True(t.Events.All(e => e.Source.ToString() == t.Coordinator.ToString()), "There were trace events from another host for coordinator " + t.Coordinator);
}
}
public void StopForce_With_Inflight_Requests(bool useStreamMode)
{
var testCluster = TestClusterManager.CreateNew(2);
const int connectionLength = 4;
var builder = Cluster.Builder()
.AddContactPoint(testCluster.InitialContactPoint)
.WithPoolingOptions(new PoolingOptions()
.SetCoreConnectionsPerHost(HostDistance.Local, connectionLength)
.SetMaxConnectionsPerHost(HostDistance.Local, connectionLength)
.SetHeartBeatInterval(0))
.WithRetryPolicy(AlwaysIgnoreRetryPolicy.Instance)
.WithSocketOptions(new SocketOptions().SetReadTimeoutMillis(0).SetStreamMode(useStreamMode))
.WithLoadBalancingPolicy(new RoundRobinPolicy());
using (var cluster = builder.Build())
{
var session = (Session)cluster.Connect();
session.Execute(string.Format(TestUtils.CreateKeyspaceSimpleFormat, "ks1", 2));
session.Execute("CREATE TABLE ks1.table1 (id1 int, id2 int, PRIMARY KEY (id1, id2))");
var ps = session.Prepare("INSERT INTO ks1.table1 (id1, id2) VALUES (?, ?)");
var t = ExecuteMultiple(testCluster, session, ps, false, 1, 100);
t.Wait();
Assert.AreEqual(2, t.Result.Length, "The 2 hosts must have been used");
// Wait for all connections to be opened
Thread.Sleep(1000);
var hosts = cluster.AllHosts().ToArray();
TestHelper.WaitUntil(() =>
hosts.Sum(h => session
.GetOrCreateConnectionPool(h, HostDistance.Local)
.OpenConnections
) == hosts.Length * connectionLength);
Assert.AreEqual(
hosts.Length * connectionLength,
hosts.Sum(h => session.GetOrCreateConnectionPool(h, HostDistance.Local).OpenConnections));
ExecuteMultiple(testCluster, session, ps, true, 8000, 200000).Wait();
}
}
public void HeartbeatShouldDetectNodeDown()
{
//Execute a couple of time
//Kill connections the node silently
//Do nothing for a while
//Check if the node is considered as down
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1);
var cluster = Cluster.Builder()
.AddContactPoint(testCluster.InitialContactPoint)
.WithPoolingOptions(
new PoolingOptions()
.SetCoreConnectionsPerHost(HostDistance.Local, 2)
.SetHeartBeatInterval(500))
.WithReconnectionPolicy(new ConstantReconnectionPolicy(Int32.MaxValue))
.Build();
var session = (Session)cluster.Connect();
for (var i = 0; i < 6; i++)
{
session.Execute("SELECT * FROM system.local");
}
var host = cluster.AllHosts().First();
var pool = session.GetOrCreateConnectionPool(host, HostDistance.Local);
Trace.TraceInformation("Killing connections");
foreach (var c in pool.OpenConnections)
{
c.Kill();
}
Trace.TraceInformation("Waiting");
for (var i = 0; i < 10; i++)
{
Thread.Sleep(1000);
}
Assert.False(cluster.AllHosts().ToList()[0].IsUp);
}
public void Token_Aware_Uses_Keyspace_From_Statement_To_Determine_Replication()
{
var testCluster = TestClusterManager.CreateNew(3, new TestClusterOptions {
UseVNodes = true
});
var cluster = Cluster.Builder().AddContactPoint(testCluster.InitialContactPoint).Build();
try
{
// Connect without a keyspace
var session = cluster.Connect();
session.Execute("CREATE KEYSPACE ks_tap_stmt_ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 2}");
session.Execute("CREATE TABLE ks_tap_stmt_ks.tbl1 (id uuid primary key)");
var ps = session.Prepare("INSERT INTO ks_tap_stmt_ks.tbl1 (id) VALUES (?)");
var id = Guid.NewGuid();
var coordinators = new HashSet <IPEndPoint>();
for (var i = 0; i < 20; i++)
{
var rs = session.Execute(ps.Bind(id));
coordinators.Add(rs.Info.QueriedHost);
}
// There should be exactly 2 different coordinators for a given token
Assert.AreEqual(2, coordinators.Count);
// Manually calculate the routing key
var routingKey = Serializer.Default.Serialize(id);
// Get the replicas
var replicas = cluster.GetReplicas("ks_tap_stmt_ks", routingKey);
Assert.AreEqual(2, replicas.Count);
CollectionAssert.AreEquivalent(replicas.Select(h => h.Address), coordinators);
}
finally
{
cluster.Dispose();
testCluster.Remove();
}
}
public void TokenAware_BindInt_NoHops()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetTestCluster(3);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()));
testCluster.InitClient();
// Test
var session = testCluster.Session;
policyTestTools.TableName = TestUtils.GetUniqueTableName();
policyTestTools.CreateSchema(session, 1);
var traces = new List <QueryTrace>();
var pstmt = session.Prepare("INSERT INTO " + policyTestTools.TableName + " (k, i) VALUES (?, ?)");
for (var i = (int)short.MinValue; i < short.MinValue + 40; i++)
{
var partitionKey = BitConverter.GetBytes(i).Reverse().ToArray();
var statement = pstmt
.Bind(i, i)
.SetRoutingKey(new RoutingKey()
{
RawRoutingKey = partitionKey
})
.EnableTracing();
var rs = session.Execute(statement);
traces.Add(rs.Info.QueryTrace);
}
//Check that there weren't any hops
foreach (var t in traces)
{
//The coordinator must be the only one executing the query
Assert.True(t.Events.All(e => e.Source.ToString() == t.Coordinator.ToString()), "There were trace events from another host for coordinator " + t.Coordinator);
}
}
public void TableMetadataClusteringOrderTest()
{
string keyspaceName = TestUtils.GetUniqueKeyspaceName();
string tableName = TestUtils.GetUniqueTableName().ToLower();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(DefaultNodeCount);
var cluster = testCluster.Cluster;
var session = testCluster.Session;
var cql = "CREATE TABLE " + tableName + " (" +
@"a text,
b int,
c text,
d text,
f text,
g text,
h timestamp,
PRIMARY KEY ((a, b), c, d)
) WITH CLUSTERING ORDER BY (c ASC, d DESC);
";
session.CreateKeyspace(keyspaceName);
session.ChangeKeyspace(keyspaceName);
session.Execute(cql);
session.Execute("INSERT INTO " + tableName + " (a, b, c, d) VALUES ('1', 2, '3', '4')");
var rs = session.Execute("select * from " + tableName);
Assert.True(rs.GetRows().Count() == 1);
var table = cluster.Metadata
.GetKeyspace(keyspaceName)
.GetTableMetadata(tableName);
Assert.NotNull(table);
Assert.True(table.TableColumns.Count() == 7);
Assert.AreEqual("a, b", String.Join(", ", table.PartitionKeys.Select(p => p.Name)));
}
public void Should_Schedule_Reconnections_In_The_Background()
{
var lbp = new RoundRobinPolicy();
var config = new Configuration(
new Cassandra.Policies(lbp, new ConstantReconnectionPolicy(1000), FallthroughRetryPolicy.Instance),
new ProtocolOptions(),
null,
new SocketOptions(),
new ClientOptions(),
NoneAuthProvider.Instance,
null,
new QueryOptions(),
new DefaultAddressTranslator());
var testCluster = TestClusterManager.GetNonShareableTestCluster(1, DefaultMaxClusterCreateRetries, true, false);
var metadata = new Metadata(config);
metadata.AddHost(new IPEndPoint(IPAddress.Parse(testCluster.InitialContactPoint), ProtocolOptions.DefaultPort));
var clusterMock = new Mock <ICluster>();
clusterMock.Setup(c => c.AllHosts()).Returns(() => metadata.Hosts.ToCollection());
lbp.Initialize(clusterMock.Object);
using (var cc = NewInstance(testCluster, config, metadata))
{
cc.Init().Wait(InitTimeout);
var host = metadata.Hosts.First();
testCluster.Stop(1);
host.SetDown();
Thread.Sleep(2000);
Assert.False(host.IsUp);
testCluster.Start(1);
host.BringUpIfDown();
//Should reconnect using timer
Thread.Sleep(5000);
Assert.DoesNotThrow(() => cc.Query("SELECT key FROM system.local", false));
}
testCluster.ShutDown();
}
public virtual void OneTimeSetUp()
{
if (_reuse && _reusableInstance != null && ReferenceEquals(_reusableInstance, TestClusterManager.LastInstance))
{
Trace.WriteLine("Reusing single node ccm instance");
TestCluster = _reusableInstance;
}
else
{
TestCluster = TestClusterManager.CreateNew(AmountOfNodes);
if (_reuse)
{
_reusableInstance = TestCluster;
}
else
{
_reusableInstance = null;
}
}
if (CreateSession)
{
Cluster = Cluster.Builder().AddContactPoint(TestCluster.InitialContactPoint)
.WithQueryTimeout(60000)
.WithSocketOptions(new SocketOptions().SetConnectTimeoutMillis(30000))
.Build();
Session = (Session)Cluster.Connect();
Session.CreateKeyspace(KeyspaceName, null, false);
Session.ChangeKeyspace(KeyspaceName);
if (SetupQueries != null)
{
foreach (var query in SetupQueries)
{
Session.Execute(query);
}
}
}
}
public void ReconnectionRecyclesPool()
{
var policy = new ConstantReconnectionPolicy(5000);
var nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(2, 1, true, false);
nonShareableTestCluster.Builder = new Builder().WithReconnectionPolicy(policy);
nonShareableTestCluster.InitClient(); // this will replace the existing session using the newly assigned Builder instance
var session = (Session)nonShareableTestCluster.Session;
var hosts = new List <IPEndPoint>();
for (var i = 0; i < 50; i++)
{
var rs = session.Execute("SELECT * FROM system.local");
if (i == 20)
{
nonShareableTestCluster.StopForce(2);
}
else if (i == 30)
{
nonShareableTestCluster.Start(2);
Thread.Sleep(5000);
}
hosts.Add(rs.Info.QueriedHost);
}
var pool = session.GetOrCreateConnectionPool(TestHelper.CreateHost(nonShareableTestCluster.InitialContactPoint), HostDistance.Local);
var connections = pool.OpenConnections.ToArray();
var expectedCoreConnections = nonShareableTestCluster.Cluster.Configuration
.GetPoolingOptions((byte)session.BinaryProtocolVersion)
.GetCoreConnectionsPerHost(HostDistance.Local);
Assert.AreEqual(expectedCoreConnections, connections.Length);
Assert.True(connections.All(c => !c.IsClosed));
}
public void Should_Throw_NoHostAvailableException_When_All_Hosts_Down()
{
var testCluster = TestClusterManager.GetNonShareableTestCluster(2, 1, true, false);
var socketOptions = new SocketOptions().SetReadTimeoutMillis(3000);
var builder = Cluster.Builder().AddContactPoint(testCluster.InitialContactPoint)
.WithSocketOptions(socketOptions);
using (var cluster = builder.Build())
{
var session = cluster.Connect();
//warmup
TestHelper.Invoke(() => session.Execute("SELECT key FROM system.local"), 10);
testCluster.PauseNode(1);
testCluster.PauseNode(2);
var ex = Assert.Throws <NoHostAvailableException>(() => session.Execute("SELECT key FROM system.local"));
Assert.AreEqual(2, ex.Errors.Count);
foreach (var innerException in ex.Errors.Values)
{
Assert.IsInstanceOf <OperationTimedOutException>(innerException);
}
testCluster.ResumeNode(1);
testCluster.ResumeNode(2);
}
}
public void SetupFixture()
{
if (CassandraVersion < Version.Parse("2.2"))
{
Assert.Ignore("Requires Cassandra version >= 2.2");
}
Diagnostics.CassandraTraceSwitch.Level = TraceLevel.Info;
_testCluster = TestClusterManager.CreateNew(1, new TestClusterOptions
{
CassandraYaml = new[]
{
"batch_size_warn_threshold_in_kb:5",
"batch_size_fail_threshold_in_kb:50"
},
//Using a mirroring handler, the server will reply providing the same payload that was sent
JvmArgs = new[] { "-Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler" }
});
_testCluster.InitClient();
Session = _testCluster.Session;
Session.Execute(String.Format(TestUtils.CreateKeyspaceSimpleFormat, Keyspace, 1));
Session.Execute(String.Format(TestUtils.CreateTableSimpleFormat, Table));
}
/// <summary>
/// Gets the latest protocol version depending on the Cassandra Version running the tests
/// </summary>
public ProtocolVersion GetProtocolVersion()
{
if (TestClusterManager.CheckDseVersion(Version.Parse("6.0"), Comparison.GreaterThanOrEqualsTo))
{
return(ProtocolVersion.DseV2);
}
var cassandraVersion = TestClusterManager.CassandraVersion;
var protocolVersion = ProtocolVersion.V1;
if (cassandraVersion >= Version.Parse("2.2"))
{
protocolVersion = ProtocolVersion.V4;
}
else if (cassandraVersion >= Version.Parse("2.1"))
{
protocolVersion = ProtocolVersion.V3;
}
else if (cassandraVersion > Version.Parse("2.0"))
{
protocolVersion = ProtocolVersion.V2;
}
return(protocolVersion);
}
public async Task Should_Add_And_Query_Newly_Bootstrapped_Node()
{
_realCluster = TestClusterManager.CreateNew();
using (var cluster = Cluster.Builder()
.WithSocketOptions(new SocketOptions().SetReadTimeoutMillis(22000).SetConnectTimeoutMillis(60000))
.AddContactPoint(_realCluster.InitialContactPoint)
.Build())
{
await Connect(cluster, false, session =>
{
Assert.AreEqual(1, cluster.AllHosts().Count);
_realCluster.BootstrapNode(2);
Trace.TraceInformation("Node bootstrapped");
var newNodeAddress = _realCluster.ClusterIpPrefix + 2;
var newNodeIpAddress = IPAddress.Parse(newNodeAddress);
TestHelper.RetryAssert(() =>
{
Assert.True(TestUtils.IsNodeReachable(newNodeIpAddress));
//New node should be part of the metadata
Assert.AreEqual(2, cluster.AllHosts().Count);
var host = cluster.AllHosts().FirstOrDefault(h => h.Address.Address.Equals(newNodeIpAddress));
Assert.IsNotNull(host);
},
2000,
90);
TestHelper.RetryAssert(() =>
{
var rs = session.Execute("SELECT key FROM system.local");
Assert.True(rs.Info.QueriedHost.Address.ToString() == newNodeAddress, "Newly bootstrapped node should be queried");
},
1,
100);
}).ConfigureAwait(false);
}
}
public void Should_Reconnect_After_Several_Failed_Attempts()
{
var lbp = new RoundRobinPolicy();
var config = new Configuration(
new Cassandra.Policies(lbp, new ConstantReconnectionPolicy(1000), FallthroughRetryPolicy.Instance),
new ProtocolOptions(),
null,
new SocketOptions(),
new ClientOptions(),
NoneAuthProvider.Instance,
null,
new QueryOptions(),
new DefaultAddressTranslator());
config.BufferPool = new Microsoft.IO.RecyclableMemoryStreamManager();
var testCluster = TestClusterManager.GetNonShareableTestCluster(1, DefaultMaxClusterCreateRetries, true, false);
var metadata = new Metadata(config);
metadata.AddHost(new IPEndPoint(IPAddress.Parse(testCluster.InitialContactPoint), ProtocolOptions.DefaultPort));
var clusterMock = new Mock <ICluster>();
clusterMock.Setup(c => c.AllHosts()).Returns(() => metadata.Hosts.ToCollection());
lbp.Initialize(clusterMock.Object);
using (var cc = NewInstance(testCluster, config))
{
cc.Init();
testCluster.Stop(1);
Assert.Throws <NoHostAvailableException>(() => TaskHelper.WaitToComplete(cc.Reconnect()));
Assert.Throws <NoHostAvailableException>(() => TaskHelper.WaitToComplete(cc.Reconnect()));
Assert.Throws <NoHostAvailableException>(() => TaskHelper.WaitToComplete(cc.Reconnect()));
Assert.Throws <NoHostAvailableException>(() => TaskHelper.WaitToComplete(cc.Reconnect()));
testCluster.Start(1);
Assert.DoesNotThrow(() => TaskHelper.WaitToComplete(cc.Reconnect()));
}
testCluster.ShutDown();
}
public void TokenAwareTest(bool usePrepared)
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1, 1, DefaultMaxClusterCreateRetries, true);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()));
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session);
//clusterInfo.Cluster.RefreshSchema();
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
// Not the best test ever, we should use OPP and check we do it the
// right nodes. But since M3P is hard-coded for now, let just check
// we just hit only one node.
int nodePosToDecommission = 2;
int nodePositionToNotDecommission = 1;
if (policyTestTools.Coordinators.ContainsKey(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort))
{
nodePosToDecommission = 1;
nodePositionToNotDecommission = 2;
}
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePosToDecommission + ":" + DefaultCassandraPort, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePositionToNotDecommission + ":" + DefaultCassandraPort, 0);
// now try again having stopped the node that was just queried
policyTestTools.ResetCoordinators();
testCluster.DecommissionNode(nodePosToDecommission);
TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + nodePosToDecommission + ":" + DefaultCassandraPort, testCluster.Cluster, 40);
policyTestTools.Query(testCluster, 12, usePrepared);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePosToDecommission + ":" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePositionToNotDecommission + ":" + DefaultCassandraPort, 12);
}
public void ReconnectionPolicyTest(Builder builder, long restartTime, long retryTime, long breakTime)
{
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1);
_policyTestTools.CreateSchema(testCluster.Session, 1);
_policyTestTools.InitPreparedStatement(testCluster, 12);
_policyTestTools.Query(testCluster, 12);
// Ensure a basic test works
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 12);
_policyTestTools.ResetCoordinators();
testCluster.StopForce(1);
// Start timing and ensure that the node is down
//long startTime = 0;
var startTime = Stopwatch.StartNew(); // = 0;
try
{
//startTime = System.nanoTime() / 1000000000;
_policyTestTools.Query(testCluster, 12);
Assert.Fail("Test race condition where node has not shut off quickly enough.");
}
catch (NoHostAvailableException) {}
long elapsedSeconds;
bool restarted = false;
while (true)
{
//thisTime = System.nanoTime() / 1000000000;
elapsedSeconds = startTime.ElapsedMilliseconds / 1000;
// Restart node at restartTime
if (!restarted && elapsedSeconds > restartTime)
{
testCluster.Start(1);
restarted = true;
}
// Continue testing queries each second
try
{
_policyTestTools.Query(testCluster, 12);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 12);
_policyTestTools.ResetCoordinators();
// Ensure the time when the query completes successfully is what was expected
Assert.True(retryTime - 6 < elapsedSeconds && elapsedSeconds < retryTime + 6, string.Format("Waited {0} seconds instead an expected {1} seconds wait", elapsedSeconds, retryTime));
}
catch (NoHostAvailableException)
{
Thread.Sleep(1000);
continue;
}
Thread.Sleep((int)(breakTime * 1000));
// The same query once more, just to be sure
_policyTestTools.Query(testCluster, 12);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 12);
_policyTestTools.ResetCoordinators();
// Ensure the reconnection times reset
testCluster.StopForce(1);
// Start timing and ensure that the node is down
//startTime = 0;
startTime.Reset();
try
{
//startTime = System.nanoTime() / 1000000000;
startTime.Start();
_policyTestTools.Query(testCluster, 12);
Assert.Fail("Test race condition where node has not shut off quickly enough.");
}
catch (NoHostAvailableException)
{
}
restarted = false;
while (true)
{
//elapsedSeconds = System.nanoTime() / 1000000000;
elapsedSeconds = startTime.ElapsedMilliseconds / 1000;
// Restart node at restartTime
if (!restarted && elapsedSeconds > restartTime)
{
testCluster.Start(1);
restarted = true;
}
// Continue testing queries each second
try
{
_policyTestTools.Query(testCluster, 12);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 12);
_policyTestTools.ResetCoordinators();
// Ensure the time when the query completes successfully is what was expected
Assert.True(retryTime - 3 < elapsedSeconds && elapsedSeconds < retryTime + 3, string.Format("Waited {0} seconds instead an expected {1} seconds wait", elapsedSeconds, retryTime));
}
catch (NoHostAvailableException)
{
Thread.Sleep(1000);
continue;
}
break;
}
break;
}
}
/// <summary>
/// Validate client behavior with replication three, multiple DCs
/// with load balancing policy TokenAware, DCAwareRoundRobin,
/// with retry policy DowngradingConsistencyRetryPolicy
/// after a node is taken down
///
/// @test_category consistency
/// @test_category connection:outage,retry_policy
/// @test_category load_balancing:round_robin,token_aware,dc_aware
/// </summary>
public void ReplicationFactorThree_TwoDcs_DcAware_DowngradingConsistencyRetryPolicy()
{
// Seetup
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(3, 3, DefaultMaxClusterCreateRetries, true);
testCluster.Builder = Cluster.Builder()
.WithLoadBalancingPolicy(new TokenAwarePolicy(new DCAwareRoundRobinPolicy("dc2")))
.WithRetryPolicy(DowngradingConsistencyRetryPolicy.Instance);
testCluster.InitClient();
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "1", DefaultCassandraPort, 30);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "2", DefaultCassandraPort, 30);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "3", DefaultCassandraPort, 30);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "4", DefaultCassandraPort, 30);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "5", DefaultCassandraPort, 30);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "6", DefaultCassandraPort, 30);
// Test
_policyTestTools.CreateMultiDcSchema(testCluster.Session, 3, 3);
_policyTestTools.InitPreparedStatement(testCluster, 12, ConsistencyLevel.Two);
_policyTestTools.Query(testCluster, 12, ConsistencyLevel.Two);
// Validate expected number of host / query counts
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 0);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 0);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, 4);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "5:" + DefaultCassandraPort, 4);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "6:" + DefaultCassandraPort, 4);
_policyTestTools.ResetCoordinators();
testCluster.StopForce(2);
// FIXME: This sleep is needed to allow the waitFor() to work
TestUtils.WaitForDownWithWait(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 5);
var acceptedList = new List <ConsistencyLevel>
{
ConsistencyLevel.Any,
ConsistencyLevel.One,
ConsistencyLevel.Two,
ConsistencyLevel.Quorum,
ConsistencyLevel.Three,
ConsistencyLevel.All,
ConsistencyLevel.LocalQuorum,
ConsistencyLevel.EachQuorum
};
var failList = new List <ConsistencyLevel>();
// Test successful writes
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
}
catch (Exception e)
{
Assert.Fail(String.Format("Test failed at CL.{0} with message: {1}", consistencyLevel, e.Message));
}
}
// Test successful reads
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List <string>
{
"EACH_QUORUM ConsistencyLevel is only supported for writes",
"ANY ConsistencyLevel is only supported for writes"
};
Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message));
}
}
// Test writes which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel));
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
catch (WriteTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
}
// Test reads which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel));
}
catch (ReadTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
}
}
public void ReplicationFactorThree_TwoDCs_DowngradingConsistencyRetryPolicy()
{
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(3, 3, DefaultMaxClusterCreateRetries, true);
testCluster.Builder = Cluster.Builder()
.WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()))
.WithRetryPolicy(DowngradingConsistencyRetryPolicy.Instance);
testCluster.InitClient();
_policyTestTools.CreateMultiDcSchema(testCluster.Session, 3, 3);
_policyTestTools.InitPreparedStatement(testCluster, 12, ConsistencyLevel.Two);
// a maximum of 4 IPs should have returned values for the query -- two copies per each of the two DCs
int queriesPerIteration = 4;
int queriesCompleted = 0;
int actualTries = 0;
int maxTries = 20;
while (_policyTestTools.Coordinators.Count() < 4 && actualTries < maxTries)
{
_policyTestTools.Query(testCluster, queriesPerIteration, ConsistencyLevel.Two);
queriesCompleted += queriesPerIteration;
}
Assert.IsTrue(_policyTestTools.Coordinators.Count() >= 4, "The minimum number of hosts queried was not met!");
int totalQueriesForAllHosts = _policyTestTools.Coordinators.Sum(c => c.Value);
Assert.AreEqual(queriesCompleted, totalQueriesForAllHosts,
"The sum of queries for all hosts should equal the number of queries recorded by the calling test!");
_policyTestTools.ResetCoordinators();
testCluster.StopForce(2);
// FIXME: This sleep is needed to allow the waitFor() to work
TestUtils.WaitForDownWithWait(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 5);
var acceptedList = new List <ConsistencyLevel>
{
ConsistencyLevel.Any,
ConsistencyLevel.One,
ConsistencyLevel.Two,
ConsistencyLevel.Quorum,
ConsistencyLevel.Three,
ConsistencyLevel.All,
ConsistencyLevel.LocalQuorum,
ConsistencyLevel.EachQuorum
};
var failList = new List <ConsistencyLevel>();
// Test successful writes
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
}
catch (Exception e)
{
Assert.Fail(String.Format("Test failed at CL.{0} with message: {1}", consistencyLevel, e.Message));
}
}
// Test successful reads
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List <string>
{
"EACH_QUORUM ConsistencyLevel is only supported for writes",
"ANY ConsistencyLevel is only supported for writes"
};
Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message));
}
}
// Test writes which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
Assert.Fail("Expected Exception was not thrown for ConsistencyLevel :" + consistencyLevel);
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
catch (WriteTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
}
// Test reads which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel));
}
catch (ReadTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
}
}
public void ReplicationFactorTwo_DowngradingConsistencyRetryPolicy()
{
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(3);
testCluster.Builder = Cluster.Builder()
.WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()))
.WithRetryPolicy(DowngradingConsistencyRetryPolicy.Instance);
testCluster.InitClient();
_policyTestTools.CreateSchema(testCluster.Session, 2);
_policyTestTools.InitPreparedStatement(testCluster, 12, ConsistencyLevel.Two);
_policyTestTools.Query(testCluster, 12, ConsistencyLevel.Two);
string coordinatorHostQueried = _policyTestTools.Coordinators.First().Key.Split(':').First();;
int awareCoord = int.Parse(coordinatorHostQueried.Split('.').Last());
int coordinatorsWithMoreThanZeroQueries = 0;
foreach (var coordinator in _policyTestTools.Coordinators)
{
coordinatorsWithMoreThanZeroQueries++;
_policyTestTools.AssertQueried(coordinator.Key.ToString(), 6);
}
Assert.AreEqual(2, coordinatorsWithMoreThanZeroQueries);
_policyTestTools.ResetCoordinators();
testCluster.StopForce(awareCoord);
TestUtils.WaitForDownWithWait(testCluster.ClusterIpPrefix + awareCoord, testCluster.Cluster, 30);
var acceptedList = new List <ConsistencyLevel>
{
ConsistencyLevel.Any,
ConsistencyLevel.One,
ConsistencyLevel.Two,
ConsistencyLevel.Quorum,
ConsistencyLevel.Three,
ConsistencyLevel.All
};
var failList = new List <ConsistencyLevel>();
// Test successful writes
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
}
catch (Exception e)
{
Assert.Fail(String.Format("Test failed at CL.{0} with message: {1}", consistencyLevel, e.Message));
}
}
// Test successful reads
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List <string>
{
"ANY ConsistencyLevel is only supported for writes"
};
Assert.True(acceptableErrorMessages.Contains(e.Message));
}
}
// Test writes which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel));
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List <string>
{
"consistency level LOCAL_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)",
"consistency level EACH_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)"
};
Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message));
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
catch (WriteTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
}
// Test reads which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel));
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List <string>
{
"consistency level LOCAL_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)",
"EACH_QUORUM ConsistencyLevel is only supported for writes"
};
Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message));
}
catch (ReadTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
}
}
public void RoundRobin_TwoDCs_EachDcHasOneNodeAddedAndDecommissioned()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1, 1, DefaultMaxClusterCreateRetries, true);
testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new RoundRobinPolicy());
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session);
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
// Validate that all host were queried equally
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6);
// Add new node to the end of first cluster, remove node from beginning of first cluster
policyTestTools.ResetCoordinators();
// Bootstrap step
testCluster.BootstrapNode(3, "dc1");
string newlyBootstrappedIp = testCluster.ClusterIpPrefix + "3";
TestUtils.WaitForUp(newlyBootstrappedIp, DefaultCassandraPort, 30);
// Validate expected nodes where queried
policyTestTools.WaitForPolicyToolsQueryToHitBootstrappedIp(testCluster, newlyBootstrappedIp);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 4);
// Remove node from beginning of first cluster
policyTestTools.ResetCoordinators();
testCluster.DecommissionNode(1);
TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, testCluster.Cluster, 20);
// Validate expected nodes where queried
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 6);
// Add new node to the end of second cluster, remove node from beginning of first cluster
policyTestTools.ResetCoordinators();
testCluster.BootstrapNode(4, "dc2");
newlyBootstrappedIp = testCluster.ClusterIpPrefix + "4";
TestUtils.WaitForUp(newlyBootstrappedIp, DefaultCassandraPort, 30);
policyTestTools.ResetCoordinators();
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 4);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, 4);
// Remove node from beginning of second cluster
policyTestTools.ResetCoordinators();
testCluster.DecommissionNode(2);
TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 20);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, 6);
}
public void SetupFixture()
{
_testCluster = TestClusterManager.CreateNew();
}
public void FailoverThenReconnect()
{
var parallelOptions = new ParallelOptions
{
TaskScheduler = new ThreadPerTaskScheduler(),
MaxDegreeOfParallelism = 100
};
var policy = new ConstantReconnectionPolicy(500);
var nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(4, 1, true, false);
using (var cluster = Cluster.Builder().AddContactPoint(nonShareableTestCluster.InitialContactPoint).WithReconnectionPolicy(policy).Build())
{
var session = cluster.Connect();
// Check query to host distribution before killing nodes
var queriedHosts = new List <string>();
DateTime futureDateTime = DateTime.Now.AddSeconds(120);
while ((from singleHost in queriedHosts select singleHost).Distinct().Count() < 4 && DateTime.Now < futureDateTime)
{
var rs = session.Execute("SELECT * FROM system.local");
queriedHosts.Add(rs.Info.QueriedHost.ToString());
Thread.Sleep(50);
}
Assert.AreEqual(4, (from singleHost in queriedHosts select singleHost).Distinct().Count(), "All hosts should have been queried!");
// Create list of actions
Action selectAction = () =>
{
var rs = session.Execute("SELECT * FROM system.local");
Assert.Greater(rs.Count(), 0);
};
var actions = new List <Action>();
for (var i = 0; i < 100; i++)
{
actions.Add(selectAction);
//Check that the control connection is using first host
StringAssert.StartsWith(nonShareableTestCluster.ClusterIpPrefix + "1", nonShareableTestCluster.Cluster.Metadata.ControlConnection.Address.ToString());
//Kill some nodes
//Including the one used by the control connection
actions.Insert(20, () => nonShareableTestCluster.Stop(1));
actions.Insert(20, () => nonShareableTestCluster.Stop(2));
actions.Insert(80, () => nonShareableTestCluster.Stop(3));
//Execute in parallel more than 100 actions
Parallel.Invoke(parallelOptions, actions.ToArray());
//Wait for the nodes to be killed
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "1", nonShareableTestCluster.Cluster, 20);
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "2", nonShareableTestCluster.Cluster, 20);
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "3", nonShareableTestCluster.Cluster, 20);
actions = new List <Action>();
for (var j = 0; j < 100; j++)
{
actions.Add(selectAction);
}
//Check that the control connection is using first host
//bring back some nodes
actions.Insert(3, () => nonShareableTestCluster.Start(3));
actions.Insert(50, () => nonShareableTestCluster.Start(2));
actions.Insert(50, () => nonShareableTestCluster.Start(1));
//Execute in parallel more than 100 actions
Trace.TraceInformation("Start invoking with restart nodes");
Parallel.Invoke(parallelOptions, actions.ToArray());
//Wait for the nodes to be restarted
TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "1", DefaultCassandraPort, 30);
TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "2", DefaultCassandraPort, 30);
TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "3", DefaultCassandraPort, 30);
queriedHosts.Clear();
// keep querying hosts until they are all queried, or time runs out
futureDateTime = DateTime.Now.AddSeconds(120);
while ((from singleHost in queriedHosts select singleHost).Distinct().Count() < 4 && DateTime.Now < futureDateTime)
{
var rs = session.Execute("SELECT * FROM system.local");
queriedHosts.Add(rs.Info.QueriedHost.ToString());
Thread.Sleep(50);
}
//Check that one of the restarted nodes were queried
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, queriedHosts);
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, queriedHosts);
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, queriedHosts);
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, queriedHosts);
//Check that the control connection is still using last host
StringAssert.StartsWith(nonShareableTestCluster.ClusterIpPrefix + "4", nonShareableTestCluster.Cluster.Metadata.ControlConnection.Address.ToString());
}
}
}
public void SetupFixture()
{
_session = TestClusterManager.GetTestCluster(1).Session;
}
public void Should_UseNewHostInQueryPlans_When_HostIsDecommissionedAndJoinsAgain()
{
var testCluster = _realCluster.Value;
using (var cluster =
Cluster.Builder()
.AddContactPoint(testCluster.InitialContactPoint)
.WithSocketOptions(new SocketOptions().SetReadTimeoutMillis(22000).SetConnectTimeoutMillis(60000))
.WithPoolingOptions(
new PoolingOptions()
.SetCoreConnectionsPerHost(HostDistance.Local, 2)
.SetMaxConnectionsPerHost(HostDistance.Local, 2))
.Build())
{
var session = (IInternalSession)cluster.Connect();
session.CreateKeyspaceIfNotExists("testks");
session.ChangeKeyspace("testks");
session.Execute("CREATE TABLE test_table (id text, PRIMARY KEY (id))");
// Assert that there are 2 pools, one for each host
var hosts = session.Cluster.AllHosts().ToList();
var pool1 = session.GetExistingPool(hosts[0].Address);
Assert.AreEqual(2, pool1.OpenConnections);
var pool2 = session.GetExistingPool(hosts[1].Address);
Assert.AreEqual(2, pool2.OpenConnections);
// Assert that both hosts are used in queries
var set = new HashSet <IPEndPoint>();
foreach (var i in Enumerable.Range(1, 100))
{
var rs = session.Execute($"INSERT INTO test_table(id) VALUES ('{i}')");
set.Add(rs.Info.QueriedHost);
}
Assert.AreEqual(2, set.Count);
// Decommission node
if (!TestClusterManager.SupportsDecommissionForcefully())
{
testCluster.DecommissionNode(1);
}
else
{
testCluster.DecommissionNodeForcefully(1);
}
testCluster.Stop(1);
// Assert that only one host is used in queries
set.Clear();
foreach (var i in Enumerable.Range(1, 100))
{
var rs = session.Execute($"INSERT INTO test_table(id) VALUES ('{i}')");
set.Add(rs.Info.QueriedHost);
}
Assert.AreEqual(1, set.Count);
var removedHost = hosts.Single(h => !h.Address.Equals(set.First()));
// Bring back the decommissioned node
testCluster.Start(1, "--jvm_arg=\"-Dcassandra.override_decommission=true\"");
// Assert that there are 2 hosts
TestHelper.RetryAssert(() =>
{
Assert.AreEqual(2, cluster.AllHosts().Count);
}, 1000, 180);
// Assert that queries use both hosts again
set.Clear();
var idx = 1;
TestHelper.RetryAssert(() =>
{
var rs = session.Execute($"INSERT INTO test_table(id) VALUES ('{idx++}')");
set.Add(rs.Info.QueriedHost);
Assert.AreEqual(2, set.Count);
}, 500, 240);
TestHelper.RetryAssert(() =>
{
pool2 = session.GetExistingPool(removedHost.Address);
Assert.IsNotNull(pool2);
Assert.AreEqual(2, pool2.OpenConnections);
}, 500, 60);
}
}
public void SetupFixture()
{
// we just need to make sure that there is a query-able cluster
TestClusterManager.GetTestCluster(1, DefaultMaxClusterCreateRetries, true, false);
}
public void Executions_After_Reconnection_Resizes_Pool()
{
var testCluster = TestClusterManager.CreateNew(2);
using (var cluster = Cluster.Builder()
.AddContactPoint(testCluster.InitialContactPoint)
.WithPoolingOptions(
new PoolingOptions()
.SetCoreConnectionsPerHost(HostDistance.Local, 2)
.SetMaxConnectionsPerHost(HostDistance.Local, 2)
.SetHeartBeatInterval(0))
.WithReconnectionPolicy(new ConstantReconnectionPolicy(1000))
.Build())
{
var session1 = (Session)cluster.Connect();
var session2 = (Session)cluster.Connect();
TestHelper.Invoke(() => session1.Execute("SELECT * FROM system.local"), 10);
TestHelper.Invoke(() => session2.Execute("SELECT * FROM system.local"), 10);
var host1 = cluster.AllHosts().First(h => TestHelper.GetLastAddressByte(h) == 1);
var host2 = cluster.AllHosts().First(h => TestHelper.GetLastAddressByte(h) == 2);
var upCounter = new ConcurrentDictionary <byte, int>();
var downCounter = new ConcurrentDictionary <byte, int>();
cluster.Metadata.HostsEvent += (sender, e) =>
{
if (e.What == HostsEventArgs.Kind.Up)
{
upCounter.AddOrUpdate(TestHelper.GetLastAddressByte(e.Address), 1, (k, v) => ++ v);
return;
}
downCounter.AddOrUpdate(TestHelper.GetLastAddressByte(e.Address), 1, (k, v) => ++ v);
};
Assert.True(host1.IsUp);
Assert.True(host2.IsUp);
Trace.TraceInformation("Stopping node #1");
testCluster.Stop(1);
Trace.TraceInformation("Stopping node #2");
testCluster.Stop(2);
//Force to be considered down
Assert.Throws <NoHostAvailableException>(() => session1.Execute("SELECT * FROM system.local"));
Assert.AreEqual(1, downCounter.GetOrAdd(1, 0));
Assert.AreEqual(0, upCounter.GetOrAdd(1, 0));
Assert.AreEqual(1, downCounter.GetOrAdd(2, 0));
Assert.AreEqual(0, upCounter.GetOrAdd(2, 0));
Trace.TraceInformation("Restarting node #1");
testCluster.Start(1);
Trace.TraceInformation("Restarting node #2");
testCluster.Start(2);
Trace.TraceInformation("Waiting for few more seconds");
Thread.Sleep(6000);
Assert.True(host1.IsUp);
Assert.True(host2.IsUp);
Assert.AreEqual(1, downCounter.GetOrAdd(1, 0));
Assert.AreEqual(1, upCounter.GetOrAdd(1, 0));
Assert.AreEqual(1, downCounter.GetOrAdd(2, 0));
Assert.AreEqual(1, upCounter.GetOrAdd(2, 0));
TestHelper.Invoke(() => session1.Execute("SELECT * FROM system.local"), 10);
TestHelper.Invoke(() => session2.Execute("SELECT * FROM system.local"), 10);
Trace.TraceInformation("Waiting for few more seconds");
Thread.Sleep(6000);
var pool1 = session1.GetOrCreateConnectionPool(host1, HostDistance.Local);
Assert.AreEqual(2, pool1.OpenConnections);
var pool2 = session1.GetOrCreateConnectionPool(host2, HostDistance.Local);
Assert.AreEqual(2, pool2.OpenConnections);
}
}
public void Reconnection_Attempted_Multiple_Times_On_Multiple_Nodes([Range(1, 3)] int repeating)
{
var testCluster = TestClusterManager.CreateNew(2);
using (var cluster = Cluster.Builder()
.AddContactPoint(testCluster.InitialContactPoint)
.WithPoolingOptions(
new PoolingOptions()
.SetCoreConnectionsPerHost(HostDistance.Local, 2)
.SetHeartBeatInterval(0))
.WithReconnectionPolicy(new ConstantReconnectionPolicy(1000))
.Build())
{
var session = cluster.Connect();
TestHelper.Invoke(() => session.Execute("SELECT * FROM system.local"), 10);
//Another session to have multiple pools
var dummySession = cluster.Connect();
TestHelper.Invoke(() => dummySession.Execute("SELECT * FROM system.local"), 10);
var host1 = cluster.AllHosts().First(h => TestHelper.GetLastAddressByte(h) == 1);
var host2 = cluster.AllHosts().First(h => TestHelper.GetLastAddressByte(h) == 2);
var upCounter = new ConcurrentDictionary <byte, int>();
var downCounter = new ConcurrentDictionary <byte, int>();
cluster.Metadata.HostsEvent += (sender, e) =>
{
if (e.What == HostsEventArgs.Kind.Up)
{
upCounter.AddOrUpdate(TestHelper.GetLastAddressByte(e.Address), 1, (k, v) => ++ v);
return;
}
downCounter.AddOrUpdate(TestHelper.GetLastAddressByte(e.Address), 1, (k, v) => ++ v);
};
Assert.True(host1.IsUp);
Trace.TraceInformation("Stopping node #1");
testCluster.Stop(1);
// Make sure the node is considered down
Assert.DoesNotThrow(() => TestHelper.Invoke(() => session.Execute("SELECT * FROM system.local"), 6));
Thread.Sleep(1000);
Assert.False(host1.IsUp);
Assert.AreEqual(1, downCounter.GetOrAdd(1, 0));
Assert.AreEqual(0, upCounter.GetOrAdd(1, 0));
Trace.TraceInformation("Stopping node #2");
testCluster.Stop(2);
Assert.Throws <NoHostAvailableException>(() => TestHelper.Invoke(() => session.Execute("SELECT * FROM system.local"), 6));
Thread.Sleep(1000);
Assert.False(host2.IsUp);
Assert.AreEqual(1, downCounter.GetOrAdd(1, 0));
Assert.AreEqual(0, upCounter.GetOrAdd(1, 0));
Assert.AreEqual(1, downCounter.GetOrAdd(2, 0));
Assert.AreEqual(0, upCounter.GetOrAdd(2, 0));
Trace.TraceInformation("Waiting for few seconds");
Thread.Sleep(8000);
Assert.False(host1.IsUp);
Assert.False(host2.IsUp);
Trace.TraceInformation("Restarting node #1");
testCluster.Start(1);
Trace.TraceInformation("Restarting node #2");
testCluster.Start(2);
Trace.TraceInformation("Waiting for few more seconds");
TestHelper.WaitUntil(() => host1.IsUp && host2.IsUp, 1000, 20);
Assert.True(host1.IsUp, "Host 1 should be UP after restarting");
Assert.True(host2.IsUp, "Host 2 should be UP after restarting");
Assert.AreEqual(1, downCounter.GetOrAdd(1, 0));
Assert.AreEqual(1, upCounter.GetOrAdd(1, 0));
Assert.AreEqual(1, downCounter.GetOrAdd(2, 0));
Assert.AreEqual(1, upCounter.GetOrAdd(2, 0));
}
}
protected virtual ITestCluster CreateNew(int nodeLength, TestClusterOptions options, bool startCluster)
{
return(TestClusterManager.CreateNew(nodeLength, options, startCluster));
}
public void Jira_CSHARP_40()
{
ITestCluster nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(1);
var session = nonShareableTestCluster.Session;
string keyspaceName = "excelsior";
session.CreateKeyspaceIfNotExists(keyspaceName);
session.ChangeKeyspace(keyspaceName);
const string cqlQuery = "SELECT * from system.local";
var query = new SimpleStatement(cqlQuery).EnableTracing();
{
var result = session.Execute(query);
Assert.Greater(result.Count(), 0, "It should return rows");
}
nonShareableTestCluster.StopForce(1);
// now wait until node is down
bool noHostAvailableExceptionWasCaught = false;
while (!noHostAvailableExceptionWasCaught)
{
try
{
nonShareableTestCluster.Cluster.Connect();
}
catch (Exception e)
{
if (e.GetType() == typeof(NoHostAvailableException))
{
noHostAvailableExceptionWasCaught = true;
}
else
{
Trace.TraceWarning("Something other than a NoHostAvailableException was thrown: " + e.GetType() + ", waiting another second ...");
Thread.Sleep(1000);
}
}
}
// now restart the node
nonShareableTestCluster.Start(1);
bool hostWasReconnected = false;
DateTime timeInTheFuture = DateTime.Now.AddSeconds(20);
while (!hostWasReconnected && DateTime.Now < timeInTheFuture)
{
try
{
session.Execute(query);
hostWasReconnected = true;
}
catch (Exception e)
{
if (e.GetType() == typeof(NoHostAvailableException))
{
Trace.TraceInformation("Host still not up yet, waiting another one second ... ");
Thread.Sleep(1000);
}
else
{
throw e;
}
}
}
RowSet rowSet = session.Execute(query);
Assert.True(rowSet.GetRows().Count() > 0, "It should return rows");
}
public void OneTimeSetUp()
{
_testCluster = TestClusterManager.CreateNew(1, new TestClusterOptions
{
CassandraYaml = new[]
{
"authenticator: com.datastax.bdp.cassandra.auth.DseAuthenticator",
"authorizer:com.datastax.bdp.cassandra.auth.DseAuthorizer"
},
DseYaml = new []
{
"authentication_options.default_scheme: internal",
"authorization_options.enabled:true",
"authentication_options.enabled:true",
"audit_logging_options.enabled:true"
},
JvmArgs = new[] { "-Dcassandra.superuser_setup_delay_ms=0" }
});
var authProvider = new DsePlainTextAuthProvider("cassandra", "cassandra");
using (var cluster = Cluster.Builder()
.AddContactPoint(_testCluster.InitialContactPoint)
.WithAuthProvider(authProvider)
.Build())
{
var session = cluster.Connect();
var queries = new[]
{
"CREATE ROLE IF NOT EXISTS alice WITH PASSWORD = '******' AND LOGIN = FALSE",
"CREATE ROLE IF NOT EXISTS ben WITH PASSWORD = '******' AND LOGIN = TRUE",
"CREATE ROLE IF NOT EXISTS '*****@*****.**' WITH LOGIN = TRUE",
"CREATE ROLE IF NOT EXISTS '*****@*****.**' WITH PASSWORD = '******' AND LOGIN = TRUE",
"CREATE ROLE IF NOT EXISTS steve WITH PASSWORD = '******' AND LOGIN = TRUE",
"CREATE ROLE IF NOT EXISTS paul WITH PASSWORD = '******' AND LOGIN = TRUE",
"CREATE ROLE IF NOT EXISTS aliceselect WITH PASSWORD = '******' AND LOGIN = FALSE",
"CREATE KEYSPACE IF NOT EXISTS aliceks " +
"WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor':'1'}",
"CREATE TABLE IF NOT EXISTS aliceks.alicetable (key text PRIMARY KEY, value text)",
"INSERT INTO aliceks.alicetable (key, value) VALUES ('hello', 'world')",
"GRANT ALL ON KEYSPACE aliceks TO alice",
"GRANT SELECT ON KEYSPACE aliceks TO aliceselect",
"GRANT EXECUTE ON ALL AUTHENTICATION SCHEMES TO 'ben'",
"GRANT EXECUTE ON ALL AUTHENTICATION SCHEMES TO '*****@*****.**'",
"GRANT EXECUTE ON ALL AUTHENTICATION SCHEMES TO 'steve'",
"GRANT EXECUTE ON ALL AUTHENTICATION SCHEMES TO '*****@*****.**'",
"GRANT EXECUTE ON ALL AUTHENTICATION SCHEMES TO 'aliceselect'",
"GRANT PROXY.LOGIN ON ROLE 'alice' TO 'ben'",
"GRANT PROXY.LOGIN ON ROLE 'alice' TO '*****@*****.**'",
"GRANT PROXY.LOGIN ON ROLE 'alice' TO 'paul'",
"GRANT PROXY.EXECUTE ON ROLE 'alice' TO 'steve'",
"GRANT PROXY.EXECUTE ON ROLE 'alice' TO '*****@*****.**'",
"GRANT PROXY.EXECUTE ON ROLE 'alice' TO 'paul'",
"GRANT PROXY.EXECUTE ON ROLE 'aliceselect' TO 'ben'",
"GRANT PROXY.EXECUTE ON ROLE 'aliceselect' TO 'steve'",
};
foreach (var q in queries)
{
session.Execute(q);
}
}
}
