public void FailoverTest()
{
var parallelOptions = new ParallelOptions();
parallelOptions.TaskScheduler = new ThreadPerTaskScheduler();
parallelOptions.MaxDegreeOfParallelism = 100;
var policy = new ConstantReconnectionPolicy(Int32.MaxValue);
ITestCluster nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(4);
nonShareableTestCluster.Builder = new Builder().WithReconnectionPolicy(policy);
nonShareableTestCluster.InitClient(); // this will replace the existing session using the newly assigned Builder instance
var session = nonShareableTestCluster.Session;
// Check query to host distribution before killing nodes
var queriedHosts = new List <string>();
DateTime futureDateTime = DateTime.Now.AddSeconds(120);
while ((from singleHost in queriedHosts select singleHost).Distinct().Count() < 4 && DateTime.Now < futureDateTime)
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
queriedHosts.Add(rs.Info.QueriedHost.ToString());
Thread.Sleep(50);
}
Assert.AreEqual(4, (from singleHost in queriedHosts select singleHost).Distinct().Count(), "All hosts should have been queried!");
// Create List of actions
Action selectAction = () =>
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
Assert.Greater(rs.Count(), 0);
};
var actions = new List <Action>();
for (var i = 0; i < 100; i++)
{
actions.Add(selectAction);
}
//kill some nodes.
actions.Insert(20, () => nonShareableTestCluster.StopForce(1));
actions.Insert(20, () => nonShareableTestCluster.StopForce(2));
actions.Insert(80, () => nonShareableTestCluster.StopForce(3));
//Execute in parallel more than 100 actions
Parallel.Invoke(parallelOptions, actions.ToArray());
// Wait for the nodes to be killed
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "1", nonShareableTestCluster.Cluster, 20);
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "2", nonShareableTestCluster.Cluster, 20);
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "3", nonShareableTestCluster.Cluster, 20);
// Execute some more SELECTs
for (var i = 0; i < 250; i++)
{
var rowSet2 = session.Execute("SELECT * FROM system.schema_columnfamilies");
Assert.Greater(rowSet2.Count(), 0);
StringAssert.StartsWith(nonShareableTestCluster.ClusterIpPrefix + "4", rowSet2.Info.QueriedHost.ToString());
}
}
public void ReadTimeoutException()
{
ITestCluster nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(2);
ISession session = nonShareableTestCluster.Session;
string keyspace = "TestKeyspace_" + Randomm.RandomAlphaNum(10);
string table = "TestTable_" + Randomm.RandomAlphaNum(10);
int replicationFactor = 2;
string key = "1";
session.Execute(String.Format(TestUtils.CreateKeyspaceSimpleFormat, keyspace, replicationFactor));
Thread.Sleep(5000);
session.ChangeKeyspace(keyspace);
session.Execute(String.Format(TestUtils.CreateTableSimpleFormat, table));
Thread.Sleep(3000);
session.Execute(
new SimpleStatement(String.Format(TestUtils.INSERT_FORMAT, table, key, "foo", 42, 24.03f)).SetConsistencyLevel(
ConsistencyLevel.All));
session.Execute(new SimpleStatement(String.Format(TestUtils.SELECT_ALL_FORMAT, table)).SetConsistencyLevel(ConsistencyLevel.All));
nonShareableTestCluster.StopForce(2);
var ex = Assert.Throws <ReadTimeoutException>(() =>
session.Execute(new SimpleStatement(String.Format(TestUtils.SELECT_ALL_FORMAT, table)).SetConsistencyLevel(ConsistencyLevel.All)));
Assert.AreEqual(ex.ConsistencyLevel, ConsistencyLevel.All);
Assert.AreEqual(ex.ReceivedAcknowledgements, 1);
Assert.AreEqual(ex.RequiredAcknowledgements, 2);
Assert.AreEqual(ex.WasDataRetrieved, true);
}
public void HostDownViaMetadataEvents()
{
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(2);
var cluster = testCluster.Cluster;
var downEventFired = false;
cluster.Metadata.HostsEvent += (sender, e) =>
{
if (e.What == HostsEventArgs.Kind.Down)
{
downEventFired = true;
}
};
//The control connection is connected to host 1
//All host are up
Assert.True(cluster.AllHosts().All(h => h.IsUp));
testCluster.StopForce(2);
var counter = 0;
const int maxWait = 100;
//No query to avoid getting a socket exception
while (counter++ < maxWait)
{
if (cluster.AllHosts().Any(h => TestHelper.GetLastAddressByte(h) == 2 && !h.IsUp))
{
break;
}
Thread.Sleep(1000);
}
Assert.True(cluster.AllHosts().Any(h => TestHelper.GetLastAddressByte(h) == 2 && !h.IsUp));
Assert.AreNotEqual(counter, maxWait, "Waited but it was never notified via events");
Assert.True(downEventFired);
}
public void ClusterInitializationRecoversFromNoHostAvailable()
{
ITestCluster nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(1);
nonShareableTestCluster.StopForce(1);
var cluster = Cluster.Builder()
.AddContactPoint(nonShareableTestCluster.InitialContactPoint)
.Build();
//initially it will throw as there is no node reachable
Assert.Throws <NoHostAvailableException>(() => cluster.Connect());
// wait for the node to be up
nonShareableTestCluster.Start(1);
DateTime timeInTheFuture = DateTime.Now.AddSeconds(60);
bool clusterIsUp = false;
while (!clusterIsUp && DateTime.Now < timeInTheFuture)
{
try
{
cluster.Connect();
clusterIsUp = true;
}
catch (NoHostAvailableException) { }
}
//Now the node is ready to accept connections
var session = cluster.Connect("system");
TestHelper.ParallelInvoke(() => session.Execute("SELECT * from local"), 20);
}
public void WriteTimeoutException()
{
ITestCluster nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(2);
ISession session = nonShareableTestCluster.Session;
string keyspace = "TestKeyspace_" + Randomm.RandomAlphaNum(10);
string table = "TestTable_" + Randomm.RandomAlphaNum(10);
int replicationFactor = 2;
string key = "1";
session.Execute(String.Format(TestUtils.CreateKeyspaceSimpleFormat, keyspace, replicationFactor));
session.ChangeKeyspace(keyspace);
session.Execute(String.Format(TestUtils.CreateTableSimpleFormat, table));
session.Execute(
new SimpleStatement(
String.Format(TestUtils.INSERT_FORMAT, table, key, "foo", 42, 24.03f)).SetConsistencyLevel(ConsistencyLevel.All));
session.Execute(new SimpleStatement(String.Format(TestUtils.SELECT_ALL_FORMAT, table)).SetConsistencyLevel(ConsistencyLevel.All));
nonShareableTestCluster.StopForce(2);
try
{
session.Execute(
new SimpleStatement(String.Format(TestUtils.INSERT_FORMAT, table, key, "foo", 42, 24.03f)).SetConsistencyLevel(
ConsistencyLevel.All));
}
catch (WriteTimeoutException e)
{
Assert.AreEqual(e.ConsistencyLevel, ConsistencyLevel.All);
Assert.AreEqual(1, e.ReceivedAcknowledgements);
Assert.AreEqual(2, e.RequiredAcknowledgements);
Assert.AreEqual(e.WriteType, "SIMPLE");
}
}
private Task <string[]> ExecuteMultiple(ITestCluster testCluster, Session session, PreparedStatement ps, bool stopNode, int maxConcurrency, int repeatLength)
{
var hosts = new ConcurrentDictionary <string, bool>();
var tcs = new TaskCompletionSource <string[]>();
var receivedCounter = 0L;
var sendCounter = 0L;
var currentlySentCounter = 0L;
var stopMark = repeatLength / 8L;
Action sendNew = null;
sendNew = () =>
{
var sent = Interlocked.Increment(ref sendCounter);
if (sent > repeatLength)
{
return;
}
Interlocked.Increment(ref currentlySentCounter);
var statement = ps.Bind(DateTime.Now.Millisecond, (int)sent);
var executeTask = session.ExecuteAsync(statement);
executeTask.ContinueWith(t =>
{
if (t.Exception != null)
{
tcs.TrySetException(t.Exception.InnerException);
return;
}
hosts.AddOrUpdate(t.Result.Info.QueriedHost.ToString(), true, (k, v) => v);
var received = Interlocked.Increment(ref receivedCounter);
if (stopNode && received == stopMark)
{
Task.Factory.StartNew(() =>
{
testCluster.StopForce(2);
}, TaskCreationOptions.LongRunning);
}
if (received == repeatLength)
{
// Mark this as finished
tcs.TrySetResult(hosts.Keys.ToArray());
return;
}
sendNew();
}, TaskContinuationOptions.ExecuteSynchronously);
};
for (var i = 0; i < maxConcurrency; i++)
{
sendNew();
}
return(tcs.Task);
}
public void RoundRobin_OneDc_AllNodesForceStoppedOneAtATime()
{
// Setup
PolicyTestTools policyTestTools = new PolicyTestTools();
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(2);
testCluster.Builder = Cluster.Builder()
.WithLoadBalancingPolicy(new RoundRobinPolicy())
.WithQueryTimeout(10000);
testCluster.InitClient();
policyTestTools.CreateSchema(testCluster.Session);
policyTestTools.InitPreparedStatement(testCluster, 12);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 6);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6);
policyTestTools.ResetCoordinators();
testCluster.StopForce(1);
TestUtils.WaitForDown(testCluster.ClusterIpPrefix + "1", testCluster.Cluster, 20);
policyTestTools.Query(testCluster, 12);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 12);
testCluster.StopForce(2);
TestUtils.WaitForDown(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 20);
try
{
policyTestTools.Query(testCluster, 3);
Assert.Fail("Exception should have been thrown, but wasn't!");
}
catch (NoHostAvailableException)
{
Trace.TraceInformation("Expected NoHostAvailableException exception was thrown.");
}
}
public void UnavailableException()
{
ITestCluster nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(2);
ISession session = nonShareableTestCluster.Session;
string keyspaceName = "TestKeyspace_" + Randomm.RandomAlphaNum(10);
string tableName = "TestTable_" + Randomm.RandomAlphaNum(10);
int replicationFactor = 2;
string key = "1";
session.Execute(String.Format(TestUtils.CreateKeyspaceSimpleFormat, keyspaceName, replicationFactor));
session.ChangeKeyspace(keyspaceName);
session.Execute(String.Format(TestUtils.CreateTableSimpleFormat, tableName));
session.Execute(
new SimpleStatement(String.Format(TestUtils.INSERT_FORMAT, tableName, key, "foo", 42, 24.03f)).SetConsistencyLevel(ConsistencyLevel.All));
session.Execute(new SimpleStatement(String.Format(TestUtils.SELECT_ALL_FORMAT, tableName)).SetConsistencyLevel(ConsistencyLevel.All));
nonShareableTestCluster.StopForce(2);
// Ensure that gossip has reported the node as down.
bool expectedExceptionWasCaught = false;
int readTimeoutWasCaught = 0;
int maxReadTimeouts = 6; // as long as we're getting Read Timeouts, then we're on the right track
while (!expectedExceptionWasCaught && readTimeoutWasCaught < maxReadTimeouts)
{
try
{
session.Execute(new SimpleStatement(String.Format(TestUtils.SELECT_ALL_FORMAT, tableName)).SetConsistencyLevel(ConsistencyLevel.All));
}
catch (UnavailableException e)
{
Assert.AreEqual(e.Consistency, ConsistencyLevel.All);
Assert.AreEqual(e.RequiredReplicas, replicationFactor);
Assert.AreEqual(e.AliveReplicas, replicationFactor - 1);
expectedExceptionWasCaught = true;
}
catch (ReadTimeoutException e)
{
Assert.AreEqual(e.ConsistencyLevel, ConsistencyLevel.All);
Trace.TraceInformation("We caught a ReadTimeoutException as expected, extending the total time to wait ... ");
readTimeoutWasCaught++;
}
Trace.TraceInformation("Expected exception was not thrown, trying again ... ");
}
Assert.True(expectedExceptionWasCaught,
string.Format("Expected exception {0} was not caught after {1} read timeouts were caught!", "UnavailableException", maxReadTimeouts));
}
public void MetadataMethodReconnects()
{
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(2);
var cluster = testCluster.Cluster;
//The control connection is connected to host 1
Assert.AreEqual(1, TestHelper.GetLastAddressByte(cluster.Metadata.ControlConnection.EndPoint.GetHostIpEndPointWithFallback()));
testCluster.StopForce(1);
Thread.Sleep(10000);
//The control connection is still connected to host 1
Assert.AreEqual(1, TestHelper.GetLastAddressByte(cluster.Metadata.ControlConnection.EndPoint.GetHostIpEndPointWithFallback()));
var t = cluster.Metadata.GetTable("system", "local");
Assert.NotNull(t);
//The control connection should be connected to host 2
Assert.AreEqual(2, TestHelper.GetLastAddressByte(cluster.Metadata.ControlConnection.EndPoint.GetHostIpEndPointWithFallback()));
}
public void MetadataMethodReconnects()
{
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(2);
var cluster = testCluster.Cluster;
//The control connection is connected to host 1
Assert.AreEqual(1, TestHelper.GetLastAddressByte(cluster.Metadata.ControlConnection.BindAddress));
testCluster.StopForce(1);
Thread.Sleep(10000);
//The control connection is still connected to host 1
Assert.AreEqual(1, TestHelper.GetLastAddressByte(cluster.Metadata.ControlConnection.BindAddress));
var t = cluster.Metadata.GetTable("system", "schema_columnfamilies");
Assert.NotNull(t);
//The control connection should be connected to host 2
Assert.AreEqual(2, TestHelper.GetLastAddressByte(cluster.Metadata.ControlConnection.BindAddress));
}
public void ReconnectionRecyclesPool()
{
var policy = new ConstantReconnectionPolicy(5000);
ITestCluster nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(2);
nonShareableTestCluster.Builder = new Builder().WithReconnectionPolicy(policy);
nonShareableTestCluster.InitClient(); // this will replace the existing session using the newly assigned Builder instance
var session = (Session)nonShareableTestCluster.Session;
var hosts = new List <IPEndPoint>();
for (var i = 0; i < 50; i++)
{
var rs = session.Execute("SELECT * FROM system.schema_columnfamilies");
if (i == 20)
{
nonShareableTestCluster.StopForce(2);
}
else if (i == 30)
{
nonShareableTestCluster.Start(2);
Thread.Sleep(5000);
}
hosts.Add(rs.Info.QueriedHost);
}
var pool = session.GetOrCreateConnectionPool(TestHelper.CreateHost(nonShareableTestCluster.InitialContactPoint), HostDistance.Local);
var connections = pool.OpenConnections.ToArray();
var expectedCoreConnections = nonShareableTestCluster.Cluster.Configuration
.GetPoolingOptions(connections.First().ProtocolVersion)
.GetCoreConnectionsPerHost(HostDistance.Local);
Assert.AreEqual(expectedCoreConnections, connections.Length);
Assert.True(connections.All(c => !c.IsClosed));
}
public void ReconnectionPolicyTest(Builder builder, long restartTime, long retryTime, long breakTime)
{
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1);
_policyTestTools.CreateSchema(testCluster.Session, 1);
_policyTestTools.InitPreparedStatement(testCluster, 12);
_policyTestTools.Query(testCluster, 12);
// Ensure a basic test works
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 12);
_policyTestTools.ResetCoordinators();
testCluster.StopForce(1);
// Start timing and ensure that the node is down
//long startTime = 0;
var startTime = Stopwatch.StartNew(); // = 0;
try
{
//startTime = System.nanoTime() / 1000000000;
_policyTestTools.Query(testCluster, 12);
Assert.Fail("Test race condition where node has not shut off quickly enough.");
}
catch (NoHostAvailableException) {}
long elapsedSeconds;
bool restarted = false;
while (true)
{
//thisTime = System.nanoTime() / 1000000000;
elapsedSeconds = startTime.ElapsedMilliseconds / 1000;
// Restart node at restartTime
if (!restarted && elapsedSeconds > restartTime)
{
testCluster.Start(1);
restarted = true;
}
// Continue testing queries each second
try
{
_policyTestTools.Query(testCluster, 12);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 12);
_policyTestTools.ResetCoordinators();
// Ensure the time when the query completes successfully is what was expected
Assert.True(retryTime - 6 < elapsedSeconds && elapsedSeconds < retryTime + 6, string.Format("Waited {0} seconds instead an expected {1} seconds wait", elapsedSeconds, retryTime));
}
catch (NoHostAvailableException)
{
Thread.Sleep(1000);
continue;
}
Thread.Sleep((int)(breakTime * 1000));
// The same query once more, just to be sure
_policyTestTools.Query(testCluster, 12);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 12);
_policyTestTools.ResetCoordinators();
// Ensure the reconnection times reset
testCluster.StopForce(1);
// Start timing and ensure that the node is down
//startTime = 0;
startTime.Reset();
try
{
//startTime = System.nanoTime() / 1000000000;
startTime.Start();
_policyTestTools.Query(testCluster, 12);
Assert.Fail("Test race condition where node has not shut off quickly enough.");
}
catch (NoHostAvailableException)
{
}
restarted = false;
while (true)
{
//elapsedSeconds = System.nanoTime() / 1000000000;
elapsedSeconds = startTime.ElapsedMilliseconds / 1000;
// Restart node at restartTime
if (!restarted && elapsedSeconds > restartTime)
{
testCluster.Start(1);
restarted = true;
}
// Continue testing queries each second
try
{
_policyTestTools.Query(testCluster, 12);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 12);
_policyTestTools.ResetCoordinators();
// Ensure the time when the query completes successfully is what was expected
Assert.True(retryTime - 3 < elapsedSeconds && elapsedSeconds < retryTime + 3, string.Format("Waited {0} seconds instead an expected {1} seconds wait", elapsedSeconds, retryTime));
}
catch (NoHostAvailableException)
{
Thread.Sleep(1000);
continue;
}
break;
}
break;
}
}
public void Jira_CSHARP_40()
{
ITestCluster nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(1);
var session = nonShareableTestCluster.Session;
string keyspaceName = "excelsior";
session.CreateKeyspaceIfNotExists(keyspaceName);
session.ChangeKeyspace(keyspaceName);
const string cqlQuery = "SELECT * from system.local";
var query = new SimpleStatement(cqlQuery).EnableTracing();
{
var result = session.Execute(query);
Assert.Greater(result.Count(), 0, "It should return rows");
}
nonShareableTestCluster.StopForce(1);
// now wait until node is down
bool noHostAvailableExceptionWasCaught = false;
while (!noHostAvailableExceptionWasCaught)
{
try
{
nonShareableTestCluster.Cluster.Connect();
}
catch (Exception e)
{
if (e.GetType() == typeof(NoHostAvailableException))
{
noHostAvailableExceptionWasCaught = true;
}
else
{
Trace.TraceWarning("Something other than a NoHostAvailableException was thrown: " + e.GetType() + ", waiting another second ...");
Thread.Sleep(1000);
}
}
}
// now restart the node
nonShareableTestCluster.Start(1);
bool hostWasReconnected = false;
DateTime timeInTheFuture = DateTime.Now.AddSeconds(20);
while (!hostWasReconnected && DateTime.Now < timeInTheFuture)
{
try
{
session.Execute(query);
hostWasReconnected = true;
}
catch (Exception e)
{
if (e.GetType() == typeof(NoHostAvailableException))
{
Trace.TraceInformation("Host still not up yet, waiting another one second ... ");
Thread.Sleep(1000);
}
else
{
throw e;
}
}
}
RowSet rowSet = session.Execute(query);
Assert.True(rowSet.GetRows().Count() > 0, "It should return rows");
}
/// <summary>
/// Validate client behavior with replication three, multiple DCs
/// with load balancing policy TokenAware, DCAwareRoundRobin,
/// with retry policy DowngradingConsistencyRetryPolicy
/// after a node is taken down
///
/// @test_category consistency
/// @test_category connection:outage,retry_policy
/// @test_category load_balancing:round_robin,token_aware,dc_aware
/// </summary>
public void ReplicationFactorThree_TwoDcs_DcAware_DowngradingConsistencyRetryPolicy()
{
// Seetup
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(3, 3, DefaultMaxClusterCreateRetries, true);
testCluster.Builder = Cluster.Builder()
.WithLoadBalancingPolicy(new TokenAwarePolicy(new DCAwareRoundRobinPolicy("dc2")))
.WithRetryPolicy(DowngradingConsistencyRetryPolicy.Instance);
testCluster.InitClient();
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "1", DefaultCassandraPort, 30);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "2", DefaultCassandraPort, 30);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "3", DefaultCassandraPort, 30);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "4", DefaultCassandraPort, 30);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "5", DefaultCassandraPort, 30);
TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "6", DefaultCassandraPort, 30);
// Test
_policyTestTools.CreateMultiDcSchema(testCluster.Session, 3, 3);
_policyTestTools.InitPreparedStatement(testCluster, 12, ConsistencyLevel.Two);
_policyTestTools.Query(testCluster, 12, ConsistencyLevel.Two);
// Validate expected number of host / query counts
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 0);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 0);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, 4);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "5:" + DefaultCassandraPort, 4);
_policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "6:" + DefaultCassandraPort, 4);
_policyTestTools.ResetCoordinators();
testCluster.StopForce(2);
// FIXME: This sleep is needed to allow the waitFor() to work
TestUtils.WaitForDownWithWait(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 5);
var acceptedList = new List <ConsistencyLevel>
{
ConsistencyLevel.Any,
ConsistencyLevel.One,
ConsistencyLevel.Two,
ConsistencyLevel.Quorum,
ConsistencyLevel.Three,
ConsistencyLevel.All,
ConsistencyLevel.LocalQuorum,
ConsistencyLevel.EachQuorum
};
var failList = new List <ConsistencyLevel>();
// Test successful writes
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
}
catch (Exception e)
{
Assert.Fail(String.Format("Test failed at CL.{0} with message: {1}", consistencyLevel, e.Message));
}
}
// Test successful reads
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List <string>
{
"EACH_QUORUM ConsistencyLevel is only supported for writes",
"ANY ConsistencyLevel is only supported for writes"
};
Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message));
}
}
// Test writes which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel));
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
catch (WriteTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
}
// Test reads which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel));
}
catch (ReadTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
}
}
public void ReplicationFactorThree_TwoDCs_DowngradingConsistencyRetryPolicy()
{
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(3, 3, DefaultMaxClusterCreateRetries, true);
testCluster.Builder = Cluster.Builder()
.WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()))
.WithRetryPolicy(DowngradingConsistencyRetryPolicy.Instance);
testCluster.InitClient();
_policyTestTools.CreateMultiDcSchema(testCluster.Session, 3, 3);
_policyTestTools.InitPreparedStatement(testCluster, 12, ConsistencyLevel.Two);
// a maximum of 4 IPs should have returned values for the query -- two copies per each of the two DCs
int queriesPerIteration = 4;
int queriesCompleted = 0;
int actualTries = 0;
int maxTries = 20;
while (_policyTestTools.Coordinators.Count() < 4 && actualTries < maxTries)
{
_policyTestTools.Query(testCluster, queriesPerIteration, ConsistencyLevel.Two);
queriesCompleted += queriesPerIteration;
}
Assert.IsTrue(_policyTestTools.Coordinators.Count() >= 4, "The minimum number of hosts queried was not met!");
int totalQueriesForAllHosts = _policyTestTools.Coordinators.Sum(c => c.Value);
Assert.AreEqual(queriesCompleted, totalQueriesForAllHosts,
"The sum of queries for all hosts should equal the number of queries recorded by the calling test!");
_policyTestTools.ResetCoordinators();
testCluster.StopForce(2);
// FIXME: This sleep is needed to allow the waitFor() to work
TestUtils.WaitForDownWithWait(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 5);
var acceptedList = new List <ConsistencyLevel>
{
ConsistencyLevel.Any,
ConsistencyLevel.One,
ConsistencyLevel.Two,
ConsistencyLevel.Quorum,
ConsistencyLevel.Three,
ConsistencyLevel.All,
ConsistencyLevel.LocalQuorum,
ConsistencyLevel.EachQuorum
};
var failList = new List <ConsistencyLevel>();
// Test successful writes
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
}
catch (Exception e)
{
Assert.Fail(String.Format("Test failed at CL.{0} with message: {1}", consistencyLevel, e.Message));
}
}
// Test successful reads
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List <string>
{
"EACH_QUORUM ConsistencyLevel is only supported for writes",
"ANY ConsistencyLevel is only supported for writes"
};
Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message));
}
}
// Test writes which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
Assert.Fail("Expected Exception was not thrown for ConsistencyLevel :" + consistencyLevel);
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
catch (WriteTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
}
// Test reads which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel));
}
catch (ReadTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
}
}
public void ReplicationFactorTwo_DowngradingConsistencyRetryPolicy()
{
ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(3);
testCluster.Builder = Cluster.Builder()
.WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()))
.WithRetryPolicy(DowngradingConsistencyRetryPolicy.Instance);
testCluster.InitClient();
_policyTestTools.CreateSchema(testCluster.Session, 2);
_policyTestTools.InitPreparedStatement(testCluster, 12, ConsistencyLevel.Two);
_policyTestTools.Query(testCluster, 12, ConsistencyLevel.Two);
string coordinatorHostQueried = _policyTestTools.Coordinators.First().Key.Split(':').First();;
int awareCoord = int.Parse(coordinatorHostQueried.Split('.').Last());
int coordinatorsWithMoreThanZeroQueries = 0;
foreach (var coordinator in _policyTestTools.Coordinators)
{
coordinatorsWithMoreThanZeroQueries++;
_policyTestTools.AssertQueried(coordinator.Key.ToString(), 6);
}
Assert.AreEqual(2, coordinatorsWithMoreThanZeroQueries);
_policyTestTools.ResetCoordinators();
testCluster.StopForce(awareCoord);
TestUtils.WaitForDownWithWait(testCluster.ClusterIpPrefix + awareCoord, testCluster.Cluster, 30);
var acceptedList = new List <ConsistencyLevel>
{
ConsistencyLevel.Any,
ConsistencyLevel.One,
ConsistencyLevel.Two,
ConsistencyLevel.Quorum,
ConsistencyLevel.Three,
ConsistencyLevel.All
};
var failList = new List <ConsistencyLevel>();
// Test successful writes
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
}
catch (Exception e)
{
Assert.Fail(String.Format("Test failed at CL.{0} with message: {1}", consistencyLevel, e.Message));
}
}
// Test successful reads
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List <string>
{
"ANY ConsistencyLevel is only supported for writes"
};
Assert.True(acceptableErrorMessages.Contains(e.Message));
}
}
// Test writes which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel));
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List <string>
{
"consistency level LOCAL_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)",
"consistency level EACH_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)"
};
Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message));
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
catch (WriteTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
}
// Test reads which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel));
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List <string>
{
"consistency level LOCAL_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)",
"EACH_QUORUM ConsistencyLevel is only supported for writes"
};
Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message));
}
catch (ReadTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
}
}
//////////////////////////////
/// Test Helpers
//////////////////////////////
public void TestReplicationFactorThree(ITestCluster testCluster)
{
_policyTestTools.CreateSchema(testCluster.Session, 3);
_policyTestTools.InitPreparedStatement(testCluster, 12, ConsistencyLevel.Three);
_policyTestTools.Query(testCluster, 12, ConsistencyLevel.Three);
_policyTestTools.ResetCoordinators();
testCluster.StopForce(2);
TestUtils.WaitForDownWithWait(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 5);
var acceptedList = new List <ConsistencyLevel>
{
ConsistencyLevel.Any,
ConsistencyLevel.One,
ConsistencyLevel.Two,
ConsistencyLevel.Quorum,
ConsistencyLevel.Three,
ConsistencyLevel.All
};
var failList = new List <ConsistencyLevel>();
// Test successful writes
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
}
catch (Exception e)
{
Assert.Fail(String.Format("Test failed at CL.{0} with message: {1}", consistencyLevel, e.Message));
}
}
// Test successful reads
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List <string>
{
"ANY ConsistencyLevel is only supported for writes"
};
Assert.True(acceptableErrorMessages.Contains(e.Message));
}
}
// Test writes which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel));
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List <string>
{
"consistency level LOCAL_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)",
"consistency level EACH_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)"
};
Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message));
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
catch (WriteTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
}
// Test reads which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel));
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List <string>
{
"consistency level LOCAL_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)",
"EACH_QUORUM ConsistencyLevel is only supported for writes"
};
Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message));
}
catch (ReadTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
}
}
//////////////////////////////
/// Test Helpers
//////////////////////////////
public void TestReplicationFactorThree(ITestCluster testCluster)
{
_policyTestTools.CreateSchema(testCluster.Session, 3);
_policyTestTools.InitPreparedStatement(testCluster, 12, ConsistencyLevel.Three);
_policyTestTools.Query(testCluster, 12, ConsistencyLevel.Three);
_policyTestTools.ResetCoordinators();
testCluster.StopForce(2);
TestUtils.WaitForDownWithWait(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 5);
var acceptedList = new List<ConsistencyLevel>
{
ConsistencyLevel.Any,
ConsistencyLevel.One,
ConsistencyLevel.Two,
ConsistencyLevel.Quorum,
ConsistencyLevel.Three,
ConsistencyLevel.All
};
var failList = new List<ConsistencyLevel>();
// Test successful writes
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
}
catch (Exception e)
{
Assert.Fail(String.Format("Test failed at CL.{0} with message: {1}", consistencyLevel, e.Message));
}
}
// Test successful reads
foreach (ConsistencyLevel consistencyLevel in acceptedList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List<string>
{
"ANY ConsistencyLevel is only supported for writes"
};
Assert.True(acceptableErrorMessages.Contains(e.Message));
}
}
// Test writes which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel);
Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel));
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List<string>
{
"consistency level LOCAL_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)",
"consistency level EACH_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)"
};
Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message));
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
catch (WriteTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
}
// Test reads which should fail
foreach (ConsistencyLevel consistencyLevel in failList)
{
try
{
_policyTestTools.Query(testCluster, 12, consistencyLevel);
Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel));
}
catch (InvalidQueryException e)
{
var acceptableErrorMessages = new List<string>
{
"consistency level LOCAL_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)",
"EACH_QUORUM ConsistencyLevel is only supported for writes"
};
Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message));
}
catch (ReadTimeoutException)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
catch (UnavailableException)
{
// expected to fail when the client has already marked the
// node as DOWN
}
}
}