public void DCAwareRoundRobinTestCCM()
{
var builder = Cluster.Builder().WithLoadBalancingPolicy(new DCAwareRoundRobinPolicy("dc2"));
CCMBridge.CCMCluster c = CCMBridge.CCMCluster.Create(2, 2, builder);
createMultiDCSchema(c.Session);
try
{
init(c, 12);
query(c, 12);
assertQueried(Options.Default.IP_PREFIX + "1", 0);
assertQueried(Options.Default.IP_PREFIX + "2", 0);
assertQueried(Options.Default.IP_PREFIX + "3", 6);
assertQueried(Options.Default.IP_PREFIX + "4", 6);
}
catch (Exception e)
{
c.ErrorOut();
throw e;
}
finally
{
resetCoordinators();
c.Discard();
}
}
public void tokenAwareWithRF2TestCCM()
{
var builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()));
CCMBridge.CCMCluster c = CCMBridge.CCMCluster.Create(2, builder);
createSchema(c.Session, 2);
try
{
init(c, 12);
query(c, 12);
// Not the best test ever, we should use OPP and check we do it the
// right nodes. But since M3P is hard-coded for now, let just check
// we just hit only one node.
assertQueried(Options.Default.IP_PREFIX + "1", 0);
assertQueried(Options.Default.IP_PREFIX + "2", 12);
assertQueried(Options.Default.IP_PREFIX + "3", 0);
resetCoordinators();
c.CCMBridge.BootstrapNode(3);
TestUtils.waitFor(Options.Default.IP_PREFIX + "3", c.Cluster, 60);
query(c, 12);
// We should still be hitting only one node
assertQueried(Options.Default.IP_PREFIX + "1", 0);
assertQueried(Options.Default.IP_PREFIX + "2", 12);
assertQueried(Options.Default.IP_PREFIX + "3", 0);
resetCoordinators();
c.CCMBridge.Stop(2);
TestUtils.waitForDown(Options.Default.IP_PREFIX + "2", c.Cluster, 60);
query(c, 12);
assertQueried(Options.Default.IP_PREFIX + "1", 6);
assertQueried(Options.Default.IP_PREFIX + "2", 0);
assertQueried(Options.Default.IP_PREFIX + "3", 6);
}
catch (Exception e)
{
c.ErrorOut();
throw e;
}
finally
{
resetCoordinators();
c.Discard();
}
}
public void forceStopCCM()
{
var builder = Cluster.Builder().WithLoadBalancingPolicy(new RoundRobinPolicy());
builder.WithQueryTimeout(10000);
CCMBridge.CCMCluster c = CCMBridge.CCMCluster.Create(4, builder);
createSchema(c.Session);
try
{
init(c, 12);
query(c, 12);
resetCoordinators();
c.CCMBridge.ForceStop(1);
c.CCMBridge.ForceStop(2);
TestUtils.waitForDown(Options.Default.IP_PREFIX + "1", c.Cluster, 40);
TestUtils.waitForDown(Options.Default.IP_PREFIX + "2", c.Cluster, 40);
query(c, 12);
c.CCMBridge.ForceStop(3);
c.CCMBridge.ForceStop(4);
TestUtils.waitForDown(Options.Default.IP_PREFIX + "3", c.Cluster, 40);
TestUtils.waitForDown(Options.Default.IP_PREFIX + "4", c.Cluster, 40);
try
{
query(c, 12);
Assert.Fail();
}
catch (NoHostAvailableException e)
{
// No more nodes so ...
}
}
catch (Exception e)
{
c.ErrorOut();
throw e;
}
finally
{
resetCoordinators();
c.Discard();
}
}
public void roundRobinTestCCM()
{
var builder = Cluster.Builder().WithLoadBalancingPolicy(new RoundRobinPolicy());
CCMBridge.CCMCluster c = CCMBridge.CCMCluster.Create(2, builder);
createSchema(c.Session);
try
{
init(c, 12);
query(c, 12);
assertQueried(Options.Default.IP_PREFIX + "1", 6);
assertQueried(Options.Default.IP_PREFIX + "2", 6);
resetCoordinators();
c.CCMBridge.BootstrapNode(3);
TestUtils.waitFor(Options.Default.IP_PREFIX + "3", c.Cluster, 60);
query(c, 12);
assertQueried(Options.Default.IP_PREFIX + "1", 4);
assertQueried(Options.Default.IP_PREFIX + "2", 4);
assertQueried(Options.Default.IP_PREFIX + "3", 4);
resetCoordinators();
c.CCMBridge.DecommissionNode(1);
TestUtils.waitForDecommission(Options.Default.IP_PREFIX + "1", c.Cluster, 60);
query(c, 12);
assertQueried(Options.Default.IP_PREFIX + "2", 6);
assertQueried(Options.Default.IP_PREFIX + "3", 6);
}
catch (Exception e)
{
c.ErrorOut();
throw e;
}
finally
{
resetCoordinators();
c.Discard();
}
}
/// <summary>
/// Tests DowngradingConsistencyRetryPolicy
/// </summary>
public void downgradingConsistencyRetryPolicy(Builder builder)
{
CCMBridge.CCMCluster c = CCMBridge.CCMCluster.Create(3, builder);
createSchema(c.Session, 3);
// FIXME: Race condition where the nodes are not fully up yet and assertQueried reports slightly different numbers
Thread.Sleep(5000);
try
{
init(c, 12, ConsistencyLevel.All);
query(c, 12, ConsistencyLevel.All);
assertQueried(Options.Default.IP_PREFIX + "1", 4);
assertQueried(Options.Default.IP_PREFIX + "2", 4);
assertQueried(Options.Default.IP_PREFIX + "3", 4);
resetCoordinators();
c.CCMBridge.ForceStop(2);
TestUtils.waitForDownWithWait(Options.Default.IP_PREFIX + "2", c.Cluster, 10);
query(c, 12, ConsistencyLevel.All);
assertQueried(Options.Default.IP_PREFIX + "1", 6);
assertQueried(Options.Default.IP_PREFIX + "2", 0);
assertQueried(Options.Default.IP_PREFIX + "3", 6);
assertAchievedConsistencyLevel(ConsistencyLevel.Two);
resetCoordinators();
c.CCMBridge.ForceStop(1);
TestUtils.waitForDownWithWait(Options.Default.IP_PREFIX + "1", c.Cluster, 5);
Thread.Sleep(5000);
try
{
query(c, 12, ConsistencyLevel.All);
}
catch (ReadTimeoutException e)
{
// assertEquals("Cassandra timeout during read query at consistency TWO (2 responses were required but only 1 replica responded)", e.getMessage());
}
query(c, 12, ConsistencyLevel.Quorum);
assertQueried(Options.Default.IP_PREFIX + "1", 0);
assertQueried(Options.Default.IP_PREFIX + "2", 0);
assertQueried(Options.Default.IP_PREFIX + "3", 12);
assertAchievedConsistencyLevel(ConsistencyLevel.One);
resetCoordinators();
query(c, 12, ConsistencyLevel.Two);
assertQueried(Options.Default.IP_PREFIX + "1", 0);
assertQueried(Options.Default.IP_PREFIX + "2", 0);
assertQueried(Options.Default.IP_PREFIX + "3", 12);
assertAchievedConsistencyLevel(ConsistencyLevel.One);
resetCoordinators();
query(c, 12, ConsistencyLevel.One);
assertQueried(Options.Default.IP_PREFIX + "1", 0);
assertQueried(Options.Default.IP_PREFIX + "2", 0);
assertQueried(Options.Default.IP_PREFIX + "3", 12);
assertAchievedConsistencyLevel(ConsistencyLevel.One);
}
catch (Exception e)
{
c.ErrorOut();
throw e;
}
finally
{
resetCoordinators();
c.Discard();
}
}
public void reconnectionPolicyTest(Builder builder, long restartTime, long retryTime, long breakTime)
{
CCMBridge.CCMCluster c = CCMBridge.CCMCluster.Create(1, builder);
createSchema(c.Session, 1);
try
{
init(c, 12);
query(c, 12);
// Ensure a basic test works
assertQueried(CCMBridge.IP_PREFIX + "1", 12);
resetCoordinators();
c.CassandraCluster.ForceStop(1);
// Start timing and ensure that the node is down
//long startTime = 0;
Stopwatch startTime = Stopwatch.StartNew(); // = 0;
try
{
//startTime = System.nanoTime() / 1000000000;
query(c, 12);
Assert.Fail("Test race condition where node has not shut off quickly enough.");
}
catch (NoHostAvailableException e) { }
long elapsedSeconds;
bool restarted = false;
while (true)
{
//thisTime = System.nanoTime() / 1000000000;
elapsedSeconds = startTime.ElapsedMilliseconds / 1000;
// Restart node at restartTime
if (!restarted && elapsedSeconds > restartTime)
{
c.CassandraCluster.Start(1);
restarted = true;
}
// Continue testing queries each second
try
{
query(c, 12);
assertQueried(CCMBridge.IP_PREFIX + "1", 12);
resetCoordinators();
// Ensure the time when the query completes successfully is what was expected
Assert.True(retryTime - 2 < elapsedSeconds && elapsedSeconds < retryTime + 2, String.Format("Waited {0} seconds instead an expected {1} seconds wait", elapsedSeconds, retryTime));
}
catch (NoHostAvailableException e)
{
Thread.Sleep(1000);
continue;
}
Thread.Sleep((int)(breakTime * 1000));
// The same query once more, just to be sure
query(c, 12);
assertQueried(CCMBridge.IP_PREFIX + "1", 12);
resetCoordinators();
// Ensure the reconnection times reset
c.CassandraCluster.ForceStop(1);
// Start timing and ensure that the node is down
//startTime = 0;
startTime.Reset();
try
{
//startTime = System.nanoTime() / 1000000000;
startTime.Start();
query(c, 12);
Assert.Fail("Test race condition where node has not shut off quickly enough.");
}
catch (NoHostAvailableException e) { }
restarted = false;
while (true)
{
//elapsedSeconds = System.nanoTime() / 1000000000;
elapsedSeconds = startTime.ElapsedMilliseconds / 1000;
// Restart node at restartTime
if (!restarted && elapsedSeconds > restartTime)
{
c.CassandraCluster.Start(1);
restarted = true;
}
// Continue testing queries each second
try
{
query(c, 12);
assertQueried(CCMBridge.IP_PREFIX + "1", 12);
resetCoordinators();
// Ensure the time when the query completes successfully is what was expected
Assert.True(retryTime - 2 < elapsedSeconds && elapsedSeconds < retryTime + 2, String.Format("Waited {0} seconds instead an expected {1} seconds wait", elapsedSeconds, retryTime));
}
catch (NoHostAvailableException e)
{
Thread.Sleep(1000);
continue;
}
break;
}
break;
}
}
catch (Exception e)
{
c.ErrorOut();
throw e;
}
finally
{
resetCoordinators();
c.Discard();
}
}
public void mixedDurationTestCCM()
{
if (CCMCluster != null)
CCMCluster.Discard();
CCMCluster = CCMBridge.CCMCluster.Create(3, Cluster.Builder());
cluster = CCMCluster.Cluster;
session = CCMCluster.Session;
session.CreateKeyspace("large_data", ReplicationStrategies.CreateSimpleStrategyReplicationProperty(3));
session.ChangeKeyspace("large_data");
session.WaitForSchemaAgreement(
session.Execute(String.Format("CREATE TABLE {0} (k INT, i INT, PRIMARY KEY(k, i))", "wide_rows")));
session.WaitForSchemaAgreement(
session.Execute(String.Format("CREATE TABLE {0} (k INT, i INT, PRIMARY KEY(k, i))", "wide_batch_rows")));
session.WaitForSchemaAgreement(
session.Execute(String.Format("CREATE TABLE {0} (k INT, i BLOB, PRIMARY KEY(k, i))", "wide_byte_rows")));
session.WaitForSchemaAgreement(
session.Execute(String.Format("CREATE TABLE {0} (k int PRIMARY KEY, i text)", "large_text")));
// Create the extra wide table definition
StringBuilder tableDeclaration = new StringBuilder();
tableDeclaration.Append("CREATE TABLE wide_table (");
tableDeclaration.Append("k INT PRIMARY KEY");
for (int i = 0; i < 330; ++i)
{
tableDeclaration.Append(String.Format(", {0} INT", createColumnName(i)));
}
tableDeclaration.Append(")");
session.WaitForSchemaAgreement(
session.Execute(tableDeclaration.ToString())
);
Random rndm = new Random(DateTime.Now.Millisecond);
try
{
for (int i = 0; i < 10; ++i)
{
switch ((int)rndm.Next(0,5))
{
case 0: testWideRows(); break;
case 1: testWideBatchRows(); break;
case 2: testByteRows(); break;
case 3: testLargeText(); break;
case 4: testWideTable(); break;
default: break;
}
}
}
catch (Exception e)
{
CCMCluster.ErrorOut();
throw e;
}
finally
{
CCMCluster.Discard();
}
}
public void testRFThreeDowngradingCLTwoDCsDCAware()
{
var builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new DCAwareRoundRobinPolicy("dc2"))).WithRetryPolicy(DowngradingConsistencyRetryPolicy.Instance);
CCMBridge.CCMCluster c = CCMBridge.CCMCluster.Create(3, 3, builder);
createMultiDCSchema(c.Session, 3, 3);
try
{
init(c, 12, ConsistencyLevel.Two);
query(c, 12, ConsistencyLevel.Two);
assertQueried(Options.Default.IP_PREFIX + "1", 0);
assertQueried(Options.Default.IP_PREFIX + "2", 0);
assertQueried(Options.Default.IP_PREFIX + "3", 0);
// BUG: JAVA-88
//assertQueried(Options.Default.IP_PREFIX + "4", 12);
//assertQueried(Options.Default.IP_PREFIX + "5", 0);
//assertQueried(Options.Default.IP_PREFIX + "6", 0);
resetCoordinators();
c.CCMBridge.ForceStop(2);
// FIXME: This sleep is needed to allow the waitFor() to work
Thread.Sleep(20000);
TestUtils.waitForDownWithWait(Options.Default.IP_PREFIX + "2", c.Cluster, 5);
List <ConsistencyLevel> acceptedList = new List <ConsistencyLevel>()
{
ConsistencyLevel.Any,
ConsistencyLevel.One,
ConsistencyLevel.Two,
ConsistencyLevel.Quorum,
ConsistencyLevel.Three,
ConsistencyLevel.All,
ConsistencyLevel.LocalQuorum,
ConsistencyLevel.EachQuorum
};
List <ConsistencyLevel> failList = new List <ConsistencyLevel>();
// Test successful writes
foreach (var cl in acceptedList)
{
try
{
init(c, 12, cl);
}
catch (Exception e)
{
Assert.Fail(String.Format("Test failed at CL.{0} with message: {1}", cl, e.Message));
}
}
// Test successful reads
foreach (var cl in acceptedList)
{
try
{
query(c, 12, cl);
}
catch (InvalidQueryException e)
{
List <String> acceptableErrorMessages = new List <string>()
{
"EACH_QUORUM ConsistencyLevel is only supported for writes",
"ANY ConsistencyLevel is only supported for writes"
};
Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message));
}
}
// Test writes which should fail
foreach (var cl in failList)
{
try
{
init(c, 12, cl);
Assert.Fail(String.Format("Test passed at CL.{0}.", cl));
}
catch (UnavailableException e)
{
// expected to fail when the client has already marked the
// node as DOWN
}
catch (WriteTimeoutException e)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
}
// Test reads which should fail
foreach (var cl in failList)
{
try
{
query(c, 12, cl);
Assert.Fail(String.Format("Test passed at CL.{0}.", cl));
}
catch (ReadTimeoutException e)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
catch (UnavailableException e)
{
// expected to fail when the client has already marked the
// node as DOWN
}
}
}
catch (Exception e)
{
c.ErrorOut();
throw e;
}
finally
{
resetCoordinators();
c.Discard();
}
}
public void testRFThreeDowngradingCL(Builder builder)
{
CCMBridge.CCMCluster c = CCMBridge.CCMCluster.Create(3, builder);
createSchema(c.Session, 3);
try
{
init(c, 12, ConsistencyLevel.All);
query(c, 12, ConsistencyLevel.All);
resetCoordinators();
c.CCMBridge.ForceStop(2);
TestUtils.waitForDownWithWait(Options.Default.IP_PREFIX + "2", c.Cluster, 5);
List <ConsistencyLevel> acceptedList = new List <ConsistencyLevel>()
{
ConsistencyLevel.Any,
ConsistencyLevel.One,
ConsistencyLevel.Two,
ConsistencyLevel.Quorum,
ConsistencyLevel.Three,
ConsistencyLevel.All
};
List <ConsistencyLevel> failList = new List <ConsistencyLevel>()
{
};
// Test successful writes
foreach (var cl in acceptedList)
{
try
{
init(c, 12, cl);
}
catch (Exception e)
{
Assert.Fail(String.Format("Test failed at CL.{0} with message: {1}", cl, e.Message));
}
}
// Test successful reads
foreach (var cl in acceptedList)
{
try
{
query(c, 12, cl);
}
catch (InvalidQueryException e)
{
List <String> acceptableErrorMessages = new List <string>()
{
"ANY ConsistencyLevel is only supported for writes"
};
Assert.True(acceptableErrorMessages.Contains(e.Message));
}
}
// Test writes which should fail
foreach (var cl in failList)
{
try
{
init(c, 12, cl);
Assert.Fail(String.Format("Test passed at CL.{0}.", cl));
}
catch (InvalidQueryException e)
{
List <String> acceptableErrorMessages = new List <string>()
{
"consistency level LOCAL_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)",
"consistency level EACH_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)"
};
Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message));
}
catch (UnavailableException e)
{
// expected to fail when the client has already marked the
// node as DOWN
}
catch (WriteTimeoutException e)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
}
// Test reads which should fail
foreach (var cl in failList)
{
try
{
query(c, 12, cl);
Assert.Fail(String.Format("Test passed at CL.{0}.", cl));
}
catch (InvalidQueryException e)
{
List <String> acceptableErrorMessages = new List <string>()
{
"consistency level LOCAL_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)",
"EACH_QUORUM ConsistencyLevel is only supported for writes"
};
Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message));
}
catch (ReadTimeoutException e)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
catch (UnavailableException e)
{
// expected to fail when the client has already marked the
// node as DOWN
}
}
}
catch (Exception e)
{
c.ErrorOut();
throw e;
}
finally
{
resetCoordinators();
c.Discard();
}
}
public void dcAwareRoundRobinTestWithOneRemoteHostCCM()
{
var builder = Cluster.Builder().WithLoadBalancingPolicy(new DCAwareRoundRobinPolicy("dc2", 1));
CCMBridge.CCMCluster c = CCMBridge.CCMCluster.Create(2, 2, builder);
createMultiDCSchema(c.Session);
try
{
init(c, 12);
query(c, 12);
assertQueried(Options.Default.IP_PREFIX + "1", 0);
assertQueried(Options.Default.IP_PREFIX + "2", 0);
assertQueried(Options.Default.IP_PREFIX + "3", 6);
assertQueried(Options.Default.IP_PREFIX + "4", 6);
assertQueried(Options.Default.IP_PREFIX + "5", 0);
resetCoordinators();
c.CCMBridge.BootstrapNode(5, "dc3");
TestUtils.waitFor(Options.Default.IP_PREFIX + "5", c.Cluster, 60);
query(c, 12);
assertQueried(Options.Default.IP_PREFIX + "1", 0);
assertQueried(Options.Default.IP_PREFIX + "2", 0);
assertQueried(Options.Default.IP_PREFIX + "3", 6);
assertQueried(Options.Default.IP_PREFIX + "4", 6);
assertQueried(Options.Default.IP_PREFIX + "5", 0);
resetCoordinators();
c.CCMBridge.DecommissionNode(3);
c.CCMBridge.DecommissionNode(4);
TestUtils.waitForDecommission(Options.Default.IP_PREFIX + "3", c.Cluster, 20);
TestUtils.waitForDecommission(Options.Default.IP_PREFIX + "4", c.Cluster, 20);
query(c, 12);
assertQueried(Options.Default.IP_PREFIX + "1", 0);
assertQueried(Options.Default.IP_PREFIX + "2", 0);
assertQueried(Options.Default.IP_PREFIX + "3", 0);
assertQueried(Options.Default.IP_PREFIX + "4", 0);
assertQueried(Options.Default.IP_PREFIX + "5", 12);
resetCoordinators();
c.CCMBridge.DecommissionNode(5);
TestUtils.waitForDecommission(Options.Default.IP_PREFIX + "5", c.Cluster, 20);
query(c, 12);
assertQueried(Options.Default.IP_PREFIX + "1", 0);
assertQueried(Options.Default.IP_PREFIX + "2", 12);
assertQueried(Options.Default.IP_PREFIX + "3", 0);
assertQueried(Options.Default.IP_PREFIX + "4", 0);
assertQueried(Options.Default.IP_PREFIX + "5", 0);
resetCoordinators();
c.CCMBridge.DecommissionNode(2);
TestUtils.waitForDecommission(Options.Default.IP_PREFIX + "2", c.Cluster, 20);
query(c, 12);
assertQueried(Options.Default.IP_PREFIX + "1", 12);
assertQueried(Options.Default.IP_PREFIX + "2", 0);
assertQueried(Options.Default.IP_PREFIX + "3", 0);
assertQueried(Options.Default.IP_PREFIX + "4", 0);
assertQueried(Options.Default.IP_PREFIX + "5", 0);
resetCoordinators();
c.CCMBridge.ForceStop(1);
TestUtils.waitForDown(Options.Default.IP_PREFIX + "2", c.Cluster, 20);
try
{
query(c, 12);
Assert.Fail();
}
catch (NoHostAvailableException e)
{
// No more nodes so ...
}
}
catch (Exception e)
{
c.ErrorOut();
throw e;
}
finally
{
resetCoordinators();
c.Discard();
}
}
public void alwaysRetryRetryPolicyTest()
{
Console.Write("MainThread is");
Console.Write("[");
Console.Write(Thread.CurrentThread.ManagedThreadId);
Console.WriteLine("]");
var builder = Cluster.Builder().WithRetryPolicy(new LoggingRetryPolicy(AlwaysRetryRetryPolicy.Instance));
CCMBridge.CCMCluster c = CCMBridge.CCMCluster.Create(2, builder);
createSchema(c.Session);
try
{
init(c, 12);
query(c, 12);
assertQueried(Options.Default.IP_PREFIX + "1", 6);
assertQueried(Options.Default.IP_PREFIX + "2", 6);
resetCoordinators();
// Test failed reads
c.CCMBridge.ForceStop(2);
Thread t1 = new Thread(() =>
{
Console.Write("Thread started");
Console.Write("[");
Console.Write(Thread.CurrentThread.ManagedThreadId);
Console.WriteLine("]");
try
{
query(c, 12);
}
catch (ThreadInterruptedException)
{
Console.Write("Thread broke");
Console.Write("[");
Console.Write(Thread.CurrentThread.ManagedThreadId);
Console.WriteLine("]");
}
Console.Write("Thread finished");
Console.Write("[");
Console.Write(Thread.CurrentThread.ManagedThreadId);
Console.WriteLine("]");
});
t1.Start();
t1.Join(10000);
if (t1.IsAlive)
{
t1.Interrupt();
}
t1.Join();
// A weak test to ensure that the nodes were contacted
assertQueried(Options.Default.IP_PREFIX + "1", 0);
assertQueried(Options.Default.IP_PREFIX + "2", 0);
resetCoordinators();
c.CCMBridge.Start(2);
TestUtils.waitFor(Options.Default.IP_PREFIX + "2", c.Cluster, 30);
Console.Write("MainThread started");
Console.Write("[");
Console.Write(Thread.CurrentThread.ManagedThreadId);
Console.WriteLine("]");
// Test successful reads
for (int i = 0; i < 10; ++i)
{
try
{
query(c, 12);
}
catch (ThreadInterruptedException)
{
Console.Write("Main Thread broke");
Console.Write("[");
Console.Write(Thread.CurrentThread.ManagedThreadId);
Console.WriteLine("]");
}
}
Console.Write("Main Thread finished");
Console.Write("[");
Console.Write(Thread.CurrentThread.ManagedThreadId);
Console.WriteLine("]");
// A weak test to ensure that the nodes were contacted
assertQueriedAtLeast(Options.Default.IP_PREFIX + "1", 1);
assertQueriedAtLeast(Options.Default.IP_PREFIX + "2", 1);
resetCoordinators();
// Test writes
for (int i = 0; i < 100; ++i)
{
init(c, 12);
}
// TODO: Missing test to see if nodes were written to
// Test failed writes
c.CCMBridge.ForceStop(2);
Thread t2 = new Thread(() =>
{
Console.WriteLine("2 Thread started");
try
{
init(c, 12);
Assert.Fail();
}
catch (ThreadInterruptedException)
{
Console.WriteLine("2 Thread async call broke");
}
catch (NoHostAvailableException)
{
Console.WriteLine("2 Thread no host");
}
Console.WriteLine("2 Thread finished");
});
t2.Start();
t2.Join(10000);
if (t2.IsAlive)
{
t2.Interrupt();
}
t2.Join();
// TODO: Missing test to see if nodes were written to
}
catch (Exception e)
{
c.ErrorOut();
throw e;
}
finally
{
resetCoordinators();
c.Discard();
}
}
public void tokenAwareTest(bool usePrepared)
{
var builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy()));
CCMBridge.CCMCluster c = CCMBridge.CCMCluster.Create(2, builder);
createSchema(c.Session);
try
{
//c.Cluster.RefreshSchema();
init(c, 12);
query(c, 12);
// Not the best test ever, we should use OPP and check we do it the
// right nodes. But since M3P is hard-coded for now, let just check
// we just hit only one node.
assertQueried(Options.Default.IP_PREFIX + "1", 0);
assertQueried(Options.Default.IP_PREFIX + "2", 12);
resetCoordinators();
query(c, 12);
assertQueried(Options.Default.IP_PREFIX + "1", 0);
assertQueried(Options.Default.IP_PREFIX + "2", 12);
resetCoordinators();
c.CCMBridge.ForceStop(2);
TestUtils.waitForDown(Options.Default.IP_PREFIX + "2", c.Cluster, 60);
try
{
query(c, 12, usePrepared);
Assert.Fail();
}
catch (UnavailableException e)
{
}
catch (ReadTimeoutException e)
{
}
resetCoordinators();
c.CCMBridge.Start(2);
TestUtils.waitFor(Options.Default.IP_PREFIX + "2", c.Cluster, 60);
query(c, 12);
assertQueried(Options.Default.IP_PREFIX + "1", 0);
assertQueried(Options.Default.IP_PREFIX + "2", 12);
resetCoordinators();
c.CCMBridge.DecommissionNode(2);
TestUtils.waitForDecommission(Options.Default.IP_PREFIX + "2", c.Cluster, 60);
query(c, 12);
assertQueried(Options.Default.IP_PREFIX + "1", 12);
assertQueried(Options.Default.IP_PREFIX + "2", 0);
}
catch (Exception e)
{
c.ErrorOut();
throw e;
}
finally
{
resetCoordinators();
c.Discard();
}
}
public void dcAwareRoundRobinTestWithOneRemoteHostCCM()
{
var builder = Cluster.Builder().WithLoadBalancingPolicy(new DCAwareRoundRobinPolicy("dc2", 1));
CCMBridge.CCMCluster c = CCMBridge.CCMCluster.Create(2, 2, builder);
createMultiDCSchema(c.Session);
try
{
init(c, 12);
query(c, 12);
var hosts = new string[] {
"unused",
Options.Default.IP_PREFIX + "1",
Options.Default.IP_PREFIX + "2",
Options.Default.IP_PREFIX + "3",
Options.Default.IP_PREFIX + "4",
Options.Default.IP_PREFIX + "5"
};
var hostsDc1 = new string[] { hosts[1], hosts[2] };
var hostsDc2 = new string[] { hosts[3], hosts[4] };
// verify queries went to local DC; therein distributed equally
assertQueriedSet(hostsDc1, 0);
assertQueried(hosts[3], 6);
assertQueried(hosts[4], 6);
assertQueried(hosts[5], 0);
resetCoordinators();
c.CCMBridge.BootstrapNode(5, "dc3");
TestUtils.waitFor(hosts[5], c.Cluster, 60);
query(c, 12);
// verify queries went to local DC; therein distributed equally
assertQueriedSet(hostsDc1, 0);
assertQueried(hosts[3], 6);
assertQueried(hosts[4], 6);
assertQueried(hosts[5], 0);
resetCoordinators();
c.CCMBridge.DecommissionNode(3);
c.CCMBridge.DecommissionNode(4);
TestUtils.waitForDecommission(hosts[3], c.Cluster, 20);
TestUtils.waitForDecommission(hosts[4], c.Cluster, 20);
query(c, 12);
// verify queries distributed equally across remote DCs
assertQueriedSet(hostsDc1, 6);
assertQueriedSet(hostsDc2, 0);
assertQueried(hosts[5], 6);
resetCoordinators();
c.CCMBridge.DecommissionNode(5);
TestUtils.waitForDecommission(hosts[5], c.Cluster, 20);
query(c, 12);
// verify queries went to the only live DC
assertQueriedSet(hostsDc1, 12);
assertQueriedSet(hostsDc2, 0);
assertQueried(hosts[5], 0);
resetCoordinators();
c.CCMBridge.DecommissionNode(2);
TestUtils.waitForDecommission(hosts[2], c.Cluster, 20);
query(c, 12);
// verify queries went to the only live node
assertQueried(hosts[1], 12);
assertQueried(hosts[2], 0);
assertQueriedSet(hostsDc2, 0);
assertQueried(hosts[5], 0);
resetCoordinators();
c.CCMBridge.ForceStop(1);
TestUtils.waitForDown(hosts[2], c.Cluster, 20);
// verify no host exception with all nodes down
try
{
query(c, 12);
Assert.Fail();
}
catch (NoHostAvailableException e)
{
// No more nodes so ...
}
}
catch (Exception e)
{
c.ErrorOut();
throw e;
}
finally
{
resetCoordinators();
c.Discard();
}
}
/// <summary>
/// Tests DowngradingConsistencyRetryPolicy
/// </summary>
public void downgradingConsistencyRetryPolicy(Builder builder)
{
CCMBridge.CCMCluster c = CCMBridge.CCMCluster.Create(3, builder);
createSchema(c.Session, 3);
// FIXME: Race condition where the nodes are not fully up yet and assertQueried reports slightly different numbers
Thread.Sleep(5000);
try
{
init(c, 12, ConsistencyLevel.All);
query(c, 12, ConsistencyLevel.All);
var hosts = new string[] {
"unused",
Options.Default.IP_PREFIX + "1",
Options.Default.IP_PREFIX + "2",
Options.Default.IP_PREFIX + "3"
};
assertQueried(hosts[1], 4);
assertQueried(hosts[2], 4);
assertQueried(hosts[3], 4);
resetCoordinators();
c.CCMBridge.ForceStop(2);
TestUtils.waitForDownWithWait(hosts[2], c.Cluster, 10);
query(c, 12, ConsistencyLevel.All);
// counts are allowed to be off by 1, due to roundrobin modulo shift when "node down" update becomes visible.
assertQueriedAtLeast(hosts[1], 5);
assertQueried(hosts[2], 0);
assertQueriedAtLeast(hosts[3], 5);
assertQueriedSet(new string[] { hosts[1], hosts[3] }, 12);
assertAchievedConsistencyLevel(ConsistencyLevel.Two);
resetCoordinators();
c.CCMBridge.ForceStop(1);
TestUtils.waitForDownWithWait(hosts[1], c.Cluster, 5);
Thread.Sleep(5000);
try
{
query(c, 12, ConsistencyLevel.All);
}
catch (ReadTimeoutException e)
{
// assertEquals("Cassandra timeout during read query at consistency TWO (2 responses were required but only 1 replica responded)", e.getMessage());
}
query(c, 12, ConsistencyLevel.Quorum);
assertQueried(hosts[1], 0);
assertQueried(hosts[2], 0);
assertQueried(hosts[3], 12);
assertAchievedConsistencyLevel(ConsistencyLevel.One);
resetCoordinators();
query(c, 12, ConsistencyLevel.Two);
assertQueried(hosts[1], 0);
assertQueried(hosts[2], 0);
assertQueried(hosts[3], 12);
assertAchievedConsistencyLevel(ConsistencyLevel.One);
resetCoordinators();
query(c, 12, ConsistencyLevel.One);
assertQueried(hosts[1], 0);
assertQueried(hosts[2], 0);
assertQueried(hosts[3], 12);
assertAchievedConsistencyLevel(ConsistencyLevel.One);
}
catch (Exception e)
{
c.ErrorOut();
throw e;
}
finally
{
resetCoordinators();
c.Discard();
}
}
public void mixedDurationTestCCM()
{
if (CCMCluster != null)
{
CCMCluster.Discard();
}
CCMCluster = CCMBridge.CCMCluster.Create(3, Cluster.Builder());
cluster = CCMCluster.Cluster;
session = CCMCluster.Session;
session.CreateKeyspace("large_data", ReplicationStrategies.CreateSimpleStrategyReplicationProperty(3));
session.ChangeKeyspace("large_data");
session.WaitForSchemaAgreement(
session.Execute(String.Format("CREATE TABLE {0} (k INT, i INT, PRIMARY KEY(k, i))", "wide_rows")));
session.WaitForSchemaAgreement(
session.Execute(String.Format("CREATE TABLE {0} (k INT, i INT, PRIMARY KEY(k, i))", "wide_batch_rows")));
session.WaitForSchemaAgreement(
session.Execute(String.Format("CREATE TABLE {0} (k INT, i BLOB, PRIMARY KEY(k, i))", "wide_byte_rows")));
session.WaitForSchemaAgreement(
session.Execute(String.Format("CREATE TABLE {0} (k int PRIMARY KEY, i text)", "large_text")));
// Create the extra wide table definition
StringBuilder tableDeclaration = new StringBuilder();
tableDeclaration.Append("CREATE TABLE wide_table (");
tableDeclaration.Append("k INT PRIMARY KEY");
for (int i = 0; i < 330; ++i)
{
tableDeclaration.Append(String.Format(", \"{0}\" INT", createColumnName(i)));
}
tableDeclaration.Append(")");
session.WaitForSchemaAgreement(
session.Execute(tableDeclaration.ToString())
);
Random rndm = new Random(DateTime.Now.Millisecond);
try
{
for (int i = 0; i < 10; ++i)
{
switch ((int)rndm.Next(0, 5))
{
case 0: testWideRows(); break;
case 1: testWideBatchRows(); break;
case 2: testByteRows(); break;
case 3: testLargeText(); break;
case 4: testWideTable(); break;
default: break;
}
}
}
catch (Exception e)
{
CCMCluster.ErrorOut();
throw e;
}
finally
{
CCMCluster.Discard();
}
}
public void alwaysIgnoreRetryPolicyTest()
{
var builder = Cluster.Builder().WithRetryPolicy(new LoggingRetryPolicy(AlwaysIgnoreRetryPolicy.Instance));
CCMBridge.CCMCluster c = CCMBridge.CCMCluster.Create(2, builder);
createSchema(c.Session);
try
{
init(c, 12);
query(c, 12);
assertQueried(Options.Default.IP_PREFIX + "1", 6);
assertQueried(Options.Default.IP_PREFIX + "2", 6);
resetCoordinators();
// Test failed reads
c.CCMBridge.ForceStop(2);
for (int i = 0; i < 10; ++i)
{
query(c, 12);
}
// A weak test to ensure that the nodes were contacted
assertQueried(Options.Default.IP_PREFIX + "1", 120);
assertQueried(Options.Default.IP_PREFIX + "2", 0);
resetCoordinators();
c.CCMBridge.Start(2);
TestUtils.waitFor(Options.Default.IP_PREFIX + "2", c.Cluster, 30);
// Test successful reads
for (int i = 0; i < 10; ++i)
{
query(c, 12);
}
// A weak test to ensure that the nodes were contacted
assertQueriedAtLeast(Options.Default.IP_PREFIX + "1", 1);
assertQueriedAtLeast(Options.Default.IP_PREFIX + "2", 1);
resetCoordinators();
// Test writes
for (int i = 0; i < 100; ++i)
{
init(c, 12);
}
// TODO: Missing test to see if nodes were written to
// Test failed writes
c.CCMBridge.ForceStop(2);
for (int i = 0; i < 100; ++i)
{
init(c, 12);
}
// TODO: Missing test to see if nodes were written to
}
catch (Exception e)
{
c.ErrorOut();
throw e;
}
finally
{
resetCoordinators();
c.Discard();
}
}
public void testRFTwoDowngradingCL()
{
var builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy())).WithRetryPolicy(DowngradingConsistencyRetryPolicy.Instance);
CCMBridge.CCMCluster c = CCMBridge.CCMCluster.Create(3, builder);
createSchema(c.Session, 2);
try
{
init(c, 12, ConsistencyLevel.Two);
query(c, 12, ConsistencyLevel.Two);
var assC = coordinators.First().Key.ToString();
var awareCoord = int.Parse(assC.Substring(assC.Length - 1));
assertQueried(Options.Default.IP_PREFIX + awareCoord.ToString(), 12);
resetCoordinators();
c.CCMBridge.ForceStop(awareCoord);
TestUtils.waitForDownWithWait(Options.Default.IP_PREFIX + awareCoord.ToString(), c.Cluster, 30);
List <ConsistencyLevel> acceptedList = new List <ConsistencyLevel>()
{
ConsistencyLevel.Any,
ConsistencyLevel.One,
ConsistencyLevel.Two,
ConsistencyLevel.Quorum,
ConsistencyLevel.Three,
ConsistencyLevel.All
};
List <ConsistencyLevel> failList = new List <ConsistencyLevel>()
{
};
// Test successful writes
foreach (var cl in acceptedList)
{
try
{
init(c, 12, cl);
}
catch (Exception e)
{
Assert.Fail(String.Format("Test failed at CL.{0} with message: {1}", cl, e.Message));
}
}
// Test successful reads
foreach (var cl in acceptedList)
{
try
{
query(c, 12, cl);
}
catch (InvalidQueryException e)
{
List <String> acceptableErrorMessages = new List <string>()
{
"ANY ConsistencyLevel is only supported for writes"
};
Assert.True(acceptableErrorMessages.Contains(e.Message));
}
}
// Test writes which should fail
foreach (var cl in failList)
{
try
{
init(c, 12, cl);
Assert.Fail(String.Format("Test passed at CL.{0}.", cl));
}
catch (InvalidQueryException e)
{
List <String> acceptableErrorMessages = new List <string>()
{
"consistency level LOCAL_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)",
"consistency level EACH_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)"
};
Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message));
}
catch (UnavailableException e)
{
// expected to fail when the client has already marked the
// node as DOWN
}
catch (WriteTimeoutException e)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
}
// Test reads which should fail
foreach (var cl in failList)
{
try
{
query(c, 12, cl);
Assert.Fail(String.Format("Test passed at CL.{0}.", cl));
}
catch (InvalidQueryException e)
{
List <String> acceptableErrorMessages = new List <string>()
{
"consistency level LOCAL_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)",
"EACH_QUORUM ConsistencyLevel is only supported for writes"
};
Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message));
}
catch (ReadTimeoutException e)
{
// expected to fail when the client hasn't marked the'
// node as DOWN yet
}
catch (UnavailableException e)
{
// expected to fail when the client has already marked the
// node as DOWN
}
}
}
catch (Exception e)
{
c.ErrorOut();
throw e;
}
finally
{
resetCoordinators();
c.Discard();
}
}
public void defaultPolicyTest(Builder builder)
{
CCMBridge.CCMCluster c = CCMBridge.CCMCluster.Create(2, builder);
createSchema(c.Session);
// FIXME: Race condition where the nodes are not fully up yet and assertQueried reports slightly different numbers with fallthrough*Policy
Thread.Sleep(5000);
try
{
init(c, 12);
query(c, 12);
assertQueried(Options.Default.IP_PREFIX + "1", 6);
assertQueried(Options.Default.IP_PREFIX + "2", 6);
resetCoordinators();
// Test reads
bool successfulQuery = false;
bool readTimeoutOnce = false;
bool unavailableOnce = false;
bool restartOnce = false;
for (int i = 0; i < 100; ++i)
{
try
{
// Force a ReadTimeoutException to be performed once
if (!readTimeoutOnce)
{
c.CCMBridge.ForceStop(2);
}
// Force an UnavailableException to be performed once
if (readTimeoutOnce && !unavailableOnce)
{
TestUtils.waitForDownWithWait(Options.Default.IP_PREFIX + "2", c.Cluster, 5);
}
// Bring back node to ensure other errors are not thrown on restart
if (unavailableOnce && !restartOnce)
{
c.CCMBridge.Start(2);
restartOnce = true;
}
query(c, 12);
if (restartOnce)
{
successfulQuery = true;
}
}
catch (UnavailableException e)
{
// Assert.Equal("Not enough replica available for query at consistency ONE (1 required but only 0 alive)".ToLower(), e.Message.ToLower());
unavailableOnce = true;
}
catch (ReadTimeoutException e)
{
// Assert.Equal("Cassandra timeout during read query at consistency ONE (1 responses were required but only 0 replica responded)".ToLower(), e.Message.ToLower());
readTimeoutOnce = true;
}
}
// Ensure the full cycle was completed
Assert.True(successfulQuery, "Hit testing race condition. [Never completed successfully.] (Shouldn't be an issue.):\n");
Assert.True(readTimeoutOnce, "Hit testing race condition. [Never encountered a ReadTimeoutException.] (Shouldn't be an issue.):\n");
Assert.True(unavailableOnce, "Hit testing race condition. [Never encountered an UnavailableException.] (Shouldn't be an issue.):\n");
// A weak test to ensure that the nodes were contacted
assertQueriedAtLeast(Options.Default.IP_PREFIX + "1", 1);
assertQueriedAtLeast(Options.Default.IP_PREFIX + "2", 1);
resetCoordinators();
// Test writes
successfulQuery = false;
bool writeTimeoutOnce = false;
unavailableOnce = false;
restartOnce = false;
for (int i = 0; i < 100; ++i)
{
try
{
// Force a WriteTimeoutException to be performed once
if (!writeTimeoutOnce)
{
c.CCMBridge.ForceStop(2);
}
// Force an UnavailableException to be performed once
if (writeTimeoutOnce && !unavailableOnce)
{
TestUtils.waitForDownWithWait(Options.Default.IP_PREFIX + "2", c.Cluster, 5);
}
// Bring back node to ensure other errors are not thrown on restart
if (unavailableOnce && !restartOnce)
{
c.CCMBridge.Start(2);
restartOnce = true;
}
init(c, 12);
if (restartOnce)
{
successfulQuery = true;
}
}
catch (UnavailableException e)
{
// Assert.Equal("Not enough replica available for query at consistency ONE (1 required but only 0 alive)".ToLower(), e.Message.ToLower());
unavailableOnce = true;
}
catch (WriteTimeoutException e)
{
// Assert.Equal("Cassandra timeout during write query at consistency ONE (1 replica were required but only 0 acknowledged the write)".ToLower(), e.Message.ToLower());
writeTimeoutOnce = true;
}
}
// Ensure the full cycle was completed
Assert.True(successfulQuery, "Hit testing race condition. [Never completed successfully.] (Shouldn't be an issue.):\n");
Assert.True(writeTimeoutOnce, "Hit testing race condition. [Never encountered a ReadTimeoutException.] (Shouldn't be an issue.):\n");
Assert.True(unavailableOnce, "Hit testing race condition. [Never encountered an UnavailableException.] (Shouldn't be an issue.):\n");
// TODO: Missing test to see if nodes were written to
// Test batch writes
successfulQuery = false;
writeTimeoutOnce = false;
unavailableOnce = false;
restartOnce = false;
for (int i = 0; i < 100; ++i)
{
try
{
// Force a WriteTimeoutException to be performed once
if (!writeTimeoutOnce)
{
c.CCMBridge.ForceStop(2);
}
// Force an UnavailableException to be performed once
if (writeTimeoutOnce && !unavailableOnce)
{
TestUtils.waitForDownWithWait(Options.Default.IP_PREFIX + "2", c.Cluster, 5);
}
// Bring back node to ensure other errors are not thrown on restart
if (unavailableOnce && !restartOnce)
{
c.CCMBridge.Start(2);
restartOnce = true;
}
init(c, 12, true);
if (restartOnce)
{
successfulQuery = true;
}
}
catch (UnavailableException e)
{
// Assert.Equal("Not enough replica available for query at consistency ONE (1 required but only 0 alive)", e.Message);
unavailableOnce = true;
}
catch (WriteTimeoutException e)
{
// Assert.Equal("Cassandra timeout during write query at consistency ONE (1 replica were required but only 0 acknowledged the write)", e.Message);
writeTimeoutOnce = true;
}
}
// Ensure the full cycle was completed
Assert.True(successfulQuery, "Hit testing race condition. [Never completed successfully.] (Shouldn't be an issue.):\n");
Assert.True(writeTimeoutOnce, "Hit testing race condition. [Never encountered a ReadTimeoutException.] (Shouldn't be an issue.):\n");
Assert.True(unavailableOnce, "Hit testing race condition. [Never encountered an UnavailableException.] (Shouldn't be an issue.):\n");
// TODO: Missing test to see if nodes were written to
}
catch (Exception e)
{
c.ErrorOut();
throw e;
}
finally
{
resetCoordinators();
c.Discard();
}
}
public void dcAwareRoundRobinTestWithOneRemoteHostCCM()
{
var builder = Cluster.Builder().WithLoadBalancingPolicy(new DCAwareRoundRobinPolicy("dc2", 1));
CCMBridge.CCMCluster c = CCMBridge.CCMCluster.Create(2, 1, builder);
createMultiDCSchema(c.Session);
try
{
init(c, 12);
query(c, 12);
assertQueried(CCMBridge.IP_PREFIX + "1", 0);
assertQueried(CCMBridge.IP_PREFIX + "2", 0);
assertQueried(CCMBridge.IP_PREFIX + "3", 12);
assertQueried(CCMBridge.IP_PREFIX + "4", 0);
resetCoordinators();
c.CassandraCluster.BootstrapNode(4, "dc3");
c.Session.Execute("ALTER KEYSPACE " + TestUtils.SIMPLE_KEYSPACE + " WITH REPLICATION ={'class' : 'NetworkTopologyStrategy','dc1' : 1, 'dc2' : 1, 'dc3' : 1}");
TestUtils.waitFor(CCMBridge.IP_PREFIX + "4", c.Cluster, 60);
query(c, 12);
assertQueried(CCMBridge.IP_PREFIX + "1", 0);
assertQueried(CCMBridge.IP_PREFIX + "2", 0);
assertQueried(CCMBridge.IP_PREFIX + "3", 12);
assertQueried(CCMBridge.IP_PREFIX + "4", 0);
resetCoordinators();
c.CassandraCluster.DecommissionNode(3);
TestUtils.waitForDecommission(CCMBridge.IP_PREFIX + "3", c.Cluster, 20);
query(c, 12);
assertQueried(CCMBridge.IP_PREFIX + "1", 6);
assertQueried(CCMBridge.IP_PREFIX + "2", 0);
assertQueried(CCMBridge.IP_PREFIX + "3", 0);
assertQueried(CCMBridge.IP_PREFIX + "4", 6);
resetCoordinators();
c.CassandraCluster.DecommissionNode(4);
TestUtils.waitForDecommission(CCMBridge.IP_PREFIX + "4", c.Cluster, 20);
query(c, 12);
assertQueried(CCMBridge.IP_PREFIX + "1", 12);
assertQueried(CCMBridge.IP_PREFIX + "2", 0);
assertQueried(CCMBridge.IP_PREFIX + "3", 0);
assertQueried(CCMBridge.IP_PREFIX + "4", 0);
resetCoordinators();
c.CassandraCluster.DecommissionNode(1);
TestUtils.waitForDecommission(CCMBridge.IP_PREFIX + "1", c.Cluster, 20);
query(c, 12);
assertQueried(CCMBridge.IP_PREFIX + "1", 0);
assertQueried(CCMBridge.IP_PREFIX + "2", 12);
assertQueried(CCMBridge.IP_PREFIX + "3", 0);
assertQueried(CCMBridge.IP_PREFIX + "4", 0);
resetCoordinators();
c.CassandraCluster.ForceStop(2);
TestUtils.waitForDown(CCMBridge.IP_PREFIX + "2", c.Cluster, 20);
try
{
query(c, 12);
Assert.Fail();
}
catch (NoHostAvailableException e)
{
// No more nodes so ...
}
}
catch (Exception e)
{
c.ErrorOut();
throw e;
}
finally
{
resetCoordinators();
c.Discard();
}
}