public void TokenAware_BindInt_NoHops() { // Setup PolicyTestTools policyTestTools = new PolicyTestTools(); ITestCluster testCluster = TestClusterManager.CreateNew(3); testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy())); testCluster.InitClient(); // Test var session = testCluster.Session; policyTestTools.TableName = TestUtils.GetUniqueTableName(); policyTestTools.CreateSchema(session, 1); var traces = new List <QueryTrace>(); var pstmt = session.Prepare("INSERT INTO " + policyTestTools.TableName + " (k, i) VALUES (?, ?)"); for (var i = (int)short.MinValue; i < short.MinValue + 40; i++) { var statement = pstmt .Bind(i, i) .EnableTracing(); var rs = session.Execute(statement); traces.Add(rs.Info.QueryTrace); } //Check that there weren't any hops foreach (var t in traces) { //The coordinator must be the only one executing the query Assert.True(t.Events.All(e => e.Source.ToString() == t.Coordinator.ToString()), "There were trace events from another host for coordinator " + t.Coordinator); } }
public void SetupFixture() { if (TestClusterManager.CheckCassandraVersion(false, Version.Parse("2.2"), Comparison.LessThan)) { Assert.Ignore("Requires Cassandra version >= 2.2"); return; } Diagnostics.CassandraTraceSwitch.Level = TraceLevel.Info; _testCluster = TestClusterManager.CreateNew(1, new TestClusterOptions { CassandraYaml = new[] { "batch_size_warn_threshold_in_kb:5", "batch_size_fail_threshold_in_kb:50" }, //Using a mirroring handler, the server will reply providing the same payload that was sent JvmArgs = new[] { "-Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler" } }); _testCluster.InitClient(); Session = _testCluster.Session; Session.Execute(string.Format(TestUtils.CreateKeyspaceSimpleFormat, Keyspace, 1)); Session.Execute(string.Format(TestUtils.CreateTableSimpleFormat, Table)); }
public void TokenAware_Prepared_Composite_NoHops() { // Setup PolicyTestTools policyTestTools = new PolicyTestTools(); ITestCluster testCluster = TestClusterManager.GetTestCluster(3); testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy())); testCluster.InitClient(); // Test var session = testCluster.Session; policyTestTools.CreateSchema(session); policyTestTools.TableName = TestUtils.GetUniqueTableName(); session.Execute(String.Format("CREATE TABLE {0} (k1 text, k2 int, i int, PRIMARY KEY ((k1, k2)))", policyTestTools.TableName)); Thread.Sleep(1000); var ps = session.Prepare(String.Format("INSERT INTO {0} (k1, k2, i) VALUES (?, ?, ?)", policyTestTools.TableName)); var traces = new List <QueryTrace>(); for (var i = 0; i < 10; i++) { var statement = ps.Bind(i.ToString(), i, i).EnableTracing(); //Routing key is calculated by the driver Assert.NotNull(statement.RoutingKey); var rs = session.Execute(statement); traces.Add(rs.Info.QueryTrace); } //Check that there weren't any hops foreach (var t in traces) { //The coordinator must be the only one executing the query Assert.True(t.Events.All(e => e.Source.ToString() == t.Coordinator.ToString()), "There were trace events from another host for coordinator " + t.Coordinator); } }
public void FailoverTest() { var parallelOptions = new ParallelOptions(); parallelOptions.TaskScheduler = new ThreadPerTaskScheduler(); parallelOptions.MaxDegreeOfParallelism = 100; var policy = new ConstantReconnectionPolicy(Int32.MaxValue); ITestCluster nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(4); nonShareableTestCluster.Builder = new Builder().WithReconnectionPolicy(policy); nonShareableTestCluster.InitClient(); // this will replace the existing session using the newly assigned Builder instance var session = nonShareableTestCluster.Session; // Check query to host distribution before killing nodes var queriedHosts = new List <string>(); DateTime futureDateTime = DateTime.Now.AddSeconds(120); while ((from singleHost in queriedHosts select singleHost).Distinct().Count() < 4 && DateTime.Now < futureDateTime) { var rs = session.Execute("SELECT * FROM system.schema_columnfamilies"); queriedHosts.Add(rs.Info.QueriedHost.ToString()); Thread.Sleep(50); } Assert.AreEqual(4, (from singleHost in queriedHosts select singleHost).Distinct().Count(), "All hosts should have been queried!"); // Create List of actions Action selectAction = () => { var rs = session.Execute("SELECT * FROM system.schema_columnfamilies"); Assert.Greater(rs.Count(), 0); }; var actions = new List <Action>(); for (var i = 0; i < 100; i++) { actions.Add(selectAction); } //kill some nodes. actions.Insert(20, () => nonShareableTestCluster.StopForce(1)); actions.Insert(20, () => nonShareableTestCluster.StopForce(2)); actions.Insert(80, () => nonShareableTestCluster.StopForce(3)); //Execute in parallel more than 100 actions Parallel.Invoke(parallelOptions, actions.ToArray()); // Wait for the nodes to be killed TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "1", nonShareableTestCluster.Cluster, 20); TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "2", nonShareableTestCluster.Cluster, 20); TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "3", nonShareableTestCluster.Cluster, 20); // Execute some more SELECTs for (var i = 0; i < 250; i++) { var rowSet2 = session.Execute("SELECT * FROM system.schema_columnfamilies"); Assert.Greater(rowSet2.Count(), 0); StringAssert.StartsWith(nonShareableTestCluster.ClusterIpPrefix + "4", rowSet2.Info.QueriedHost.ToString()); } }
/// <summary> /// Tests DowngradingConsistencyRetryPolicy /// /// @test_category connection:retry_policy /// </summary> public void DowngradingConsistencyRetryPolicyTest(Builder builder) { PolicyTestTools policyTestTools = new PolicyTestTools(); ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(3); testCluster.Builder = builder; testCluster.InitClient(); policyTestTools.CreateSchema(testCluster.Session, 3); // FIXME: Race condition where the nodes are not fully up yet and assertQueried reports slightly different numbers TestUtils.WaitForSchemaAgreement(testCluster.Cluster); policyTestTools.InitPreparedStatement(testCluster, 12, ConsistencyLevel.All); policyTestTools.Query(testCluster, 12, ConsistencyLevel.All); policyTestTools.AssertAchievedConsistencyLevel(ConsistencyLevel.All); //Kill one node: 2 nodes alive testCluster.Stop(2); TestUtils.WaitForDown(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 20); //After killing one node, the achieved consistency level should be downgraded policyTestTools.ResetCoordinators(); policyTestTools.Query(testCluster, 12, ConsistencyLevel.All); policyTestTools.AssertAchievedConsistencyLevel(ConsistencyLevel.Two); }
public void TokenAware_TargetPartition_NoHops() { // Setup PolicyTestTools policyTestTools = new PolicyTestTools(); ITestCluster testCluster = TestClusterManager.GetTestCluster(3); testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy())); testCluster.InitClient(); // Test policyTestTools.TableName = TestUtils.GetUniqueTableName(); policyTestTools.CreateSchema(testCluster.Session, 1); var traces = new List <QueryTrace>(); for (var i = -10; i < 10; i++) { var partitionKey = BitConverter.GetBytes(i).Reverse().ToArray(); var statement = new SimpleStatement(String.Format("INSERT INTO " + policyTestTools.TableName + " (k, i) VALUES ({0}, {0})", i)) .SetRoutingKey(new RoutingKey() { RawRoutingKey = partitionKey }) .EnableTracing(); var rs = testCluster.Session.Execute(statement); traces.Add(rs.Info.QueryTrace); } //Check that there weren't any hops foreach (var t in traces) { //The coordinator must be the only one executing the query Assert.True(t.Events.All(e => e.Source.ToString() == t.Coordinator.ToString()), "There were trace events from another host for coordinator " + t.Coordinator); } }
public void TokenAware_TargetWrongPartition_HopsOccur() { // Setup PolicyTestTools policyTestTools = new PolicyTestTools(); ITestCluster testCluster = TestClusterManager.GetTestCluster(3); testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy())); testCluster.InitClient(); var session = testCluster.Session; policyTestTools.TableName = TestUtils.GetUniqueTableName(); policyTestTools.CreateSchema(session, 1); var traces = new List <QueryTrace>(); for (var i = 1; i < 10; i++) { //The partition key is wrongly calculated var statement = new SimpleStatement(String.Format("INSERT INTO " + policyTestTools.TableName + " (k, i) VALUES ({0}, {0})", i)) .SetRoutingKey(new RoutingKey() { RawRoutingKey = new byte[] { 0, 0, 0, 0 } }) .EnableTracing(); var rs = session.Execute(statement); traces.Add(rs.Info.QueryTrace); } //Check that there were hops var hopsPerQuery = traces.Select(t => t.Events.Any(e => e.Source.ToString() == t.Coordinator.ToString())); Assert.True(hopsPerQuery.Any(v => v)); }
/// <summary> /// Validate client behavior with replication three, only one DC /// with load balancing policy TokenAware, DCAwareRoundRobin, /// with retry policy DowngradingConsistencyRetryPolicy /// after a node is taken down /// /// @test_category consistency /// @test_category connection:outage,retry_policy /// @test_category load_balancing:round_robin /// </summary> public void ReplicationFactorThree_RoundRobin_DowngradingConsistencyRetryPolicy() { ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(3); testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new RoundRobinPolicy()).WithRetryPolicy(DowngradingConsistencyRetryPolicy.Instance); testCluster.InitClient(); TestReplicationFactorThree(testCluster); }
public void AlwaysIgnoreRetryPolicyTest() { ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(2); testCluster.Builder = Cluster.Builder() .WithRetryPolicy(new LoggingRetryPolicy(AlwaysIgnoreRetryPolicy.Instance)) .AddContactPoint(testCluster.ClusterIpPrefix + "1") .AddContactPoint(testCluster.ClusterIpPrefix + "2"); testCluster.InitClient(); RetryPolicyTest(testCluster); }
public void BootstrappedNode() { var policy = new ConstantReconnectionPolicy(500); ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1); testCluster.Builder = new Builder().WithReconnectionPolicy(policy); testCluster.InitClient(); // this will replace the existing session using the newly assigned Builder instance testCluster.BootstrapNode(2); TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "2", DefaultCassandraPort, 60); //Wait for the join to be online string newlyBootstrappedHost = testCluster.ClusterIpPrefix + 2; TestUtils.ValidateBootStrappedNodeIsQueried(testCluster, 2, newlyBootstrappedHost); }
public void TryNextHostRetryPolicyTest() { ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(2); var socketOptions = new SocketOptions().SetReadTimeoutMillis(2000); testCluster.Builder = Cluster.Builder() .WithRetryPolicy(new LoggingRetryPolicy(TryNextHostRetryPolicy.Instance)) .AddContactPoint(testCluster.ClusterIpPrefix + "1") .AddContactPoint(testCluster.ClusterIpPrefix + "2") .WithSocketOptions(socketOptions); testCluster.InitClient(); // Setup cluster PolicyTestTools policyTestTools = new PolicyTestTools(); policyTestTools.CreateSchema(testCluster.Session, 2); policyTestTools.InitPreparedStatement(testCluster, 12); // Try with both hosts policyTestTools.Query(testCluster, 10); policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + 1 + ":" + DefaultCassandraPort, 5); policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + 2 + ":" + DefaultCassandraPort, 5); // Try with host 1 policyTestTools.ResetCoordinators(); testCluster.PauseNode(2); policyTestTools.Query(testCluster, 10); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 1 + ":" + DefaultCassandraPort, 10); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 2 + ":" + DefaultCassandraPort, 0); testCluster.ResumeNode(2); // Try with host 2 policyTestTools.ResetCoordinators(); testCluster.PauseNode(1); policyTestTools.Query(testCluster, 10); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 1 + ":" + DefaultCassandraPort, 0); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 2 + ":" + DefaultCassandraPort, 10); // Try with 0 hosts policyTestTools.ResetCoordinators(); testCluster.PauseNode(2); Assert.Throws <NoHostAvailableException>(() => policyTestTools.Query(testCluster, 10)); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 1 + ":" + DefaultCassandraPort, 0); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + 2 + ":" + DefaultCassandraPort, 0); testCluster.ResumeNode(1); testCluster.ResumeNode(2); }
private void ReprepareTest(bool useKeyspace) { string keyspace = DefaultKeyspaceName; ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1); testCluster.InitClient(); // make sure client session was just created var nonShareableSession = testCluster.Session; string fqKeyspaceName = ""; if (useKeyspace) { nonShareableSession.ChangeKeyspace(keyspace); } else { fqKeyspaceName = keyspace + "."; } try { nonShareableSession.Execute("CREATE TABLE " + fqKeyspaceName + "test(k text PRIMARY KEY, i int)"); } catch (AlreadyExistsException) { } nonShareableSession.Execute("INSERT INTO " + fqKeyspaceName + "test (k, i) VALUES ('123', 17)"); nonShareableSession.Execute("INSERT INTO " + fqKeyspaceName + "test (k, i) VALUES ('124', 18)"); PreparedStatement ps = nonShareableSession.Prepare("SELECT * FROM " + fqKeyspaceName + "test WHERE k = ?"); var rs = nonShareableSession.Execute(ps.Bind("123")); Assert.AreEqual(rs.First().GetValue <int>("i"), 17); testCluster.Stop(1); TestUtils.WaitForDown(testCluster.ClusterIpPrefix + "1", testCluster.Cluster, 40); testCluster.Start(1); TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "1", DefaultCassandraPort, 60); Assert.True(nonShareableSession.Cluster.AllHosts().Select(h => h.IsUp).Any(), "There should be one node up"); for (var i = 0; i < 10; i++) { var rowset = nonShareableSession.Execute(ps.Bind("124")); Assert.AreEqual(rowset.First().GetValue <int>("i"), 18); } }
public void RoundRobin_TwoDCs_DcAware() { // Setup PolicyTestTools policyTestTools = new PolicyTestTools(); ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1, 1, DefaultMaxClusterCreateRetries, true); testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new DCAwareRoundRobinPolicy("dc2")); testCluster.InitClient(); policyTestTools.CreateMultiDcSchema(testCluster.Session); policyTestTools.InitPreparedStatement(testCluster, 12); policyTestTools.Query(testCluster, 12); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 12); }
public void RoundRobin_DcAware_BuildClusterWithNonExistentDc() { ITestCluster testCluster = TestClusterManager.GetTestCluster(1); testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new DCAwareRoundRobinPolicy("dc2")); try { testCluster.InitClient(); Assert.Fail("Expected exception was not thrown!"); } catch (ArgumentException e) { string expectedErrMsg = "Datacenter dc2 does not match any of the nodes, available datacenters: datacenter1."; Assert.IsTrue(e.Message.Contains(expectedErrMsg)); } }
public void RoundRobin_TokenAware_TwoDCsWithOneNodeEach_ReplicationFactorTwo() { // Setup PolicyTestTools policyTestTools = new PolicyTestTools(); ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1, 1, DefaultMaxClusterCreateRetries, true); testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy())); testCluster.InitClient(); policyTestTools.CreateSchema(testCluster.Session, 2); policyTestTools.InitPreparedStatement(testCluster, 12); policyTestTools.Query(testCluster, 12); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 6); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6); }
public void RoundRobin_OneDc_OneNodeAdded_OneNodeDecommissioned() { // Setup PolicyTestTools policyTestTools = new PolicyTestTools(); ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1); testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new RoundRobinPolicy()); testCluster.InitClient(); policyTestTools.CreateSchema(testCluster.Session); policyTestTools.InitPreparedStatement(testCluster, 12); policyTestTools.Query(testCluster, 12); // Validate that all host were queried equally policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 12); // Add new node to the end of second cluster, remove node from beginning of first cluster policyTestTools.ResetCoordinators(); // Bootstrap step int bootStrapPos = 2; testCluster.BootstrapNode(bootStrapPos); string newlyBootstrappedIp = testCluster.ClusterIpPrefix + bootStrapPos; TestUtils.WaitForUp(newlyBootstrappedIp, DefaultCassandraPort, 30); // Validate expected nodes where queried policyTestTools.WaitForPolicyToolsQueryToHitBootstrappedIp(testCluster, newlyBootstrappedIp); policyTestTools.Query(testCluster, 12); policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 6); policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6); // decommission old node policyTestTools.ResetCoordinators(); testCluster.DecommissionNode(1); TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, testCluster.Cluster, 60); policyTestTools.Query(testCluster, 12); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 12); }
public void TokenAware_Composite_NoHops() { // Setup PolicyTestTools policyTestTools = new PolicyTestTools(); ITestCluster testCluster = TestClusterManager.GetTestCluster(3); testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy())); testCluster.InitClient(); // Test var session = testCluster.Session; policyTestTools.CreateSchema(session); policyTestTools.TableName = TestUtils.GetUniqueTableName(); session.Execute(String.Format("CREATE TABLE {0} (k1 text, k2 int, i int, PRIMARY KEY ((k1, k2)))", policyTestTools.TableName)); var traces = new List <QueryTrace>(); for (var i = 0; i < 10; i++) { var statement = new SimpleStatement(String.Format("INSERT INTO " + policyTestTools.TableName + " (k1, k2, i) VALUES ('{0}', {0}, {0})", i)) .SetRoutingKey( new RoutingKey() { RawRoutingKey = Encoding.UTF8.GetBytes(i.ToString()) }, new RoutingKey() { RawRoutingKey = BitConverter.GetBytes(i).Reverse().ToArray() }) .EnableTracing(); var rs = session.Execute(statement); traces.Add(rs.Info.QueryTrace); } //Check that there weren't any hops foreach (var t in traces) { //The coordinator must be the only one executing the query Assert.True(t.Events.All(e => e.Source.ToString() == t.Coordinator.ToString()), "There were trace events from another host for coordinator " + t.Coordinator); } }
private bool TryToInititalizeClusterClient(ITestCluster testCluster) { try { foreach (string host in testCluster.ExpectedInitialHosts) { TestUtils.WaitForUp(host, DefaultCassandraPort, 30); } // at this point we expect all the nodes to be up testCluster.InitClient(); return(true); } catch (Exception e) { Trace.TraceError("Unexpected Error occurred when trying to get shared test cluster with InitialContactPoint: " + testCluster.InitialContactPoint + ", name: " + testCluster.Name); Trace.TraceError("Error Message: " + e.Message); Trace.TraceError("Error Stack Trace: " + e.StackTrace); Trace.TraceError("Removing this cluster, and looping back to create a new one ... "); } return(false); }
public void RoundRobin_OneDc_AllNodesForceStoppedOneAtATime() { // Setup PolicyTestTools policyTestTools = new PolicyTestTools(); ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(2); testCluster.Builder = Cluster.Builder() .WithLoadBalancingPolicy(new RoundRobinPolicy()) .WithQueryTimeout(10000); testCluster.InitClient(); policyTestTools.CreateSchema(testCluster.Session); policyTestTools.InitPreparedStatement(testCluster, 12); policyTestTools.Query(testCluster, 12); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 6); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6); policyTestTools.ResetCoordinators(); testCluster.StopForce(1); TestUtils.WaitForDown(testCluster.ClusterIpPrefix + "1", testCluster.Cluster, 20); policyTestTools.Query(testCluster, 12); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 12); testCluster.StopForce(2); TestUtils.WaitForDown(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 20); try { policyTestTools.Query(testCluster, 3); Assert.Fail("Exception should have been thrown, but wasn't!"); } catch (NoHostAvailableException) { Trace.TraceInformation("Expected NoHostAvailableException exception was thrown."); } }
public void TokenAware_BindString_NoHops() { // Setup PolicyTestTools policyTestTools = new PolicyTestTools(); ITestCluster testCluster = TestClusterManager.GetTestCluster(3); testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy())); testCluster.InitClient(); // Test var session = testCluster.Session; policyTestTools.CreateSchema(session); policyTestTools.TableName = TestUtils.GetUniqueTableName(); session.Execute(String.Format("CREATE TABLE {0} (k text PRIMARY KEY, i int)", policyTestTools.TableName)); var traces = new List <QueryTrace>(); string key = "value"; for (var i = 100; i < 140; i++) { key += (char)i; var partitionKey = Encoding.UTF8.GetBytes(key); var statement = new SimpleStatement("INSERT INTO " + policyTestTools.TableName + " (k, i) VALUES (?, ?)", key, i) .SetRoutingKey(new RoutingKey() { RawRoutingKey = partitionKey }) .EnableTracing(); var rs = session.Execute(statement); traces.Add(rs.Info.QueryTrace); } //Check that there weren't any hops foreach (var t in traces) { //The coordinator must be the only one executing the query Assert.True(t.Events.All(e => e.Source.ToString() == t.Coordinator.ToString()), "There were trace events from another host for coordinator " + t.Coordinator); } }
public void TokenAware_Guid_NoHops() { // Setup PolicyTestTools policyTestTools = new PolicyTestTools(); ITestCluster testCluster = TestClusterManager.GetTestCluster(3); testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy())); testCluster.InitClient(); // Test var session = testCluster.Session; string uniqueTableName = TestUtils.GetUniqueTableName(); policyTestTools.CreateSchema(session); session.Execute(String.Format("CREATE TABLE {0} (k uuid PRIMARY KEY, i int)", uniqueTableName)); var traces = new List <QueryTrace>(); for (var i = 0; i < 10; i++) { var key = Guid.NewGuid(); var statement = new SimpleStatement(String.Format("INSERT INTO " + uniqueTableName + " (k, i) VALUES ({0}, {1})", key, i)) .SetRoutingKey( new RoutingKey() { RawRoutingKey = TypeCodec.GuidShuffle(key.ToByteArray()) }) .EnableTracing(); var rs = session.Execute(statement); traces.Add(rs.Info.QueryTrace); } //Check that there weren't any hops foreach (var t in traces) { //The coordinator must be the only one executing the query Assert.True(t.Events.All(e => e.Source.ToString() == t.Coordinator.ToString()), "There were trace events from another host for coordinator " + t.Coordinator); } }
public void ReconnectionRecyclesPool() { var policy = new ConstantReconnectionPolicy(5000); ITestCluster nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(2); nonShareableTestCluster.Builder = new Builder().WithReconnectionPolicy(policy); nonShareableTestCluster.InitClient(); // this will replace the existing session using the newly assigned Builder instance var session = (Session)nonShareableTestCluster.Session; var hosts = new List <IPEndPoint>(); for (var i = 0; i < 50; i++) { var rs = session.Execute("SELECT * FROM system.schema_columnfamilies"); if (i == 20) { nonShareableTestCluster.StopForce(2); } else if (i == 30) { nonShareableTestCluster.Start(2); Thread.Sleep(5000); } hosts.Add(rs.Info.QueriedHost); } var pool = session.GetOrCreateConnectionPool(TestHelper.CreateHost(nonShareableTestCluster.InitialContactPoint), HostDistance.Local); var connections = pool.OpenConnections.ToArray(); var expectedCoreConnections = nonShareableTestCluster.Cluster.Configuration .GetPoolingOptions(connections.First().ProtocolVersion) .GetCoreConnectionsPerHost(HostDistance.Local); Assert.AreEqual(expectedCoreConnections, connections.Length); Assert.True(connections.All(c => !c.IsClosed)); }
public void TokenAwareTest(bool usePrepared) { // Setup PolicyTestTools policyTestTools = new PolicyTestTools(); ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1, 1, DefaultMaxClusterCreateRetries, true); testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy())); testCluster.InitClient(); policyTestTools.CreateSchema(testCluster.Session); //clusterInfo.Cluster.RefreshSchema(); policyTestTools.InitPreparedStatement(testCluster, 12); policyTestTools.Query(testCluster, 12); // Not the best test ever, we should use OPP and check we do it the // right nodes. But since M3P is hard-coded for now, let just check // we just hit only one node. int nodePosToDecommission = 2; int nodePositionToNotDecommission = 1; if (policyTestTools.Coordinators.ContainsKey(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort)) { nodePosToDecommission = 1; nodePositionToNotDecommission = 2; } policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePosToDecommission + ":" + DefaultCassandraPort, 12); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePositionToNotDecommission + ":" + DefaultCassandraPort, 0); // now try again having stopped the node that was just queried policyTestTools.ResetCoordinators(); testCluster.DecommissionNode(nodePosToDecommission); TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + nodePosToDecommission + ":" + DefaultCassandraPort, testCluster.Cluster, 40); policyTestTools.Query(testCluster, 12, usePrepared); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePosToDecommission + ":" + DefaultCassandraPort, 0); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + nodePositionToNotDecommission + ":" + DefaultCassandraPort, 12); }
public void Parallel_Insert_And_Select_Sync() { var originalTraceLevel = Diagnostics.CassandraTraceSwitch.Level; ITestCluster testCluster = TestClusterManager.GetTestCluster(3); Diagnostics.CassandraTraceSwitch.Level = TraceLevel.Warning; testCluster.Builder = Cluster.Builder().WithRetryPolicy(DowngradingConsistencyRetryPolicy.Instance); testCluster.InitClient(); ISession session = testCluster.Session; string uniqueKsName = "keyspace_" + Randomm.RandomAlphaNum(10); session.Execute(@"CREATE KEYSPACE " + uniqueKsName + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 3};"); TestUtils.WaitForSchemaAgreement(testCluster.Cluster); session.ChangeKeyspace(uniqueKsName); string tableName = "table_" + Guid.NewGuid().ToString("N").ToLower(); session.Execute(String.Format(TestUtils.CREATE_TABLE_TIME_SERIES, tableName)); TestUtils.WaitForSchemaAgreement(testCluster.Cluster); var insertQuery = String.Format("INSERT INTO {0} (id, event_time, text_sample) VALUES (?, ?, ?)", tableName); var insertQueryPrepared = session.Prepare(insertQuery); var selectQuery = String.Format("SELECT * FROM {0} LIMIT 10000", tableName); const int rowsPerId = 1000; object insertQueryStatement = new SimpleStatement(insertQuery); if (CassandraVersion.Major < 2) { //Use prepared statements all the way as it is not possible to bind on a simple statement with C* 1.2 insertQueryStatement = session.Prepare(insertQuery); } var actionInsert = GetInsertAction(session, insertQueryStatement, ConsistencyLevel.Quorum, rowsPerId); var actionInsertPrepared = GetInsertAction(session, insertQueryPrepared, ConsistencyLevel.Quorum, rowsPerId); var actionSelect = GetSelectAction(session, selectQuery, ConsistencyLevel.Quorum, 10); //Execute insert sync to have some records actionInsert(); //Execute select sync to assert that everything is going OK actionSelect(); var actions = new List <Action>(); for (var i = 0; i < 10; i++) { //Add 10 actions to execute actions.AddRange(new[] { actionInsert, actionSelect, actionInsertPrepared }); actions.AddRange(new[] { actionSelect, actionInsert, actionInsertPrepared, actionInsert }); actions.AddRange(new[] { actionInsertPrepared, actionInsertPrepared, actionSelect }); } //Execute in parallel the 100 actions var parallelOptions = new ParallelOptions(); parallelOptions.TaskScheduler = new ThreadPerTaskScheduler(); parallelOptions.MaxDegreeOfParallelism = 300; Parallel.Invoke(parallelOptions, actions.ToArray()); Parallel.Invoke(actions.ToArray()); Diagnostics.CassandraTraceSwitch.Level = originalTraceLevel; }
/// <summary> /// Validate client behavior with replication three, multiple DCs /// with load balancing policy TokenAware, DCAwareRoundRobin, /// with retry policy DowngradingConsistencyRetryPolicy /// after a node is taken down /// /// @test_category consistency /// @test_category connection:outage,retry_policy /// @test_category load_balancing:round_robin,token_aware,dc_aware /// </summary> public void ReplicationFactorThree_TwoDcs_DcAware_DowngradingConsistencyRetryPolicy() { // Seetup ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(3, 3, DefaultMaxClusterCreateRetries, true); testCluster.Builder = Cluster.Builder() .WithLoadBalancingPolicy(new TokenAwarePolicy(new DCAwareRoundRobinPolicy("dc2"))) .WithRetryPolicy(DowngradingConsistencyRetryPolicy.Instance); testCluster.InitClient(); TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "1", DefaultCassandraPort, 30); TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "2", DefaultCassandraPort, 30); TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "3", DefaultCassandraPort, 30); TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "4", DefaultCassandraPort, 30); TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "5", DefaultCassandraPort, 30); TestUtils.WaitForUp(testCluster.ClusterIpPrefix + "6", DefaultCassandraPort, 30); // Test _policyTestTools.CreateMultiDcSchema(testCluster.Session, 3, 3); _policyTestTools.InitPreparedStatement(testCluster, 12, ConsistencyLevel.Two); _policyTestTools.Query(testCluster, 12, ConsistencyLevel.Two); // Validate expected number of host / query counts _policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0); _policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 0); _policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 0); _policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, 4); _policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "5:" + DefaultCassandraPort, 4); _policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "6:" + DefaultCassandraPort, 4); _policyTestTools.ResetCoordinators(); testCluster.StopForce(2); // FIXME: This sleep is needed to allow the waitFor() to work TestUtils.WaitForDownWithWait(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 5); var acceptedList = new List <ConsistencyLevel> { ConsistencyLevel.Any, ConsistencyLevel.One, ConsistencyLevel.Two, ConsistencyLevel.Quorum, ConsistencyLevel.Three, ConsistencyLevel.All, ConsistencyLevel.LocalQuorum, ConsistencyLevel.EachQuorum }; var failList = new List <ConsistencyLevel>(); // Test successful writes foreach (ConsistencyLevel consistencyLevel in acceptedList) { try { _policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel); } catch (Exception e) { Assert.Fail(String.Format("Test failed at CL.{0} with message: {1}", consistencyLevel, e.Message)); } } // Test successful reads foreach (ConsistencyLevel consistencyLevel in acceptedList) { try { _policyTestTools.Query(testCluster, 12, consistencyLevel); } catch (InvalidQueryException e) { var acceptableErrorMessages = new List <string> { "EACH_QUORUM ConsistencyLevel is only supported for writes", "ANY ConsistencyLevel is only supported for writes" }; Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message)); } } // Test writes which should fail foreach (ConsistencyLevel consistencyLevel in failList) { try { _policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel); Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel)); } catch (UnavailableException) { // expected to fail when the client has already marked the // node as DOWN } catch (WriteTimeoutException) { // expected to fail when the client hasn't marked the' // node as DOWN yet } } // Test reads which should fail foreach (ConsistencyLevel consistencyLevel in failList) { try { _policyTestTools.Query(testCluster, 12, consistencyLevel); Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel)); } catch (ReadTimeoutException) { // expected to fail when the client hasn't marked the' // node as DOWN yet } catch (UnavailableException) { // expected to fail when the client has already marked the // node as DOWN } } }
public void ReplicationFactorThree_TwoDCs_DowngradingConsistencyRetryPolicy() { ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(3, 3, DefaultMaxClusterCreateRetries, true); testCluster.Builder = Cluster.Builder() .WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy())) .WithRetryPolicy(DowngradingConsistencyRetryPolicy.Instance); testCluster.InitClient(); _policyTestTools.CreateMultiDcSchema(testCluster.Session, 3, 3); _policyTestTools.InitPreparedStatement(testCluster, 12, ConsistencyLevel.Two); // a maximum of 4 IPs should have returned values for the query -- two copies per each of the two DCs int queriesPerIteration = 4; int queriesCompleted = 0; int actualTries = 0; int maxTries = 20; while (_policyTestTools.Coordinators.Count() < 4 && actualTries < maxTries) { _policyTestTools.Query(testCluster, queriesPerIteration, ConsistencyLevel.Two); queriesCompleted += queriesPerIteration; } Assert.IsTrue(_policyTestTools.Coordinators.Count() >= 4, "The minimum number of hosts queried was not met!"); int totalQueriesForAllHosts = _policyTestTools.Coordinators.Sum(c => c.Value); Assert.AreEqual(queriesCompleted, totalQueriesForAllHosts, "The sum of queries for all hosts should equal the number of queries recorded by the calling test!"); _policyTestTools.ResetCoordinators(); testCluster.StopForce(2); // FIXME: This sleep is needed to allow the waitFor() to work TestUtils.WaitForDownWithWait(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 5); var acceptedList = new List <ConsistencyLevel> { ConsistencyLevel.Any, ConsistencyLevel.One, ConsistencyLevel.Two, ConsistencyLevel.Quorum, ConsistencyLevel.Three, ConsistencyLevel.All, ConsistencyLevel.LocalQuorum, ConsistencyLevel.EachQuorum }; var failList = new List <ConsistencyLevel>(); // Test successful writes foreach (ConsistencyLevel consistencyLevel in acceptedList) { try { _policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel); } catch (Exception e) { Assert.Fail(String.Format("Test failed at CL.{0} with message: {1}", consistencyLevel, e.Message)); } } // Test successful reads foreach (ConsistencyLevel consistencyLevel in acceptedList) { try { _policyTestTools.Query(testCluster, 12, consistencyLevel); } catch (InvalidQueryException e) { var acceptableErrorMessages = new List <string> { "EACH_QUORUM ConsistencyLevel is only supported for writes", "ANY ConsistencyLevel is only supported for writes" }; Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message)); } } // Test writes which should fail foreach (ConsistencyLevel consistencyLevel in failList) { try { _policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel); Assert.Fail("Expected Exception was not thrown for ConsistencyLevel :" + consistencyLevel); } catch (UnavailableException) { // expected to fail when the client has already marked the // node as DOWN } catch (WriteTimeoutException) { // expected to fail when the client hasn't marked the' // node as DOWN yet } } // Test reads which should fail foreach (ConsistencyLevel consistencyLevel in failList) { try { _policyTestTools.Query(testCluster, 12, consistencyLevel); Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel)); } catch (ReadTimeoutException) { // expected to fail when the client hasn't marked the' // node as DOWN yet } catch (UnavailableException) { // expected to fail when the client has already marked the // node as DOWN } } }
public void ReplicationFactorTwo_DowngradingConsistencyRetryPolicy() { ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(3); testCluster.Builder = Cluster.Builder() .WithLoadBalancingPolicy(new TokenAwarePolicy(new RoundRobinPolicy())) .WithRetryPolicy(DowngradingConsistencyRetryPolicy.Instance); testCluster.InitClient(); _policyTestTools.CreateSchema(testCluster.Session, 2); _policyTestTools.InitPreparedStatement(testCluster, 12, ConsistencyLevel.Two); _policyTestTools.Query(testCluster, 12, ConsistencyLevel.Two); string coordinatorHostQueried = _policyTestTools.Coordinators.First().Key.Split(':').First();; int awareCoord = int.Parse(coordinatorHostQueried.Split('.').Last()); int coordinatorsWithMoreThanZeroQueries = 0; foreach (var coordinator in _policyTestTools.Coordinators) { coordinatorsWithMoreThanZeroQueries++; _policyTestTools.AssertQueried(coordinator.Key.ToString(), 6); } Assert.AreEqual(2, coordinatorsWithMoreThanZeroQueries); _policyTestTools.ResetCoordinators(); testCluster.StopForce(awareCoord); TestUtils.WaitForDownWithWait(testCluster.ClusterIpPrefix + awareCoord, testCluster.Cluster, 30); var acceptedList = new List <ConsistencyLevel> { ConsistencyLevel.Any, ConsistencyLevel.One, ConsistencyLevel.Two, ConsistencyLevel.Quorum, ConsistencyLevel.Three, ConsistencyLevel.All }; var failList = new List <ConsistencyLevel>(); // Test successful writes foreach (ConsistencyLevel consistencyLevel in acceptedList) { try { _policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel); } catch (Exception e) { Assert.Fail(String.Format("Test failed at CL.{0} with message: {1}", consistencyLevel, e.Message)); } } // Test successful reads foreach (ConsistencyLevel consistencyLevel in acceptedList) { try { _policyTestTools.Query(testCluster, 12, consistencyLevel); } catch (InvalidQueryException e) { var acceptableErrorMessages = new List <string> { "ANY ConsistencyLevel is only supported for writes" }; Assert.True(acceptableErrorMessages.Contains(e.Message)); } } // Test writes which should fail foreach (ConsistencyLevel consistencyLevel in failList) { try { _policyTestTools.InitPreparedStatement(testCluster, 12, consistencyLevel); Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel)); } catch (InvalidQueryException e) { var acceptableErrorMessages = new List <string> { "consistency level LOCAL_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)", "consistency level EACH_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)" }; Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message)); } catch (UnavailableException) { // expected to fail when the client has already marked the // node as DOWN } catch (WriteTimeoutException) { // expected to fail when the client hasn't marked the' // node as DOWN yet } } // Test reads which should fail foreach (ConsistencyLevel consistencyLevel in failList) { try { _policyTestTools.Query(testCluster, 12, consistencyLevel); Assert.Fail(String.Format("Test passed at CL.{0}.", consistencyLevel)); } catch (InvalidQueryException e) { var acceptableErrorMessages = new List <string> { "consistency level LOCAL_QUORUM not compatible with replication strategy (org.apache.cassandra.locator.SimpleStrategy)", "EACH_QUORUM ConsistencyLevel is only supported for writes" }; Assert.True(acceptableErrorMessages.Contains(e.Message), String.Format("Received: {0}", e.Message)); } catch (ReadTimeoutException) { // expected to fail when the client hasn't marked the' // node as DOWN yet } catch (UnavailableException) { // expected to fail when the client has already marked the // node as DOWN } } }
public void RoundRobin_TwoDCs_EachDcHasOneNodeAddedAndDecommissioned() { // Setup PolicyTestTools policyTestTools = new PolicyTestTools(); ITestCluster testCluster = TestClusterManager.GetNonShareableTestCluster(1, 1, DefaultMaxClusterCreateRetries, true); testCluster.Builder = Cluster.Builder().WithLoadBalancingPolicy(new RoundRobinPolicy()); testCluster.InitClient(); policyTestTools.CreateSchema(testCluster.Session); policyTestTools.InitPreparedStatement(testCluster, 12); policyTestTools.Query(testCluster, 12); // Validate that all host were queried equally policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 6); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6); // Add new node to the end of first cluster, remove node from beginning of first cluster policyTestTools.ResetCoordinators(); // Bootstrap step testCluster.BootstrapNode(3, "dc1"); string newlyBootstrappedIp = testCluster.ClusterIpPrefix + "3"; TestUtils.WaitForUp(newlyBootstrappedIp, DefaultCassandraPort, 30); // Validate expected nodes where queried policyTestTools.WaitForPolicyToolsQueryToHitBootstrappedIp(testCluster, newlyBootstrappedIp); policyTestTools.Query(testCluster, 12); policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 4); policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 4); policyTestTools.AssertQueriedAtLeast(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 4); // Remove node from beginning of first cluster policyTestTools.ResetCoordinators(); testCluster.DecommissionNode(1); TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, testCluster.Cluster, 20); // Validate expected nodes where queried policyTestTools.Query(testCluster, 12); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 6); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 6); // Add new node to the end of second cluster, remove node from beginning of first cluster policyTestTools.ResetCoordinators(); testCluster.BootstrapNode(4, "dc2"); newlyBootstrappedIp = testCluster.ClusterIpPrefix + "4"; TestUtils.WaitForUp(newlyBootstrappedIp, DefaultCassandraPort, 30); policyTestTools.ResetCoordinators(); policyTestTools.Query(testCluster, 12); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 4); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 4); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, 4); // Remove node from beginning of second cluster policyTestTools.ResetCoordinators(); testCluster.DecommissionNode(2); TestUtils.waitForDecommission(testCluster.ClusterIpPrefix + "2", testCluster.Cluster, 20); policyTestTools.Query(testCluster, 12); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, 0); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, 0); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, 6); policyTestTools.AssertQueried(testCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, 6); }
private bool TryToInititalizeClusterClient(ITestCluster testCluster) { try { foreach (string host in testCluster.ExpectedInitialHosts) TestUtils.WaitForUp(host, DefaultCassandraPort, 30); // at this point we expect all the nodes to be up testCluster.InitClient(); return true; } catch (Exception e) { Trace.TraceError("Unexpected Error occurred when trying to get shared test cluster with InitialContactPoint: " + testCluster.InitialContactPoint + ", name: " + testCluster.Name); Trace.TraceError("Error Message: " + e.Message); Trace.TraceError("Error Stack Trace: " + e.StackTrace); Trace.TraceError("Removing this cluster, and looping back to create a new one ... "); } return false; }
public void FailoverThenReconnect() { var parallelOptions = new ParallelOptions { TaskScheduler = new ThreadPerTaskScheduler(), MaxDegreeOfParallelism = 100 }; var policy = new ConstantReconnectionPolicy(500); ITestCluster nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(4); nonShareableTestCluster.Builder = new Builder().WithReconnectionPolicy(policy); nonShareableTestCluster.InitClient(); // this will replace the existing session using the newly assigned Builder instance var session = nonShareableTestCluster.Session; // Check query to host distribution before killing nodes var queriedHosts = new List <string>(); DateTime futureDateTime = DateTime.Now.AddSeconds(120); while ((from singleHost in queriedHosts select singleHost).Distinct().Count() < 4 && DateTime.Now < futureDateTime) { var rs = session.Execute("SELECT * FROM system.schema_columnfamilies"); queriedHosts.Add(rs.Info.QueriedHost.ToString()); Thread.Sleep(50); } Assert.AreEqual(4, (from singleHost in queriedHosts select singleHost).Distinct().Count(), "All hosts should have been queried!"); // Create list of actions Action selectAction = () => { var rs = session.Execute("SELECT * FROM system.schema_columnfamilies"); Assert.Greater(rs.Count(), 0); }; var actions = new List <Action>(); for (var i = 0; i < 100; i++) { actions.Add(selectAction); } //Check that the control connection is using first host StringAssert.StartsWith(nonShareableTestCluster.ClusterIpPrefix + "1", nonShareableTestCluster.Cluster.Metadata.ControlConnection.BindAddress.ToString()); //Kill some nodes //Including the one used by the control connection actions.Insert(20, () => nonShareableTestCluster.Stop(1)); actions.Insert(20, () => nonShareableTestCluster.Stop(2)); actions.Insert(80, () => nonShareableTestCluster.Stop(3)); //Execute in parallel more than 100 actions Parallel.Invoke(parallelOptions, actions.ToArray()); //Wait for the nodes to be killed TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "1", nonShareableTestCluster.Cluster, 20); TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "2", nonShareableTestCluster.Cluster, 20); TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "3", nonShareableTestCluster.Cluster, 20); actions = new List <Action>(); for (var i = 0; i < 100; i++) { actions.Add(selectAction); } //Check that the control connection is using first host //bring back some nodes actions.Insert(3, () => nonShareableTestCluster.Start(3)); actions.Insert(50, () => nonShareableTestCluster.Start(2)); actions.Insert(50, () => nonShareableTestCluster.Start(1)); //Execute in parallel more than 100 actions Trace.TraceInformation("Start invoking with restart nodes"); Parallel.Invoke(parallelOptions, actions.ToArray()); //Wait for the nodes to be restarted TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "1", DefaultCassandraPort, 30); TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "2", DefaultCassandraPort, 30); TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "3", DefaultCassandraPort, 30); queriedHosts.Clear(); // keep querying hosts until they are all queried, or time runs out futureDateTime = DateTime.Now.AddSeconds(120); while ((from singleHost in queriedHosts select singleHost).Distinct().Count() < 4 && DateTime.Now < futureDateTime) { var rs = session.Execute("SELECT * FROM system.schema_columnfamilies"); queriedHosts.Add(rs.Info.QueriedHost.ToString()); Thread.Sleep(50); } //Check that one of the restarted nodes were queried Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, queriedHosts); Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, queriedHosts); Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, queriedHosts); Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, queriedHosts); //Check that the control connection is still using last host StringAssert.StartsWith(nonShareableTestCluster.ClusterIpPrefix + "4", nonShareableTestCluster.Cluster.Metadata.ControlConnection.BindAddress.ToString()); }