public async Task Cluster_Should_StopSendingPeersV2Requests_When_InvalidQueryIsThrown()
{
var oldLevel = Diagnostics.CassandraTraceSwitch.Level;
var listener = new TestTraceListener();
Trace.Listeners.Add(listener);
Diagnostics.CassandraTraceSwitch.Level = TraceLevel.Verbose;
try
{
TestCluster.PrimeFluent(
p => p.WhenQuery("SELECT * FROM system.peers_v2")
.ThenServerError(ServerError.Invalid, "error"));
SetupNewSession(b => b.WithPoolingOptions(new PoolingOptions().SetHeartBeatInterval(3000)));
var peersV2Queries = TestCluster.GetQueries("SELECT * FROM system.peers_v2");
var peersQueries = TestCluster.GetQueries("SELECT * FROM system.peers");
await TestCluster.GetNode(Session.Cluster.Metadata.ControlConnection.Host.Address).Stop().ConfigureAwait(false);
// wait until control connection reconnection is done
TestHelper.RetryAssert(
() =>
{
Assert.AreEqual(1, Session.Cluster.AllHosts().Count(h => !h.IsUp));
Assert.IsTrue(Session.Cluster.Metadata.ControlConnection.Host.IsUp);
},
200,
100);
await TestCluster.GetNode(Session.Cluster.Metadata.ControlConnection.Host.Address).Stop().ConfigureAwait(false);
// wait until control connection reconnection is done
TestHelper.RetryAssert(
() =>
{
Assert.AreEqual(2, Session.Cluster.AllHosts().Count(h => !h.IsUp));
Assert.IsTrue(Session.Cluster.Metadata.ControlConnection.Host.IsUp);
},
200,
100);
var afterPeersV2Queries = TestCluster.GetQueries("SELECT * FROM system.peers_v2");
var afterPeersQueries = TestCluster.GetQueries("SELECT * FROM system.peers");
Assert.AreEqual(peersV2Queries.Count, afterPeersV2Queries.Count);
Assert.AreEqual(peersQueries.Count + 2, afterPeersQueries.Count);
}
catch (Exception ex)
{
Trace.Flush();
Assert.Fail(ex.ToString() + Environment.NewLine + string.Join(Environment.NewLine, listener.Queue.ToList()));
}
finally
{
Trace.Listeners.Remove(listener);
Diagnostics.CassandraTraceSwitch.Level = oldLevel;
}
}
public override void SetUp()
{
base.SetUp();
var contactPoint = TestCluster.GetNode(1).ContactPoint;
var separatorIndex = contactPoint.IndexOf(":", StringComparison.Ordinal);
var address = contactPoint.Substring(0, separatorIndex);
_addressNode2 = IPAddress.Parse(address);
}
public async Task Should_RetryOnNextNodes_When_ANodeIsPaused(bool streamMode)
{
var pausedNode = TestCluster.GetNode(2);
SetupNewSession(b =>
b.WithPoolingOptions(
new PoolingOptions()
.SetCoreConnectionsPerHost(HostDistance.Local, 1)
.SetMaxConnectionsPerHost(HostDistance.Local, 1))
.WithSocketOptions(
new SocketOptions()
.SetReadTimeoutMillis(2000)
.SetStreamMode(streamMode)
.SetDefunctReadTimeoutThreshold(int.MaxValue)));
var maxRequestsPerConnection =
Session.Cluster.Configuration
.GetOrCreatePoolingOptions(Session.Cluster.Metadata.ControlConnection.ProtocolVersion)
.GetMaxRequestsPerConnection();
var tenKbBuffer = new byte[10240];
await pausedNode.PauseReadsAsync().ConfigureAwait(false);
// send number of requests = max pending
var requests =
Enumerable.Repeat(0, maxRequestsPerConnection * Session.Cluster.AllHosts().Count)
.Select(i => Session.ExecuteAsync(new SimpleStatement("INSERT INTO table1 (id) VALUES (?)", tenKbBuffer))).ToList();
var pools = InternalSession.GetPools().ToList();
var runningNodesPools = pools.Where(kvp => !kvp.Key.Equals(pausedNode.IpEndPoint));
var pausedNodePool = pools.Single(kvp => kvp.Key.Equals(pausedNode.IpEndPoint));
var connections = pools.SelectMany(kvp => kvp.Value.ConnectionsSnapshot).ToList();
var runningNodesConnections = runningNodesPools.SelectMany(kvp => kvp.Value.ConnectionsSnapshot).ToList();
var pausedNodeConnections = pausedNodePool.Value.ConnectionsSnapshot;
await Task.WhenAll(requests).ConfigureAwait(false);
await AssertRetryUntilWriteQueueStabilizesAsync(connections).ConfigureAwait(false);
TestHelper.RetryAssert(
() =>
{
Assert.IsTrue(runningNodesConnections.All(c => c.InFlight == 0));
Assert.IsTrue(runningNodesConnections.All(c => c.WriteQueueLength == 0));
Assert.IsTrue(runningNodesConnections.All(c => c.PendingOperationsMapLength == 0));
},
100,
100);
Assert.IsTrue(pausedNodeConnections.All(c => c.InFlight > 0));
Assert.IsTrue(pausedNodeConnections.All(c => c.WriteQueueLength > 0));
Assert.IsTrue(pausedNodeConnections.All(c => c.PendingOperationsMapLength > 0));
}
public void SpeculativeExecution_Should_Execute_On_Next_Node()
{
var session = GetSession(new ConstantSpeculativeExecutionPolicy(50L, 1));
TestCluster.GetNode(1).PrimeFluent(
b => b.WhenQuery(QueryLocal)
.ThenRowsSuccess(new[] { "key" }, r => r.WithRow("local")).WithDelayInMs(10000));
TestHelper.ParallelInvoke(() =>
{
var rs = session.Execute(new SimpleStatement(QueryLocal).SetIdempotence(true));
Assert.AreNotEqual(_addressNode2, rs.Info.QueriedHost.Address);
}, 10);
}
public async Task SpeculativeExecution_Should_Not_Execute_On_Next_Node_When_Not_Idempotent()
{
var lbp = new OrderedLoadBalancingPolicy(
TestCluster.GetNode(1).Address.ToString(), TestCluster.GetNode(0).Address.ToString());
var session = GetSession(new ConstantSpeculativeExecutionPolicy(50L, 1), true, lbp);
TestCluster.GetNode(1).PrimeFluent(
b => b.WhenQuery(QueryLocal)
.ThenRowsSuccess(new[] { "key" }, r => r.WithRow("local")).WithDelayInMs(1000));
var rs = await session.ExecuteAsync(new SimpleStatement(QueryLocal).SetIdempotence(false)).ConfigureAwait(false);
// Used the first host in the query plan
Assert.AreEqual(TestCluster.GetNode(1).IpEndPoint, rs.Info.QueriedHost);
}
public async Task Cluster_Should_StopSendingPeersV2Requests_When_InvalidQueryIsThrown()
{
TestCluster.PrimeFluent(
p => p.WhenQuery("SELECT * FROM system.peers_v2")
.ThenServerError(ServerError.Invalid, "error"));
SetupNewSession();
var peersV2Queries = TestCluster.GetQueries("SELECT * FROM system.peers_v2");
var peersQueries = TestCluster.GetQueries("SELECT * FROM system.peers");
await TestCluster.GetNode(Session.Cluster.Metadata.ControlConnection.Host.Address).Stop().ConfigureAwait(false);
// wait until control connection reconnection is done
TestHelper.RetryAssert(
() =>
{
Assert.AreEqual(1, Session.Cluster.AllHosts().Count(h => !h.IsUp));
Assert.IsTrue(Session.Cluster.Metadata.ControlConnection.Host.IsUp);
},
100,
50);
await TestCluster.GetNode(Session.Cluster.Metadata.ControlConnection.Host.Address).Stop().ConfigureAwait(false);
// wait until control connection reconnection is done
TestHelper.RetryAssert(
() =>
{
Assert.AreEqual(2, Session.Cluster.AllHosts().Count(h => !h.IsUp));
Assert.IsTrue(Session.Cluster.Metadata.ControlConnection.Host.IsUp);
},
100,
50);
var afterPeersV2Queries = TestCluster.GetQueries("SELECT * FROM system.peers_v2");
var afterPeersQueries = TestCluster.GetQueries("SELECT * FROM system.peers");
Assert.AreEqual(peersV2Queries.Count, afterPeersV2Queries.Count);
Assert.AreEqual(peersQueries.Count + 2, afterPeersQueries.Count);
}
public async Task Should_ContinueRoutingTrafficToNonPausedNodes_When_ANodeIsPaused(bool streamMode)
{
var pausedNode = TestCluster.GetNode(2);
const string profileName = "running-nodes";
SetupNewSession(b =>
b.WithPoolingOptions(
new PoolingOptions()
.SetCoreConnectionsPerHost(HostDistance.Local, 1)
.SetMaxConnectionsPerHost(HostDistance.Local, 1))
.WithSocketOptions(
new SocketOptions()
.SetReadTimeoutMillis(120000)
.SetStreamMode(streamMode))
.WithExecutionProfiles(opt => opt
.WithProfile(profileName, profile => profile
.WithLoadBalancingPolicy(
new TestDisallowListLbp(
Cassandra.Policies.NewDefaultLoadBalancingPolicy("dc1"))))));
var maxRequestsPerConnection =
Session.Cluster.Configuration
.GetOrCreatePoolingOptions(Session.Cluster.Metadata.ControlConnection.ProtocolVersion)
.GetMaxRequestsPerConnection();
var tenKbBuffer = new byte[10240];
await pausedNode.PauseReadsAsync().ConfigureAwait(false);
// send number of requests = max pending
var requests =
Enumerable.Repeat(0, maxRequestsPerConnection * Session.Cluster.AllHosts().Count)
.Select(i => Session.ExecuteAsync(new SimpleStatement("INSERT INTO table1 (id) VALUES (?)", tenKbBuffer))).ToList();
try
{
var pools = InternalSession.GetPools().ToList();
var runningNodesPools = pools.Where(kvp => !kvp.Key.Equals(pausedNode.IpEndPoint));
var pausedNodePool = pools.Single(kvp => kvp.Key.Equals(pausedNode.IpEndPoint));
var connections = pools.SelectMany(kvp => kvp.Value.ConnectionsSnapshot).ToList();
var runningNodesConnections = runningNodesPools.SelectMany(kvp => kvp.Value.ConnectionsSnapshot).ToList();
var pausedNodeConnections = pausedNodePool.Value.ConnectionsSnapshot;
await AssertRetryUntilWriteQueueStabilizesAsync(connections).ConfigureAwait(false);
TestHelper.RetryAssert(
() =>
{
Assert.IsTrue(runningNodesConnections.All(c => c.InFlight == 0));
Assert.IsTrue(runningNodesConnections.All(c => c.WriteQueueLength == 0));
Assert.IsTrue(runningNodesConnections.All(c => c.PendingOperationsMapLength == 0));
},
100,
100);
Assert.IsTrue(pausedNodeConnections.All(c => c.InFlight > 0));
Assert.IsTrue(pausedNodeConnections.All(c => c.WriteQueueLength > 0));
Assert.IsTrue(pausedNodeConnections.All(c => c.PendingOperationsMapLength > 0));
var writeQueueLengths = pausedNodeConnections.Select(c => c.WriteQueueLength);
Assert.AreEqual(pausedNodeConnections.Sum(c => c.InFlight), requests.Count(t => !t.IsCompleted && !t.IsFaulted));
// these should succeed because we are not hitting the paused node with the custom profile
var moreRequests =
Enumerable.Range(0, 100)
.Select(i => Session.ExecuteAsync(
new SimpleStatement("INSERT INTO table1 (id) VALUES (?)", tenKbBuffer),
profileName))
.ToList();
await Task.WhenAll(moreRequests).ConfigureAwait(false);
Assert.IsTrue(moreRequests.All(t => t.IsCompleted && !t.IsFaulted && !t.IsCanceled));
CollectionAssert.AreEqual(writeQueueLengths, pausedNodeConnections.Select(c => c.WriteQueueLength));
}
finally
{
await TestCluster.ResumeReadsAsync().ConfigureAwait(false);
await(await Task.WhenAny(Task.WhenAll(requests), Task.Delay(5000)).ConfigureAwait(false)).ConfigureAwait(false);
Assert.IsTrue(requests.All(t => t.IsCompleted && !t.IsFaulted && !t.IsCanceled));
}
}