public async Task Should_KeepOperationsInWriteQueue_When_ServerAppliesTcpBackPressure(bool streamMode)
{
SetupNewSession(b =>
b.WithPoolingOptions(
new PoolingOptions()
.SetCoreConnectionsPerHost(HostDistance.Local, 1)
.SetMaxConnectionsPerHost(HostDistance.Local, 1))
.WithSocketOptions(new SocketOptions()
.SetReadTimeoutMillis(360000)
.SetStreamMode(streamMode)));
var maxRequestsPerConnection = Session.Cluster.Configuration
.GetOrCreatePoolingOptions(Session.Cluster.Metadata.ControlConnection.ProtocolVersion)
.GetMaxRequestsPerConnection();
var tenKbBuffer = new byte[10240];
await TestCluster.PauseReadsAsync().ConfigureAwait(false);
var pools = InternalSession.GetPools().ToList();
var connections = pools.SelectMany(kvp => kvp.Value.ConnectionsSnapshot).ToList();
var requests = new List <Task>();
using (var cts = new CancellationTokenSource())
{
var task = Task.Run(
async() =>
{
while (!cts.IsCancellationRequested)
{
requests.Add(Session.ExecuteAsync(new SimpleStatement("INSERT INTO table1 (id) VALUES (?)", tenKbBuffer)));
await Task.Yield();
}
},
cts.Token);
await AssertRetryUntilWriteQueueStabilizesAsync(connections, maxRequestsPerConnection).ConfigureAwait(false);
cts.Cancel();
await task.ConfigureAwait(false);
}
Assert.IsTrue(connections.All(c => c.WriteQueueLength > 0));
var writeQueueSizes = connections.ToDictionary(c => c, c => c.WriteQueueLength, ReferenceEqualityComparer <IConnection> .Instance);
var pendingOps = connections.ToDictionary(c => c, c => c.PendingOperationsMapLength, ReferenceEqualityComparer <IConnection> .Instance);
// these should fail because we have hit max pending ops
var moreRequests =
Enumerable.Range(0, 100)
.Select(i => Task.Run(() => Session.ExecuteAsync(new SimpleStatement("INSERT INTO table1 (id) VALUES (?)", tenKbBuffer))))
.ToList();
try
{
try
{
await(await Task.WhenAny(Task.WhenAll(moreRequests), Task.Delay(15000)).ConfigureAwait(false)).ConfigureAwait(false);
Assert.Fail("Should throw exception.");
}
catch (NoHostAvailableException)
{
// ignored
}
var moreFailedRequests = moreRequests.Where(t => t.IsFaulted).ToList();
Assert.Greater(moreFailedRequests.Count, 1);
Assert.AreEqual(moreRequests.Count, moreFailedRequests.Count);
Assert.GreaterOrEqual(connections.Sum(c => c.InFlight), maxRequestsPerConnection * Session.Cluster.AllHosts().Count);
// ReSharper disable once PossibleNullReferenceException
Assert.IsTrue(moreFailedRequests.All(t => t.IsFaulted && ((NoHostAvailableException)t.Exception.InnerException).Errors.All(e => e.Value is BusyPoolException)));
var newWriteQueueSizes =
connections.ToDictionary(c => c, c => c.WriteQueueLength, ReferenceEqualityComparer <IConnection> .Instance);
var newPendingsOps =
connections.ToDictionary(c => c, c => c.PendingOperationsMapLength, ReferenceEqualityComparer <IConnection> .Instance);
foreach (var kvp in writeQueueSizes)
{
Assert.GreaterOrEqual(newWriteQueueSizes[kvp.Key], kvp.Value);
Assert.Greater(newWriteQueueSizes[kvp.Key], 1);
}
foreach (var kvp in pendingOps)
{
Assert.AreEqual(newPendingsOps[kvp.Key], kvp.Value);
Assert.Greater(newPendingsOps[kvp.Key], 1);
}
}
finally
{
await TestCluster.ResumeReadsAsync().ConfigureAwait(false);
try
{
await(await Task.WhenAny(Task.WhenAll(requests), Task.Delay(15000)).ConfigureAwait(false)).ConfigureAwait(false);
}
catch (NoHostAvailableException)
{
}
Assert.AreEqual(
requests.Count,
requests.Count(t => t.IsCompleted && !t.IsFaulted && !t.IsCanceled)
+ requests.Count(t => t.IsFaulted &&
((NoHostAvailableException)t.Exception.InnerException)
.Errors.All(e => e.Value is BusyPoolException)));
}
}
public async Task Should_ContinueRoutingTrafficToNonPausedNodes_When_ANodeIsPaused(bool streamMode)
{
var pausedNode = TestCluster.GetNode(2);
const string profileName = "running-nodes";
SetupNewSession(b =>
b.WithPoolingOptions(
new PoolingOptions()
.SetCoreConnectionsPerHost(HostDistance.Local, 1)
.SetMaxConnectionsPerHost(HostDistance.Local, 1))
.WithSocketOptions(
new SocketOptions()
.SetReadTimeoutMillis(120000)
.SetStreamMode(streamMode))
.WithExecutionProfiles(opt => opt
.WithProfile(profileName, profile => profile
.WithLoadBalancingPolicy(
new TestDisallowListLbp(
Cassandra.Policies.NewDefaultLoadBalancingPolicy("dc1"))))));
var maxRequestsPerConnection =
Session.Cluster.Configuration
.GetOrCreatePoolingOptions(Session.Cluster.Metadata.ControlConnection.ProtocolVersion)
.GetMaxRequestsPerConnection();
var tenKbBuffer = new byte[10240];
await pausedNode.PauseReadsAsync().ConfigureAwait(false);
// send number of requests = max pending
var requests =
Enumerable.Repeat(0, maxRequestsPerConnection * Session.Cluster.AllHosts().Count)
.Select(i => Session.ExecuteAsync(new SimpleStatement("INSERT INTO table1 (id) VALUES (?)", tenKbBuffer))).ToList();
try
{
var pools = InternalSession.GetPools().ToList();
var runningNodesPools = pools.Where(kvp => !kvp.Key.Equals(pausedNode.IpEndPoint));
var pausedNodePool = pools.Single(kvp => kvp.Key.Equals(pausedNode.IpEndPoint));
var connections = pools.SelectMany(kvp => kvp.Value.ConnectionsSnapshot).ToList();
var runningNodesConnections = runningNodesPools.SelectMany(kvp => kvp.Value.ConnectionsSnapshot).ToList();
var pausedNodeConnections = pausedNodePool.Value.ConnectionsSnapshot;
await AssertRetryUntilWriteQueueStabilizesAsync(connections).ConfigureAwait(false);
TestHelper.RetryAssert(
() =>
{
Assert.IsTrue(runningNodesConnections.All(c => c.InFlight == 0));
Assert.IsTrue(runningNodesConnections.All(c => c.WriteQueueLength == 0));
Assert.IsTrue(runningNodesConnections.All(c => c.PendingOperationsMapLength == 0));
},
100,
100);
Assert.IsTrue(pausedNodeConnections.All(c => c.InFlight > 0));
Assert.IsTrue(pausedNodeConnections.All(c => c.WriteQueueLength > 0));
Assert.IsTrue(pausedNodeConnections.All(c => c.PendingOperationsMapLength > 0));
var writeQueueLengths = pausedNodeConnections.Select(c => c.WriteQueueLength);
Assert.AreEqual(pausedNodeConnections.Sum(c => c.InFlight), requests.Count(t => !t.IsCompleted && !t.IsFaulted));
// these should succeed because we are not hitting the paused node with the custom profile
var moreRequests =
Enumerable.Range(0, 100)
.Select(i => Session.ExecuteAsync(
new SimpleStatement("INSERT INTO table1 (id) VALUES (?)", tenKbBuffer),
profileName))
.ToList();
await Task.WhenAll(moreRequests).ConfigureAwait(false);
Assert.IsTrue(moreRequests.All(t => t.IsCompleted && !t.IsFaulted && !t.IsCanceled));
CollectionAssert.AreEqual(writeQueueLengths, pausedNodeConnections.Select(c => c.WriteQueueLength));
}
finally
{
await TestCluster.ResumeReadsAsync().ConfigureAwait(false);
await(await Task.WhenAny(Task.WhenAll(requests), Task.Delay(5000)).ConfigureAwait(false)).ConfigureAwait(false);
Assert.IsTrue(requests.All(t => t.IsCompleted && !t.IsFaulted && !t.IsCanceled));
}
}