public async Task Should_ThrowOperationTimedOut_When_ServerAppliesTcpBackPressure(bool streamMode)
{
SetupNewSession(b =>
b.WithPoolingOptions(
new PoolingOptions()
.SetCoreConnectionsPerHost(HostDistance.Local, 1)
.SetMaxConnectionsPerHost(HostDistance.Local, 1))
.WithSocketOptions(new SocketOptions()
.SetReadTimeoutMillis(1000)
.SetStreamMode(streamMode)
.SetDefunctReadTimeoutThreshold(int.MaxValue)));
var maxRequestsPerConnection = Session.Cluster.Configuration
.GetOrCreatePoolingOptions(Session.Cluster.Metadata.ControlConnection.ProtocolVersion)
.GetMaxRequestsPerConnection();
var tenKbBuffer = new byte[10240];
await TestCluster.PauseReadsAsync().ConfigureAwait(false);
// send number of requests = max pending
var requests =
Enumerable.Repeat(0, maxRequestsPerConnection * Session.Cluster.AllHosts().Count)
.Select(i => Session.ExecuteAsync(new SimpleStatement("INSERT INTO table1 (id) VALUES (?)", tenKbBuffer))).ToList();
var taskAll = Task.WhenAll(requests);
try
{
await(await Task.WhenAny(taskAll, Task.Delay(60000)).ConfigureAwait(false)).ConfigureAwait(false);
Assert.Fail("Should time out.");
}
catch (NoHostAvailableException)
{
// ignored
}
requests = requests.Where(t => t.IsFaulted).ToList();
Assert.Greater(requests.Count, 1);
Assert.IsTrue(requests.All(
t => ((NoHostAvailableException)t.Exception.InnerException)
.Errors.Any(e => e.Value is OperationTimedOutException)));
}
public async Task Should_KeepOperationsInWriteQueue_When_ServerAppliesTcpBackPressure(bool streamMode)
{
SetupNewSession(b =>
b.WithPoolingOptions(
new PoolingOptions()
.SetCoreConnectionsPerHost(HostDistance.Local, 1)
.SetMaxConnectionsPerHost(HostDistance.Local, 1))
.WithSocketOptions(new SocketOptions()
.SetReadTimeoutMillis(360000)
.SetStreamMode(streamMode)));
var maxRequestsPerConnection = Session.Cluster.Configuration
.GetOrCreatePoolingOptions(Session.Cluster.Metadata.ControlConnection.ProtocolVersion)
.GetMaxRequestsPerConnection();
var tenKbBuffer = new byte[10240];
await TestCluster.PauseReadsAsync().ConfigureAwait(false);
var pools = InternalSession.GetPools().ToList();
var connections = pools.SelectMany(kvp => kvp.Value.ConnectionsSnapshot).ToList();
var requests = new List <Task>();
using (var cts = new CancellationTokenSource())
{
var task = Task.Run(
async() =>
{
while (!cts.IsCancellationRequested)
{
requests.Add(Session.ExecuteAsync(new SimpleStatement("INSERT INTO table1 (id) VALUES (?)", tenKbBuffer)));
await Task.Yield();
}
},
cts.Token);
await AssertRetryUntilWriteQueueStabilizesAsync(connections, maxRequestsPerConnection).ConfigureAwait(false);
cts.Cancel();
await task.ConfigureAwait(false);
}
Assert.IsTrue(connections.All(c => c.WriteQueueLength > 0));
var writeQueueSizes = connections.ToDictionary(c => c, c => c.WriteQueueLength, ReferenceEqualityComparer <IConnection> .Instance);
var pendingOps = connections.ToDictionary(c => c, c => c.PendingOperationsMapLength, ReferenceEqualityComparer <IConnection> .Instance);
// these should fail because we have hit max pending ops
var moreRequests =
Enumerable.Range(0, 100)
.Select(i => Task.Run(() => Session.ExecuteAsync(new SimpleStatement("INSERT INTO table1 (id) VALUES (?)", tenKbBuffer))))
.ToList();
try
{
try
{
await(await Task.WhenAny(Task.WhenAll(moreRequests), Task.Delay(15000)).ConfigureAwait(false)).ConfigureAwait(false);
Assert.Fail("Should throw exception.");
}
catch (NoHostAvailableException)
{
// ignored
}
var moreFailedRequests = moreRequests.Where(t => t.IsFaulted).ToList();
Assert.Greater(moreFailedRequests.Count, 1);
Assert.AreEqual(moreRequests.Count, moreFailedRequests.Count);
Assert.GreaterOrEqual(connections.Sum(c => c.InFlight), maxRequestsPerConnection * Session.Cluster.AllHosts().Count);
// ReSharper disable once PossibleNullReferenceException
Assert.IsTrue(moreFailedRequests.All(t => t.IsFaulted && ((NoHostAvailableException)t.Exception.InnerException).Errors.All(e => e.Value is BusyPoolException)));
var newWriteQueueSizes =
connections.ToDictionary(c => c, c => c.WriteQueueLength, ReferenceEqualityComparer <IConnection> .Instance);
var newPendingsOps =
connections.ToDictionary(c => c, c => c.PendingOperationsMapLength, ReferenceEqualityComparer <IConnection> .Instance);
foreach (var kvp in writeQueueSizes)
{
Assert.GreaterOrEqual(newWriteQueueSizes[kvp.Key], kvp.Value);
Assert.Greater(newWriteQueueSizes[kvp.Key], 1);
}
foreach (var kvp in pendingOps)
{
Assert.AreEqual(newPendingsOps[kvp.Key], kvp.Value);
Assert.Greater(newPendingsOps[kvp.Key], 1);
}
}
finally
{
await TestCluster.ResumeReadsAsync().ConfigureAwait(false);
try
{
await(await Task.WhenAny(Task.WhenAll(requests), Task.Delay(15000)).ConfigureAwait(false)).ConfigureAwait(false);
}
catch (NoHostAvailableException)
{
}
Assert.AreEqual(
requests.Count,
requests.Count(t => t.IsCompleted && !t.IsFaulted && !t.IsCanceled)
+ requests.Count(t => t.IsFaulted &&
((NoHostAvailableException)t.Exception.InnerException)
.Errors.All(e => e.Value is BusyPoolException)));
}
}
