public void Stop_should_trigger_immidiate_maintenace_call(
[Values(false, true)] bool checkOutConnection,
[Values(false, true)] bool closeInUseConnection)
{
var eventCapturer = new EventCapturer()
.Capture <ConnectionPoolAddedConnectionEvent>()
.Capture <ConnectionPoolRemovedConnectionEvent>();
int connectionId = 0;
using (var pool = CreatePool(
eventCapturer,
minPoolSize: 1,
connectionFactoryConfigurator: (factory) =>
{
factory
.Setup(f => f.CreateConnection(__serverId, __endPoint))
.Returns(() => new MockConnection(new ConnectionId(__serverId, ++connectionId), new ConnectionSettings(), eventCapturer));
})) // use to ensure that Maintenance attempt has been called
{
var subject = pool._maintenanceHelper();
var maitenanceInPlayTimeout = TimeSpan.FromMilliseconds(50);
eventCapturer.WaitForEventOrThrowIfTimeout <ConnectionPoolAddedConnectionEvent>(maitenanceInPlayTimeout);
eventCapturer.Next().Should().BeOfType <ConnectionPoolAddedConnectionEvent>().Which.ConnectionId.LocalValue.Should().Be(1); // minPoolSize has been enrolled
eventCapturer.Any().Should().BeFalse();
SpinWait.SpinUntil(() => pool.ConnectionHolder._connections().Count > 0, TimeSpan.FromSeconds(1)).Should().BeTrue(); // wait until connection 1 has been returned to the pool after minPoolSize logic
IConnection acquiredConnection = null;
if (checkOutConnection)
{
acquiredConnection = pool.AcquireConnection(CancellationToken.None);
acquiredConnection.ConnectionId.LocalValue.Should().Be(1);
}
IncrementGeneration(pool);
subject.Stop(maxGenerationToReap: closeInUseConnection ? pool.Generation : null);
var requestInPlayTimeout = TimeSpan.FromMilliseconds(100);
if (!closeInUseConnection && checkOutConnection)
{
// connection in progress should be not touched
Thread.Sleep(requestInPlayTimeout);
}
else
{
eventCapturer.WaitForOrThrowIfTimeout((events) => events.OfType <ConnectionPoolRemovedConnectionEvent>().Count() >= 1, requestInPlayTimeout);
eventCapturer.Next().Should().BeOfType <ConnectionPoolRemovedConnectionEvent>();
}
eventCapturer.Any().Should().BeFalse();
pool.AvailableCount.Should().Be(checkOutConnection ? pool.Settings.MaxConnections - 1 : pool.Settings.MaxConnections);
pool.CreatedCount.Should().Be(checkOutConnection ? 1 : 0);
pool.DormantCount.Should().Be(0);
pool.PendingCount.Should().Be(0);
pool.UsedCount.Should().Be(checkOutConnection ? 1 : 0);
}
}
private void Wait()
{
var eventCondition = MapEventNameToCondition(_event);
Func <IEnumerable <object>, bool> eventsConditionWithFilterByCount = (events) => events.Count(eventCondition) >= _count;
_eventCapturer.WaitForOrThrowIfTimeout(
eventsConditionWithFilterByCount,
TimeSpan.FromSeconds(10),
(timeout) =>
{
var triggeredEventsCount = _eventCapturer.Events.Count(eventCondition);
return($"Waiting for {_count} {_event} exceeded the timeout {timeout}. The number of triggered events is {triggeredEventsCount}.");
});
}
public void Heartbeat_should_work_as_expected()
{
var heartbeatSuceededTimestamps = new ConcurrentQueue <DateTime>();
var eventCapturer = new EventCapturer()
.Capture <ServerHeartbeatSucceededEvent>(
(@event) =>
{
heartbeatSuceededTimestamps.Enqueue(DateTime.UtcNow);
return(true);
}
);
var heartbeatInterval = TimeSpan.FromMilliseconds(500);
eventCapturer.Clear();
using (var client = CreateClient(eventCapturer, heartbeatInterval))
{
eventCapturer.WaitForOrThrowIfTimeout(
events => events.Count() > 3, // wait for at least 3 events
TimeSpan.FromSeconds(10),
(timeout) =>
{
return($"Waiting for the expected events exceeded the timeout {timeout}. The number of triggered events is {eventCapturer.Events.ToList().Count}.");
});
}
var heartbeatSuceededTimestampsList = heartbeatSuceededTimestamps.ToList();
// we have at least 3 items here
// Skip the first event because we have nothing to compare it to
for (int i = 1; i < heartbeatSuceededTimestampsList.Count; i++)
{
var attemptDuration = heartbeatSuceededTimestampsList[i] - heartbeatSuceededTimestampsList[i - 1];
attemptDuration
.Should()
.BeLessThan(TimeSpan.FromSeconds(2));
// Assert the client processes isMaster replies more frequently than 10 secs (approximately every 500ms)
}
}
public void Heartbeat_should_make_immediate_next_attempt_for_streaming_protocol(string exceptionType, bool?moreToCome)
{
var capturedEvents = new EventCapturer()
.Capture <ServerHeartbeatSucceededEvent>()
.Capture <ServerHeartbeatFailedEvent>()
.Capture <ServerDescriptionChangedEvent>();
var subject = CreateSubject(out var mockConnection, out _, out var mockRoundTimeTripMonitor, capturedEvents);
subject.DescriptionChanged +=
(o, e) =>
{
capturedEvents.TryGetEventHandler <ServerDescriptionChangedEvent>(out var eventHandler);
eventHandler(new ServerDescriptionChangedEvent(e.OldServerDescription, e.NewServerDescription));
};
SetupHeartbeatConnection(mockConnection, isStreamable: true, autoFillStreamingResponses: false);
Exception exception = null;
switch (exceptionType)
{
case null:
mockConnection.EnqueueCommandResponseMessage(CreateStreamableCommandResponseMessage(moreToCome.Value), null);
break;
case "MongoConnectionException":
// previousDescription type is "Known" for this case
mockConnection.EnqueueCommandResponseMessage(
exception = CoreExceptionHelper.CreateException(exceptionType));
break;
}
// 10 seconds delay. Not expected to be processed
mockConnection.EnqueueCommandResponseMessage(CreateStreamableCommandResponseMessage(), TimeSpan.FromSeconds(10));
subject.Initialize();
var expectedServerDescriptionChangedEventCount = exception != null
? 3 // +1 event because a connection initialized event doesn't have waiting
: 2;
capturedEvents.WaitForOrThrowIfTimeout(
events =>
events.Count(e => e is ServerDescriptionChangedEvent) >= expectedServerDescriptionChangedEventCount, // the connection has been initialized and the first heatbeat event has been fired
TimeSpan.FromSeconds(10));
capturedEvents.Next().Should().BeOfType <ServerDescriptionChangedEvent>(); // connection initialized
AssertHeartbeatAttempt();
capturedEvents.Any().Should().BeFalse(); // the next attempt will be in 10 seconds because the second stremable respone has 10 seconds delay
void AssertHeartbeatAttempt()
{
if (exception != null)
{
mockRoundTimeTripMonitor.Verify(c => c.Reset(), Times.Once);
var serverHeartbeatFailedEvent = capturedEvents.Next().Should().BeOfType <ServerHeartbeatFailedEvent>().Subject; // updating the server based on the heartbeat
serverHeartbeatFailedEvent.Exception.Should().Be(exception);
var serverDescriptionChangedEvent = capturedEvents.Next().Should().BeOfType <ServerDescriptionChangedEvent>().Subject;
serverDescriptionChangedEvent.NewDescription.HeartbeatException.Should().Be(exception);
serverDescriptionChangedEvent = capturedEvents.Next().Should().BeOfType <ServerDescriptionChangedEvent>().Subject; // when we catch exceptions, we close the current connection, so opening connection will trigger one more ServerDescriptionChangedEvent
serverDescriptionChangedEvent.OldDescription.HeartbeatException.Should().Be(exception);
serverDescriptionChangedEvent.NewDescription.HeartbeatException.Should().BeNull();
}
else
{
mockRoundTimeTripMonitor.Verify(c => c.Reset(), Times.Never);
capturedEvents.Next().Should().BeOfType <ServerHeartbeatSucceededEvent>();
var serverDescriptionChangedEvent = capturedEvents.Next().Should().BeOfType <ServerDescriptionChangedEvent>().Subject; // updating the server based on the heartbeat
serverDescriptionChangedEvent.NewDescription.HeartbeatException.Should().BeNull();
}
}
}
public async Task PoolClearedError_write_retryablity_test([Values(false, true)] bool async)
{
RequireServer.Check()
.Supports(Feature.FailPointsBlockConnection)
.ClusterTypes(ClusterType.ReplicaSet, ClusterType.Sharded);
var heartbeatInterval = TimeSpan.FromMilliseconds(50);
var eventsWaitTimeout = TimeSpan.FromMilliseconds(5000);
var failPointCommand = BsonDocument.Parse(
$@"{{
configureFailPoint : 'failCommand',
mode : {{ 'times' : 1 }},
data :
{{
failCommands : [ 'insert' ],
errorCode : 91,
blockConnection: true,
blockTimeMS: 1000,
errorLabels: [""RetryableWriteError""]
}}
}}");
IServerSelector failPointSelector = WritableServerSelector.Instance;
var settings = DriverTestConfiguration.GetClientSettings();
if (CoreTestConfiguration.Cluster.Description.Type == ClusterType.Sharded)
{
var serverAddress = settings.Servers.First();
settings.Servers = new[] { serverAddress };
// set settings.DirectConnection = true after removing obsolete ConnectionMode
#pragma warning disable CS0618 // Type or member is obsolete
settings.ConnectionMode = ConnectionMode.Direct;
#pragma warning restore CS0618 // Type or member is obsolete
failPointSelector = new EndPointServerSelector(new DnsEndPoint(serverAddress.Host, serverAddress.Port));
}
settings.MaxConnectionPoolSize = 1;
settings.RetryWrites = true;
var eventCapturer = new EventCapturer()
.Capture <ConnectionPoolClearedEvent>()
.Capture <ConnectionPoolCheckedOutConnectionEvent>()
.Capture <ConnectionPoolCheckingOutConnectionFailedEvent>()
.CaptureCommandEvents("insert");
var failpointServer = DriverTestConfiguration.Client.Cluster.SelectServer(failPointSelector, default);
using var failPoint = FailPoint.Configure(failpointServer, NoCoreSession.NewHandle(), failPointCommand);
using var client = CreateClient(settings, eventCapturer, heartbeatInterval);
var database = client.GetDatabase(DriverTestConfiguration.DatabaseNamespace.DatabaseName);
var collection = database.GetCollection <BsonDocument>(DriverTestConfiguration.CollectionNamespace.CollectionName);
eventCapturer.Clear();
if (async)
{
await ThreadingUtilities.ExecuteTasksOnNewThreads(2, async _ =>
{
await collection.InsertOneAsync(new BsonDocument("x", 1));
});
}
else
{
ThreadingUtilities.ExecuteOnNewThreads(2, _ =>
{
collection.InsertOne(new BsonDocument("x", 1));
});
}
// wait for 2 CommandSucceededEvent events, meaning that all other events should be received
eventCapturer.WaitForOrThrowIfTimeout(
events => events.OfType <CommandSucceededEvent>().Count() == 2,
eventsWaitTimeout);
eventCapturer.Events.OfType <CommandStartedEvent>().Count().Should().Be(3);
eventCapturer.Events.OfType <CommandFailedEvent>().Count().Should().Be(1);
eventCapturer.Events.OfType <CommandSucceededEvent>().Count().Should().Be(2);
eventCapturer.Events.OfType <ConnectionPoolClearedEvent>().Count().Should().Be(1);
eventCapturer.Events.OfType <ConnectionPoolCheckedOutConnectionEvent>().Count().Should().Be(3);
eventCapturer.Events.OfType <ConnectionPoolCheckingOutConnectionFailedEvent>().Count().Should().Be(1);
}
public async Task Ensure_server_session_are_allocated_only_on_connection_checkout([Values(true, false)] bool async)
{
var eventCapturer = new EventCapturer()
.Capture <CommandStartedEvent>();
using var client = DriverTestConfiguration.CreateDisposableClient(
(MongoClientSettings settings) =>
{
settings.RetryWrites = true;
settings.MaxConnectionPoolSize = 1;
settings.ClusterConfigurator = c => c.Subscribe(eventCapturer);
},
logger: null);
var database = client.GetDatabase("test");
database.DropCollection("inventory");
var collection = database.GetCollection <BsonDocument>("inventory");
const int operationsCount = 8;
var singleSessionUsed = false;
for (int i = 0; i < 5; i++)
{
eventCapturer.Clear();
await ThreadingUtilities.ExecuteTasksOnNewThreads(operationsCount, async i =>
{
switch (i)
{
case 0:
if (async)
{
await collection.InsertOneAsync(new BsonDocument("x", 0));
}
else
{
collection.InsertOne(new BsonDocument("x", 0));
}
break;
case 1:
if (async)
{
await collection.DeleteOneAsync(Builders <BsonDocument> .Filter.Eq("_id", 1));
}
else
{
collection.DeleteOne(Builders <BsonDocument> .Filter.Eq("_id", 1));
}
break;
case 2:
if (async)
{
await collection.UpdateOneAsync(Builders <BsonDocument> .Filter.Empty, Builders <BsonDocument> .Update.Set("a", 1));
}
else
{
collection.UpdateOne(Builders <BsonDocument> .Filter.Empty, Builders <BsonDocument> .Update.Set("a", 1));
}
break;
case 3:
var bulkWriteRequests = new WriteModel <BsonDocument>[]
{
new UpdateOneModel <BsonDocument>(Builders <BsonDocument> .Filter.Empty, new BsonDocument("$set", new BsonDocument("1", 1)))
};
if (async)
{
await collection.BulkWriteAsync(bulkWriteRequests);
}
else
{
collection.BulkWrite(bulkWriteRequests);
}
break;
case 4:
if (async)
{
await collection.FindOneAndDeleteAsync(Builders <BsonDocument> .Filter.Empty);
}
else
{
collection.FindOneAndDelete(Builders <BsonDocument> .Filter.Empty);
}
break;
case 5:
if (async)
{
await collection.FindOneAndUpdateAsync(Builders <BsonDocument> .Filter.Empty, Builders <BsonDocument> .Update.Set("a", 1));
}
else
{
collection.FindOneAndUpdate(Builders <BsonDocument> .Filter.Empty, Builders <BsonDocument> .Update.Set("a", 1));
}
break;
case 6:
if (async)
{
await collection.FindOneAndReplaceAsync(Builders <BsonDocument> .Filter.Empty, new BsonDocument("x", 0));
}
else
{
collection.FindOneAndReplace(Builders <BsonDocument> .Filter.Empty, new BsonDocument("x", 0));
}
break;
case 7:
if (async)
{
var cursor = await collection.FindAsync(Builders <BsonDocument> .Filter.Empty);
_ = await cursor.ToListAsync();
}
else
{
_ = collection.Find(Builders <BsonDocument> .Filter.Empty).ToList();
}
break;
}
});
eventCapturer.WaitForOrThrowIfTimeout(e => e.OfType <CommandStartedEvent>().Count() >= operationsCount, TimeSpan.FromSeconds(10));
var lsids = eventCapturer.Events.OfType <CommandStartedEvent>().Select(c => c.Command["lsid"]).ToArray();
var distinctLsidsCount = lsids.Distinct().Count();
distinctLsidsCount.Should().BeLessThan(operationsCount);
if (distinctLsidsCount == 1)
{
singleSessionUsed = true;
break;
}
}
singleSessionUsed.Should().BeTrue("At least one iteration should use single session");
}
public void Ensure_command_network_error_before_hadnshake_is_correctly_handled([Values(false, true)] bool async, [Values(false, true)] bool streamable)
{
var eventCapturer = new EventCapturer().Capture <ServerDescriptionChangedEvent>();
// ensure that hello or legacy hello check response is finished only after network error
var hasNetworkErrorBeenTriggered = new TaskCompletionSource <bool>();
// ensure that there are no unexpected events between test ending and cluster disposing
var hasClusterBeenDisposed = new TaskCompletionSource <bool>();
EndPoint initialSelectedEndpoint = null;
using (var cluster = CreateAndSetupCluster(hasNetworkErrorBeenTriggered, hasClusterBeenDisposed, eventCapturer, streamable))
{
ForceClusterId(cluster, __clusterId);
// 0. Initial heartbeat via `connection.Open`
// The next hello or legacy hello response will be delayed because the Task.WaitAny in the mock.Returns
cluster.Initialize();
var selectedServer = cluster.SelectServer(CreateWritableServerAndEndPointSelector(__endPoint1), CancellationToken.None);
initialSelectedEndpoint = selectedServer.EndPoint;
initialSelectedEndpoint.Should().Be(__endPoint1);
// make sure the next hello or legacy hello check has been called
Thread.Sleep(__heartbeatInterval + TimeSpan.FromMilliseconds(50));
// 1. Trigger the command network error BEFORE handshake. At this time hello or legacy hello response is already delayed until `hasNetworkErrorBeenTriggered.SetResult`
Exception exception;
if (async)
{
exception = Record.Exception(() => selectedServer.GetChannelAsync(CancellationToken.None).GetAwaiter().GetResult());
}
else
{
exception = Record.Exception(() => selectedServer.GetChannel(CancellationToken.None));
}
var e = exception.Should().BeOfType <MongoConnectionException>().Subject;
e.Message.Should().Be("DnsException");
// 2. Waiting for the hello or legacy hello check
hasNetworkErrorBeenTriggered.SetResult(true); // unlock the in-progress hello or legacy hello response
Thread.Sleep(100); // make sure the delayed hello or legacy hello check had time to change description if there is a bug
var knownServers = cluster.Description.Servers.Where(s => s.Type != ServerType.Unknown);
if (knownServers.Select(s => s.EndPoint).Contains(initialSelectedEndpoint))
{
throw new Exception($"The type of failed server {initialSelectedEndpoint} has not been changed to Unknown.");
}
// ensure that a new server can be selected
selectedServer = cluster.SelectServer(WritableServerSelector.Instance, CancellationToken.None);
// ensure that the selected server is not the same as the initial
selectedServer.EndPoint.Should().Be(__endPoint2);
// the 4th event is MongoConnectionException which will trigger the next hello or legacy hello check immediately
eventCapturer.WaitForOrThrowIfTimeout(events => events.Count() >= 4, TimeSpan.FromSeconds(5));
}
hasClusterBeenDisposed.SetCanceled(); // Cut off not related events. Stop waiting in the latest mock.Returns for OpenAsync
// Events asserting
var initialHeartbeatEvents = new[]
{
// endpoints can be in random order
eventCapturer.Next().Should().BeOfType <ServerDescriptionChangedEvent>().Subject,
eventCapturer.Next().Should().BeOfType <ServerDescriptionChangedEvent>().Subject
}
.OrderBy(c => GetPort(c.NewDescription.EndPoint))
.ToList();
AssertEvent(initialHeartbeatEvents[0], __endPoint1, ServerType.ShardRouter, "Heartbeat");
AssertEvent(initialHeartbeatEvents[1], __endPoint2, ServerType.ShardRouter, "Heartbeat"); // the next 27018 events will be suppressed
AssertNextEvent(eventCapturer, initialSelectedEndpoint, ServerType.Unknown, "InvalidatedBecause:ChannelException during handshake: MongoDB.Driver.MongoConnectionException: DnsException");
AssertNextEvent(eventCapturer, initialSelectedEndpoint, ServerType.Unknown, "Heartbeat", typeof(MongoConnectionException));
eventCapturer.Any().Should().BeFalse();
int GetPort(EndPoint endpoint) => ((DnsEndPoint)endpoint).Port;
}
private (IConnectionPool, FailPoint, ICluster, Func <object, bool>) SetupConnectionData(BsonDocument test, EventCapturer eventCapturer, bool isUnit)
{
ParseSettings(test, out var connectionPoolSettings, out var connectionSettings);
IConnectionPool connectionPool;
ICluster cluster = null;
FailPoint failPoint = null;
Func <object, bool> eventsFilter = _ => true;
if (isUnit)
{
var endPoint = new DnsEndPoint("localhost", 27017);
var serverId = new ServerId(new ClusterId(), endPoint);
var connectionFactory = new Mock <IConnectionFactory>();
var connectionExceptionHandler = new Mock <IConnectionExceptionHandler>();
connectionFactory
.Setup(c => c.CreateConnection(serverId, endPoint))
.Returns(() =>
{
var connection = new MockConnection(serverId, connectionSettings, eventCapturer);
return(connection);
});
connectionPool = new ExclusiveConnectionPool(
serverId,
endPoint,
connectionPoolSettings,
connectionFactory.Object,
eventCapturer,
connectionExceptionHandler.Object);
connectionPool.Initialize();
}
else
{
var async = test.GetValue(Schema.async).ToBoolean();
cluster = CoreTestConfiguration.CreateCluster(b => b
.ConfigureServer(s => s.With(
heartbeatInterval: TimeSpan.FromMinutes(10)))
.ConfigureConnectionPool(c => c.With(
maxConnecting: connectionPoolSettings.MaxConnecting,
maxConnections: connectionPoolSettings.MaxConnections,
minConnections: connectionPoolSettings.MinConnections,
maintenanceInterval: connectionPoolSettings.MaintenanceInterval,
waitQueueTimeout: connectionPoolSettings.WaitQueueTimeout))
.ConfigureConnection(s => s.With(applicationName: $"{connectionSettings.ApplicationName}_async_{async}"))
.Subscribe(eventCapturer));
var server = cluster.SelectServer(WritableServerSelector.Instance, CancellationToken.None);
connectionPool = server._connectionPool();
if (test.TryGetValue(Schema.Intergration.failPoint, out var failPointDocument))
{
if (failPointDocument.AsBsonDocument.Contains("data"))
{
var data = failPointDocument["data"].AsBsonDocument;
if (data.TryGetValue("appName", out var appNameValue))
{
data["appName"] = $"{appNameValue}_async_{async}";
}
}
var resetPool = connectionPoolSettings.MinConnections > 0;
if (resetPool)
{
eventCapturer.WaitForOrThrowIfTimeout(events => events.Any(e => e is ConnectionCreatedEvent), TimeSpan.FromMilliseconds(500));
var connectionIdsToIgnore = new HashSet <int>(eventCapturer.Events
.OfType <ConnectionCreatedEvent>()
.Select(c => c.ConnectionId.LocalValue)
.ToList());
eventsFilter = o =>
{
if (o is ConnectionOpenedEvent
or ConnectionClosedEvent
or ConnectionCreatedEvent
or ConnectionFailedEvent)
{
var connectionId = o.ConnectionId();
return(!connectionIdsToIgnore.Contains(connectionId.LocalValue) &&
EndPointHelper.Equals(connectionId.ServerId.EndPoint, server.EndPoint));
}
if (o is ConnectionPoolReadyEvent
or ConnectionPoolClearedEvent)
{
var serverId = o.ServerId();
return(EndPointHelper.Equals(serverId.EndPoint, server.EndPoint));
}
return(true);
};
connectionPool.Clear(closeInUseConnections: false);
eventCapturer.WaitForOrThrowIfTimeout(events => events.Any(e => e is ConnectionPoolClearedEvent), TimeSpan.FromMilliseconds(500));
}
var failPointServer = CoreTestConfiguration.Cluster.SelectServer(new EndPointServerSelector(server.EndPoint), default);
failPoint = FailPoint.Configure(failPointServer, NoCoreSession.NewHandle(), failPointDocument.AsBsonDocument);
if (resetPool)
{
eventCapturer.Clear();
connectionPool.SetReady();
}
}
}
return(connectionPool, failPoint, cluster, eventsFilter);
}
