public void Execute_should_throw_when_maxTime_is_exceeded(
[Values(false, true)] bool async)
{
RequireServer.Check().ClusterTypes(ClusterType.Standalone, ClusterType.ReplicaSet).VersionLessThan("4.8.0");
var subject = new GeoSearchOperation <BsonDocument>(
_collectionNamespace,
new BsonArray {
1, 2
},
BsonDocumentSerializer.Instance,
_messageEncoderSettings)
{
MaxDistance = 20,
Search = new BsonDocument("Type", "restaraunt")
};
subject.MaxTime = TimeSpan.FromSeconds(9001);
using (var failPoint = FailPoint.ConfigureAlwaysOn(_cluster, _session, FailPointName.MaxTimeAlwaysTimeout))
{
var exception = Record.Exception(() => ExecuteOperation(subject, failPoint.Binding, async));
exception.Should().BeOfType <MongoExecutionTimeoutException>();
}
}
// private methods
private FailPoint ConfigureFailPoint(string failpointCommand)
{
var cluster = DriverTestConfiguration.Client.Cluster;
var session = NoCoreSession.NewHandle();
return(FailPoint.Configure(cluster, session, BsonDocument.Parse(failpointCommand)));
}
// public methods
public void ConfigureFailPoint(IServer server, ICoreSessionHandle session, BsonDocument failCommand)
{
ConfigureFailPointCommand(failCommand);
var failPoint = FailPoint.Configure(server, session, failCommand);
AddDisposable(failPoint);
}
public async Task ConfigureFailPointAsync(IServer server, ICoreSessionHandle session, BsonDocument failCommand)
{
ConfigureFailPointCommand(failCommand);
var failPoint = await Task.Run(() => FailPoint.Configure(server, session, failCommand)).ConfigureAwait(false);
AddDisposable(failPoint);
}
public void Execute(out FailPoint failPoint)
{
var cluster = _client.Cluster;
var session = NoCoreSession.NewHandle();
failPoint = FailPoint.Configure(cluster, session, _failPointCommand);
}
private FailPoint ConfigureFailPoint(BsonDocument test, ICluster cluster)
{
if (test.TryGetValue(Schema.Intergration.failPoint, out var failPoint))
{
var command = failPoint.AsBsonDocument;
return(FailPoint.Configure(cluster, NoCoreSession.NewHandle(), command));
}
return(null);
}
public void Execute(out FailPoint failPoint)
{
var pinnedServer = _session?.WrappedCoreSession?.CurrentTransaction?.PinnedServer;
if (pinnedServer == null)
{
throw new InvalidOperationException("UnifiedTargetedFailPointOperation requires a pinned server.");
}
var session = NoCoreSession.NewHandle();
failPoint = FailPoint.Configure(pinnedServer, session, _failPointCommand);
}
protected FailPoint ConfigureFailPoint(BsonDocument test)
{
if (test.TryGetValue(FailPointKey, out var failPoint))
{
var cluster = DriverTestConfiguration.Client.Cluster;
var server = cluster.SelectServer(WritableServerSelector.Instance, CancellationToken.None);
var session = NoCoreSession.NewHandle();
var command = failPoint.AsBsonDocument;
return(FailPoint.Configure(cluster, session, command));
}
return(null);
}
private (IConnectionPool, FailPoint, ICluster) SetupConnectionData(BsonDocument test, IEventSubscriber eventSubscriber, bool isUnit)
{
ParseSettings(test, out var connectionPoolSettings, out var connectionSettings);
IConnectionPool connectionPool;
ICluster cluster = null;
FailPoint failPoint = null;
if (isUnit)
{
var endPoint = new DnsEndPoint("localhost", 27017);
var serverId = new ServerId(new ClusterId(), endPoint);
var connectionFactory = new Mock <IConnectionFactory>();
connectionFactory
.Setup(c => c.CreateConnection(serverId, endPoint))
.Returns(() =>
{
var connection = new MockConnection(serverId, connectionSettings, eventSubscriber);
connection.Open(CancellationToken.None);
return(connection);
});
connectionPool = new ExclusiveConnectionPool(
serverId,
endPoint,
connectionPoolSettings,
connectionFactory.Object,
eventSubscriber);
connectionPool.Initialize();
}
else
{
failPoint = ConfigureFailPoint(test, CoreTestConfiguration.Cluster);
cluster = CoreTestConfiguration.CreateCluster(b =>
b.ConfigureConnectionPool(c => c.With(
maxConnections: connectionPoolSettings.MaxConnections,
minConnections: connectionPoolSettings.MinConnections,
waitQueueTimeout: connectionPoolSettings.WaitQueueTimeout))
.Subscribe(eventSubscriber));
var server = cluster.SelectServer(WritableServerSelector.Instance, CancellationToken.None);
connectionPool = server._connectionPool();
}
return(connectionPool, failPoint, cluster);
}
public void Execute_should_throw_when_maxTime_is_exceeded(
[Values(false, true)] bool async)
{
RequireServer.Check().Supports(Feature.FailPoints).ClusterTypes(ClusterType.Standalone, ClusterType.ReplicaSet);
var subject = new FindOpcodeOperation <BsonDocument>(_collectionNamespace, BsonDocumentSerializer.Instance, _messageEncoderSettings);
subject.MaxTime = TimeSpan.FromSeconds(9001);
using (var failPoint = FailPoint.ConfigureAlwaysOn(_cluster, _session, FailPointName.MaxTimeAlwaysTimeout))
{
var exception = Record.Exception(() => ExecuteOperation(subject, failPoint.Binding, async));
exception.Should().BeOfType <MongoExecutionTimeoutException>();
}
}
public void Execute_should_throw_when_maxTime_is_exceeded(
[Values(false, true)] bool async)
{
RequireServer.Check().ClusterTypes(ClusterType.Standalone, ClusterType.ReplicaSet);
#pragma warning disable CS0618 // Type or member is obsolete
var subject = new MapReduceOperation <BsonDocument>(_collectionNamespace, _mapFunction, _reduceFunction, _resultSerializer, _messageEncoderSettings);
#pragma warning restore CS0618 // Type or member is obsolete
subject.MaxTime = TimeSpan.FromSeconds(9001);
using (var failPoint = FailPoint.ConfigureAlwaysOn(_cluster, _session, FailPointName.MaxTimeAlwaysTimeout))
{
var exception = Record.Exception(() => ExecuteOperation(subject, failPoint.Binding, async));
exception.Should().BeOfType <MongoExecutionTimeoutException>();
}
}
public void Execute_should_throw_when_maxTime_is_exceeded(
[Values(false, true)] bool async)
{
RequireServer.Check().ClusterTypes(ClusterType.Standalone, ClusterType.ReplicaSet);
var filter = BsonDocument.Parse("{ asdf : 1 }");
var subject = new FindOneAndUpdateOperation <BsonDocument>(_collectionNamespace, filter, _update, _findAndModifyValueDeserializer, _messageEncoderSettings)
{
MaxTime = TimeSpan.FromSeconds(9001)
};
using (var failPoint = FailPoint.ConfigureAlwaysOn(_cluster, _session, FailPointName.MaxTimeAlwaysTimeout))
{
var exception = Record.Exception(() => ExecuteOperation(subject, failPoint.Binding, async));
exception.Should().BeOfType <MongoExecutionTimeoutException>();
}
}
public void Execute_should_throw_when_maxTime_is_exceeded(
[Values(false, true)] bool async)
{
RequireServer.Check().Supports(Feature.AggregateExplain, Feature.FailPoints).ClusterTypes(ClusterType.Standalone, ClusterType.ReplicaSet);
var subject = new AggregateExplainOperation(_collectionNamespace, __pipeline, _messageEncoderSettings)
{
MaxTime = TimeSpan.FromSeconds(9001)
};
using (var failPoint = FailPoint.ConfigureAlwaysOn(CoreTestConfiguration.Cluster, _session, FailPointName.MaxTimeAlwaysTimeout))
{
var exception = Record.Exception(() => ExecuteOperation(subject, failPoint.Binding, async));
exception.Should().BeOfType <MongoExecutionTimeoutException>();
}
}
public void Execute_should_throw_when_maxTime_is_exceeded(
[Values(false, true)] bool async)
{
RequireServer.Check().Supports(Feature.Eval, Feature.FailPoints).Authentication(false).ClusterTypes(ClusterType.Standalone, ClusterType.ReplicaSet);
var function = "return 1";
var subject = new EvalOperation(_adminDatabaseNamespace, function, _messageEncoderSettings)
{
MaxTime = TimeSpan.FromSeconds(9001)
};
using (var failPoint = FailPoint.ConfigureAlwaysOn(_cluster, _session, FailPointName.MaxTimeAlwaysTimeout))
{
var exception = Record.Exception(() => ExecuteOperation(subject, failPoint.Binding, async));
exception.Should().BeOfType <MongoExecutionTimeoutException>();
}
}
public void Execute_should_throw_when_maxTime_is_exceeded(
[Values(false, true)] bool async)
{
RequireServer.Check().Supports(Feature.CreateIndexesCommand, Feature.FailPoints).ClusterTypes(ClusterType.Standalone, ClusterType.ReplicaSet);
var requests = new[] { new CreateIndexRequest(new BsonDocument("x", 1)) };
var subject = new CreateIndexesUsingCommandOperation(_collectionNamespace, requests, _messageEncoderSettings)
{
MaxTime = TimeSpan.FromSeconds(9001)
};
using (var failPoint = FailPoint.ConfigureAlwaysOn(_cluster, _session, FailPointName.MaxTimeAlwaysTimeout))
{
var exception = Record.Exception(() => ExecuteOperation(subject, failPoint.Binding, async));
exception.Should().BeOfType <MongoExecutionTimeoutException>();
}
}
private FailPoint ConfigureFailPoint(BsonDocument test)
{
BsonValue failPoint;
if (test.TryGetValue("failPoint", out failPoint))
{
var cluster = DriverTestConfiguration.Client.Cluster;
var server = cluster.SelectServer(WritableServerSelector.Instance, CancellationToken.None);
var session = NoCoreSession.NewHandle();
var command = failPoint.AsBsonDocument;
Logger.Debug("Configuring failpoint");
return(FailPoint.Configure(cluster, session, command));
}
return(null);
}
public void Monitor_sleep_at_least_minHeartbeatFreqencyMS_between_checks()
{
var minVersion = new SemanticVersion(4, 9, 0, "");
RequireServer.Check().VersionGreaterThanOrEqualTo(minVersion);
const string appName = "SDAMMinHeartbeatFrequencyTest";
var failPointCommand = BsonDocument.Parse(
$@"{{
configureFailPoint : 'failCommand',
mode : {{ 'times' : 5 }},
data :
{{
failCommands : [ 'isMaster' ],
errorCode : 1234,
appName : '{appName}'
}}
}}");
var settings = DriverTestConfiguration.GetClientSettings();
var serverAddress = settings.Servers.First();
settings.Servers = new[] { serverAddress };
// set settings.DirectConnection = true after removing obsolete ConnectionMode
#pragma warning disable CS0618 // Type or member is obsolete
settings.ConnectionMode = ConnectionMode.Direct;
#pragma warning restore CS0618 // Type or member is obsolete
settings.ApplicationName = appName;
settings.ServerSelectionTimeout = TimeSpan.FromSeconds(5);
var server = DriverTestConfiguration.Client.Cluster.SelectServer(new EndPointServerSelector(new DnsEndPoint(serverAddress.Host, serverAddress.Port)), default);
using var failPoint = FailPoint.Configure(server, NoCoreSession.NewHandle(), failPointCommand);
using var client = DriverTestConfiguration.CreateDisposableClient(settings);
var database = client.GetDatabase(DriverTestConfiguration.DatabaseNamespace.DatabaseName);
var sw = Stopwatch.StartNew();
_ = database.RunCommand <BsonDocument>("{ ping : 1 }");
sw.Stop();
sw.ElapsedMilliseconds.Should().BeInRange(2000, 3500);
}
public void Execute_should_throw_when_maxTime_is_exceeded(
[Values(false, true)] bool async)
{
RequireServer.Check().Supports(Feature.FailPoints, Feature.AggregateOut).ClusterTypes(ClusterType.Standalone, ClusterType.ReplicaSet);
var pipeline = new[]
{
BsonDocument.Parse("{ $match : { x : \"x\" } }"),
BsonDocument.Parse("{ $out : \"awesome\" }")
};
var subject = new AggregateToCollectionOperation(_collectionNamespace, pipeline, _messageEncoderSettings)
{
MaxTime = TimeSpan.FromSeconds(9001)
};
using (var failPoint = FailPoint.ConfigureAlwaysOn(_cluster, _session, FailPointName.MaxTimeAlwaysTimeout))
{
var exception = Record.Exception(() => ExecuteOperation(subject, failPoint.Binding, async));
exception.Should().BeOfType <MongoExecutionTimeoutException>();
}
}
private DisposableBundle ConfigureFailPointOnPrimaryOrShardRoutersIfNeeded(IMongoClient client, BsonDocument test)
{
if (!test.TryGetValue("failPoint", out var failPoint))
{
return(null);
}
var cluster = client.Cluster;
var timeOut = TimeSpan.FromSeconds(60);
SpinWait.SpinUntil(() => cluster.Description.Type != ClusterType.Unknown, timeOut).Should().BeTrue();
List <IServer> failPointServers;
switch (cluster.Description.Type)
{
case ClusterType.ReplicaSet:
var primary = cluster.SelectServer(WritableServerSelector.Instance, CancellationToken.None);
failPointServers = new List <IServer> {
primary
};
break;
case ClusterType.Sharded:
failPointServers =
cluster.Description.Servers
.Select(server => server.EndPoint)
.Select(endPoint => cluster.SelectServer(new EndPointServerSelector(endPoint), CancellationToken.None))
.ToList();
break;
default:
throw new Exception($"Unsupported cluster type: {cluster.Description.Type}");
}
var session = NoCoreSession.NewHandle();
var failPoints = failPointServers.Select(s => FailPoint.Configure(s, session, failPoint.AsBsonDocument)).ToList();
return(new DisposableBundle(failPoints));
}
protected FailPoint ConfigureFailPoint(BsonDocument test, IMongoClient client)
{
if (test.TryGetValue(FailPointKey, out var failPoint))
{
Logger.Debug("Configuring failpoint");
ConfigureFailPointCommand(failPoint.AsBsonDocument);
var cluster = client.Cluster;
var settings = client.Settings.Clone();
ConfigureClientSettings(settings, test);
#pragma warning disable CS0618 // Type or member is obsolete
if (settings.ConnectionModeSwitch == ConnectionModeSwitch.UseDirectConnection &&
#pragma warning restore CS0618 // Type or member is obsolete
settings.DirectConnection == true)
{
var serverAddress = EndPointHelper.Parse(settings.Server.ToString());
var selector = new EndPointServerSelector(serverAddress);
_failPointServer = cluster.SelectServer(selector, CancellationToken.None);
}
else
{
_failPointServer = cluster.SelectServer(WritableServerSelector.Instance, CancellationToken.None);
}
var session = NoCoreSession.NewHandle();
var command = failPoint.AsBsonDocument;
return(FailPoint.Configure(_failPointServer, session, command));
}
return(null);
}
public async Task PoolClearedError_write_retryablity_test([Values(false, true)] bool async)
{
RequireServer.Check()
.Supports(Feature.FailPointsBlockConnection)
.ClusterTypes(ClusterType.ReplicaSet, ClusterType.Sharded);
var heartbeatInterval = TimeSpan.FromMilliseconds(50);
var eventsWaitTimeout = TimeSpan.FromMilliseconds(5000);
var failPointCommand = BsonDocument.Parse(
$@"{{
configureFailPoint : 'failCommand',
mode : {{ 'times' : 1 }},
data :
{{
failCommands : [ 'insert' ],
errorCode : 91,
blockConnection: true,
blockTimeMS: 1000,
errorLabels: [""RetryableWriteError""]
}}
}}");
IServerSelector failPointSelector = WritableServerSelector.Instance;
var settings = DriverTestConfiguration.GetClientSettings();
if (CoreTestConfiguration.Cluster.Description.Type == ClusterType.Sharded)
{
var serverAddress = settings.Servers.First();
settings.Servers = new[] { serverAddress };
// set settings.DirectConnection = true after removing obsolete ConnectionMode
#pragma warning disable CS0618 // Type or member is obsolete
settings.ConnectionMode = ConnectionMode.Direct;
#pragma warning restore CS0618 // Type or member is obsolete
failPointSelector = new EndPointServerSelector(new DnsEndPoint(serverAddress.Host, serverAddress.Port));
}
settings.MaxConnectionPoolSize = 1;
settings.RetryWrites = true;
var eventCapturer = new EventCapturer()
.Capture <ConnectionPoolClearedEvent>()
.Capture <ConnectionPoolCheckedOutConnectionEvent>()
.Capture <ConnectionPoolCheckingOutConnectionFailedEvent>()
.CaptureCommandEvents("insert");
var failpointServer = DriverTestConfiguration.Client.Cluster.SelectServer(failPointSelector, default);
using var failPoint = FailPoint.Configure(failpointServer, NoCoreSession.NewHandle(), failPointCommand);
using var client = CreateClient(settings, eventCapturer, heartbeatInterval);
var database = client.GetDatabase(DriverTestConfiguration.DatabaseNamespace.DatabaseName);
var collection = database.GetCollection <BsonDocument>(DriverTestConfiguration.CollectionNamespace.CollectionName);
eventCapturer.Clear();
if (async)
{
await ThreadingUtilities.ExecuteTasksOnNewThreads(2, async _ =>
{
await collection.InsertOneAsync(new BsonDocument("x", 1));
});
}
else
{
ThreadingUtilities.ExecuteOnNewThreads(2, _ =>
{
collection.InsertOne(new BsonDocument("x", 1));
});
}
// wait for 2 CommandSucceededEvent events, meaning that all other events should be received
eventCapturer.WaitForOrThrowIfTimeout(
events => events.OfType <CommandSucceededEvent>().Count() == 2,
eventsWaitTimeout);
eventCapturer.Events.OfType <CommandStartedEvent>().Count().Should().Be(3);
eventCapturer.Events.OfType <CommandFailedEvent>().Count().Should().Be(1);
eventCapturer.Events.OfType <CommandSucceededEvent>().Count().Should().Be(2);
eventCapturer.Events.OfType <ConnectionPoolClearedEvent>().Count().Should().Be(1);
eventCapturer.Events.OfType <ConnectionPoolCheckedOutConnectionEvent>().Count().Should().Be(3);
eventCapturer.Events.OfType <ConnectionPoolCheckingOutConnectionFailedEvent>().Count().Should().Be(1);
}
private (IConnectionPool, FailPoint, ICluster, Func <object, bool>) SetupConnectionData(BsonDocument test, EventCapturer eventCapturer, bool isUnit)
{
ParseSettings(test, out var connectionPoolSettings, out var connectionSettings);
IConnectionPool connectionPool;
ICluster cluster = null;
FailPoint failPoint = null;
Func <object, bool> eventsFilter = _ => true;
if (isUnit)
{
var endPoint = new DnsEndPoint("localhost", 27017);
var serverId = new ServerId(new ClusterId(), endPoint);
var connectionFactory = new Mock <IConnectionFactory>();
var connectionExceptionHandler = new Mock <IConnectionExceptionHandler>();
connectionFactory
.Setup(c => c.CreateConnection(serverId, endPoint))
.Returns(() =>
{
var connection = new MockConnection(serverId, connectionSettings, eventCapturer);
return(connection);
});
connectionPool = new ExclusiveConnectionPool(
serverId,
endPoint,
connectionPoolSettings,
connectionFactory.Object,
eventCapturer,
connectionExceptionHandler.Object);
connectionPool.Initialize();
}
else
{
var async = test.GetValue(Schema.async).ToBoolean();
cluster = CoreTestConfiguration.CreateCluster(b => b
.ConfigureServer(s => s.With(
heartbeatInterval: TimeSpan.FromMinutes(10)))
.ConfigureConnectionPool(c => c.With(
maxConnecting: connectionPoolSettings.MaxConnecting,
maxConnections: connectionPoolSettings.MaxConnections,
minConnections: connectionPoolSettings.MinConnections,
maintenanceInterval: connectionPoolSettings.MaintenanceInterval,
waitQueueTimeout: connectionPoolSettings.WaitQueueTimeout))
.ConfigureConnection(s => s.With(applicationName: $"{connectionSettings.ApplicationName}_async_{async}"))
.Subscribe(eventCapturer));
var server = cluster.SelectServer(WritableServerSelector.Instance, CancellationToken.None);
connectionPool = server._connectionPool();
if (test.TryGetValue(Schema.Intergration.failPoint, out var failPointDocument))
{
if (failPointDocument.AsBsonDocument.Contains("data"))
{
var data = failPointDocument["data"].AsBsonDocument;
if (data.TryGetValue("appName", out var appNameValue))
{
data["appName"] = $"{appNameValue}_async_{async}";
}
}
var resetPool = connectionPoolSettings.MinConnections > 0;
if (resetPool)
{
eventCapturer.WaitForOrThrowIfTimeout(events => events.Any(e => e is ConnectionCreatedEvent), TimeSpan.FromMilliseconds(500));
var connectionIdsToIgnore = new HashSet <int>(eventCapturer.Events
.OfType <ConnectionCreatedEvent>()
.Select(c => c.ConnectionId.LocalValue)
.ToList());
eventsFilter = o =>
{
if (o is ConnectionOpenedEvent
or ConnectionClosedEvent
or ConnectionCreatedEvent
or ConnectionFailedEvent)
{
var connectionId = o.ConnectionId();
return(!connectionIdsToIgnore.Contains(connectionId.LocalValue) &&
EndPointHelper.Equals(connectionId.ServerId.EndPoint, server.EndPoint));
}
if (o is ConnectionPoolReadyEvent
or ConnectionPoolClearedEvent)
{
var serverId = o.ServerId();
return(EndPointHelper.Equals(serverId.EndPoint, server.EndPoint));
}
return(true);
};
connectionPool.Clear(closeInUseConnections: false);
eventCapturer.WaitForOrThrowIfTimeout(events => events.Any(e => e is ConnectionPoolClearedEvent), TimeSpan.FromMilliseconds(500));
}
var failPointServer = CoreTestConfiguration.Cluster.SelectServer(new EndPointServerSelector(server.EndPoint), default);
failPoint = FailPoint.Configure(failPointServer, NoCoreSession.NewHandle(), failPointDocument.AsBsonDocument);
if (resetPool)
{
eventCapturer.Clear();
connectionPool.SetReady();
}
}
}
return(connectionPool, failPoint, cluster, eventsFilter);
}
