public void TestRemotelyCloseConnectionDuringSessionCreation()
{
using (TestAmqpPeer testAmqpPeer = new TestAmqpPeer(Address, User, Password))
{
testAmqpPeer.Open();
NmsConnectionFactory factory = new NmsConnectionFactory(Address);
IConnection connection = factory.CreateConnection(User, Password);
// We need to set request timeout because there may be a deadlock when we try to
// create amqplite session before underlying connection changes its state do disconnected
connection.RequestTimeout = TimeSpan.FromMilliseconds(1000);
// Explicitly close the connection with an error
testAmqpPeer.Close();
try
{
connection.CreateSession();
Assert.Fail("Expected exception to be thrown");
}
catch (NMSException e)
{
Console.WriteLine(e);
}
connection.Close();
}
}
public void TestStartMaxReconnectAttemptsTriggeredWhenRemotesAreRejecting()
{
using (TestAmqpPeer firstPeer = new TestAmqpPeer())
using (TestAmqpPeer secondPeer = new TestAmqpPeer())
using (TestAmqpPeer thirdPeer = new TestAmqpPeer())
using (TestAmqpPeer fourthPeer = new TestAmqpPeer())
{
ManualResetEvent failedConnection = new ManualResetEvent(false);
firstPeer.RejectConnect(AmqpError.NOT_FOUND, "Resource could not be located");
secondPeer.RejectConnect(AmqpError.NOT_FOUND, "Resource could not be located");
thirdPeer.RejectConnect(AmqpError.NOT_FOUND, "Resource could not be located");
// This shouldn't get hit, but if it does accept the connect so we don't pass the failed
// to connect assertion.
fourthPeer.ExpectSaslAnonymous();
fourthPeer.ExpectOpen();
fourthPeer.ExpectBegin();
fourthPeer.ExpectClose();
NmsConnection connection = EstablishAnonymousConnection("failover.startupMaxReconnectAttempts=3&failover.reconnectDelay=15&failover.useReconnectBackOff=false",
firstPeer, secondPeer, thirdPeer, fourthPeer);
Mock <INmsConnectionListener> connectionListener = new Mock <INmsConnectionListener>();
connectionListener
.Setup(listener => listener.OnConnectionFailure(It.IsAny <NMSException>()))
.Callback(() => { failedConnection.Set(); });
connection.AddConnectionListener(connectionListener.Object);
Assert.Catch <NMSException>(() => connection.Start(), "Should not be able to connect");
Assert.True(failedConnection.WaitOne(TimeSpan.FromSeconds(5)));
try
{
connection.Close();
}
catch (NMSException e)
{
}
firstPeer.WaitForAllMatchersToComplete(2000);
secondPeer.WaitForAllMatchersToComplete(2000);
thirdPeer.WaitForAllMatchersToComplete(2000);
// Shut down last peer and verify no connection made to it
fourthPeer.PurgeExpectations();
fourthPeer.Close();
Assert.NotNull(firstPeer.ClientSocket, "Peer 1 should have accepted a TCP connection");
Assert.NotNull(secondPeer.ClientSocket, "Peer 2 should have accepted a TCP connection");
Assert.NotNull(thirdPeer.ClientSocket, "Peer 3 should have accepted a TCP connection");
Assert.IsNull(fourthPeer.ClientSocket, "Peer 4 should not have accepted any TCP connection");
}
}
public void TestRemotelyEndConnectionWithSessionWithConsumer()
{
using (TestAmqpPeer testAmqpPeer = new TestAmqpPeer(Address, User, Password))
{
testAmqpPeer.Open();
NmsConnectionFactory factory = new NmsConnectionFactory(Address);
IConnection connection = factory.CreateConnection(User, Password);
connection.Start();
ISession session = connection.CreateSession(AcknowledgementMode.AutoAcknowledge);
IQueue queue = session.GetQueue("myQueue");
var consumer = session.CreateConsumer(queue);
// Explicitly close the connection with an error
testAmqpPeer.Close();
Assert.That(() => ((NmsConnection)connection).IsConnected, Is.False.After(200, 50), "Connection never closed");
Assert.That(() => ((NmsConnection)connection).IsClosed, Is.True.After(200, 50), "Connection never closed");
try
{
connection.CreateSession(AcknowledgementMode.AutoAcknowledge);
Assert.Fail("Expected ISE to be thrown due to being closed");
}
catch (IllegalStateException)
{
}
// Verify the session is now marked closed
try
{
session.CreateConsumer(queue);
Assert.Fail("Expected ISE to be thrown due to being closed");
}
catch (IllegalStateException)
{
}
// Verify the consumer is now marked closed
try
{
consumer.Listener += message => { };
Assert.Fail("Expected ISE to be thrown due to being closed");
}
catch (IllegalStateException)
{
}
// Try closing them explicitly, should effectively no-op in client.
// The test peer will throw during close if it sends anything.
consumer.Close();
session.Close();
connection.Close();
}
}
private void DoTestConsumerReceiveThrowsIfConnectionLost(bool useTimeout)
{
using (TestAmqpPeer testAmqpPeer = new TestAmqpPeer(Address, User, Password))
{
string topicName = "myTopic";
testAmqpPeer.Open();
ManualResetEvent disconnected = new ManualResetEvent(false);
NmsConnection connection = (NmsConnection)EstablishConnection();
Mock <INmsConnectionListener> connectionListener = new Mock <INmsConnectionListener>();
connectionListener
.Setup(listener => listener.OnConnectionFailure(It.IsAny <NMSException>()))
.Callback(() => { disconnected.Set(); });
connection.AddConnectionListener(connectionListener.Object);
connection.Start();
ISession session = connection.CreateSession(AcknowledgementMode.ClientAcknowledge);
ITopic topic = session.GetTopic(topicName);
testAmqpPeer.SendMessage(topicName, "test");
IMessageConsumer consumer = session.CreateConsumer(topic);
testAmqpPeer.Close();
Assert.True(disconnected.WaitOne(), "Connection should be disconnected");
try
{
if (useTimeout)
{
consumer.Receive(TimeSpan.FromMilliseconds(1000));
}
else
{
consumer.Receive();
}
Assert.Fail("An exception should have been thrown");
}
catch (NMSException)
{
// Expected
}
}
}
public void TestTempDestinationRecreatedAfterConnectionFailsOver()
{
using (TestAmqpPeer originalPeer = new TestAmqpPeer(Address1, User, Password))
using (TestAmqpPeer finalPeer = new TestAmqpPeer(Address2, User, Password))
{
originalPeer.RegisterLinkProcessor(new TestLinkProcessor());
finalPeer.RegisterLinkProcessor(new TestLinkProcessor());
ManualResetEvent originalConnected = new ManualResetEvent(false);
ManualResetEvent finalConnected = new ManualResetEvent(false);
// Create a peer to connect to, then one to reconnect to
originalPeer.Open();
finalPeer.Open();
NmsConnection connection = EstablishConnection(originalPeer, finalPeer);
Mock <INmsConnectionListener> connectionListener = new Mock <INmsConnectionListener>();
connectionListener
.Setup(listener => listener.OnConnectionEstablished(It.Is <Uri>(uri => originalPeer.Address == uri)))
.Callback(() => { originalConnected.Set(); });
connectionListener
.Setup(listener => listener.OnConnectionRestored(It.Is <Uri>(uri => finalPeer.Address == uri)))
.Callback(() => { finalConnected.Set(); });
connection.AddConnectionListener(connectionListener.Object);
connection.Start();
Assert.True(originalConnected.WaitOne(TimeSpan.FromSeconds(5)), "Should connect to original peer");
ISession session = connection.CreateSession(AcknowledgementMode.AutoAcknowledge);
ITemporaryTopic temporaryTopic = session.CreateTemporaryTopic();
originalPeer.Close();
Assert.True(finalConnected.WaitOne(TimeSpan.FromSeconds(5)), "Should connect to final peer");
temporaryTopic.Delete();
connection.Close();
}
}
public void TestCreateProducerAfterConnectionDrops()
{
using (TestAmqpPeer originalPeer = new TestAmqpPeer(Address1, User, Password))
using (TestAmqpPeer finalPeer = new TestAmqpPeer(Address2, User, Password))
{
originalPeer.RegisterLinkProcessor(new TestLinkProcessor());
finalPeer.RegisterLinkProcessor(new TestLinkProcessor());
ManualResetEvent originalConnected = new ManualResetEvent(false);
ManualResetEvent finalConnected = new ManualResetEvent(false);
// Create a peer to connect to, then one to reconnect to
originalPeer.Open();
finalPeer.Open();
NmsConnection connection = EstablishConnection(originalPeer, finalPeer);
Mock <INmsConnectionListener> connectionListener = new Mock <INmsConnectionListener>();
connectionListener
.Setup(listener => listener.OnConnectionEstablished(It.Is <Uri>(uri => originalPeer.Address == uri)))
.Callback(() => { originalConnected.Set(); });
connectionListener
.Setup(listener => listener.OnConnectionRestored(It.Is <Uri>(uri => finalPeer.Address == uri)))
.Callback(() => { finalConnected.Set(); });
connection.AddConnectionListener(connectionListener.Object);
connection.Start();
Assert.True(originalConnected.WaitOne(TimeSpan.FromSeconds(5)), "Should connect to original peer");
originalPeer.Close();
ISession session = connection.CreateSession();
IQueue queue = session.GetQueue("myQueue");
IMessageProducer producer = session.CreateProducer(queue);
Assert.True(finalConnected.WaitOne(TimeSpan.FromSeconds(5)), "Should connect to final peer");
producer.Close();
connection.Close();
}
}
public void TestRemotelyEndConnectionListenerInvoked()
{
using (TestAmqpPeer testAmqpPeer = new TestAmqpPeer(Address, User, Password))
{
testAmqpPeer.Open();
NmsConnectionFactory factory = new NmsConnectionFactory(Address);
IConnection connection = factory.CreateConnection(User, Password);
bool done = false;
connection.ExceptionListener += exception => { done = true; };
connection.Start();
// Explicitly close the connection with an error
testAmqpPeer.Close();
Assert.That(() => done, Is.True.After(1000));
}
}
public void TestFailoverHandlesDropThenRejectionCloseAfterConnect()
{
using (TestAmqpPeer originalPeer = new TestAmqpPeer(Address1, User, Password))
using (TestAmqpPeer rejectingPeer = new TestAmqpPeer(Address2, User, Password))
using (TestAmqpPeer finalPeer = new TestAmqpPeer(Address3, User, Password))
{
ManualResetEvent originalConnected = new ManualResetEvent(false);
ManualResetEvent finalConnected = new ManualResetEvent(false);
// Create a peer to connect to, one to fail to reconnect to, and a final one to reconnect to
originalPeer.Open();
finalPeer.Open();
long ird = 0;
long rd = 2000;
NmsConnection connection = EstablishConnection("failover.initialReconnectDelay=" + ird + "&failover.reconnectDelay=" + rd + "&failover.maxReconnectAttempts=10", originalPeer, rejectingPeer, finalPeer);
Mock <INmsConnectionListener> connectionListener = new Mock <INmsConnectionListener>();
connectionListener
.Setup(listener => listener.OnConnectionEstablished(It.Is <Uri>(uri => originalPeer.Address == uri)))
.Callback(() => { originalConnected.Set(); });
connectionListener
.Setup(listener => listener.OnConnectionRestored(It.Is <Uri>(uri => finalPeer.Address == uri)))
.Callback(() => { finalConnected.Set(); });
connection.AddConnectionListener(connectionListener.Object);
connection.Start();
Assert.True(originalConnected.WaitOne(TimeSpan.FromSeconds(5)), "Should connect to original peer");
Assert.False(finalConnected.WaitOne(TimeSpan.FromMilliseconds(100)), "Should not yet have connected to final peer");
// Close the original peer and wait for things to shake out.
originalPeer.Close();
Assert.True(finalConnected.WaitOne(TimeSpan.FromSeconds(5)), "Should connect to final peer");
connection.Close();
}
}
public void TestFailoverEnforcesSendTimeout()
{
using (TestAmqpPeer testPeer = new TestAmqpPeer(Address1, User, Password))
{
ManualResetEvent connected = new ManualResetEvent(false);
ManualResetEvent disconnected = new ManualResetEvent(false);
testPeer.RegisterLinkProcessor(new TestLinkProcessor());
testPeer.Open();
NmsConnection connection = EstablishConnection("nms.sendTimeout=1000&failover.reconnectDelay=2000&failover.maxReconnectAttempts=60", testPeer);
Mock <INmsConnectionListener> connectionListener = new Mock <INmsConnectionListener>();
connectionListener
.Setup(listener => listener.OnConnectionEstablished(It.IsAny <Uri>()))
.Callback(() => { connected.Set(); });
connectionListener
.Setup(listener => listener.OnConnectionInterrupted(It.IsAny <Uri>()))
.Callback(() => { disconnected.Set(); });
connection.AddConnectionListener(connectionListener.Object);
connection.Start();
Assert.True(connected.WaitOne(TimeSpan.FromSeconds(5)), "Should connect to peer");
ISession session = connection.CreateSession(AcknowledgementMode.AutoAcknowledge);
IQueue queue = session.GetQueue("myQueue");
IMessageProducer producer = session.CreateProducer(queue);
testPeer.Close();
Assert.True(disconnected.WaitOne(TimeSpan.FromSeconds(5)), "Should lose connection to peer");
Assert.Catch <NMSException>(() => producer.Send(producer.CreateTextMessage("test")));
connection.Close();
}
}
public void TestFailoverHandlesDropAfterSessionCloseRequested()
{
using (TestAmqpPeer testPeer = new TestAmqpPeer(Address1, User, Password))
{
ManualResetEvent connected = new ManualResetEvent(false);
testPeer.RegisterLinkProcessor(new MockLinkProcessor(context => context.Complete(new Error(new Symbol("error")))));
testPeer.Open();
NmsConnection connection = EstablishConnection(testPeer);
Mock <INmsConnectionListener> connectionListener = new Mock <INmsConnectionListener>();
connectionListener
.Setup(listener => listener.OnConnectionEstablished(It.IsAny <Uri>()))
.Callback(() => { connected.Set(); });
connection.AddConnectionListener(connectionListener.Object);
connection.Start();
Assert.True(connected.WaitOne(TimeSpan.FromSeconds(5)), "Should connect to peer");
ISession session = connection.CreateSession(AcknowledgementMode.AutoAcknowledge);
testPeer.Close();
try
{
session.Close();
}
catch (NMSException)
{
}
catch (Exception)
{
Assert.Fail("Session close should have completed normally.");
}
connection.Close();
}
}
public void TestFailoverHandlesDropWithModifiedInitialReconnectDelay()
{
using (TestAmqpPeer originalPeer = new TestAmqpPeer(Address1, User, Password))
using (TestAmqpPeer finalPeer = new TestAmqpPeer(Address3, User, Password))
{
ManualResetEvent originalConnected = new ManualResetEvent(false);
ManualResetEvent finalConnected = new ManualResetEvent(false);
// Create a peer to connect to, then one to reconnect to
originalPeer.Open();
finalPeer.Open();
NmsConnection connection = EstablishConnection("failover.initialReconnectDelay=" + 1 + "&failover.reconnectDelay=" + 600 + "&failover.maxReconnectAttempts=10", originalPeer, finalPeer);
Mock <INmsConnectionListener> connectionListener = new Mock <INmsConnectionListener>();
connectionListener
.Setup(listener => listener.OnConnectionEstablished(It.Is <Uri>(uri => originalPeer.Address == uri)))
.Callback(() => { originalConnected.Set(); });
connectionListener
.Setup(listener => listener.OnConnectionRestored(It.Is <Uri>(uri => finalPeer.Address == uri)))
.Callback(() => { finalConnected.Set(); });
connection.AddConnectionListener(connectionListener.Object);
connection.Start();
Assert.True(originalConnected.WaitOne(TimeSpan.FromSeconds(5)), "Should connect to original peer");
// Close the original peer
originalPeer.Close();
connection.CreateSession(AcknowledgementMode.AutoAcknowledge);
Assert.True(finalConnected.WaitOne(TimeSpan.FromSeconds(5)), "Should connect to final peer");
connection.Close();
}
}
public void TestFailoverDoesNotFailPendingSend()
{
using (TestAmqpPeer originalPeer = new TestAmqpPeer(Address1, User, Password))
using (TestAmqpPeer finalPeer = new TestAmqpPeer(Address3, User, Password))
{
ManualResetEvent messageReceived = new ManualResetEvent(false);
finalPeer.RegisterMessageProcessor("q1", context =>
{
messageReceived.Set();
context.Complete();
});
originalPeer.Open();
finalPeer.Open();
NmsConnection connection = EstablishConnection("failover.initialReconnectDelay=10000", finalPeer);
connection.Start();
ISession session = connection.CreateSession();
IQueue queue = session.GetQueue("q1");
IMessageProducer producer = session.CreateProducer(queue);
originalPeer.Close();
Assert.DoesNotThrow(() =>
{
ITextMessage message = session.CreateTextMessage("test");
producer.Send(message);
});
Assert.True(messageReceived.WaitOne(TimeSpan.FromSeconds(5)), "Message should be delivered to final peer.");
connection.Close();
}
}
public void TestFailoverHandlesDropThenRejectionCloseAfterConnect()
{
using (TestAmqpPeer originalPeer = new TestAmqpPeer())
using (TestAmqpPeer rejectingPeer = new TestAmqpPeer())
using (TestAmqpPeer finalPeer = new TestAmqpPeer())
{
ManualResetEvent originalConnected = new ManualResetEvent(false);
ManualResetEvent finalConnected = new ManualResetEvent(false);
// Create a peer to connect to, one to fail to reconnect to, and a final one to reconnect to
var originalUri = CreatePeerUri(originalPeer);
var rejectingUri = CreatePeerUri(rejectingPeer);
var finalUri = CreatePeerUri(finalPeer);
// Connect to the first
originalPeer.ExpectSaslAnonymous();
originalPeer.ExpectOpen();
originalPeer.ExpectBegin();
long ird = 0;
long rd = 2000;
DateTime start = DateTime.UtcNow;
NmsConnection connection = EstablishAnonymousConnection("failover.initialReconnectDelay=" + ird + "&failover.reconnectDelay=" + rd + "&failover.maxReconnectAttempts=10", originalPeer,
rejectingPeer, finalPeer);
Mock <INmsConnectionListener> connectionListener = new Mock <INmsConnectionListener>();
connectionListener
.Setup(listener => listener.OnConnectionEstablished(It.Is <Uri>(uri => originalUri == uri.ToString())))
.Callback(() => { originalConnected.Set(); });
connectionListener
.Setup(listener => listener.OnConnectionRestored(It.Is <Uri>(uri => finalUri == uri.ToString())))
.Callback(() => { finalConnected.Set(); });
connection.AddConnectionListener(connectionListener.Object);
connection.Start();
Assert.True(originalConnected.WaitOne(TimeSpan.FromSeconds(5)), "Should connect to original peer");
Assert.False(finalConnected.WaitOne(TimeSpan.FromMilliseconds(100)), "Should not yet have connected to final peer");
// Set expectations on rejecting and final peer
rejectingPeer.RejectConnect(AmqpError.NOT_FOUND, "Resource could not be located");
finalPeer.ExpectSaslAnonymous();
finalPeer.ExpectOpen();
finalPeer.ExpectBegin();
// Close the original peer and wait for things to shake out.
originalPeer.Close(sendClose: true);
rejectingPeer.WaitForAllMatchersToComplete(1000);
Assert.True(finalConnected.WaitOne(TimeSpan.FromSeconds(5)), "Should connect to final peer");
DateTime end = DateTime.UtcNow;
long margin = 2000;
// TODO: It is failing because, we are not handling rejected connection properly, when socket connection is established
// but broker replies with amqp:connection-establishment-failed. Initially connection is treated as successful, which resets
// the attempts counter. As a result next connect attempt is being made without any delay.
// Assert.That((end - start).TotalMilliseconds, Is.GreaterThanOrEqualTo(ird + rd).And.LessThanOrEqualTo(ird + rd + margin), "Elapsed time outwith expected range for reconnect");
finalPeer.ExpectClose();
connection.Close();
finalPeer.WaitForAllMatchersToComplete(1000);
}
}
public void TestFailoverHandlesDropThenRejectionCloseAfterConnect()
{
using (TestAmqpPeer originalPeer = new TestAmqpPeer())
using (TestAmqpPeer rejectingPeer = new TestAmqpPeer())
using (TestAmqpPeer finalPeer = new TestAmqpPeer())
{
ManualResetEvent originalConnected = new ManualResetEvent(false);
ManualResetEvent finalConnected = new ManualResetEvent(false);
// Create a peer to connect to, one to fail to reconnect to, and a final one to reconnect to
var originalUri = CreatePeerUri(originalPeer);
var rejectingUri = CreatePeerUri(rejectingPeer);
var finalUri = CreatePeerUri(finalPeer);
Logger.Info($"Original peer is at: {originalUri}");
Logger.Info($"Rejecting peer is at: {rejectingUri}");
Logger.Info($"Final peer is at: {finalUri}");
// Connect to the first
originalPeer.ExpectSaslAnonymous();
originalPeer.ExpectOpen();
originalPeer.ExpectBegin();
long ird = 0;
long rd = 2000;
NmsConnection connection = EstablishAnonymousConnection("failover.initialReconnectDelay=" + ird + "&failover.reconnectDelay=" + rd + "&failover.maxReconnectAttempts=10", originalPeer,
rejectingPeer, finalPeer);
Mock <INmsConnectionListener> connectionListener = new Mock <INmsConnectionListener>();
connectionListener
.Setup(listener => listener.OnConnectionEstablished(It.Is <Uri>(uri => originalUri == uri.ToString())))
.Callback(() => { originalConnected.Set(); });
connectionListener
.Setup(listener => listener.OnConnectionRestored(It.Is <Uri>(uri => finalUri == uri.ToString())))
.Callback(() => { finalConnected.Set(); });
connection.AddConnectionListener(connectionListener.Object);
connection.Start();
Assert.True(originalConnected.WaitOne(TimeSpan.FromSeconds(5)), "Should connect to original peer");
Assert.False(finalConnected.WaitOne(TimeSpan.FromMilliseconds(100)), "Should not yet have connected to final peer");
// Set expectations on rejecting and final peer
rejectingPeer.RejectConnect(AmqpError.NOT_FOUND, "Resource could not be located");
finalPeer.ExpectSaslAnonymous();
finalPeer.ExpectOpen();
finalPeer.ExpectBegin();
// Close the original peer and wait for things to shake out.
originalPeer.Close(sendClose: true);
rejectingPeer.WaitForAllMatchersToComplete(2000);
Assert.True(finalConnected.WaitOne(TimeSpan.FromSeconds(10)), "Should connect to final peer");
finalPeer.ExpectClose();
connection.Close();
finalPeer.WaitForAllMatchersToComplete(1000);
}
}