public void TestFailoverHandlesDropAfterSessionCloseRequested()
{
using (TestAmqpPeer originalPeer = new TestAmqpPeer())
{
ManualResetEvent originalConnected = new ManualResetEvent(false);
// Create a peer to connect to
var originalUri = CreatePeerUri(originalPeer);
// Connect to the first peer
originalPeer.ExpectSaslAnonymous();
originalPeer.ExpectOpen();
originalPeer.ExpectBegin();
Mock <INmsConnectionListener> connectionListener = new Mock <INmsConnectionListener>();
connectionListener
.Setup(listener => listener.OnConnectionEstablished(It.Is <Uri>(uri => originalUri == uri.ToString())))
.Callback(() => { originalConnected.Set(); });
NmsConnection connection = EstablishAnonymousConnection(originalPeer);
connection.AddConnectionListener(connectionListener.Object);
connection.Start();
Assert.True(originalConnected.WaitOne(TimeSpan.FromSeconds(5)), "Should connect to peer");
originalPeer.ExpectBegin();
originalPeer.ExpectEnd(sendResponse: false);
originalPeer.DropAfterLastMatcher();
ISession session = connection.CreateSession(AcknowledgementMode.AutoAcknowledge);
ManualResetEvent sessionCloseCompleted = new ManualResetEvent(false);
Exception sessionClosedThrew = null;
Task.Run(() =>
{
try
{
session.Close();
}
catch (Exception e)
{
sessionClosedThrew = e;
}
finally
{
sessionCloseCompleted.Set();
}
});
originalPeer.WaitForAllMatchersToComplete(2000);
Assert.IsTrue(sessionCloseCompleted.WaitOne(TimeSpan.FromSeconds(3)), "Session close should have completed by now");
Assert.IsNull(sessionClosedThrew, "Session close should have completed normally");
connection.Close();
}
}
public void TestCreateProducerAfterConnectionDrops()
{
using (TestAmqpPeer originalPeer = new TestAmqpPeer())
using (TestAmqpPeer finalPeer = new TestAmqpPeer())
{
ManualResetEvent originalConnected = new ManualResetEvent(false);
ManualResetEvent finalConnected = new ManualResetEvent(false);
// Create a peer to connect to, then one to reconnect to
var originalUri = CreatePeerUri(originalPeer);
var finalUri = CreatePeerUri(finalPeer);
// Connect to the first peer
originalPeer.ExpectSaslAnonymous();
originalPeer.ExpectOpen();
originalPeer.ExpectBegin();
originalPeer.ExpectBegin();
originalPeer.DropAfterLastMatcher();
NmsConnection connection = EstablishAnonymousConnection(originalPeer, finalPeer);
Mock <INmsConnectionListener> connectionListener = new Mock <INmsConnectionListener>();
connectionListener
.Setup(listener => listener.OnConnectionEstablished(It.Is <Uri>(uri => originalUri == uri.ToString())))
.Callback(() => { originalConnected.Set(); });
connectionListener
.Setup(listener => listener.OnConnectionRestored(It.Is <Uri>(uri => finalUri == uri.ToString())))
.Callback(() => { finalConnected.Set(); });
connection.AddConnectionListener(connectionListener.Object);
connection.Start();
Assert.True(originalConnected.WaitOne(TimeSpan.FromSeconds(5)), "Should connect to original peer");
// Post Failover Expectations of FinalPeer
finalPeer.ExpectSaslAnonymous();
finalPeer.ExpectOpen();
finalPeer.ExpectBegin();
finalPeer.ExpectBegin();
finalPeer.ExpectSenderAttach();
finalPeer.ExpectDetach(expectClosed: true, sendResponse: true, replyClosed: true);
finalPeer.ExpectClose();
ISession session = connection.CreateSession();
IQueue queue = session.GetQueue("myQueue");
IMessageProducer producer = session.CreateProducer(queue);
Assert.True(finalConnected.WaitOne(TimeSpan.FromSeconds(5)), "Should connect to final peer");
producer.Close();
connection.Close();
finalPeer.WaitForAllMatchersToComplete(1000);
}
}
public void TestFailoverHandlesDropWithModifiedInitialReconnectDelay()
{
using (TestAmqpPeer originalPeer = new TestAmqpPeer())
using (TestAmqpPeer finalPeer = new TestAmqpPeer())
{
ManualResetEvent originalConnected = new ManualResetEvent(false);
ManualResetEvent finalConnected = new ManualResetEvent(false);
// Create a peer to connect to, then one to reconnect to
var originalUri = CreatePeerUri(originalPeer);
var finalUri = CreatePeerUri(finalPeer);
// Connect to the first peer
originalPeer.ExpectSaslAnonymous();
originalPeer.ExpectOpen();
originalPeer.ExpectBegin();
originalPeer.ExpectBegin();
originalPeer.DropAfterLastMatcher();
NmsConnection connection = EstablishAnonymousConnection("failover.initialReconnectDelay=1&failover.reconnectDelay=600&failover.maxReconnectAttempts=10", originalPeer, finalPeer);
Mock <INmsConnectionListener> connectionListener = new Mock <INmsConnectionListener>();
connectionListener
.Setup(listener => listener.OnConnectionEstablished(It.Is <Uri>(uri => originalUri == uri.ToString())))
.Callback(() => { originalConnected.Set(); });
connectionListener
.Setup(listener => listener.OnConnectionRestored(It.Is <Uri>(uri => finalUri == uri.ToString())))
.Callback(() => { finalConnected.Set(); });
connection.AddConnectionListener(connectionListener.Object);
connection.Start();
Assert.True(originalConnected.WaitOne(TimeSpan.FromSeconds(5)), "Should connect to original peer");
// Post Failover Expectations of FinalPeer
finalPeer.ExpectSaslAnonymous();
finalPeer.ExpectOpen();
finalPeer.ExpectBegin();
finalPeer.ExpectBegin();
connection.CreateSession(AcknowledgementMode.AutoAcknowledge);
Assert.True(finalConnected.WaitOne(TimeSpan.FromSeconds(5)), "Should connect to final peer");
// Shut it down
finalPeer.ExpectClose();
connection.Close();
finalPeer.WaitForAllMatchersToComplete(1000);
}
}
private async Task DoTestConsumerReceiveThrowsIfConnectionLost(bool useTimeout)
{
ManualResetEvent consumerReady = new ManualResetEvent(false);
using (TestAmqpPeer testPeer = new TestAmqpPeer())
{
IConnection connection = await EstablishConnectionAsync(testPeer);
testPeer.ExpectBegin();
ISession session = await connection.CreateSessionAsync(AcknowledgementMode.AutoAcknowledge);
IQueue queue = await session.GetQueueAsync("queue");
await connection.StartAsync();
testPeer.ExpectReceiverAttach();
testPeer.ExpectLinkFlow();
testPeer.RunAfterLastHandler(() => { consumerReady.WaitOne(2000); });
testPeer.DropAfterLastMatcher(delay: 10);
IMessageConsumer consumer = await session.CreateConsumerAsync(queue);
consumerReady.Set();
try
{
if (useTimeout)
{
await consumer.ReceiveAsync(TimeSpan.FromMilliseconds(10_0000));
}
else
{
await consumer.ReceiveAsync();
}
Assert.Fail("An exception should have been thrown");
}
catch (NMSException)
{
// Expected
}
testPeer.WaitForAllMatchersToComplete(1000);
}
}
public void TestFailoverEnforcesSendTimeout()
{
using (TestAmqpPeer testPeer = new TestAmqpPeer())
{
ManualResetEvent connected = new ManualResetEvent(false);
ManualResetEvent disconnected = new ManualResetEvent(false);
// Connect to the test peer
testPeer.ExpectSaslAnonymous();
testPeer.ExpectOpen();
testPeer.ExpectBegin();
testPeer.ExpectBegin();
testPeer.ExpectSenderAttach();
testPeer.DropAfterLastMatcher();
NmsConnection connection = EstablishAnonymousConnection("nms.sendTimeout=1000&failover.reconnectDelay=2000&failover.maxReconnectAttempts=60", testPeer);
Mock <INmsConnectionListener> connectionListener = new Mock <INmsConnectionListener>();
connectionListener
.Setup(listener => listener.OnConnectionEstablished(It.IsAny <Uri>()))
.Callback(() => { connected.Set(); });
connectionListener
.Setup(listener => listener.OnConnectionInterrupted(It.IsAny <Uri>()))
.Callback(() => { disconnected.Set(); });
connection.AddConnectionListener(connectionListener.Object);
connection.Start();
Assert.True(connected.WaitOne(TimeSpan.FromSeconds(5)), "Should connect to peer");
ISession session = connection.CreateSession(AcknowledgementMode.AutoAcknowledge);
IQueue queue = session.GetQueue("myQueue");
IMessageProducer producer = session.CreateProducer(queue);
Assert.True(disconnected.WaitOne(TimeSpan.FromSeconds(5)), "Should lose connection to peer");
Assert.Catch <NMSException>(() => producer.Send(producer.CreateTextMessage("test")));
connection.Close();
testPeer.WaitForAllMatchersToComplete(1000);
}
}
public void TestFailoverDoesNotFailPendingSend()
{
using (TestAmqpPeer originalPeer = new TestAmqpPeer())
using (TestAmqpPeer finalPeer = new TestAmqpPeer())
{
originalPeer.ExpectSaslAnonymous();
originalPeer.ExpectOpen();
originalPeer.ExpectBegin();
originalPeer.ExpectBegin();
// Ensure our send blocks in the provider waiting for credit so that on failover
// the message will actually get sent from the Failover bits once we grant some
// credit for the recovered sender.
originalPeer.ExpectSenderAttachWithoutGrantingCredit();
originalPeer.DropAfterLastMatcher(delay: 10); // Wait for sender to get into wait state
// Post Failover Expectations of sender
finalPeer.ExpectSaslAnonymous();
finalPeer.ExpectOpen();
finalPeer.ExpectBegin();
finalPeer.ExpectBegin();
finalPeer.ExpectSenderAttach();
finalPeer.ExpectTransfer(messageMatcher: Assert.IsNotNull);
finalPeer.ExpectClose();
NmsConnection connection = EstablishAnonymousConnection("failover.initialReconnectDelay=25", originalPeer, finalPeer);
ISession session = connection.CreateSession();
IQueue queue = session.GetQueue("myQueue");
IMessageProducer producer = session.CreateProducer(queue);
// Create and transfer a new message
string text = "myMessage";
ITextMessage message = session.CreateTextMessage(text);
Assert.DoesNotThrow(() =>
{
producer.Send(message);
});
connection.Close();
finalPeer.WaitForAllMatchersToComplete(1000);
}
}
private void DoTestConsumerReceiveThrowsIfConnectionLost(bool useTimeout)
{
ManualResetEvent consumerReady = new ManualResetEvent(false);
using (TestAmqpPeer testPeer = new TestAmqpPeer())
{
var context = EstablishNMSContext(testPeer);
testPeer.ExpectBegin();
IQueue queue = context.GetQueue("queue");
context.Start();
testPeer.ExpectReceiverAttach();
testPeer.ExpectLinkFlow();
testPeer.RunAfterLastHandler(() => { consumerReady.WaitOne(2000); });
testPeer.DropAfterLastMatcher(delay: 10);
var consumer = context.CreateConsumer(queue);
consumerReady.Set();
try
{
if (useTimeout)
{
consumer.Receive(TimeSpan.FromMilliseconds(10_0000));
}
else
{
consumer.Receive();
}
Assert.Fail("An exception should have been thrown");
}
catch (NMSException)
{
// Expected
}
testPeer.WaitForAllMatchersToComplete(1000);
}
}
public void TestTempDestinationRecreatedAfterConnectionFailsOver()
{
using (TestAmqpPeer originalPeer = new TestAmqpPeer())
using (TestAmqpPeer finalPeer = new TestAmqpPeer())
{
ManualResetEvent originalConnected = new ManualResetEvent(false);
ManualResetEvent finalConnected = new ManualResetEvent(false);
// Create a peer to connect to, then one to reconnect to
var originalUri = CreatePeerUri(originalPeer);
var finalUri = CreatePeerUri(finalPeer);
originalPeer.ExpectSaslAnonymous();
originalPeer.ExpectOpen();
originalPeer.ExpectBegin();
originalPeer.ExpectBegin();
string dynamicAddress1 = "myTempTopicAddress";
originalPeer.ExpectTempTopicCreationAttach(dynamicAddress1);
originalPeer.DropAfterLastMatcher();
NmsConnection connection = EstablishAnonymousConnection(originalPeer, finalPeer);
Mock <INmsConnectionListener> connectionListener = new Mock <INmsConnectionListener>();
connectionListener
.Setup(listener => listener.OnConnectionEstablished(It.Is <Uri>(uri => originalUri == uri.ToString())))
.Callback(() => { originalConnected.Set(); });
connectionListener
.Setup(listener => listener.OnConnectionRestored(It.Is <Uri>(uri => finalUri == uri.ToString())))
.Callback(() => { finalConnected.Set(); });
connection.AddConnectionListener(connectionListener.Object);
connection.Start();
Assert.True(originalConnected.WaitOne(TimeSpan.FromSeconds(5)), "Should connect to original peer");
// Post Failover Expectations of FinalPeer
finalPeer.ExpectSaslAnonymous();
finalPeer.ExpectOpen();
finalPeer.ExpectBegin();
String dynamicAddress2 = "myTempTopicAddress2";
finalPeer.ExpectTempTopicCreationAttach(dynamicAddress2);
// Session is recreated after previous temporary destinations are recreated on failover.
finalPeer.ExpectBegin();
ISession session = connection.CreateSession(AcknowledgementMode.AutoAcknowledge);
ITemporaryTopic temporaryTopic = session.CreateTemporaryTopic();
Assert.True(finalConnected.WaitOne(TimeSpan.FromSeconds(5)), "Should connect to final peer");
// Delete the temporary Topic and close the session.
finalPeer.ExpectDetach(expectClosed: true, sendResponse: true, replyClosed: true);
finalPeer.ExpectEnd();
temporaryTopic.Delete();
session.Close();
// Shut it down
finalPeer.ExpectClose();
connection.Close();
originalPeer.WaitForAllMatchersToComplete(2000);
finalPeer.WaitForAllMatchersToComplete(1000);
}
}
public void TestConsumerCanReceivesMessagesWhenConnectionLostDuringAutoAck()
{
using (TestAmqpPeer originalPeer = new TestAmqpPeer())
using (TestAmqpPeer finalPeer = new TestAmqpPeer())
{
ManualResetEvent consumerReady = new ManualResetEvent(false);
ManualResetEvent originalConnected = new ManualResetEvent(false);
ManualResetEvent finalConnected = new ManualResetEvent(false);
// Connect to the first peer
originalPeer.ExpectSaslAnonymous();
originalPeer.ExpectOpen();
originalPeer.ExpectBegin();
originalPeer.ExpectBegin();
NmsConnection connection = EstablishAnonymousConnection(originalPeer, finalPeer);
Mock <INmsConnectionListener> connectionListener = new Mock <INmsConnectionListener>();
connectionListener
.Setup(listener => listener.OnConnectionEstablished(It.IsAny <Uri>()))
.Callback(() => { originalConnected.Set(); });
connectionListener
.Setup(listener => listener.OnConnectionRestored(It.IsAny <Uri>()))
.Callback(() => { finalConnected.Set(); });
connection.AddConnectionListener(connectionListener.Object);
connection.Start();
Assert.True(originalConnected.WaitOne(TimeSpan.FromSeconds(5)), "Should connect to original peer");
originalPeer.ExpectReceiverAttach();
originalPeer.ExpectLinkFlowRespondWithTransfer(message: CreateMessageWithContent(), 1);
originalPeer.RunAfterLastHandler(() => consumerReady.WaitOne(TimeSpan.FromSeconds(2)));
originalPeer.DropAfterLastMatcher();
// Post Failover Expectations of FinalPeer
finalPeer.ExpectSaslAnonymous();
finalPeer.ExpectOpen();
finalPeer.ExpectBegin();
finalPeer.ExpectBegin();
finalPeer.ExpectReceiverAttach();
finalPeer.ExpectLinkFlowRespondWithTransfer(message: CreateMessageWithContent(), 1);
finalPeer.ExpectDispositionThatIsAcceptedAndSettled();
ISession session = connection.CreateSession(AcknowledgementMode.AutoAcknowledge);
IQueue queue = session.GetQueue("myQueue");
IMessageConsumer messageConsumer = session.CreateConsumer(queue);
CountdownEvent msgReceivedLatch = new CountdownEvent(2);
messageConsumer.Listener += message =>
{
if (msgReceivedLatch.CurrentCount == 2)
{
consumerReady.Set();
finalConnected.WaitOne(2000);
}
msgReceivedLatch.Signal();
};
finalPeer.WaitForAllMatchersToComplete(5000);
Assert.IsTrue(msgReceivedLatch.Wait(TimeSpan.FromSeconds(10)), $"Expected 2 messages, but got {2 - msgReceivedLatch.CurrentCount}");
}
}