public const int TestSuccessRate = 80; // 4 out of 5 (80%) test runs should pass (even after accounting for network instability issues). public static async Task TestPoolAmqpAsync( string devicePrefix, Client.TransportType transport, int poolSize, int devicesCount, Func <DeviceClient, TestDevice, Task> initOperation, Func <DeviceClient, TestDevice, Task> testOperation, Func <Task> cleanupOperation, ConnectionStringAuthScope authScope, bool ignoreConnectionStatus, MsTestLogger logger) { var transportSettings = new ITransportSettings[] { new AmqpTransportSettings(transport) { AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings() { MaxPoolSize = unchecked ((uint)poolSize), Pooling = true } } }; int totalRuns = 0; int successfulRuns = 0; int currentSuccessRate = 0; bool reRunTest = false; IList <TestDevice> testDevices = new List <TestDevice>(); IList <DeviceClient> deviceClients = new List <DeviceClient>(); IList <AmqpConnectionStatusChange> amqpConnectionStatuses = new List <AmqpConnectionStatusChange>(); IList <Task> operations = new List <Task>(); do { totalRuns++; // Arrange // Initialize the test device client instances // Set the device client connection status change handler logger.Trace($">>> {nameof(PoolingOverAmqp)} Initializing Device Clients for multiplexing test - Test run {totalRuns}"); for (int i = 0; i < devicesCount; i++) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(logger, $"{devicePrefix}_{i}_").ConfigureAwait(false); DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope); var amqpConnectionStatusChange = new AmqpConnectionStatusChange(logger); deviceClient.SetConnectionStatusChangesHandler(amqpConnectionStatusChange.ConnectionStatusChangesHandler); testDevices.Add(testDevice); deviceClients.Add(deviceClient); amqpConnectionStatuses.Add(amqpConnectionStatusChange); if (initOperation != null) { operations.Add(initOperation(deviceClient, testDevice)); } } await Task.WhenAll(operations).ConfigureAwait(false); operations.Clear(); try { for (int i = 0; i < devicesCount; i++) { operations.Add(testOperation(deviceClients[i], testDevices[i])); } await Task.WhenAll(operations).ConfigureAwait(false); operations.Clear(); // Close the device client instances and verify the connection status change checks bool deviceConnectionStatusAsExpected = true; for (int i = 0; i < devicesCount; i++) { await deviceClients[i].CloseAsync().ConfigureAwait(false); if (!ignoreConnectionStatus) { // The connection status change count should be 2: connect (open) and disabled (close) if (amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount != 2) { deviceConnectionStatusAsExpected = false; } // The connection status should be "Disabled", with connection status change reason "Client_close" Assert.AreEqual( ConnectionStatus.Disabled, amqpConnectionStatuses[i].LastConnectionStatus, $"The actual connection status is = {amqpConnectionStatuses[i].LastConnectionStatus}"); Assert.AreEqual( ConnectionStatusChangeReason.Client_Close, amqpConnectionStatuses[i].LastConnectionStatusChangeReason, $"The actual connection status change reason is = {amqpConnectionStatuses[i].LastConnectionStatusChangeReason}"); } } if (deviceConnectionStatusAsExpected) { successfulRuns++; } currentSuccessRate = (int)((double)successfulRuns / totalRuns * 100); reRunTest = currentSuccessRate < TestSuccessRate; } finally { // Close the service-side components and dispose the device client instances. if (cleanupOperation != null) { await cleanupOperation().ConfigureAwait(false); } foreach (DeviceClient deviceClient in deviceClients) { deviceClient.Dispose(); } // Clean up the local lists testDevices.Clear(); deviceClients.Clear(); amqpConnectionStatuses.Clear(); } } while (reRunTest && totalRuns < MaxTestRunCount); Assert.IsFalse(reRunTest, $"Device client instances got disconnected in {totalRuns - successfulRuns} runs out of {totalRuns}; current testSuccessRate = {currentSuccessRate}%."); }
public static async Task TestFaultInjectionPoolAmqpAsync( string devicePrefix, Client.TransportType transport, int poolSize, int devicesCount, string faultType, string reason, int delayInSec, int durationInSec, Func <DeviceClient, TestDevice, Task> initOperation, Func <DeviceClient, TestDevice, Task> testOperation, Func <IList <DeviceClient>, Task> cleanupOperation, ConnectionStringAuthScope authScope) { var transportSettings = new ITransportSettings[] { new AmqpTransportSettings(transport) { AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings() { MaxPoolSize = unchecked ((uint)poolSize), Pooling = true } } }; IList <TestDevice> testDevices = new List <TestDevice>(); IList <DeviceClient> deviceClients = new List <DeviceClient>(); IList <AmqpConnectionStatusChange> amqpConnectionStatuses = new List <AmqpConnectionStatusChange>(); IList <Task> operations = new List <Task>(); // Arrange // Initialize the test device client instances // Set the device client connection status change handler s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)} Initializing Device Clients for multiplexing test."); for (int i = 0; i < devicesCount; i++) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{devicePrefix}_{i}_").ConfigureAwait(false); DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope); var amqpConnectionStatusChange = new AmqpConnectionStatusChange(testDevice.Id); deviceClient.SetConnectionStatusChangesHandler(amqpConnectionStatusChange.ConnectionStatusChangesHandler); testDevices.Add(testDevice); deviceClients.Add(deviceClient); amqpConnectionStatuses.Add(amqpConnectionStatusChange); operations.Add(initOperation(deviceClient, testDevice)); } await Task.WhenAll(operations).ConfigureAwait(false); operations.Clear(); var watch = new Stopwatch(); try { // Act-Assert // Perform the test operation and verify the operation is successful // Perform baseline test operation for (int i = 0; i < devicesCount; i++) { s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing baseline operation for device {i}."); operations.Add(testOperation(deviceClients[i], testDevices[i])); } await Task.WhenAll(operations).ConfigureAwait(false); operations.Clear(); // Inject the fault into device 0 watch.Start(); s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Device {0} Requesting fault injection type={faultType} reason={reason}, delay={delayInSec}s, duration={durationInSec}s"); var faultInjectionMessage = FaultInjection.ComposeErrorInjectionProperties(faultType, reason, delayInSec, durationInSec); await deviceClients[0].SendEventAsync(faultInjectionMessage).ConfigureAwait(false); int delay = FaultInjection.WaitForReconnectMilliseconds - (int)watch.ElapsedMilliseconds; if (delay < 0) { delay = 0; } s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Waiting for fault injection to be active and device to be connected: {delay}ms"); await Task.Delay(delay).ConfigureAwait(false); // Perform the test operation for all devices for (int i = 0; i < devicesCount; i++) { s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation for device {i}."); operations.Add(testOperation(deviceClients[i], testDevices[i])); } await Task.WhenAll(operations).ConfigureAwait(false); operations.Clear(); // Close the device client instances for (int i = 0; i < devicesCount; i++) { operations.Add(deviceClients[i].CloseAsync()); } await Task.WhenAll(operations).ConfigureAwait(false); operations.Clear(); // Verify the connection status change checks. // For all of the devices - last connection status should be "Disabled", with reason "Client_close" for (int i = 0; i < devicesCount; i++) { // For the faulted device [device 0] - verify the connection status change count. if (i == 0) { if (FaultInjection.FaultShouldDisconnect(faultType)) { // 4 is the minimum notification count: connect, fault, reconnect, disable. Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount >= 4, $"The actual connection status change count for faulted device[0] is = {amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount}"); } else { // 2 is the minimum notification count: connect, disable. Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount == 2, $"The actual connection status change count for for faulted device[0] is = {amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount}"); } } Assert.AreEqual(ConnectionStatus.Disabled, amqpConnectionStatuses[i].LastConnectionStatus); Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, amqpConnectionStatuses[i].LastConnectionStatusChangeReason); } } finally { // Close the service-side components and dispose the device client instances. await cleanupOperation(deviceClients).ConfigureAwait(false); watch.Stop(); int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds; if (timeToFinishFaultInjection > 0) { s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed."); await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false); } } }
public static async Task TestFaultInjectionPoolAmqpAsync( string devicePrefix, Client.TransportType transport, string proxyAddress, int poolSize, int devicesCount, string faultType, string reason, int delayInSec, int durationInSec, Func <DeviceClient, TestDevice, TestDeviceCallbackHandler, Task> initOperation, Func <DeviceClient, TestDevice, TestDeviceCallbackHandler, Task> testOperation, Func <IList <DeviceClient>, Task> cleanupOperation, ConnectionStringAuthScope authScope, MsTestLogger logger) { var transportSettings = new ITransportSettings[] { new AmqpTransportSettings(transport) { AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings() { MaxPoolSize = unchecked ((uint)poolSize), Pooling = true, }, Proxy = proxyAddress == null ? null : new WebProxy(proxyAddress), } }; IList <TestDevice> testDevices = new List <TestDevice>(); IList <DeviceClient> deviceClients = new List <DeviceClient>(); IList <TestDeviceCallbackHandler> testDeviceCallbackHandlers = new List <TestDeviceCallbackHandler>(); IList <AmqpConnectionStatusChange> amqpConnectionStatuses = new List <AmqpConnectionStatusChange>(); IList <Task> operations = new List <Task>(); // Arrange // Initialize the test device client instances // Set the device client connection status change handler logger.Trace($">>> {nameof(FaultInjectionPoolingOverAmqp)} Initializing Device Clients for multiplexing test."); for (int i = 0; i < devicesCount; i++) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(logger, $"{devicePrefix}_{i}_").ConfigureAwait(false); DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope); var amqpConnectionStatusChange = new AmqpConnectionStatusChange(testDevice.Id, logger); deviceClient.SetConnectionStatusChangesHandler(amqpConnectionStatusChange.ConnectionStatusChangesHandler); var testDeviceCallbackHandler = new TestDeviceCallbackHandler(deviceClient, testDevice, logger); testDevices.Add(testDevice); deviceClients.Add(deviceClient); testDeviceCallbackHandlers.Add(testDeviceCallbackHandler); amqpConnectionStatuses.Add(amqpConnectionStatusChange); operations.Add(initOperation(deviceClient, testDevice, testDeviceCallbackHandler)); } await Task.WhenAll(operations).ConfigureAwait(false); operations.Clear(); var watch = new Stopwatch(); try { // Act-Assert // Perform the test operation and verify the operation is successful // Perform baseline test operation for (int i = 0; i < devicesCount; i++) { logger.Trace($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing baseline operation for device {i}."); operations.Add(testOperation(deviceClients[i], testDevices[i], testDeviceCallbackHandlers[i])); } await Task.WhenAll(operations).ConfigureAwait(false); operations.Clear(); int countBeforeFaultInjection = amqpConnectionStatuses[0].ConnectionStatusChangeCount; // Inject the fault into device 0 watch.Start(); logger.Trace($"{nameof(FaultInjectionPoolingOverAmqp)}: {testDevices[0].Id} Requesting fault injection type={faultType} reason={reason}, delay={delayInSec}s, duration={durationInSec}s"); var faultInjectionMessage = FaultInjection.ComposeErrorInjectionProperties(faultType, reason, delayInSec, durationInSec); await deviceClients[0].SendEventAsync(faultInjectionMessage).ConfigureAwait(false); logger.Trace($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {delayInSec} seconds."); await Task.Delay(TimeSpan.FromSeconds(delayInSec)).ConfigureAwait(false); // For disconnect type faults, the faulted device should disconnect and all devices should recover. if (FaultInjection.FaultShouldDisconnect(faultType)) { logger.Trace($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirming fault injection has been actived."); // Check that service issued the fault to the faulting device [device 0] bool isFaulted = false; for (int i = 0; i < FaultInjection.LatencyTimeBufferInSec; i++) { if (amqpConnectionStatuses[0].ConnectionStatusChangeCount > countBeforeFaultInjection) { isFaulted = true; break; } await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); } Assert.IsTrue(isFaulted, $"The device {testDevices[0].Id} did not get faulted with fault type: {faultType}"); logger.Trace($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirmed fault injection has been actived."); // Check all devices are back online logger.Trace($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirming all devices back online."); bool notRecovered = true; int j = 0; for (int i = 0; notRecovered && i < durationInSec + FaultInjection.LatencyTimeBufferInSec; i++) { notRecovered = false; for (j = 0; j < devicesCount; j++) { if (amqpConnectionStatuses[j].LastConnectionStatus != ConnectionStatus.Connected) { await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); notRecovered = true; break; } } } if (notRecovered) { Assert.Fail($"{testDevices[j].Id} did not reconnect."); } logger.Trace($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirmed all devices back online."); // Perform the test operation for all devices for (int i = 0; i < devicesCount; i++) { logger.Trace($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation for device {i}."); operations.Add(testOperation(deviceClients[i], testDevices[i], testDeviceCallbackHandlers[i])); } await Task.WhenAll(operations).ConfigureAwait(false); operations.Clear(); } else { logger.Trace($"{nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation while fault injection is being activated."); // Perform the test operation for the faulted device multi times. for (int i = 0; i < FaultInjection.LatencyTimeBufferInSec; i++) { logger.Trace($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation for device 0 - Run {i}."); await testOperation(deviceClients[0], testDevices[0], testDeviceCallbackHandlers[0]).ConfigureAwait(false); await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); } } // Close the device client instances for (int i = 0; i < devicesCount; i++) { operations.Add(deviceClients[i].CloseAsync()); } await Task.WhenAll(operations).ConfigureAwait(false); operations.Clear(); // Verify the connection status change checks. // For all of the devices - last connection status should be "Disabled", with reason "Client_close" for (int i = 0; i < devicesCount; i++) { // For the faulted device [device 0] - verify the connection status change count. if (i == 0) { if (FaultInjection.FaultShouldDisconnect(faultType)) { // 4 is the minimum notification count: connect, fault, reconnect, disable. Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangeCount >= 4, $"The expected connection status change count for {testDevices[i].Id} should equals or greater than 4 but was {amqpConnectionStatuses[i].ConnectionStatusChangeCount}"); } else { // 2 is the minimum notification count: connect, disable. Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangeCount >= 2, $"The expected connection status change count for {testDevices[i].Id} should be 2 but was {amqpConnectionStatuses[i].ConnectionStatusChangeCount}"); } } Assert.AreEqual(ConnectionStatus.Disabled, amqpConnectionStatuses[i].LastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Disabled} but was {amqpConnectionStatuses[i].LastConnectionStatus}"); Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, amqpConnectionStatuses[i].LastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Client_Close} but was {amqpConnectionStatuses[i].LastConnectionStatusChangeReason}"); } } finally { // Close the service-side components and dispose the device client instances. await cleanupOperation(deviceClients).ConfigureAwait(false); watch.Stop(); int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds; if (timeToFinishFaultInjection > 0) { logger.Trace($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed."); await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false); } } }
private async Task ReuseAuthenticationMethod_MuxedDevices(Client.TransportType transport, int devicesCount) { IList <TestDevice> testDevices = new List <TestDevice>(); IList <DeviceClient> deviceClients = new List <DeviceClient>(); IList <AuthenticationWithTokenRefresh> authenticationMethods = new List <AuthenticationWithTokenRefresh>(); IList <AmqpConnectionStatusChange> amqpConnectionStatuses = new List <AmqpConnectionStatusChange>(); // Set up amqp transport settings to multiplex all device sessions over the same amqp connection. var amqpTransportSettings = new AmqpTransportSettings(transport) { AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings { Pooling = true, MaxPoolSize = 1, }, }; for (int i = 0; i < devicesCount; i++) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(Logger, _devicePrefix).ConfigureAwait(false); #pragma warning disable CA2000 // Dispose objects before losing scope - the authentication method is disposed at the end of the test. var authenticationMethod = new DeviceAuthenticationSasToken(testDevice.ConnectionString, disposeWithClient: false); #pragma warning restore CA2000 // Dispose objects before losing scope testDevices.Add(testDevice); authenticationMethods.Add(authenticationMethod); } // Initialize the client instances, set the connection status change handler and open the connection. for (int i = 0; i < devicesCount; i++) { #pragma warning disable CA2000 // Dispose objects before losing scope - the client instance is disposed during the course of the test. DeviceClient deviceClient = DeviceClient.Create(testDevices[i].IoTHubHostName, authenticationMethods[i], new ITransportSettings[] { amqpTransportSettings }); #pragma warning restore CA2000 // Dispose objects before losing scope var amqpConnectionStatusChange = new AmqpConnectionStatusChange(testDevices[i].Id, Logger); deviceClient.SetConnectionStatusChangesHandler(amqpConnectionStatusChange.ConnectionStatusChangesHandler); amqpConnectionStatuses.Add(amqpConnectionStatusChange); await deviceClient.OpenAsync().ConfigureAwait(false); deviceClients.Add(deviceClient); } // Close and dispose client instance 1. // The closed client should report a status of "disabled" while the rest of them should be connected. // This is to ensure that disposal on one multiplexed device doesn't cause cascading failures // in the rest of the devices on the same tcp connection. await deviceClients[0].CloseAsync().ConfigureAwait(false); deviceClients[0].Dispose(); amqpConnectionStatuses[0].LastConnectionStatus.Should().Be(ConnectionStatus.Disabled); Logger.Trace($"{nameof(ReuseAuthenticationMethod_MuxedDevices)}: Confirming the rest of the multiplexed devices are online and operational."); bool notRecovered = true; var sw = Stopwatch.StartNew(); while (notRecovered && sw.Elapsed < MaxWaitTime) { notRecovered = false; for (int i = 1; i < devicesCount; i++) { if (amqpConnectionStatuses[i].LastConnectionStatus != ConnectionStatus.Connected) { await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); notRecovered = true; break; } } } notRecovered.Should().BeFalse(); // Send a message through the rest of the multiplexed client instances. var message = new Client.Message(); for (int i = 1; i < devicesCount; i++) { await deviceClients[i].SendEventAsync(message).ConfigureAwait(false); Logger.Trace($"Test with client {i} completed."); } message.Dispose(); // Close and dispose all of the client instances. for (int i = 1; i < devicesCount; i++) { await deviceClients[i].CloseAsync().ConfigureAwait(false); deviceClients[i].Dispose(); } deviceClients.Clear(); amqpConnectionStatuses.Clear(); // Initialize the client instances by reusing the created authentication methods and open the connection. for (int i = 0; i < devicesCount; i++) { #pragma warning disable CA2000 // Dispose objects before losing scope - the client instance is disposed at the end of the test. DeviceClient deviceClient = DeviceClient.Create(testDevices[i].IoTHubHostName, authenticationMethods[i], new ITransportSettings[] { amqpTransportSettings }); #pragma warning restore CA2000 // Dispose objects before losing scope var amqpConnectionStatusChange = new AmqpConnectionStatusChange(testDevices[i].Id, Logger); deviceClient.SetConnectionStatusChangesHandler(amqpConnectionStatusChange.ConnectionStatusChangesHandler); amqpConnectionStatuses.Add(amqpConnectionStatusChange); await deviceClient.OpenAsync().ConfigureAwait(false); deviceClients.Add(deviceClient); } // Ensure that all clients are connected successfully, and the close and dispose the instances. // Also dispose the authentication methods created. for (int i = 0; i < devicesCount; i++) { amqpConnectionStatuses[i].LastConnectionStatus.Should().Be(ConnectionStatus.Connected); await deviceClients[i].CloseAsync(); deviceClients[i].Dispose(); authenticationMethods[i].Dispose(); amqpConnectionStatuses[i].LastConnectionStatus.Should().Be(ConnectionStatus.Disabled); } }