public async Task DeviceClient_TokenConnectionDoubleRelease_Ok()
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);

            string deviceConnectionString = testDevice.ConnectionString;

            var    config   = new Configuration.IoTHub.DeviceConnectionStringParser(deviceConnectionString);
            string iotHub   = config.IoTHub;
            string deviceId = config.DeviceID;
            string key      = config.SharedAccessKey;

            SharedAccessSignatureBuilder builder = new SharedAccessSignatureBuilder()
            {
                Key        = key,
                TimeToLive = new TimeSpan(0, 10, 0),
                Target     = $"{iotHub}/devices/{WebUtility.UrlEncode(deviceId)}",
            };

            DeviceAuthenticationWithToken auth = new DeviceAuthenticationWithToken(deviceId, builder.ToSignature());

            using (DeviceClient deviceClient = DeviceClient.Create(iotHub, auth, Client.TransportType.Amqp_Tcp_Only))
            {
                _log.WriteLine($"Created {nameof(DeviceClient)} ID={TestLogging.IdOf(deviceClient)}");

                Console.WriteLine("DeviceClient OpenAsync.");
                await deviceClient.OpenAsync().ConfigureAwait(false);

                Console.WriteLine("DeviceClient SendEventAsync.");
                await deviceClient.SendEventAsync(new Client.Message(Encoding.UTF8.GetBytes("TestMessage"))).ConfigureAwait(false);

                Console.WriteLine("DeviceClient CloseAsync.");
                await deviceClient.CloseAsync().ConfigureAwait(false);   // First release
            } // Second release
        }
Пример #2
0
        private async Task ModuleClient_Gives_ConnectionStatus_DeviceDisabled_Base(
            Client.TransportType protocol, Func <RegistryManager, string, Task> registryManagerOperation)
        {
            AmqpTransportSettings amqpTransportSettings = new AmqpTransportSettings(protocol);

            ITransportSettings[] transportSettings = new ITransportSettings[] { amqpTransportSettings };

            TestModule testModule = await TestModule.GetTestModuleAsync(DevicePrefix + $"_{Guid.NewGuid()}", ModulePrefix).ConfigureAwait(false);

            ConnectionStatus?            status             = null;
            ConnectionStatusChangeReason?statusChangeReason = null;
            int deviceDisabledReceivedCount = 0;
            ConnectionStatusChangesHandler statusChangeHandler = (s, r) =>
            {
                if (r == ConnectionStatusChangeReason.Device_Disabled)
                {
                    status             = s;
                    statusChangeReason = r;
                    deviceDisabledReceivedCount++;
                }
            };

            using (ModuleClient moduleClient = ModuleClient.CreateFromConnectionString(testModule.ConnectionString, transportSettings))
            {
                moduleClient.SetConnectionStatusChangesHandler(statusChangeHandler);
                _log.WriteLine($"Created {nameof(DeviceClient)} ID={TestLogging.IdOf(moduleClient)}");

                Console.WriteLine("ModuleClient OpenAsync.");
                await moduleClient.OpenAsync().ConfigureAwait(false);

                // Receiving the module twin should succeed right now.
                Console.WriteLine("ModuleClient GetTwinAsync.");
                var twin = await moduleClient.GetTwinAsync().ConfigureAwait(false);

                Assert.IsNotNull(twin);

                // Delete/disable the device in IoT Hub.
                using (RegistryManager registryManager = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString))
                {
                    await registryManagerOperation(registryManager, testModule.DeviceId).ConfigureAwait(false);
                }

                // Artificial sleep waiting for the connection status change handler to get triggered.
                int sleepCount = 50;
                for (int i = 0; i < sleepCount; i++)
                {
                    await Task.Delay(TimeSpan.FromSeconds(10)).ConfigureAwait(false);

                    if (deviceDisabledReceivedCount == 1)
                    {
                        break;
                    }
                }

                Assert.AreEqual(1, deviceDisabledReceivedCount);
                Assert.AreEqual(ConnectionStatus.Disconnected, status);
                Assert.AreEqual(ConnectionStatusChangeReason.Device_Disabled, statusChangeReason);
            }
        }
Пример #3
0
        /// <summary>
        /// Factory method.
        /// IMPORTANT: Not thread safe!
        /// </summary>
        /// <param name="namePrefix"></param>
        /// <param name="type"></param>
        /// <returns></returns>
        public static async Task <TestDevice> GetTestDeviceAsync(string namePrefix, TestDeviceType type = TestDeviceType.Sasl)
        {
            var    log    = TestLogging.GetInstance();
            string prefix = namePrefix + type + "_";

            if (!s_deviceCache.TryGetValue(prefix, out TestDevice testDevice))
            {
                string deviceName = prefix + Guid.NewGuid();
                log.WriteLine($"{nameof(GetTestDeviceAsync)}: Device with prefix {prefix} not found. Removing old devices.");

                // Delete existing devices named this way and create a new one.
                RegistryManager rm = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);
                await RemoveDevicesAsync(prefix, rm).ConfigureAwait(false);

                log.WriteLine($"{nameof(GetTestDeviceAsync)}: Creating device {deviceName} with type {type}.");

                Device requestDevice = new Device(deviceName);
                if (type == TestDeviceType.X509)
                {
                    requestDevice.Authentication = new AuthenticationMechanism()
                    {
                        X509Thumbprint = new X509Thumbprint()
                        {
                            PrimaryThumbprint = Configuration.IoTHub.GetCertificateWithPrivateKey().Thumbprint
                        }
                    };
                }

                Device device = await rm.AddDeviceAsync(requestDevice).ConfigureAwait(false);

                log.WriteLine($"{nameof(GetTestDeviceAsync)}: Pausing for {DelayAfterDeviceCreationSeconds} after device was created.");
                await Task.Delay(DelayAfterDeviceCreationSeconds * 1000).ConfigureAwait(false);

                s_deviceCache[prefix] = new TestDevice(device);

                await rm.CloseAsync().ConfigureAwait(false);
            }

            TestDevice ret = s_deviceCache[prefix];

            log.WriteLine($"{nameof(GetTestDeviceAsync)}: Using device {ret.Id}.");
            return(ret);
        }
Пример #4
0
        /// <summary>
        /// Factory method.
        /// </summary>
        /// <param name="namePrefix"></param>
        /// <param name="type"></param>
        public static async Task <TestModule> GetTestModuleAsync(string deviceNamePrefix, string moduleNamePrefix)
        {
            var        log        = TestLogging.GetInstance();
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(deviceNamePrefix).ConfigureAwait(false);

            string deviceName = testDevice.Id;
            string moduleName = moduleNamePrefix + Guid.NewGuid();

            using var rm = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);
            log.WriteLine($"{nameof(GetTestModuleAsync)}: Creating module for device {deviceName}.");

            var    requestModule = new Module(deviceName, moduleName);
            Module module        = await rm.AddModuleAsync(requestModule).ConfigureAwait(false);

            await rm.CloseAsync().ConfigureAwait(false);

            var ret = new TestModule(module);

            log.WriteLine($"{nameof(GetTestModuleAsync)}: Using device {ret.DeviceId} with module {ret.Id}.");
            return(ret);
        }
Пример #5
0
        private static async Task RemoveDevicesAsync(string devicePrefix, RegistryManager rm)
        {
            var log = TestLogging.GetInstance();

            log.WriteLine($"{nameof(RemoveDevicesAsync)} Enumerating devices.");

            IQuery q = rm.CreateQuery("SELECT * FROM devices", 100);

            while (q.HasMoreResults)
            {
                IEnumerable <Twin> results = await q.GetNextAsTwinAsync().ConfigureAwait(false);

                foreach (Twin t in results)
                {
                    if (t.DeviceId.StartsWith(devicePrefix, StringComparison.OrdinalIgnoreCase))
                    {
                        log.WriteLine($"{nameof(RemoveDevicesAsync)} Removing device: {t.DeviceId}");
                        await rm.RemoveDeviceAsync(t.DeviceId).ConfigureAwait(false);
                    }
                }
            }
        }
 public DeviceTokenRefreshE2ETests()
 {
     _listener = TestConfig.StartEventListener();
     _log      = TestLogging.GetInstance();
 }
Пример #7
0
        private async Task DeviceCombinedClientOperations(
            Client.TransportType transport,
            int poolSize,
            int devicesCount,
            ConnectionStringAuthScope authScope
            )
        {
            var transportSettings = new ITransportSettings[]
            {
                new AmqpTransportSettings(transport)
                {
                    AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
                    {
                        MaxPoolSize = unchecked ((uint)poolSize),
                        Pooling     = true
                    }
                }
            };

            // Initialize service client for service-side operations
            ServiceClient serviceClient = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);

            ICollection <DeviceClient>     deviceClients = new List <DeviceClient>();
            Dictionary <DeviceClient, int> deviceClientConnectionStatusChangeCount = new Dictionary <DeviceClient, int>();

            try
            {
                _log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Starting the test execution for {devicesCount} devices");

                for (int i = 0; i < devicesCount; i++)
                {
                    ConnectionStatus?            lastConnectionStatus             = null;
                    ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
                    int setConnectionStatusChangesHandlerCount = 0;

                    TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{DevicePrefix}_{i}_").ConfigureAwait(false);

                    DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
                    deviceClients.Add(deviceClient);

                    // Set the connection status change handler
                    deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
                    {
                        setConnectionStatusChangesHandlerCount++;
                        lastConnectionStatus             = status;
                        lastConnectionStatusChangeReason = statusChangeReason;
                        _log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}");
                        deviceClientConnectionStatusChangeCount[deviceClient] = setConnectionStatusChangesHandlerCount;
                    });

                    // Perform D2C Operation
                    _log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Preparing to send message for device {i}");
                    await deviceClient.OpenAsync().ConfigureAwait(false);

                    await MessageSend.SendSingleMessageAndVerifyAsync(deviceClient, testDevice.Id).ConfigureAwait(false);

                    // Perform C2D Operation
                    _log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Setting the device {i} to receive C2D message.");
                    string  payload, messageId, p1Value;
                    Message msg = MessageReceive.ComposeC2DTestMessage(out payload, out messageId, out p1Value);
                    await serviceClient.SendAsync(testDevice.Id, msg).ConfigureAwait(false);

                    await MessageReceive.VerifyReceivedC2DMessageAsync(transport, deviceClient, payload, p1Value).ConfigureAwait(false);

                    // Invoke direct methods
                    _log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Testing direct methods for device {i}");
                    Task methodReceivedTask = await MethodOperation.SetDeviceReceiveMethod(deviceClient).ConfigureAwait(false);

                    await Task.WhenAll(
                        MethodOperation.ServiceSendMethodAndVerifyResponse(testDevice.Id),
                        methodReceivedTask).ConfigureAwait(false);

                    // Set reported twin properties
                    _log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Setting reported Twin properties for device {i}");
                    await TwinOperation.Twin_DeviceSetsReportedPropertyAndGetsItBack(deviceClient).ConfigureAwait(false);

                    // Receive set desired twin properties
                    _log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Received set desired tein properties for device {i}");
                    var  propName           = Guid.NewGuid().ToString();
                    var  propValue          = Guid.NewGuid().ToString();
                    Task updateReceivedTask = await TwinOperation.SetTwinPropertyUpdateCallbackHandlerAsync(deviceClient, propName, propValue).ConfigureAwait(false);

                    await Task.WhenAll(
                        TwinOperation.RegistryManagerUpdateDesiredPropertyAsync(testDevice.Id, propName, propValue),
                        updateReceivedTask).ConfigureAwait(false);
                }
            }
            finally
            {
                // Close and dispose all of the device client instances here
                foreach (DeviceClient deviceClient in deviceClients)
                {
                    await deviceClient.CloseAsync().ConfigureAwait(false);

                    // The connection status change count should be 2: connect (open) and disabled (close)
                    Assert.IsTrue(deviceClientConnectionStatusChangeCount[deviceClient] == 2);

                    _log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
                    deviceClient.Dispose();
                }
            }
        }
        // Error injection template method.
        public static async Task TestErrorInjectionAsync(
            string devicePrefix,
            TestDeviceType type,
            Client.TransportType transport,
            string faultType,
            string reason,
            int delayInSec,
            int durationInSec,
            Func <DeviceClient, TestDevice, Task> initOperation,
            Func <DeviceClient, TestDevice, Task> testOperation,
            Func <Task> cleanupOperation)
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(devicePrefix, type).ConfigureAwait(false);

            DeviceClient deviceClient = testDevice.CreateDeviceClient(transport);

            ConnectionStatus?            lastConnectionStatus             = null;
            ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
            int setConnectionStatusChangesHandlerCount = 0;

            deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
            {
                setConnectionStatusChangesHandlerCount++;
                lastConnectionStatus             = status;
                lastConnectionStatusChangeReason = statusChangeReason;
                s_log.WriteLine($"{nameof(FaultInjection)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}");
            });

            var watch = new Stopwatch();

            try
            {
                await deviceClient.OpenAsync().ConfigureAwait(false);

                if (transport != Client.TransportType.Http1)
                {
                    Assert.IsTrue(setConnectionStatusChangesHandlerCount >= 1); // Normally one connection but in some cases, due to network issues we might have already retried several times to connect.
                    Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus);
                    Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason);
                }

                await initOperation(deviceClient, testDevice).ConfigureAwait(false);

                s_log.WriteLine($">>> {nameof(FaultInjection)} Testing baseline");
                await testOperation(deviceClient, testDevice).ConfigureAwait(false);

                watch.Start();
                s_log.WriteLine($">>> {nameof(FaultInjection)} Testing fault handling");
                await ActivateFaultInjection(transport, faultType, reason, delayInSec, durationInSec, deviceClient).ConfigureAwait(false);

                int delay = FaultInjection.WaitForDisconnectMilliseconds - (int)watch.ElapsedMilliseconds;
                if (delay < 0)
                {
                    delay = 0;
                }

                s_log.WriteLine($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {delay}ms");
                await Task.Delay(delay).ConfigureAwait(false);

                await testOperation(deviceClient, testDevice).ConfigureAwait(false);

                await deviceClient.CloseAsync().ConfigureAwait(false);

                if (transport != Client.TransportType.Http1)
                {
                    if (FaultShouldDisconnect(faultType))
                    {
                        // 4 is the minimum notification count: connect, fault, reconnect, disable.
                        // There are cases where the retry must be timed out (i.e. very likely for MQTT where otherwise
                        // we would attempt to send the fault injection forever.)
                        Assert.IsTrue(setConnectionStatusChangesHandlerCount >= 4);
                    }
                    else
                    {
                        // 2 is the minimum notification count: connect, disable.
                        // We will monitor the test environment real network stability and switch to >=2 if necessary to
                        // account for real network issues.
                        Assert.IsTrue(setConnectionStatusChangesHandlerCount == 2);
                    }

                    Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus);
                    Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason);
                }
            }
            finally
            {
                await cleanupOperation().ConfigureAwait(false);

                s_log.WriteLine($"{nameof(FaultInjection)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
                deviceClient.Dispose();

                watch.Stop();

                int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds;
                if (timeToFinishFaultInjection > 0)
                {
                    s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed.");
                    await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false);
                }
            }
        }
        private async Task DeviceClient_TokenIsRefreshed_Internal(Client.TransportType transport, int ttl = 20)
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);

            int           buffer             = 50;
            Device        device             = testDevice.Identity;
            SemaphoreSlim deviceDisconnected = new SemaphoreSlim(0);

            var refresher = new TestTokenRefresher(
                device.Id,
                device.Authentication.SymmetricKey.PrimaryKey,
                ttl,
                buffer,
                transport);

            using (DeviceClient deviceClient = DeviceClient.Create(testDevice.IoTHubHostName, refresher, transport))
            {
                _log.WriteLine($"Created {nameof(DeviceClient)} ID={TestLogging.IdOf(deviceClient)}");

                if (transport == Client.TransportType.Mqtt)
                {
                    deviceClient.SetConnectionStatusChangesHandler((ConnectionStatus status, ConnectionStatusChangeReason reason) =>
                    {
                        _log.WriteLine($"{nameof(ConnectionStatusChangesHandler)}: {status}; {reason}");
                        if (status == ConnectionStatus.Disconnected_Retrying || status == ConnectionStatus.Disconnected)
                        {
                            deviceDisconnected.Release();
                        }
                    });
                }

                var message = new Client.Message(Encoding.UTF8.GetBytes("Hello"));

                using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(ttl * 10)))
                {
                    try
                    {
                        // Create the first Token.
                        Console.WriteLine($"[{DateTime.UtcNow}] OpenAsync");
                        await deviceClient.OpenAsync(cts.Token).ConfigureAwait(false);

                        Console.WriteLine($"[{DateTime.UtcNow}] SendEventAsync (1)");
                        await deviceClient.SendEventAsync(message, cts.Token).ConfigureAwait(false);

                        await refresher.WaitForTokenRefreshAsync(cts.Token).ConfigureAwait(false);
                    }
                    catch (OperationCanceledException ex)
                    {
                        Assert.Fail($"{TestLogging.IdOf(deviceClient)} did not get the initial token. {ex}");
                        throw;
                    }

                    // Wait for the Token to expire.
                    if (transport == Client.TransportType.Http1)
                    {
                        float waitTime = (float)ttl * ((float)buffer / 100) + 1;
                        Console.WriteLine($"[{DateTime.UtcNow}] Waiting {waitTime} seconds.");
                        await Task.Delay(TimeSpan.FromSeconds(waitTime)).ConfigureAwait(false);
                    }
                    else if (transport == Client.TransportType.Mqtt)
                    {
                        Console.WriteLine($"[{DateTime.UtcNow}] Waiting for device disconnect.");
                        await deviceDisconnected.WaitAsync(cts.Token).ConfigureAwait(false);
                    }

                    try
                    {
                        Console.WriteLine($"[{DateTime.UtcNow}] SendEventAsync (2)");
                        await deviceClient.SendEventAsync(message, cts.Token).ConfigureAwait(false);

                        await refresher.WaitForTokenRefreshAsync(cts.Token).ConfigureAwait(false);
                    }
                    catch (OperationCanceledException ex)
                    {
                        Assert.Fail($"{TestLogging.IdOf(deviceClient)} did not refresh token after {refresher.DetectedRefreshInterval}. {ex}");
                        throw;
                    }

                    // Ensure that the token was refreshed.
                    Console.WriteLine($"[{DateTime.UtcNow}] Token was refreshed after {refresher.DetectedRefreshInterval} (ttl = {ttl} seconds).");
                    Assert.IsTrue(
                        refresher.DetectedRefreshInterval.TotalSeconds < (float)ttl * (1 + (float)buffer / 100), // Wait for more than what we expect.
                        $"Token was refreshed after {refresher.DetectedRefreshInterval} although ttl={ttl} seconds.");

                    Console.WriteLine($"[{DateTime.UtcNow}] CloseAsync");
                    await deviceClient.CloseAsync().ConfigureAwait(false);
                }
            }
        }
Пример #10
0
        public DeviceClient CreateDeviceClient(Client.TransportType transport)
        {
            DeviceClient deviceClient = null;

            if (_authenticationMethod == null)
            {
                deviceClient = DeviceClient.CreateFromConnectionString(ConnectionString, transport);
                s_log.WriteLine($"{nameof(CreateDeviceClient)}: Created {nameof(DeviceClient)} {_device.Id} from connection string: {transport} ID={TestLogging.IdOf(deviceClient)}");
            }
            else
            {
                deviceClient = DeviceClient.Create(IoTHubHostName, AuthenticationMethod, transport);
                s_log.WriteLine($"{nameof(CreateDeviceClient)}: Created {nameof(DeviceClient)} {_device.Id} from IAuthenticationMethod: {transport} ID={TestLogging.IdOf(deviceClient)}");
            }

            return(deviceClient);
        }
Пример #11
0
        public DeviceClient CreateDeviceClient(ITransportSettings[] transportSettings, ConnectionStringAuthScope authScope = ConnectionStringAuthScope.Device)
        {
            DeviceClient deviceClient = null;

            if (_authenticationMethod == null)
            {
                if (authScope == ConnectionStringAuthScope.Device)
                {
                    deviceClient = DeviceClient.CreateFromConnectionString(ConnectionString, transportSettings);
                    s_log.WriteLine($"{nameof(CreateDeviceClient)}: Created {nameof(DeviceClient)} {_device.Id} from device connection string: ID={TestLogging.IdOf(deviceClient)}");
                }
                else
                {
                    deviceClient = DeviceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString, _device.Id, transportSettings);
                    s_log.WriteLine($"{nameof(CreateDeviceClient)}: Created {nameof(DeviceClient)} {_device.Id} from IoTHub connection string: ID={TestLogging.IdOf(deviceClient)}");
                }
            }
            else
            {
                deviceClient = DeviceClient.Create(IoTHubHostName, AuthenticationMethod, transportSettings);
                s_log.WriteLine($"{nameof(CreateDeviceClient)}: Created {nameof(DeviceClient)} {_device.Id} from IAuthenticationMethod: ID={TestLogging.IdOf(deviceClient)}");
            }

            return(deviceClient);
        }
        private async Task SendMessageMuxedOverAmqp(
            TestDeviceType type,
            Client.TransportType transport,
            int poolSize,
            int devicesCount,
            ConnectionStringAuthScope authScope = ConnectionStringAuthScope.Device)
        {
            var transportSettings = new ITransportSettings[]
            {
                new AmqpTransportSettings(transport)
                {
                    AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
                    {
                        MaxPoolSize = unchecked ((uint)poolSize),
                        Pooling     = true
                    }
                }
            };

            ICollection <DeviceClient>     deviceClients = new List <DeviceClient>();
            Dictionary <DeviceClient, int> deviceClientConnectionStatusChangeCount = new Dictionary <DeviceClient, int>();

            try
            {
                _log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}: Starting the test execution for {devicesCount} devices");

                for (int i = 0; i < devicesCount; i++)
                {
                    ConnectionStatus?            lastConnectionStatus             = null;
                    ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
                    int setConnectionStatusChangesHandlerCount = 0;

                    TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{DevicePrefix}_{i}_", type).ConfigureAwait(false);

                    DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
                    deviceClients.Add(deviceClient);

                    deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
                    {
                        setConnectionStatusChangesHandlerCount++;
                        lastConnectionStatus             = status;
                        lastConnectionStatusChangeReason = statusChangeReason;
                        _log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}");
                        deviceClientConnectionStatusChangeCount[deviceClient] = setConnectionStatusChangesHandlerCount;
                    });

                    _log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}: Preparing to send message for device {i}");
                    await deviceClient.OpenAsync().ConfigureAwait(false);

                    await MessageSend.SendSingleMessageAndVerifyAsync(deviceClient, testDevice.Id).ConfigureAwait(false);
                }
            }
            finally
            {
                // Close and dispose all of the device client instances here
                foreach (DeviceClient deviceClient in deviceClients)
                {
                    await deviceClient.CloseAsync().ConfigureAwait(false);

                    // The connection status change count should be 2: connect (open) and disabled (close)
                    Assert.IsTrue(deviceClientConnectionStatusChangeCount[deviceClient] == 2, $"Connection status change count for deviceClient {TestLogging.GetHashCode(deviceClient)} is {deviceClientConnectionStatusChangeCount[deviceClient]}");

                    _log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
                    deviceClient.Dispose();
                }
            }
        }
        private async Task SendMessageModuleMuxedOverAmqp(
            Client.TransportType transport,
            int poolSize,
            int devicesCount,
            bool useSameDevice = false)
        {
            var transportSettings = new ITransportSettings[]
            {
                new AmqpTransportSettings(transport)
                {
                    AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
                    {
                        MaxPoolSize = unchecked ((uint)poolSize),
                        Pooling     = true
                    }
                }
            };

            ICollection <ModuleClient>     moduleClients = new List <ModuleClient>();
            Dictionary <ModuleClient, int> moduleClientConnectionStatusChangeCount = new Dictionary <ModuleClient, int>();

            try
            {
                _log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}: Starting the test execution for {devicesCount} modules");

                for (int i = 0; i < devicesCount; i++)
                {
                    ConnectionStatus?            lastConnectionStatus             = null;
                    ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
                    int setConnectionStatusChangesHandlerCount = 0;

                    string devicePrefix = useSameDevice ? DevicePrefix : $"{DevicePrefix}_{i}_";
                    string modulePrefix = useSameDevice ? $"{ModulePrefix}_{i}_" : ModulePrefix;

                    TestModule testModule = await TestModule.GetTestModuleAsync(devicePrefix, modulePrefix).ConfigureAwait(false);

                    ModuleClient moduleClient = ModuleClient.CreateFromConnectionString(testModule.ConnectionString, transportSettings);
                    moduleClients.Add(moduleClient);

                    moduleClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
                    {
                        setConnectionStatusChangesHandlerCount++;
                        lastConnectionStatus             = status;
                        lastConnectionStatusChangeReason = statusChangeReason;
                        _log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}");
                        moduleClientConnectionStatusChangeCount[moduleClient] = setConnectionStatusChangesHandlerCount;
                    });

                    _log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}: Preparing to send message for module {i}");
                    await moduleClient.OpenAsync().ConfigureAwait(false);

                    await MessageSend.SendSingleMessageModuleAndVerifyAsync(moduleClient, testModule.DeviceId).ConfigureAwait(false);
                }
            }
            finally
            {
                // Close and dispose all of the module client instances here
                foreach (ModuleClient moduleClient in moduleClients)
                {
                    await moduleClient.CloseAsync().ConfigureAwait(false);

                    // The connection status change count should be 2: connect (open) and disabled (close)
                    Assert.IsTrue(moduleClientConnectionStatusChangeCount[moduleClient] == 2, $"Connection status change count for deviceClient {TestLogging.GetHashCode(moduleClient)} is {moduleClientConnectionStatusChangeCount[moduleClient]}");

                    _log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}: Disposing moduleClient {TestLogging.GetHashCode(moduleClient)}");
                    moduleClient.Dispose();
                }
            }
        }
        private async Task ReceiveMessageMuxedOverAmqp(
            Client.TransportType transport,
            int poolSize,
            int devicesCount,
            ConnectionStringAuthScope authScope = ConnectionStringAuthScope.Device)
        {
            var transportSettings = new ITransportSettings[]
            {
                new AmqpTransportSettings(transport)
                {
                    AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
                    {
                        MaxPoolSize = unchecked ((uint)poolSize),
                        Pooling     = true
                    }
                }
            };

            ICollection <DeviceClient>     deviceClients = new List <DeviceClient>();
            Dictionary <DeviceClient, int> deviceClientConnectionStatusChangeCount = new Dictionary <DeviceClient, int>();
            ServiceClient serviceClient = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);

            try
            {
                _log.WriteLine($"{nameof(MessageReceiveE2EMultiplexingTests)}: Starting the test execution for {devicesCount} devices");
                await serviceClient.OpenAsync().ConfigureAwait(false);

                for (int i = 0; i < devicesCount; i++)
                {
                    ConnectionStatus?            lastConnectionStatus             = null;
                    ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
                    int setConnectionStatusChangesHandlerCount = 0;

                    TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{DevicePrefix}_{i}_").ConfigureAwait(false);

                    DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
                    deviceClients.Add(deviceClient);

                    deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
                    {
                        setConnectionStatusChangesHandlerCount++;
                        lastConnectionStatus             = status;
                        lastConnectionStatusChangeReason = statusChangeReason;
                        _log.WriteLine($"{nameof(MessageReceiveE2EMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}");
                        deviceClientConnectionStatusChangeCount[deviceClient] = setConnectionStatusChangesHandlerCount;
                    });

                    _log.WriteLine($"{nameof(MessageReceiveE2EMultiplexingTests)}: Preparing to receive message for device {i}");
                    await deviceClient.OpenAsync().ConfigureAwait(false);

                    string payload, messageId, p1Value;

                    Message msg = MessageReceive.ComposeC2DTestMessage(out payload, out messageId, out p1Value);
                    await serviceClient.SendAsync(testDevice.Id, msg).ConfigureAwait(false);

                    await MessageReceive.VerifyReceivedC2DMessageAsync(transport, deviceClient, payload, p1Value).ConfigureAwait(false);
                }
            }
            finally
            {
                await serviceClient.CloseAsync().ConfigureAwait(false);

                serviceClient.Dispose();

                // Close and dispose all of the device client instances here
                foreach (DeviceClient deviceClient in deviceClients)
                {
                    await deviceClient.CloseAsync().ConfigureAwait(false);

                    // The connection status change count should be 2: connect (open) and disabled (close)
                    Assert.IsTrue(deviceClientConnectionStatusChangeCount[deviceClient] == 2);

                    _log.WriteLine($"{nameof(MessageReceiveE2EMultiplexingTests)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
                    deviceClient.Dispose();
                }
            }
        }
Пример #15
0
        // Error injection template method.
        public static async Task TestErrorInjectionAsync(
            string devicePrefix,
            TestDeviceType type,
            Client.TransportType transport,
            string faultType,
            string reason,
            int delayInSec,
            int durationInSec,
            Func <DeviceClient, TestDevice, Task> initOperation,
            Func <DeviceClient, TestDevice, Task> testOperation,
            Func <Task> cleanupOperation)
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(devicePrefix, type).ConfigureAwait(false);

            DeviceClient deviceClient = testDevice.CreateDeviceClient(transport);

            ConnectionStatus?            lastConnectionStatus             = null;
            ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
            int setConnectionStatusChangesHandlerCount = 0;

            deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
            {
                s_log.WriteLine($"{nameof(FaultInjection)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}");
                lastConnectionStatus             = status;
                lastConnectionStatusChangeReason = statusChangeReason;
                setConnectionStatusChangesHandlerCount++;
            });

            var watch = new Stopwatch();

            try
            {
                await deviceClient.OpenAsync().ConfigureAwait(false);

                if (transport != Client.TransportType.Http1)
                {
                    Assert.AreEqual(1, setConnectionStatusChangesHandlerCount);
                    Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus);
                    Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason);
                }

                await initOperation(deviceClient, testDevice).ConfigureAwait(false);

                s_log.WriteLine($">>> {nameof(FaultInjection)} Testing baseline");
                await testOperation(deviceClient, testDevice).ConfigureAwait(false);

                await ActivateFaultInjection(transport, faultType, reason, delayInSec, durationInSec, deviceClient).ConfigureAwait(false);

                s_log.WriteLine($">>> {nameof(FaultInjection)} Testing fault handling");
                watch.Start();
                s_log.WriteLine($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {FaultInjection.WaitForDisconnectMilliseconds}ms");
                await Task.Delay(FaultInjection.WaitForDisconnectMilliseconds).ConfigureAwait(false);

                await testOperation(deviceClient, testDevice).ConfigureAwait(false);

                await deviceClient.CloseAsync().ConfigureAwait(false);

                if (transport == Client.TransportType.Mqtt ||
                    transport == Client.TransportType.Mqtt_Tcp_Only ||
                    transport == Client.TransportType.Mqtt_WebSocket_Only)
                {
                    // Our fault injection is only terminating the connection for MQTT. (HTTP is not connection-oriented, AMQP is not actually terminating the TCP layer.)
                    Assert.IsTrue(setConnectionStatusChangesHandlerCount >= 4);
                    Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus);
                    Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason);
                }
            }
            finally
            {
                await cleanupOperation().ConfigureAwait(false);

                s_log.WriteLine($"{nameof(FaultInjection)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
                deviceClient.Dispose();

                watch.Stop();

                int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds;
                if (timeToFinishFaultInjection > 0)
                {
                    s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed.");
                    await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false);
                }
            }
        }
        // Error injection template method.
        public static async Task TestErrorInjectionAsync(
            string devicePrefix,
            TestDeviceType type,
            Client.TransportType transport,
            string faultType,
            string reason,
            int delayInSec,
            int durationInSec,
            Func <DeviceClient, TestDevice, Task> initOperation,
            Func <DeviceClient, TestDevice, Task> testOperation,
            Func <Task> cleanupOperation)
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(devicePrefix, type).ConfigureAwait(false);

            DeviceClient deviceClient = testDevice.CreateDeviceClient(transport);

            ConnectionStatus?            lastConnectionStatus             = null;
            ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
            int connectionStatusChangeCount = 0;

            deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
            {
                connectionStatusChangeCount++;
                lastConnectionStatus             = status;
                lastConnectionStatusChangeReason = statusChangeReason;
                s_log.WriteLine($"{nameof(FaultInjection)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={connectionStatusChangeCount}");
            });

            var watch = new Stopwatch();

            try
            {
                await deviceClient.OpenAsync().ConfigureAwait(false);

                if (transport != Client.TransportType.Http1)
                {
                    Assert.IsTrue(connectionStatusChangeCount >= 1, $"The expected connection status change should be equal or greater than 1 but was {connectionStatusChangeCount}"); // Normally one connection but in some cases, due to network issues we might have already retried several times to connect.
                    Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Connected} but was {lastConnectionStatus}");
                    Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Connection_Ok} but was {lastConnectionStatusChangeReason}");
                }

                await initOperation(deviceClient, testDevice).ConfigureAwait(false);

                s_log.WriteLine($">>> {nameof(FaultInjection)} Testing baseline");
                await testOperation(deviceClient, testDevice).ConfigureAwait(false);

                watch.Start();
                s_log.WriteLine($">>> {nameof(FaultInjection)} Testing fault handling");
                await ActivateFaultInjection(transport, faultType, reason, delayInSec, durationInSec, deviceClient).ConfigureAwait(false);

                s_log.WriteLine($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {delayInSec} seconds.");
                await Task.Delay(TimeSpan.FromSeconds(delayInSec)).ConfigureAwait(false);

                // For disconnect type faults, the device should disconnect and recover.
                if (FaultShouldDisconnect(faultType))
                {
                    s_log.WriteLine($"{nameof(FaultInjection)}: Confirming fault injection has been actived.");
                    // Check that service issued the fault to the faulting device
                    bool isFaulted = false;
                    for (int i = 0; i < LatencyTimeBufferInSec; i++)
                    {
                        if (connectionStatusChangeCount >= 2)
                        {
                            isFaulted = true;
                            break;
                        }

                        await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
                    }

                    Assert.IsTrue(isFaulted, $"The device {testDevice.Id} did not get faulted with fault type: {faultType}");
                    s_log.WriteLine($"{nameof(FaultInjection)}: Confirmed fault injection has been actived.");

                    // Check the device is back online
                    s_log.WriteLine($"{nameof(FaultInjection)}: Confirming device back online.");
                    for (int i = 0; lastConnectionStatus != ConnectionStatus.Connected && i < durationInSec + LatencyTimeBufferInSec; i++)
                    {
                        await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
                    }

                    Assert.AreEqual(lastConnectionStatus, ConnectionStatus.Connected, $"{testDevice.Id} did not reconnect.");
                    s_log.WriteLine($"{nameof(FaultInjection)}: Confirmed device back online.");

                    // Perform the test operation.
                    s_log.WriteLine($">>> {nameof(FaultInjection)}: Performing test operation for device {testDevice.Id}.");
                    await testOperation(deviceClient, testDevice).ConfigureAwait(false);
                }
                else
                {
                    s_log.WriteLine($"{nameof(FaultInjection)}: Performing test operation while fault injection is being activated.");
                    // Perform the test operation for the faulted device multi times.
                    for (int i = 0; i < LatencyTimeBufferInSec; i++)
                    {
                        s_log.WriteLine($">>> {nameof(FaultInjection)}: Performing test operation for device - Run {i}.");
                        await testOperation(deviceClient, testDevice).ConfigureAwait(false);

                        await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
                    }
                }

                await deviceClient.CloseAsync().ConfigureAwait(false);

                if (transport != Client.TransportType.Http1)
                {
                    if (FaultInjection.FaultShouldDisconnect(faultType))
                    {
                        // 4 is the minimum notification count: connect, fault, reconnect, disable.
                        // There are cases where the retry must be timed out (i.e. very likely for MQTT where otherwise
                        // we would attempt to send the fault injection forever.)
                        Assert.IsTrue(connectionStatusChangeCount >= 4, $"The expected connection status change count for {testDevice.Id} should be equal or greater than 4 but was {connectionStatusChangeCount}");
                    }
                    else
                    {
                        // 2 is the minimum notification count: connect, disable.
                        // We will monitor the test environment real network stability and switch to >=2 if necessary to
                        // account for real network issues.
                        Assert.IsTrue(connectionStatusChangeCount == 2, $"The expected connection status change count for {testDevice.Id}  should be 2 but was {connectionStatusChangeCount}");
                    }
                    Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Disabled} but was {lastConnectionStatus}");
                    Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Client_Close} but was {lastConnectionStatusChangeReason}");
                }
            }
            finally
            {
                await cleanupOperation().ConfigureAwait(false);

                s_log.WriteLine($"{nameof(FaultInjection)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
                deviceClient.Dispose();

                watch.Stop();

                int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds;
                if (timeToFinishFaultInjection > 0)
                {
                    s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed.");
                    await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false);
                }
            }
        }
Пример #17
0
        private async Task Twin_ServiceSetsDesiredPropertyAndDeviceReceivesEventMuxedOverAmqp(
            ConnectionStringAuthScope authScope,
            Client.TransportType transport,
            int poolSize,
            int devicesCount,
            Func <DeviceClient, string, string, Task <Task> > setTwinPropertyUpdateCallbackAsync
            )
        {
            var propName  = Guid.NewGuid().ToString();
            var propValue = Guid.NewGuid().ToString();

            var transportSettings = new ITransportSettings[]
            {
                new AmqpTransportSettings(transport)
                {
                    AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
                    {
                        MaxPoolSize = unchecked ((uint)poolSize),
                        Pooling     = true
                    }
                }
            };

            ICollection <DeviceClient>     deviceClients = new List <DeviceClient>();
            Dictionary <DeviceClient, int> deviceClientConnectionStatusChangeCount = new Dictionary <DeviceClient, int>();

            try
            {
                _log.WriteLine($"{nameof(TwinE2EMultiplexingTests)}: Starting the test execution for {devicesCount} devices");

                for (int i = 0; i < devicesCount; i++)
                {
                    ConnectionStatus?            lastConnectionStatus             = null;
                    ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
                    int setConnectionStatusChangesHandlerCount = 0;

                    TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{DevicePrefix}_{i}_").ConfigureAwait(false);

                    DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
                    deviceClients.Add(deviceClient);

                    deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
                    {
                        setConnectionStatusChangesHandlerCount++;
                        lastConnectionStatus             = status;
                        lastConnectionStatusChangeReason = statusChangeReason;
                        _log.WriteLine($"{nameof(TwinE2EMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}");
                        deviceClientConnectionStatusChangeCount[deviceClient] = setConnectionStatusChangesHandlerCount;
                    });

                    _log.WriteLine($"{nameof(Twin_ServiceSetsDesiredPropertyAndDeviceReceivesEventMuxedOverAmqp)}: name={propName}, value={propValue}");
                    Task updateReceivedTask = await setTwinPropertyUpdateCallbackAsync(deviceClient, propName, propValue).ConfigureAwait(false);

                    await Task.WhenAll(
                        TwinOperation.RegistryManagerUpdateDesiredPropertyAsync(testDevice.Id, propName, propValue),
                        updateReceivedTask).ConfigureAwait(false);
                }
            }
            finally
            {
                // Close and dispose all of the device client instances here
                foreach (DeviceClient deviceClient in deviceClients)
                {
                    await deviceClient.CloseAsync().ConfigureAwait(false);

                    // The connection status change count should be 2: connect (open) and disabled (close)
                    Assert.IsTrue(deviceClientConnectionStatusChangeCount[deviceClient] == 2);

                    _log.WriteLine($"{nameof(TwinE2EMultiplexingTests)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
                    deviceClient.Dispose();
                }
            }
        }
 public DeviceTokenRefreshE2ETests()
 {
     _listener = new ConsoleEventListener("Microsoft-Azure-");
     _log      = TestLogging.GetInstance();
 }
Пример #19
0
        async Task ModuleClient_Gives_ConnectionStatus_DeviceDisabled_Base(
            Client.TransportType protocol, Func <RegistryManager, string, Task> registryManagerOperation)
        {
            AmqpTransportSettings amqpTransportSettings = new AmqpTransportSettings(protocol);

            ITransportSettings[] transportSettings = new ITransportSettings[] { amqpTransportSettings };

            TestModule testModule = await TestModule.GetTestModuleAsync(DevicePrefix + $"_{Guid.NewGuid()}", ModulePrefix).ConfigureAwait(false);

            ConnectionStatus?            status             = null;
            ConnectionStatusChangeReason?statusChangeReason = null;
            int deviceDisabledReceivedCount = 0;
            ConnectionStatusChangesHandler statusChangeHandler = (s, r) =>
            {
                if (r == ConnectionStatusChangeReason.Device_Disabled)
                {
                    status             = s;
                    statusChangeReason = r;
                    deviceDisabledReceivedCount++;
                }
            };

            using (ModuleClient moduleClient = ModuleClient.CreateFromConnectionString(testModule.ConnectionString, transportSettings))
            {
                moduleClient.SetConnectionStatusChangesHandler(statusChangeHandler);
                _log.WriteLine($"Created {nameof(DeviceClient)} ID={TestLogging.IdOf(moduleClient)}");

                Console.WriteLine("ModuleClient OpenAsync.");
                await moduleClient.OpenAsync().ConfigureAwait(false);

                // Receiving the module twin should succeed right now.
                Console.WriteLine("ModuleClient GetTwinAsync.");
                var twin = await moduleClient.GetTwinAsync().ConfigureAwait(false);

                Assert.IsNotNull(twin);

                // Delete the device in IoT Hub.
                using (RegistryManager registryManager = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString))
                {
                    await registryManagerOperation(registryManager, testModule.DeviceId).ConfigureAwait(false);
                }

                // Periodically keep retrieving the device twin to keep connection alive.
                // The ConnectionStatusChangesHandler should be triggered when the connection is closed from IoT hub with an
                // exception thrown.
                int twinRetrievals = 50;
                for (int i = 0; i < twinRetrievals; i++)
                {
                    try
                    {
                        await Task.Delay(TimeSpan.FromSeconds(10)).ConfigureAwait(false);

                        if (deviceDisabledReceivedCount == 1)
                        {
                            // Call an API on the client again to trigger the ConnectionStatusChangesHandler once again with the
                            // Device_Disabled status.
                            // This currently does not work due to some issues with IoT hub allowing new connections even when the
                            // device is deleted/disabled. Once that problem is investigated and fixed, we can re-enable this call
                            // and test for multiple invocations of the ConnectionStatusChangeHandler.
                            // await moduleClient.GetTwinAsync().ConfigureAwait(false);
                            break;
                        }
                    }
                    catch (IotHubException ex)
                    {
                        _log.WriteLine($"Exception occurred while retrieving module twin: {ex}");
                        Assert.IsInstanceOfType(ex.InnerException, typeof(DeviceNotFoundException));
                    }
                }

                Assert.AreEqual(1, deviceDisabledReceivedCount);
                Assert.AreEqual(ConnectionStatus.Disconnected, status);
                Assert.AreEqual(ConnectionStatusChangeReason.Device_Disabled, statusChangeReason);
            }
        }
        private async Task SendMethodAndRespondMuxedOverAmqp(
            ConnectionStringAuthScope authScope,
            Client.TransportType transport,
            int poolSize,
            int devicesCount,
            Func <DeviceClient, Task <Task> > setDeviceReceiveMethod
            )
        {
            var transportSettings = new ITransportSettings[]
            {
                new AmqpTransportSettings(transport)
                {
                    AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
                    {
                        MaxPoolSize = unchecked ((uint)poolSize),
                        Pooling     = true
                    }
                }
            };

            ICollection <DeviceClient>     deviceClients = new List <DeviceClient>();
            Dictionary <DeviceClient, int> deviceClientConnectionStatusChangeCount = new Dictionary <DeviceClient, int>();

            try
            {
                for (int i = 0; i < devicesCount; i++)
                {
                    ConnectionStatus?            lastConnectionStatus             = null;
                    ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
                    int setConnectionStatusChangesHandlerCount = 0;

                    TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{DevicePrefix}_{i}_").ConfigureAwait(false);

                    DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
                    deviceClients.Add(deviceClient);

                    deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
                    {
                        setConnectionStatusChangesHandlerCount++;
                        lastConnectionStatus             = status;
                        lastConnectionStatusChangeReason = statusChangeReason;
                        _log.WriteLine($"{nameof(MethodE2EMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}");
                        deviceClientConnectionStatusChangeCount[deviceClient] = setConnectionStatusChangesHandlerCount;
                    });

                    Task methodReceivedTask = await setDeviceReceiveMethod(deviceClient).ConfigureAwait(false);

                    await Task.WhenAll(
                        MethodOperation.ServiceSendMethodAndVerifyResponse(testDevice.Id),
                        methodReceivedTask).ConfigureAwait(false);
                }
            }
            finally
            {
                // Close and dispose all of the device client instances here
                foreach (DeviceClient deviceClient in deviceClients)
                {
                    await deviceClient.CloseAsync().ConfigureAwait(false);

                    // The connection status change count should be 2: connect (open) and disabled (close)
                    Assert.IsTrue(deviceClientConnectionStatusChangeCount[deviceClient] == 2, $"Connection status change count for deviceClient {TestLogging.GetHashCode(deviceClient)} is {deviceClientConnectionStatusChangeCount[deviceClient]}");

                    _log.WriteLine($"{nameof(MethodE2EMultiplexingTests)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
                    deviceClient.Dispose();
                }
            }
        }