Example #1
0
        public async Task Message_ClientThrowsForMqttTopicNameTooLong()
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);

            using (DeviceClient deviceClient = testDevice.CreateDeviceClient(Client.TransportType.Mqtt))
            {
                await deviceClient.OpenAsync().ConfigureAwait(false);

                Client.Message msg = new Client.Message(Encoding.UTF8.GetBytes("testMessage"));
                //Mqtt topic name consists of, among other things, system properties and user properties
                // setting lots of very long user properties should cause a MessageTooLargeException explaining
                // that the topic name is too long to publish over mqtt
                for (int i = 0; i < 100; i++)
                {
                    msg.Properties.Add(Guid.NewGuid().ToString(), new string('1', 1024));
                }

                await deviceClient.SendEventAsync(msg).ConfigureAwait(false);
            }
        }
        private async Task Twin_ServiceSetsDesiredPropertyAndDeviceReceivesEvent(Client.TransportType transport, Func <DeviceClient, string, string, Task <Task> > setTwinPropertyUpdateCallbackAsync)
        {
            var propName  = Guid.NewGuid().ToString();
            var propValue = Guid.NewGuid().ToString();

            _log.WriteLine($"{nameof(Twin_ServiceSetsDesiredPropertyAndDeviceReceivesEvent)}: name={propName}, value={propValue}");

            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);

            using (DeviceClient deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport))
            {
                Task updateReceivedTask = await setTwinPropertyUpdateCallbackAsync(deviceClient, propName, propValue).ConfigureAwait(false);

                await Task.WhenAll(
                    RegistryManagerUpdateDesiredPropertyAsync(testDevice.Id, propName, propValue),
                    updateReceivedTask).ConfigureAwait(false);

                await deviceClient.CloseAsync().ConfigureAwait(false);
            }
        }
Example #3
0
        private async Task ReceiveMessageWithTimeout(TestDeviceType type, Client.TransportType transport, double time)
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix, type).ConfigureAwait(false);

            using (DeviceClient deviceClient = testDevice.CreateDeviceClient(transport))
            {
                await deviceClient.OpenAsync().ConfigureAwait(false);

                if (transport == Client.TransportType.Mqtt_Tcp_Only ||
                    transport == Client.TransportType.Mqtt_WebSocket_Only)
                {
                    // Dummy ReceiveAsync to ensure mqtt subscription registration before SendAsync() is called on service client.
                    await deviceClient.ReceiveAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false);
                }

                await ReceiveMessageTimeoutCheck(deviceClient, time).ConfigureAwait(false);

                await deviceClient.CloseAsync().ConfigureAwait(false);
            }
        }
        private async Task Twin_DeviceSetsReportedPropertyAndGetsItBack(Client.TransportType transport)
        {
            var propName  = Guid.NewGuid().ToString();
            var propValue = Guid.NewGuid().ToString();

            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);

            using (DeviceClient deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport))
            {
                TwinCollection props = new TwinCollection();
                props[propName] = propValue;
                await deviceClient.UpdateReportedPropertiesAsync(props).ConfigureAwait(false);

                Twin deviceTwin = await deviceClient.GetTwinAsync().ConfigureAwait(false);

                Assert.AreEqual <String>(deviceTwin.Properties.Reported[propName].ToString(), propValue);

                await deviceClient.CloseAsync().ConfigureAwait(false);
            }
        }
        private async Task TestSecurityMessage(Client.TransportType transport)
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(_devicePrefix).ConfigureAwait(false);

            EventHubTestListener testListener = await EventHubTestListener.CreateListener(testDevice.Id).ConfigureAwait(false);

            using (DeviceClient deviceClient = testDevice.CreateDeviceClient(transport))
            {
                try
                {
                    await SendSingleSecurityMessage(deviceClient, testDevice.Id, testListener, _logAnalyticsClient).ConfigureAwait(false);
                }
                finally
                {
                    await deviceClient.CloseAsync().ConfigureAwait(false);

                    await testListener.CloseAsync().ConfigureAwait(false);
                }
            }
        }
        internal async Task SendSingleMessageX509(Client.TransportType transport)
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix, TestDeviceType.X509).ConfigureAwait(false);

            EventHubReceiver eventHubReceiver = await CreateEventHubReceiver(testDevice.Id).ConfigureAwait(false);

            X509Certificate2 cert = Configuration.IoTHub.GetCertificateWithPrivateKey();

            var auth         = new DeviceAuthenticationWithX509Certificate(testDevice.Id, cert);
            var deviceClient = DeviceClient.Create(testDevice.IoTHubHostName, auth, transport);

            try
            {
                await deviceClient.OpenAsync().ConfigureAwait(false);

                string         payload;
                string         p1Value;
                Client.Message testMessage = ComposeD2CTestMessage(out payload, out p1Value);
                await deviceClient.SendEventAsync(testMessage).ConfigureAwait(false);

                bool      isReceived = false;
                Stopwatch sw         = new Stopwatch();
                sw.Start();
                while (!isReceived && sw.Elapsed.Minutes < 1)
                {
                    var events = await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5)).ConfigureAwait(false);

                    isReceived = VerifyTestMessage(events, testDevice.Id, payload, p1Value);
                }

                sw.Stop();

                Assert.IsTrue(isReceived, "Message is not received.");
            }
            finally
            {
                await deviceClient.CloseAsync().ConfigureAwait(false);

                await eventHubReceiver.CloseAsync().ConfigureAwait(false);
            }
        }
        internal async Task SendMessageThrottledForHttp()
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);

            PartitionReceiver eventHubReceiver = await CreateEventHubReceiver(testDevice.Id).ConfigureAwait(false);

            var deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, Client.TransportType.Http1);

            try
            {
                deviceClient.OperationTimeoutInMilliseconds = (uint)FaultInjection.ShortRetryInMilliSec;
                await deviceClient.OpenAsync().ConfigureAwait(false);

                string         payload, p1Value;
                Client.Message testMessage = ComposeD2CTestMessage(out payload, out p1Value);
                await deviceClient.SendEventAsync(testMessage).ConfigureAwait(false);

                bool      isReceived = false;
                Stopwatch sw         = new Stopwatch();
                sw.Start();
                while (!isReceived && sw.Elapsed.Minutes < 1)
                {
                    var events = await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5)).ConfigureAwait(false);

                    isReceived = VerifyTestMessage(events, testDevice.Id, payload, p1Value);
                }
                sw.Stop();

                // Implementation of error injection of throttling on http is that it will throttle the
                // fault injection message itself only.  The duration of fault has no effect on http throttle.
                // Client is supposed to retry sending the throttling fault message until operation timeout.
                await deviceClient.SendEventAsync(FaultInjection.ComposeErrorInjectionProperties(FaultInjection.FaultType_Throttle,
                                                                                                 FaultInjection.FaultCloseReason_Boom, FaultInjection.DefaultDelayInSec, FaultInjection.DefaultDurationInSec)).ConfigureAwait(false);
            }
            finally
            {
                await deviceClient.CloseAsync().ConfigureAwait(false);

                await eventHubReceiver.CloseAsync().ConfigureAwait(false);
            }
        }
        private async Task ReceiveMessageRecovery(Client.TransportType transport, string faultType, string reason,
            int delayInSec)
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);

            ServiceClient serviceClient = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);
            var deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport);

            try
            {
                await deviceClient.OpenAsync().ConfigureAwait(false);
                if (transport == Client.TransportType.Mqtt_Tcp_Only ||
                    transport == Client.TransportType.Mqtt_WebSocket_Only)
                {
                    // Dummy ReceiveAsync to ensure mqtt subscription registration before SendAsync() is called on service client.
                    await deviceClient.ReceiveAsync(TimeSpan.FromSeconds(2)).ConfigureAwait(false);
                }

                string payload, messageId, p1Value;
                await serviceClient.OpenAsync().ConfigureAwait(false);
                await serviceClient.SendAsync(
                    testDevice.Id,
                    ComposeC2DTestMessage(out payload, out messageId, out p1Value)).ConfigureAwait(false);
                await VerifyReceivedC2DMessage(transport, deviceClient, payload, p1Value).ConfigureAwait(false);

                // send error command
                await deviceClient.SendEventAsync(
                    FaultInjection.ComposeErrorInjectionProperties(faultType, reason, delayInSec)).ConfigureAwait(false);

                await Task.Delay(FaultInjection.WaitForDisconnectMilliseconds).ConfigureAwait(false);
                await serviceClient.SendAsync(
                    testDevice.Id,
                    ComposeC2DTestMessage(out payload, out messageId, out p1Value)).ConfigureAwait(false);
                await VerifyReceivedC2DMessage(transport, deviceClient, payload, p1Value).ConfigureAwait(false);
            }
            finally
            {
                await deviceClient.CloseAsync().ConfigureAwait(false);
                await serviceClient.CloseAsync().ConfigureAwait(false);
            }
        }
        private async Task UploadFile(Client.TransportType transport, string filename, bool x509auth = false)
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(
                DevicePrefix, 
                x509auth ? TestDeviceType.X509 : TestDeviceType.Sasl).ConfigureAwait(false);

            DeviceClient deviceClient;
            if (x509auth)
            {
                X509Certificate2 cert = Configuration.IoTHub.GetCertificateWithPrivateKey();

                var auth = new DeviceAuthenticationWithX509Certificate(testDevice.Id, cert);
                deviceClient = DeviceClient.Create(testDevice.IoTHubHostName, auth, transport);
            }
            else
            {
                deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport);
            }

            using(deviceClient)
            using (ServiceClient serviceClient = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString))
            {
                FileNotificationReceiver<FileNotification> notificationReceiver = serviceClient.GetFileNotificationReceiver();

                
                using (FileStream fileStreamSource = new FileStream(filename, FileMode.Open, FileAccess.Read))
                {
                    await deviceClient.UploadToBlobAsync(filename, fileStreamSource).ConfigureAwait(false);
                }

                FileNotification fileNotification = await VerifyFileNotification(notificationReceiver, testDevice.Id).ConfigureAwait(false);

                Assert.IsNotNull(fileNotification, "FileNotification is not received.");
                Assert.AreEqual(testDevice.Id + "/" + filename, fileNotification.BlobName, "Uploaded file name mismatch in notifications");
                Assert.AreEqual(new FileInfo(filename).Length, fileNotification.BlobSizeInBytes, "Uploaded file size mismatch in notifications");
                Assert.IsFalse(string.IsNullOrEmpty(fileNotification.BlobUri), "File notification blob uri is null or empty");

                await deviceClient.CloseAsync().ConfigureAwait(false);
                await serviceClient.CloseAsync().ConfigureAwait(false);
            }
        }
Example #10
0
        private async Task Twin_ServiceSetsDesiredPropertyAndDeviceReceivesEvent_WithObseleteCallbackSetter(Client.TransportType transport)
        {
            var tcs = new TaskCompletionSource<bool>();
            var propName = Guid.NewGuid().ToString();
            var propValue = Guid.NewGuid().ToString();

            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);
            RegistryManager registryManager = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);

            var deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport);

#pragma warning disable CS0618
            await deviceClient.SetDesiredPropertyUpdateCallback((patch, context) =>
            {
                return Task.Run(() =>
                {
                    try
                    {
                        Assert.AreEqual(patch[propName].ToString(), propValue);
                    }
                    catch (Exception e)
                    {
                        tcs.SetException(e);
                    }
                    finally
                    {
                        tcs.SetResult(true);
                    }
                });
            }, null).ConfigureAwait(false);
#pragma warning restore CS0618

            var twinPatch = new Twin();
            twinPatch.Properties.Desired[propName] = propValue;
            await registryManager.UpdateTwinAsync(testDevice.Id, twinPatch, "*").ConfigureAwait(false);
            
            await tcs.Task.ConfigureAwait(false);
            await deviceClient.CloseAsync().ConfigureAwait(false);
            await registryManager.CloseAsync().ConfigureAwait(false);

        }
Example #11
0
        private async Task TestTimeout(TimeSpan?timeout)
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);

            using (ServiceClient sender = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString))
            {
                Stopwatch sw = new Stopwatch();
                sw.Start();

                _log.WriteLine($"Testing ServiceClient SendAsync() timeout in ticks={timeout?.Ticks}");
                try
                {
                    await sender.SendAsync(testDevice.Id, new Message(Encoding.ASCII.GetBytes("Dummy Message")), timeout).ConfigureAwait(false);
                }
                finally
                {
                    sw.Stop();
                    _log.WriteLine($"Testing ServiceClient SendAsync(): exiting test after time={sw.Elapsed}; ticks={sw.ElapsedTicks}");
                }
            }
        }
        /// <summary>
        /// Factory method.
        /// </summary>
        /// <param name="namePrefix"></param>
        /// <param name="type"></param>
        public static async Task <TestModule> GetTestModuleAsync(string deviceNamePrefix, string moduleNamePrefix)
        {
            var        log        = TestLogging.GetInstance();
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(deviceNamePrefix).ConfigureAwait(false);

            string deviceName = testDevice.Id;
            string moduleName = moduleNamePrefix + Guid.NewGuid();

            using var rm = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);
            log.WriteLine($"{nameof(GetTestModuleAsync)}: Creating module for device {deviceName}.");

            var    requestModule = new Module(deviceName, moduleName);
            Module module        = await rm.AddModuleAsync(requestModule).ConfigureAwait(false);

            await rm.CloseAsync().ConfigureAwait(false);

            var ret = new TestModule(module);

            log.WriteLine($"{nameof(GetTestModuleAsync)}: Using device {ret.DeviceId} with module {ret.Id}.");
            return(ret);
        }
        private async Task ReceiveMessageInOperationTimeout(TestDeviceType type, Client.TransportType transport)
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix, type).ConfigureAwait(false);

            using (DeviceClient deviceClient = testDevice.CreateDeviceClient(transport))
            {
                _log.WriteLine($"{nameof(ReceiveMessageInOperationTimeout)} - calling OpenAsync() for transport={transport}");
                await deviceClient.OpenAsync().ConfigureAwait(false);

                if (transport == Client.TransportType.Mqtt_Tcp_Only ||
                    transport == Client.TransportType.Mqtt_WebSocket_Only)
                {
                    // Dummy ReceiveAsync to ensure mqtt subscription registration before SendAsync() is called on service client.
                    await deviceClient.ReceiveAsync(TIMESPAN_FIVE_SECONDS).ConfigureAwait(false);
                }

                try
                {
                    deviceClient.OperationTimeoutInMilliseconds = Convert.ToUInt32(TIMESPAN_ONE_MINUTE.TotalMilliseconds);
                    _log.WriteLine($"{nameof(ReceiveMessageInOperationTimeout)} - setting device client default operation timeout={deviceClient.OperationTimeoutInMilliseconds} ms");

                    if (transport == Client.TransportType.Amqp || transport == Client.TransportType.Amqp_Tcp_Only || transport == Client.TransportType.Amqp_WebSocket_Only)
                    {
                        // For AMQP because of static 1 min interval check the cancellation token, in worst case it will block upto extra 1 min to return
                        await ReceiveMessageWithoutTimeoutCheck(deviceClient, TIMESPAN_ONE_MINUTE).ConfigureAwait(false);
                    }
                    else
                    {
                        await ReceiveMessageWithoutTimeoutCheck(deviceClient, TIMESPAN_FIVE_SECONDS).ConfigureAwait(false);
                    }
                }
                finally
                {
                    _log.WriteLine($"{nameof(ReceiveMessageInOperationTimeout)} - calling CloseAsync() for transport={transport}");
                    deviceClient.OperationTimeoutInMilliseconds = DeviceClient.DefaultOperationTimeoutInMilliseconds;
                    await deviceClient.CloseAsync().ConfigureAwait(false);
                }
            }
        }
Example #14
0
        private async Task Twin_ClientHandlesRejectionInvalidPropertyName(Client.TransportType transport)
        {
            var propName1 = "$" + Guid.NewGuid().ToString();
            var propName2 = Guid.NewGuid().ToString();

            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);

            RegistryManager registryManager = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);

            using (var deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport))
            {
                var exceptionThrown = false;
                try
                {
                    await deviceClient.UpdateReportedPropertiesAsync(new TwinCollection
                    {
                        [propName1] = 123,
                        [propName2] = "abcd"
                    }).ConfigureAwait(false);
                }
                catch (Exception)
                {
                    exceptionThrown = true;
                }

                if (!exceptionThrown)
                {
                    throw new AssertFailedException("Exception was expected, but not thrown.");
                }

                var serviceTwin = await registryManager.GetTwinAsync(testDevice.Id).ConfigureAwait(false);

                Assert.IsFalse(serviceTwin.Properties.Reported.Contains(propName1));

                await deviceClient.CloseAsync().ConfigureAwait(false);
            }

            await registryManager.CloseAsync().ConfigureAwait(false);
        }
        private async Task Twin_ServiceSetsDesiredPropertyAndDeviceReceivesItOnNextGet(Client.TransportType transport)
        {
            var propName  = Guid.NewGuid().ToString();
            var propValue = Guid.NewGuid().ToString();

            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);

            using (RegistryManager registryManager = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString))
                using (DeviceClient deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport))
                {
                    var twinPatch = new Twin();
                    twinPatch.Properties.Desired[propName] = propValue;
                    await registryManager.UpdateTwinAsync(testDevice.Id, twinPatch, "*").ConfigureAwait(false);

                    var deviceTwin = await deviceClient.GetTwinAsync().ConfigureAwait(false);

                    Assert.AreEqual <string>(deviceTwin.Properties.Desired[propName].ToString(), propValue);

                    await deviceClient.CloseAsync().ConfigureAwait(false);

                    await registryManager.CloseAsync().ConfigureAwait(false);
                }
        }
        private async Task SendMethodAndRespondWithObseletedSetMethodHandler(Client.TransportType transport)
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);

            var assertResult = new TaskCompletionSource <Tuple <bool, bool> >();
            var deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport);

#pragma warning disable CS0618

            deviceClient?.SetMethodHandler(MethodName, (request, context) =>
            {
                assertResult.TrySetResult(new Tuple <bool, bool>(request.Name.Equals(MethodName), request.DataAsJson.Equals(ServiceRequestJson)));
                return(Task.FromResult(new MethodResponse(Encoding.UTF8.GetBytes(DeviceResponseJson), 200)));
            }, null);
#pragma warning restore CS0618

            // sleep to ensure async tasks started in SetMethodHandler has completed
            Thread.Sleep(5000);

            await ServiceSendMethodAndVerifyResponse(testDevice.Id, MethodName, DeviceResponseJson, ServiceRequestJson, assertResult).ConfigureAwait(false);

            await deviceClient.CloseAsync().ConfigureAwait(false);
        }
        /// <summary>
        /// Factory method.
        /// </summary>
        /// <param name="namePrefix"></param>
        /// <param name="type"></param>
        public static async Task <TestDevice> GetTestDeviceAsync(string namePrefix, TestDeviceType type = TestDeviceType.Sasl)
        {
            string prefix = namePrefix + type + "_";

            try
            {
                await s_semaphore.WaitAsync().ConfigureAwait(false);

                if (!s_deviceCache.TryGetValue(prefix, out TestDevice testDevice))
                {
                    await CreateDeviceAsync(type, prefix).ConfigureAwait(false);
                }

                TestDevice ret = s_deviceCache[prefix];

                s_log.WriteLine($"{nameof(GetTestDeviceAsync)}: Using device {ret.Id}.");
                return(ret);
            }
            finally
            {
                s_semaphore.Release();
            }
        }
        private async Task Twin_DeviceSetsReportedPropertyAndServiceReceivesIt(Client.TransportType transport)
        {
            var propName  = Guid.NewGuid().ToString();
            var propValue = Guid.NewGuid().ToString();

            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);

            using (RegistryManager registryManager = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString))
                using (DeviceClient deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport))
                {
                    var patch = new TwinCollection();
                    patch[propName] = propValue;
                    await deviceClient.UpdateReportedPropertiesAsync(patch).ConfigureAwait(false);

                    await deviceClient.CloseAsync().ConfigureAwait(false);

                    var serviceTwin = await registryManager.GetTwinAsync(testDevice.Id).ConfigureAwait(false);

                    Assert.AreEqual <string>(serviceTwin.Properties.Reported[propName].ToString(), propValue);

                    _log.WriteLine("verified " + serviceTwin.Properties.Reported[propName].ToString() + "=" + propValue);
                }
        }
        internal async Task SendSingleMessage(Client.TransportType transport)
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);

            EventHubClient   eventHubClient;
            EventHubReceiver eventHubReceiver = CreateEventHubReceiver(testDevice.Id, out eventHubClient);
            var deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport);

            try
            {
                await deviceClient.OpenAsync().ConfigureAwait(false);

                string         payload;
                string         p1Value;
                Client.Message testMessage = ComposeD2CTestMessage(out payload, out p1Value);
                await deviceClient.SendEventAsync(testMessage).ConfigureAwait(false);

                bool      isReceived = false;
                Stopwatch sw         = new Stopwatch();
                sw.Start();
                while (!isReceived && sw.Elapsed.Minutes < 1)
                {
                    var events = await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5)).ConfigureAwait(false);

                    isReceived = VerifyTestMessage(events, testDevice.Id, payload, p1Value);
                }
                sw.Stop();

                Assert.IsTrue(isReceived, "Message is not received.");
            }
            finally
            {
                await deviceClient.CloseAsync().ConfigureAwait(false);

                await eventHubReceiver.CloseAsync().ConfigureAwait(false);
            }
        }
        // This function create a device with x509 cert and connect to the iothub on the transport specified.
        // Then a message is send from the service client.
        // It then verifies the message is received on the device.
        private async Task ReceiveSingleMessageX509(Client.TransportType transport)
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix, TestDeviceType.X509).ConfigureAwait(false);

            ServiceClient serviceClient = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);

            X509Certificate2 cert = Configuration.IoTHub.GetCertificateWithPrivateKey();

            var auth         = new DeviceAuthenticationWithX509Certificate(testDevice.Id, cert);
            var deviceClient = DeviceClient.Create(testDevice.IoTHubHostName, auth, transport);

            try
            {
                await deviceClient.OpenAsync().ConfigureAwait(false);

                if (transport == Client.TransportType.Mqtt_Tcp_Only ||
                    transport == Client.TransportType.Mqtt_WebSocket_Only)
                {
                    // Dummy ReceiveAsync to ensure mqtt subscription registration before SendAsync() is called on service client.
                    await deviceClient.ReceiveAsync(TimeSpan.FromSeconds(2)).ConfigureAwait(false);
                }

                string payload, messageId, p1Value;
                await serviceClient.OpenAsync().ConfigureAwait(false);

                await serviceClient.SendAsync(testDevice.Id,
                                              ComposeC2DTestMessage(out payload, out messageId, out p1Value)).ConfigureAwait(false);

                await VerifyReceivedC2DMessage(transport, deviceClient, payload, p1Value).ConfigureAwait(false);
            }
            finally
            {
                await deviceClient.CloseAsync().ConfigureAwait(false);

                await serviceClient.CloseAsync().ConfigureAwait(false);
            }
        }
        private static async Task CreateDeviceAsync(TestDeviceType type, string prefix)
        {
            string deviceName = prefix + Guid.NewGuid();

            s_log.WriteLine($"{nameof(GetTestDeviceAsync)}: Device with prefix {prefix} not found.");

            // Delete existing devices named this way and create a new one.
            using (RegistryManager rm = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString))
            {
                s_log.WriteLine($"{nameof(GetTestDeviceAsync)}: Creating device {deviceName} with type {type}.");

                Client.IAuthenticationMethod auth = null;

                Device requestDevice = new Device(deviceName);
                if (type == TestDeviceType.X509)
                {
                    requestDevice.Authentication = new AuthenticationMechanism()
                    {
                        X509Thumbprint = new X509Thumbprint()
                        {
                            PrimaryThumbprint = Configuration.IoTHub.GetCertificateWithPrivateKey().Thumbprint
                        }
                    };

                    auth = new DeviceAuthenticationWithX509Certificate(deviceName, Configuration.IoTHub.GetCertificateWithPrivateKey());
                }

                Device device = await rm.AddDeviceAsync(requestDevice).ConfigureAwait(false);

                s_log.WriteLine($"{nameof(GetTestDeviceAsync)}: Pausing for {DelayAfterDeviceCreationSeconds}s after device was created.");
                await Task.Delay(DelayAfterDeviceCreationSeconds * 1000).ConfigureAwait(false);

                s_deviceCache[prefix] = new TestDevice(device, auth);

                await rm.CloseAsync().ConfigureAwait(false);
            }
        }
        private async Task UploadFileDisconnectTransport(
            Client.TransportType transport,
            string filename,
            string faultType,
            string reason,
            int delayInSec,
            int durationInSec           = 0,
            int retryDurationInMilliSec = FaultInjection.RecoveryTimeMilliseconds)
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);

            using (var deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport))
            {
                deviceClient.OperationTimeoutInMilliseconds = (uint)retryDurationInMilliSec;

                await FileNotificationTestListener.InitAsync().ConfigureAwait(false);

                using (var fileStreamSource = new FileStream(filename, FileMode.Open, FileAccess.Read))
                {
                    Task fileUploadTask     = deviceClient.UploadToBlobAsync(filename, fileStreamSource);
                    Task errorInjectionTask = SendErrorInjectionMessageAsync(deviceClient, faultType, reason, delayInSec, durationInSec);
                    await Task.WhenAll(fileUploadTask, errorInjectionTask).ConfigureAwait(false);

                    await FileNotificationTestListener.VerifyFileNotification(filename, testDevice.Id).ConfigureAwait(false);
                }

                try
                {
                    await deviceClient.CloseAsync().ConfigureAwait(false);
                }
                catch
                {
                    // catch and ignore exceptions resulted incase device client close failed while offline
                }
            }
        }
Example #23
0
        private async Task GetSasUriAsync(Client.TransportType transport, string blobName, bool useX509auth = false)
        {
            using TestDevice testDevice = await TestDevice
                                          .GetTestDeviceAsync(
                      Logger,
                      _devicePrefix,
                      useX509auth
                      ?TestDeviceType.X509
                      : TestDeviceType.Sasl)
                                          .ConfigureAwait(false);

            DeviceClient     deviceClient;
            X509Certificate2 cert = null;
            DeviceAuthenticationWithX509Certificate x509Auth = null;

            if (useX509auth)
            {
                cert     = s_selfSignedCertificate;
                x509Auth = new DeviceAuthenticationWithX509Certificate(testDevice.Id, cert);

                deviceClient = DeviceClient.Create(testDevice.IoTHubHostName, x509Auth, transport);
            }
            else
            {
                deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport);
            }

            using (deviceClient)
            {
                FileUploadSasUriResponse sasUriResponse = await deviceClient.GetFileUploadSasUriAsync(new FileUploadSasUriRequest { BlobName = blobName });

                await deviceClient.CloseAsync().ConfigureAwait(false);
            }

            x509Auth?.Dispose();
        }
        private async Task DeviceClient_TokenIsRefreshed_Internal(Client.TransportType transport, int ttl = 20)
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);

            int           buffer             = 50;
            Device        device             = testDevice.Identity;
            SemaphoreSlim deviceDisconnected = new SemaphoreSlim(0);

            var refresher = new TestTokenRefresher(
                device.Id,
                device.Authentication.SymmetricKey.PrimaryKey,
                ttl,
                buffer,
                transport);

            using (DeviceClient deviceClient = DeviceClient.Create(testDevice.IoTHubHostName, refresher, transport))
            {
                _log.WriteLine($"Created {nameof(DeviceClient)} ID={TestLogging.IdOf(deviceClient)}");

                if (transport == Client.TransportType.Mqtt)
                {
                    deviceClient.SetConnectionStatusChangesHandler((ConnectionStatus status, ConnectionStatusChangeReason reason) =>
                    {
                        _log.WriteLine($"{nameof(ConnectionStatusChangesHandler)}: {status}; {reason}");
                        if (status == ConnectionStatus.Disconnected_Retrying || status == ConnectionStatus.Disconnected)
                        {
                            deviceDisconnected.Release();
                        }
                    });
                }

                var message = new Client.Message(Encoding.UTF8.GetBytes("Hello"));

                using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(ttl * 10)))
                {
                    try
                    {
                        // Create the first Token.
                        Console.WriteLine($"[{DateTime.UtcNow}] OpenAsync");
                        await deviceClient.OpenAsync(cts.Token).ConfigureAwait(false);

                        Console.WriteLine($"[{DateTime.UtcNow}] SendEventAsync (1)");
                        await deviceClient.SendEventAsync(message, cts.Token).ConfigureAwait(false);

                        await refresher.WaitForTokenRefreshAsync(cts.Token).ConfigureAwait(false);
                    }
                    catch (OperationCanceledException ex)
                    {
                        Assert.Fail($"{TestLogging.IdOf(deviceClient)} did not get the initial token. {ex}");
                        throw;
                    }

                    // Wait for the Token to expire.
                    if (transport == Client.TransportType.Http1)
                    {
                        float waitTime = (float)ttl * ((float)buffer / 100) + 1;
                        Console.WriteLine($"[{DateTime.UtcNow}] Waiting {waitTime} seconds.");
                        await Task.Delay(TimeSpan.FromSeconds(waitTime)).ConfigureAwait(false);
                    }
                    else if (transport == Client.TransportType.Mqtt)
                    {
                        Console.WriteLine($"[{DateTime.UtcNow}] Waiting for device disconnect.");
                        await deviceDisconnected.WaitAsync(cts.Token).ConfigureAwait(false);
                    }

                    try
                    {
                        Console.WriteLine($"[{DateTime.UtcNow}] SendEventAsync (2)");
                        await deviceClient.SendEventAsync(message, cts.Token).ConfigureAwait(false);

                        await refresher.WaitForTokenRefreshAsync(cts.Token).ConfigureAwait(false);
                    }
                    catch (OperationCanceledException ex)
                    {
                        Assert.Fail($"{TestLogging.IdOf(deviceClient)} did not refresh token after {refresher.DetectedRefreshInterval}. {ex}");
                        throw;
                    }

                    // Ensure that the token was refreshed.
                    Console.WriteLine($"[{DateTime.UtcNow}] Token was refreshed after {refresher.DetectedRefreshInterval} (ttl = {ttl} seconds).");
                    Assert.IsTrue(
                        refresher.DetectedRefreshInterval.TotalSeconds < (float)ttl * (1 + (float)buffer / 100), // Wait for more than what we expect.
                        $"Token was refreshed after {refresher.DetectedRefreshInterval} although ttl={ttl} seconds.");

                    Console.WriteLine($"[{DateTime.UtcNow}] CloseAsync");
                    await deviceClient.CloseAsync().ConfigureAwait(false);
                }
            }
        }
Example #25
0
        public async Task DuplicateDevice_NoRetry_NoPingpong_OpenAsync()
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(Logger, _devicePrefix, TestDeviceType.Sasl).ConfigureAwait(false);

            Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: 2 device client instances with the same deviceId={testDevice.Id}.");

            using DeviceClient deviceClient1 = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only);
            using DeviceClient deviceClient2 = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only);

            Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: set device client instance 1 to no retry.");
            deviceClient1.SetRetryPolicy(new NoRetry());

            ConnectionStatus?lastConnectionStatusDevice1 = null;
            var connectionStatusChangesDevice1           = new Dictionary <ConnectionStatus, int>();

            deviceClient1.SetConnectionStatusChangesHandler((status, reason) =>
            {
                connectionStatusChangesDevice1.TryGetValue(status, out int count);
                count++;
                connectionStatusChangesDevice1[status] = count;
                lastConnectionStatusDevice1            = status;
            });

            ConnectionStatus?lastConnectionStatusDevice2 = null;
            var connectionStatusChangesDevice2           = new Dictionary <ConnectionStatus, int>();

            deviceClient2.SetConnectionStatusChangesHandler((status, reason) =>
            {
                connectionStatusChangesDevice2.TryGetValue(status, out int count);
                count++;
                connectionStatusChangesDevice2[status] = count;
                lastConnectionStatusDevice2            = status;
            });

            Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: device client instance 1 calling OpenAsync...");
            await deviceClient1.OpenAsync().ConfigureAwait(false);

            await deviceClient1
            .SetMethodHandlerAsync(
                "empty_method",
                (methodRequest, userContext) => Task.FromResult(new MethodResponse(200)),
                deviceClient1)
            .ConfigureAwait(false);

            Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: device client instance 2 calling OpenAsync...");
            await deviceClient2.OpenAsync().ConfigureAwait(false);

            await deviceClient2
            .SetMethodHandlerAsync(
                "empty_method",
                (methodRequest, userContext) => Task.FromResult(new MethodResponse(200)),
                deviceClient2)
            .ConfigureAwait(false);

            Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: waiting device client instance 1 to be kicked off...");
            var sw = Stopwatch.StartNew();

            while (sw.Elapsed < FaultInjection.LatencyTimeBuffer)
            {
                if (connectionStatusChangesDevice1.ContainsKey(ConnectionStatus.Disconnected))
                {
                    break;
                }
                await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
            }
            sw.Reset();

            lastConnectionStatusDevice1.Should().Be(ConnectionStatus.Disconnected, $"Excpected device 1 to be {ConnectionStatus.Disconnected} but was {lastConnectionStatusDevice1}.");
            connectionStatusChangesDevice1.Should().NotContainKey(ConnectionStatus.Disconnected_Retrying, $"Shouldn't get {ConnectionStatus.Disconnected_Retrying} status change.");
            int connected = connectionStatusChangesDevice1[ConnectionStatus.Connected];

            connected.Should().Be(1, $"Should get {ConnectionStatus.Connected} once but got it {connected} times.");
            int disconnected = connectionStatusChangesDevice1[ConnectionStatus.Disconnected];

            disconnected.Should().Be(1, $"Should get {ConnectionStatus.Disconnected} once but got it {disconnected} times.");

            lastConnectionStatusDevice2.Should().Be(ConnectionStatus.Connected, $"Expected device 2 to be {ConnectionStatus.Connected} but was {lastConnectionStatusDevice2}.");
        }
Example #26
0
        public async Task FaultInjection_NoRetry_NoRecovery_OpenAsync()
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(Logger, _devicePrefix, TestDeviceType.Sasl).ConfigureAwait(false);

            using DeviceClient deviceClient = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only);

            Logger.Trace($"{nameof(FaultInjection_NoRetry_NoRecovery_OpenAsync)}: deviceId={testDevice.Id}");
            deviceClient.SetRetryPolicy(new NoRetry());

            ConnectionStatus?            lastConnectionStatus             = null;
            ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
            var connectionStatusChanges = new Dictionary <ConnectionStatus, int>();

            deviceClient.SetConnectionStatusChangesHandler((status, reason) =>
            {
                connectionStatusChanges.TryGetValue(status, out int count);
                count++;
                connectionStatusChanges[status]  = count;
                lastConnectionStatus             = status;
                lastConnectionStatusChangeReason = reason;
            });

            Logger.Trace($"{nameof(FaultInjection_NoRetry_NoRecovery_OpenAsync)}: calling OpenAsync...");
            await deviceClient.OpenAsync().ConfigureAwait(false);

            Logger.Trace($"{nameof(FaultInjection_NoRetry_NoRecovery_OpenAsync)}: injecting fault {FaultInjection.FaultType_Tcp}...");
            await FaultInjection
            .ActivateFaultInjectionAsync(
                Client.TransportType.Amqp_Tcp_Only,
                FaultInjection.FaultType_Tcp,
                FaultInjection.FaultCloseReason_Boom,
                FaultInjection.DefaultFaultDelay,
                FaultInjection.DefaultFaultDuration,
                deviceClient,
                Logger)
            .ConfigureAwait(false);

            await Task.Delay(FaultInjection.DefaultFaultDelay).ConfigureAwait(false);

            Logger.Trace($"{nameof(FaultInjection_NoRetry_NoRecovery_OpenAsync)}: waiting fault injection occurs...");
            var sw = Stopwatch.StartNew();

            while (sw.Elapsed < FaultInjection.LatencyTimeBuffer)
            {
                if (connectionStatusChanges.ContainsKey(ConnectionStatus.Disconnected))
                {
                    break;
                }
                await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
            }
            sw.Reset();

            lastConnectionStatus.Should().Be(ConnectionStatus.Disconnected, $"Expected device to be {ConnectionStatus.Disconnected} but was {lastConnectionStatus}.");
            lastConnectionStatusChangeReason.Should().Be(ConnectionStatusChangeReason.Retry_Expired, $"Expected device to be {ConnectionStatusChangeReason.Retry_Expired} but was {lastConnectionStatusChangeReason}.");
            connectionStatusChanges.Should().NotContainKey(ConnectionStatus.Disconnected_Retrying, $"Shouldn't get {ConnectionStatus.Disconnected_Retrying} status change.");
            int connected = connectionStatusChanges[ConnectionStatus.Connected];

            connected.Should().Be(1, $"Should get {ConnectionStatus.Connected} once but got it {connected} times.");
            int disconnected = connectionStatusChanges[ConnectionStatus.Disconnected];

            disconnected.Should().Be(1, $"Should get {ConnectionStatus.Disconnected} once but got it {disconnected} times.");
        }
        // Error injection template method.
        public static async Task TestErrorInjectionAsync(
            string devicePrefix,
            TestDeviceType type,
            Client.TransportType transport,
            string faultType,
            string reason,
            int delayInSec,
            int durationInSec,
            Func <DeviceClient, TestDevice, Task> initOperation,
            Func <DeviceClient, TestDevice, Task> testOperation,
            Func <Task> cleanupOperation)
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(devicePrefix, type).ConfigureAwait(false);

            DeviceClient deviceClient = testDevice.CreateDeviceClient(transport);

            ConnectionStatus?            lastConnectionStatus             = null;
            ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
            int setConnectionStatusChangesHandlerCount = 0;

            deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
            {
                setConnectionStatusChangesHandlerCount++;
                lastConnectionStatus             = status;
                lastConnectionStatusChangeReason = statusChangeReason;
                s_log.WriteLine($"{nameof(FaultInjection)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}");
            });

            var watch = new Stopwatch();

            try
            {
                await deviceClient.OpenAsync().ConfigureAwait(false);

                if (transport != Client.TransportType.Http1)
                {
                    Assert.IsTrue(setConnectionStatusChangesHandlerCount >= 1); // Normally one connection but in some cases, due to network issues we might have already retried several times to connect.
                    Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus);
                    Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason);
                }

                await initOperation(deviceClient, testDevice).ConfigureAwait(false);

                s_log.WriteLine($">>> {nameof(FaultInjection)} Testing baseline");
                await testOperation(deviceClient, testDevice).ConfigureAwait(false);

                watch.Start();
                s_log.WriteLine($">>> {nameof(FaultInjection)} Testing fault handling");
                await ActivateFaultInjection(transport, faultType, reason, delayInSec, durationInSec, deviceClient).ConfigureAwait(false);

                int delay = FaultInjection.WaitForDisconnectMilliseconds - (int)watch.ElapsedMilliseconds;
                if (delay < 0)
                {
                    delay = 0;
                }

                s_log.WriteLine($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {delay}ms");
                await Task.Delay(delay).ConfigureAwait(false);

                await testOperation(deviceClient, testDevice).ConfigureAwait(false);

                await deviceClient.CloseAsync().ConfigureAwait(false);

                if (transport != Client.TransportType.Http1)
                {
                    if (FaultShouldDisconnect(faultType))
                    {
                        // 4 is the minimum notification count: connect, fault, reconnect, disable.
                        // There are cases where the retry must be timed out (i.e. very likely for MQTT where otherwise
                        // we would attempt to send the fault injection forever.)
                        Assert.IsTrue(setConnectionStatusChangesHandlerCount >= 4);
                    }
                    else
                    {
                        // 2 is the minimum notification count: connect, disable.
                        // We will monitor the test environment real network stability and switch to >=2 if necessary to
                        // account for real network issues.
                        Assert.IsTrue(setConnectionStatusChangesHandlerCount == 2);
                    }

                    Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus);
                    Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason);
                }
            }
            finally
            {
                await cleanupOperation().ConfigureAwait(false);

                s_log.WriteLine($"{nameof(FaultInjection)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
                deviceClient.Dispose();

                watch.Stop();

                int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds;
                if (timeToFinishFaultInjection > 0)
                {
                    s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed.");
                    await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false);
                }
            }
        }
        public static async Task TestPoolAmqpAsync(
            string devicePrefix,
            Client.TransportType transport,
            int poolSize,
            int devicesCount,
            Func <DeviceClient, TestDevice, Task> initOperation,
            Func <DeviceClient, TestDevice, Task> testOperation,
            Func <Task> cleanupOperation,
            ConnectionStringAuthScope authScope,
            bool ignoreConnectionStatus)
        {
            var transportSettings = new ITransportSettings[]
            {
                new AmqpTransportSettings(transport)
                {
                    AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
                    {
                        MaxPoolSize = unchecked ((uint)poolSize),
                        Pooling     = true
                    }
                }
            };

            int  totalRuns          = 0;
            int  successfulRuns     = 0;
            int  currentSuccessRate = 0;
            bool reRunTest          = false;

            IList <TestDevice>   testDevices   = new List <TestDevice>();
            IList <DeviceClient> deviceClients = new List <DeviceClient>();
            IList <AmqpConnectionStatusChange> amqpConnectionStatuses = new List <AmqpConnectionStatusChange>();
            IList <Task> operations = new List <Task>();

            do
            {
                totalRuns++;

                // Arrange
                // Initialize the test device client instances
                // Set the device client connection status change handler
                s_log.WriteLine($">>> {nameof(PoolingOverAmqp)} Initializing Device Clients for multiplexing test - Test run {totalRuns}");
                for (int i = 0; i < devicesCount; i++)
                {
                    TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{devicePrefix}_{i}_").ConfigureAwait(false);

                    DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);

                    var amqpConnectionStatusChange = new AmqpConnectionStatusChange();
                    deviceClient.SetConnectionStatusChangesHandler(amqpConnectionStatusChange.ConnectionStatusChangesHandler);

                    testDevices.Add(testDevice);
                    deviceClients.Add(deviceClient);
                    amqpConnectionStatuses.Add(amqpConnectionStatusChange);

                    if (initOperation != null)
                    {
                        operations.Add(initOperation(deviceClient, testDevice));
                    }
                }

                await Task.WhenAll(operations).ConfigureAwait(false);

                operations.Clear();

                try
                {
                    for (int i = 0; i < devicesCount; i++)
                    {
                        operations.Add(testOperation(deviceClients[i], testDevices[i]));
                    }
                    await Task.WhenAll(operations).ConfigureAwait(false);

                    operations.Clear();

                    // Close the device client instances and verify the connection status change checks
                    bool deviceConnectionStatusAsExpected = true;
                    for (int i = 0; i < devicesCount; i++)
                    {
                        await deviceClients[i].CloseAsync().ConfigureAwait(false);

                        if (!ignoreConnectionStatus)
                        {
                            // The connection status change count should be 2: connect (open) and disabled (close)
                            if (amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount != 2)
                            {
                                deviceConnectionStatusAsExpected = false;
                            }

                            // The connection status should be "Disabled", with connection status change reason "Client_close"
                            Assert.AreEqual(
                                ConnectionStatus.Disabled,
                                amqpConnectionStatuses[i].LastConnectionStatus,
                                $"The actual connection status is = {amqpConnectionStatuses[i].LastConnectionStatus}");
                            Assert.AreEqual(
                                ConnectionStatusChangeReason.Client_Close,
                                amqpConnectionStatuses[i].LastConnectionStatusChangeReason,
                                $"The actual connection status change reason is = {amqpConnectionStatuses[i].LastConnectionStatusChangeReason}");
                        }
                    }
                    if (deviceConnectionStatusAsExpected)
                    {
                        successfulRuns++;
                    }

                    currentSuccessRate = (int)((double)successfulRuns / totalRuns * 100);
                    reRunTest          = currentSuccessRate < TestSuccessRate;
                }
                finally
                {
                    // Close the service-side components and dispose the device client instances.
                    if (cleanupOperation != null)
                    {
                        await cleanupOperation().ConfigureAwait(false);
                    }

                    foreach (DeviceClient deviceClient in deviceClients)
                    {
                        deviceClient.Dispose();
                    }

                    // Clean up the local lists
                    testDevices.Clear();
                    deviceClients.Clear();
                    amqpConnectionStatuses.Clear();
                }
            } while (reRunTest && totalRuns < MaxTestRunCount);

            Assert.IsFalse(reRunTest, $"Device client instances got disconnected in {totalRuns - successfulRuns} runs out of {totalRuns}; current testSuccessRate = {currentSuccessRate}%.");
        }
        private async Task SendMethodAndRespondRecovery(Client.TransportType transport, string faultType, string reason, int delayInSec)
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);

            var          assertResult = new TaskCompletionSource <Tuple <bool, bool> >();
            DeviceClient deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport);

            ConnectionStatus?            lastConnectionStatus             = null;
            ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
            int setConnectionStatusChangesHandlerCount = 0;
            var tcsConnected    = new TaskCompletionSource <bool>();
            var tcsDisconnected = new TaskCompletionSource <bool>();

            deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
            {
                _log.WriteLine("Connection Changed to {0} because {1}", status, statusChangeReason);

                if (status == ConnectionStatus.Disconnected_Retrying)
                {
                    tcsDisconnected.TrySetResult(true);
                    Assert.AreEqual(ConnectionStatusChangeReason.No_Network, statusChangeReason);
                }
                else if (status == ConnectionStatus.Connected)
                {
                    tcsConnected.TrySetResult(true);
                }

                lastConnectionStatus             = status;
                lastConnectionStatusChangeReason = statusChangeReason;
                setConnectionStatusChangesHandlerCount++;
            });

            await deviceClient.SetMethodHandlerAsync(MethodName,
                                                     (request, context) =>
            {
                assertResult.TrySetResult(new Tuple <bool, bool>(request.Name.Equals(MethodName),
                                                                 request.DataAsJson.Equals(ServiceRequestJson)));
                return(Task.FromResult(new MethodResponse(Encoding.UTF8.GetBytes(DeviceResponseJson), 200)));
            },
                                                     null).ConfigureAwait(false);

            // assert on successful connection
            await Task.WhenAny(Task.Delay(1000), tcsConnected.Task).ConfigureAwait(false);

            Assert.IsTrue(tcsConnected.Task.IsCompleted, "Initial connection failed");

            if (transport != Client.TransportType.Http1)
            {
                Assert.AreEqual(1, setConnectionStatusChangesHandlerCount);
                Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus);
                Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason);
            }

            // check on normal operation
            await
            ServiceSendMethodAndVerifyResponse(testDevice.Id, MethodName, DeviceResponseJson,
                                               ServiceRequestJson, assertResult).ConfigureAwait(false);

            // reset ConnectionStatusChangesHandler data
            setConnectionStatusChangesHandlerCount = 0;
            tcsConnected    = new TaskCompletionSource <bool>();
            tcsDisconnected = new TaskCompletionSource <bool>();

            // send error command
            await deviceClient.SendEventAsync(FaultInjection.ComposeErrorInjectionProperties(faultType, reason, delayInSec)).ConfigureAwait(false);

            // wait for disconnection
            await Task.WhenAny(
                Task.Run(async() =>
            {
                await Task.Delay(TimeSpan.FromSeconds(10)).ConfigureAwait(false);
            }), tcsDisconnected.Task).ConfigureAwait(false);

            Assert.IsTrue(tcsDisconnected.Task.IsCompleted, "Error injection did not interrupt the device");

            await Task.WhenAny(
                Task.Run(async() =>
            {
                await Task.Delay(TimeSpan.FromSeconds(MaximumRecoveryTimeSeconds)).ConfigureAwait(false);
                return(Task.FromResult(true));
            }), tcsConnected.Task).ConfigureAwait(false);

            Assert.IsTrue(tcsConnected.Task.IsCompleted, "Recovery connection failed");

            assertResult = new TaskCompletionSource <Tuple <bool, bool> >();
            await
            ServiceSendMethodAndVerifyResponse(testDevice.Id, MethodName, DeviceResponseJson,
                                               ServiceRequestJson, assertResult).ConfigureAwait(false);

            setConnectionStatusChangesHandlerCount = 0;

            //remove and CloseAsync
            await deviceClient.SetMethodHandlerAsync(MethodName, null, null).ConfigureAwait(false);

            if (transport != Client.TransportType.Http1)
            {
                Assert.AreEqual(1, setConnectionStatusChangesHandlerCount);
                Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus);
                Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason);
            }
        }
        internal async Task SendMessageRecovery(Client.TransportType transport,
                                                string faultType, string reason, int delayInSec, int durationInSec = 0, int retryDurationInMilliSec = 240000)
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);

            var              deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport);
            EventHubClient   eventHubClient;
            EventHubReceiver eventHubReceiver = CreateEventHubReceiver(testDevice.Id, out eventHubClient);

            try
            {
                deviceClient.OperationTimeoutInMilliseconds = (uint)retryDurationInMilliSec;

                ConnectionStatus?            lastConnectionStatus             = null;
                ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
                int setConnectionStatusChangesHandlerCount = 0;

                deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
                {
                    lastConnectionStatus             = status;
                    lastConnectionStatusChangeReason = statusChangeReason;
                    setConnectionStatusChangesHandlerCount++;
                });

                await deviceClient.OpenAsync().ConfigureAwait(false);

                if (transport != Client.TransportType.Http1)
                {
                    Assert.AreEqual(1, setConnectionStatusChangesHandlerCount);
                    Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus);
                    Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason);
                }

                string         payload, p1Value;
                Client.Message testMessage = ComposeD2CTestMessage(out payload, out p1Value);
                await deviceClient.SendEventAsync(testMessage).ConfigureAwait(false);


                bool      isReceived = false;
                Stopwatch sw         = new Stopwatch();
                sw.Start();
                while (!isReceived && sw.Elapsed.Minutes < 1)
                {
                    var events = await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5)).ConfigureAwait(false);

                    isReceived = VerifyTestMessage(events, testDevice.Id, payload, p1Value);
                }
                sw.Stop();

                // send error command and clear eventHubReceiver of the fault injection message
                await deviceClient.SendEventAsync(FaultInjection.ComposeErrorInjectionProperties(faultType, reason,
                                                                                                 delayInSec,
                                                                                                 durationInSec)).ConfigureAwait(false);

                await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5)).ConfigureAwait(false);

                Thread.Sleep(1000);

                testMessage = ComposeD2CTestMessage(out payload, out p1Value);
                await deviceClient.SendEventAsync(testMessage).ConfigureAwait(false);

                sw.Reset();
                sw.Start();
                while (!isReceived && sw.Elapsed.Minutes < 1)
                {
                    var events = await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5)).ConfigureAwait(false);

                    isReceived = VerifyTestMessage(events, testDevice.Id, payload, p1Value);
                }
                sw.Stop();

                await deviceClient.CloseAsync().ConfigureAwait(false);

                if (transport != Client.TransportType.Http1)
                {
                    Assert.AreEqual(2, setConnectionStatusChangesHandlerCount);
                    Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus);
                    Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason);
                }
            }
            finally
            {
                await deviceClient.CloseAsync().ConfigureAwait(false);

                await eventHubReceiver.CloseAsync().ConfigureAwait(false);
            }
        }