public static async Task TestFaultInjectionPoolAmqpAsync(
            string devicePrefix,
            Client.TransportType transport,
            int poolSize,
            int devicesCount,
            string faultType,
            string reason,
            int delayInSec,
            int durationInSec,
            Func <DeviceClient, TestDevice, Task> initOperation,
            Func <DeviceClient, TestDevice, Task> testOperation,
            Func <IList <DeviceClient>, Task> cleanupOperation,
            ConnectionStringAuthScope authScope)
        {
            var transportSettings = new ITransportSettings[]
            {
                new AmqpTransportSettings(transport)
                {
                    AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
                    {
                        MaxPoolSize = unchecked ((uint)poolSize),
                        Pooling     = true
                    }
                }
            };

            IList <TestDevice>   testDevices   = new List <TestDevice>();
            IList <DeviceClient> deviceClients = new List <DeviceClient>();
            IList <AmqpConnectionStatusChange> amqpConnectionStatuses = new List <AmqpConnectionStatusChange>();
            IList <Task> operations = new List <Task>();

            // Arrange
            // Initialize the test device client instances
            // Set the device client connection status change handler
            s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)} Initializing Device Clients for multiplexing test.");
            for (int i = 0; i < devicesCount; i++)
            {
                TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{devicePrefix}_{i}_").ConfigureAwait(false);

                DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);

                var amqpConnectionStatusChange = new AmqpConnectionStatusChange(testDevice.Id);
                deviceClient.SetConnectionStatusChangesHandler(amqpConnectionStatusChange.ConnectionStatusChangesHandler);

                testDevices.Add(testDevice);
                deviceClients.Add(deviceClient);
                amqpConnectionStatuses.Add(amqpConnectionStatusChange);

                operations.Add(initOperation(deviceClient, testDevice));
            }
            await Task.WhenAll(operations).ConfigureAwait(false);

            operations.Clear();

            var watch = new Stopwatch();

            try
            {
                // Act-Assert
                // Perform the test operation and verify the operation is successful

                // Perform baseline test operation
                for (int i = 0; i < devicesCount; i++)
                {
                    s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing baseline operation for device {i}.");
                    operations.Add(testOperation(deviceClients[i], testDevices[i]));
                }
                await Task.WhenAll(operations).ConfigureAwait(false);

                operations.Clear();

                int countBeforeFaultInjection = amqpConnectionStatuses[0].ConnectionStatusChangeCount;
                // Inject the fault into device 0
                watch.Start();

                s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: {testDevices[0].Id} Requesting fault injection type={faultType} reason={reason}, delay={delayInSec}s, duration={durationInSec}s");
                var   faultInjectionMessage = FaultInjection.ComposeErrorInjectionProperties(faultType, reason, delayInSec, durationInSec);
                await deviceClients[0].SendEventAsync(faultInjectionMessage).ConfigureAwait(false);

                s_log.WriteLine($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {delayInSec} seconds.");
                await Task.Delay(TimeSpan.FromSeconds(delayInSec)).ConfigureAwait(false);

                // For disconnect type faults, the faulted device should disconnect and all devices should recover.
                if (FaultInjection.FaultShouldDisconnect(faultType))
                {
                    s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirming fault injection has been actived.");
                    // Check that service issued the fault to the faulting device [device 0]
                    bool isFaulted = false;
                    for (int i = 0; i < FaultInjection.LatencyTimeBufferInSec; i++)
                    {
                        if (amqpConnectionStatuses[0].ConnectionStatusChangeCount > countBeforeFaultInjection)
                        {
                            isFaulted = true;
                            break;
                        }

                        await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
                    }

                    Assert.IsTrue(isFaulted, $"The device {testDevices[0].Id} did not get faulted with fault type: {faultType}");
                    s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirmed fault injection has been actived.");

                    // Check all devices are back online
                    s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirming all devices back online.");
                    bool notRecovered = true;
                    int  j            = 0;
                    for (int i = 0; notRecovered && i < durationInSec + FaultInjection.LatencyTimeBufferInSec; i++)
                    {
                        notRecovered = false;
                        for (j = 0; j < devicesCount; j++)
                        {
                            if (amqpConnectionStatuses[j].LastConnectionStatus != ConnectionStatus.Connected)
                            {
                                await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);

                                notRecovered = true;
                                break;
                            }
                        }
                    }

                    if (notRecovered)
                    {
                        Assert.Fail($"{testDevices[j].Id} did not reconnect.");
                    }
                    s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirmed all devices back online.");

                    // Perform the test operation for all devices
                    for (int i = 0; i < devicesCount; i++)
                    {
                        s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation for device {i}.");
                        operations.Add(testOperation(deviceClients[i], testDevices[i]));
                    }
                    await Task.WhenAll(operations).ConfigureAwait(false);

                    operations.Clear();
                }
                else
                {
                    s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation while fault injection is being activated.");
                    // Perform the test operation for the faulted device multi times.
                    for (int i = 0; i < FaultInjection.LatencyTimeBufferInSec; i++)
                    {
                        s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation for device 0 - Run {i}.");
                        await testOperation(deviceClients[0], testDevices[0]).ConfigureAwait(false);

                        await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
                    }
                }

                // Close the device client instances
                for (int i = 0; i < devicesCount; i++)
                {
                    operations.Add(deviceClients[i].CloseAsync());
                }
                await Task.WhenAll(operations).ConfigureAwait(false);

                operations.Clear();

                // Verify the connection status change checks.
                // For all of the devices - last connection status should be "Disabled", with reason "Client_close"
                for (int i = 0; i < devicesCount; i++)
                {
                    // For the faulted device [device 0] - verify the connection status change count.
                    if (i == 0)
                    {
                        if (FaultInjection.FaultShouldDisconnect(faultType))
                        {
                            // 4 is the minimum notification count: connect, fault, reconnect, disable.
                            Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangeCount >= 4, $"The expected connection status change count for {testDevices[i].Id} should equals or greater than 4 but was {amqpConnectionStatuses[i].ConnectionStatusChangeCount}");
                        }
                        else
                        {
                            // 2 is the minimum notification count: connect, disable.
                            Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangeCount == 2, $"The expected connection status change count for {testDevices[i].Id}  should be 2 but was {amqpConnectionStatuses[i].ConnectionStatusChangeCount}");
                        }
                    }
                    Assert.AreEqual(ConnectionStatus.Disabled, amqpConnectionStatuses[i].LastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Disabled} but was {amqpConnectionStatuses[i].LastConnectionStatus}");
                    Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, amqpConnectionStatuses[i].LastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Client_Close} but was {amqpConnectionStatuses[i].LastConnectionStatusChangeReason}");
                }
            }
            finally
            {
                // Close the service-side components and dispose the device client instances.
                await cleanupOperation(deviceClients).ConfigureAwait(false);

                watch.Stop();

                int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds;
                if (timeToFinishFaultInjection > 0)
                {
                    s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed.");
                    await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false);
                }
            }
        }
Exemple #2
0
        // Error injection template method.
        public static async Task TestErrorInjectionAsync(
            string devicePrefix,
            TestDeviceType type,
            Client.TransportType transport,
            string faultType,
            string reason,
            int delayInSec,
            int durationInSec,
            Func <DeviceClient, TestDevice, Task> initOperation,
            Func <DeviceClient, TestDevice, Task> testOperation,
            Func <Task> cleanupOperation)
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(devicePrefix, type).ConfigureAwait(false);

            DeviceClient deviceClient = testDevice.CreateDeviceClient(transport);

            ConnectionStatus?            lastConnectionStatus             = null;
            ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
            int connectionStatusChangeCount = 0;

            deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
            {
                connectionStatusChangeCount++;
                lastConnectionStatus             = status;
                lastConnectionStatusChangeReason = statusChangeReason;
                s_log.WriteLine($"{nameof(FaultInjection)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={connectionStatusChangeCount}");
            });

            var watch = new Stopwatch();

            try
            {
                await deviceClient.OpenAsync().ConfigureAwait(false);

                if (transport != Client.TransportType.Http1)
                {
                    Assert.IsTrue(connectionStatusChangeCount >= 1, $"The expected connection status change should be equal or greater than 1 but was {connectionStatusChangeCount}"); // Normally one connection but in some cases, due to network issues we might have already retried several times to connect.
                    Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Connected} but was {lastConnectionStatus}");
                    Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Connection_Ok} but was {lastConnectionStatusChangeReason}");
                }

                await initOperation(deviceClient, testDevice).ConfigureAwait(false);

                s_log.WriteLine($">>> {nameof(FaultInjection)} Testing baseline");
                await testOperation(deviceClient, testDevice).ConfigureAwait(false);

                int countBeforeFaultInjection = connectionStatusChangeCount;
                watch.Start();
                s_log.WriteLine($">>> {nameof(FaultInjection)} Testing fault handling");
                await ActivateFaultInjection(transport, faultType, reason, delayInSec, durationInSec, deviceClient).ConfigureAwait(false);

                s_log.WriteLine($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {delayInSec} seconds.");
                await Task.Delay(TimeSpan.FromSeconds(delayInSec)).ConfigureAwait(false);

                // For disconnect type faults, the device should disconnect and recover.
                if (FaultShouldDisconnect(faultType))
                {
                    s_log.WriteLine($"{nameof(FaultInjection)}: Confirming fault injection has been actived.");
                    // Check that service issued the fault to the faulting device
                    bool isFaulted = false;
                    for (int i = 0; i < LatencyTimeBufferInSec; i++)
                    {
                        if (connectionStatusChangeCount > countBeforeFaultInjection)
                        {
                            isFaulted = true;
                            break;
                        }

                        await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
                    }

                    Assert.IsTrue(isFaulted, $"The device {testDevice.Id} did not get faulted with fault type: {faultType}");
                    s_log.WriteLine($"{nameof(FaultInjection)}: Confirmed fault injection has been actived.");

                    // Check the device is back online
                    s_log.WriteLine($"{nameof(FaultInjection)}: Confirming device back online.");
                    for (int i = 0; lastConnectionStatus != ConnectionStatus.Connected && i < durationInSec + LatencyTimeBufferInSec; i++)
                    {
                        await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
                    }

                    Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus, $"{testDevice.Id} did not reconnect.");
                    s_log.WriteLine($"{nameof(FaultInjection)}: Confirmed device back online.");

                    // Perform the test operation.
                    s_log.WriteLine($">>> {nameof(FaultInjection)}: Performing test operation for device {testDevice.Id}.");
                    await testOperation(deviceClient, testDevice).ConfigureAwait(false);
                }
                else
                {
                    s_log.WriteLine($"{nameof(FaultInjection)}: Performing test operation while fault injection is being activated.");
                    // Perform the test operation for the faulted device multi times.
                    for (int i = 0; i < LatencyTimeBufferInSec; i++)
                    {
                        s_log.WriteLine($">>> {nameof(FaultInjection)}: Performing test operation for device - Run {i}.");
                        await testOperation(deviceClient, testDevice).ConfigureAwait(false);

                        await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
                    }
                }

                await deviceClient.CloseAsync().ConfigureAwait(false);

                if (transport != Client.TransportType.Http1)
                {
                    if (FaultInjection.FaultShouldDisconnect(faultType))
                    {
                        // 4 is the minimum notification count: connect, fault, reconnect, disable.
                        // There are cases where the retry must be timed out (i.e. very likely for MQTT where otherwise
                        // we would attempt to send the fault injection forever.)
                        Assert.IsTrue(connectionStatusChangeCount >= 4, $"The expected connection status change count for {testDevice.Id} should be equal or greater than 4 but was {connectionStatusChangeCount}");
                    }
                    else
                    {
                        // 2 is the minimum notification count: connect, disable.
                        // We will monitor the test environment real network stability and switch to >=2 if necessary to
                        // account for real network issues.
                        Assert.IsTrue(connectionStatusChangeCount == 2, $"The expected connection status change count for {testDevice.Id}  should be 2 but was {connectionStatusChangeCount}");
                    }
                    Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Disabled} but was {lastConnectionStatus}");
                    Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Client_Close} but was {lastConnectionStatusChangeReason}");
                }
            }
            finally
            {
                await cleanupOperation().ConfigureAwait(false);

                s_log.WriteLine($"{nameof(FaultInjection)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
                deviceClient.Dispose();

                watch.Stop();

                int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds;
                if (timeToFinishFaultInjection > 0)
                {
                    s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed.");
                    await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false);
                }
            }
        }
        private async Task Twin_DeviceDesiredPropertyUpdateRecovery(Client.TransportType transport, string faultType, string reason, int delayInSec)
        {
            var tcs       = new TaskCompletionSource <bool>();
            var propName  = Guid.NewGuid().ToString();
            var propValue = Guid.NewGuid().ToString();

            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);

            RegistryManager registryManager = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);

            var deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport);

            ConnectionStatus?            lastConnectionStatus             = null;
            ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
            int setConnectionStatusChangesHandlerCount = 0;
            var tcsConnected    = new TaskCompletionSource <bool>();
            var tcsDisconnected = new TaskCompletionSource <bool>();

            deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
            {
                _log.WriteLine("Connection Changed to {0} because {1}", status, statusChangeReason);

                if (status == ConnectionStatus.Disconnected_Retrying)
                {
                    tcsDisconnected.TrySetResult(true);
                    Assert.AreEqual(ConnectionStatusChangeReason.No_Network, statusChangeReason);
                }
                else if (status == ConnectionStatus.Connected)
                {
                    tcsConnected.TrySetResult(true);
                }

                lastConnectionStatus             = status;
                lastConnectionStatusChangeReason = statusChangeReason;
                setConnectionStatusChangesHandlerCount++;
            });

            await deviceClient.SetDesiredPropertyUpdateCallbackAsync((patch, context) =>
            {
                return(Task.Run(() =>
                {
                    try
                    {
                        Assert.AreEqual(patch[propName].ToString(), propValue);
                    }
                    catch (Exception e)
                    {
                        tcs.SetException(e);
                    }
                    finally
                    {
                        tcs.SetResult(true);
                    }
                }));
            }, null).ConfigureAwait(false);

            // assert on successful connection
            await Task.WhenAny(
                Task.Run(async() =>
            {
                await Task.Delay(1000).ConfigureAwait(false);
            }), tcsConnected.Task).ConfigureAwait(false);

            Assert.IsTrue(tcsConnected.Task.IsCompleted, "Initial connection failed");
            if (transport != Client.TransportType.Http1)
            {
                Assert.AreEqual(1, setConnectionStatusChangesHandlerCount);
                Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus);
                Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason);
            }

            var twinPatch = new Twin();

            twinPatch.Properties.Desired[propName] = propValue;
            await registryManager.UpdateTwinAsync(testDevice.Id, twinPatch, "*").ConfigureAwait(false);

            await tcs.Task.ConfigureAwait(false);

            // send error command
            await deviceClient.SendEventAsync(FaultInjection.ComposeErrorInjectionProperties(faultType, reason, delayInSec)).ConfigureAwait(false);

            // reset ConnectionStatusChangesHandler data
            setConnectionStatusChangesHandlerCount = 0;
            tcsConnected    = new TaskCompletionSource <bool>();
            tcsDisconnected = new TaskCompletionSource <bool>();

            // wait for disconnection
            await Task.WhenAny(
                Task.Run(async() =>
            {
                await Task.Delay(TimeSpan.FromSeconds(10)).ConfigureAwait(false);
            }), tcsDisconnected.Task).ConfigureAwait(false);

            Assert.IsTrue(tcsDisconnected.Task.IsCompleted, "Error injection did not interrupt the device");

            await Task.WhenAny(
                Task.Run(async() =>
            {
                await Task.Delay(TimeSpan.FromSeconds(MaximumRecoveryTimeSeconds)).ConfigureAwait(false);
                return(Task.FromResult(true));
            }), tcsConnected.Task).ConfigureAwait(false);

            Assert.IsTrue(tcsConnected.Task.IsCompleted, "Recovery connection failed");

            tcs       = new TaskCompletionSource <bool>();
            twinPatch = new Twin();
            twinPatch.Properties.Desired[propName] = propValue;
            await registryManager.UpdateTwinAsync(testDevice.Id, twinPatch, "*").ConfigureAwait(false);

            await tcs.Task.ConfigureAwait(false);

            await deviceClient.CloseAsync().ConfigureAwait(false);

            await registryManager.CloseAsync().ConfigureAwait(false);
        }
        private async Task SendMethodAndRespondRecovery(Client.TransportType transport, string faultType, string reason, int delayInSec)
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);

            var          assertResult = new TaskCompletionSource <Tuple <bool, bool> >();
            DeviceClient deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport);

            ConnectionStatus?            lastConnectionStatus             = null;
            ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
            int setConnectionStatusChangesHandlerCount = 0;
            var tcsConnected    = new TaskCompletionSource <bool>();
            var tcsDisconnected = new TaskCompletionSource <bool>();

            deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
            {
                _log.WriteLine("Connection Changed to {0} because {1}", status, statusChangeReason);

                if (status == ConnectionStatus.Disconnected_Retrying)
                {
                    tcsDisconnected.TrySetResult(true);
                    Assert.AreEqual(ConnectionStatusChangeReason.No_Network, statusChangeReason);
                }
                else if (status == ConnectionStatus.Connected)
                {
                    tcsConnected.TrySetResult(true);
                }

                lastConnectionStatus             = status;
                lastConnectionStatusChangeReason = statusChangeReason;
                setConnectionStatusChangesHandlerCount++;
            });

            await deviceClient.SetMethodHandlerAsync(MethodName,
                                                     (request, context) =>
            {
                assertResult.TrySetResult(new Tuple <bool, bool>(request.Name.Equals(MethodName),
                                                                 request.DataAsJson.Equals(ServiceRequestJson)));
                return(Task.FromResult(new MethodResponse(Encoding.UTF8.GetBytes(DeviceResponseJson), 200)));
            },
                                                     null).ConfigureAwait(false);

            // assert on successful connection
            await Task.WhenAny(Task.Delay(FaultInjection.ShortRetryInMilliSec), tcsConnected.Task).ConfigureAwait(false);

            Assert.IsTrue(tcsConnected.Task.IsCompleted, "Initial connection failed");

            if (transport != Client.TransportType.Http1)
            {
                Assert.AreEqual(1, setConnectionStatusChangesHandlerCount);
                Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus);
                Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason);
            }

            // check on normal operation
            await
            ServiceSendMethodAndVerifyResponse(testDevice.Id, MethodName, DeviceResponseJson,
                                               ServiceRequestJson, assertResult).ConfigureAwait(false);

            // reset ConnectionStatusChangesHandler data
            setConnectionStatusChangesHandlerCount = 0;
            tcsConnected    = new TaskCompletionSource <bool>();
            tcsDisconnected = new TaskCompletionSource <bool>();

            // send error command
            await deviceClient.SendEventAsync(FaultInjection.ComposeErrorInjectionProperties(faultType, reason, delayInSec)).ConfigureAwait(false);

            // wait for disconnection
            await Task.WhenAny(Task.Delay(FaultInjection.WaitForDisconnectMilliseconds), tcsDisconnected.Task).ConfigureAwait(false);

            Assert.IsTrue(tcsDisconnected.Task.IsCompleted, "Error injection did not interrupt the device");

            await Task.WhenAny(Task.Delay(FaultInjection.RecoveryTimeMilliseconds), tcsConnected.Task).ConfigureAwait(false);

            Assert.IsTrue(tcsConnected.Task.IsCompleted, "Recovery connection failed");

            assertResult = new TaskCompletionSource <Tuple <bool, bool> >();
            await
            ServiceSendMethodAndVerifyResponse(testDevice.Id, MethodName, DeviceResponseJson,
                                               ServiceRequestJson, assertResult).ConfigureAwait(false);

            setConnectionStatusChangesHandlerCount = 0;

            //remove and CloseAsync
            await deviceClient.SetMethodHandlerAsync(MethodName, null, null).ConfigureAwait(false);

            if (transport != Client.TransportType.Http1)
            {
                Assert.AreEqual(1, setConnectionStatusChangesHandlerCount);
                Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus);
                Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason);
            }
        }
Exemple #5
0
        internal async Task SendMessageRecovery(Client.TransportType transport,
                                                string faultType, string reason, int delayInSec, int durationInSec = 0, int retryDurationInMilliSec = 240000)
        {
            TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);

            var              deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport);
            EventHubClient   eventHubClient;
            EventHubReceiver eventHubReceiver = CreateEventHubReceiver(testDevice.Id, out eventHubClient);

            try
            {
                deviceClient.OperationTimeoutInMilliseconds = (uint)retryDurationInMilliSec;

                ConnectionStatus?            lastConnectionStatus             = null;
                ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
                int setConnectionStatusChangesHandlerCount = 0;

                deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
                {
                    lastConnectionStatus             = status;
                    lastConnectionStatusChangeReason = statusChangeReason;
                    setConnectionStatusChangesHandlerCount++;
                });

                await deviceClient.OpenAsync().ConfigureAwait(false);

                if (transport != Client.TransportType.Http1)
                {
                    Assert.AreEqual(1, setConnectionStatusChangesHandlerCount);
                    Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus);
                    Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason);
                }

                string         payload, p1Value;
                Client.Message testMessage = ComposeD2CTestMessage(out payload, out p1Value);
                await deviceClient.SendEventAsync(testMessage).ConfigureAwait(false);


                bool      isReceived = false;
                Stopwatch sw         = new Stopwatch();
                sw.Start();
                while (!isReceived && sw.Elapsed.Minutes < 1)
                {
                    var events = await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5)).ConfigureAwait(false);

                    isReceived = VerifyTestMessage(events, testDevice.Id, payload, p1Value);
                }
                sw.Stop();

                // send error command and clear eventHubReceiver of the fault injection message
                await deviceClient.SendEventAsync(FaultInjection.ComposeErrorInjectionProperties(faultType, reason,
                                                                                                 delayInSec,
                                                                                                 durationInSec)).ConfigureAwait(false);

                await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5)).ConfigureAwait(false);

                Thread.Sleep(1000);

                testMessage = ComposeD2CTestMessage(out payload, out p1Value);
                await deviceClient.SendEventAsync(testMessage).ConfigureAwait(false);

                sw.Reset();
                sw.Start();
                while (!isReceived && sw.Elapsed.Minutes < 1)
                {
                    var events = await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5)).ConfigureAwait(false);

                    isReceived = VerifyTestMessage(events, testDevice.Id, payload, p1Value);
                }
                sw.Stop();

                await deviceClient.CloseAsync().ConfigureAwait(false);

                if (transport != Client.TransportType.Http1)
                {
                    Assert.AreEqual(2, setConnectionStatusChangesHandlerCount);
                    Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus);
                    Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason);
                }
            }
            finally
            {
                await deviceClient.CloseAsync().ConfigureAwait(false);

                await eventHubReceiver.CloseAsync().ConfigureAwait(false);

                _log.WriteLine($"Waiting for fault injection interval to finish {FaultInjection.DefaultDelayInSec}s.");
                await Task.Delay(FaultInjection.DefaultDurationInSec * 1000).ConfigureAwait(false);
            }
        }