private async Task ReceiveMessageInOperationTimeout(TestDeviceType type, Client.TransportType transport) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix, type).ConfigureAwait(false); using (DeviceClient deviceClient = testDevice.CreateDeviceClient(transport)) { await deviceClient.OpenAsync().ConfigureAwait(false); if (transport == Client.TransportType.Mqtt_Tcp_Only || transport == Client.TransportType.Mqtt_WebSocket_Only) { // Dummy ReceiveAsync to ensure mqtt subscription registration before SendAsync() is called on service client. await deviceClient.ReceiveAsync(TIMESPAN_FIVE_SECONDS).ConfigureAwait(false); } try { deviceClient.OperationTimeoutInMilliseconds = Convert.ToUInt32(TIMESPAN_ONE_MINUTE.TotalMilliseconds); if (transport == Client.TransportType.Amqp || transport == Client.TransportType.Amqp_Tcp_Only || transport == Client.TransportType.Amqp_WebSocket_Only) { // For AMQP because of static 1 min interval check the cancellation token, in worst case it will block upto extra 1 min to return await ReceiveMessageWithoutTimeoutCheck(deviceClient, TIMESPAN_ONE_MINUTE).ConfigureAwait(false); } else { await ReceiveMessageWithoutTimeoutCheck(deviceClient, TIMESPAN_FIVE_SECONDS).ConfigureAwait(false); } } finally { deviceClient.OperationTimeoutInMilliseconds = DeviceClient.DefaultOperationTimeoutInMilliseconds; await deviceClient.CloseAsync().ConfigureAwait(false); } } }
private async Task ReceiveSingleMessage(TestDeviceType type, Client.TransportType transport) { ServiceClient serviceClient = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString); TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix, type).ConfigureAwait(false); DeviceClient deviceClient = testDevice.CreateDeviceClient(transport); try { await deviceClient.OpenAsync().ConfigureAwait(false); if (transport == Client.TransportType.Mqtt_Tcp_Only || transport == Client.TransportType.Mqtt_WebSocket_Only) { _log.WriteLine("Dummy ReceiveAsync to ensure mqtt subscription registration before SendAsync() is called on service client."); await deviceClient.ReceiveAsync(TimeSpan.FromSeconds(2)).ConfigureAwait(false); } string payload, messageId, p1Value; await serviceClient.OpenAsync().ConfigureAwait(false); Message msg = ComposeC2DTestMessage(out payload, out messageId, out p1Value); await serviceClient.SendAsync(testDevice.Id, msg).ConfigureAwait(false); await VerifyReceivedC2DMessage(transport, deviceClient, payload, p1Value).ConfigureAwait(false); } finally { await deviceClient.CloseAsync().ConfigureAwait(false); await serviceClient.CloseAsync().ConfigureAwait(false); } }
private async Task ReceiveSingleMessage(TestDeviceType type, Client.TransportType transport) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix, type).ConfigureAwait(false); using (DeviceClient deviceClient = testDevice.CreateDeviceClient(transport)) using (ServiceClient serviceClient = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString)) { await deviceClient.OpenAsync().ConfigureAwait(false); if (transport == Client.TransportType.Mqtt_Tcp_Only || transport == Client.TransportType.Mqtt_WebSocket_Only) { // Dummy ReceiveAsync to ensure mqtt subscription registration before SendAsync() is called on service client. await deviceClient.ReceiveAsync(TimeSpan.FromSeconds(1)).ConfigureAwait(false); } await serviceClient.OpenAsync().ConfigureAwait(false); (Message msg, string messageId, string payload, string p1Value) = ComposeC2DTestMessage(); await serviceClient.SendAsync(testDevice.Id, msg).ConfigureAwait(false); await VerifyReceivedC2DMessageAsync(transport, deviceClient, testDevice.Id, payload, p1Value).ConfigureAwait(false); await deviceClient.CloseAsync().ConfigureAwait(false); await serviceClient.CloseAsync().ConfigureAwait(false); } }
private async Task SendSingleMessage(TestDeviceType type, Client.TransportType transport) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix, type).ConfigureAwait(false); DeviceClient deviceClient = testDevice.CreateDeviceClient(transport); await SendSingleMessage(deviceClient, testDevice.Id).ConfigureAwait(false); }
public async Task Duplicated_NoPingpong() { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix, TestDeviceType.Sasl).ConfigureAwait(false); _log.WriteLine($"{nameof(Duplicated_NoPingpong)}: 2 device client instances with the same deviceId={testDevice.Id}."); using (DeviceClient deviceClient1 = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only), deviceClient2 = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only)) { _log.WriteLine($"{nameof(Duplicated_NoPingpong)}: set device client instance 1 to no retry."); deviceClient1.SetRetryPolicy(new NoRetry()); ConnectionStatus?lastConnectionStatus = null; Dictionary <ConnectionStatus, int> connectionStatusChanges = new Dictionary <ConnectionStatus, int>(); deviceClient1.SetConnectionStatusChangesHandler((status, reason) => { connectionStatusChanges.TryGetValue(status, out int count); count++; connectionStatusChanges[status] = count; lastConnectionStatus = status; }); _log.WriteLine($"{nameof(FaultInjection_NoRecovery)}: device client instance 1 calling OpenAsync..."); await deviceClient1.OpenAsync().ConfigureAwait(false); await deviceClient1.SetMethodHandlerAsync( "dummy_method", (methodRequest, userContext) => Task.FromResult(new MethodResponse(200)), deviceClient1 ).ConfigureAwait(false); _log.WriteLine($"{nameof(FaultInjection_NoRecovery)}: device client instance 2 calling OpenAsync..."); await deviceClient2.OpenAsync().ConfigureAwait(false); await deviceClient2.SetMethodHandlerAsync( "dummy_method", (methodRequest, userContext) => Task.FromResult(new MethodResponse(200)), deviceClient2 ).ConfigureAwait(false); _log.WriteLine($"{nameof(Duplicated_NoPingpong)}: waiting device client instance 1 to be kicked off..."); for (int i = 0; i < FaultInjection.LatencyTimeBufferInSec; i++) { if (connectionStatusChanges.ContainsKey(ConnectionStatus.Disconnected)) { break; } await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); } Assert.AreEqual(ConnectionStatus.Disconnected, lastConnectionStatus, $"Excepeted device to be {ConnectionStatus.Disconnected} but was {lastConnectionStatus}."); Assert.IsFalse(connectionStatusChanges.ContainsKey(ConnectionStatus.Disconnected_Retrying), $"Shouldn't get {ConnectionStatus.Disconnected_Retrying} status change."); int connected = connectionStatusChanges[ConnectionStatus.Connected]; Assert.AreEqual(1, connected, $"Should get {ConnectionStatus.Connected} once but was {connected}."); int disconnected = connectionStatusChanges[ConnectionStatus.Disconnected]; Assert.AreEqual(1, disconnected, $"Should get {ConnectionStatus.Disconnected} once but was {disconnected}."); } }
public async Task FaultInjection_NoRecovery() { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(Logger, DevicePrefix, TestDeviceType.Sasl).ConfigureAwait(false); using DeviceClient deviceClient = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only); Logger.Trace($"{nameof(FaultInjection_NoRecovery)}: deviceId={testDevice.Id}"); deviceClient.SetRetryPolicy(new NoRetry()); ConnectionStatus?lastConnectionStatus = null; Dictionary <ConnectionStatus, int> connectionStatusChanges = new Dictionary <ConnectionStatus, int>(); deviceClient.SetConnectionStatusChangesHandler((status, reason) => { connectionStatusChanges.TryGetValue(status, out int count); count++; connectionStatusChanges[status] = count; lastConnectionStatus = status; }); Logger.Trace($"{nameof(FaultInjection_NoRecovery)}: calling OpenAsync..."); await deviceClient.OpenAsync().ConfigureAwait(false); Logger.Trace($"{nameof(FaultInjection_NoRecovery)}: injecting fault {FaultInjection.FaultType_Tcp}..."); await FaultInjection .ActivateFaultInjectionAsync( Client.TransportType.Amqp_Tcp_Only, FaultInjection.FaultType_Tcp, FaultInjection.FaultCloseReason_Boom, FaultInjection.DefaultDelayInSec, FaultInjection.DefaultDurationInSec, deviceClient, Logger) .ConfigureAwait(false); await Task.Delay(FaultInjection.DefaultDelayInSec).ConfigureAwait(false); Logger.Trace($"{nameof(FaultInjection_NoRecovery)}: waiting fault injection occurs..."); for (int i = 0; i < FaultInjection.LatencyTimeBufferInSec; i++) { if (connectionStatusChanges.ContainsKey(ConnectionStatus.Disconnected)) { break; } await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); } Assert.AreEqual(ConnectionStatus.Disconnected, lastConnectionStatus, $"Excepeted device to be {ConnectionStatus.Disconnected} but was {lastConnectionStatus}."); Assert.IsFalse(connectionStatusChanges.ContainsKey(ConnectionStatus.Disconnected_Retrying), $"Shouldn't get {ConnectionStatus.Disconnected_Retrying} status change."); int connected = connectionStatusChanges[ConnectionStatus.Connected]; Assert.AreEqual(1, connected, $"Should get {ConnectionStatus.Connected} once but was {connected}."); int disconnected = connectionStatusChanges[ConnectionStatus.Disconnected]; Assert.AreEqual(1, disconnected, $"Should get {ConnectionStatus.Disconnected} once but was {disconnected}."); }
private async Task SendMessageTest(ITransportSettings transportSetting) { using TestDevice testDevice = await TestDevice.GetTestDeviceAsync(Logger, s_devicePrefix, TestDeviceType.X509).ConfigureAwait(false); using DeviceClient deviceClient = testDevice.CreateDeviceClient(new[] { transportSetting }); await deviceClient.OpenAsync().ConfigureAwait(false); await MessageSendE2ETests.SendSingleMessageAsync(deviceClient, testDevice.Id, Logger).ConfigureAwait(false); await deviceClient.CloseAsync().ConfigureAwait(false); }
public async Task DeviceClient_CreateFromConnectionString_TokenIsRefreshed_Mqtt() { var sasTokenTimeToLive = TimeSpan.FromSeconds(10); int sasTokenRenewalBuffer = 50; using var deviceDisconnected = new SemaphoreSlim(0); int operationTimeoutInMilliseconds = (int)sasTokenTimeToLive.TotalMilliseconds * 2; // Service allows a buffer time of upto 10mins before dropping connections that are authenticated with an expired sas tokens. using var tokenRefreshCts = new CancellationTokenSource((int)(sasTokenTimeToLive.TotalMilliseconds * 2 + TimeSpan.FromMinutes(10).TotalMilliseconds)); TestDevice testDevice = await TestDevice.GetTestDeviceAsync(Logger, DevicePrefix).ConfigureAwait(false); var options = new ClientOptions { SasTokenTimeToLive = sasTokenTimeToLive, SasTokenRenewalBuffer = sasTokenRenewalBuffer, }; using DeviceClient deviceClient = testDevice.CreateDeviceClient(Client.TransportType.Mqtt, options); Logger.Trace($"Created {nameof(DeviceClient)} instance for {testDevice.Id}."); deviceClient.SetConnectionStatusChangesHandler((ConnectionStatus status, ConnectionStatusChangeReason reason) => { Logger.Trace($"{nameof(ConnectionStatusChangesHandler)}: {status}; {reason}"); if (status == ConnectionStatus.Disconnected_Retrying || status == ConnectionStatus.Disconnected) { deviceDisconnected.Release(); } }); deviceClient.OperationTimeoutInMilliseconds = (uint)operationTimeoutInMilliseconds; var message = new Client.Message(Encoding.UTF8.GetBytes("Hello")); Logger.Trace($"[{testDevice.Id}]: SendEventAsync (1)"); await deviceClient.SendEventAsync(message).ConfigureAwait(false); // Wait for the Token to expire. Logger.Trace($"[{testDevice.Id}]: Waiting for device disconnect."); await deviceDisconnected.WaitAsync(tokenRefreshCts.Token).ConfigureAwait(false); try { Logger.Trace($"[{testDevice.Id}]: SendEventAsync (2)"); await deviceClient.SendEventAsync(message).ConfigureAwait(false); } catch (OperationCanceledException ex) { Assert.Fail($"{testDevice.Id} did not refresh token after expected ttl of {sasTokenTimeToLive}: {ex}"); throw; } }
private async Task SendBatchMessages(TestDeviceType type, Client.TransportType transport) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix, type).ConfigureAwait(false); using (DeviceClient deviceClient = testDevice.CreateDeviceClient(transport)) { await deviceClient.OpenAsync().ConfigureAwait(false); await SendSendBatchMessagesAndVerifyAsync(deviceClient, testDevice.Id).ConfigureAwait(false); await deviceClient.CloseAsync().ConfigureAwait(false); } }
private async Task SendSingleMessage(TestDeviceType type, ITransportSettings[] transportSettings) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix, type).ConfigureAwait(false); using (DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings)) { await deviceClient.OpenAsync().ConfigureAwait(false); await SendSingleMessageAndVerifyAsync(deviceClient, testDevice.Id).ConfigureAwait(false); await deviceClient.CloseAsync().ConfigureAwait(false); } }
private async Task SendSingleMessage(TestDeviceType type, Client.TransportType transport, int messageSize = 0) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix, type).ConfigureAwait(false); using DeviceClient deviceClient = testDevice.CreateDeviceClient(transport); await deviceClient.OpenAsync().ConfigureAwait(false); await SendSingleMessageAndVerifyAsync(deviceClient, testDevice.Id, messageSize).ConfigureAwait(false); await deviceClient.CloseAsync().ConfigureAwait(false); }
private async Task TestSecurityMessageAsync(Client.TransportType transport) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(_devicePrefix).ConfigureAwait(false); using DeviceClient deviceClient = testDevice.CreateDeviceClient(transport); try { await SendSingleSecurityMessageAsync(deviceClient, testDevice.Id, _logAnalyticsClient).ConfigureAwait(false); } finally { await deviceClient.CloseAsync().ConfigureAwait(false); } }
private static async Task CompleteMessageMixOrder(TestDeviceType type, Client.TransportType transport) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix, type).ConfigureAwait(false); using (DeviceClient deviceClient = testDevice.CreateDeviceClient(transport)) using (ServiceClient serviceClient = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString)) { await deviceClient.OpenAsync().ConfigureAwait(false); if (transport == Client.TransportType.Mqtt_Tcp_Only || transport == Client.TransportType.Mqtt_WebSocket_Only) { // Dummy ReceiveAsync to ensure mqtt subscription registration before SendAsync() is called on service client. await deviceClient.ReceiveAsync(TIMESPAN_FIVE_SECONDS).ConfigureAwait(false); } await serviceClient.OpenAsync().ConfigureAwait(false); var messages = new List <Client.Message>(); for (int i = 0; i < MESSAGE_COUNT; i++) { (Message msg, string messageId, string payload, string p1Value) = MessageReceiveE2ETests.ComposeC2DTestMessage(); await serviceClient.SendAsync(testDevice.Id, msg).ConfigureAwait(false); Client.Message message = await deviceClient.ReceiveAsync(TIMESPAN_ONE_MINUTE).ConfigureAwait(false); if (message == null) { Assert.Fail("No message received."); } messages.Add(message); } for (int i = 0; i < MESSAGE_COUNT; i++) { Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); await deviceClient.CompleteAsync(messages[MESSAGE_COUNT - 1 - i]).ConfigureAwait(false); stopwatch.Stop(); Assert.IsFalse(stopwatch.ElapsedMilliseconds > deviceClient.OperationTimeoutInMilliseconds, $"CompleteAsync is over {deviceClient.OperationTimeoutInMilliseconds}"); } await deviceClient.CloseAsync().ConfigureAwait(false); await serviceClient.CloseAsync().ConfigureAwait(false); } }
private async Task ReceiveMessageInOperationTimeoutAsync(TestDeviceType type, Client.TransportType transport) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(s_devicePrefix, type).ConfigureAwait(false); using DeviceClient deviceClient = testDevice.CreateDeviceClient(transport); s_log.WriteLine($"{nameof(ReceiveMessageInOperationTimeoutAsync)} - calling OpenAsync() for transport={transport}"); await deviceClient.OpenAsync().ConfigureAwait(false); if (transport == Client.TransportType.Mqtt_Tcp_Only || transport == Client.TransportType.Mqtt_WebSocket_Only) { // Dummy ReceiveAsync to ensure mqtt subscription registration before SendAsync() is called on service client. await deviceClient.ReceiveAsync(s_fiveSeconds).ConfigureAwait(false); } try { deviceClient.OperationTimeoutInMilliseconds = Convert.ToUInt32(s_oneMinute.TotalMilliseconds); s_log.WriteLine($"{nameof(ReceiveMessageInOperationTimeoutAsync)} - setting device client default operation timeout={deviceClient.OperationTimeoutInMilliseconds} ms"); if (transport == Client.TransportType.Amqp || transport == Client.TransportType.Amqp_Tcp_Only || transport == Client.TransportType.Amqp_WebSocket_Only) { // TODO: this extra minute on the timeout is undesirable by customers, and tests seems to be failing on a slight timing issue. // For now, add an additional 5 second buffer to prevent tests from failing, and meanwhile address issue 1203. // For AMQP because of static 1 min interval check the cancellation token, in worst case it will block upto extra 1 min to return await ReceiveMessageWithoutTimeoutCheckAsync(deviceClient, s_oneMinute + TimeSpan.FromSeconds(5)).ConfigureAwait(false); } else { await ReceiveMessageWithoutTimeoutCheckAsync(deviceClient, s_fiveSeconds).ConfigureAwait(false); } } finally { s_log.WriteLine($"{nameof(ReceiveMessageInOperationTimeoutAsync)} - calling CloseAsync() for transport={transport}"); deviceClient.OperationTimeoutInMilliseconds = DeviceClient.DefaultOperationTimeoutInMilliseconds; await deviceClient.CloseAsync().ConfigureAwait(false); } }
public async Task Message_ClientThrowsForMqttTopicNameTooLong() { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false); using DeviceClient deviceClient = testDevice.CreateDeviceClient(Client.TransportType.Mqtt); await deviceClient.OpenAsync().ConfigureAwait(false); using var msg = new Client.Message(Encoding.UTF8.GetBytes("testMessage")); //Mqtt topic name consists of, among other things, system properties and user properties // setting lots of very long user properties should cause a MessageTooLargeException explaining // that the topic name is too long to publish over mqtt for (int i = 0; i < 100; i++) { msg.Properties.Add(Guid.NewGuid().ToString(), new string('1', 1024)); } await deviceClient.SendEventAsync(msg).ConfigureAwait(false); }
private async Task ReceiveMessageWithTimeoutAsync(TestDeviceType type, Client.TransportType transport, TimeSpan timeout) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(s_devicePrefix, type).ConfigureAwait(false); using DeviceClient deviceClient = testDevice.CreateDeviceClient(transport); await deviceClient.OpenAsync().ConfigureAwait(false); if (transport == Client.TransportType.Mqtt_Tcp_Only || transport == Client.TransportType.Mqtt_WebSocket_Only) { // Dummy ReceiveAsync to ensure mqtt subscription registration before SendAsync() is called on service client. await deviceClient.ReceiveAsync(s_fiveSeconds).ConfigureAwait(false); } await ReceiveMessageWithTimeoutCheckAsync(deviceClient, timeout).ConfigureAwait(false); await deviceClient.CloseAsync().ConfigureAwait(false); }
private async Task TestSecurityMessage(Client.TransportType transport) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(_devicePrefix).ConfigureAwait(false); EventHubTestListener testListener = await EventHubTestListener.CreateListener(testDevice.Id).ConfigureAwait(false); using (DeviceClient deviceClient = testDevice.CreateDeviceClient(transport)) { try { await SendSingleSecurityMessage(deviceClient, testDevice.Id, testListener, _logAnalyticsClient).ConfigureAwait(false); } finally { await deviceClient.CloseAsync().ConfigureAwait(false); await testListener.CloseAsync().ConfigureAwait(false); } } }
// Error injection template method. public static async Task TestErrorInjectionAsync( string devicePrefix, TestDeviceType type, Client.TransportType transport, string faultType, string reason, int delayInSec, int durationInSec, Func <DeviceClient, TestDevice, Task> initOperation, Func <DeviceClient, TestDevice, Task> testOperation, Func <Task> cleanupOperation) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(devicePrefix, type).ConfigureAwait(false); DeviceClient deviceClient = testDevice.CreateDeviceClient(transport); ConnectionStatus? lastConnectionStatus = null; ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null; int setConnectionStatusChangesHandlerCount = 0; deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) => { setConnectionStatusChangesHandlerCount++; lastConnectionStatus = status; lastConnectionStatusChangeReason = statusChangeReason; s_log.WriteLine($"{nameof(FaultInjection)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}"); }); var watch = new Stopwatch(); try { await deviceClient.OpenAsync().ConfigureAwait(false); if (transport != Client.TransportType.Http1) { Assert.IsTrue(setConnectionStatusChangesHandlerCount >= 1); // Normally one connection but in some cases, due to network issues we might have already retried several times to connect. Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus); Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason); } await initOperation(deviceClient, testDevice).ConfigureAwait(false); s_log.WriteLine($">>> {nameof(FaultInjection)} Testing baseline"); await testOperation(deviceClient, testDevice).ConfigureAwait(false); watch.Start(); s_log.WriteLine($">>> {nameof(FaultInjection)} Testing fault handling"); await ActivateFaultInjection(transport, faultType, reason, delayInSec, durationInSec, deviceClient).ConfigureAwait(false); int delay = FaultInjection.WaitForDisconnectMilliseconds - (int)watch.ElapsedMilliseconds; if (delay < 0) { delay = 0; } s_log.WriteLine($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {delay}ms"); await Task.Delay(delay).ConfigureAwait(false); await testOperation(deviceClient, testDevice).ConfigureAwait(false); await deviceClient.CloseAsync().ConfigureAwait(false); if (transport != Client.TransportType.Http1) { if (FaultShouldDisconnect(faultType)) { // 4 is the minimum notification count: connect, fault, reconnect, disable. // There are cases where the retry must be timed out (i.e. very likely for MQTT where otherwise // we would attempt to send the fault injection forever.) Assert.IsTrue(setConnectionStatusChangesHandlerCount >= 4); } else { // 2 is the minimum notification count: connect, disable. // We will monitor the test environment real network stability and switch to >=2 if necessary to // account for real network issues. Assert.IsTrue(setConnectionStatusChangesHandlerCount == 2); } Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus); Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason); } } finally { await cleanupOperation().ConfigureAwait(false); s_log.WriteLine($"{nameof(FaultInjection)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}"); deviceClient.Dispose(); watch.Stop(); int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds; if (timeToFinishFaultInjection > 0) { s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed."); await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false); } } }
public async Task FaultInjection_NoRetry_NoRecovery_OpenAsync() { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(Logger, _devicePrefix, TestDeviceType.Sasl).ConfigureAwait(false); using DeviceClient deviceClient = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only); Logger.Trace($"{nameof(FaultInjection_NoRetry_NoRecovery_OpenAsync)}: deviceId={testDevice.Id}"); deviceClient.SetRetryPolicy(new NoRetry()); ConnectionStatus? lastConnectionStatus = null; ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null; var connectionStatusChanges = new Dictionary <ConnectionStatus, int>(); deviceClient.SetConnectionStatusChangesHandler((status, reason) => { connectionStatusChanges.TryGetValue(status, out int count); count++; connectionStatusChanges[status] = count; lastConnectionStatus = status; lastConnectionStatusChangeReason = reason; }); Logger.Trace($"{nameof(FaultInjection_NoRetry_NoRecovery_OpenAsync)}: calling OpenAsync..."); await deviceClient.OpenAsync().ConfigureAwait(false); Logger.Trace($"{nameof(FaultInjection_NoRetry_NoRecovery_OpenAsync)}: injecting fault {FaultInjection.FaultType_Tcp}..."); await FaultInjection .ActivateFaultInjectionAsync( Client.TransportType.Amqp_Tcp_Only, FaultInjection.FaultType_Tcp, FaultInjection.FaultCloseReason_Boom, FaultInjection.DefaultFaultDelay, FaultInjection.DefaultFaultDuration, deviceClient, Logger) .ConfigureAwait(false); await Task.Delay(FaultInjection.DefaultFaultDelay).ConfigureAwait(false); Logger.Trace($"{nameof(FaultInjection_NoRetry_NoRecovery_OpenAsync)}: waiting fault injection occurs..."); var sw = Stopwatch.StartNew(); while (sw.Elapsed < FaultInjection.LatencyTimeBuffer) { if (connectionStatusChanges.ContainsKey(ConnectionStatus.Disconnected)) { break; } await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); } sw.Reset(); lastConnectionStatus.Should().Be(ConnectionStatus.Disconnected, $"Expected device to be {ConnectionStatus.Disconnected} but was {lastConnectionStatus}."); lastConnectionStatusChangeReason.Should().Be(ConnectionStatusChangeReason.Retry_Expired, $"Expected device to be {ConnectionStatusChangeReason.Retry_Expired} but was {lastConnectionStatusChangeReason}."); connectionStatusChanges.Should().NotContainKey(ConnectionStatus.Disconnected_Retrying, $"Shouldn't get {ConnectionStatus.Disconnected_Retrying} status change."); int connected = connectionStatusChanges[ConnectionStatus.Connected]; connected.Should().Be(1, $"Should get {ConnectionStatus.Connected} once but got it {connected} times."); int disconnected = connectionStatusChanges[ConnectionStatus.Disconnected]; disconnected.Should().Be(1, $"Should get {ConnectionStatus.Disconnected} once but got it {disconnected} times."); }
private async Task SendMessageMuxedOverAmqp( TestDeviceType type, Client.TransportType transport, int poolSize, int devicesCount, ConnectionStringAuthScope authScope = ConnectionStringAuthScope.Device) { var transportSettings = new ITransportSettings[] { new AmqpTransportSettings(transport) { AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings() { MaxPoolSize = unchecked ((uint)poolSize), Pooling = true } } }; ICollection <DeviceClient> deviceClients = new List <DeviceClient>(); Dictionary <DeviceClient, int> deviceClientConnectionStatusChangeCount = new Dictionary <DeviceClient, int>(); try { _log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}: Starting the test execution for {devicesCount} devices"); for (int i = 0; i < devicesCount; i++) { ConnectionStatus? lastConnectionStatus = null; ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null; int setConnectionStatusChangesHandlerCount = 0; TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{DevicePrefix}_{i}_", type).ConfigureAwait(false); DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope); deviceClients.Add(deviceClient); deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) => { setConnectionStatusChangesHandlerCount++; lastConnectionStatus = status; lastConnectionStatusChangeReason = statusChangeReason; _log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}"); deviceClientConnectionStatusChangeCount[deviceClient] = setConnectionStatusChangesHandlerCount; }); _log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}: Preparing to send message for device {i}"); await deviceClient.OpenAsync().ConfigureAwait(false); await MessageSend.SendSingleMessageAndVerifyAsync(deviceClient, testDevice.Id).ConfigureAwait(false); } } finally { // Close and dispose all of the device client instances here foreach (DeviceClient deviceClient in deviceClients) { await deviceClient.CloseAsync().ConfigureAwait(false); // The connection status change count should be 2: connect (open) and disabled (close) Assert.IsTrue(deviceClientConnectionStatusChangeCount[deviceClient] == 2, $"Connection status change count for deviceClient {TestLogging.GetHashCode(deviceClient)} is {deviceClientConnectionStatusChangeCount[deviceClient]}"); _log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}"); deviceClient.Dispose(); } } }
public static async Task TestPoolAmqpAsync( string devicePrefix, Client.TransportType transport, int poolSize, int devicesCount, Func <DeviceClient, TestDevice, Task> initOperation, Func <DeviceClient, TestDevice, Task> testOperation, Func <Task> cleanupOperation, ConnectionStringAuthScope authScope, bool ignoreConnectionStatus) { var transportSettings = new ITransportSettings[] { new AmqpTransportSettings(transport) { AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings() { MaxPoolSize = unchecked ((uint)poolSize), Pooling = true } } }; int totalRuns = 0; int successfulRuns = 0; int currentSuccessRate = 0; bool reRunTest = false; IList <TestDevice> testDevices = new List <TestDevice>(); IList <DeviceClient> deviceClients = new List <DeviceClient>(); IList <AmqpConnectionStatusChange> amqpConnectionStatuses = new List <AmqpConnectionStatusChange>(); IList <Task> operations = new List <Task>(); do { totalRuns++; // Arrange // Initialize the test device client instances // Set the device client connection status change handler s_log.WriteLine($">>> {nameof(PoolingOverAmqp)} Initializing Device Clients for multiplexing test - Test run {totalRuns}"); for (int i = 0; i < devicesCount; i++) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{devicePrefix}_{i}_").ConfigureAwait(false); DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope); var amqpConnectionStatusChange = new AmqpConnectionStatusChange(); deviceClient.SetConnectionStatusChangesHandler(amqpConnectionStatusChange.ConnectionStatusChangesHandler); testDevices.Add(testDevice); deviceClients.Add(deviceClient); amqpConnectionStatuses.Add(amqpConnectionStatusChange); if (initOperation != null) { operations.Add(initOperation(deviceClient, testDevice)); } } await Task.WhenAll(operations).ConfigureAwait(false); operations.Clear(); try { for (int i = 0; i < devicesCount; i++) { operations.Add(testOperation(deviceClients[i], testDevices[i])); } await Task.WhenAll(operations).ConfigureAwait(false); operations.Clear(); // Close the device client instances and verify the connection status change checks bool deviceConnectionStatusAsExpected = true; for (int i = 0; i < devicesCount; i++) { await deviceClients[i].CloseAsync().ConfigureAwait(false); if (!ignoreConnectionStatus) { // The connection status change count should be 2: connect (open) and disabled (close) if (amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount != 2) { deviceConnectionStatusAsExpected = false; } // The connection status should be "Disabled", with connection status change reason "Client_close" Assert.AreEqual( ConnectionStatus.Disabled, amqpConnectionStatuses[i].LastConnectionStatus, $"The actual connection status is = {amqpConnectionStatuses[i].LastConnectionStatus}"); Assert.AreEqual( ConnectionStatusChangeReason.Client_Close, amqpConnectionStatuses[i].LastConnectionStatusChangeReason, $"The actual connection status change reason is = {amqpConnectionStatuses[i].LastConnectionStatusChangeReason}"); } } if (deviceConnectionStatusAsExpected) { successfulRuns++; } currentSuccessRate = (int)((double)successfulRuns / totalRuns * 100); reRunTest = currentSuccessRate < TestSuccessRate; } finally { // Close the service-side components and dispose the device client instances. if (cleanupOperation != null) { await cleanupOperation().ConfigureAwait(false); } foreach (DeviceClient deviceClient in deviceClients) { deviceClient.Dispose(); } // Clean up the local lists testDevices.Clear(); deviceClients.Clear(); amqpConnectionStatuses.Clear(); } } while (reRunTest && totalRuns < MaxTestRunCount); Assert.IsFalse(reRunTest, $"Device client instances got disconnected in {totalRuns - successfulRuns} runs out of {totalRuns}; current testSuccessRate = {currentSuccessRate}%."); }
private async Task SendMethodAndRespondMuxedOverAmqp( ConnectionStringAuthScope authScope, Client.TransportType transport, int poolSize, int devicesCount, Func <DeviceClient, Task <Task> > setDeviceReceiveMethod ) { var transportSettings = new ITransportSettings[] { new AmqpTransportSettings(transport) { AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings() { MaxPoolSize = unchecked ((uint)poolSize), Pooling = true } } }; ICollection <DeviceClient> deviceClients = new List <DeviceClient>(); Dictionary <DeviceClient, int> deviceClientConnectionStatusChangeCount = new Dictionary <DeviceClient, int>(); try { for (int i = 0; i < devicesCount; i++) { ConnectionStatus? lastConnectionStatus = null; ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null; int setConnectionStatusChangesHandlerCount = 0; TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{DevicePrefix}_{i}_").ConfigureAwait(false); DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope); deviceClients.Add(deviceClient); deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) => { setConnectionStatusChangesHandlerCount++; lastConnectionStatus = status; lastConnectionStatusChangeReason = statusChangeReason; _log.WriteLine($"{nameof(MethodE2EMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}"); deviceClientConnectionStatusChangeCount[deviceClient] = setConnectionStatusChangesHandlerCount; }); Task methodReceivedTask = await setDeviceReceiveMethod(deviceClient).ConfigureAwait(false); await Task.WhenAll( MethodOperation.ServiceSendMethodAndVerifyResponse(testDevice.Id), methodReceivedTask).ConfigureAwait(false); } } finally { // Close and dispose all of the device client instances here foreach (DeviceClient deviceClient in deviceClients) { await deviceClient.CloseAsync().ConfigureAwait(false); // The connection status change count should be 2: connect (open) and disabled (close) Assert.IsTrue(deviceClientConnectionStatusChangeCount[deviceClient] == 2, $"Connection status change count for deviceClient {TestLogging.GetHashCode(deviceClient)} is {deviceClientConnectionStatusChangeCount[deviceClient]}"); _log.WriteLine($"{nameof(MethodE2EMultiplexingTests)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}"); deviceClient.Dispose(); } } }
// Error injection template method. public static async Task TestErrorInjectionAsync( string devicePrefix, TestDeviceType type, Client.TransportType transport, string faultType, string reason, int delayInSec, int durationInSec, Func <DeviceClient, TestDevice, Task> initOperation, Func <DeviceClient, TestDevice, Task> testOperation, Func <Task> cleanupOperation) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(devicePrefix, type).ConfigureAwait(false); DeviceClient deviceClient = testDevice.CreateDeviceClient(transport); ConnectionStatus? lastConnectionStatus = null; ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null; int setConnectionStatusChangesHandlerCount = 0; deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) => { s_log.WriteLine($"{nameof(FaultInjection)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}"); lastConnectionStatus = status; lastConnectionStatusChangeReason = statusChangeReason; setConnectionStatusChangesHandlerCount++; }); var watch = new Stopwatch(); try { await deviceClient.OpenAsync().ConfigureAwait(false); if (transport != Client.TransportType.Http1) { Assert.AreEqual(1, setConnectionStatusChangesHandlerCount); Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus); Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason); } await initOperation(deviceClient, testDevice).ConfigureAwait(false); s_log.WriteLine($">>> {nameof(FaultInjection)} Testing baseline"); await testOperation(deviceClient, testDevice).ConfigureAwait(false); await ActivateFaultInjection(transport, faultType, reason, delayInSec, durationInSec, deviceClient).ConfigureAwait(false); s_log.WriteLine($">>> {nameof(FaultInjection)} Testing fault handling"); watch.Start(); s_log.WriteLine($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {FaultInjection.WaitForDisconnectMilliseconds}ms"); await Task.Delay(FaultInjection.WaitForDisconnectMilliseconds).ConfigureAwait(false); await testOperation(deviceClient, testDevice).ConfigureAwait(false); await deviceClient.CloseAsync().ConfigureAwait(false); if (transport == Client.TransportType.Mqtt || transport == Client.TransportType.Mqtt_Tcp_Only || transport == Client.TransportType.Mqtt_WebSocket_Only) { // Our fault injection is only terminating the connection for MQTT. (HTTP is not connection-oriented, AMQP is not actually terminating the TCP layer.) Assert.IsTrue(setConnectionStatusChangesHandlerCount >= 4); Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus); Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason); } } finally { await cleanupOperation().ConfigureAwait(false); s_log.WriteLine($"{nameof(FaultInjection)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}"); deviceClient.Dispose(); watch.Stop(); int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds; if (timeToFinishFaultInjection > 0) { s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed."); await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false); } } }
public static async Task TestFaultInjectionPoolAmqpAsync( string devicePrefix, Client.TransportType transport, int poolSize, int devicesCount, string faultType, string reason, int delayInSec, int durationInSec, Func <DeviceClient, TestDevice, Task> initOperation, Func <DeviceClient, TestDevice, Task> testOperation, Func <IList <DeviceClient>, Task> cleanupOperation, ConnectionStringAuthScope authScope) { var transportSettings = new ITransportSettings[] { new AmqpTransportSettings(transport) { AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings() { MaxPoolSize = unchecked ((uint)poolSize), Pooling = true } } }; IList <TestDevice> testDevices = new List <TestDevice>(); IList <DeviceClient> deviceClients = new List <DeviceClient>(); IList <AmqpConnectionStatusChange> amqpConnectionStatuses = new List <AmqpConnectionStatusChange>(); IList <Task> operations = new List <Task>(); // Arrange // Initialize the test device client instances // Set the device client connection status change handler s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)} Initializing Device Clients for multiplexing test."); for (int i = 0; i < devicesCount; i++) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{devicePrefix}_{i}_").ConfigureAwait(false); DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope); var amqpConnectionStatusChange = new AmqpConnectionStatusChange(testDevice.Id); deviceClient.SetConnectionStatusChangesHandler(amqpConnectionStatusChange.ConnectionStatusChangesHandler); testDevices.Add(testDevice); deviceClients.Add(deviceClient); amqpConnectionStatuses.Add(amqpConnectionStatusChange); operations.Add(initOperation(deviceClient, testDevice)); } await Task.WhenAll(operations).ConfigureAwait(false); operations.Clear(); var watch = new Stopwatch(); try { // Act-Assert // Perform the test operation and verify the operation is successful // Perform baseline test operation for (int i = 0; i < devicesCount; i++) { s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing baseline operation for device {i}."); operations.Add(testOperation(deviceClients[i], testDevices[i])); } await Task.WhenAll(operations).ConfigureAwait(false); operations.Clear(); // Inject the fault into device 0 watch.Start(); s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: {testDevices[0].Id} Requesting fault injection type={faultType} reason={reason}, delay={delayInSec}s, duration={durationInSec}s"); var faultInjectionMessage = FaultInjection.ComposeErrorInjectionProperties(faultType, reason, delayInSec, durationInSec); await deviceClients[0].SendEventAsync(faultInjectionMessage).ConfigureAwait(false); s_log.WriteLine($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {delayInSec} seconds."); await Task.Delay(TimeSpan.FromSeconds(delayInSec)).ConfigureAwait(false); // For disconnect type faults, the faulted device should disconnect and all devices should recover. if (FaultInjection.FaultShouldDisconnect(faultType)) { s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirming fault injection has been actived."); // Check that service issued the fault to the faulting device [device 0] bool isFaulted = false; for (int i = 0; i < FaultInjection.LatencyTimeBufferInSec; i++) { if (amqpConnectionStatuses[0].ConnectionStatusChangeCount >= 2) { isFaulted = true; break; } await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); } Assert.IsTrue(isFaulted, $"The device {testDevices[0].Id} did not get faulted with fault type: {faultType}"); s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirmed fault injection has been actived."); // Check all devices are back online s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirming all devices back online."); bool notRecovered = true; int j = 0; for (int i = 0; notRecovered && i < durationInSec + FaultInjection.LatencyTimeBufferInSec; i++) { notRecovered = false; for (j = 0; j < devicesCount; j++) { if (amqpConnectionStatuses[j].LastConnectionStatus != ConnectionStatus.Connected) { await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); notRecovered = true; break; } } } if (notRecovered) { Assert.Fail($"{testDevices[j].Id} did not reconnect."); } s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirmed all devices back online."); // Perform the test operation for all devices for (int i = 0; i < devicesCount; i++) { s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation for device {i}."); operations.Add(testOperation(deviceClients[i], testDevices[i])); } await Task.WhenAll(operations).ConfigureAwait(false); operations.Clear(); } else { s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation while fault injection is being activated."); // Perform the test operation for the faulted device multi times. for (int i = 0; i < FaultInjection.LatencyTimeBufferInSec; i++) { s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation for device 0 - Run {i}."); await testOperation(deviceClients[0], testDevices[0]).ConfigureAwait(false); await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); } } // Close the device client instances for (int i = 0; i < devicesCount; i++) { operations.Add(deviceClients[i].CloseAsync()); } await Task.WhenAll(operations).ConfigureAwait(false); operations.Clear(); // Verify the connection status change checks. // For all of the devices - last connection status should be "Disabled", with reason "Client_close" for (int i = 0; i < devicesCount; i++) { // For the faulted device [device 0] - verify the connection status change count. if (i == 0) { if (FaultInjection.FaultShouldDisconnect(faultType)) { // 4 is the minimum notification count: connect, fault, reconnect, disable. Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangeCount >= 4, $"The expected connection status change count for {testDevices[i].Id} should equals or greater than 4 but was {amqpConnectionStatuses[i].ConnectionStatusChangeCount}"); } else { // 2 is the minimum notification count: connect, disable. Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangeCount == 2, $"The expected connection status change count for {testDevices[i].Id} should be 2 but was {amqpConnectionStatuses[i].ConnectionStatusChangeCount}"); } } Assert.AreEqual(ConnectionStatus.Disabled, amqpConnectionStatuses[i].LastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Disabled} but was {amqpConnectionStatuses[i].LastConnectionStatus}"); Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, amqpConnectionStatuses[i].LastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Client_Close} but was {amqpConnectionStatuses[i].LastConnectionStatusChangeReason}"); } } finally { // Close the service-side components and dispose the device client instances. await cleanupOperation(deviceClients).ConfigureAwait(false); watch.Stop(); int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds; if (timeToFinishFaultInjection > 0) { s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed."); await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false); } } }
private async Task DeviceCombinedClientOperations( Client.TransportType transport, int poolSize, int devicesCount, ConnectionStringAuthScope authScope ) { var transportSettings = new ITransportSettings[] { new AmqpTransportSettings(transport) { AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings() { MaxPoolSize = unchecked ((uint)poolSize), Pooling = true } } }; // Initialize service client for service-side operations ServiceClient serviceClient = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString); ICollection <DeviceClient> deviceClients = new List <DeviceClient>(); Dictionary <DeviceClient, int> deviceClientConnectionStatusChangeCount = new Dictionary <DeviceClient, int>(); try { _log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Starting the test execution for {devicesCount} devices"); for (int i = 0; i < devicesCount; i++) { ConnectionStatus? lastConnectionStatus = null; ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null; int setConnectionStatusChangesHandlerCount = 0; TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{DevicePrefix}_{i}_").ConfigureAwait(false); DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope); deviceClients.Add(deviceClient); // Set the connection status change handler deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) => { setConnectionStatusChangesHandlerCount++; lastConnectionStatus = status; lastConnectionStatusChangeReason = statusChangeReason; _log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}"); deviceClientConnectionStatusChangeCount[deviceClient] = setConnectionStatusChangesHandlerCount; }); // Perform D2C Operation _log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Preparing to send message for device {i}"); await deviceClient.OpenAsync().ConfigureAwait(false); await MessageSend.SendSingleMessageAndVerifyAsync(deviceClient, testDevice.Id).ConfigureAwait(false); // Perform C2D Operation _log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Setting the device {i} to receive C2D message."); string payload, messageId, p1Value; Message msg = MessageReceive.ComposeC2DTestMessage(out payload, out messageId, out p1Value); await serviceClient.SendAsync(testDevice.Id, msg).ConfigureAwait(false); await MessageReceive.VerifyReceivedC2DMessageAsync(transport, deviceClient, payload, p1Value).ConfigureAwait(false); // Invoke direct methods _log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Testing direct methods for device {i}"); Task methodReceivedTask = await MethodOperation.SetDeviceReceiveMethod(deviceClient).ConfigureAwait(false); await Task.WhenAll( MethodOperation.ServiceSendMethodAndVerifyResponse(testDevice.Id), methodReceivedTask).ConfigureAwait(false); // Set reported twin properties _log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Setting reported Twin properties for device {i}"); await TwinOperation.Twin_DeviceSetsReportedPropertyAndGetsItBack(deviceClient).ConfigureAwait(false); // Receive set desired twin properties _log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Received set desired tein properties for device {i}"); var propName = Guid.NewGuid().ToString(); var propValue = Guid.NewGuid().ToString(); Task updateReceivedTask = await TwinOperation.SetTwinPropertyUpdateCallbackHandlerAsync(deviceClient, propName, propValue).ConfigureAwait(false); await Task.WhenAll( TwinOperation.RegistryManagerUpdateDesiredPropertyAsync(testDevice.Id, propName, propValue), updateReceivedTask).ConfigureAwait(false); } } finally { // Close and dispose all of the device client instances here foreach (DeviceClient deviceClient in deviceClients) { await deviceClient.CloseAsync().ConfigureAwait(false); // The connection status change count should be 2: connect (open) and disabled (close) Assert.IsTrue(deviceClientConnectionStatusChangeCount[deviceClient] == 2); _log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}"); deviceClient.Dispose(); } } }
private async Task Twin_ServiceSetsDesiredPropertyAndDeviceReceivesEventMuxedOverAmqp( ConnectionStringAuthScope authScope, Client.TransportType transport, int poolSize, int devicesCount, Func <DeviceClient, string, string, Task <Task> > setTwinPropertyUpdateCallbackAsync ) { var propName = Guid.NewGuid().ToString(); var propValue = Guid.NewGuid().ToString(); var transportSettings = new ITransportSettings[] { new AmqpTransportSettings(transport) { AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings() { MaxPoolSize = unchecked ((uint)poolSize), Pooling = true } } }; ICollection <DeviceClient> deviceClients = new List <DeviceClient>(); Dictionary <DeviceClient, int> deviceClientConnectionStatusChangeCount = new Dictionary <DeviceClient, int>(); try { _log.WriteLine($"{nameof(TwinE2EMultiplexingTests)}: Starting the test execution for {devicesCount} devices"); for (int i = 0; i < devicesCount; i++) { ConnectionStatus? lastConnectionStatus = null; ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null; int setConnectionStatusChangesHandlerCount = 0; TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{DevicePrefix}_{i}_").ConfigureAwait(false); DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope); deviceClients.Add(deviceClient); deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) => { setConnectionStatusChangesHandlerCount++; lastConnectionStatus = status; lastConnectionStatusChangeReason = statusChangeReason; _log.WriteLine($"{nameof(TwinE2EMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}"); deviceClientConnectionStatusChangeCount[deviceClient] = setConnectionStatusChangesHandlerCount; }); _log.WriteLine($"{nameof(Twin_ServiceSetsDesiredPropertyAndDeviceReceivesEventMuxedOverAmqp)}: name={propName}, value={propValue}"); Task updateReceivedTask = await setTwinPropertyUpdateCallbackAsync(deviceClient, propName, propValue).ConfigureAwait(false); await Task.WhenAll( TwinOperation.RegistryManagerUpdateDesiredPropertyAsync(testDevice.Id, propName, propValue), updateReceivedTask).ConfigureAwait(false); } } finally { // Close and dispose all of the device client instances here foreach (DeviceClient deviceClient in deviceClients) { await deviceClient.CloseAsync().ConfigureAwait(false); // The connection status change count should be 2: connect (open) and disabled (close) Assert.IsTrue(deviceClientConnectionStatusChangeCount[deviceClient] == 2); _log.WriteLine($"{nameof(TwinE2EMultiplexingTests)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}"); deviceClient.Dispose(); } } }
public async Task DuplicateDevice_NoRetry_NoPingpong_OpenAsync() { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(Logger, _devicePrefix, TestDeviceType.Sasl).ConfigureAwait(false); Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: 2 device client instances with the same deviceId={testDevice.Id}."); using DeviceClient deviceClient1 = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only); using DeviceClient deviceClient2 = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only); Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: set device client instance 1 to no retry."); deviceClient1.SetRetryPolicy(new NoRetry()); ConnectionStatus?lastConnectionStatusDevice1 = null; var connectionStatusChangesDevice1 = new Dictionary <ConnectionStatus, int>(); deviceClient1.SetConnectionStatusChangesHandler((status, reason) => { connectionStatusChangesDevice1.TryGetValue(status, out int count); count++; connectionStatusChangesDevice1[status] = count; lastConnectionStatusDevice1 = status; }); ConnectionStatus?lastConnectionStatusDevice2 = null; var connectionStatusChangesDevice2 = new Dictionary <ConnectionStatus, int>(); deviceClient2.SetConnectionStatusChangesHandler((status, reason) => { connectionStatusChangesDevice2.TryGetValue(status, out int count); count++; connectionStatusChangesDevice2[status] = count; lastConnectionStatusDevice2 = status; }); Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: device client instance 1 calling OpenAsync..."); await deviceClient1.OpenAsync().ConfigureAwait(false); await deviceClient1 .SetMethodHandlerAsync( "empty_method", (methodRequest, userContext) => Task.FromResult(new MethodResponse(200)), deviceClient1) .ConfigureAwait(false); Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: device client instance 2 calling OpenAsync..."); await deviceClient2.OpenAsync().ConfigureAwait(false); await deviceClient2 .SetMethodHandlerAsync( "empty_method", (methodRequest, userContext) => Task.FromResult(new MethodResponse(200)), deviceClient2) .ConfigureAwait(false); Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: waiting device client instance 1 to be kicked off..."); var sw = Stopwatch.StartNew(); while (sw.Elapsed < FaultInjection.LatencyTimeBuffer) { if (connectionStatusChangesDevice1.ContainsKey(ConnectionStatus.Disconnected)) { break; } await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); } sw.Reset(); lastConnectionStatusDevice1.Should().Be(ConnectionStatus.Disconnected, $"Excpected device 1 to be {ConnectionStatus.Disconnected} but was {lastConnectionStatusDevice1}."); connectionStatusChangesDevice1.Should().NotContainKey(ConnectionStatus.Disconnected_Retrying, $"Shouldn't get {ConnectionStatus.Disconnected_Retrying} status change."); int connected = connectionStatusChangesDevice1[ConnectionStatus.Connected]; connected.Should().Be(1, $"Should get {ConnectionStatus.Connected} once but got it {connected} times."); int disconnected = connectionStatusChangesDevice1[ConnectionStatus.Disconnected]; disconnected.Should().Be(1, $"Should get {ConnectionStatus.Disconnected} once but got it {disconnected} times."); lastConnectionStatusDevice2.Should().Be(ConnectionStatus.Connected, $"Expected device 2 to be {ConnectionStatus.Connected} but was {lastConnectionStatusDevice2}."); }
private async Task ReceiveMessageMuxedOverAmqp( Client.TransportType transport, int poolSize, int devicesCount, ConnectionStringAuthScope authScope = ConnectionStringAuthScope.Device) { var transportSettings = new ITransportSettings[] { new AmqpTransportSettings(transport) { AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings() { MaxPoolSize = unchecked ((uint)poolSize), Pooling = true } } }; ICollection <DeviceClient> deviceClients = new List <DeviceClient>(); Dictionary <DeviceClient, int> deviceClientConnectionStatusChangeCount = new Dictionary <DeviceClient, int>(); ServiceClient serviceClient = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString); try { _log.WriteLine($"{nameof(MessageReceiveE2EMultiplexingTests)}: Starting the test execution for {devicesCount} devices"); await serviceClient.OpenAsync().ConfigureAwait(false); for (int i = 0; i < devicesCount; i++) { ConnectionStatus? lastConnectionStatus = null; ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null; int setConnectionStatusChangesHandlerCount = 0; TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{DevicePrefix}_{i}_").ConfigureAwait(false); DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope); deviceClients.Add(deviceClient); deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) => { setConnectionStatusChangesHandlerCount++; lastConnectionStatus = status; lastConnectionStatusChangeReason = statusChangeReason; _log.WriteLine($"{nameof(MessageReceiveE2EMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}"); deviceClientConnectionStatusChangeCount[deviceClient] = setConnectionStatusChangesHandlerCount; }); _log.WriteLine($"{nameof(MessageReceiveE2EMultiplexingTests)}: Preparing to receive message for device {i}"); await deviceClient.OpenAsync().ConfigureAwait(false); string payload, messageId, p1Value; Message msg = MessageReceive.ComposeC2DTestMessage(out payload, out messageId, out p1Value); await serviceClient.SendAsync(testDevice.Id, msg).ConfigureAwait(false); await MessageReceive.VerifyReceivedC2DMessageAsync(transport, deviceClient, payload, p1Value).ConfigureAwait(false); } } finally { await serviceClient.CloseAsync().ConfigureAwait(false); serviceClient.Dispose(); // Close and dispose all of the device client instances here foreach (DeviceClient deviceClient in deviceClients) { await deviceClient.CloseAsync().ConfigureAwait(false); // The connection status change count should be 2: connect (open) and disabled (close) Assert.IsTrue(deviceClientConnectionStatusChangeCount[deviceClient] == 2); _log.WriteLine($"{nameof(MessageReceiveE2EMultiplexingTests)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}"); deviceClient.Dispose(); } } }
// Error injection template method. public static async Task TestErrorInjectionAsync( string devicePrefix, TestDeviceType type, Client.TransportType transport, string faultType, string reason, int delayInSec, int durationInSec, Func <DeviceClient, TestDevice, Task> initOperation, Func <DeviceClient, TestDevice, Task> testOperation, Func <Task> cleanupOperation) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(devicePrefix, type).ConfigureAwait(false); DeviceClient deviceClient = testDevice.CreateDeviceClient(transport); ConnectionStatus? lastConnectionStatus = null; ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null; int connectionStatusChangeCount = 0; deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) => { connectionStatusChangeCount++; lastConnectionStatus = status; lastConnectionStatusChangeReason = statusChangeReason; s_log.WriteLine($"{nameof(FaultInjection)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={connectionStatusChangeCount}"); }); var watch = new Stopwatch(); try { await deviceClient.OpenAsync().ConfigureAwait(false); if (transport != Client.TransportType.Http1) { Assert.IsTrue(connectionStatusChangeCount >= 1, $"The expected connection status change should be equal or greater than 1 but was {connectionStatusChangeCount}"); // Normally one connection but in some cases, due to network issues we might have already retried several times to connect. Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Connected} but was {lastConnectionStatus}"); Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Connection_Ok} but was {lastConnectionStatusChangeReason}"); } await initOperation(deviceClient, testDevice).ConfigureAwait(false); s_log.WriteLine($">>> {nameof(FaultInjection)} Testing baseline"); await testOperation(deviceClient, testDevice).ConfigureAwait(false); watch.Start(); s_log.WriteLine($">>> {nameof(FaultInjection)} Testing fault handling"); await ActivateFaultInjection(transport, faultType, reason, delayInSec, durationInSec, deviceClient).ConfigureAwait(false); s_log.WriteLine($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {delayInSec} seconds."); await Task.Delay(TimeSpan.FromSeconds(delayInSec)).ConfigureAwait(false); // For disconnect type faults, the device should disconnect and recover. if (FaultShouldDisconnect(faultType)) { s_log.WriteLine($"{nameof(FaultInjection)}: Confirming fault injection has been actived."); // Check that service issued the fault to the faulting device bool isFaulted = false; for (int i = 0; i < LatencyTimeBufferInSec; i++) { if (connectionStatusChangeCount >= 2) { isFaulted = true; break; } await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); } Assert.IsTrue(isFaulted, $"The device {testDevice.Id} did not get faulted with fault type: {faultType}"); s_log.WriteLine($"{nameof(FaultInjection)}: Confirmed fault injection has been actived."); // Check the device is back online s_log.WriteLine($"{nameof(FaultInjection)}: Confirming device back online."); for (int i = 0; lastConnectionStatus != ConnectionStatus.Connected && i < durationInSec + LatencyTimeBufferInSec; i++) { await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); } Assert.AreEqual(lastConnectionStatus, ConnectionStatus.Connected, $"{testDevice.Id} did not reconnect."); s_log.WriteLine($"{nameof(FaultInjection)}: Confirmed device back online."); // Perform the test operation. s_log.WriteLine($">>> {nameof(FaultInjection)}: Performing test operation for device {testDevice.Id}."); await testOperation(deviceClient, testDevice).ConfigureAwait(false); } else { s_log.WriteLine($"{nameof(FaultInjection)}: Performing test operation while fault injection is being activated."); // Perform the test operation for the faulted device multi times. for (int i = 0; i < LatencyTimeBufferInSec; i++) { s_log.WriteLine($">>> {nameof(FaultInjection)}: Performing test operation for device - Run {i}."); await testOperation(deviceClient, testDevice).ConfigureAwait(false); await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); } } await deviceClient.CloseAsync().ConfigureAwait(false); if (transport != Client.TransportType.Http1) { if (FaultInjection.FaultShouldDisconnect(faultType)) { // 4 is the minimum notification count: connect, fault, reconnect, disable. // There are cases where the retry must be timed out (i.e. very likely for MQTT where otherwise // we would attempt to send the fault injection forever.) Assert.IsTrue(connectionStatusChangeCount >= 4, $"The expected connection status change count for {testDevice.Id} should be equal or greater than 4 but was {connectionStatusChangeCount}"); } else { // 2 is the minimum notification count: connect, disable. // We will monitor the test environment real network stability and switch to >=2 if necessary to // account for real network issues. Assert.IsTrue(connectionStatusChangeCount == 2, $"The expected connection status change count for {testDevice.Id} should be 2 but was {connectionStatusChangeCount}"); } Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Disabled} but was {lastConnectionStatus}"); Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Client_Close} but was {lastConnectionStatusChangeReason}"); } } finally { await cleanupOperation().ConfigureAwait(false); s_log.WriteLine($"{nameof(FaultInjection)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}"); deviceClient.Dispose(); watch.Stop(); int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds; if (timeToFinishFaultInjection > 0) { s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed."); await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false); } } }
public static async Task TestFaultInjectionPoolAmqpAsync( string devicePrefix, Client.TransportType transport, int poolSize, int devicesCount, string faultType, string reason, int delayInSec, int durationInSec, Func <DeviceClient, TestDevice, Task> initOperation, Func <DeviceClient, TestDevice, Task> testOperation, Func <IList <DeviceClient>, Task> cleanupOperation, ConnectionStringAuthScope authScope) { var transportSettings = new ITransportSettings[] { new AmqpTransportSettings(transport) { AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings() { MaxPoolSize = unchecked ((uint)poolSize), Pooling = true } } }; IList <TestDevice> testDevices = new List <TestDevice>(); IList <DeviceClient> deviceClients = new List <DeviceClient>(); IList <AmqpConnectionStatusChange> amqpConnectionStatuses = new List <AmqpConnectionStatusChange>(); IList <Task> operations = new List <Task>(); // Arrange // Initialize the test device client instances // Set the device client connection status change handler s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)} Initializing Device Clients for multiplexing test."); for (int i = 0; i < devicesCount; i++) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{devicePrefix}_{i}_").ConfigureAwait(false); DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope); var amqpConnectionStatusChange = new AmqpConnectionStatusChange(testDevice.Id); deviceClient.SetConnectionStatusChangesHandler(amqpConnectionStatusChange.ConnectionStatusChangesHandler); testDevices.Add(testDevice); deviceClients.Add(deviceClient); amqpConnectionStatuses.Add(amqpConnectionStatusChange); operations.Add(initOperation(deviceClient, testDevice)); } await Task.WhenAll(operations).ConfigureAwait(false); operations.Clear(); var watch = new Stopwatch(); try { // Act-Assert // Perform the test operation and verify the operation is successful // Perform baseline test operation for (int i = 0; i < devicesCount; i++) { s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing baseline operation for device {i}."); operations.Add(testOperation(deviceClients[i], testDevices[i])); } await Task.WhenAll(operations).ConfigureAwait(false); operations.Clear(); // Inject the fault into device 0 watch.Start(); s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Device {0} Requesting fault injection type={faultType} reason={reason}, delay={delayInSec}s, duration={durationInSec}s"); var faultInjectionMessage = FaultInjection.ComposeErrorInjectionProperties(faultType, reason, delayInSec, durationInSec); await deviceClients[0].SendEventAsync(faultInjectionMessage).ConfigureAwait(false); int delay = FaultInjection.WaitForReconnectMilliseconds - (int)watch.ElapsedMilliseconds; if (delay < 0) { delay = 0; } s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Waiting for fault injection to be active and device to be connected: {delay}ms"); await Task.Delay(delay).ConfigureAwait(false); // Perform the test operation for all devices for (int i = 0; i < devicesCount; i++) { s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation for device {i}."); operations.Add(testOperation(deviceClients[i], testDevices[i])); } await Task.WhenAll(operations).ConfigureAwait(false); operations.Clear(); // Close the device client instances for (int i = 0; i < devicesCount; i++) { operations.Add(deviceClients[i].CloseAsync()); } await Task.WhenAll(operations).ConfigureAwait(false); operations.Clear(); // Verify the connection status change checks. // For all of the devices - last connection status should be "Disabled", with reason "Client_close" for (int i = 0; i < devicesCount; i++) { // For the faulted device [device 0] - verify the connection status change count. if (i == 0) { if (FaultInjection.FaultShouldDisconnect(faultType)) { // 4 is the minimum notification count: connect, fault, reconnect, disable. Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount >= 4, $"The actual connection status change count for faulted device[0] is = {amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount}"); } else { // 2 is the minimum notification count: connect, disable. Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount == 2, $"The actual connection status change count for for faulted device[0] is = {amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount}"); } } Assert.AreEqual(ConnectionStatus.Disabled, amqpConnectionStatuses[i].LastConnectionStatus); Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, amqpConnectionStatuses[i].LastConnectionStatusChangeReason); } } finally { // Close the service-side components and dispose the device client instances. await cleanupOperation(deviceClients).ConfigureAwait(false); watch.Stop(); int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds; if (timeToFinishFaultInjection > 0) { s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed."); await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false); } } }