public async Task FaultInjection_NoRetry_NoRecovery_OpenAsync() { using TestDevice testDevice = await TestDevice.GetTestDeviceAsync(Logger, _devicePrefix, TestDeviceType.Sasl).ConfigureAwait(false); using DeviceClient deviceClient = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only); Logger.Trace($"{nameof(FaultInjection_NoRetry_NoRecovery_OpenAsync)}: deviceId={testDevice.Id}"); deviceClient.SetRetryPolicy(new NoRetry()); ConnectionStatus? lastConnectionStatus = null; ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null; var connectionStatusChanges = new Dictionary <ConnectionStatus, int>(); deviceClient.SetConnectionStatusChangesHandler((status, reason) => { connectionStatusChanges.TryGetValue(status, out int count); count++; connectionStatusChanges[status] = count; lastConnectionStatus = status; lastConnectionStatusChangeReason = reason; }); Logger.Trace($"{nameof(FaultInjection_NoRetry_NoRecovery_OpenAsync)}: calling OpenAsync..."); await deviceClient.OpenAsync().ConfigureAwait(false); Logger.Trace($"{nameof(FaultInjection_NoRetry_NoRecovery_OpenAsync)}: injecting fault {FaultInjection.FaultType_Tcp}..."); await FaultInjection .ActivateFaultInjectionAsync( Client.TransportType.Amqp_Tcp_Only, FaultInjection.FaultType_Tcp, FaultInjection.FaultCloseReason_Boom, FaultInjection.DefaultFaultDelay, FaultInjection.DefaultFaultDuration, deviceClient, Logger) .ConfigureAwait(false); await Task.Delay(FaultInjection.DefaultFaultDelay).ConfigureAwait(false); Logger.Trace($"{nameof(FaultInjection_NoRetry_NoRecovery_OpenAsync)}: waiting fault injection occurs..."); var sw = Stopwatch.StartNew(); while (sw.Elapsed < FaultInjection.LatencyTimeBuffer) { if (connectionStatusChanges.ContainsKey(ConnectionStatus.Disconnected)) { break; } await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); } sw.Reset(); lastConnectionStatus.Should().Be(ConnectionStatus.Disconnected, $"Expected device to be {ConnectionStatus.Disconnected} but was {lastConnectionStatus}."); lastConnectionStatusChangeReason.Should().Be(ConnectionStatusChangeReason.Retry_Expired, $"Expected device to be {ConnectionStatusChangeReason.Retry_Expired} but was {lastConnectionStatusChangeReason}."); connectionStatusChanges.Should().NotContainKey(ConnectionStatus.Disconnected_Retrying, $"Shouldn't get {ConnectionStatus.Disconnected_Retrying} status change."); int connected = connectionStatusChanges[ConnectionStatus.Connected]; connected.Should().Be(1, $"Should get {ConnectionStatus.Connected} once but got it {connected} times."); int disconnected = connectionStatusChanges[ConnectionStatus.Disconnected]; disconnected.Should().Be(1, $"Should get {ConnectionStatus.Disconnected} once but got it {disconnected} times."); }
public async Task DuplicateDevice_NoRetry_NoPingpong_OpenAsync() { using TestDevice testDevice = await TestDevice.GetTestDeviceAsync(Logger, _devicePrefix, TestDeviceType.Sasl).ConfigureAwait(false); Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: 2 device client instances with the same deviceId={testDevice.Id}."); using DeviceClient deviceClient1 = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only); using DeviceClient deviceClient2 = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only); Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: set device client instance 1 to no retry."); deviceClient1.SetRetryPolicy(new NoRetry()); ConnectionStatus?lastConnectionStatusDevice1 = null; var connectionStatusChangesDevice1 = new Dictionary <ConnectionStatus, int>(); deviceClient1.SetConnectionStatusChangesHandler((status, reason) => { connectionStatusChangesDevice1.TryGetValue(status, out int count); count++; connectionStatusChangesDevice1[status] = count; lastConnectionStatusDevice1 = status; }); ConnectionStatus?lastConnectionStatusDevice2 = null; var connectionStatusChangesDevice2 = new Dictionary <ConnectionStatus, int>(); deviceClient2.SetConnectionStatusChangesHandler((status, reason) => { connectionStatusChangesDevice2.TryGetValue(status, out int count); count++; connectionStatusChangesDevice2[status] = count; lastConnectionStatusDevice2 = status; }); Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: device client instance 1 calling OpenAsync..."); await deviceClient1.OpenAsync().ConfigureAwait(false); await deviceClient1 .SetMethodHandlerAsync( "empty_method", (methodRequest, userContext) => Task.FromResult(new MethodResponse(200)), deviceClient1) .ConfigureAwait(false); Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: device client instance 2 calling OpenAsync..."); await deviceClient2.OpenAsync().ConfigureAwait(false); await deviceClient2 .SetMethodHandlerAsync( "empty_method", (methodRequest, userContext) => Task.FromResult(new MethodResponse(200)), deviceClient2) .ConfigureAwait(false); Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: waiting device client instance 1 to be kicked off..."); var sw = Stopwatch.StartNew(); while (sw.Elapsed < FaultInjection.LatencyTimeBuffer) { if (connectionStatusChangesDevice1.ContainsKey(ConnectionStatus.Disconnected)) { break; } await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); } sw.Reset(); lastConnectionStatusDevice1.Should().Be(ConnectionStatus.Disconnected, $"Excpected device 1 to be {ConnectionStatus.Disconnected} but was {lastConnectionStatusDevice1}."); connectionStatusChangesDevice1.Should().NotContainKey(ConnectionStatus.Disconnected_Retrying, $"Shouldn't get {ConnectionStatus.Disconnected_Retrying} status change."); int connected = connectionStatusChangesDevice1[ConnectionStatus.Connected]; connected.Should().Be(1, $"Should get {ConnectionStatus.Connected} once but got it {connected} times."); int disconnected = connectionStatusChangesDevice1[ConnectionStatus.Disconnected]; disconnected.Should().Be(1, $"Should get {ConnectionStatus.Disconnected} once but got it {disconnected} times."); lastConnectionStatusDevice2.Should().Be(ConnectionStatus.Connected, $"Expected device 2 to be {ConnectionStatus.Connected} but was {lastConnectionStatusDevice2}."); }
private async Task Twin_DeviceDesiredPropertyUpdateRecovery(Client.TransportType transport, string faultType, string reason, int delayInSec) { var tcs = new TaskCompletionSource <bool>(); var propName = Guid.NewGuid().ToString(); var propValue = Guid.NewGuid().ToString(); TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false); RegistryManager registryManager = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString); var deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport); ConnectionStatus? lastConnectionStatus = null; ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null; int setConnectionStatusChangesHandlerCount = 0; var tcsConnected = new TaskCompletionSource <bool>(); var tcsDisconnected = new TaskCompletionSource <bool>(); deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) => { _log.WriteLine("Connection Changed to {0} because {1}", status, statusChangeReason); if (status == ConnectionStatus.Disconnected_Retrying) { tcsDisconnected.TrySetResult(true); Assert.AreEqual(ConnectionStatusChangeReason.No_Network, statusChangeReason); } else if (status == ConnectionStatus.Connected) { tcsConnected.TrySetResult(true); } lastConnectionStatus = status; lastConnectionStatusChangeReason = statusChangeReason; setConnectionStatusChangesHandlerCount++; }); await deviceClient.SetDesiredPropertyUpdateCallbackAsync((patch, context) => { return(Task.Run(() => { try { Assert.AreEqual(patch[propName].ToString(), propValue); } catch (Exception e) { tcs.SetException(e); } finally { tcs.SetResult(true); } })); }, null).ConfigureAwait(false); // assert on successful connection await Task.WhenAny( Task.Run(async() => { await Task.Delay(1000).ConfigureAwait(false); }), tcsConnected.Task).ConfigureAwait(false); Assert.IsTrue(tcsConnected.Task.IsCompleted, "Initial connection failed"); if (transport != Client.TransportType.Http1) { Assert.AreEqual(1, setConnectionStatusChangesHandlerCount); Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus); Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason); } var twinPatch = new Twin(); twinPatch.Properties.Desired[propName] = propValue; await registryManager.UpdateTwinAsync(testDevice.Id, twinPatch, "*").ConfigureAwait(false); await tcs.Task.ConfigureAwait(false); // send error command await deviceClient.SendEventAsync(FaultInjection.ComposeErrorInjectionProperties(faultType, reason, delayInSec)).ConfigureAwait(false); // reset ConnectionStatusChangesHandler data setConnectionStatusChangesHandlerCount = 0; tcsConnected = new TaskCompletionSource <bool>(); tcsDisconnected = new TaskCompletionSource <bool>(); // wait for disconnection await Task.WhenAny( Task.Run(async() => { await Task.Delay(TimeSpan.FromSeconds(10)).ConfigureAwait(false); }), tcsDisconnected.Task).ConfigureAwait(false); Assert.IsTrue(tcsDisconnected.Task.IsCompleted, "Error injection did not interrupt the device"); await Task.WhenAny( Task.Run(async() => { await Task.Delay(TimeSpan.FromSeconds(MaximumRecoveryTimeSeconds)).ConfigureAwait(false); return(Task.FromResult(true)); }), tcsConnected.Task).ConfigureAwait(false); Assert.IsTrue(tcsConnected.Task.IsCompleted, "Recovery connection failed"); tcs = new TaskCompletionSource <bool>(); twinPatch = new Twin(); twinPatch.Properties.Desired[propName] = propValue; await registryManager.UpdateTwinAsync(testDevice.Id, twinPatch, "*").ConfigureAwait(false); await tcs.Task.ConfigureAwait(false); await deviceClient.CloseAsync().ConfigureAwait(false); await registryManager.CloseAsync().ConfigureAwait(false); }
private async Task _Twin_DeviceDesiredPropertyUpdateRecovery(Client.TransportType transport, string faultType, string reason, int delayInSec) { var tcs = new TaskCompletionSource <bool>(); var propName = Guid.NewGuid().ToString(); var propValue = Guid.NewGuid().ToString(); Tuple <string, string> deviceInfo = TestUtil.CreateDevice(DevicePrefix, hostName, registryManager); var deviceClient = DeviceClient.CreateFromConnectionString(deviceInfo.Item2, transport); ConnectionStatus? lastConnectionStatus = null; ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null; int setConnectionStatusChangesHandlerCount = 0; var tcsConnected = new TaskCompletionSource <bool>(); var tcsDisconnected = new TaskCompletionSource <bool>(); deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) => { if (status == ConnectionStatus.Disconnected_Retrying) { tcsDisconnected.TrySetResult(true); Assert.AreEqual(ConnectionStatusChangeReason.No_Network, statusChangeReason); } else if (status == ConnectionStatus.Connected) { tcsConnected.TrySetResult(true); } lastConnectionStatus = status; lastConnectionStatusChangeReason = statusChangeReason; setConnectionStatusChangesHandlerCount++; }); await deviceClient.SetDesiredPropertyUpdateCallbackAsync((patch, context) => { return(Task.Run(() => { try { Assert.AreEqual(patch[propName].ToString(), propValue); } catch (Exception e) { tcs.SetException(e); } finally { tcs.SetResult(true); } })); }, null); // assert on successfuly connection await Task.WhenAny( Task.Run(async() => { await Task.Delay(1000); }), tcsConnected.Task); Assert.IsTrue(tcsConnected.Task.IsCompleted, "Initial connection failed"); if (transport != Client.TransportType.Http1) { Assert.AreEqual(1, setConnectionStatusChangesHandlerCount); Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus); Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason); } var twinPatch = new Twin(); twinPatch.Properties.Desired[propName] = propValue; await registryManager.UpdateTwinAsync(deviceInfo.Item1, twinPatch, "*"); await tcs.Task; // send error command await deviceClient.SendEventAsync(TestUtil.ComposeErrorInjectionProperties(faultType, reason, delayInSec)); // reset ConnectionStatusChangesHandler data setConnectionStatusChangesHandlerCount = 0; tcsConnected = new TaskCompletionSource <bool>(); tcsDisconnected = new TaskCompletionSource <bool>(); // wait for disconnection await Task.WhenAny( Task.Run(async() => { await Task.Delay(TimeSpan.FromSeconds(10)); }), tcsDisconnected.Task); Assert.IsTrue(tcsDisconnected.Task.IsCompleted, "Error injection did not interrupt the device"); // allow max 30s for connection recovery await Task.WhenAny( Task.Run(async() => { await Task.Delay(TimeSpan.FromSeconds(10)); return(Task.FromResult(true)); }), tcsConnected.Task); Assert.IsTrue(tcsConnected.Task.IsCompleted, "Recovery connection failed"); tcs = new TaskCompletionSource <bool>(); twinPatch = new Twin(); twinPatch.Properties.Desired[propName] = propValue; await registryManager.UpdateTwinAsync(deviceInfo.Item1, twinPatch, "*"); await tcs.Task; await deviceClient.CloseAsync(); await TestUtil.RemoveDeviceAsync(deviceInfo.Item1, registryManager); }
private async Task SendMessageRecovery(Client.TransportType transport, string faultType, string reason, int delayInSec, int durationInSec = 0, int retryDurationInMilliSec = 240000) { await sequentialTestSemaphore.WaitAsync(); Tuple <string, string> deviceInfo = TestUtil.CreateDevice(DevicePrefix, hostName, registryManager); var deviceClient = DeviceClient.CreateFromConnectionString(deviceInfo.Item2, transport); EventHubClient eventHubClient; EventHubReceiver eventHubReceiver = CreateEventHubReceiver(deviceInfo.Item1, out eventHubClient); try { deviceClient.OperationTimeoutInMilliseconds = (uint)retryDurationInMilliSec; ConnectionStatus? lastConnectionStatus = null; ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null; int setConnectionStatusChangesHandlerCount = 0; deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) => { lastConnectionStatus = status; lastConnectionStatusChangeReason = statusChangeReason; setConnectionStatusChangesHandlerCount++; }); await deviceClient.OpenAsync(); if (transport != Client.TransportType.Http1) { Assert.AreEqual(1, setConnectionStatusChangesHandlerCount); Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus); Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason); } string payload, p1Value; Client.Message testMessage = ComposeD2CTestMessage(out payload, out p1Value); await deviceClient.SendEventAsync(testMessage); bool isReceived = false; Stopwatch sw = new Stopwatch(); sw.Start(); while (!isReceived && sw.Elapsed.Minutes < 1) { var events = await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5)); isReceived = VerifyTestMessage(events, deviceInfo.Item1, payload, p1Value); } sw.Stop(); // send error command and clear eventHubReceiver of the fault injection message await deviceClient.SendEventAsync(TestUtil.ComposeErrorInjectionProperties(faultType, reason, delayInSec, durationInSec)); await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5)); Thread.Sleep(1000); testMessage = ComposeD2CTestMessage(out payload, out p1Value); await deviceClient.SendEventAsync(testMessage); sw.Reset(); sw.Start(); while (!isReceived && sw.Elapsed.Minutes < 1) { var events = await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5)); isReceived = VerifyTestMessage(events, deviceInfo.Item1, payload, p1Value); } sw.Stop(); await deviceClient.CloseAsync(); if (transport != Client.TransportType.Http1) { Assert.AreEqual(2, setConnectionStatusChangesHandlerCount); Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus); Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason); } } finally { await deviceClient.CloseAsync(); await eventHubReceiver.CloseAsync(); await eventHubClient.CloseAsync(); await TestUtil.RemoveDeviceAsync(deviceInfo.Item1, registryManager); sequentialTestSemaphore.Release(1); } }
private async Task SendMethodAndRespondMuxedOverAmqp( ConnectionStringAuthScope authScope, Client.TransportType transport, int poolSize, int devicesCount, Func <DeviceClient, Task <Task> > setDeviceReceiveMethod ) { var transportSettings = new ITransportSettings[] { new AmqpTransportSettings(transport) { AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings() { MaxPoolSize = unchecked ((uint)poolSize), Pooling = true } } }; ICollection <DeviceClient> deviceClients = new List <DeviceClient>(); Dictionary <DeviceClient, int> deviceClientConnectionStatusChangeCount = new Dictionary <DeviceClient, int>(); try { for (int i = 0; i < devicesCount; i++) { ConnectionStatus? lastConnectionStatus = null; ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null; int setConnectionStatusChangesHandlerCount = 0; TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{DevicePrefix}_{i}_").ConfigureAwait(false); DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope); deviceClients.Add(deviceClient); deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) => { setConnectionStatusChangesHandlerCount++; lastConnectionStatus = status; lastConnectionStatusChangeReason = statusChangeReason; _log.WriteLine($"{nameof(MethodE2EMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}"); deviceClientConnectionStatusChangeCount[deviceClient] = setConnectionStatusChangesHandlerCount; }); Task methodReceivedTask = await setDeviceReceiveMethod(deviceClient).ConfigureAwait(false); await Task.WhenAll( MethodOperation.ServiceSendMethodAndVerifyResponse(testDevice.Id), methodReceivedTask).ConfigureAwait(false); } } finally { // Close and dispose all of the device client instances here foreach (DeviceClient deviceClient in deviceClients) { await deviceClient.CloseAsync().ConfigureAwait(false); // The connection status change count should be 2: connect (open) and disabled (close) Assert.IsTrue(deviceClientConnectionStatusChangeCount[deviceClient] == 2, $"Connection status change count for deviceClient {TestLogging.GetHashCode(deviceClient)} is {deviceClientConnectionStatusChangeCount[deviceClient]}"); _log.WriteLine($"{nameof(MethodE2EMultiplexingTests)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}"); deviceClient.Dispose(); } } }
public static async Task TestErrorInjectionTemplate( string devicePrefix, TestDeviceType type, Client.TransportType transport, string faultType, string reason, int delayInSec, int durationInSec, Func <DeviceClient, TestDevice, Task> initOperation, Func <DeviceClient, TestDevice, Task> testOperation, Func <Task> cleanupOperation) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(devicePrefix, type).ConfigureAwait(false); DeviceClient deviceClient = testDevice.CreateDeviceClient(transport); ConnectionStatus? lastConnectionStatus = null; ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null; int setConnectionStatusChangesHandlerCount = 0; deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) => { s_log.WriteLine($"{nameof(FaultInjection)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}"); lastConnectionStatus = status; lastConnectionStatusChangeReason = statusChangeReason; setConnectionStatusChangesHandlerCount++; }); var watch = new Stopwatch(); try { await deviceClient.OpenAsync().ConfigureAwait(false); if (transport != Client.TransportType.Http1) { Assert.AreEqual(1, setConnectionStatusChangesHandlerCount); Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus); Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason); } await initOperation(deviceClient, testDevice).ConfigureAwait(false); s_log.WriteLine($">>> {nameof(FaultInjection)} Testing baseline"); await testOperation(deviceClient, testDevice).ConfigureAwait(false); await ActivateFaultInjection(transport, faultType, reason, delayInSec, durationInSec, deviceClient).ConfigureAwait(false); s_log.WriteLine($">>> {nameof(FaultInjection)} Testing fault handling"); watch.Start(); s_log.WriteLine($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {FaultInjection.WaitForDisconnectMilliseconds}ms"); await Task.Delay(FaultInjection.WaitForDisconnectMilliseconds).ConfigureAwait(false); await testOperation(deviceClient, testDevice).ConfigureAwait(false); await deviceClient.CloseAsync().ConfigureAwait(false); if (transport == Client.TransportType.Mqtt || transport == Client.TransportType.Mqtt_Tcp_Only || transport == Client.TransportType.Mqtt_WebSocket_Only) { // Our fault injection is only terminating the connection for MQTT. (HTTP is not connection-oriented, AMQP is not actually terminating the TCP layer.) Assert.IsTrue(setConnectionStatusChangesHandlerCount >= 4); Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus); Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason); } } finally { await cleanupOperation().ConfigureAwait(false); await deviceClient.CloseAsync().ConfigureAwait(false); watch.Stop(); int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds; if (timeToFinishFaultInjection > 0) { s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed."); await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false); } } }
private async Task Twin_ServiceSetsDesiredPropertyAndDeviceReceivesEventMuxedOverAmqp( ConnectionStringAuthScope authScope, Client.TransportType transport, int poolSize, int devicesCount, Func <DeviceClient, string, string, Task <Task> > setTwinPropertyUpdateCallbackAsync ) { var propName = Guid.NewGuid().ToString(); var propValue = Guid.NewGuid().ToString(); var transportSettings = new ITransportSettings[] { new AmqpTransportSettings(transport) { AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings() { MaxPoolSize = unchecked ((uint)poolSize), Pooling = true } } }; ICollection <DeviceClient> deviceClients = new List <DeviceClient>(); Dictionary <DeviceClient, int> deviceClientConnectionStatusChangeCount = new Dictionary <DeviceClient, int>(); try { _log.WriteLine($"{nameof(TwinE2EMultiplexingTests)}: Starting the test execution for {devicesCount} devices"); for (int i = 0; i < devicesCount; i++) { ConnectionStatus? lastConnectionStatus = null; ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null; int setConnectionStatusChangesHandlerCount = 0; TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{DevicePrefix}_{i}_").ConfigureAwait(false); DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope); deviceClients.Add(deviceClient); deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) => { setConnectionStatusChangesHandlerCount++; lastConnectionStatus = status; lastConnectionStatusChangeReason = statusChangeReason; _log.WriteLine($"{nameof(TwinE2EMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}"); deviceClientConnectionStatusChangeCount[deviceClient] = setConnectionStatusChangesHandlerCount; }); _log.WriteLine($"{nameof(Twin_ServiceSetsDesiredPropertyAndDeviceReceivesEventMuxedOverAmqp)}: name={propName}, value={propValue}"); Task updateReceivedTask = await setTwinPropertyUpdateCallbackAsync(deviceClient, propName, propValue).ConfigureAwait(false); await Task.WhenAll( TwinOperation.RegistryManagerUpdateDesiredPropertyAsync(testDevice.Id, propName, propValue), updateReceivedTask).ConfigureAwait(false); } } finally { // Close and dispose all of the device client instances here foreach (DeviceClient deviceClient in deviceClients) { await deviceClient.CloseAsync().ConfigureAwait(false); // The connection status change count should be 2: connect (open) and disabled (close) Assert.IsTrue(deviceClientConnectionStatusChangeCount[deviceClient] == 2, $"Connection status change count for deviceClient {TestLogging.GetHashCode(deviceClient)} is {deviceClientConnectionStatusChangeCount[deviceClient]}"); _log.WriteLine($"{nameof(TwinE2EMultiplexingTests)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}"); deviceClient.Dispose(); } } }
async Task ModuleClient_Gives_ConnectionStatus_DeviceDisabled_Base( Client.TransportType protocol, Func <RegistryManager, string, Task> registryManagerOperation) { AmqpTransportSettings amqpTransportSettings = new AmqpTransportSettings(protocol); ITransportSettings[] transportSettings = new ITransportSettings[] { amqpTransportSettings }; TestModule testModule = await TestModule.GetTestModuleAsync(DevicePrefix + $"_{Guid.NewGuid()}", ModulePrefix).ConfigureAwait(false); ConnectionStatus? status = null; ConnectionStatusChangeReason?statusChangeReason = null; int deviceDisabledReceivedCount = 0; ConnectionStatusChangesHandler statusChangeHandler = (s, r) => { if (r == ConnectionStatusChangeReason.Device_Disabled) { status = s; statusChangeReason = r; deviceDisabledReceivedCount++; } }; using (ModuleClient moduleClient = ModuleClient.CreateFromConnectionString(testModule.ConnectionString, transportSettings)) { moduleClient.SetConnectionStatusChangesHandler(statusChangeHandler); _log.WriteLine($"Created {nameof(DeviceClient)} ID={TestLogging.IdOf(moduleClient)}"); Console.WriteLine("ModuleClient OpenAsync."); await moduleClient.OpenAsync().ConfigureAwait(false); // Receiving the module twin should succeed right now. Console.WriteLine("ModuleClient GetTwinAsync."); var twin = await moduleClient.GetTwinAsync().ConfigureAwait(false); Assert.IsNotNull(twin); // Delete the device in IoT Hub. using (RegistryManager registryManager = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString)) { await registryManagerOperation(registryManager, testModule.DeviceId).ConfigureAwait(false); } // Periodically keep retrieving the device twin to keep connection alive. // The ConnectionStatusChangesHandler should be triggered when the connection is closed from IoT hub with an // exception thrown. int twinRetrievals = 50; for (int i = 0; i < twinRetrievals; i++) { try { await Task.Delay(TimeSpan.FromSeconds(10)).ConfigureAwait(false); if (deviceDisabledReceivedCount == 1) { // Call an API on the client again to trigger the ConnectionStatusChangesHandler once again with the // Device_Disabled status. // This currently does not work due to some issues with IoT hub allowing new connections even when the // device is deleted/disabled. Once that problem is investigated and fixed, we can re-enable this call // and test for multiple invocations of the ConnectionStatusChangeHandler. // await moduleClient.GetTwinAsync().ConfigureAwait(false); break; } } catch (IotHubException ex) { _log.WriteLine($"Exception occurred while retrieving module twin: {ex}"); Assert.IsInstanceOfType(ex.InnerException, typeof(DeviceNotFoundException)); } } Assert.AreEqual(1, deviceDisabledReceivedCount); Assert.AreEqual(ConnectionStatus.Disconnected, status); Assert.AreEqual(ConnectionStatusChangeReason.Device_Disabled, statusChangeReason); } }
private async Task SendMethodAndRespondRecovery(Client.TransportType transport, string faultType, string reason, int delayInSec) { Tuple <string, string> deviceInfo = TestUtil.CreateDevice(DevicePrefix, hostName, registryManager); var assertResult = new TaskCompletionSource <Tuple <bool, bool> >(); DeviceClient deviceClient = DeviceClient.CreateFromConnectionString(deviceInfo.Item2, transport); ConnectionStatus? lastConnectionStatus = null; ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null; int setConnectionStatusChangesHandlerCount = 0; bool assertStatusChange = false; deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) => { if (assertStatusChange) { if (setConnectionStatusChangesHandlerCount == 0) { Assert.AreEqual(ConnectionStatus.Disconnected_Retrying, status); Assert.AreEqual(ConnectionStatusChangeReason.No_Network, statusChangeReason); } else { Assert.AreEqual(ConnectionStatus.Connected, status); Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, statusChangeReason); } } lastConnectionStatus = status; lastConnectionStatusChangeReason = statusChangeReason; setConnectionStatusChangesHandlerCount++; }); await deviceClient.SetMethodHandlerAsync(MethodName, (request, context) => { assertResult.SetResult(new Tuple <bool, bool>(request.Name.Equals(MethodName), request.DataAsJson.Equals(ServiceRequestJson))); return(Task.FromResult(new MethodResponse(Encoding.UTF8.GetBytes(DeviceResponseJson), 200))); }, null); if (transport != Client.TransportType.Http1) { Assert.AreEqual(1, setConnectionStatusChangesHandlerCount); Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus); Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason); } await ServiceSendMethodAndVerifyResponse(deviceInfo.Item1, MethodName, DeviceResponseJson, ServiceRequestJson, assertResult); // allow time for connection recovery setConnectionStatusChangesHandlerCount = 0; assertStatusChange = true; // send error command await deviceClient.SendEventAsync(TestUtil.ComposeErrorInjectionProperties(faultType, reason, delayInSec)); Stopwatch sw = new Stopwatch(); sw.Start(); while (sw.Elapsed.Minutes < 3 && setConnectionStatusChangesHandlerCount < 2) { await Task.Delay(1000); } assertResult = new TaskCompletionSource <Tuple <bool, bool> >(); await ServiceSendMethodAndVerifyResponse(deviceInfo.Item1, MethodName, DeviceResponseJson, ServiceRequestJson, assertResult); setConnectionStatusChangesHandlerCount = 0; assertStatusChange = false; await deviceClient.SetMethodHandlerAsync(MethodName, null, null); if (transport != Client.TransportType.Http1) { Assert.AreEqual(1, setConnectionStatusChangesHandlerCount); Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus); Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason); } TestUtil.RemoveDevice(deviceInfo.Item1, registryManager); }
private async Task ReceiveMessageMuxedOverAmqp( Client.TransportType transport, int poolSize, int devicesCount, ConnectionStringAuthScope authScope = ConnectionStringAuthScope.Device) { var transportSettings = new ITransportSettings[] { new AmqpTransportSettings(transport) { AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings() { MaxPoolSize = unchecked ((uint)poolSize), Pooling = true } } }; ICollection <DeviceClient> deviceClients = new List <DeviceClient>(); Dictionary <DeviceClient, int> deviceClientConnectionStatusChangeCount = new Dictionary <DeviceClient, int>(); ServiceClient serviceClient = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString); try { _log.WriteLine($"{nameof(MessageReceiveE2EMultiplexingTests)}: Starting the test execution for {devicesCount} devices"); await serviceClient.OpenAsync().ConfigureAwait(false); for (int i = 0; i < devicesCount; i++) { ConnectionStatus? lastConnectionStatus = null; ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null; int setConnectionStatusChangesHandlerCount = 0; TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{DevicePrefix}_{i}_").ConfigureAwait(false); DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope); deviceClients.Add(deviceClient); deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) => { setConnectionStatusChangesHandlerCount++; lastConnectionStatus = status; lastConnectionStatusChangeReason = statusChangeReason; _log.WriteLine($"{nameof(MessageReceiveE2EMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}"); deviceClientConnectionStatusChangeCount[deviceClient] = setConnectionStatusChangesHandlerCount; }); _log.WriteLine($"{nameof(MessageReceiveE2EMultiplexingTests)}: Preparing to receive message for device {i}"); await deviceClient.OpenAsync().ConfigureAwait(false); string payload, messageId, p1Value; Message msg = MessageReceive.ComposeC2DTestMessage(out payload, out messageId, out p1Value); await serviceClient.SendAsync(testDevice.Id, msg).ConfigureAwait(false); await MessageReceive.VerifyReceivedC2DMessageAsync(transport, deviceClient, payload, p1Value).ConfigureAwait(false); } } finally { await serviceClient.CloseAsync().ConfigureAwait(false); serviceClient.Dispose(); // Close and dispose all of the device client instances here foreach (DeviceClient deviceClient in deviceClients) { await deviceClient.CloseAsync().ConfigureAwait(false); // The connection status change count should be 2: connect (open) and disabled (close) Assert.IsTrue(deviceClientConnectionStatusChangeCount[deviceClient] == 2); _log.WriteLine($"{nameof(MessageReceiveE2EMultiplexingTests)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}"); deviceClient.Dispose(); } } }
public ConnectionStatusChangeResult ChangeTo(ConnectionType connectionType, ConnectionStatus toState, ConnectionStatus?fromState = null) { if (toState == ConnectionStatus.Disabled) { return(Disable(connectionType, true)); } var changeResult = new ConnectionStatusChangeResult { ClientStatus = this.status }; lock (lockObject) { if (connections.ContainsKey(connectionType)) { ConnectionStatus existingConnectionState = connections[connectionType].Item1; CancellationTokenSource existingCancellationTokenSource = connections[connectionType].Item2; if (((existingConnectionState != ConnectionStatus.Disconnected && existingConnectionState != ConnectionStatus.Disabled) || toState == ConnectionStatus.Connected) && (!fromState.HasValue || fromState.Value == existingConnectionState)) { CancellationTokenSource changeStatusCancellationTokenSource = null; if (toState == ConnectionStatus.Disconnected_Retrying) { changeStatusCancellationTokenSource = new CancellationTokenSource(); } connections[connectionType] = new Tuple <ConnectionStatus, CancellationTokenSource>(toState, changeStatusCancellationTokenSource); changeResult.StatusChangeCancellationTokenSource = changeStatusCancellationTokenSource; changeResult.IsConnectionStatusChanged = true; // When status is changed from Disconnected_Retrying, dispose cancellation token as it is no longer valid. if (existingConnectionState == ConnectionStatus.Disconnected_Retrying && existingCancellationTokenSource != null) { existingCancellationTokenSource.Dispose(); } } } else { if (toState == ConnectionStatus.Connected && (!fromState.HasValue || fromState.Value == ConnectionStatus.Disconnected)) { connections.Add(connectionType, new Tuple <ConnectionStatus, CancellationTokenSource>(toState, null)); changeResult.IsConnectionStatusChanged = true; } } if (changeResult.IsConnectionStatusChanged) { Tuple <ConnectionStatus, ConnectionStatus> beforeAndAfterState = UpdateDeviceClientState(); changeResult.IsClientStatusChanged = beforeAndAfterState.Item1 != beforeAndAfterState.Item2; changeResult.ClientStatus = beforeAndAfterState.Item2; } } return(changeResult); }
private async Task SendMessageModuleMuxedOverAmqp( Client.TransportType transport, int poolSize, int devicesCount, bool useSameDevice = false) { var transportSettings = new ITransportSettings[] { new AmqpTransportSettings(transport) { AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings() { MaxPoolSize = unchecked ((uint)poolSize), Pooling = true } } }; ICollection <ModuleClient> moduleClients = new List <ModuleClient>(); Dictionary <ModuleClient, int> moduleClientConnectionStatusChangeCount = new Dictionary <ModuleClient, int>(); try { _log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}: Starting the test execution for {devicesCount} modules"); for (int i = 0; i < devicesCount; i++) { ConnectionStatus? lastConnectionStatus = null; ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null; int setConnectionStatusChangesHandlerCount = 0; string devicePrefix = useSameDevice ? DevicePrefix : $"{DevicePrefix}_{i}_"; string modulePrefix = useSameDevice ? $"{ModulePrefix}_{i}_" : ModulePrefix; TestModule testModule = await TestModule.GetTestModuleAsync(devicePrefix, modulePrefix).ConfigureAwait(false); ModuleClient moduleClient = ModuleClient.CreateFromConnectionString(testModule.ConnectionString, transportSettings); moduleClients.Add(moduleClient); moduleClient.SetConnectionStatusChangesHandler((status, statusChangeReason) => { setConnectionStatusChangesHandlerCount++; lastConnectionStatus = status; lastConnectionStatusChangeReason = statusChangeReason; _log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}"); moduleClientConnectionStatusChangeCount[moduleClient] = setConnectionStatusChangesHandlerCount; }); _log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}: Preparing to send message for module {i}"); await moduleClient.OpenAsync().ConfigureAwait(false); await MessageSend.SendSingleMessageModuleAndVerifyAsync(moduleClient, testModule.DeviceId).ConfigureAwait(false); } } finally { // Close and dispose all of the module client instances here foreach (ModuleClient moduleClient in moduleClients) { await moduleClient.CloseAsync().ConfigureAwait(false); // The connection status change count should be 2: connect (open) and disabled (close) Assert.IsTrue(moduleClientConnectionStatusChangeCount[moduleClient] == 2); _log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}: Disposing moduleClient {TestLogging.GetHashCode(moduleClient)}"); moduleClient.Dispose(); } } }
private async Task CheckPendingMessages() { LOG("CheckPendingMessages()"); TMessage[] messages; lock ( LockObject ) { if (Status != ConnectionStatus.Connected) { // Connection not ready. Leave in queue & send later return; } if (PendingMessages.Count == 0) { // Nothing to send return; } messages = PendingMessages.ToArray(); PendingMessages.Clear(); Status = ConnectionStatus.Sending; } await OnStatusChanged.Invoke(ConnectionStatus.Sending); try { var rootMessage = CreateRootMessage_MessagesList(messages); await SendRootMessage(rootMessage); } catch (System.Exception ex) { await OnInternalError.Invoke("Could not send messages", ex); } bool launchCloseConnection; string connectionID = null; { ConnectionStatus?newStatus = null; lock ( LockObject ) { switch (Status) { case ConnectionStatus.Sending: // Switch back to 'Connected' newStatus = Status = ConnectionStatus.Connected; launchCloseConnection = false; break; case ConnectionStatus.Closing: // Stop has been requested while sending launchCloseConnection = true; connectionID = ConnectionID; ConnectionID = null; break; default: // Should not happen FAIL("The property 'Status' is not supposed to have value '" + Status + "' here"); newStatus = Status = ConnectionStatus.Closing; launchCloseConnection = true; connectionID = ConnectionID; ConnectionID = null; break; } } if (newStatus != null) { await OnStatusChanged.Invoke(newStatus.Value); } } if (launchCloseConnection) { var dummy = CloseConnection(connectionID); // NB: No await so can be performed in background } }
private async Task DeviceClient_Gives_ConnectionStatus_DeviceDisabled_Base( Client.TransportType protocol, Func <RegistryManager, string, Task> registryManagerOperation) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix + $"_{Guid.NewGuid()}").ConfigureAwait(false); string deviceConnectionString = testDevice.ConnectionString; var config = new Configuration.IoTHub.DeviceConnectionStringParser(deviceConnectionString); string deviceId = config.DeviceID; ConnectionStatus? status = null; ConnectionStatusChangeReason?statusChangeReason = null; int deviceDisabledReceivedCount = 0; using (DeviceClient deviceClient = DeviceClient.CreateFromConnectionString(deviceConnectionString, protocol)) { ConnectionStatusChangesHandler statusChangeHandler = (s, r) => { if (r == ConnectionStatusChangeReason.Device_Disabled) { status = s; statusChangeReason = r; deviceDisabledReceivedCount++; } }; deviceClient.SetConnectionStatusChangesHandler(statusChangeHandler); _log.WriteLine($"Created {nameof(DeviceClient)} ID={TestLogging.IdOf(deviceClient)}"); Console.WriteLine("DeviceClient OpenAsync."); await deviceClient.OpenAsync().ConfigureAwait(false); // Receiving the module twin should succeed right now. Console.WriteLine("ModuleClient GetTwinAsync."); var twin = await deviceClient.GetTwinAsync().ConfigureAwait(false); Assert.IsNotNull(twin); // Delete/disable the device in IoT Hub. This should trigger the ConnectionStatusChangesHandler. using (RegistryManager registryManager = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString)) { await registryManagerOperation(registryManager, deviceId).ConfigureAwait(false); } // Artificial sleep waiting for the connection status change handler to get triggered. int sleepCount = 50; for (int i = 0; i < sleepCount; i++) { await Task.Delay(TimeSpan.FromSeconds(10)).ConfigureAwait(false); if (deviceDisabledReceivedCount == 1) { break; } } Assert.AreEqual(1, deviceDisabledReceivedCount); Assert.AreEqual(ConnectionStatus.Disconnected, status); Assert.AreEqual(ConnectionStatusChangeReason.Device_Disabled, statusChangeReason); } }
internal async Task SendMessageRecovery(Client.TransportType transport, string faultType, string reason, int delayInSec, int durationInSec = 0, int retryDurationInMilliSec = 240000) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false); var deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport); PartitionReceiver eventHubReceiver = await CreateEventHubReceiver(testDevice.Id).ConfigureAwait(false); try { deviceClient.OperationTimeoutInMilliseconds = (uint)retryDurationInMilliSec; ConnectionStatus? lastConnectionStatus = null; ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null; int setConnectionStatusChangesHandlerCount = 0; deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) => { lastConnectionStatus = status; lastConnectionStatusChangeReason = statusChangeReason; setConnectionStatusChangesHandlerCount++; }); await deviceClient.OpenAsync().ConfigureAwait(false); if (transport != Client.TransportType.Http1) { Assert.AreEqual(1, setConnectionStatusChangesHandlerCount); Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus); Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason); } string payload, p1Value; Client.Message testMessage = ComposeD2CTestMessage(out payload, out p1Value); await deviceClient.SendEventAsync(testMessage).ConfigureAwait(false); bool isReceived = false; Stopwatch sw = new Stopwatch(); sw.Start(); while (!isReceived && sw.Elapsed.Minutes < 1) { var events = await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5)).ConfigureAwait(false); isReceived = VerifyTestMessage(events, testDevice.Id, payload, p1Value); } sw.Stop(); // send error command and clear eventHubReceiver of the fault injection message await deviceClient.SendEventAsync(FaultInjection.ComposeErrorInjectionProperties(faultType, reason, delayInSec, durationInSec)).ConfigureAwait(false); await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5)).ConfigureAwait(false); Thread.Sleep(1000); testMessage = ComposeD2CTestMessage(out payload, out p1Value); await deviceClient.SendEventAsync(testMessage).ConfigureAwait(false); sw.Reset(); sw.Start(); while (!isReceived && sw.Elapsed.Minutes < 1) { var events = await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5)).ConfigureAwait(false); isReceived = VerifyTestMessage(events, testDevice.Id, payload, p1Value); } sw.Stop(); await deviceClient.CloseAsync().ConfigureAwait(false); if (transport != Client.TransportType.Http1) { Assert.AreEqual(2, setConnectionStatusChangesHandlerCount); Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus); Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason); } } finally { await deviceClient.CloseAsync().ConfigureAwait(false); await eventHubReceiver.CloseAsync().ConfigureAwait(false); } }
public ConnectionStatusChangeResult ChangeTo(ConnectionType connectionType, ConnectionStatus toState, ConnectionStatus?fromState = null, CancellationTokenSource cancellationTokenSource = null) { if (toState == ConnectionStatus.Disabled) { return(Disable(connectionType, true)); } Tuple <ConnectionStatus, CancellationTokenSource> connectionValue; if (toState == ConnectionStatus.Disconnected_Retrying) { if (cancellationTokenSource == null) { throw new ArgumentNullException($"{nameof(cancellationTokenSource)} should be provided for retrying state."); } connectionValue = Tuple.Create(toState, cancellationTokenSource); } else { connectionValue = new Tuple <ConnectionStatus, CancellationTokenSource>(toState, null); } var changeResult = new ConnectionStatusChangeResult { ClientStatus = this.status }; lock (lockObject) { if (connections.ContainsKey(connectionType)) { ConnectionStatus existingConnectionState = connections[connectionType].Item1; if (((existingConnectionState != ConnectionStatus.Disconnected && existingConnectionState != ConnectionStatus.Disabled) || toState == ConnectionStatus.Connected) && (!fromState.HasValue || fromState.Value == existingConnectionState)) { connections[connectionType] = connectionValue; changeResult.IsConnectionStatusChanged = true; } } else { if (toState == ConnectionStatus.Connected && (!fromState.HasValue || fromState.Value == ConnectionStatus.Disconnected)) { connections.Add(connectionType, connectionValue); changeResult.IsConnectionStatusChanged = true; } } if (changeResult.IsConnectionStatusChanged) { Tuple <ConnectionStatus, ConnectionStatus> beforeAndAfterState = UpdateDeviceClientState(); changeResult.IsClientStatusChanged = beforeAndAfterState.Item1 != beforeAndAfterState.Item2; changeResult.ClientStatus = beforeAndAfterState.Item2; } } return(changeResult); }
// Error injection template method. public static async Task TestErrorInjectionAsync( string devicePrefix, TestDeviceType type, Client.TransportType transport, string proxyAddress, string faultType, string reason, int delayInSec, int durationInSec, Func <DeviceClient, TestDevice, Task> initOperation, Func <DeviceClient, TestDevice, Task> testOperation, Func <Task> cleanupOperation, MsTestLogger logger) { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(logger, devicePrefix, type).ConfigureAwait(false); ITransportSettings transportSettings = CreateTransportSettingsFromName(transport, proxyAddress); DeviceClient deviceClient = testDevice.CreateDeviceClient(new ITransportSettings[] { transportSettings }); ConnectionStatus? lastConnectionStatus = null; ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null; int connectionStatusChangeCount = 0; deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) => { connectionStatusChangeCount++; lastConnectionStatus = status; lastConnectionStatusChangeReason = statusChangeReason; logger.Trace($"{nameof(FaultInjection)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={connectionStatusChangeCount}"); }); var watch = new Stopwatch(); try { await deviceClient.OpenAsync().ConfigureAwait(false); if (transport != Client.TransportType.Http1) { Assert.IsTrue(connectionStatusChangeCount >= 1, $"The expected connection status change should be equal or greater than 1 but was {connectionStatusChangeCount}"); // Normally one connection but in some cases, due to network issues we might have already retried several times to connect. Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Connected} but was {lastConnectionStatus}"); Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Connection_Ok} but was {lastConnectionStatusChangeReason}"); } await initOperation(deviceClient, testDevice).ConfigureAwait(false); logger.Trace($">>> {nameof(FaultInjection)} Testing baseline"); await testOperation(deviceClient, testDevice).ConfigureAwait(false); int countBeforeFaultInjection = connectionStatusChangeCount; watch.Start(); logger.Trace($">>> {nameof(FaultInjection)} Testing fault handling"); await ActivateFaultInjectionAsync(transport, faultType, reason, delayInSec, durationInSec, deviceClient, logger).ConfigureAwait(false); logger.Trace($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {delayInSec} seconds."); await Task.Delay(TimeSpan.FromSeconds(delayInSec)).ConfigureAwait(false); // For disconnect type faults, the device should disconnect and recover. if (FaultShouldDisconnect(faultType)) { logger.Trace($"{nameof(FaultInjection)}: Confirming fault injection has been actived."); // Check that service issued the fault to the faulting device bool isFaulted = false; for (int i = 0; i < LatencyTimeBufferInSec; i++) { if (connectionStatusChangeCount > countBeforeFaultInjection) { isFaulted = true; break; } await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); } Assert.IsTrue(isFaulted, $"The device {testDevice.Id} did not get faulted with fault type: {faultType}"); logger.Trace($"{nameof(FaultInjection)}: Confirmed fault injection has been actived."); // Check the device is back online logger.Trace($"{nameof(FaultInjection)}: Confirming device back online."); for (int i = 0; lastConnectionStatus != ConnectionStatus.Connected && i < durationInSec + LatencyTimeBufferInSec; i++) { await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); } Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus, $"{testDevice.Id} did not reconnect."); logger.Trace($"{nameof(FaultInjection)}: Confirmed device back online."); // Perform the test operation. logger.Trace($">>> {nameof(FaultInjection)}: Performing test operation for device {testDevice.Id}."); await testOperation(deviceClient, testDevice).ConfigureAwait(false); } else { logger.Trace($"{nameof(FaultInjection)}: Performing test operation while fault injection is being activated."); // Perform the test operation for the faulted device multi times. for (int i = 0; i < LatencyTimeBufferInSec; i++) { logger.Trace($">>> {nameof(FaultInjection)}: Performing test operation for device - Run {i}."); await testOperation(deviceClient, testDevice).ConfigureAwait(false); await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); } } await deviceClient.CloseAsync().ConfigureAwait(false); if (transport != Client.TransportType.Http1) { if (FaultInjection.FaultShouldDisconnect(faultType)) { // 4 is the minimum notification count: connect, fault, reconnect, disable. // There are cases where the retry must be timed out (i.e. very likely for MQTT where otherwise // we would attempt to send the fault injection forever.) Assert.IsTrue(connectionStatusChangeCount >= 4, $"The expected connection status change count for {testDevice.Id} should be equal or greater than 4 but was {connectionStatusChangeCount}"); } else { // 2 is the minimum notification count: connect, disable. // We will monitor the test environment real network stability and switch to >=2 if necessary to // account for real network issues. Assert.IsTrue(connectionStatusChangeCount == 2, $"The expected connection status change count for {testDevice.Id} should be 2 but was {connectionStatusChangeCount}"); } Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Disabled} but was {lastConnectionStatus}"); Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Client_Close} but was {lastConnectionStatusChangeReason}"); } } finally { await cleanupOperation().ConfigureAwait(false); logger.Trace($"{nameof(FaultInjection)}: Disposing deviceClient {TestLogger.GetHashCode(deviceClient)}"); deviceClient.Dispose(); watch.Stop(); int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds; if (timeToFinishFaultInjection > 0) { logger.Trace($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed."); await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false); } } }
internal PrivateLinkServiceConnectionState(ConnectionStatus?status, string description, ActionsRequired?actionsRequired) { Status = status; Description = description; ActionsRequired = actionsRequired; }
private async Task ModuleClient_Gives_ConnectionStatus_DeviceDisabled_Base( Client.TransportType protocol, Func <RegistryManager, string, Task> registryManagerOperation) { AmqpTransportSettings amqpTransportSettings = new AmqpTransportSettings(protocol); ITransportSettings[] transportSettings = new ITransportSettings[] { amqpTransportSettings }; TestModule testModule = await TestModule.GetTestModuleAsync(DevicePrefix + $"_{Guid.NewGuid()}", ModulePrefix, Logger).ConfigureAwait(false); ConnectionStatus? status = null; ConnectionStatusChangeReason?statusChangeReason = null; int deviceDisabledReceivedCount = 0; ConnectionStatusChangesHandler statusChangeHandler = (s, r) => { if (r == ConnectionStatusChangeReason.Device_Disabled) { status = s; statusChangeReason = r; deviceDisabledReceivedCount++; } }; using (ModuleClient moduleClient = ModuleClient.CreateFromConnectionString(testModule.ConnectionString, transportSettings)) { moduleClient.SetConnectionStatusChangesHandler(statusChangeHandler); Logger.Trace($"{nameof(ModuleClient_Gives_ConnectionStatus_DeviceDisabled_Base)}: Created {nameof(ModuleClient)} with moduleId={testModule.Id}"); await moduleClient.OpenAsync().ConfigureAwait(false); // Receiving the module twin should succeed right now. Logger.Trace($"{nameof(ModuleClient_Gives_ConnectionStatus_DeviceDisabled_Base)}: ModuleClient GetTwinAsync."); var twin = await moduleClient.GetTwinAsync().ConfigureAwait(false); Assert.IsNotNull(twin); // Delete/disable the device in IoT Hub. using (RegistryManager registryManager = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString)) { await registryManagerOperation(registryManager, testModule.DeviceId).ConfigureAwait(false); } Logger.Trace($"{nameof(ModuleClient_Gives_ConnectionStatus_DeviceDisabled_Base)}: Completed RegistryManager operation."); // Artificial sleep waiting for the connection status change handler to get triggered. int sleepCount = 50; for (int i = 0; i < sleepCount; i++) { await Task.Delay(TimeSpan.FromSeconds(10)).ConfigureAwait(false); if (deviceDisabledReceivedCount == 1) { break; } } Logger.Trace($"{nameof(ModuleClient_Gives_ConnectionStatus_DeviceDisabled_Base)}: Asserting connection status change."); Assert.AreEqual(1, deviceDisabledReceivedCount); Assert.AreEqual(ConnectionStatus.Disconnected, status); Assert.AreEqual(ConnectionStatusChangeReason.Device_Disabled, statusChangeReason); } }
public async Task Duplicated_NoPingpong() { TestDevice testDevice = await TestDevice.GetTestDeviceAsync(Logger, DevicePrefix, TestDeviceType.Sasl).ConfigureAwait(false); Logger.Trace($"{nameof(Duplicated_NoPingpong)}: 2 device client instances with the same deviceId={testDevice.Id}."); using DeviceClient deviceClient1 = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only); using DeviceClient deviceClient2 = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only); Logger.Trace($"{nameof(Duplicated_NoPingpong)}: set device client instance 1 to no retry."); deviceClient1.SetRetryPolicy(new NoRetry()); ConnectionStatus?lastConnectionStatus = null; var connectionStatusChanges = new Dictionary <ConnectionStatus, int>(); deviceClient1.SetConnectionStatusChangesHandler((status, reason) => { connectionStatusChanges.TryGetValue(status, out int count); count++; connectionStatusChanges[status] = count; lastConnectionStatus = status; }); Logger.Trace($"{nameof(FaultInjection_NoRecovery)}: device client instance 1 calling OpenAsync..."); await deviceClient1.OpenAsync().ConfigureAwait(false); await deviceClient1 .SetMethodHandlerAsync( "dummy_method", (methodRequest, userContext) => Task.FromResult(new MethodResponse(200)), deviceClient1) .ConfigureAwait(false); Logger.Trace($"{nameof(FaultInjection_NoRecovery)}: device client instance 2 calling OpenAsync..."); await deviceClient2.OpenAsync().ConfigureAwait(false); await deviceClient2 .SetMethodHandlerAsync( "dummy_method", (methodRequest, userContext) => Task.FromResult(new MethodResponse(200)), deviceClient2) .ConfigureAwait(false); Logger.Trace($"{nameof(Duplicated_NoPingpong)}: waiting device client instance 1 to be kicked off..."); for (int i = 0; i < FaultInjection.LatencyTimeBufferInSec; i++) { if (connectionStatusChanges.ContainsKey(ConnectionStatus.Disconnected)) { break; } await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); } Assert.AreEqual(ConnectionStatus.Disconnected, lastConnectionStatus, $"Excepeted device to be {ConnectionStatus.Disconnected} but was {lastConnectionStatus}."); Assert.IsFalse(connectionStatusChanges.ContainsKey(ConnectionStatus.Disconnected_Retrying), $"Shouldn't get {ConnectionStatus.Disconnected_Retrying} status change."); int connected = connectionStatusChanges[ConnectionStatus.Connected]; Assert.AreEqual(1, connected, $"Should get {ConnectionStatus.Connected} once but was {connected}."); int disconnected = connectionStatusChanges[ConnectionStatus.Disconnected]; Assert.AreEqual(1, disconnected, $"Should get {ConnectionStatus.Disconnected} once but was {disconnected}."); }
private async Task SendMethodAndRespondRecovery(Client.TransportType transport, string faultType, string reason, int delayInSec) { Tuple <string, string> deviceInfo = TestUtil.CreateDevice(DevicePrefix, hostName, registryManager); var assertResult = new TaskCompletionSource <Tuple <bool, bool> >(); DeviceClient deviceClient = DeviceClient.CreateFromConnectionString(deviceInfo.Item2, transport); ConnectionStatus? lastConnectionStatus = null; ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null; int setConnectionStatusChangesHandlerCount = 0; var tcsConnected = new TaskCompletionSource <bool>(); var tcsDisconnected = new TaskCompletionSource <bool>(); deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) => { Debug.WriteLine("Connection Changed to {0} because {1}", status, statusChangeReason); if (status == ConnectionStatus.Disconnected_Retrying) { tcsDisconnected.TrySetResult(true); Assert.AreEqual(ConnectionStatusChangeReason.No_Network, statusChangeReason); } else if (status == ConnectionStatus.Connected) { tcsConnected.TrySetResult(true); } lastConnectionStatus = status; lastConnectionStatusChangeReason = statusChangeReason; setConnectionStatusChangesHandlerCount++; }); await deviceClient.SetMethodHandlerAsync(MethodName, (request, context) => { assertResult.TrySetResult(new Tuple <bool, bool>(request.Name.Equals(MethodName), request.DataAsJson.Equals(ServiceRequestJson))); return(Task.FromResult(new MethodResponse(Encoding.UTF8.GetBytes(DeviceResponseJson), 200))); }, null).ConfigureAwait(false); // assert on successfuly connection await Task.WhenAny( Task.Run(async() => { await Task.Delay(1000).ConfigureAwait(false); }), tcsConnected.Task).ConfigureAwait(false); Assert.IsTrue(tcsConnected.Task.IsCompleted, "Initial connection failed"); if (transport != Client.TransportType.Http1) { Assert.AreEqual(1, setConnectionStatusChangesHandlerCount); Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus); Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason); } // check on normal operation await ServiceSendMethodAndVerifyResponse(deviceInfo.Item1, MethodName, DeviceResponseJson, ServiceRequestJson, assertResult).ConfigureAwait(false); // reset ConnectionStatusChangesHandler data setConnectionStatusChangesHandlerCount = 0; tcsConnected = new TaskCompletionSource <bool>(); tcsDisconnected = new TaskCompletionSource <bool>(); // send error command await deviceClient.SendEventAsync(TestUtil.ComposeErrorInjectionProperties(faultType, reason, delayInSec)).ConfigureAwait(false); // wait for disconnection await Task.WhenAny( Task.Run(async() => { await Task.Delay(TimeSpan.FromSeconds(10)).ConfigureAwait(false); }), tcsDisconnected.Task).ConfigureAwait(false); Assert.IsTrue(tcsDisconnected.Task.IsCompleted, "Error injection did not interrupt the device"); // allow max 3 minutes for connection recovery await Task.WhenAny( Task.Run(async() => { ////////////////////////change it to 30 await Task.Delay(TimeSpan.FromSeconds(10)).ConfigureAwait(false); return(Task.FromResult(true)); }), tcsConnected.Task).ConfigureAwait(false); Assert.IsTrue(tcsConnected.Task.IsCompleted, "Recovery connection failed"); assertResult = new TaskCompletionSource <Tuple <bool, bool> >(); await ServiceSendMethodAndVerifyResponse(deviceInfo.Item1, MethodName, DeviceResponseJson, ServiceRequestJson, assertResult).ConfigureAwait(false); setConnectionStatusChangesHandlerCount = 0; //remove and CloseAsync await deviceClient.SetMethodHandlerAsync(MethodName, null, null).ConfigureAwait(false); if (transport != Client.TransportType.Http1) { Assert.AreEqual(1, setConnectionStatusChangesHandlerCount); Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus); Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason); } await TestUtil.RemoveDeviceAsync(deviceInfo.Item1, registryManager).ConfigureAwait(false); }
private async Task DeviceCombinedClientOperations( Client.TransportType transport, int poolSize, int devicesCount, ConnectionStringAuthScope authScope ) { var transportSettings = new ITransportSettings[] { new AmqpTransportSettings(transport) { AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings() { MaxPoolSize = unchecked ((uint)poolSize), Pooling = true } } }; // Initialize service client for service-side operations ServiceClient serviceClient = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString); ICollection <DeviceClient> deviceClients = new List <DeviceClient>(); Dictionary <DeviceClient, int> deviceClientConnectionStatusChangeCount = new Dictionary <DeviceClient, int>(); try { _log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Starting the test execution for {devicesCount} devices"); for (int i = 0; i < devicesCount; i++) { ConnectionStatus? lastConnectionStatus = null; ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null; int setConnectionStatusChangesHandlerCount = 0; TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{DevicePrefix}_{i}_").ConfigureAwait(false); DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope); deviceClients.Add(deviceClient); // Set the connection status change handler deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) => { setConnectionStatusChangesHandlerCount++; lastConnectionStatus = status; lastConnectionStatusChangeReason = statusChangeReason; _log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}"); deviceClientConnectionStatusChangeCount[deviceClient] = setConnectionStatusChangesHandlerCount; }); // Perform D2C Operation _log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Preparing to send message for device {i}"); await deviceClient.OpenAsync().ConfigureAwait(false); await MessageSend.SendSingleMessageAndVerifyAsync(deviceClient, testDevice.Id).ConfigureAwait(false); // Perform C2D Operation _log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Setting the device {i} to receive C2D message."); string payload, messageId, p1Value; Message msg = MessageReceive.ComposeC2DTestMessage(out payload, out messageId, out p1Value); await serviceClient.SendAsync(testDevice.Id, msg).ConfigureAwait(false); await MessageReceive.VerifyReceivedC2DMessageAsync(transport, deviceClient, payload, p1Value).ConfigureAwait(false); // Invoke direct methods _log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Testing direct methods for device {i}"); Task methodReceivedTask = await MethodOperation.SetDeviceReceiveMethod(deviceClient).ConfigureAwait(false); await Task.WhenAll( MethodOperation.ServiceSendMethodAndVerifyResponse(testDevice.Id), methodReceivedTask).ConfigureAwait(false); // Set reported twin properties _log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Setting reported Twin properties for device {i}"); await TwinOperation.Twin_DeviceSetsReportedPropertyAndGetsItBack(deviceClient).ConfigureAwait(false); // Receive set desired twin properties _log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Received set desired tein properties for device {i}"); var propName = Guid.NewGuid().ToString(); var propValue = Guid.NewGuid().ToString(); Task updateReceivedTask = await TwinOperation.SetTwinPropertyUpdateCallbackHandlerAsync(deviceClient, propName, propValue).ConfigureAwait(false); await Task.WhenAll( TwinOperation.RegistryManagerUpdateDesiredPropertyAsync(testDevice.Id, propName, propValue), updateReceivedTask).ConfigureAwait(false); } } finally { // Close and dispose all of the device client instances here foreach (DeviceClient deviceClient in deviceClients) { await deviceClient.CloseAsync().ConfigureAwait(false); // The connection status change count should be 2: connect (open) and disabled (close) Assert.IsTrue(deviceClientConnectionStatusChangeCount[deviceClient] == 2, $"Connection status change count for deviceClient {TestLogging.GetHashCode(deviceClient)} is {deviceClientConnectionStatusChangeCount[deviceClient]}"); _log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}"); deviceClient.Dispose(); } } }