public async Task FaultInjection_NoRetry_NoRecovery_OpenAsync()
{
using TestDevice testDevice = await TestDevice.GetTestDeviceAsync(Logger, _devicePrefix, TestDeviceType.Sasl).ConfigureAwait(false);
using DeviceClient deviceClient = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only);
Logger.Trace($"{nameof(FaultInjection_NoRetry_NoRecovery_OpenAsync)}: deviceId={testDevice.Id}");
deviceClient.SetRetryPolicy(new NoRetry());
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
var connectionStatusChanges = new Dictionary <ConnectionStatus, int>();
deviceClient.SetConnectionStatusChangesHandler((status, reason) =>
{
connectionStatusChanges.TryGetValue(status, out int count);
count++;
connectionStatusChanges[status] = count;
lastConnectionStatus = status;
lastConnectionStatusChangeReason = reason;
});
Logger.Trace($"{nameof(FaultInjection_NoRetry_NoRecovery_OpenAsync)}: calling OpenAsync...");
await deviceClient.OpenAsync().ConfigureAwait(false);
Logger.Trace($"{nameof(FaultInjection_NoRetry_NoRecovery_OpenAsync)}: injecting fault {FaultInjection.FaultType_Tcp}...");
await FaultInjection
.ActivateFaultInjectionAsync(
Client.TransportType.Amqp_Tcp_Only,
FaultInjection.FaultType_Tcp,
FaultInjection.FaultCloseReason_Boom,
FaultInjection.DefaultFaultDelay,
FaultInjection.DefaultFaultDuration,
deviceClient,
Logger)
.ConfigureAwait(false);
await Task.Delay(FaultInjection.DefaultFaultDelay).ConfigureAwait(false);
Logger.Trace($"{nameof(FaultInjection_NoRetry_NoRecovery_OpenAsync)}: waiting fault injection occurs...");
var sw = Stopwatch.StartNew();
while (sw.Elapsed < FaultInjection.LatencyTimeBuffer)
{
if (connectionStatusChanges.ContainsKey(ConnectionStatus.Disconnected))
{
break;
}
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
sw.Reset();
lastConnectionStatus.Should().Be(ConnectionStatus.Disconnected, $"Expected device to be {ConnectionStatus.Disconnected} but was {lastConnectionStatus}.");
lastConnectionStatusChangeReason.Should().Be(ConnectionStatusChangeReason.Retry_Expired, $"Expected device to be {ConnectionStatusChangeReason.Retry_Expired} but was {lastConnectionStatusChangeReason}.");
connectionStatusChanges.Should().NotContainKey(ConnectionStatus.Disconnected_Retrying, $"Shouldn't get {ConnectionStatus.Disconnected_Retrying} status change.");
int connected = connectionStatusChanges[ConnectionStatus.Connected];
connected.Should().Be(1, $"Should get {ConnectionStatus.Connected} once but got it {connected} times.");
int disconnected = connectionStatusChanges[ConnectionStatus.Disconnected];
disconnected.Should().Be(1, $"Should get {ConnectionStatus.Disconnected} once but got it {disconnected} times.");
}
public async Task DuplicateDevice_NoRetry_NoPingpong_OpenAsync()
{
using TestDevice testDevice = await TestDevice.GetTestDeviceAsync(Logger, _devicePrefix, TestDeviceType.Sasl).ConfigureAwait(false);
Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: 2 device client instances with the same deviceId={testDevice.Id}.");
using DeviceClient deviceClient1 = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only);
using DeviceClient deviceClient2 = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only);
Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: set device client instance 1 to no retry.");
deviceClient1.SetRetryPolicy(new NoRetry());
ConnectionStatus?lastConnectionStatusDevice1 = null;
var connectionStatusChangesDevice1 = new Dictionary <ConnectionStatus, int>();
deviceClient1.SetConnectionStatusChangesHandler((status, reason) =>
{
connectionStatusChangesDevice1.TryGetValue(status, out int count);
count++;
connectionStatusChangesDevice1[status] = count;
lastConnectionStatusDevice1 = status;
});
ConnectionStatus?lastConnectionStatusDevice2 = null;
var connectionStatusChangesDevice2 = new Dictionary <ConnectionStatus, int>();
deviceClient2.SetConnectionStatusChangesHandler((status, reason) =>
{
connectionStatusChangesDevice2.TryGetValue(status, out int count);
count++;
connectionStatusChangesDevice2[status] = count;
lastConnectionStatusDevice2 = status;
});
Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: device client instance 1 calling OpenAsync...");
await deviceClient1.OpenAsync().ConfigureAwait(false);
await deviceClient1
.SetMethodHandlerAsync(
"empty_method",
(methodRequest, userContext) => Task.FromResult(new MethodResponse(200)),
deviceClient1)
.ConfigureAwait(false);
Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: device client instance 2 calling OpenAsync...");
await deviceClient2.OpenAsync().ConfigureAwait(false);
await deviceClient2
.SetMethodHandlerAsync(
"empty_method",
(methodRequest, userContext) => Task.FromResult(new MethodResponse(200)),
deviceClient2)
.ConfigureAwait(false);
Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: waiting device client instance 1 to be kicked off...");
var sw = Stopwatch.StartNew();
while (sw.Elapsed < FaultInjection.LatencyTimeBuffer)
{
if (connectionStatusChangesDevice1.ContainsKey(ConnectionStatus.Disconnected))
{
break;
}
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
sw.Reset();
lastConnectionStatusDevice1.Should().Be(ConnectionStatus.Disconnected, $"Excpected device 1 to be {ConnectionStatus.Disconnected} but was {lastConnectionStatusDevice1}.");
connectionStatusChangesDevice1.Should().NotContainKey(ConnectionStatus.Disconnected_Retrying, $"Shouldn't get {ConnectionStatus.Disconnected_Retrying} status change.");
int connected = connectionStatusChangesDevice1[ConnectionStatus.Connected];
connected.Should().Be(1, $"Should get {ConnectionStatus.Connected} once but got it {connected} times.");
int disconnected = connectionStatusChangesDevice1[ConnectionStatus.Disconnected];
disconnected.Should().Be(1, $"Should get {ConnectionStatus.Disconnected} once but got it {disconnected} times.");
lastConnectionStatusDevice2.Should().Be(ConnectionStatus.Connected, $"Expected device 2 to be {ConnectionStatus.Connected} but was {lastConnectionStatusDevice2}.");
}
private async Task Twin_DeviceDesiredPropertyUpdateRecovery(Client.TransportType transport, string faultType, string reason, int delayInSec)
{
var tcs = new TaskCompletionSource <bool>();
var propName = Guid.NewGuid().ToString();
var propValue = Guid.NewGuid().ToString();
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);
RegistryManager registryManager = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);
var deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport);
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int setConnectionStatusChangesHandlerCount = 0;
var tcsConnected = new TaskCompletionSource <bool>();
var tcsDisconnected = new TaskCompletionSource <bool>();
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
_log.WriteLine("Connection Changed to {0} because {1}", status, statusChangeReason);
if (status == ConnectionStatus.Disconnected_Retrying)
{
tcsDisconnected.TrySetResult(true);
Assert.AreEqual(ConnectionStatusChangeReason.No_Network, statusChangeReason);
}
else if (status == ConnectionStatus.Connected)
{
tcsConnected.TrySetResult(true);
}
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
setConnectionStatusChangesHandlerCount++;
});
await deviceClient.SetDesiredPropertyUpdateCallbackAsync((patch, context) =>
{
return(Task.Run(() =>
{
try
{
Assert.AreEqual(patch[propName].ToString(), propValue);
}
catch (Exception e)
{
tcs.SetException(e);
}
finally
{
tcs.SetResult(true);
}
}));
}, null).ConfigureAwait(false);
// assert on successful connection
await Task.WhenAny(
Task.Run(async() =>
{
await Task.Delay(1000).ConfigureAwait(false);
}), tcsConnected.Task).ConfigureAwait(false);
Assert.IsTrue(tcsConnected.Task.IsCompleted, "Initial connection failed");
if (transport != Client.TransportType.Http1)
{
Assert.AreEqual(1, setConnectionStatusChangesHandlerCount);
Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason);
}
var twinPatch = new Twin();
twinPatch.Properties.Desired[propName] = propValue;
await registryManager.UpdateTwinAsync(testDevice.Id, twinPatch, "*").ConfigureAwait(false);
await tcs.Task.ConfigureAwait(false);
// send error command
await deviceClient.SendEventAsync(FaultInjection.ComposeErrorInjectionProperties(faultType, reason, delayInSec)).ConfigureAwait(false);
// reset ConnectionStatusChangesHandler data
setConnectionStatusChangesHandlerCount = 0;
tcsConnected = new TaskCompletionSource <bool>();
tcsDisconnected = new TaskCompletionSource <bool>();
// wait for disconnection
await Task.WhenAny(
Task.Run(async() =>
{
await Task.Delay(TimeSpan.FromSeconds(10)).ConfigureAwait(false);
}), tcsDisconnected.Task).ConfigureAwait(false);
Assert.IsTrue(tcsDisconnected.Task.IsCompleted, "Error injection did not interrupt the device");
await Task.WhenAny(
Task.Run(async() =>
{
await Task.Delay(TimeSpan.FromSeconds(MaximumRecoveryTimeSeconds)).ConfigureAwait(false);
return(Task.FromResult(true));
}), tcsConnected.Task).ConfigureAwait(false);
Assert.IsTrue(tcsConnected.Task.IsCompleted, "Recovery connection failed");
tcs = new TaskCompletionSource <bool>();
twinPatch = new Twin();
twinPatch.Properties.Desired[propName] = propValue;
await registryManager.UpdateTwinAsync(testDevice.Id, twinPatch, "*").ConfigureAwait(false);
await tcs.Task.ConfigureAwait(false);
await deviceClient.CloseAsync().ConfigureAwait(false);
await registryManager.CloseAsync().ConfigureAwait(false);
}
private async Task _Twin_DeviceDesiredPropertyUpdateRecovery(Client.TransportType transport, string faultType, string reason, int delayInSec)
{
var tcs = new TaskCompletionSource <bool>();
var propName = Guid.NewGuid().ToString();
var propValue = Guid.NewGuid().ToString();
Tuple <string, string> deviceInfo = TestUtil.CreateDevice(DevicePrefix, hostName, registryManager);
var deviceClient = DeviceClient.CreateFromConnectionString(deviceInfo.Item2, transport);
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int setConnectionStatusChangesHandlerCount = 0;
var tcsConnected = new TaskCompletionSource <bool>();
var tcsDisconnected = new TaskCompletionSource <bool>();
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
if (status == ConnectionStatus.Disconnected_Retrying)
{
tcsDisconnected.TrySetResult(true);
Assert.AreEqual(ConnectionStatusChangeReason.No_Network, statusChangeReason);
}
else if (status == ConnectionStatus.Connected)
{
tcsConnected.TrySetResult(true);
}
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
setConnectionStatusChangesHandlerCount++;
});
await deviceClient.SetDesiredPropertyUpdateCallbackAsync((patch, context) =>
{
return(Task.Run(() =>
{
try
{
Assert.AreEqual(patch[propName].ToString(), propValue);
}
catch (Exception e)
{
tcs.SetException(e);
}
finally
{
tcs.SetResult(true);
}
}));
}, null);
// assert on successfuly connection
await Task.WhenAny(
Task.Run(async() =>
{
await Task.Delay(1000);
}), tcsConnected.Task);
Assert.IsTrue(tcsConnected.Task.IsCompleted, "Initial connection failed");
if (transport != Client.TransportType.Http1)
{
Assert.AreEqual(1, setConnectionStatusChangesHandlerCount);
Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason);
}
var twinPatch = new Twin();
twinPatch.Properties.Desired[propName] = propValue;
await registryManager.UpdateTwinAsync(deviceInfo.Item1, twinPatch, "*");
await tcs.Task;
// send error command
await deviceClient.SendEventAsync(TestUtil.ComposeErrorInjectionProperties(faultType, reason, delayInSec));
// reset ConnectionStatusChangesHandler data
setConnectionStatusChangesHandlerCount = 0;
tcsConnected = new TaskCompletionSource <bool>();
tcsDisconnected = new TaskCompletionSource <bool>();
// wait for disconnection
await Task.WhenAny(
Task.Run(async() =>
{
await Task.Delay(TimeSpan.FromSeconds(10));
}), tcsDisconnected.Task);
Assert.IsTrue(tcsDisconnected.Task.IsCompleted, "Error injection did not interrupt the device");
// allow max 30s for connection recovery
await Task.WhenAny(
Task.Run(async() =>
{
await Task.Delay(TimeSpan.FromSeconds(10));
return(Task.FromResult(true));
}), tcsConnected.Task);
Assert.IsTrue(tcsConnected.Task.IsCompleted, "Recovery connection failed");
tcs = new TaskCompletionSource <bool>();
twinPatch = new Twin();
twinPatch.Properties.Desired[propName] = propValue;
await registryManager.UpdateTwinAsync(deviceInfo.Item1, twinPatch, "*");
await tcs.Task;
await deviceClient.CloseAsync();
await TestUtil.RemoveDeviceAsync(deviceInfo.Item1, registryManager);
}
private async Task SendMessageRecovery(Client.TransportType transport,
string faultType, string reason, int delayInSec, int durationInSec = 0, int retryDurationInMilliSec = 240000)
{
await sequentialTestSemaphore.WaitAsync();
Tuple <string, string> deviceInfo = TestUtil.CreateDevice(DevicePrefix, hostName, registryManager);
var deviceClient = DeviceClient.CreateFromConnectionString(deviceInfo.Item2, transport);
EventHubClient eventHubClient;
EventHubReceiver eventHubReceiver = CreateEventHubReceiver(deviceInfo.Item1, out eventHubClient);
try
{
deviceClient.OperationTimeoutInMilliseconds = (uint)retryDurationInMilliSec;
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int setConnectionStatusChangesHandlerCount = 0;
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
setConnectionStatusChangesHandlerCount++;
});
await deviceClient.OpenAsync();
if (transport != Client.TransportType.Http1)
{
Assert.AreEqual(1, setConnectionStatusChangesHandlerCount);
Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason);
}
string payload, p1Value;
Client.Message testMessage = ComposeD2CTestMessage(out payload, out p1Value);
await deviceClient.SendEventAsync(testMessage);
bool isReceived = false;
Stopwatch sw = new Stopwatch();
sw.Start();
while (!isReceived && sw.Elapsed.Minutes < 1)
{
var events = await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5));
isReceived = VerifyTestMessage(events, deviceInfo.Item1, payload, p1Value);
}
sw.Stop();
// send error command and clear eventHubReceiver of the fault injection message
await deviceClient.SendEventAsync(TestUtil.ComposeErrorInjectionProperties(faultType, reason,
delayInSec,
durationInSec));
await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5));
Thread.Sleep(1000);
testMessage = ComposeD2CTestMessage(out payload, out p1Value);
await deviceClient.SendEventAsync(testMessage);
sw.Reset();
sw.Start();
while (!isReceived && sw.Elapsed.Minutes < 1)
{
var events = await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5));
isReceived = VerifyTestMessage(events, deviceInfo.Item1, payload, p1Value);
}
sw.Stop();
await deviceClient.CloseAsync();
if (transport != Client.TransportType.Http1)
{
Assert.AreEqual(2, setConnectionStatusChangesHandlerCount);
Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason);
}
}
finally
{
await deviceClient.CloseAsync();
await eventHubReceiver.CloseAsync();
await eventHubClient.CloseAsync();
await TestUtil.RemoveDeviceAsync(deviceInfo.Item1, registryManager);
sequentialTestSemaphore.Release(1);
}
}
private async Task SendMethodAndRespondMuxedOverAmqp(
ConnectionStringAuthScope authScope,
Client.TransportType transport,
int poolSize,
int devicesCount,
Func <DeviceClient, Task <Task> > setDeviceReceiveMethod
)
{
var transportSettings = new ITransportSettings[]
{
new AmqpTransportSettings(transport)
{
AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
{
MaxPoolSize = unchecked ((uint)poolSize),
Pooling = true
}
}
};
ICollection <DeviceClient> deviceClients = new List <DeviceClient>();
Dictionary <DeviceClient, int> deviceClientConnectionStatusChangeCount = new Dictionary <DeviceClient, int>();
try
{
for (int i = 0; i < devicesCount; i++)
{
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int setConnectionStatusChangesHandlerCount = 0;
TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{DevicePrefix}_{i}_").ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
deviceClients.Add(deviceClient);
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
setConnectionStatusChangesHandlerCount++;
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
_log.WriteLine($"{nameof(MethodE2EMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}");
deviceClientConnectionStatusChangeCount[deviceClient] = setConnectionStatusChangesHandlerCount;
});
Task methodReceivedTask = await setDeviceReceiveMethod(deviceClient).ConfigureAwait(false);
await Task.WhenAll(
MethodOperation.ServiceSendMethodAndVerifyResponse(testDevice.Id),
methodReceivedTask).ConfigureAwait(false);
}
}
finally
{
// Close and dispose all of the device client instances here
foreach (DeviceClient deviceClient in deviceClients)
{
await deviceClient.CloseAsync().ConfigureAwait(false);
// The connection status change count should be 2: connect (open) and disabled (close)
Assert.IsTrue(deviceClientConnectionStatusChangeCount[deviceClient] == 2, $"Connection status change count for deviceClient {TestLogging.GetHashCode(deviceClient)} is {deviceClientConnectionStatusChangeCount[deviceClient]}");
_log.WriteLine($"{nameof(MethodE2EMultiplexingTests)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
deviceClient.Dispose();
}
}
}
public static async Task TestErrorInjectionTemplate(
string devicePrefix,
TestDeviceType type,
Client.TransportType transport,
string faultType,
string reason,
int delayInSec,
int durationInSec,
Func <DeviceClient, TestDevice, Task> initOperation,
Func <DeviceClient, TestDevice, Task> testOperation,
Func <Task> cleanupOperation)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(devicePrefix, type).ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transport);
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int setConnectionStatusChangesHandlerCount = 0;
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
s_log.WriteLine($"{nameof(FaultInjection)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}");
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
setConnectionStatusChangesHandlerCount++;
});
var watch = new Stopwatch();
try
{
await deviceClient.OpenAsync().ConfigureAwait(false);
if (transport != Client.TransportType.Http1)
{
Assert.AreEqual(1, setConnectionStatusChangesHandlerCount);
Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason);
}
await initOperation(deviceClient, testDevice).ConfigureAwait(false);
s_log.WriteLine($">>> {nameof(FaultInjection)} Testing baseline");
await testOperation(deviceClient, testDevice).ConfigureAwait(false);
await ActivateFaultInjection(transport, faultType, reason, delayInSec, durationInSec, deviceClient).ConfigureAwait(false);
s_log.WriteLine($">>> {nameof(FaultInjection)} Testing fault handling");
watch.Start();
s_log.WriteLine($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {FaultInjection.WaitForDisconnectMilliseconds}ms");
await Task.Delay(FaultInjection.WaitForDisconnectMilliseconds).ConfigureAwait(false);
await testOperation(deviceClient, testDevice).ConfigureAwait(false);
await deviceClient.CloseAsync().ConfigureAwait(false);
if (transport == Client.TransportType.Mqtt ||
transport == Client.TransportType.Mqtt_Tcp_Only ||
transport == Client.TransportType.Mqtt_WebSocket_Only)
{
// Our fault injection is only terminating the connection for MQTT. (HTTP is not connection-oriented, AMQP is not actually terminating the TCP layer.)
Assert.IsTrue(setConnectionStatusChangesHandlerCount >= 4);
Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason);
}
}
finally
{
await cleanupOperation().ConfigureAwait(false);
await deviceClient.CloseAsync().ConfigureAwait(false);
watch.Stop();
int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds;
if (timeToFinishFaultInjection > 0)
{
s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed.");
await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false);
}
}
}
private async Task Twin_ServiceSetsDesiredPropertyAndDeviceReceivesEventMuxedOverAmqp(
ConnectionStringAuthScope authScope,
Client.TransportType transport,
int poolSize,
int devicesCount,
Func <DeviceClient, string, string, Task <Task> > setTwinPropertyUpdateCallbackAsync
)
{
var propName = Guid.NewGuid().ToString();
var propValue = Guid.NewGuid().ToString();
var transportSettings = new ITransportSettings[]
{
new AmqpTransportSettings(transport)
{
AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
{
MaxPoolSize = unchecked ((uint)poolSize),
Pooling = true
}
}
};
ICollection <DeviceClient> deviceClients = new List <DeviceClient>();
Dictionary <DeviceClient, int> deviceClientConnectionStatusChangeCount = new Dictionary <DeviceClient, int>();
try
{
_log.WriteLine($"{nameof(TwinE2EMultiplexingTests)}: Starting the test execution for {devicesCount} devices");
for (int i = 0; i < devicesCount; i++)
{
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int setConnectionStatusChangesHandlerCount = 0;
TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{DevicePrefix}_{i}_").ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
deviceClients.Add(deviceClient);
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
setConnectionStatusChangesHandlerCount++;
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
_log.WriteLine($"{nameof(TwinE2EMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}");
deviceClientConnectionStatusChangeCount[deviceClient] = setConnectionStatusChangesHandlerCount;
});
_log.WriteLine($"{nameof(Twin_ServiceSetsDesiredPropertyAndDeviceReceivesEventMuxedOverAmqp)}: name={propName}, value={propValue}");
Task updateReceivedTask = await setTwinPropertyUpdateCallbackAsync(deviceClient, propName, propValue).ConfigureAwait(false);
await Task.WhenAll(
TwinOperation.RegistryManagerUpdateDesiredPropertyAsync(testDevice.Id, propName, propValue),
updateReceivedTask).ConfigureAwait(false);
}
}
finally
{
// Close and dispose all of the device client instances here
foreach (DeviceClient deviceClient in deviceClients)
{
await deviceClient.CloseAsync().ConfigureAwait(false);
// The connection status change count should be 2: connect (open) and disabled (close)
Assert.IsTrue(deviceClientConnectionStatusChangeCount[deviceClient] == 2, $"Connection status change count for deviceClient {TestLogging.GetHashCode(deviceClient)} is {deviceClientConnectionStatusChangeCount[deviceClient]}");
_log.WriteLine($"{nameof(TwinE2EMultiplexingTests)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
deviceClient.Dispose();
}
}
}
async Task ModuleClient_Gives_ConnectionStatus_DeviceDisabled_Base(
Client.TransportType protocol, Func <RegistryManager, string, Task> registryManagerOperation)
{
AmqpTransportSettings amqpTransportSettings = new AmqpTransportSettings(protocol);
ITransportSettings[] transportSettings = new ITransportSettings[] { amqpTransportSettings };
TestModule testModule = await TestModule.GetTestModuleAsync(DevicePrefix + $"_{Guid.NewGuid()}", ModulePrefix).ConfigureAwait(false);
ConnectionStatus? status = null;
ConnectionStatusChangeReason?statusChangeReason = null;
int deviceDisabledReceivedCount = 0;
ConnectionStatusChangesHandler statusChangeHandler = (s, r) =>
{
if (r == ConnectionStatusChangeReason.Device_Disabled)
{
status = s;
statusChangeReason = r;
deviceDisabledReceivedCount++;
}
};
using (ModuleClient moduleClient = ModuleClient.CreateFromConnectionString(testModule.ConnectionString, transportSettings))
{
moduleClient.SetConnectionStatusChangesHandler(statusChangeHandler);
_log.WriteLine($"Created {nameof(DeviceClient)} ID={TestLogging.IdOf(moduleClient)}");
Console.WriteLine("ModuleClient OpenAsync.");
await moduleClient.OpenAsync().ConfigureAwait(false);
// Receiving the module twin should succeed right now.
Console.WriteLine("ModuleClient GetTwinAsync.");
var twin = await moduleClient.GetTwinAsync().ConfigureAwait(false);
Assert.IsNotNull(twin);
// Delete the device in IoT Hub.
using (RegistryManager registryManager = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString))
{
await registryManagerOperation(registryManager, testModule.DeviceId).ConfigureAwait(false);
}
// Periodically keep retrieving the device twin to keep connection alive.
// The ConnectionStatusChangesHandler should be triggered when the connection is closed from IoT hub with an
// exception thrown.
int twinRetrievals = 50;
for (int i = 0; i < twinRetrievals; i++)
{
try
{
await Task.Delay(TimeSpan.FromSeconds(10)).ConfigureAwait(false);
if (deviceDisabledReceivedCount == 1)
{
// Call an API on the client again to trigger the ConnectionStatusChangesHandler once again with the
// Device_Disabled status.
// This currently does not work due to some issues with IoT hub allowing new connections even when the
// device is deleted/disabled. Once that problem is investigated and fixed, we can re-enable this call
// and test for multiple invocations of the ConnectionStatusChangeHandler.
// await moduleClient.GetTwinAsync().ConfigureAwait(false);
break;
}
}
catch (IotHubException ex)
{
_log.WriteLine($"Exception occurred while retrieving module twin: {ex}");
Assert.IsInstanceOfType(ex.InnerException, typeof(DeviceNotFoundException));
}
}
Assert.AreEqual(1, deviceDisabledReceivedCount);
Assert.AreEqual(ConnectionStatus.Disconnected, status);
Assert.AreEqual(ConnectionStatusChangeReason.Device_Disabled, statusChangeReason);
}
}
private async Task SendMethodAndRespondRecovery(Client.TransportType transport, string faultType, string reason, int delayInSec)
{
Tuple <string, string> deviceInfo = TestUtil.CreateDevice(DevicePrefix, hostName, registryManager);
var assertResult = new TaskCompletionSource <Tuple <bool, bool> >();
DeviceClient deviceClient = DeviceClient.CreateFromConnectionString(deviceInfo.Item2, transport);
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int setConnectionStatusChangesHandlerCount = 0;
bool assertStatusChange = false;
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
if (assertStatusChange)
{
if (setConnectionStatusChangesHandlerCount == 0)
{
Assert.AreEqual(ConnectionStatus.Disconnected_Retrying, status);
Assert.AreEqual(ConnectionStatusChangeReason.No_Network, statusChangeReason);
}
else
{
Assert.AreEqual(ConnectionStatus.Connected, status);
Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, statusChangeReason);
}
}
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
setConnectionStatusChangesHandlerCount++;
});
await deviceClient.SetMethodHandlerAsync(MethodName,
(request, context) =>
{
assertResult.SetResult(new Tuple <bool, bool>(request.Name.Equals(MethodName), request.DataAsJson.Equals(ServiceRequestJson)));
return(Task.FromResult(new MethodResponse(Encoding.UTF8.GetBytes(DeviceResponseJson), 200)));
},
null);
if (transport != Client.TransportType.Http1)
{
Assert.AreEqual(1, setConnectionStatusChangesHandlerCount);
Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason);
}
await ServiceSendMethodAndVerifyResponse(deviceInfo.Item1, MethodName, DeviceResponseJson, ServiceRequestJson, assertResult);
// allow time for connection recovery
setConnectionStatusChangesHandlerCount = 0;
assertStatusChange = true;
// send error command
await deviceClient.SendEventAsync(TestUtil.ComposeErrorInjectionProperties(faultType, reason, delayInSec));
Stopwatch sw = new Stopwatch();
sw.Start();
while (sw.Elapsed.Minutes < 3 && setConnectionStatusChangesHandlerCount < 2)
{
await Task.Delay(1000);
}
assertResult = new TaskCompletionSource <Tuple <bool, bool> >();
await ServiceSendMethodAndVerifyResponse(deviceInfo.Item1, MethodName, DeviceResponseJson, ServiceRequestJson, assertResult);
setConnectionStatusChangesHandlerCount = 0;
assertStatusChange = false;
await deviceClient.SetMethodHandlerAsync(MethodName, null, null);
if (transport != Client.TransportType.Http1)
{
Assert.AreEqual(1, setConnectionStatusChangesHandlerCount);
Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason);
}
TestUtil.RemoveDevice(deviceInfo.Item1, registryManager);
}
private async Task ReceiveMessageMuxedOverAmqp(
Client.TransportType transport,
int poolSize,
int devicesCount,
ConnectionStringAuthScope authScope = ConnectionStringAuthScope.Device)
{
var transportSettings = new ITransportSettings[]
{
new AmqpTransportSettings(transport)
{
AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
{
MaxPoolSize = unchecked ((uint)poolSize),
Pooling = true
}
}
};
ICollection <DeviceClient> deviceClients = new List <DeviceClient>();
Dictionary <DeviceClient, int> deviceClientConnectionStatusChangeCount = new Dictionary <DeviceClient, int>();
ServiceClient serviceClient = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);
try
{
_log.WriteLine($"{nameof(MessageReceiveE2EMultiplexingTests)}: Starting the test execution for {devicesCount} devices");
await serviceClient.OpenAsync().ConfigureAwait(false);
for (int i = 0; i < devicesCount; i++)
{
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int setConnectionStatusChangesHandlerCount = 0;
TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{DevicePrefix}_{i}_").ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
deviceClients.Add(deviceClient);
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
setConnectionStatusChangesHandlerCount++;
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
_log.WriteLine($"{nameof(MessageReceiveE2EMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}");
deviceClientConnectionStatusChangeCount[deviceClient] = setConnectionStatusChangesHandlerCount;
});
_log.WriteLine($"{nameof(MessageReceiveE2EMultiplexingTests)}: Preparing to receive message for device {i}");
await deviceClient.OpenAsync().ConfigureAwait(false);
string payload, messageId, p1Value;
Message msg = MessageReceive.ComposeC2DTestMessage(out payload, out messageId, out p1Value);
await serviceClient.SendAsync(testDevice.Id, msg).ConfigureAwait(false);
await MessageReceive.VerifyReceivedC2DMessageAsync(transport, deviceClient, payload, p1Value).ConfigureAwait(false);
}
}
finally
{
await serviceClient.CloseAsync().ConfigureAwait(false);
serviceClient.Dispose();
// Close and dispose all of the device client instances here
foreach (DeviceClient deviceClient in deviceClients)
{
await deviceClient.CloseAsync().ConfigureAwait(false);
// The connection status change count should be 2: connect (open) and disabled (close)
Assert.IsTrue(deviceClientConnectionStatusChangeCount[deviceClient] == 2);
_log.WriteLine($"{nameof(MessageReceiveE2EMultiplexingTests)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
deviceClient.Dispose();
}
}
}
public ConnectionStatusChangeResult ChangeTo(ConnectionType connectionType, ConnectionStatus toState, ConnectionStatus?fromState = null)
{
if (toState == ConnectionStatus.Disabled)
{
return(Disable(connectionType, true));
}
var changeResult = new ConnectionStatusChangeResult
{
ClientStatus = this.status
};
lock (lockObject)
{
if (connections.ContainsKey(connectionType))
{
ConnectionStatus existingConnectionState = connections[connectionType].Item1;
CancellationTokenSource existingCancellationTokenSource = connections[connectionType].Item2;
if (((existingConnectionState != ConnectionStatus.Disconnected && existingConnectionState != ConnectionStatus.Disabled) || toState == ConnectionStatus.Connected) &&
(!fromState.HasValue || fromState.Value == existingConnectionState))
{
CancellationTokenSource changeStatusCancellationTokenSource = null;
if (toState == ConnectionStatus.Disconnected_Retrying)
{
changeStatusCancellationTokenSource = new CancellationTokenSource();
}
connections[connectionType] = new Tuple <ConnectionStatus, CancellationTokenSource>(toState, changeStatusCancellationTokenSource);
changeResult.StatusChangeCancellationTokenSource = changeStatusCancellationTokenSource;
changeResult.IsConnectionStatusChanged = true;
// When status is changed from Disconnected_Retrying, dispose cancellation token as it is no longer valid.
if (existingConnectionState == ConnectionStatus.Disconnected_Retrying && existingCancellationTokenSource != null)
{
existingCancellationTokenSource.Dispose();
}
}
}
else
{
if (toState == ConnectionStatus.Connected && (!fromState.HasValue || fromState.Value == ConnectionStatus.Disconnected))
{
connections.Add(connectionType, new Tuple <ConnectionStatus, CancellationTokenSource>(toState, null));
changeResult.IsConnectionStatusChanged = true;
}
}
if (changeResult.IsConnectionStatusChanged)
{
Tuple <ConnectionStatus, ConnectionStatus> beforeAndAfterState = UpdateDeviceClientState();
changeResult.IsClientStatusChanged = beforeAndAfterState.Item1 != beforeAndAfterState.Item2;
changeResult.ClientStatus = beforeAndAfterState.Item2;
}
}
return(changeResult);
}
private async Task SendMessageModuleMuxedOverAmqp(
Client.TransportType transport,
int poolSize,
int devicesCount,
bool useSameDevice = false)
{
var transportSettings = new ITransportSettings[]
{
new AmqpTransportSettings(transport)
{
AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
{
MaxPoolSize = unchecked ((uint)poolSize),
Pooling = true
}
}
};
ICollection <ModuleClient> moduleClients = new List <ModuleClient>();
Dictionary <ModuleClient, int> moduleClientConnectionStatusChangeCount = new Dictionary <ModuleClient, int>();
try
{
_log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}: Starting the test execution for {devicesCount} modules");
for (int i = 0; i < devicesCount; i++)
{
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int setConnectionStatusChangesHandlerCount = 0;
string devicePrefix = useSameDevice ? DevicePrefix : $"{DevicePrefix}_{i}_";
string modulePrefix = useSameDevice ? $"{ModulePrefix}_{i}_" : ModulePrefix;
TestModule testModule = await TestModule.GetTestModuleAsync(devicePrefix, modulePrefix).ConfigureAwait(false);
ModuleClient moduleClient = ModuleClient.CreateFromConnectionString(testModule.ConnectionString, transportSettings);
moduleClients.Add(moduleClient);
moduleClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
setConnectionStatusChangesHandlerCount++;
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
_log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}");
moduleClientConnectionStatusChangeCount[moduleClient] = setConnectionStatusChangesHandlerCount;
});
_log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}: Preparing to send message for module {i}");
await moduleClient.OpenAsync().ConfigureAwait(false);
await MessageSend.SendSingleMessageModuleAndVerifyAsync(moduleClient, testModule.DeviceId).ConfigureAwait(false);
}
}
finally
{
// Close and dispose all of the module client instances here
foreach (ModuleClient moduleClient in moduleClients)
{
await moduleClient.CloseAsync().ConfigureAwait(false);
// The connection status change count should be 2: connect (open) and disabled (close)
Assert.IsTrue(moduleClientConnectionStatusChangeCount[moduleClient] == 2);
_log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}: Disposing moduleClient {TestLogging.GetHashCode(moduleClient)}");
moduleClient.Dispose();
}
}
}
private async Task CheckPendingMessages()
{
LOG("CheckPendingMessages()");
TMessage[] messages;
lock ( LockObject )
{
if (Status != ConnectionStatus.Connected)
{
// Connection not ready. Leave in queue & send later
return;
}
if (PendingMessages.Count == 0)
{
// Nothing to send
return;
}
messages = PendingMessages.ToArray();
PendingMessages.Clear();
Status = ConnectionStatus.Sending;
}
await OnStatusChanged.Invoke(ConnectionStatus.Sending);
try
{
var rootMessage = CreateRootMessage_MessagesList(messages);
await SendRootMessage(rootMessage);
}
catch (System.Exception ex)
{
await OnInternalError.Invoke("Could not send messages", ex);
}
bool launchCloseConnection;
string connectionID = null;
{
ConnectionStatus?newStatus = null;
lock ( LockObject )
{
switch (Status)
{
case ConnectionStatus.Sending:
// Switch back to 'Connected'
newStatus = Status = ConnectionStatus.Connected;
launchCloseConnection = false;
break;
case ConnectionStatus.Closing:
// Stop has been requested while sending
launchCloseConnection = true;
connectionID = ConnectionID;
ConnectionID = null;
break;
default:
// Should not happen
FAIL("The property 'Status' is not supposed to have value '" + Status + "' here");
newStatus = Status = ConnectionStatus.Closing;
launchCloseConnection = true;
connectionID = ConnectionID;
ConnectionID = null;
break;
}
}
if (newStatus != null)
{
await OnStatusChanged.Invoke(newStatus.Value);
}
}
if (launchCloseConnection)
{
var dummy = CloseConnection(connectionID); // NB: No await so can be performed in background
}
}
private async Task DeviceClient_Gives_ConnectionStatus_DeviceDisabled_Base(
Client.TransportType protocol, Func <RegistryManager, string, Task> registryManagerOperation)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix + $"_{Guid.NewGuid()}").ConfigureAwait(false);
string deviceConnectionString = testDevice.ConnectionString;
var config = new Configuration.IoTHub.DeviceConnectionStringParser(deviceConnectionString);
string deviceId = config.DeviceID;
ConnectionStatus? status = null;
ConnectionStatusChangeReason?statusChangeReason = null;
int deviceDisabledReceivedCount = 0;
using (DeviceClient deviceClient = DeviceClient.CreateFromConnectionString(deviceConnectionString, protocol))
{
ConnectionStatusChangesHandler statusChangeHandler = (s, r) =>
{
if (r == ConnectionStatusChangeReason.Device_Disabled)
{
status = s;
statusChangeReason = r;
deviceDisabledReceivedCount++;
}
};
deviceClient.SetConnectionStatusChangesHandler(statusChangeHandler);
_log.WriteLine($"Created {nameof(DeviceClient)} ID={TestLogging.IdOf(deviceClient)}");
Console.WriteLine("DeviceClient OpenAsync.");
await deviceClient.OpenAsync().ConfigureAwait(false);
// Receiving the module twin should succeed right now.
Console.WriteLine("ModuleClient GetTwinAsync.");
var twin = await deviceClient.GetTwinAsync().ConfigureAwait(false);
Assert.IsNotNull(twin);
// Delete/disable the device in IoT Hub. This should trigger the ConnectionStatusChangesHandler.
using (RegistryManager registryManager = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString))
{
await registryManagerOperation(registryManager, deviceId).ConfigureAwait(false);
}
// Artificial sleep waiting for the connection status change handler to get triggered.
int sleepCount = 50;
for (int i = 0; i < sleepCount; i++)
{
await Task.Delay(TimeSpan.FromSeconds(10)).ConfigureAwait(false);
if (deviceDisabledReceivedCount == 1)
{
break;
}
}
Assert.AreEqual(1, deviceDisabledReceivedCount);
Assert.AreEqual(ConnectionStatus.Disconnected, status);
Assert.AreEqual(ConnectionStatusChangeReason.Device_Disabled, statusChangeReason);
}
}
internal async Task SendMessageRecovery(Client.TransportType transport,
string faultType, string reason, int delayInSec, int durationInSec = 0, int retryDurationInMilliSec = 240000)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);
var deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport);
PartitionReceiver eventHubReceiver = await CreateEventHubReceiver(testDevice.Id).ConfigureAwait(false);
try
{
deviceClient.OperationTimeoutInMilliseconds = (uint)retryDurationInMilliSec;
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int setConnectionStatusChangesHandlerCount = 0;
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
setConnectionStatusChangesHandlerCount++;
});
await deviceClient.OpenAsync().ConfigureAwait(false);
if (transport != Client.TransportType.Http1)
{
Assert.AreEqual(1, setConnectionStatusChangesHandlerCount);
Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason);
}
string payload, p1Value;
Client.Message testMessage = ComposeD2CTestMessage(out payload, out p1Value);
await deviceClient.SendEventAsync(testMessage).ConfigureAwait(false);
bool isReceived = false;
Stopwatch sw = new Stopwatch();
sw.Start();
while (!isReceived && sw.Elapsed.Minutes < 1)
{
var events = await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5)).ConfigureAwait(false);
isReceived = VerifyTestMessage(events, testDevice.Id, payload, p1Value);
}
sw.Stop();
// send error command and clear eventHubReceiver of the fault injection message
await deviceClient.SendEventAsync(FaultInjection.ComposeErrorInjectionProperties(faultType, reason,
delayInSec,
durationInSec)).ConfigureAwait(false);
await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5)).ConfigureAwait(false);
Thread.Sleep(1000);
testMessage = ComposeD2CTestMessage(out payload, out p1Value);
await deviceClient.SendEventAsync(testMessage).ConfigureAwait(false);
sw.Reset();
sw.Start();
while (!isReceived && sw.Elapsed.Minutes < 1)
{
var events = await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5)).ConfigureAwait(false);
isReceived = VerifyTestMessage(events, testDevice.Id, payload, p1Value);
}
sw.Stop();
await deviceClient.CloseAsync().ConfigureAwait(false);
if (transport != Client.TransportType.Http1)
{
Assert.AreEqual(2, setConnectionStatusChangesHandlerCount);
Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason);
}
}
finally
{
await deviceClient.CloseAsync().ConfigureAwait(false);
await eventHubReceiver.CloseAsync().ConfigureAwait(false);
}
}
public ConnectionStatusChangeResult ChangeTo(ConnectionType connectionType, ConnectionStatus toState, ConnectionStatus?fromState = null, CancellationTokenSource cancellationTokenSource = null)
{
if (toState == ConnectionStatus.Disabled)
{
return(Disable(connectionType, true));
}
Tuple <ConnectionStatus, CancellationTokenSource> connectionValue;
if (toState == ConnectionStatus.Disconnected_Retrying)
{
if (cancellationTokenSource == null)
{
throw new ArgumentNullException($"{nameof(cancellationTokenSource)} should be provided for retrying state.");
}
connectionValue = Tuple.Create(toState, cancellationTokenSource);
}
else
{
connectionValue = new Tuple <ConnectionStatus, CancellationTokenSource>(toState, null);
}
var changeResult = new ConnectionStatusChangeResult
{
ClientStatus = this.status
};
lock (lockObject)
{
if (connections.ContainsKey(connectionType))
{
ConnectionStatus existingConnectionState = connections[connectionType].Item1;
if (((existingConnectionState != ConnectionStatus.Disconnected && existingConnectionState != ConnectionStatus.Disabled) || toState == ConnectionStatus.Connected) &&
(!fromState.HasValue || fromState.Value == existingConnectionState))
{
connections[connectionType] = connectionValue;
changeResult.IsConnectionStatusChanged = true;
}
}
else
{
if (toState == ConnectionStatus.Connected && (!fromState.HasValue || fromState.Value == ConnectionStatus.Disconnected))
{
connections.Add(connectionType, connectionValue);
changeResult.IsConnectionStatusChanged = true;
}
}
if (changeResult.IsConnectionStatusChanged)
{
Tuple <ConnectionStatus, ConnectionStatus> beforeAndAfterState = UpdateDeviceClientState();
changeResult.IsClientStatusChanged = beforeAndAfterState.Item1 != beforeAndAfterState.Item2;
changeResult.ClientStatus = beforeAndAfterState.Item2;
}
}
return(changeResult);
}
// Error injection template method.
public static async Task TestErrorInjectionAsync(
string devicePrefix,
TestDeviceType type,
Client.TransportType transport,
string proxyAddress,
string faultType,
string reason,
int delayInSec,
int durationInSec,
Func <DeviceClient, TestDevice, Task> initOperation,
Func <DeviceClient, TestDevice, Task> testOperation,
Func <Task> cleanupOperation,
MsTestLogger logger)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(logger, devicePrefix, type).ConfigureAwait(false);
ITransportSettings transportSettings = CreateTransportSettingsFromName(transport, proxyAddress);
DeviceClient deviceClient = testDevice.CreateDeviceClient(new ITransportSettings[] { transportSettings });
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int connectionStatusChangeCount = 0;
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
connectionStatusChangeCount++;
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
logger.Trace($"{nameof(FaultInjection)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={connectionStatusChangeCount}");
});
var watch = new Stopwatch();
try
{
await deviceClient.OpenAsync().ConfigureAwait(false);
if (transport != Client.TransportType.Http1)
{
Assert.IsTrue(connectionStatusChangeCount >= 1, $"The expected connection status change should be equal or greater than 1 but was {connectionStatusChangeCount}"); // Normally one connection but in some cases, due to network issues we might have already retried several times to connect.
Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Connected} but was {lastConnectionStatus}");
Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Connection_Ok} but was {lastConnectionStatusChangeReason}");
}
await initOperation(deviceClient, testDevice).ConfigureAwait(false);
logger.Trace($">>> {nameof(FaultInjection)} Testing baseline");
await testOperation(deviceClient, testDevice).ConfigureAwait(false);
int countBeforeFaultInjection = connectionStatusChangeCount;
watch.Start();
logger.Trace($">>> {nameof(FaultInjection)} Testing fault handling");
await ActivateFaultInjectionAsync(transport, faultType, reason, delayInSec, durationInSec, deviceClient, logger).ConfigureAwait(false);
logger.Trace($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {delayInSec} seconds.");
await Task.Delay(TimeSpan.FromSeconds(delayInSec)).ConfigureAwait(false);
// For disconnect type faults, the device should disconnect and recover.
if (FaultShouldDisconnect(faultType))
{
logger.Trace($"{nameof(FaultInjection)}: Confirming fault injection has been actived.");
// Check that service issued the fault to the faulting device
bool isFaulted = false;
for (int i = 0; i < LatencyTimeBufferInSec; i++)
{
if (connectionStatusChangeCount > countBeforeFaultInjection)
{
isFaulted = true;
break;
}
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
Assert.IsTrue(isFaulted, $"The device {testDevice.Id} did not get faulted with fault type: {faultType}");
logger.Trace($"{nameof(FaultInjection)}: Confirmed fault injection has been actived.");
// Check the device is back online
logger.Trace($"{nameof(FaultInjection)}: Confirming device back online.");
for (int i = 0; lastConnectionStatus != ConnectionStatus.Connected && i < durationInSec + LatencyTimeBufferInSec; i++)
{
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus, $"{testDevice.Id} did not reconnect.");
logger.Trace($"{nameof(FaultInjection)}: Confirmed device back online.");
// Perform the test operation.
logger.Trace($">>> {nameof(FaultInjection)}: Performing test operation for device {testDevice.Id}.");
await testOperation(deviceClient, testDevice).ConfigureAwait(false);
}
else
{
logger.Trace($"{nameof(FaultInjection)}: Performing test operation while fault injection is being activated.");
// Perform the test operation for the faulted device multi times.
for (int i = 0; i < LatencyTimeBufferInSec; i++)
{
logger.Trace($">>> {nameof(FaultInjection)}: Performing test operation for device - Run {i}.");
await testOperation(deviceClient, testDevice).ConfigureAwait(false);
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
}
await deviceClient.CloseAsync().ConfigureAwait(false);
if (transport != Client.TransportType.Http1)
{
if (FaultInjection.FaultShouldDisconnect(faultType))
{
// 4 is the minimum notification count: connect, fault, reconnect, disable.
// There are cases where the retry must be timed out (i.e. very likely for MQTT where otherwise
// we would attempt to send the fault injection forever.)
Assert.IsTrue(connectionStatusChangeCount >= 4, $"The expected connection status change count for {testDevice.Id} should be equal or greater than 4 but was {connectionStatusChangeCount}");
}
else
{
// 2 is the minimum notification count: connect, disable.
// We will monitor the test environment real network stability and switch to >=2 if necessary to
// account for real network issues.
Assert.IsTrue(connectionStatusChangeCount == 2, $"The expected connection status change count for {testDevice.Id} should be 2 but was {connectionStatusChangeCount}");
}
Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Disabled} but was {lastConnectionStatus}");
Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Client_Close} but was {lastConnectionStatusChangeReason}");
}
}
finally
{
await cleanupOperation().ConfigureAwait(false);
logger.Trace($"{nameof(FaultInjection)}: Disposing deviceClient {TestLogger.GetHashCode(deviceClient)}");
deviceClient.Dispose();
watch.Stop();
int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds;
if (timeToFinishFaultInjection > 0)
{
logger.Trace($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed.");
await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false);
}
}
}
internal PrivateLinkServiceConnectionState(ConnectionStatus?status, string description, ActionsRequired?actionsRequired)
{
Status = status;
Description = description;
ActionsRequired = actionsRequired;
}
private async Task ModuleClient_Gives_ConnectionStatus_DeviceDisabled_Base(
Client.TransportType protocol, Func <RegistryManager, string, Task> registryManagerOperation)
{
AmqpTransportSettings amqpTransportSettings = new AmqpTransportSettings(protocol);
ITransportSettings[] transportSettings = new ITransportSettings[] { amqpTransportSettings };
TestModule testModule = await TestModule.GetTestModuleAsync(DevicePrefix + $"_{Guid.NewGuid()}", ModulePrefix, Logger).ConfigureAwait(false);
ConnectionStatus? status = null;
ConnectionStatusChangeReason?statusChangeReason = null;
int deviceDisabledReceivedCount = 0;
ConnectionStatusChangesHandler statusChangeHandler = (s, r) =>
{
if (r == ConnectionStatusChangeReason.Device_Disabled)
{
status = s;
statusChangeReason = r;
deviceDisabledReceivedCount++;
}
};
using (ModuleClient moduleClient = ModuleClient.CreateFromConnectionString(testModule.ConnectionString, transportSettings))
{
moduleClient.SetConnectionStatusChangesHandler(statusChangeHandler);
Logger.Trace($"{nameof(ModuleClient_Gives_ConnectionStatus_DeviceDisabled_Base)}: Created {nameof(ModuleClient)} with moduleId={testModule.Id}");
await moduleClient.OpenAsync().ConfigureAwait(false);
// Receiving the module twin should succeed right now.
Logger.Trace($"{nameof(ModuleClient_Gives_ConnectionStatus_DeviceDisabled_Base)}: ModuleClient GetTwinAsync.");
var twin = await moduleClient.GetTwinAsync().ConfigureAwait(false);
Assert.IsNotNull(twin);
// Delete/disable the device in IoT Hub.
using (RegistryManager registryManager = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString))
{
await registryManagerOperation(registryManager, testModule.DeviceId).ConfigureAwait(false);
}
Logger.Trace($"{nameof(ModuleClient_Gives_ConnectionStatus_DeviceDisabled_Base)}: Completed RegistryManager operation.");
// Artificial sleep waiting for the connection status change handler to get triggered.
int sleepCount = 50;
for (int i = 0; i < sleepCount; i++)
{
await Task.Delay(TimeSpan.FromSeconds(10)).ConfigureAwait(false);
if (deviceDisabledReceivedCount == 1)
{
break;
}
}
Logger.Trace($"{nameof(ModuleClient_Gives_ConnectionStatus_DeviceDisabled_Base)}: Asserting connection status change.");
Assert.AreEqual(1, deviceDisabledReceivedCount);
Assert.AreEqual(ConnectionStatus.Disconnected, status);
Assert.AreEqual(ConnectionStatusChangeReason.Device_Disabled, statusChangeReason);
}
}
public async Task Duplicated_NoPingpong()
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(Logger, DevicePrefix, TestDeviceType.Sasl).ConfigureAwait(false);
Logger.Trace($"{nameof(Duplicated_NoPingpong)}: 2 device client instances with the same deviceId={testDevice.Id}.");
using DeviceClient deviceClient1 = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only);
using DeviceClient deviceClient2 = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only);
Logger.Trace($"{nameof(Duplicated_NoPingpong)}: set device client instance 1 to no retry.");
deviceClient1.SetRetryPolicy(new NoRetry());
ConnectionStatus?lastConnectionStatus = null;
var connectionStatusChanges = new Dictionary <ConnectionStatus, int>();
deviceClient1.SetConnectionStatusChangesHandler((status, reason) =>
{
connectionStatusChanges.TryGetValue(status, out int count);
count++;
connectionStatusChanges[status] = count;
lastConnectionStatus = status;
});
Logger.Trace($"{nameof(FaultInjection_NoRecovery)}: device client instance 1 calling OpenAsync...");
await deviceClient1.OpenAsync().ConfigureAwait(false);
await deviceClient1
.SetMethodHandlerAsync(
"dummy_method",
(methodRequest, userContext) => Task.FromResult(new MethodResponse(200)),
deviceClient1)
.ConfigureAwait(false);
Logger.Trace($"{nameof(FaultInjection_NoRecovery)}: device client instance 2 calling OpenAsync...");
await deviceClient2.OpenAsync().ConfigureAwait(false);
await deviceClient2
.SetMethodHandlerAsync(
"dummy_method",
(methodRequest, userContext) => Task.FromResult(new MethodResponse(200)),
deviceClient2)
.ConfigureAwait(false);
Logger.Trace($"{nameof(Duplicated_NoPingpong)}: waiting device client instance 1 to be kicked off...");
for (int i = 0; i < FaultInjection.LatencyTimeBufferInSec; i++)
{
if (connectionStatusChanges.ContainsKey(ConnectionStatus.Disconnected))
{
break;
}
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
Assert.AreEqual(ConnectionStatus.Disconnected, lastConnectionStatus, $"Excepeted device to be {ConnectionStatus.Disconnected} but was {lastConnectionStatus}.");
Assert.IsFalse(connectionStatusChanges.ContainsKey(ConnectionStatus.Disconnected_Retrying), $"Shouldn't get {ConnectionStatus.Disconnected_Retrying} status change.");
int connected = connectionStatusChanges[ConnectionStatus.Connected];
Assert.AreEqual(1, connected, $"Should get {ConnectionStatus.Connected} once but was {connected}.");
int disconnected = connectionStatusChanges[ConnectionStatus.Disconnected];
Assert.AreEqual(1, disconnected, $"Should get {ConnectionStatus.Disconnected} once but was {disconnected}.");
}
private async Task SendMethodAndRespondRecovery(Client.TransportType transport, string faultType, string reason, int delayInSec)
{
Tuple <string, string> deviceInfo = TestUtil.CreateDevice(DevicePrefix, hostName, registryManager);
var assertResult = new TaskCompletionSource <Tuple <bool, bool> >();
DeviceClient deviceClient = DeviceClient.CreateFromConnectionString(deviceInfo.Item2, transport);
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int setConnectionStatusChangesHandlerCount = 0;
var tcsConnected = new TaskCompletionSource <bool>();
var tcsDisconnected = new TaskCompletionSource <bool>();
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
Debug.WriteLine("Connection Changed to {0} because {1}", status, statusChangeReason);
if (status == ConnectionStatus.Disconnected_Retrying)
{
tcsDisconnected.TrySetResult(true);
Assert.AreEqual(ConnectionStatusChangeReason.No_Network, statusChangeReason);
}
else if (status == ConnectionStatus.Connected)
{
tcsConnected.TrySetResult(true);
}
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
setConnectionStatusChangesHandlerCount++;
});
await deviceClient.SetMethodHandlerAsync(MethodName,
(request, context) =>
{
assertResult.TrySetResult(new Tuple <bool, bool>(request.Name.Equals(MethodName),
request.DataAsJson.Equals(ServiceRequestJson)));
return(Task.FromResult(new MethodResponse(Encoding.UTF8.GetBytes(DeviceResponseJson), 200)));
},
null).ConfigureAwait(false);
// assert on successfuly connection
await Task.WhenAny(
Task.Run(async() =>
{
await Task.Delay(1000).ConfigureAwait(false);
}), tcsConnected.Task).ConfigureAwait(false);
Assert.IsTrue(tcsConnected.Task.IsCompleted, "Initial connection failed");
if (transport != Client.TransportType.Http1)
{
Assert.AreEqual(1, setConnectionStatusChangesHandlerCount);
Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason);
}
// check on normal operation
await
ServiceSendMethodAndVerifyResponse(deviceInfo.Item1, MethodName, DeviceResponseJson,
ServiceRequestJson, assertResult).ConfigureAwait(false);
// reset ConnectionStatusChangesHandler data
setConnectionStatusChangesHandlerCount = 0;
tcsConnected = new TaskCompletionSource <bool>();
tcsDisconnected = new TaskCompletionSource <bool>();
// send error command
await deviceClient.SendEventAsync(TestUtil.ComposeErrorInjectionProperties(faultType, reason, delayInSec)).ConfigureAwait(false);
// wait for disconnection
await Task.WhenAny(
Task.Run(async() =>
{
await Task.Delay(TimeSpan.FromSeconds(10)).ConfigureAwait(false);
}), tcsDisconnected.Task).ConfigureAwait(false);
Assert.IsTrue(tcsDisconnected.Task.IsCompleted, "Error injection did not interrupt the device");
// allow max 3 minutes for connection recovery
await Task.WhenAny(
Task.Run(async() =>
{
////////////////////////change it to 30
await Task.Delay(TimeSpan.FromSeconds(10)).ConfigureAwait(false);
return(Task.FromResult(true));
}), tcsConnected.Task).ConfigureAwait(false);
Assert.IsTrue(tcsConnected.Task.IsCompleted, "Recovery connection failed");
assertResult = new TaskCompletionSource <Tuple <bool, bool> >();
await
ServiceSendMethodAndVerifyResponse(deviceInfo.Item1, MethodName, DeviceResponseJson,
ServiceRequestJson, assertResult).ConfigureAwait(false);
setConnectionStatusChangesHandlerCount = 0;
//remove and CloseAsync
await deviceClient.SetMethodHandlerAsync(MethodName, null, null).ConfigureAwait(false);
if (transport != Client.TransportType.Http1)
{
Assert.AreEqual(1, setConnectionStatusChangesHandlerCount);
Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason);
}
await TestUtil.RemoveDeviceAsync(deviceInfo.Item1, registryManager).ConfigureAwait(false);
}
private async Task DeviceCombinedClientOperations(
Client.TransportType transport,
int poolSize,
int devicesCount,
ConnectionStringAuthScope authScope
)
{
var transportSettings = new ITransportSettings[]
{
new AmqpTransportSettings(transport)
{
AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
{
MaxPoolSize = unchecked ((uint)poolSize),
Pooling = true
}
}
};
// Initialize service client for service-side operations
ServiceClient serviceClient = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);
ICollection <DeviceClient> deviceClients = new List <DeviceClient>();
Dictionary <DeviceClient, int> deviceClientConnectionStatusChangeCount = new Dictionary <DeviceClient, int>();
try
{
_log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Starting the test execution for {devicesCount} devices");
for (int i = 0; i < devicesCount; i++)
{
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int setConnectionStatusChangesHandlerCount = 0;
TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{DevicePrefix}_{i}_").ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
deviceClients.Add(deviceClient);
// Set the connection status change handler
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
setConnectionStatusChangesHandlerCount++;
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
_log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}");
deviceClientConnectionStatusChangeCount[deviceClient] = setConnectionStatusChangesHandlerCount;
});
// Perform D2C Operation
_log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Preparing to send message for device {i}");
await deviceClient.OpenAsync().ConfigureAwait(false);
await MessageSend.SendSingleMessageAndVerifyAsync(deviceClient, testDevice.Id).ConfigureAwait(false);
// Perform C2D Operation
_log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Setting the device {i} to receive C2D message.");
string payload, messageId, p1Value;
Message msg = MessageReceive.ComposeC2DTestMessage(out payload, out messageId, out p1Value);
await serviceClient.SendAsync(testDevice.Id, msg).ConfigureAwait(false);
await MessageReceive.VerifyReceivedC2DMessageAsync(transport, deviceClient, payload, p1Value).ConfigureAwait(false);
// Invoke direct methods
_log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Testing direct methods for device {i}");
Task methodReceivedTask = await MethodOperation.SetDeviceReceiveMethod(deviceClient).ConfigureAwait(false);
await Task.WhenAll(
MethodOperation.ServiceSendMethodAndVerifyResponse(testDevice.Id),
methodReceivedTask).ConfigureAwait(false);
// Set reported twin properties
_log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Setting reported Twin properties for device {i}");
await TwinOperation.Twin_DeviceSetsReportedPropertyAndGetsItBack(deviceClient).ConfigureAwait(false);
// Receive set desired twin properties
_log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Received set desired tein properties for device {i}");
var propName = Guid.NewGuid().ToString();
var propValue = Guid.NewGuid().ToString();
Task updateReceivedTask = await TwinOperation.SetTwinPropertyUpdateCallbackHandlerAsync(deviceClient, propName, propValue).ConfigureAwait(false);
await Task.WhenAll(
TwinOperation.RegistryManagerUpdateDesiredPropertyAsync(testDevice.Id, propName, propValue),
updateReceivedTask).ConfigureAwait(false);
}
}
finally
{
// Close and dispose all of the device client instances here
foreach (DeviceClient deviceClient in deviceClients)
{
await deviceClient.CloseAsync().ConfigureAwait(false);
// The connection status change count should be 2: connect (open) and disabled (close)
Assert.IsTrue(deviceClientConnectionStatusChangeCount[deviceClient] == 2, $"Connection status change count for deviceClient {TestLogging.GetHashCode(deviceClient)} is {deviceClientConnectionStatusChangeCount[deviceClient]}");
_log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
deviceClient.Dispose();
}
}
}
