/// <summary>
/// Rety an async operation based on the retry strategy supplied.
/// </summary>
/// <param name="asyncOperation">The async operation to be retried.</param>
/// <param name="retryPolicy">The retry policy to be applied.</param>
/// <param name="retryableExceptions">The exceptions to be retried on.</param>
/// <param name="logger">The <see cref="MsTestLogger"/> instance to be used.</param>
/// <returns></returns>
public static async Task RetryOperationsAsync(Func <Task> asyncOperation, IRetryPolicy retryPolicy, HashSet <Type> retryableExceptions, MsTestLogger logger)
{
int counter = 0;
bool shouldRetry;
do
{
TimeSpan retryInterval;
try
{
await asyncOperation().ConfigureAwait(false);
break;
}
catch (Exception ex) when(retryableExceptions.Any(e => e.IsInstanceOfType(ex)))
{
shouldRetry = retryPolicy.ShouldRetry(++counter, ex, out retryInterval);
logger.Trace($"Attempt {counter}: operation did not succeed: {ex}");
if (!shouldRetry)
{
logger.Trace($"Encountered an exception that will not be retried - attempt: {counter}; exception: {ex}");
throw;
}
}
logger.Trace($"Will retry operation in {retryInterval}.");
await Task.Delay(retryInterval).ConfigureAwait(false);
}while (shouldRetry);
}
public async Task SetDeviceReceiveMethodAsync(string methodName, string deviceResponseJson, string expectedServiceRequestJson)
{
await _deviceClient.SetMethodHandlerAsync(methodName,
(request, context) =>
{
try
{
_logger.Trace($"{nameof(SetDeviceReceiveMethodAsync)}: DeviceClient callback method: {request.Name} {request.ResponseTimeout}.");
Assert.AreEqual(methodName, request.Name, $"The expected method name should be {methodName} but was {request.Name}");
Assert.AreEqual(expectedServiceRequestJson, request.DataAsJson, $"The expected method name should be {expectedServiceRequestJson} but was {request.DataAsJson}");
return(Task.FromResult(new MethodResponse(Encoding.UTF8.GetBytes(deviceResponseJson), 200)));
}
catch (Exception ex)
{
_logger.Trace($"{nameof(SetDeviceReceiveMethodAsync)}: Error during DeviceClient callback method: {ex}.");
_methodExceptionDispatch = ExceptionDispatchInfo.Capture(ex);
return(Task.FromResult(new MethodResponse(500)));
}
finally
{
// Always notify that we got the callback.
_methodCallbackSemaphore.Release();
}
},
null).ConfigureAwait(false);
}
public static async Task ServiceSendMethodAndVerifyResponseAsync(
string deviceId,
string moduleId,
string methodName,
string respJson,
string reqJson,
MsTestLogger logger,
TimeSpan responseTimeout = default,
ServiceClientTransportSettings serviceClientTransportSettings = default)
{
ServiceClient serviceClient = serviceClientTransportSettings == default
? ServiceClient.CreateFromConnectionString(TestConfiguration.IoTHub.ConnectionString)
: ServiceClient.CreateFromConnectionString(TestConfiguration.IoTHub.ConnectionString, TransportType.Amqp, serviceClientTransportSettings);
TimeSpan methodTimeout = responseTimeout == default ? s_defaultMethodTimeoutMinutes : responseTimeout;
logger.Trace($"{nameof(ServiceSendMethodAndVerifyResponseAsync)}: Invoke method {methodName}.");
CloudToDeviceMethodResult response =
await serviceClient.InvokeDeviceMethodAsync(
deviceId,
moduleId,
new CloudToDeviceMethod(methodName, responseTimeout).SetPayloadJson(reqJson)).ConfigureAwait(false);
logger.Trace($"{nameof(ServiceSendMethodAndVerifyResponseAsync)}: Method status: {response.Status}.");
Assert.AreEqual(200, response.Status, $"The expected response status should be 200 but was {response.Status}");
string payload = response.GetPayloadAsJson();
Assert.AreEqual(respJson, payload, $"The expected response payload should be {respJson} but was {payload}");
await serviceClient.CloseAsync().ConfigureAwait(false);
serviceClient.Dispose();
}
private static async Task ReceiveMessageWithoutTimeoutCheckAsync(DeviceClient dc, TimeSpan bufferTime, MsTestLogger logger)
{
var sw = new Stopwatch();
while (true)
{
try
{
logger.Trace($"{nameof(ReceiveMessageWithoutTimeoutCheckAsync)} - Calling ReceiveAsync()");
sw.Restart();
using Client.Message message = await dc.ReceiveAsync().ConfigureAwait(false);
sw.Stop();
logger.Trace($"{nameof(ReceiveMessageWithoutTimeoutCheckAsync)} - Received message={message}; time taken={sw.ElapsedMilliseconds} ms");
if (message == null)
{
break;
}
await dc.CompleteAsync(message).ConfigureAwait(false);
}
finally
{
TimeSpan maxLatency = TimeSpan.FromMilliseconds(dc.OperationTimeoutInMilliseconds) + bufferTime;
if (sw.Elapsed > maxLatency)
{
Assert.Fail($"ReceiveAsync did not return in {maxLatency}, instead it took {sw.Elapsed}.");
}
}
}
}
// Fault timings:
// --------------------------------------------------------------------------------------------------------------------------------------------------------------------
// --- device in normal operation --- | FaultRequested | --- <delayInSec> --- | --- Device in fault mode for <durationInSec> --- | --- device in normal operation ---
// --------------------------------------------------------------------------------------------------------------------------------------------------------------------
public static async Task ActivateFaultInjectionAsync(Client.TransportType transport, string faultType, string reason, int delayInSec, int durationInSec, DeviceClient deviceClient, MsTestLogger logger)
{
logger.Trace($"{nameof(ActivateFaultInjectionAsync)}: Requesting fault injection type={faultType} reason={reason}, delay={delayInSec}s, duration={FaultInjection.DefaultDurationInSec}s");
uint oldTimeout = deviceClient.OperationTimeoutInMilliseconds;
try
{
// For MQTT FaultInjection will terminate the connection prior to a PUBACK
// which leads to an infinite loop trying to resend the FaultInjection message.
if (transport == Client.TransportType.Mqtt ||
transport == Client.TransportType.Mqtt_Tcp_Only ||
transport == Client.TransportType.Mqtt_WebSocket_Only)
{
deviceClient.OperationTimeoutInMilliseconds = (uint)delayInSec * 1000;
}
await deviceClient
.SendEventAsync(
ComposeErrorInjectionProperties(
faultType,
reason,
delayInSec,
durationInSec))
.ConfigureAwait(false);
}
catch (IotHubCommunicationException ex)
{
logger.Trace($"{nameof(ActivateFaultInjectionAsync)}: {ex}");
// For quota injection, the fault is only seen for the original HTTP request.
if (transport == Client.TransportType.Http1)
{
throw;
}
}
catch (TimeoutException ex)
{
logger.Trace($"{nameof(ActivateFaultInjectionAsync)}: {ex}");
// For quota injection, the fault is only seen for the original HTTP request.
if (transport == Client.TransportType.Http1)
{
throw;
}
}
finally
{
deviceClient.OperationTimeoutInMilliseconds = oldTimeout;
logger.Trace($"{nameof(ActivateFaultInjectionAsync)}: Fault injection requested.");
}
}
public static async Task ServiceSendMethodAndVerifyNotReceivedAsync(
string deviceId,
string methodName,
MsTestLogger logger,
TimeSpan responseTimeout = default,
ServiceClientTransportSettings serviceClientTransportSettings = default)
{
ServiceClient serviceClient = serviceClientTransportSettings == default
? ServiceClient.CreateFromConnectionString(TestConfiguration.IoTHub.ConnectionString)
: ServiceClient.CreateFromConnectionString(TestConfiguration.IoTHub.ConnectionString, TransportType.Amqp, serviceClientTransportSettings);
TimeSpan methodTimeout = responseTimeout == default ? s_defaultMethodTimeoutMinutes : responseTimeout;
logger.Trace($"{nameof(ServiceSendMethodAndVerifyResponseAsync)}: Invoke method {methodName}.");
try
{
CloudToDeviceMethodResult response = await serviceClient
.InvokeDeviceMethodAsync(
deviceId,
new CloudToDeviceMethod(methodName, methodTimeout).SetPayloadJson(null))
.ConfigureAwait(false);
}
catch (DeviceNotFoundException)
{
}
finally
{
await serviceClient.CloseAsync().ConfigureAwait(false);
serviceClient.Dispose();
}
}
public static async Task <Task> SetTwinPropertyUpdateCallbackHandlerAsync(DeviceClient deviceClient, string expectedPropName, object expectedPropValue, MsTestLogger logger)
{
var propertyUpdateReceived = new TaskCompletionSource <bool>();
string userContext = "myContext";
await deviceClient
.SetDesiredPropertyUpdateCallbackAsync(
(patch, context) =>
{
logger.Trace($"{nameof(SetTwinPropertyUpdateCallbackHandlerAsync)}: DesiredProperty: {patch}, {context}");
try
{
Assert.AreEqual(JsonConvert.SerializeObject(expectedPropValue), JsonConvert.SerializeObject(patch[expectedPropName]));
Assert.AreEqual(userContext, context, "Context");
}
catch (Exception e)
{
propertyUpdateReceived.SetException(e);
}
finally
{
propertyUpdateReceived.SetResult(true);
}
return(Task.FromResult <bool>(true));
},
userContext)
.ConfigureAwait(false);
return(propertyUpdateReceived.Task);
}
public void ConnectionStatusChangesHandler(ConnectionStatus status, ConnectionStatusChangeReason reason)
{
ConnectionStatusChangesHandlerCount++;
LastConnectionStatus = status;
LastConnectionStatusChangeReason = reason;
_logger.Trace($"{nameof(PoolingOverAmqp)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={reason} count={ConnectionStatusChangesHandlerCount}");
}
public static async Task Twin_DeviceSetsReportedPropertyAndGetsItBackAsync(DeviceClient deviceClient, string deviceId, object propValue, MsTestLogger logger)
{
var propName = Guid.NewGuid().ToString();
logger.Trace($"{nameof(Twin_DeviceSetsReportedPropertyAndGetsItBackAsync)}: name={propName}, value={propValue}");
var props = new TwinCollection();
props[propName] = propValue;
await deviceClient.UpdateReportedPropertiesAsync(props).ConfigureAwait(false);
// Validate the updated twin from the device-client
Twin deviceTwin = await deviceClient.GetTwinAsync().ConfigureAwait(false);
var actual = deviceTwin.Properties.Reported[propName];
Assert.AreEqual(JsonConvert.SerializeObject(actual), JsonConvert.SerializeObject(propValue));
// Validate the updated twin from the service-client
Twin completeTwin = await _registryManager.GetTwinAsync(deviceId).ConfigureAwait(false);
var actualProp = completeTwin.Properties.Reported[propName];
Assert.AreEqual(JsonConvert.SerializeObject(actualProp), JsonConvert.SerializeObject(propValue));
}
public void ConnectionStatusChangesHandler(ConnectionStatus status, ConnectionStatusChangeReason reason)
{
ConnectionStatusChangeCount++;
LastConnectionStatus = status;
LastConnectionStatusChangeReason = reason;
_logger.Trace($"{nameof(AmqpConnectionStatusChange)}.{nameof(ConnectionStatusChangesHandler)}: {_deviceId}: status={status} statusChangeReason={reason} count={ConnectionStatusChangeCount}");
}
private static Task <Task> SetDeviceReceiveMethodObsoleteHandler(DeviceClient deviceClient, string methodName, MsTestLogger logger)
{
var methodCallReceived = new TaskCompletionSource <bool>();
#pragma warning disable CS0618
deviceClient.SetMethodHandler(methodName, (request, context) =>
{
logger.Trace($"{nameof(SetDeviceReceiveMethodObsoleteHandler)}: DeviceClient method: {request.Name} {request.ResponseTimeout}.");
try
{
Assert.AreEqual(methodName, request.Name, $"The expected method name should be {methodName} but was {request.Name}");
Assert.AreEqual(ServiceRequestJson, request.DataAsJson, $"The expected respose payload should be {ServiceRequestJson} but was {request.DataAsJson}");
methodCallReceived.SetResult(true);
}
catch (Exception ex)
{
methodCallReceived.SetException(ex);
}
return(Task.FromResult(new MethodResponse(Encoding.UTF8.GetBytes(DeviceResponseJson), 200)));
}, null);
#pragma warning restore CS0618
return(Task.FromResult <Task>(methodCallReceived.Task));
}
public static async Task <Task> SetModuleReceiveMethodDefaultHandlerAsync(ModuleClient moduleClient, string methodName, MsTestLogger logger)
{
var methodCallReceived = new TaskCompletionSource <bool>();
await moduleClient.SetMethodDefaultHandlerAsync(
(request, context) =>
{
logger.Trace($"{nameof(SetDeviceReceiveMethodDefaultHandlerAsync)}: ModuleClient method: {request.Name} {request.ResponseTimeout}.");
try
{
Assert.AreEqual(methodName, request.Name, $"The expected method name should be {methodName} but was {request.Name}");
Assert.AreEqual(ServiceRequestJson, request.DataAsJson, $"The expected respose payload should be {ServiceRequestJson} but was {request.DataAsJson}");
methodCallReceived.SetResult(true);
}
catch (Exception ex)
{
methodCallReceived.SetException(ex);
}
return(Task.FromResult(new MethodResponse(Encoding.UTF8.GetBytes(DeviceResponseJson), 200)));
},
null).ConfigureAwait(false);
return(methodCallReceived.Task);
}
private static async Task <TestDevice> CreateDeviceAsync(TestDeviceType type, string prefix)
{
string deviceName = "E2E_" + prefix + Guid.NewGuid();
// Delete existing devices named this way and create a new one.
using var rm = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);
_logger.Trace($"{nameof(GetTestDeviceAsync)}: Creating device {deviceName} with type {type}.");
Client.IAuthenticationMethod auth = null;
var requestDevice = new Device(deviceName);
if (type == TestDeviceType.X509)
{
requestDevice.Authentication = new AuthenticationMechanism
{
X509Thumbprint = new X509Thumbprint
{
PrimaryThumbprint = Configuration.IoTHub.GetCertificateWithPrivateKey().Thumbprint
}
};
auth = new DeviceAuthenticationWithX509Certificate(deviceName, Configuration.IoTHub.GetCertificateWithPrivateKey());
}
Device device = await rm.AddDeviceAsync(requestDevice).ConfigureAwait(false);
_logger.Trace($"{nameof(GetTestDeviceAsync)}: Pausing for {DelayAfterDeviceCreationSeconds}s after device was created.");
await Task.Delay(DelayAfterDeviceCreationSeconds * 1000).ConfigureAwait(false);
await rm.CloseAsync().ConfigureAwait(false);
return(new TestDevice(device, auth));
}
public static async Task VerifyReceivedC2dMessageWithCancellationTokenAsync(Client.TransportType transport, DeviceClient dc, string deviceId, string payload, string p1Value, MsTestLogger logger)
{
var sw = new Stopwatch();
bool received = false;
sw.Start();
while (!received && sw.ElapsedMilliseconds < FaultInjection.RecoveryTimeMilliseconds)
{
logger.Trace($"Receiving messages for device {deviceId}.");
using var cts = new CancellationTokenSource(s_oneMinute);
using Client.Message receivedMessage = await dc.ReceiveAsync(cts.Token).ConfigureAwait(false);
if (receivedMessage == null)
{
Assert.Fail($"No message is received for device {deviceId} in {s_oneMinute}.");
}
try
{
// always complete message
await dc.CompleteAsync(receivedMessage).ConfigureAwait(false);
}
catch (Exception)
{
// ignore exception from CompleteAsync
}
string messageData = Encoding.ASCII.GetString(receivedMessage.GetBytes());
logger.Trace($"{nameof(VerifyReceivedC2dMessageWithCancellationTokenAsync)}: Received message: for {deviceId}: {messageData}");
if (Equals(payload, messageData))
{
Assert.AreEqual(1, receivedMessage.Properties.Count, $"The count of received properties did not match for device {deviceId}");
System.Collections.Generic.KeyValuePair <string, string> prop = receivedMessage.Properties.Single();
Assert.AreEqual("property1", prop.Key, $"The key \"property1\" did not match for device {deviceId}");
Assert.AreEqual(p1Value, prop.Value, $"The value of \"property1\" did not match for device {deviceId}");
received = true;
}
}
sw.Stop();
Assert.IsTrue(received, $"No message received for device {deviceId} with payload={payload} in {FaultInjection.RecoveryTimeMilliseconds}.");
}
/// <summary>
/// Factory method.
/// </summary>
/// <param name="namePrefix"></param>
/// <param name="type"></param>
public static async Task <TestModule> GetTestModuleAsync(string deviceNamePrefix, string moduleNamePrefix, MsTestLogger logger)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(logger, deviceNamePrefix).ConfigureAwait(false);
string deviceName = testDevice.Id;
string moduleName = moduleNamePrefix + Guid.NewGuid();
using var rm = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);
logger.Trace($"{nameof(GetTestModuleAsync)}: Creating module for device {deviceName}.");
var requestModule = new Module(deviceName, moduleName);
Module module = await rm.AddModuleAsync(requestModule).ConfigureAwait(false);
await rm.CloseAsync().ConfigureAwait(false);
var ret = new TestModule(module);
logger.Trace($"{nameof(GetTestModuleAsync)}: Using device {ret.DeviceId} with module {ret.Id}.");
return(ret);
}
public static (Client.Message message, string payload, string p1Value) ComposeD2cTestMessageOfSpecifiedSize(int messageSize, MsTestLogger logger)
{
string messageId = Guid.NewGuid().ToString();
string payload = $"{Guid.NewGuid()}_{new string('*', messageSize)}";
string p1Value = Guid.NewGuid().ToString();
logger.Trace($"{nameof(ComposeD2cTestMessageOfSpecifiedSize)}: messageId='{messageId}' payload='{payload}' p1Value='{p1Value}'");
var message = new Client.Message(Encoding.UTF8.GetBytes(payload))
{
MessageId = messageId,
};
message.Properties.Add("property1", p1Value);
message.Properties.Add("property2", null);
return(message, payload, p1Value);
}
public static (Message message, string payload, string p1Value) ComposeC2dTestMessage(MsTestLogger logger)
{
var payload = Guid.NewGuid().ToString();
var messageId = Guid.NewGuid().ToString();
var p1Value = Guid.NewGuid().ToString();
var userId = Guid.NewGuid().ToString();
logger.Trace($"{nameof(ComposeC2dTestMessage)}: messageId='{messageId}' userId='{userId}' payload='{payload}' p1Value='{p1Value}'");
var message = new Message(Encoding.UTF8.GetBytes(payload))
{
MessageId = messageId,
UserId = userId,
Properties = { ["property1"] = p1Value }
};
return(message, payload, p1Value);
}
private static async Task <TestDevice> CreateDeviceAsync(TestDeviceType type, string prefix)
{
string deviceName = "E2E_" + prefix + Guid.NewGuid();
// Delete existing devices named this way and create a new one.
using var rm = RegistryManager.CreateFromConnectionString(TestConfiguration.IoTHub.ConnectionString);
s_logger.Trace($"{nameof(GetTestDeviceAsync)}: Creating device {deviceName} with type {type}.");
Client.IAuthenticationMethod auth = null;
var requestDevice = new Device(deviceName);
X509Certificate2 authCertificate = null;
if (type == TestDeviceType.X509)
{
requestDevice.Authentication = new AuthenticationMechanism
{
X509Thumbprint = new X509Thumbprint
{
PrimaryThumbprint = TestConfiguration.IoTHub.GetCertificateWithPrivateKey().Thumbprint
}
};
#pragma warning disable CA2000 // Dispose objects before losing scope - X509Certificate and DeviceAuthenticationWithX509Certificate are disposed when TestDevice is disposed.
authCertificate = TestConfiguration.IoTHub.GetCertificateWithPrivateKey();
auth = new DeviceAuthenticationWithX509Certificate(deviceName, authCertificate);
#pragma warning restore CA2000 // Dispose objects before losing scope - X509Certificate and DeviceAuthenticationWithX509Certificate are disposed when TestDevice is disposed.
}
Device device = null;
await RetryOperationHelper
.RetryOperationsAsync(
async() =>
{
device = await rm.AddDeviceAsync(requestDevice).ConfigureAwait(false);
},
s_exponentialBackoffRetryStrategy,
s_retryableExceptions,
s_logger)
.ConfigureAwait(false);
await rm.CloseAsync().ConfigureAwait(false);
return(device == null
? throw new Exception($"Exhausted attempts for creating device {device.Id}, requests got throttled.")
: new TestDevice(device, auth)
{
_authCertificate = authCertificate,
});
}
/// <summary>
/// Factory method.
/// </summary>
/// <param name="namePrefix">The prefix to apply to your device name</param>
/// <param name="type">The way the device will authenticate</param>
public static async Task <TestDevice> GetTestDeviceAsync(MsTestLogger logger, string namePrefix, TestDeviceType type = TestDeviceType.Sasl)
{
s_logger = logger;
string prefix = namePrefix + type + "_";
try
{
await s_semaphore.WaitAsync().ConfigureAwait(false);
TestDevice ret = await CreateDeviceAsync(type, prefix).ConfigureAwait(false);
s_logger.Trace($"{nameof(GetTestDeviceAsync)}: Using device {ret.Id}.");
return(ret);
}
finally
{
s_semaphore.Release();
}
}
public static (Client.Message message, string payload, string p1Value) ComposeD2cTestMessage(MsTestLogger logger)
{
string messageId = Guid.NewGuid().ToString();
string payload = Guid.NewGuid().ToString();
string p1Value = Guid.NewGuid().ToString();
string userId = Guid.NewGuid().ToString();
logger.Trace($"{nameof(ComposeD2cTestMessage)}: messageId='{messageId}' userId='{userId}' payload='{payload}' p1Value='{p1Value}'");
var message = new Client.Message(Encoding.UTF8.GetBytes(payload))
{
MessageId = messageId,
UserId = userId,
};
message.Properties.Add("property1", p1Value);
message.Properties.Add("property2", null);
return(message, payload, p1Value);
}
public static async Task <Task> SubscribeAndUnsubscribeMethodAsync(DeviceClient deviceClient, string methodName, MsTestLogger logger)
{
var methodCallReceived = new TaskCompletionSource <bool>();
await deviceClient
.SetMethodHandlerAsync(
methodName,
(request, context) =>
{
logger.Trace($"{nameof(SubscribeAndUnsubscribeMethodAsync)}: DeviceClient method: {request.Name} {request.ResponseTimeout}.");
return(Task.FromResult(new MethodResponse(Encoding.UTF8.GetBytes(DeviceResponseJson), 200)));
},
null)
.ConfigureAwait(false);
await deviceClient.SetMethodHandlerAsync(methodName, null, null).ConfigureAwait(false);
// Return the task that tells us we have received the callback.
return(methodCallReceived.Task);
}
private static async Task <Task> SetTwinPropertyUpdateCallbackObsoleteHandlerAsync(DeviceClient deviceClient, string expectedPropName, object expectedPropValue, MsTestLogger logger)
{
#pragma warning disable CS0618
string userContext = "myContext";
var propertyUpdateReceived = new TaskCompletionSource <bool>();
await deviceClient
.SetDesiredPropertyUpdateCallback(
(patch, context) =>
{
logger.Trace($"{nameof(SetTwinPropertyUpdateCallbackHandlerAsync)}: DesiredProperty: {patch}, {context}");
try
{
Assert.AreEqual(expectedPropValue, patch[expectedPropName].ToString());
Assert.AreEqual(userContext, context, "Context");
}
catch (Exception e)
{
propertyUpdateReceived.SetException(e);
}
finally
{
propertyUpdateReceived.SetResult(true);
}
return(Task.FromResult <bool>(true));
},
userContext)
.ConfigureAwait(false);
#pragma warning restore CS0618
return(propertyUpdateReceived.Task);
}
public const int TestSuccessRate = 80; // 4 out of 5 (80%) test runs should pass (even after accounting for network instability issues).
public static async Task TestPoolAmqpAsync(
string devicePrefix,
Client.TransportType transport,
int poolSize,
int devicesCount,
Func <DeviceClient, TestDevice, Task> initOperation,
Func <DeviceClient, TestDevice, Task> testOperation,
Func <Task> cleanupOperation,
ConnectionStringAuthScope authScope,
bool ignoreConnectionStatus,
MsTestLogger logger)
{
var transportSettings = new ITransportSettings[]
{
new AmqpTransportSettings(transport)
{
AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
{
MaxPoolSize = unchecked ((uint)poolSize),
Pooling = true
}
}
};
int totalRuns = 0;
int successfulRuns = 0;
int currentSuccessRate = 0;
bool reRunTest = false;
IList <TestDevice> testDevices = new List <TestDevice>();
IList <DeviceClient> deviceClients = new List <DeviceClient>();
IList <AmqpConnectionStatusChange> amqpConnectionStatuses = new List <AmqpConnectionStatusChange>();
IList <Task> operations = new List <Task>();
do
{
totalRuns++;
// Arrange
// Initialize the test device client instances
// Set the device client connection status change handler
logger.Trace($">>> {nameof(PoolingOverAmqp)} Initializing Device Clients for multiplexing test - Test run {totalRuns}");
for (int i = 0; i < devicesCount; i++)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(logger, $"{devicePrefix}_{i}_").ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
var amqpConnectionStatusChange = new AmqpConnectionStatusChange(logger);
deviceClient.SetConnectionStatusChangesHandler(amqpConnectionStatusChange.ConnectionStatusChangesHandler);
testDevices.Add(testDevice);
deviceClients.Add(deviceClient);
amqpConnectionStatuses.Add(amqpConnectionStatusChange);
if (initOperation != null)
{
operations.Add(initOperation(deviceClient, testDevice));
}
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
try
{
for (int i = 0; i < devicesCount; i++)
{
operations.Add(testOperation(deviceClients[i], testDevices[i]));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
// Close the device client instances and verify the connection status change checks
bool deviceConnectionStatusAsExpected = true;
for (int i = 0; i < devicesCount; i++)
{
await deviceClients[i].CloseAsync().ConfigureAwait(false);
if (!ignoreConnectionStatus)
{
// The connection status change count should be 2: connect (open) and disabled (close)
if (amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount != 2)
{
deviceConnectionStatusAsExpected = false;
}
// The connection status should be "Disabled", with connection status change reason "Client_close"
Assert.AreEqual(
ConnectionStatus.Disabled,
amqpConnectionStatuses[i].LastConnectionStatus,
$"The actual connection status is = {amqpConnectionStatuses[i].LastConnectionStatus}");
Assert.AreEqual(
ConnectionStatusChangeReason.Client_Close,
amqpConnectionStatuses[i].LastConnectionStatusChangeReason,
$"The actual connection status change reason is = {amqpConnectionStatuses[i].LastConnectionStatusChangeReason}");
}
}
if (deviceConnectionStatusAsExpected)
{
successfulRuns++;
}
currentSuccessRate = (int)((double)successfulRuns / totalRuns * 100);
reRunTest = currentSuccessRate < TestSuccessRate;
}
finally
{
// Close the service-side components and dispose the device client instances.
if (cleanupOperation != null)
{
await cleanupOperation().ConfigureAwait(false);
}
foreach (DeviceClient deviceClient in deviceClients)
{
deviceClient.Dispose();
}
// Clean up the local lists
testDevices.Clear();
deviceClients.Clear();
amqpConnectionStatuses.Clear();
}
} while (reRunTest && totalRuns < MaxTestRunCount);
Assert.IsFalse(reRunTest, $"Device client instances got disconnected in {totalRuns - successfulRuns} runs out of {totalRuns}; current testSuccessRate = {currentSuccessRate}%.");
}
private static void GenerateSignedCertificateFiles(
string leafCertificateSubject,
string signingIntermediateCertificateSubject,
DirectoryInfo destinationCertificateFolder,
MsTestLogger logger)
{
string signingCertificateKeyFile = Path.Combine(destinationCertificateFolder.FullName, $"{signingIntermediateCertificateSubject}.key");
string signingCertificateCerFile = Path.Combine(destinationCertificateFolder.FullName, $"{signingIntermediateCertificateSubject}.cer");
string leafCertificateKeyFile = Path.Combine(destinationCertificateFolder.FullName, $"{leafCertificateSubject}.key");
string leafCertificateCsrFile = Path.Combine(destinationCertificateFolder.FullName, $"{leafCertificateSubject}.csr");
string leafCertificateCerFile = Path.Combine(destinationCertificateFolder.FullName, $"{leafCertificateSubject}.cer");
string leafCertificatePfxFile = Path.Combine(destinationCertificateFolder.FullName, $"{leafCertificateSubject}.pfx");
// Generate the private key for the certificate
logger.Trace($"Generating the private key for the certificate with subject {leafCertificateSubject} using ...\n");
string keyGen = $"genpkey" +
$" -out \"{leafCertificateKeyFile}\"" +
$" -algorithm RSA" +
$" -pkeyopt rsa_keygen_bits:2048";
logger.Trace($"openssl {keyGen}\n");
using Process keyGenCmdProcess = CreateErrorObservantProcess("openssl", keyGen);
keyGenCmdProcess.Start();
keyGenCmdProcess.WaitForExit();
keyGenCmdProcess.ExitCode.Should().Be(0, $"\"{keyGen}\" exited with error {keyGenCmdProcess.StandardError.ReadToEnd()}.");
// Generate the certificate signing request for the certificate
logger.Trace($"Generating the certificate signing request for the certificate with subject {leafCertificateSubject} using ...\n");
string csrGen = $"req" +
$" -new" +
$" -subj /CN={leafCertificateSubject}" +
$" -key \"{leafCertificateKeyFile}\"" +
$" -out \"{leafCertificateCsrFile}\"";
logger.Trace($"openssl {csrGen}\n");
using Process csrGenCmdProcess = CreateErrorObservantProcess("openssl", csrGen);
csrGenCmdProcess.Start();
csrGenCmdProcess.WaitForExit();
csrGenCmdProcess.ExitCode.Should().Be(0, $"\"{csrGen}\" exited with error {csrGenCmdProcess.StandardError.ReadToEnd()}.");
string signGen;
// This is a request to generate a self-signed certificate.
if (string.IsNullOrWhiteSpace(signingIntermediateCertificateSubject))
{
// Self-sign the certificate signing request generating a file containing the public certificate information
logger.Trace($"Self-sign the certificate with subject {leafCertificateSubject} using ...\n");
signGen = $"x509" +
$" -req" +
$" -days 7" +
$" -in \"{leafCertificateCsrFile}\"" +
$" -signkey \"{leafCertificateKeyFile}\"" +
$" -out \"{leafCertificateCerFile}\"";
}
// This is a request to generate a certificate signed by a verified intermediate certificate
else
{
// Use the public certificate and private keys from the intermediate certificate to sign the leaf device certificate.
logger.Trace($"Sign the certificate with subject {leafCertificateSubject} using the keys from intermediate certificate with subject {signingIntermediateCertificateSubject} ...\n");
signGen = $"x509" +
$" -req" +
$" -days 7" +
$" -in \"{leafCertificateCsrFile}\"" +
$" -CA \"{signingCertificateCerFile}\"" +
$" -CAkey \"{signingCertificateKeyFile}\"" +
$" -CAcreateserial" +
$" -out \"{leafCertificateCerFile}\"";
}
logger.Trace($"openssl {signGen}\n");
using Process signGenCmdProcess = CreateErrorObservantProcess("openssl", signGen);
signGenCmdProcess.Start();
signGenCmdProcess.WaitForExit();
signGenCmdProcess.ExitCode.Should().Be(0, $"\"{signGen}\" exited with error {signGenCmdProcess.StandardError.ReadToEnd()}.");
// Generate the pfx file containing both public certificate and private key information
logger.Trace($"Generating {leafCertificateSubject}.pfx file using ...\n");
string pfxGen = $"pkcs12" +
$" -export" +
$" -in \"{leafCertificateCerFile}\"" +
$" -inkey \"{leafCertificateKeyFile}\"" +
$" -out \"{leafCertificatePfxFile}\"" +
$" -passout pass:"******"openssl {pfxGen}\n");
using Process pfxGenCmdProcess = CreateErrorObservantProcess("openssl", pfxGen);
pfxGenCmdProcess.Start();
pfxGenCmdProcess.WaitForExit();
pfxGenCmdProcess.ExitCode.Should().Be(0, $"\"{pfxGen}\" exited with error {pfxGenCmdProcess.StandardError.ReadToEnd()}.");
}
// Error injection template method.
public static async Task TestErrorInjectionAsync(
string devicePrefix,
TestDeviceType type,
Client.TransportType transport,
string proxyAddress,
string faultType,
string reason,
int delayInSec,
int durationInSec,
Func <DeviceClient, TestDevice, Task> initOperation,
Func <DeviceClient, TestDevice, Task> testOperation,
Func <Task> cleanupOperation,
MsTestLogger logger)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(logger, devicePrefix, type).ConfigureAwait(false);
ITransportSettings transportSettings = CreateTransportSettingsFromName(transport, proxyAddress);
DeviceClient deviceClient = testDevice.CreateDeviceClient(new ITransportSettings[] { transportSettings });
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int connectionStatusChangeCount = 0;
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
connectionStatusChangeCount++;
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
logger.Trace($"{nameof(FaultInjection)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={connectionStatusChangeCount}");
});
var watch = new Stopwatch();
try
{
await deviceClient.OpenAsync().ConfigureAwait(false);
if (transport != Client.TransportType.Http1)
{
Assert.IsTrue(connectionStatusChangeCount >= 1, $"The expected connection status change should be equal or greater than 1 but was {connectionStatusChangeCount}"); // Normally one connection but in some cases, due to network issues we might have already retried several times to connect.
Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Connected} but was {lastConnectionStatus}");
Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Connection_Ok} but was {lastConnectionStatusChangeReason}");
}
await initOperation(deviceClient, testDevice).ConfigureAwait(false);
logger.Trace($">>> {nameof(FaultInjection)} Testing baseline");
await testOperation(deviceClient, testDevice).ConfigureAwait(false);
int countBeforeFaultInjection = connectionStatusChangeCount;
watch.Start();
logger.Trace($">>> {nameof(FaultInjection)} Testing fault handling");
await ActivateFaultInjectionAsync(transport, faultType, reason, delayInSec, durationInSec, deviceClient, logger).ConfigureAwait(false);
logger.Trace($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {delayInSec} seconds.");
await Task.Delay(TimeSpan.FromSeconds(delayInSec)).ConfigureAwait(false);
// For disconnect type faults, the device should disconnect and recover.
if (FaultShouldDisconnect(faultType))
{
logger.Trace($"{nameof(FaultInjection)}: Confirming fault injection has been activated.");
// Check that service issued the fault to the faulting device
bool isFaulted = false;
for (int i = 0; i < LatencyTimeBufferInSec; i++)
{
if (connectionStatusChangeCount > countBeforeFaultInjection)
{
isFaulted = true;
break;
}
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
Assert.IsTrue(isFaulted, $"The device {testDevice.Id} did not get faulted with fault type: {faultType}");
logger.Trace($"{nameof(FaultInjection)}: Confirmed fault injection has been activated.");
// Check the device is back online
logger.Trace($"{nameof(FaultInjection)}: Confirming device back online.");
for (int i = 0; lastConnectionStatus != ConnectionStatus.Connected && i < durationInSec + LatencyTimeBufferInSec; i++)
{
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus, $"{testDevice.Id} did not reconnect.");
logger.Trace($"{nameof(FaultInjection)}: Confirmed device back online.");
// Perform the test operation.
logger.Trace($">>> {nameof(FaultInjection)}: Performing test operation for device {testDevice.Id}.");
await testOperation(deviceClient, testDevice).ConfigureAwait(false);
}
else
{
logger.Trace($"{nameof(FaultInjection)}: Performing test operation while fault injection is being activated.");
// Perform the test operation for the faulted device multi times.
for (int i = 0; i < LatencyTimeBufferInSec; i++)
{
logger.Trace($">>> {nameof(FaultInjection)}: Performing test operation for device - Run {i}.");
await testOperation(deviceClient, testDevice).ConfigureAwait(false);
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
}
await deviceClient.CloseAsync().ConfigureAwait(false);
if (transport != Client.TransportType.Http1)
{
if (FaultInjection.FaultShouldDisconnect(faultType))
{
// 4 is the minimum notification count: connect, fault, reconnect, disable.
// There are cases where the retry must be timed out (i.e. very likely for MQTT where otherwise
// we would attempt to send the fault injection forever.)
Assert.IsTrue(connectionStatusChangeCount >= 4, $"The expected connection status change count for {testDevice.Id} should be equal or greater than 4 but was {connectionStatusChangeCount}");
}
else
{
// 2 is the minimum notification count: connect, disable.
// We will monitor the test environment real network stability and switch to >=2 if necessary to
// account for real network issues.
Assert.IsTrue(connectionStatusChangeCount == 2, $"The expected connection status change count for {testDevice.Id} should be 2 but was {connectionStatusChangeCount}");
}
Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Disabled} but was {lastConnectionStatus}");
Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Client_Close} but was {lastConnectionStatusChangeReason}");
}
}
finally
{
await cleanupOperation().ConfigureAwait(false);
logger.Trace($"{nameof(FaultInjection)}: Disposing deviceClient {TestLogger.GetHashCode(deviceClient)}");
deviceClient.Dispose();
watch.Stop();
int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds;
if (timeToFinishFaultInjection > 0)
{
logger.Trace($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed.");
await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false);
}
}
}
public static async Task VerifyReceivedC2DMessageAsync(Client.TransportType transport, DeviceClient dc, string deviceId, Message message, string payload, MsTestLogger logger)
{
string receivedMessageDestination = $"/devices/{deviceId}/messages/deviceBound";
var sw = new Stopwatch();
bool received = false;
sw.Start();
while (!received && sw.ElapsedMilliseconds < FaultInjection.RecoveryTimeMilliseconds)
{
Client.Message receivedMessage = null;
try
{
logger.Trace($"Receiving messages for device {deviceId}.");
if (transport == Client.TransportType.Http1)
{
// timeout on HTTP is not supported
receivedMessage = await dc.ReceiveAsync().ConfigureAwait(false);
}
else
{
receivedMessage = await dc.ReceiveAsync(s_oneMinute).ConfigureAwait(false);
}
if (receivedMessage == null)
{
Assert.Fail($"No message is received for device {deviceId} in {s_oneMinute}.");
}
try
{
// always complete message
await dc.CompleteAsync(receivedMessage).ConfigureAwait(false);
}
catch (Exception)
{
// ignore exception from CompleteAsync
}
Assert.AreEqual(receivedMessage.MessageId, message.MessageId, "Recieved message Id is not what was sent by service");
Assert.AreEqual(receivedMessage.UserId, message.UserId, "Recieved user Id is not what was sent by service");
Assert.AreEqual(receivedMessage.To, receivedMessageDestination, "Recieved message destination is not what was sent by service");
string messageData = Encoding.ASCII.GetString(receivedMessage.GetBytes());
logger.Trace($"{nameof(VerifyReceivedC2DMessageAsync)}: Received message: for {deviceId}: {messageData}");
if (Equals(payload, messageData))
{
Assert.AreEqual(1, receivedMessage.Properties.Count, $"The count of received properties did not match for device {deviceId}");
System.Collections.Generic.KeyValuePair <string, string> prop = receivedMessage.Properties.Single();
string propertyKey = "property1";
Assert.AreEqual(propertyKey, prop.Key, $"The key \"property1\" did not match for device {deviceId}");
Assert.AreEqual(message.Properties[propertyKey], prop.Value, $"The value of \"property1\" did not match for device {deviceId}");
received = true;
}
}
finally
{
receivedMessage?.Dispose();
}
}
sw.Stop();
Assert.IsTrue(received, $"No message received for device {deviceId} with payload={payload} in {FaultInjection.RecoveryTimeMilliseconds}.");
}
public bool IsBypassed(Uri host)
{
Interlocked.Increment(ref _counter);
_logger.Trace($"{nameof(CustomWebProxy)}.{nameof(IsBypassed)} Uri = {host}");
return(false);
}
public static async Task TestFaultInjectionPoolAmqpAsync(
string devicePrefix,
Client.TransportType transport,
string proxyAddress,
int poolSize,
int devicesCount,
string faultType,
string reason,
int delayInSec,
int durationInSec,
Func <DeviceClient, TestDevice, TestDeviceCallbackHandler, Task> initOperation,
Func <DeviceClient, TestDevice, TestDeviceCallbackHandler, Task> testOperation,
Func <IList <DeviceClient>, Task> cleanupOperation,
ConnectionStringAuthScope authScope,
MsTestLogger logger)
{
var transportSettings = new ITransportSettings[]
{
new AmqpTransportSettings(transport)
{
AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
{
MaxPoolSize = unchecked ((uint)poolSize),
Pooling = true,
},
Proxy = proxyAddress == null ? null : new WebProxy(proxyAddress),
}
};
IList <TestDevice> testDevices = new List <TestDevice>();
IList <DeviceClient> deviceClients = new List <DeviceClient>();
IList <TestDeviceCallbackHandler> testDeviceCallbackHandlers = new List <TestDeviceCallbackHandler>();
IList <AmqpConnectionStatusChange> amqpConnectionStatuses = new List <AmqpConnectionStatusChange>();
IList <Task> operations = new List <Task>();
// Arrange
// Initialize the test device client instances
// Set the device client connection status change handler
logger.Trace($">>> {nameof(FaultInjectionPoolingOverAmqp)} Initializing Device Clients for multiplexing test.");
for (int i = 0; i < devicesCount; i++)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(logger, $"{devicePrefix}_{i}_").ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
var amqpConnectionStatusChange = new AmqpConnectionStatusChange(testDevice.Id, logger);
deviceClient.SetConnectionStatusChangesHandler(amqpConnectionStatusChange.ConnectionStatusChangesHandler);
var testDeviceCallbackHandler = new TestDeviceCallbackHandler(deviceClient, testDevice, logger);
testDevices.Add(testDevice);
deviceClients.Add(deviceClient);
testDeviceCallbackHandlers.Add(testDeviceCallbackHandler);
amqpConnectionStatuses.Add(amqpConnectionStatusChange);
operations.Add(initOperation(deviceClient, testDevice, testDeviceCallbackHandler));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
var watch = new Stopwatch();
try
{
// Act-Assert
// Perform the test operation and verify the operation is successful
// Perform baseline test operation
for (int i = 0; i < devicesCount; i++)
{
logger.Trace($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing baseline operation for device {i}.");
operations.Add(testOperation(deviceClients[i], testDevices[i], testDeviceCallbackHandlers[i]));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
int countBeforeFaultInjection = amqpConnectionStatuses[0].ConnectionStatusChangeCount;
// Inject the fault into device 0
watch.Start();
logger.Trace($"{nameof(FaultInjectionPoolingOverAmqp)}: {testDevices[0].Id} Requesting fault injection type={faultType} reason={reason}, delay={delayInSec}s, duration={durationInSec}s");
var faultInjectionMessage = FaultInjection.ComposeErrorInjectionProperties(faultType, reason, delayInSec, durationInSec);
await deviceClients[0].SendEventAsync(faultInjectionMessage).ConfigureAwait(false);
logger.Trace($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {delayInSec} seconds.");
await Task.Delay(TimeSpan.FromSeconds(delayInSec)).ConfigureAwait(false);
// For disconnect type faults, the faulted device should disconnect and all devices should recover.
if (FaultInjection.FaultShouldDisconnect(faultType))
{
logger.Trace($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirming fault injection has been actived.");
// Check that service issued the fault to the faulting device [device 0]
bool isFaulted = false;
for (int i = 0; i < FaultInjection.LatencyTimeBufferInSec; i++)
{
if (amqpConnectionStatuses[0].ConnectionStatusChangeCount > countBeforeFaultInjection)
{
isFaulted = true;
break;
}
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
Assert.IsTrue(isFaulted, $"The device {testDevices[0].Id} did not get faulted with fault type: {faultType}");
logger.Trace($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirmed fault injection has been actived.");
// Check all devices are back online
logger.Trace($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirming all devices back online.");
bool notRecovered = true;
int j = 0;
for (int i = 0; notRecovered && i < durationInSec + FaultInjection.LatencyTimeBufferInSec; i++)
{
notRecovered = false;
for (j = 0; j < devicesCount; j++)
{
if (amqpConnectionStatuses[j].LastConnectionStatus != ConnectionStatus.Connected)
{
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
notRecovered = true;
break;
}
}
}
if (notRecovered)
{
Assert.Fail($"{testDevices[j].Id} did not reconnect.");
}
logger.Trace($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirmed all devices back online.");
// Perform the test operation for all devices
for (int i = 0; i < devicesCount; i++)
{
logger.Trace($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation for device {i}.");
operations.Add(testOperation(deviceClients[i], testDevices[i], testDeviceCallbackHandlers[i]));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
}
else
{
logger.Trace($"{nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation while fault injection is being activated.");
// Perform the test operation for the faulted device multi times.
for (int i = 0; i < FaultInjection.LatencyTimeBufferInSec; i++)
{
logger.Trace($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation for device 0 - Run {i}.");
await testOperation(deviceClients[0], testDevices[0], testDeviceCallbackHandlers[0]).ConfigureAwait(false);
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
}
// Close the device client instances
for (int i = 0; i < devicesCount; i++)
{
operations.Add(deviceClients[i].CloseAsync());
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
// Verify the connection status change checks.
// For all of the devices - last connection status should be "Disabled", with reason "Client_close"
for (int i = 0; i < devicesCount; i++)
{
// For the faulted device [device 0] - verify the connection status change count.
if (i == 0)
{
if (FaultInjection.FaultShouldDisconnect(faultType))
{
// 4 is the minimum notification count: connect, fault, reconnect, disable.
Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangeCount >= 4, $"The expected connection status change count for {testDevices[i].Id} should equals or greater than 4 but was {amqpConnectionStatuses[i].ConnectionStatusChangeCount}");
}
else
{
// 2 is the minimum notification count: connect, disable.
Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangeCount >= 2, $"The expected connection status change count for {testDevices[i].Id} should be 2 but was {amqpConnectionStatuses[i].ConnectionStatusChangeCount}");
}
}
Assert.AreEqual(ConnectionStatus.Disabled, amqpConnectionStatuses[i].LastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Disabled} but was {amqpConnectionStatuses[i].LastConnectionStatus}");
Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, amqpConnectionStatuses[i].LastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Client_Close} but was {amqpConnectionStatuses[i].LastConnectionStatusChangeReason}");
}
}
finally
{
// Close the service-side components and dispose the device client instances.
await cleanupOperation(deviceClients).ConfigureAwait(false);
watch.Stop();
int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds;
if (timeToFinishFaultInjection > 0)
{
logger.Trace($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed.");
await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false);
}
}
}