public const int TestSuccessRate = 80; // 4 out of 5 (80%) test runs should pass (even after accounting for network instability issues).
public static async Task TestPoolAmqpAsync(
string devicePrefix,
Client.TransportType transport,
int poolSize,
int devicesCount,
Func <DeviceClient, TestDevice, Task> initOperation,
Func <DeviceClient, TestDevice, Task> testOperation,
Func <Task> cleanupOperation,
ConnectionStringAuthScope authScope,
bool ignoreConnectionStatus,
MsTestLogger logger)
{
var transportSettings = new ITransportSettings[]
{
new AmqpTransportSettings(transport)
{
AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
{
MaxPoolSize = unchecked ((uint)poolSize),
Pooling = true
}
}
};
int totalRuns = 0;
int successfulRuns = 0;
int currentSuccessRate = 0;
bool reRunTest = false;
IList <TestDevice> testDevices = new List <TestDevice>();
IList <DeviceClient> deviceClients = new List <DeviceClient>();
IList <AmqpConnectionStatusChange> amqpConnectionStatuses = new List <AmqpConnectionStatusChange>();
IList <Task> operations = new List <Task>();
do
{
totalRuns++;
// Arrange
// Initialize the test device client instances
// Set the device client connection status change handler
logger.Trace($">>> {nameof(PoolingOverAmqp)} Initializing Device Clients for multiplexing test - Test run {totalRuns}");
for (int i = 0; i < devicesCount; i++)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(logger, $"{devicePrefix}_{i}_").ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
var amqpConnectionStatusChange = new AmqpConnectionStatusChange(logger);
deviceClient.SetConnectionStatusChangesHandler(amqpConnectionStatusChange.ConnectionStatusChangesHandler);
testDevices.Add(testDevice);
deviceClients.Add(deviceClient);
amqpConnectionStatuses.Add(amqpConnectionStatusChange);
if (initOperation != null)
{
operations.Add(initOperation(deviceClient, testDevice));
}
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
try
{
for (int i = 0; i < devicesCount; i++)
{
operations.Add(testOperation(deviceClients[i], testDevices[i]));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
// Close the device client instances and verify the connection status change checks
bool deviceConnectionStatusAsExpected = true;
for (int i = 0; i < devicesCount; i++)
{
await deviceClients[i].CloseAsync().ConfigureAwait(false);
if (!ignoreConnectionStatus)
{
// The connection status change count should be 2: connect (open) and disabled (close)
if (amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount != 2)
{
deviceConnectionStatusAsExpected = false;
}
// The connection status should be "Disabled", with connection status change reason "Client_close"
Assert.AreEqual(
ConnectionStatus.Disabled,
amqpConnectionStatuses[i].LastConnectionStatus,
$"The actual connection status is = {amqpConnectionStatuses[i].LastConnectionStatus}");
Assert.AreEqual(
ConnectionStatusChangeReason.Client_Close,
amqpConnectionStatuses[i].LastConnectionStatusChangeReason,
$"The actual connection status change reason is = {amqpConnectionStatuses[i].LastConnectionStatusChangeReason}");
}
}
if (deviceConnectionStatusAsExpected)
{
successfulRuns++;
}
currentSuccessRate = (int)((double)successfulRuns / totalRuns * 100);
reRunTest = currentSuccessRate < TestSuccessRate;
}
finally
{
// Close the service-side components and dispose the device client instances.
if (cleanupOperation != null)
{
await cleanupOperation().ConfigureAwait(false);
}
foreach (DeviceClient deviceClient in deviceClients)
{
deviceClient.Dispose();
}
// Clean up the local lists
testDevices.Clear();
deviceClients.Clear();
amqpConnectionStatuses.Clear();
}
} while (reRunTest && totalRuns < MaxTestRunCount);
Assert.IsFalse(reRunTest, $"Device client instances got disconnected in {totalRuns - successfulRuns} runs out of {totalRuns}; current testSuccessRate = {currentSuccessRate}%.");
}
public static async Task TestFaultInjectionPoolAmqpAsync(
string devicePrefix,
Client.TransportType transport,
int poolSize,
int devicesCount,
string faultType,
string reason,
int delayInSec,
int durationInSec,
Func <DeviceClient, TestDevice, Task> initOperation,
Func <DeviceClient, TestDevice, Task> testOperation,
Func <IList <DeviceClient>, Task> cleanupOperation,
ConnectionStringAuthScope authScope)
{
var transportSettings = new ITransportSettings[]
{
new AmqpTransportSettings(transport)
{
AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
{
MaxPoolSize = unchecked ((uint)poolSize),
Pooling = true
}
}
};
IList <TestDevice> testDevices = new List <TestDevice>();
IList <DeviceClient> deviceClients = new List <DeviceClient>();
IList <AmqpConnectionStatusChange> amqpConnectionStatuses = new List <AmqpConnectionStatusChange>();
IList <Task> operations = new List <Task>();
// Arrange
// Initialize the test device client instances
// Set the device client connection status change handler
s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)} Initializing Device Clients for multiplexing test.");
for (int i = 0; i < devicesCount; i++)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{devicePrefix}_{i}_").ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
var amqpConnectionStatusChange = new AmqpConnectionStatusChange(testDevice.Id);
deviceClient.SetConnectionStatusChangesHandler(amqpConnectionStatusChange.ConnectionStatusChangesHandler);
testDevices.Add(testDevice);
deviceClients.Add(deviceClient);
amqpConnectionStatuses.Add(amqpConnectionStatusChange);
operations.Add(initOperation(deviceClient, testDevice));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
var watch = new Stopwatch();
try
{
// Act-Assert
// Perform the test operation and verify the operation is successful
// Perform baseline test operation
for (int i = 0; i < devicesCount; i++)
{
s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing baseline operation for device {i}.");
operations.Add(testOperation(deviceClients[i], testDevices[i]));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
// Inject the fault into device 0
watch.Start();
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Device {0} Requesting fault injection type={faultType} reason={reason}, delay={delayInSec}s, duration={durationInSec}s");
var faultInjectionMessage = FaultInjection.ComposeErrorInjectionProperties(faultType, reason, delayInSec, durationInSec);
await deviceClients[0].SendEventAsync(faultInjectionMessage).ConfigureAwait(false);
int delay = FaultInjection.WaitForReconnectMilliseconds - (int)watch.ElapsedMilliseconds;
if (delay < 0)
{
delay = 0;
}
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Waiting for fault injection to be active and device to be connected: {delay}ms");
await Task.Delay(delay).ConfigureAwait(false);
// Perform the test operation for all devices
for (int i = 0; i < devicesCount; i++)
{
s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation for device {i}.");
operations.Add(testOperation(deviceClients[i], testDevices[i]));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
// Close the device client instances
for (int i = 0; i < devicesCount; i++)
{
operations.Add(deviceClients[i].CloseAsync());
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
// Verify the connection status change checks.
// For all of the devices - last connection status should be "Disabled", with reason "Client_close"
for (int i = 0; i < devicesCount; i++)
{
// For the faulted device [device 0] - verify the connection status change count.
if (i == 0)
{
if (FaultInjection.FaultShouldDisconnect(faultType))
{
// 4 is the minimum notification count: connect, fault, reconnect, disable.
Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount >= 4, $"The actual connection status change count for faulted device[0] is = {amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount}");
}
else
{
// 2 is the minimum notification count: connect, disable.
Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount == 2, $"The actual connection status change count for for faulted device[0] is = {amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount}");
}
}
Assert.AreEqual(ConnectionStatus.Disabled, amqpConnectionStatuses[i].LastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, amqpConnectionStatuses[i].LastConnectionStatusChangeReason);
}
}
finally
{
// Close the service-side components and dispose the device client instances.
await cleanupOperation(deviceClients).ConfigureAwait(false);
watch.Stop();
int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds;
if (timeToFinishFaultInjection > 0)
{
s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed.");
await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false);
}
}
}
public static async Task TestFaultInjectionPoolAmqpAsync(
string devicePrefix,
Client.TransportType transport,
string proxyAddress,
int poolSize,
int devicesCount,
string faultType,
string reason,
int delayInSec,
int durationInSec,
Func <DeviceClient, TestDevice, TestDeviceCallbackHandler, Task> initOperation,
Func <DeviceClient, TestDevice, TestDeviceCallbackHandler, Task> testOperation,
Func <IList <DeviceClient>, Task> cleanupOperation,
ConnectionStringAuthScope authScope,
MsTestLogger logger)
{
var transportSettings = new ITransportSettings[]
{
new AmqpTransportSettings(transport)
{
AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
{
MaxPoolSize = unchecked ((uint)poolSize),
Pooling = true,
},
Proxy = proxyAddress == null ? null : new WebProxy(proxyAddress),
}
};
IList <TestDevice> testDevices = new List <TestDevice>();
IList <DeviceClient> deviceClients = new List <DeviceClient>();
IList <TestDeviceCallbackHandler> testDeviceCallbackHandlers = new List <TestDeviceCallbackHandler>();
IList <AmqpConnectionStatusChange> amqpConnectionStatuses = new List <AmqpConnectionStatusChange>();
IList <Task> operations = new List <Task>();
// Arrange
// Initialize the test device client instances
// Set the device client connection status change handler
logger.Trace($">>> {nameof(FaultInjectionPoolingOverAmqp)} Initializing Device Clients for multiplexing test.");
for (int i = 0; i < devicesCount; i++)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(logger, $"{devicePrefix}_{i}_").ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
var amqpConnectionStatusChange = new AmqpConnectionStatusChange(testDevice.Id, logger);
deviceClient.SetConnectionStatusChangesHandler(amqpConnectionStatusChange.ConnectionStatusChangesHandler);
var testDeviceCallbackHandler = new TestDeviceCallbackHandler(deviceClient, testDevice, logger);
testDevices.Add(testDevice);
deviceClients.Add(deviceClient);
testDeviceCallbackHandlers.Add(testDeviceCallbackHandler);
amqpConnectionStatuses.Add(amqpConnectionStatusChange);
operations.Add(initOperation(deviceClient, testDevice, testDeviceCallbackHandler));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
var watch = new Stopwatch();
try
{
// Act-Assert
// Perform the test operation and verify the operation is successful
// Perform baseline test operation
for (int i = 0; i < devicesCount; i++)
{
logger.Trace($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing baseline operation for device {i}.");
operations.Add(testOperation(deviceClients[i], testDevices[i], testDeviceCallbackHandlers[i]));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
int countBeforeFaultInjection = amqpConnectionStatuses[0].ConnectionStatusChangeCount;
// Inject the fault into device 0
watch.Start();
logger.Trace($"{nameof(FaultInjectionPoolingOverAmqp)}: {testDevices[0].Id} Requesting fault injection type={faultType} reason={reason}, delay={delayInSec}s, duration={durationInSec}s");
var faultInjectionMessage = FaultInjection.ComposeErrorInjectionProperties(faultType, reason, delayInSec, durationInSec);
await deviceClients[0].SendEventAsync(faultInjectionMessage).ConfigureAwait(false);
logger.Trace($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {delayInSec} seconds.");
await Task.Delay(TimeSpan.FromSeconds(delayInSec)).ConfigureAwait(false);
// For disconnect type faults, the faulted device should disconnect and all devices should recover.
if (FaultInjection.FaultShouldDisconnect(faultType))
{
logger.Trace($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirming fault injection has been actived.");
// Check that service issued the fault to the faulting device [device 0]
bool isFaulted = false;
for (int i = 0; i < FaultInjection.LatencyTimeBufferInSec; i++)
{
if (amqpConnectionStatuses[0].ConnectionStatusChangeCount > countBeforeFaultInjection)
{
isFaulted = true;
break;
}
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
Assert.IsTrue(isFaulted, $"The device {testDevices[0].Id} did not get faulted with fault type: {faultType}");
logger.Trace($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirmed fault injection has been actived.");
// Check all devices are back online
logger.Trace($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirming all devices back online.");
bool notRecovered = true;
int j = 0;
for (int i = 0; notRecovered && i < durationInSec + FaultInjection.LatencyTimeBufferInSec; i++)
{
notRecovered = false;
for (j = 0; j < devicesCount; j++)
{
if (amqpConnectionStatuses[j].LastConnectionStatus != ConnectionStatus.Connected)
{
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
notRecovered = true;
break;
}
}
}
if (notRecovered)
{
Assert.Fail($"{testDevices[j].Id} did not reconnect.");
}
logger.Trace($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirmed all devices back online.");
// Perform the test operation for all devices
for (int i = 0; i < devicesCount; i++)
{
logger.Trace($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation for device {i}.");
operations.Add(testOperation(deviceClients[i], testDevices[i], testDeviceCallbackHandlers[i]));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
}
else
{
logger.Trace($"{nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation while fault injection is being activated.");
// Perform the test operation for the faulted device multi times.
for (int i = 0; i < FaultInjection.LatencyTimeBufferInSec; i++)
{
logger.Trace($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation for device 0 - Run {i}.");
await testOperation(deviceClients[0], testDevices[0], testDeviceCallbackHandlers[0]).ConfigureAwait(false);
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
}
// Close the device client instances
for (int i = 0; i < devicesCount; i++)
{
operations.Add(deviceClients[i].CloseAsync());
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
// Verify the connection status change checks.
// For all of the devices - last connection status should be "Disabled", with reason "Client_close"
for (int i = 0; i < devicesCount; i++)
{
// For the faulted device [device 0] - verify the connection status change count.
if (i == 0)
{
if (FaultInjection.FaultShouldDisconnect(faultType))
{
// 4 is the minimum notification count: connect, fault, reconnect, disable.
Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangeCount >= 4, $"The expected connection status change count for {testDevices[i].Id} should equals or greater than 4 but was {amqpConnectionStatuses[i].ConnectionStatusChangeCount}");
}
else
{
// 2 is the minimum notification count: connect, disable.
Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangeCount >= 2, $"The expected connection status change count for {testDevices[i].Id} should be 2 but was {amqpConnectionStatuses[i].ConnectionStatusChangeCount}");
}
}
Assert.AreEqual(ConnectionStatus.Disabled, amqpConnectionStatuses[i].LastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Disabled} but was {amqpConnectionStatuses[i].LastConnectionStatus}");
Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, amqpConnectionStatuses[i].LastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Client_Close} but was {amqpConnectionStatuses[i].LastConnectionStatusChangeReason}");
}
}
finally
{
// Close the service-side components and dispose the device client instances.
await cleanupOperation(deviceClients).ConfigureAwait(false);
watch.Stop();
int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds;
if (timeToFinishFaultInjection > 0)
{
logger.Trace($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed.");
await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false);
}
}
}
private async Task ReuseAuthenticationMethod_MuxedDevices(Client.TransportType transport, int devicesCount)
{
IList <TestDevice> testDevices = new List <TestDevice>();
IList <DeviceClient> deviceClients = new List <DeviceClient>();
IList <AuthenticationWithTokenRefresh> authenticationMethods = new List <AuthenticationWithTokenRefresh>();
IList <AmqpConnectionStatusChange> amqpConnectionStatuses = new List <AmqpConnectionStatusChange>();
// Set up amqp transport settings to multiplex all device sessions over the same amqp connection.
var amqpTransportSettings = new AmqpTransportSettings(transport)
{
AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings
{
Pooling = true,
MaxPoolSize = 1,
},
};
for (int i = 0; i < devicesCount; i++)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(Logger, _devicePrefix).ConfigureAwait(false);
#pragma warning disable CA2000 // Dispose objects before losing scope - the authentication method is disposed at the end of the test.
var authenticationMethod = new DeviceAuthenticationSasToken(testDevice.ConnectionString, disposeWithClient: false);
#pragma warning restore CA2000 // Dispose objects before losing scope
testDevices.Add(testDevice);
authenticationMethods.Add(authenticationMethod);
}
// Initialize the client instances, set the connection status change handler and open the connection.
for (int i = 0; i < devicesCount; i++)
{
#pragma warning disable CA2000 // Dispose objects before losing scope - the client instance is disposed during the course of the test.
DeviceClient deviceClient = DeviceClient.Create(testDevices[i].IoTHubHostName, authenticationMethods[i], new ITransportSettings[] { amqpTransportSettings });
#pragma warning restore CA2000 // Dispose objects before losing scope
var amqpConnectionStatusChange = new AmqpConnectionStatusChange(testDevices[i].Id, Logger);
deviceClient.SetConnectionStatusChangesHandler(amqpConnectionStatusChange.ConnectionStatusChangesHandler);
amqpConnectionStatuses.Add(amqpConnectionStatusChange);
await deviceClient.OpenAsync().ConfigureAwait(false);
deviceClients.Add(deviceClient);
}
// Close and dispose client instance 1.
// The closed client should report a status of "disabled" while the rest of them should be connected.
// This is to ensure that disposal on one multiplexed device doesn't cause cascading failures
// in the rest of the devices on the same tcp connection.
await deviceClients[0].CloseAsync().ConfigureAwait(false);
deviceClients[0].Dispose();
amqpConnectionStatuses[0].LastConnectionStatus.Should().Be(ConnectionStatus.Disabled);
Logger.Trace($"{nameof(ReuseAuthenticationMethod_MuxedDevices)}: Confirming the rest of the multiplexed devices are online and operational.");
bool notRecovered = true;
var sw = Stopwatch.StartNew();
while (notRecovered && sw.Elapsed < MaxWaitTime)
{
notRecovered = false;
for (int i = 1; i < devicesCount; i++)
{
if (amqpConnectionStatuses[i].LastConnectionStatus != ConnectionStatus.Connected)
{
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
notRecovered = true;
break;
}
}
}
notRecovered.Should().BeFalse();
// Send a message through the rest of the multiplexed client instances.
var message = new Client.Message();
for (int i = 1; i < devicesCount; i++)
{
await deviceClients[i].SendEventAsync(message).ConfigureAwait(false);
Logger.Trace($"Test with client {i} completed.");
}
message.Dispose();
// Close and dispose all of the client instances.
for (int i = 1; i < devicesCount; i++)
{
await deviceClients[i].CloseAsync().ConfigureAwait(false);
deviceClients[i].Dispose();
}
deviceClients.Clear();
amqpConnectionStatuses.Clear();
// Initialize the client instances by reusing the created authentication methods and open the connection.
for (int i = 0; i < devicesCount; i++)
{
#pragma warning disable CA2000 // Dispose objects before losing scope - the client instance is disposed at the end of the test.
DeviceClient deviceClient = DeviceClient.Create(testDevices[i].IoTHubHostName, authenticationMethods[i], new ITransportSettings[] { amqpTransportSettings });
#pragma warning restore CA2000 // Dispose objects before losing scope
var amqpConnectionStatusChange = new AmqpConnectionStatusChange(testDevices[i].Id, Logger);
deviceClient.SetConnectionStatusChangesHandler(amqpConnectionStatusChange.ConnectionStatusChangesHandler);
amqpConnectionStatuses.Add(amqpConnectionStatusChange);
await deviceClient.OpenAsync().ConfigureAwait(false);
deviceClients.Add(deviceClient);
}
// Ensure that all clients are connected successfully, and the close and dispose the instances.
// Also dispose the authentication methods created.
for (int i = 0; i < devicesCount; i++)
{
amqpConnectionStatuses[i].LastConnectionStatus.Should().Be(ConnectionStatus.Connected);
await deviceClients[i].CloseAsync();
deviceClients[i].Dispose();
authenticationMethods[i].Dispose();
amqpConnectionStatuses[i].LastConnectionStatus.Should().Be(ConnectionStatus.Disabled);
}
}
