public static async Task TestFaultInjectionPoolAmqpAsync(
string devicePrefix,
Client.TransportType transport,
int poolSize,
int devicesCount,
string faultType,
string reason,
int delayInSec,
int durationInSec,
Func <DeviceClient, TestDevice, Task> initOperation,
Func <DeviceClient, TestDevice, Task> testOperation,
Func <IList <DeviceClient>, Task> cleanupOperation,
ConnectionStringAuthScope authScope)
{
var transportSettings = new ITransportSettings[]
{
new AmqpTransportSettings(transport)
{
AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
{
MaxPoolSize = unchecked ((uint)poolSize),
Pooling = true
}
}
};
IList <TestDevice> testDevices = new List <TestDevice>();
IList <DeviceClient> deviceClients = new List <DeviceClient>();
IList <AmqpConnectionStatusChange> amqpConnectionStatuses = new List <AmqpConnectionStatusChange>();
IList <Task> operations = new List <Task>();
// Arrange
// Initialize the test device client instances
// Set the device client connection status change handler
s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)} Initializing Device Clients for multiplexing test.");
for (int i = 0; i < devicesCount; i++)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{devicePrefix}_{i}_").ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
var amqpConnectionStatusChange = new AmqpConnectionStatusChange(testDevice.Id);
deviceClient.SetConnectionStatusChangesHandler(amqpConnectionStatusChange.ConnectionStatusChangesHandler);
testDevices.Add(testDevice);
deviceClients.Add(deviceClient);
amqpConnectionStatuses.Add(amqpConnectionStatusChange);
operations.Add(initOperation(deviceClient, testDevice));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
var watch = new Stopwatch();
try
{
// Act-Assert
// Perform the test operation and verify the operation is successful
// Perform baseline test operation
for (int i = 0; i < devicesCount; i++)
{
s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing baseline operation for device {i}.");
operations.Add(testOperation(deviceClients[i], testDevices[i]));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
// Inject the fault into device 0
watch.Start();
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: {testDevices[0].Id} Requesting fault injection type={faultType} reason={reason}, delay={delayInSec}s, duration={durationInSec}s");
var faultInjectionMessage = FaultInjection.ComposeErrorInjectionProperties(faultType, reason, delayInSec, durationInSec);
await deviceClients[0].SendEventAsync(faultInjectionMessage).ConfigureAwait(false);
s_log.WriteLine($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {delayInSec} seconds.");
await Task.Delay(TimeSpan.FromSeconds(delayInSec)).ConfigureAwait(false);
// For disconnect type faults, the faulted device should disconnect and all devices should recover.
if (FaultInjection.FaultShouldDisconnect(faultType))
{
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirming fault injection has been actived.");
// Check that service issued the fault to the faulting device [device 0]
bool isFaulted = false;
for (int i = 0; i < FaultInjection.LatencyTimeBufferInSec; i++)
{
if (amqpConnectionStatuses[0].ConnectionStatusChangeCount >= 2)
{
isFaulted = true;
break;
}
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
Assert.IsTrue(isFaulted, $"The device {testDevices[0].Id} did not get faulted with fault type: {faultType}");
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirmed fault injection has been actived.");
// Check all devices are back online
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirming all devices back online.");
bool notRecovered = true;
int j = 0;
for (int i = 0; notRecovered && i < durationInSec + FaultInjection.LatencyTimeBufferInSec; i++)
{
notRecovered = false;
for (j = 0; j < devicesCount; j++)
{
if (amqpConnectionStatuses[j].LastConnectionStatus != ConnectionStatus.Connected)
{
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
notRecovered = true;
break;
}
}
}
if (notRecovered)
{
Assert.Fail($"{testDevices[j].Id} did not reconnect.");
}
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirmed all devices back online.");
// Perform the test operation for all devices
for (int i = 0; i < devicesCount; i++)
{
s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation for device {i}.");
operations.Add(testOperation(deviceClients[i], testDevices[i]));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
}
else
{
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation while fault injection is being activated.");
// Perform the test operation for the faulted device multi times.
for (int i = 0; i < FaultInjection.LatencyTimeBufferInSec; i++)
{
s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation for device 0 - Run {i}.");
await testOperation(deviceClients[0], testDevices[0]).ConfigureAwait(false);
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
}
// Close the device client instances
for (int i = 0; i < devicesCount; i++)
{
operations.Add(deviceClients[i].CloseAsync());
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
// Verify the connection status change checks.
// For all of the devices - last connection status should be "Disabled", with reason "Client_close"
for (int i = 0; i < devicesCount; i++)
{
// For the faulted device [device 0] - verify the connection status change count.
if (i == 0)
{
if (FaultInjection.FaultShouldDisconnect(faultType))
{
// 4 is the minimum notification count: connect, fault, reconnect, disable.
Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangeCount >= 4, $"The expected connection status change count for {testDevices[i].Id} should equals or greater than 4 but was {amqpConnectionStatuses[i].ConnectionStatusChangeCount}");
}
else
{
// 2 is the minimum notification count: connect, disable.
Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangeCount == 2, $"The expected connection status change count for {testDevices[i].Id} should be 2 but was {amqpConnectionStatuses[i].ConnectionStatusChangeCount}");
}
}
Assert.AreEqual(ConnectionStatus.Disabled, amqpConnectionStatuses[i].LastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Disabled} but was {amqpConnectionStatuses[i].LastConnectionStatus}");
Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, amqpConnectionStatuses[i].LastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Client_Close} but was {amqpConnectionStatuses[i].LastConnectionStatusChangeReason}");
}
}
finally
{
// Close the service-side components and dispose the device client instances.
await cleanupOperation(deviceClients).ConfigureAwait(false);
watch.Stop();
int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds;
if (timeToFinishFaultInjection > 0)
{
s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed.");
await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false);
}
}
}
// Error injection template method.
public static async Task TestErrorInjectionAsync(
string devicePrefix,
TestDeviceType type,
Client.TransportType transport,
string faultType,
string reason,
int delayInSec,
int durationInSec,
Func <DeviceClient, TestDevice, Task> initOperation,
Func <DeviceClient, TestDevice, Task> testOperation,
Func <Task> cleanupOperation)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(devicePrefix, type).ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transport);
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int connectionStatusChangeCount = 0;
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
connectionStatusChangeCount++;
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
s_log.WriteLine($"{nameof(FaultInjection)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={connectionStatusChangeCount}");
});
var watch = new Stopwatch();
try
{
await deviceClient.OpenAsync().ConfigureAwait(false);
if (transport != Client.TransportType.Http1)
{
Assert.IsTrue(connectionStatusChangeCount >= 1, $"The expected connection status change should be equal or greater than 1 but was {connectionStatusChangeCount}"); // Normally one connection but in some cases, due to network issues we might have already retried several times to connect.
Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Connected} but was {lastConnectionStatus}");
Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Connection_Ok} but was {lastConnectionStatusChangeReason}");
}
await initOperation(deviceClient, testDevice).ConfigureAwait(false);
s_log.WriteLine($">>> {nameof(FaultInjection)} Testing baseline");
await testOperation(deviceClient, testDevice).ConfigureAwait(false);
watch.Start();
s_log.WriteLine($">>> {nameof(FaultInjection)} Testing fault handling");
await ActivateFaultInjection(transport, faultType, reason, delayInSec, durationInSec, deviceClient).ConfigureAwait(false);
s_log.WriteLine($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {delayInSec} seconds.");
await Task.Delay(TimeSpan.FromSeconds(delayInSec)).ConfigureAwait(false);
// For disconnect type faults, the device should disconnect and recover.
if (FaultShouldDisconnect(faultType))
{
s_log.WriteLine($"{nameof(FaultInjection)}: Confirming fault injection has been actived.");
// Check that service issued the fault to the faulting device
bool isFaulted = false;
for (int i = 0; i < LatencyTimeBufferInSec; i++)
{
if (connectionStatusChangeCount >= 2)
{
isFaulted = true;
break;
}
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
Assert.IsTrue(isFaulted, $"The device {testDevice.Id} did not get faulted with fault type: {faultType}");
s_log.WriteLine($"{nameof(FaultInjection)}: Confirmed fault injection has been actived.");
// Check the device is back online
s_log.WriteLine($"{nameof(FaultInjection)}: Confirming device back online.");
for (int i = 0; lastConnectionStatus != ConnectionStatus.Connected && i < durationInSec + LatencyTimeBufferInSec; i++)
{
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
Assert.AreEqual(lastConnectionStatus, ConnectionStatus.Connected, $"{testDevice.Id} did not reconnect.");
s_log.WriteLine($"{nameof(FaultInjection)}: Confirmed device back online.");
// Perform the test operation.
s_log.WriteLine($">>> {nameof(FaultInjection)}: Performing test operation for device {testDevice.Id}.");
await testOperation(deviceClient, testDevice).ConfigureAwait(false);
}
else
{
s_log.WriteLine($"{nameof(FaultInjection)}: Performing test operation while fault injection is being activated.");
// Perform the test operation for the faulted device multi times.
for (int i = 0; i < LatencyTimeBufferInSec; i++)
{
s_log.WriteLine($">>> {nameof(FaultInjection)}: Performing test operation for device - Run {i}.");
await testOperation(deviceClient, testDevice).ConfigureAwait(false);
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
}
await deviceClient.CloseAsync().ConfigureAwait(false);
if (transport != Client.TransportType.Http1)
{
if (FaultInjection.FaultShouldDisconnect(faultType))
{
// 4 is the minimum notification count: connect, fault, reconnect, disable.
// There are cases where the retry must be timed out (i.e. very likely for MQTT where otherwise
// we would attempt to send the fault injection forever.)
Assert.IsTrue(connectionStatusChangeCount >= 4, $"The expected connection status change count for {testDevice.Id} should be equal or greater than 4 but was {connectionStatusChangeCount}");
}
else
{
// 2 is the minimum notification count: connect, disable.
// We will monitor the test environment real network stability and switch to >=2 if necessary to
// account for real network issues.
Assert.IsTrue(connectionStatusChangeCount == 2, $"The expected connection status change count for {testDevice.Id} should be 2 but was {connectionStatusChangeCount}");
}
Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Disabled} but was {lastConnectionStatus}");
Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Client_Close} but was {lastConnectionStatusChangeReason}");
}
}
finally
{
await cleanupOperation().ConfigureAwait(false);
s_log.WriteLine($"{nameof(FaultInjection)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
deviceClient.Dispose();
watch.Stop();
int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds;
if (timeToFinishFaultInjection > 0)
{
s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed.");
await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false);
}
}
}
public static async Task TestFaultInjectionPoolAmqpAsync(
string devicePrefix,
Client.TransportType transport,
int poolSize,
int devicesCount,
string faultType,
string reason,
int delayInSec,
int durationInSec,
Func <DeviceClient, TestDevice, Task> initOperation,
Func <DeviceClient, TestDevice, Task> testOperation,
Func <IList <DeviceClient>, Task> cleanupOperation,
ConnectionStringAuthScope authScope)
{
var transportSettings = new ITransportSettings[]
{
new AmqpTransportSettings(transport)
{
AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
{
MaxPoolSize = unchecked ((uint)poolSize),
Pooling = true
}
}
};
IList <TestDevice> testDevices = new List <TestDevice>();
IList <DeviceClient> deviceClients = new List <DeviceClient>();
IList <AmqpConnectionStatusChange> amqpConnectionStatuses = new List <AmqpConnectionStatusChange>();
IList <Task> operations = new List <Task>();
// Arrange
// Initialize the test device client instances
// Set the device client connection status change handler
s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)} Initializing Device Clients for multiplexing test.");
for (int i = 0; i < devicesCount; i++)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{devicePrefix}_{i}_").ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
var amqpConnectionStatusChange = new AmqpConnectionStatusChange(testDevice.Id);
deviceClient.SetConnectionStatusChangesHandler(amqpConnectionStatusChange.ConnectionStatusChangesHandler);
testDevices.Add(testDevice);
deviceClients.Add(deviceClient);
amqpConnectionStatuses.Add(amqpConnectionStatusChange);
operations.Add(initOperation(deviceClient, testDevice));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
var watch = new Stopwatch();
try
{
// Act-Assert
// Perform the test operation and verify the operation is successful
// Perform baseline test operation
for (int i = 0; i < devicesCount; i++)
{
s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing baseline operation for device {i}.");
operations.Add(testOperation(deviceClients[i], testDevices[i]));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
// Inject the fault into device 0
watch.Start();
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Device {0} Requesting fault injection type={faultType} reason={reason}, delay={delayInSec}s, duration={durationInSec}s");
var faultInjectionMessage = FaultInjection.ComposeErrorInjectionProperties(faultType, reason, delayInSec, durationInSec);
await deviceClients[0].SendEventAsync(faultInjectionMessage).ConfigureAwait(false);
int delay = FaultInjection.WaitForReconnectMilliseconds - (int)watch.ElapsedMilliseconds;
if (delay < 0)
{
delay = 0;
}
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Waiting for fault injection to be active and device to be connected: {delay}ms");
await Task.Delay(delay).ConfigureAwait(false);
// Perform the test operation for all devices
for (int i = 0; i < devicesCount; i++)
{
s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation for device {i}.");
operations.Add(testOperation(deviceClients[i], testDevices[i]));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
// Close the device client instances
for (int i = 0; i < devicesCount; i++)
{
operations.Add(deviceClients[i].CloseAsync());
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
// Verify the connection status change checks.
// For all of the devices - last connection status should be "Disabled", with reason "Client_close"
for (int i = 0; i < devicesCount; i++)
{
// For the faulted device [device 0] - verify the connection status change count.
if (i == 0)
{
if (FaultInjection.FaultShouldDisconnect(faultType))
{
// 4 is the minimum notification count: connect, fault, reconnect, disable.
Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount >= 4, $"The actual connection status change count for faulted device[0] is = {amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount}");
}
else
{
// 2 is the minimum notification count: connect, disable.
Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount == 2, $"The actual connection status change count for for faulted device[0] is = {amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount}");
}
}
Assert.AreEqual(ConnectionStatus.Disabled, amqpConnectionStatuses[i].LastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, amqpConnectionStatuses[i].LastConnectionStatusChangeReason);
}
}
finally
{
// Close the service-side components and dispose the device client instances.
await cleanupOperation(deviceClients).ConfigureAwait(false);
watch.Stop();
int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds;
if (timeToFinishFaultInjection > 0)
{
s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed.");
await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false);
}
}
}