public static async Task TestFaultInjectionPoolAmqpAsync(
string devicePrefix,
Client.TransportType transport,
int poolSize,
int devicesCount,
string faultType,
string reason,
int delayInSec,
int durationInSec,
Func <DeviceClient, TestDevice, Task> initOperation,
Func <DeviceClient, TestDevice, Task> testOperation,
Func <IList <DeviceClient>, Task> cleanupOperation,
ConnectionStringAuthScope authScope)
{
var transportSettings = new ITransportSettings[]
{
new AmqpTransportSettings(transport)
{
AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
{
MaxPoolSize = unchecked ((uint)poolSize),
Pooling = true
}
}
};
IList <TestDevice> testDevices = new List <TestDevice>();
IList <DeviceClient> deviceClients = new List <DeviceClient>();
IList <AmqpConnectionStatusChange> amqpConnectionStatuses = new List <AmqpConnectionStatusChange>();
IList <Task> operations = new List <Task>();
// Arrange
// Initialize the test device client instances
// Set the device client connection status change handler
s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)} Initializing Device Clients for multiplexing test.");
for (int i = 0; i < devicesCount; i++)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{devicePrefix}_{i}_").ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
var amqpConnectionStatusChange = new AmqpConnectionStatusChange(testDevice.Id);
deviceClient.SetConnectionStatusChangesHandler(amqpConnectionStatusChange.ConnectionStatusChangesHandler);
testDevices.Add(testDevice);
deviceClients.Add(deviceClient);
amqpConnectionStatuses.Add(amqpConnectionStatusChange);
operations.Add(initOperation(deviceClient, testDevice));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
var watch = new Stopwatch();
try
{
// Act-Assert
// Perform the test operation and verify the operation is successful
// Perform baseline test operation
for (int i = 0; i < devicesCount; i++)
{
s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing baseline operation for device {i}.");
operations.Add(testOperation(deviceClients[i], testDevices[i]));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
int countBeforeFaultInjection = amqpConnectionStatuses[0].ConnectionStatusChangeCount;
// Inject the fault into device 0
watch.Start();
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: {testDevices[0].Id} Requesting fault injection type={faultType} reason={reason}, delay={delayInSec}s, duration={durationInSec}s");
var faultInjectionMessage = FaultInjection.ComposeErrorInjectionProperties(faultType, reason, delayInSec, durationInSec);
await deviceClients[0].SendEventAsync(faultInjectionMessage).ConfigureAwait(false);
s_log.WriteLine($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {delayInSec} seconds.");
await Task.Delay(TimeSpan.FromSeconds(delayInSec)).ConfigureAwait(false);
// For disconnect type faults, the faulted device should disconnect and all devices should recover.
if (FaultInjection.FaultShouldDisconnect(faultType))
{
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirming fault injection has been actived.");
// Check that service issued the fault to the faulting device [device 0]
bool isFaulted = false;
for (int i = 0; i < FaultInjection.LatencyTimeBufferInSec; i++)
{
if (amqpConnectionStatuses[0].ConnectionStatusChangeCount > countBeforeFaultInjection)
{
isFaulted = true;
break;
}
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
Assert.IsTrue(isFaulted, $"The device {testDevices[0].Id} did not get faulted with fault type: {faultType}");
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirmed fault injection has been actived.");
// Check all devices are back online
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirming all devices back online.");
bool notRecovered = true;
int j = 0;
for (int i = 0; notRecovered && i < durationInSec + FaultInjection.LatencyTimeBufferInSec; i++)
{
notRecovered = false;
for (j = 0; j < devicesCount; j++)
{
if (amqpConnectionStatuses[j].LastConnectionStatus != ConnectionStatus.Connected)
{
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
notRecovered = true;
break;
}
}
}
if (notRecovered)
{
Assert.Fail($"{testDevices[j].Id} did not reconnect.");
}
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirmed all devices back online.");
// Perform the test operation for all devices
for (int i = 0; i < devicesCount; i++)
{
s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation for device {i}.");
operations.Add(testOperation(deviceClients[i], testDevices[i]));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
}
else
{
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation while fault injection is being activated.");
// Perform the test operation for the faulted device multi times.
for (int i = 0; i < FaultInjection.LatencyTimeBufferInSec; i++)
{
s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation for device 0 - Run {i}.");
await testOperation(deviceClients[0], testDevices[0]).ConfigureAwait(false);
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
}
// Close the device client instances
for (int i = 0; i < devicesCount; i++)
{
operations.Add(deviceClients[i].CloseAsync());
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
// Verify the connection status change checks.
// For all of the devices - last connection status should be "Disabled", with reason "Client_close"
for (int i = 0; i < devicesCount; i++)
{
// For the faulted device [device 0] - verify the connection status change count.
if (i == 0)
{
if (FaultInjection.FaultShouldDisconnect(faultType))
{
// 4 is the minimum notification count: connect, fault, reconnect, disable.
Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangeCount >= 4, $"The expected connection status change count for {testDevices[i].Id} should equals or greater than 4 but was {amqpConnectionStatuses[i].ConnectionStatusChangeCount}");
}
else
{
// 2 is the minimum notification count: connect, disable.
Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangeCount == 2, $"The expected connection status change count for {testDevices[i].Id} should be 2 but was {amqpConnectionStatuses[i].ConnectionStatusChangeCount}");
}
}
Assert.AreEqual(ConnectionStatus.Disabled, amqpConnectionStatuses[i].LastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Disabled} but was {amqpConnectionStatuses[i].LastConnectionStatus}");
Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, amqpConnectionStatuses[i].LastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Client_Close} but was {amqpConnectionStatuses[i].LastConnectionStatusChangeReason}");
}
}
finally
{
// Close the service-side components and dispose the device client instances.
await cleanupOperation(deviceClients).ConfigureAwait(false);
watch.Stop();
int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds;
if (timeToFinishFaultInjection > 0)
{
s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed.");
await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false);
}
}
}
// Error injection template method.
public static async Task TestErrorInjectionAsync(
string devicePrefix,
TestDeviceType type,
Client.TransportType transport,
string faultType,
string reason,
int delayInSec,
int durationInSec,
Func <DeviceClient, TestDevice, Task> initOperation,
Func <DeviceClient, TestDevice, Task> testOperation,
Func <Task> cleanupOperation)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(devicePrefix, type).ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transport);
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int connectionStatusChangeCount = 0;
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
connectionStatusChangeCount++;
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
s_log.WriteLine($"{nameof(FaultInjection)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={connectionStatusChangeCount}");
});
var watch = new Stopwatch();
try
{
await deviceClient.OpenAsync().ConfigureAwait(false);
if (transport != Client.TransportType.Http1)
{
Assert.IsTrue(connectionStatusChangeCount >= 1, $"The expected connection status change should be equal or greater than 1 but was {connectionStatusChangeCount}"); // Normally one connection but in some cases, due to network issues we might have already retried several times to connect.
Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Connected} but was {lastConnectionStatus}");
Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Connection_Ok} but was {lastConnectionStatusChangeReason}");
}
await initOperation(deviceClient, testDevice).ConfigureAwait(false);
s_log.WriteLine($">>> {nameof(FaultInjection)} Testing baseline");
await testOperation(deviceClient, testDevice).ConfigureAwait(false);
int countBeforeFaultInjection = connectionStatusChangeCount;
watch.Start();
s_log.WriteLine($">>> {nameof(FaultInjection)} Testing fault handling");
await ActivateFaultInjection(transport, faultType, reason, delayInSec, durationInSec, deviceClient).ConfigureAwait(false);
s_log.WriteLine($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {delayInSec} seconds.");
await Task.Delay(TimeSpan.FromSeconds(delayInSec)).ConfigureAwait(false);
// For disconnect type faults, the device should disconnect and recover.
if (FaultShouldDisconnect(faultType))
{
s_log.WriteLine($"{nameof(FaultInjection)}: Confirming fault injection has been actived.");
// Check that service issued the fault to the faulting device
bool isFaulted = false;
for (int i = 0; i < LatencyTimeBufferInSec; i++)
{
if (connectionStatusChangeCount > countBeforeFaultInjection)
{
isFaulted = true;
break;
}
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
Assert.IsTrue(isFaulted, $"The device {testDevice.Id} did not get faulted with fault type: {faultType}");
s_log.WriteLine($"{nameof(FaultInjection)}: Confirmed fault injection has been actived.");
// Check the device is back online
s_log.WriteLine($"{nameof(FaultInjection)}: Confirming device back online.");
for (int i = 0; lastConnectionStatus != ConnectionStatus.Connected && i < durationInSec + LatencyTimeBufferInSec; i++)
{
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus, $"{testDevice.Id} did not reconnect.");
s_log.WriteLine($"{nameof(FaultInjection)}: Confirmed device back online.");
// Perform the test operation.
s_log.WriteLine($">>> {nameof(FaultInjection)}: Performing test operation for device {testDevice.Id}.");
await testOperation(deviceClient, testDevice).ConfigureAwait(false);
}
else
{
s_log.WriteLine($"{nameof(FaultInjection)}: Performing test operation while fault injection is being activated.");
// Perform the test operation for the faulted device multi times.
for (int i = 0; i < LatencyTimeBufferInSec; i++)
{
s_log.WriteLine($">>> {nameof(FaultInjection)}: Performing test operation for device - Run {i}.");
await testOperation(deviceClient, testDevice).ConfigureAwait(false);
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
}
await deviceClient.CloseAsync().ConfigureAwait(false);
if (transport != Client.TransportType.Http1)
{
if (FaultInjection.FaultShouldDisconnect(faultType))
{
// 4 is the minimum notification count: connect, fault, reconnect, disable.
// There are cases where the retry must be timed out (i.e. very likely for MQTT where otherwise
// we would attempt to send the fault injection forever.)
Assert.IsTrue(connectionStatusChangeCount >= 4, $"The expected connection status change count for {testDevice.Id} should be equal or greater than 4 but was {connectionStatusChangeCount}");
}
else
{
// 2 is the minimum notification count: connect, disable.
// We will monitor the test environment real network stability and switch to >=2 if necessary to
// account for real network issues.
Assert.IsTrue(connectionStatusChangeCount == 2, $"The expected connection status change count for {testDevice.Id} should be 2 but was {connectionStatusChangeCount}");
}
Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Disabled} but was {lastConnectionStatus}");
Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Client_Close} but was {lastConnectionStatusChangeReason}");
}
}
finally
{
await cleanupOperation().ConfigureAwait(false);
s_log.WriteLine($"{nameof(FaultInjection)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
deviceClient.Dispose();
watch.Stop();
int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds;
if (timeToFinishFaultInjection > 0)
{
s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed.");
await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false);
}
}
}
private async Task Twin_DeviceDesiredPropertyUpdateRecovery(Client.TransportType transport, string faultType, string reason, int delayInSec)
{
var tcs = new TaskCompletionSource <bool>();
var propName = Guid.NewGuid().ToString();
var propValue = Guid.NewGuid().ToString();
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);
RegistryManager registryManager = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);
var deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport);
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int setConnectionStatusChangesHandlerCount = 0;
var tcsConnected = new TaskCompletionSource <bool>();
var tcsDisconnected = new TaskCompletionSource <bool>();
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
_log.WriteLine("Connection Changed to {0} because {1}", status, statusChangeReason);
if (status == ConnectionStatus.Disconnected_Retrying)
{
tcsDisconnected.TrySetResult(true);
Assert.AreEqual(ConnectionStatusChangeReason.No_Network, statusChangeReason);
}
else if (status == ConnectionStatus.Connected)
{
tcsConnected.TrySetResult(true);
}
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
setConnectionStatusChangesHandlerCount++;
});
await deviceClient.SetDesiredPropertyUpdateCallbackAsync((patch, context) =>
{
return(Task.Run(() =>
{
try
{
Assert.AreEqual(patch[propName].ToString(), propValue);
}
catch (Exception e)
{
tcs.SetException(e);
}
finally
{
tcs.SetResult(true);
}
}));
}, null).ConfigureAwait(false);
// assert on successful connection
await Task.WhenAny(
Task.Run(async() =>
{
await Task.Delay(1000).ConfigureAwait(false);
}), tcsConnected.Task).ConfigureAwait(false);
Assert.IsTrue(tcsConnected.Task.IsCompleted, "Initial connection failed");
if (transport != Client.TransportType.Http1)
{
Assert.AreEqual(1, setConnectionStatusChangesHandlerCount);
Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason);
}
var twinPatch = new Twin();
twinPatch.Properties.Desired[propName] = propValue;
await registryManager.UpdateTwinAsync(testDevice.Id, twinPatch, "*").ConfigureAwait(false);
await tcs.Task.ConfigureAwait(false);
// send error command
await deviceClient.SendEventAsync(FaultInjection.ComposeErrorInjectionProperties(faultType, reason, delayInSec)).ConfigureAwait(false);
// reset ConnectionStatusChangesHandler data
setConnectionStatusChangesHandlerCount = 0;
tcsConnected = new TaskCompletionSource <bool>();
tcsDisconnected = new TaskCompletionSource <bool>();
// wait for disconnection
await Task.WhenAny(
Task.Run(async() =>
{
await Task.Delay(TimeSpan.FromSeconds(10)).ConfigureAwait(false);
}), tcsDisconnected.Task).ConfigureAwait(false);
Assert.IsTrue(tcsDisconnected.Task.IsCompleted, "Error injection did not interrupt the device");
await Task.WhenAny(
Task.Run(async() =>
{
await Task.Delay(TimeSpan.FromSeconds(MaximumRecoveryTimeSeconds)).ConfigureAwait(false);
return(Task.FromResult(true));
}), tcsConnected.Task).ConfigureAwait(false);
Assert.IsTrue(tcsConnected.Task.IsCompleted, "Recovery connection failed");
tcs = new TaskCompletionSource <bool>();
twinPatch = new Twin();
twinPatch.Properties.Desired[propName] = propValue;
await registryManager.UpdateTwinAsync(testDevice.Id, twinPatch, "*").ConfigureAwait(false);
await tcs.Task.ConfigureAwait(false);
await deviceClient.CloseAsync().ConfigureAwait(false);
await registryManager.CloseAsync().ConfigureAwait(false);
}
private async Task SendMethodAndRespondRecovery(Client.TransportType transport, string faultType, string reason, int delayInSec)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);
var assertResult = new TaskCompletionSource <Tuple <bool, bool> >();
DeviceClient deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport);
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int setConnectionStatusChangesHandlerCount = 0;
var tcsConnected = new TaskCompletionSource <bool>();
var tcsDisconnected = new TaskCompletionSource <bool>();
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
_log.WriteLine("Connection Changed to {0} because {1}", status, statusChangeReason);
if (status == ConnectionStatus.Disconnected_Retrying)
{
tcsDisconnected.TrySetResult(true);
Assert.AreEqual(ConnectionStatusChangeReason.No_Network, statusChangeReason);
}
else if (status == ConnectionStatus.Connected)
{
tcsConnected.TrySetResult(true);
}
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
setConnectionStatusChangesHandlerCount++;
});
await deviceClient.SetMethodHandlerAsync(MethodName,
(request, context) =>
{
assertResult.TrySetResult(new Tuple <bool, bool>(request.Name.Equals(MethodName),
request.DataAsJson.Equals(ServiceRequestJson)));
return(Task.FromResult(new MethodResponse(Encoding.UTF8.GetBytes(DeviceResponseJson), 200)));
},
null).ConfigureAwait(false);
// assert on successful connection
await Task.WhenAny(Task.Delay(FaultInjection.ShortRetryInMilliSec), tcsConnected.Task).ConfigureAwait(false);
Assert.IsTrue(tcsConnected.Task.IsCompleted, "Initial connection failed");
if (transport != Client.TransportType.Http1)
{
Assert.AreEqual(1, setConnectionStatusChangesHandlerCount);
Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason);
}
// check on normal operation
await
ServiceSendMethodAndVerifyResponse(testDevice.Id, MethodName, DeviceResponseJson,
ServiceRequestJson, assertResult).ConfigureAwait(false);
// reset ConnectionStatusChangesHandler data
setConnectionStatusChangesHandlerCount = 0;
tcsConnected = new TaskCompletionSource <bool>();
tcsDisconnected = new TaskCompletionSource <bool>();
// send error command
await deviceClient.SendEventAsync(FaultInjection.ComposeErrorInjectionProperties(faultType, reason, delayInSec)).ConfigureAwait(false);
// wait for disconnection
await Task.WhenAny(Task.Delay(FaultInjection.WaitForDisconnectMilliseconds), tcsDisconnected.Task).ConfigureAwait(false);
Assert.IsTrue(tcsDisconnected.Task.IsCompleted, "Error injection did not interrupt the device");
await Task.WhenAny(Task.Delay(FaultInjection.RecoveryTimeMilliseconds), tcsConnected.Task).ConfigureAwait(false);
Assert.IsTrue(tcsConnected.Task.IsCompleted, "Recovery connection failed");
assertResult = new TaskCompletionSource <Tuple <bool, bool> >();
await
ServiceSendMethodAndVerifyResponse(testDevice.Id, MethodName, DeviceResponseJson,
ServiceRequestJson, assertResult).ConfigureAwait(false);
setConnectionStatusChangesHandlerCount = 0;
//remove and CloseAsync
await deviceClient.SetMethodHandlerAsync(MethodName, null, null).ConfigureAwait(false);
if (transport != Client.TransportType.Http1)
{
Assert.AreEqual(1, setConnectionStatusChangesHandlerCount);
Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason);
}
}
internal async Task SendMessageRecovery(Client.TransportType transport,
string faultType, string reason, int delayInSec, int durationInSec = 0, int retryDurationInMilliSec = 240000)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);
var deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport);
EventHubClient eventHubClient;
EventHubReceiver eventHubReceiver = CreateEventHubReceiver(testDevice.Id, out eventHubClient);
try
{
deviceClient.OperationTimeoutInMilliseconds = (uint)retryDurationInMilliSec;
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int setConnectionStatusChangesHandlerCount = 0;
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
setConnectionStatusChangesHandlerCount++;
});
await deviceClient.OpenAsync().ConfigureAwait(false);
if (transport != Client.TransportType.Http1)
{
Assert.AreEqual(1, setConnectionStatusChangesHandlerCount);
Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason);
}
string payload, p1Value;
Client.Message testMessage = ComposeD2CTestMessage(out payload, out p1Value);
await deviceClient.SendEventAsync(testMessage).ConfigureAwait(false);
bool isReceived = false;
Stopwatch sw = new Stopwatch();
sw.Start();
while (!isReceived && sw.Elapsed.Minutes < 1)
{
var events = await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5)).ConfigureAwait(false);
isReceived = VerifyTestMessage(events, testDevice.Id, payload, p1Value);
}
sw.Stop();
// send error command and clear eventHubReceiver of the fault injection message
await deviceClient.SendEventAsync(FaultInjection.ComposeErrorInjectionProperties(faultType, reason,
delayInSec,
durationInSec)).ConfigureAwait(false);
await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5)).ConfigureAwait(false);
Thread.Sleep(1000);
testMessage = ComposeD2CTestMessage(out payload, out p1Value);
await deviceClient.SendEventAsync(testMessage).ConfigureAwait(false);
sw.Reset();
sw.Start();
while (!isReceived && sw.Elapsed.Minutes < 1)
{
var events = await eventHubReceiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(5)).ConfigureAwait(false);
isReceived = VerifyTestMessage(events, testDevice.Id, payload, p1Value);
}
sw.Stop();
await deviceClient.CloseAsync().ConfigureAwait(false);
if (transport != Client.TransportType.Http1)
{
Assert.AreEqual(2, setConnectionStatusChangesHandlerCount);
Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason);
}
}
finally
{
await deviceClient.CloseAsync().ConfigureAwait(false);
await eventHubReceiver.CloseAsync().ConfigureAwait(false);
_log.WriteLine($"Waiting for fault injection interval to finish {FaultInjection.DefaultDelayInSec}s.");
await Task.Delay(FaultInjection.DefaultDurationInSec * 1000).ConfigureAwait(false);
}
}