private async Task ReceiveMessageInOperationTimeout(TestDeviceType type, Client.TransportType transport)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix, type).ConfigureAwait(false);
using (DeviceClient deviceClient = testDevice.CreateDeviceClient(transport))
{
await deviceClient.OpenAsync().ConfigureAwait(false);
if (transport == Client.TransportType.Mqtt_Tcp_Only ||
transport == Client.TransportType.Mqtt_WebSocket_Only)
{
// Dummy ReceiveAsync to ensure mqtt subscription registration before SendAsync() is called on service client.
await deviceClient.ReceiveAsync(TIMESPAN_FIVE_SECONDS).ConfigureAwait(false);
}
try
{
deviceClient.OperationTimeoutInMilliseconds = Convert.ToUInt32(TIMESPAN_ONE_MINUTE.TotalMilliseconds);
if (transport == Client.TransportType.Amqp || transport == Client.TransportType.Amqp_Tcp_Only || transport == Client.TransportType.Amqp_WebSocket_Only)
{
// For AMQP because of static 1 min interval check the cancellation token, in worst case it will block upto extra 1 min to return
await ReceiveMessageWithoutTimeoutCheck(deviceClient, TIMESPAN_ONE_MINUTE).ConfigureAwait(false);
}
else
{
await ReceiveMessageWithoutTimeoutCheck(deviceClient, TIMESPAN_FIVE_SECONDS).ConfigureAwait(false);
}
}
finally
{
deviceClient.OperationTimeoutInMilliseconds = DeviceClient.DefaultOperationTimeoutInMilliseconds;
await deviceClient.CloseAsync().ConfigureAwait(false);
}
}
}
private async Task ReceiveSingleMessage(TestDeviceType type, Client.TransportType transport)
{
ServiceClient serviceClient = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix, type).ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transport);
try
{
await deviceClient.OpenAsync().ConfigureAwait(false);
if (transport == Client.TransportType.Mqtt_Tcp_Only ||
transport == Client.TransportType.Mqtt_WebSocket_Only)
{
_log.WriteLine("Dummy ReceiveAsync to ensure mqtt subscription registration before SendAsync() is called on service client.");
await deviceClient.ReceiveAsync(TimeSpan.FromSeconds(2)).ConfigureAwait(false);
}
string payload, messageId, p1Value;
await serviceClient.OpenAsync().ConfigureAwait(false);
Message msg = ComposeC2DTestMessage(out payload, out messageId, out p1Value);
await serviceClient.SendAsync(testDevice.Id, msg).ConfigureAwait(false);
await VerifyReceivedC2DMessage(transport, deviceClient, payload, p1Value).ConfigureAwait(false);
}
finally
{
await deviceClient.CloseAsync().ConfigureAwait(false);
await serviceClient.CloseAsync().ConfigureAwait(false);
}
}
private async Task ReceiveSingleMessage(TestDeviceType type, Client.TransportType transport)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix, type).ConfigureAwait(false);
using (DeviceClient deviceClient = testDevice.CreateDeviceClient(transport))
using (ServiceClient serviceClient = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString))
{
await deviceClient.OpenAsync().ConfigureAwait(false);
if (transport == Client.TransportType.Mqtt_Tcp_Only ||
transport == Client.TransportType.Mqtt_WebSocket_Only)
{
// Dummy ReceiveAsync to ensure mqtt subscription registration before SendAsync() is called on service client.
await deviceClient.ReceiveAsync(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
await serviceClient.OpenAsync().ConfigureAwait(false);
(Message msg, string messageId, string payload, string p1Value) = ComposeC2DTestMessage();
await serviceClient.SendAsync(testDevice.Id, msg).ConfigureAwait(false);
await VerifyReceivedC2DMessageAsync(transport, deviceClient, testDevice.Id, payload, p1Value).ConfigureAwait(false);
await deviceClient.CloseAsync().ConfigureAwait(false);
await serviceClient.CloseAsync().ConfigureAwait(false);
}
}
private async Task SendSingleMessage(TestDeviceType type, Client.TransportType transport)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix, type).ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transport);
await SendSingleMessage(deviceClient, testDevice.Id).ConfigureAwait(false);
}
public async Task Duplicated_NoPingpong()
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix, TestDeviceType.Sasl).ConfigureAwait(false);
_log.WriteLine($"{nameof(Duplicated_NoPingpong)}: 2 device client instances with the same deviceId={testDevice.Id}.");
using (DeviceClient deviceClient1 = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only), deviceClient2 = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only))
{
_log.WriteLine($"{nameof(Duplicated_NoPingpong)}: set device client instance 1 to no retry.");
deviceClient1.SetRetryPolicy(new NoRetry());
ConnectionStatus?lastConnectionStatus = null;
Dictionary <ConnectionStatus, int> connectionStatusChanges = new Dictionary <ConnectionStatus, int>();
deviceClient1.SetConnectionStatusChangesHandler((status, reason) =>
{
connectionStatusChanges.TryGetValue(status, out int count);
count++;
connectionStatusChanges[status] = count;
lastConnectionStatus = status;
});
_log.WriteLine($"{nameof(FaultInjection_NoRecovery)}: device client instance 1 calling OpenAsync...");
await deviceClient1.OpenAsync().ConfigureAwait(false);
await deviceClient1.SetMethodHandlerAsync(
"dummy_method",
(methodRequest, userContext) => Task.FromResult(new MethodResponse(200)),
deviceClient1
).ConfigureAwait(false);
_log.WriteLine($"{nameof(FaultInjection_NoRecovery)}: device client instance 2 calling OpenAsync...");
await deviceClient2.OpenAsync().ConfigureAwait(false);
await deviceClient2.SetMethodHandlerAsync(
"dummy_method",
(methodRequest, userContext) => Task.FromResult(new MethodResponse(200)),
deviceClient2
).ConfigureAwait(false);
_log.WriteLine($"{nameof(Duplicated_NoPingpong)}: waiting device client instance 1 to be kicked off...");
for (int i = 0; i < FaultInjection.LatencyTimeBufferInSec; i++)
{
if (connectionStatusChanges.ContainsKey(ConnectionStatus.Disconnected))
{
break;
}
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
Assert.AreEqual(ConnectionStatus.Disconnected, lastConnectionStatus, $"Excepeted device to be {ConnectionStatus.Disconnected} but was {lastConnectionStatus}.");
Assert.IsFalse(connectionStatusChanges.ContainsKey(ConnectionStatus.Disconnected_Retrying), $"Shouldn't get {ConnectionStatus.Disconnected_Retrying} status change.");
int connected = connectionStatusChanges[ConnectionStatus.Connected];
Assert.AreEqual(1, connected, $"Should get {ConnectionStatus.Connected} once but was {connected}.");
int disconnected = connectionStatusChanges[ConnectionStatus.Disconnected];
Assert.AreEqual(1, disconnected, $"Should get {ConnectionStatus.Disconnected} once but was {disconnected}.");
}
}
public async Task FaultInjection_NoRecovery()
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(Logger, DevicePrefix, TestDeviceType.Sasl).ConfigureAwait(false);
using DeviceClient deviceClient = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only);
Logger.Trace($"{nameof(FaultInjection_NoRecovery)}: deviceId={testDevice.Id}");
deviceClient.SetRetryPolicy(new NoRetry());
ConnectionStatus?lastConnectionStatus = null;
Dictionary <ConnectionStatus, int> connectionStatusChanges = new Dictionary <ConnectionStatus, int>();
deviceClient.SetConnectionStatusChangesHandler((status, reason) =>
{
connectionStatusChanges.TryGetValue(status, out int count);
count++;
connectionStatusChanges[status] = count;
lastConnectionStatus = status;
});
Logger.Trace($"{nameof(FaultInjection_NoRecovery)}: calling OpenAsync...");
await deviceClient.OpenAsync().ConfigureAwait(false);
Logger.Trace($"{nameof(FaultInjection_NoRecovery)}: injecting fault {FaultInjection.FaultType_Tcp}...");
await FaultInjection
.ActivateFaultInjectionAsync(
Client.TransportType.Amqp_Tcp_Only,
FaultInjection.FaultType_Tcp,
FaultInjection.FaultCloseReason_Boom,
FaultInjection.DefaultDelayInSec,
FaultInjection.DefaultDurationInSec,
deviceClient,
Logger)
.ConfigureAwait(false);
await Task.Delay(FaultInjection.DefaultDelayInSec).ConfigureAwait(false);
Logger.Trace($"{nameof(FaultInjection_NoRecovery)}: waiting fault injection occurs...");
for (int i = 0; i < FaultInjection.LatencyTimeBufferInSec; i++)
{
if (connectionStatusChanges.ContainsKey(ConnectionStatus.Disconnected))
{
break;
}
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
Assert.AreEqual(ConnectionStatus.Disconnected, lastConnectionStatus, $"Excepeted device to be {ConnectionStatus.Disconnected} but was {lastConnectionStatus}.");
Assert.IsFalse(connectionStatusChanges.ContainsKey(ConnectionStatus.Disconnected_Retrying), $"Shouldn't get {ConnectionStatus.Disconnected_Retrying} status change.");
int connected = connectionStatusChanges[ConnectionStatus.Connected];
Assert.AreEqual(1, connected, $"Should get {ConnectionStatus.Connected} once but was {connected}.");
int disconnected = connectionStatusChanges[ConnectionStatus.Disconnected];
Assert.AreEqual(1, disconnected, $"Should get {ConnectionStatus.Disconnected} once but was {disconnected}.");
}
private async Task SendMessageTest(ITransportSettings transportSetting)
{
using TestDevice testDevice = await TestDevice.GetTestDeviceAsync(Logger, s_devicePrefix, TestDeviceType.X509).ConfigureAwait(false);
using DeviceClient deviceClient = testDevice.CreateDeviceClient(new[] { transportSetting });
await deviceClient.OpenAsync().ConfigureAwait(false);
await MessageSendE2ETests.SendSingleMessageAsync(deviceClient, testDevice.Id, Logger).ConfigureAwait(false);
await deviceClient.CloseAsync().ConfigureAwait(false);
}
public async Task DeviceClient_CreateFromConnectionString_TokenIsRefreshed_Mqtt()
{
var sasTokenTimeToLive = TimeSpan.FromSeconds(10);
int sasTokenRenewalBuffer = 50;
using var deviceDisconnected = new SemaphoreSlim(0);
int operationTimeoutInMilliseconds = (int)sasTokenTimeToLive.TotalMilliseconds * 2;
// Service allows a buffer time of upto 10mins before dropping connections that are authenticated with an expired sas tokens.
using var tokenRefreshCts = new CancellationTokenSource((int)(sasTokenTimeToLive.TotalMilliseconds * 2 + TimeSpan.FromMinutes(10).TotalMilliseconds));
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(Logger, DevicePrefix).ConfigureAwait(false);
var options = new ClientOptions
{
SasTokenTimeToLive = sasTokenTimeToLive,
SasTokenRenewalBuffer = sasTokenRenewalBuffer,
};
using DeviceClient deviceClient = testDevice.CreateDeviceClient(Client.TransportType.Mqtt, options);
Logger.Trace($"Created {nameof(DeviceClient)} instance for {testDevice.Id}.");
deviceClient.SetConnectionStatusChangesHandler((ConnectionStatus status, ConnectionStatusChangeReason reason) =>
{
Logger.Trace($"{nameof(ConnectionStatusChangesHandler)}: {status}; {reason}");
if (status == ConnectionStatus.Disconnected_Retrying || status == ConnectionStatus.Disconnected)
{
deviceDisconnected.Release();
}
});
deviceClient.OperationTimeoutInMilliseconds = (uint)operationTimeoutInMilliseconds;
var message = new Client.Message(Encoding.UTF8.GetBytes("Hello"));
Logger.Trace($"[{testDevice.Id}]: SendEventAsync (1)");
await deviceClient.SendEventAsync(message).ConfigureAwait(false);
// Wait for the Token to expire.
Logger.Trace($"[{testDevice.Id}]: Waiting for device disconnect.");
await deviceDisconnected.WaitAsync(tokenRefreshCts.Token).ConfigureAwait(false);
try
{
Logger.Trace($"[{testDevice.Id}]: SendEventAsync (2)");
await deviceClient.SendEventAsync(message).ConfigureAwait(false);
}
catch (OperationCanceledException ex)
{
Assert.Fail($"{testDevice.Id} did not refresh token after expected ttl of {sasTokenTimeToLive}: {ex}");
throw;
}
}
private async Task SendBatchMessages(TestDeviceType type, Client.TransportType transport)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix, type).ConfigureAwait(false);
using (DeviceClient deviceClient = testDevice.CreateDeviceClient(transport))
{
await deviceClient.OpenAsync().ConfigureAwait(false);
await SendSendBatchMessagesAndVerifyAsync(deviceClient, testDevice.Id).ConfigureAwait(false);
await deviceClient.CloseAsync().ConfigureAwait(false);
}
}
private async Task SendSingleMessage(TestDeviceType type, ITransportSettings[] transportSettings)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix, type).ConfigureAwait(false);
using (DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings))
{
await deviceClient.OpenAsync().ConfigureAwait(false);
await SendSingleMessageAndVerifyAsync(deviceClient, testDevice.Id).ConfigureAwait(false);
await deviceClient.CloseAsync().ConfigureAwait(false);
}
}
private async Task SendSingleMessage(TestDeviceType type, Client.TransportType transport, int messageSize = 0)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix, type).ConfigureAwait(false);
using DeviceClient deviceClient = testDevice.CreateDeviceClient(transport);
await deviceClient.OpenAsync().ConfigureAwait(false);
await SendSingleMessageAndVerifyAsync(deviceClient, testDevice.Id, messageSize).ConfigureAwait(false);
await deviceClient.CloseAsync().ConfigureAwait(false);
}
private async Task TestSecurityMessageAsync(Client.TransportType transport)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(_devicePrefix).ConfigureAwait(false);
using DeviceClient deviceClient = testDevice.CreateDeviceClient(transport);
try
{
await SendSingleSecurityMessageAsync(deviceClient, testDevice.Id, _logAnalyticsClient).ConfigureAwait(false);
}
finally
{
await deviceClient.CloseAsync().ConfigureAwait(false);
}
}
private static async Task CompleteMessageMixOrder(TestDeviceType type, Client.TransportType transport)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix, type).ConfigureAwait(false);
using (DeviceClient deviceClient = testDevice.CreateDeviceClient(transport))
using (ServiceClient serviceClient = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString))
{
await deviceClient.OpenAsync().ConfigureAwait(false);
if (transport == Client.TransportType.Mqtt_Tcp_Only ||
transport == Client.TransportType.Mqtt_WebSocket_Only)
{
// Dummy ReceiveAsync to ensure mqtt subscription registration before SendAsync() is called on service client.
await deviceClient.ReceiveAsync(TIMESPAN_FIVE_SECONDS).ConfigureAwait(false);
}
await serviceClient.OpenAsync().ConfigureAwait(false);
var messages = new List <Client.Message>();
for (int i = 0; i < MESSAGE_COUNT; i++)
{
(Message msg, string messageId, string payload, string p1Value) = MessageReceiveE2ETests.ComposeC2DTestMessage();
await serviceClient.SendAsync(testDevice.Id, msg).ConfigureAwait(false);
Client.Message message = await deviceClient.ReceiveAsync(TIMESPAN_ONE_MINUTE).ConfigureAwait(false);
if (message == null)
{
Assert.Fail("No message received.");
}
messages.Add(message);
}
for (int i = 0; i < MESSAGE_COUNT; i++)
{
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
await deviceClient.CompleteAsync(messages[MESSAGE_COUNT - 1 - i]).ConfigureAwait(false);
stopwatch.Stop();
Assert.IsFalse(stopwatch.ElapsedMilliseconds > deviceClient.OperationTimeoutInMilliseconds, $"CompleteAsync is over {deviceClient.OperationTimeoutInMilliseconds}");
}
await deviceClient.CloseAsync().ConfigureAwait(false);
await serviceClient.CloseAsync().ConfigureAwait(false);
}
}
private async Task ReceiveMessageInOperationTimeoutAsync(TestDeviceType type, Client.TransportType transport)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(s_devicePrefix, type).ConfigureAwait(false);
using DeviceClient deviceClient = testDevice.CreateDeviceClient(transport);
s_log.WriteLine($"{nameof(ReceiveMessageInOperationTimeoutAsync)} - calling OpenAsync() for transport={transport}");
await deviceClient.OpenAsync().ConfigureAwait(false);
if (transport == Client.TransportType.Mqtt_Tcp_Only ||
transport == Client.TransportType.Mqtt_WebSocket_Only)
{
// Dummy ReceiveAsync to ensure mqtt subscription registration before SendAsync() is called on service client.
await deviceClient.ReceiveAsync(s_fiveSeconds).ConfigureAwait(false);
}
try
{
deviceClient.OperationTimeoutInMilliseconds = Convert.ToUInt32(s_oneMinute.TotalMilliseconds);
s_log.WriteLine($"{nameof(ReceiveMessageInOperationTimeoutAsync)} - setting device client default operation timeout={deviceClient.OperationTimeoutInMilliseconds} ms");
if (transport == Client.TransportType.Amqp ||
transport == Client.TransportType.Amqp_Tcp_Only ||
transport == Client.TransportType.Amqp_WebSocket_Only)
{
// TODO: this extra minute on the timeout is undesirable by customers, and tests seems to be failing on a slight timing issue.
// For now, add an additional 5 second buffer to prevent tests from failing, and meanwhile address issue 1203.
// For AMQP because of static 1 min interval check the cancellation token, in worst case it will block upto extra 1 min to return
await ReceiveMessageWithoutTimeoutCheckAsync(deviceClient, s_oneMinute + TimeSpan.FromSeconds(5)).ConfigureAwait(false);
}
else
{
await ReceiveMessageWithoutTimeoutCheckAsync(deviceClient, s_fiveSeconds).ConfigureAwait(false);
}
}
finally
{
s_log.WriteLine($"{nameof(ReceiveMessageInOperationTimeoutAsync)} - calling CloseAsync() for transport={transport}");
deviceClient.OperationTimeoutInMilliseconds = DeviceClient.DefaultOperationTimeoutInMilliseconds;
await deviceClient.CloseAsync().ConfigureAwait(false);
}
}
public async Task Message_ClientThrowsForMqttTopicNameTooLong()
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);
using DeviceClient deviceClient = testDevice.CreateDeviceClient(Client.TransportType.Mqtt);
await deviceClient.OpenAsync().ConfigureAwait(false);
using var msg = new Client.Message(Encoding.UTF8.GetBytes("testMessage"));
//Mqtt topic name consists of, among other things, system properties and user properties
// setting lots of very long user properties should cause a MessageTooLargeException explaining
// that the topic name is too long to publish over mqtt
for (int i = 0; i < 100; i++)
{
msg.Properties.Add(Guid.NewGuid().ToString(), new string('1', 1024));
}
await deviceClient.SendEventAsync(msg).ConfigureAwait(false);
}
private async Task ReceiveMessageWithTimeoutAsync(TestDeviceType type, Client.TransportType transport, TimeSpan timeout)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(s_devicePrefix, type).ConfigureAwait(false);
using DeviceClient deviceClient = testDevice.CreateDeviceClient(transport);
await deviceClient.OpenAsync().ConfigureAwait(false);
if (transport == Client.TransportType.Mqtt_Tcp_Only ||
transport == Client.TransportType.Mqtt_WebSocket_Only)
{
// Dummy ReceiveAsync to ensure mqtt subscription registration before SendAsync() is called on service client.
await deviceClient.ReceiveAsync(s_fiveSeconds).ConfigureAwait(false);
}
await ReceiveMessageWithTimeoutCheckAsync(deviceClient, timeout).ConfigureAwait(false);
await deviceClient.CloseAsync().ConfigureAwait(false);
}
private async Task TestSecurityMessage(Client.TransportType transport)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(_devicePrefix).ConfigureAwait(false);
EventHubTestListener testListener = await EventHubTestListener.CreateListener(testDevice.Id).ConfigureAwait(false);
using (DeviceClient deviceClient = testDevice.CreateDeviceClient(transport))
{
try
{
await SendSingleSecurityMessage(deviceClient, testDevice.Id, testListener, _logAnalyticsClient).ConfigureAwait(false);
}
finally
{
await deviceClient.CloseAsync().ConfigureAwait(false);
await testListener.CloseAsync().ConfigureAwait(false);
}
}
}
// Error injection template method.
public static async Task TestErrorInjectionAsync(
string devicePrefix,
TestDeviceType type,
Client.TransportType transport,
string faultType,
string reason,
int delayInSec,
int durationInSec,
Func <DeviceClient, TestDevice, Task> initOperation,
Func <DeviceClient, TestDevice, Task> testOperation,
Func <Task> cleanupOperation)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(devicePrefix, type).ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transport);
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int setConnectionStatusChangesHandlerCount = 0;
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
setConnectionStatusChangesHandlerCount++;
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
s_log.WriteLine($"{nameof(FaultInjection)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}");
});
var watch = new Stopwatch();
try
{
await deviceClient.OpenAsync().ConfigureAwait(false);
if (transport != Client.TransportType.Http1)
{
Assert.IsTrue(setConnectionStatusChangesHandlerCount >= 1); // Normally one connection but in some cases, due to network issues we might have already retried several times to connect.
Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason);
}
await initOperation(deviceClient, testDevice).ConfigureAwait(false);
s_log.WriteLine($">>> {nameof(FaultInjection)} Testing baseline");
await testOperation(deviceClient, testDevice).ConfigureAwait(false);
watch.Start();
s_log.WriteLine($">>> {nameof(FaultInjection)} Testing fault handling");
await ActivateFaultInjection(transport, faultType, reason, delayInSec, durationInSec, deviceClient).ConfigureAwait(false);
int delay = FaultInjection.WaitForDisconnectMilliseconds - (int)watch.ElapsedMilliseconds;
if (delay < 0)
{
delay = 0;
}
s_log.WriteLine($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {delay}ms");
await Task.Delay(delay).ConfigureAwait(false);
await testOperation(deviceClient, testDevice).ConfigureAwait(false);
await deviceClient.CloseAsync().ConfigureAwait(false);
if (transport != Client.TransportType.Http1)
{
if (FaultShouldDisconnect(faultType))
{
// 4 is the minimum notification count: connect, fault, reconnect, disable.
// There are cases where the retry must be timed out (i.e. very likely for MQTT where otherwise
// we would attempt to send the fault injection forever.)
Assert.IsTrue(setConnectionStatusChangesHandlerCount >= 4);
}
else
{
// 2 is the minimum notification count: connect, disable.
// We will monitor the test environment real network stability and switch to >=2 if necessary to
// account for real network issues.
Assert.IsTrue(setConnectionStatusChangesHandlerCount == 2);
}
Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason);
}
}
finally
{
await cleanupOperation().ConfigureAwait(false);
s_log.WriteLine($"{nameof(FaultInjection)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
deviceClient.Dispose();
watch.Stop();
int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds;
if (timeToFinishFaultInjection > 0)
{
s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed.");
await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false);
}
}
}
public async Task FaultInjection_NoRetry_NoRecovery_OpenAsync()
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(Logger, _devicePrefix, TestDeviceType.Sasl).ConfigureAwait(false);
using DeviceClient deviceClient = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only);
Logger.Trace($"{nameof(FaultInjection_NoRetry_NoRecovery_OpenAsync)}: deviceId={testDevice.Id}");
deviceClient.SetRetryPolicy(new NoRetry());
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
var connectionStatusChanges = new Dictionary <ConnectionStatus, int>();
deviceClient.SetConnectionStatusChangesHandler((status, reason) =>
{
connectionStatusChanges.TryGetValue(status, out int count);
count++;
connectionStatusChanges[status] = count;
lastConnectionStatus = status;
lastConnectionStatusChangeReason = reason;
});
Logger.Trace($"{nameof(FaultInjection_NoRetry_NoRecovery_OpenAsync)}: calling OpenAsync...");
await deviceClient.OpenAsync().ConfigureAwait(false);
Logger.Trace($"{nameof(FaultInjection_NoRetry_NoRecovery_OpenAsync)}: injecting fault {FaultInjection.FaultType_Tcp}...");
await FaultInjection
.ActivateFaultInjectionAsync(
Client.TransportType.Amqp_Tcp_Only,
FaultInjection.FaultType_Tcp,
FaultInjection.FaultCloseReason_Boom,
FaultInjection.DefaultFaultDelay,
FaultInjection.DefaultFaultDuration,
deviceClient,
Logger)
.ConfigureAwait(false);
await Task.Delay(FaultInjection.DefaultFaultDelay).ConfigureAwait(false);
Logger.Trace($"{nameof(FaultInjection_NoRetry_NoRecovery_OpenAsync)}: waiting fault injection occurs...");
var sw = Stopwatch.StartNew();
while (sw.Elapsed < FaultInjection.LatencyTimeBuffer)
{
if (connectionStatusChanges.ContainsKey(ConnectionStatus.Disconnected))
{
break;
}
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
sw.Reset();
lastConnectionStatus.Should().Be(ConnectionStatus.Disconnected, $"Expected device to be {ConnectionStatus.Disconnected} but was {lastConnectionStatus}.");
lastConnectionStatusChangeReason.Should().Be(ConnectionStatusChangeReason.Retry_Expired, $"Expected device to be {ConnectionStatusChangeReason.Retry_Expired} but was {lastConnectionStatusChangeReason}.");
connectionStatusChanges.Should().NotContainKey(ConnectionStatus.Disconnected_Retrying, $"Shouldn't get {ConnectionStatus.Disconnected_Retrying} status change.");
int connected = connectionStatusChanges[ConnectionStatus.Connected];
connected.Should().Be(1, $"Should get {ConnectionStatus.Connected} once but got it {connected} times.");
int disconnected = connectionStatusChanges[ConnectionStatus.Disconnected];
disconnected.Should().Be(1, $"Should get {ConnectionStatus.Disconnected} once but got it {disconnected} times.");
}
private async Task SendMessageMuxedOverAmqp(
TestDeviceType type,
Client.TransportType transport,
int poolSize,
int devicesCount,
ConnectionStringAuthScope authScope = ConnectionStringAuthScope.Device)
{
var transportSettings = new ITransportSettings[]
{
new AmqpTransportSettings(transport)
{
AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
{
MaxPoolSize = unchecked ((uint)poolSize),
Pooling = true
}
}
};
ICollection <DeviceClient> deviceClients = new List <DeviceClient>();
Dictionary <DeviceClient, int> deviceClientConnectionStatusChangeCount = new Dictionary <DeviceClient, int>();
try
{
_log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}: Starting the test execution for {devicesCount} devices");
for (int i = 0; i < devicesCount; i++)
{
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int setConnectionStatusChangesHandlerCount = 0;
TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{DevicePrefix}_{i}_", type).ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
deviceClients.Add(deviceClient);
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
setConnectionStatusChangesHandlerCount++;
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
_log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}");
deviceClientConnectionStatusChangeCount[deviceClient] = setConnectionStatusChangesHandlerCount;
});
_log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}: Preparing to send message for device {i}");
await deviceClient.OpenAsync().ConfigureAwait(false);
await MessageSend.SendSingleMessageAndVerifyAsync(deviceClient, testDevice.Id).ConfigureAwait(false);
}
}
finally
{
// Close and dispose all of the device client instances here
foreach (DeviceClient deviceClient in deviceClients)
{
await deviceClient.CloseAsync().ConfigureAwait(false);
// The connection status change count should be 2: connect (open) and disabled (close)
Assert.IsTrue(deviceClientConnectionStatusChangeCount[deviceClient] == 2, $"Connection status change count for deviceClient {TestLogging.GetHashCode(deviceClient)} is {deviceClientConnectionStatusChangeCount[deviceClient]}");
_log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
deviceClient.Dispose();
}
}
}
public static async Task TestPoolAmqpAsync(
string devicePrefix,
Client.TransportType transport,
int poolSize,
int devicesCount,
Func <DeviceClient, TestDevice, Task> initOperation,
Func <DeviceClient, TestDevice, Task> testOperation,
Func <Task> cleanupOperation,
ConnectionStringAuthScope authScope,
bool ignoreConnectionStatus)
{
var transportSettings = new ITransportSettings[]
{
new AmqpTransportSettings(transport)
{
AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
{
MaxPoolSize = unchecked ((uint)poolSize),
Pooling = true
}
}
};
int totalRuns = 0;
int successfulRuns = 0;
int currentSuccessRate = 0;
bool reRunTest = false;
IList <TestDevice> testDevices = new List <TestDevice>();
IList <DeviceClient> deviceClients = new List <DeviceClient>();
IList <AmqpConnectionStatusChange> amqpConnectionStatuses = new List <AmqpConnectionStatusChange>();
IList <Task> operations = new List <Task>();
do
{
totalRuns++;
// Arrange
// Initialize the test device client instances
// Set the device client connection status change handler
s_log.WriteLine($">>> {nameof(PoolingOverAmqp)} Initializing Device Clients for multiplexing test - Test run {totalRuns}");
for (int i = 0; i < devicesCount; i++)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{devicePrefix}_{i}_").ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
var amqpConnectionStatusChange = new AmqpConnectionStatusChange();
deviceClient.SetConnectionStatusChangesHandler(amqpConnectionStatusChange.ConnectionStatusChangesHandler);
testDevices.Add(testDevice);
deviceClients.Add(deviceClient);
amqpConnectionStatuses.Add(amqpConnectionStatusChange);
if (initOperation != null)
{
operations.Add(initOperation(deviceClient, testDevice));
}
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
try
{
for (int i = 0; i < devicesCount; i++)
{
operations.Add(testOperation(deviceClients[i], testDevices[i]));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
// Close the device client instances and verify the connection status change checks
bool deviceConnectionStatusAsExpected = true;
for (int i = 0; i < devicesCount; i++)
{
await deviceClients[i].CloseAsync().ConfigureAwait(false);
if (!ignoreConnectionStatus)
{
// The connection status change count should be 2: connect (open) and disabled (close)
if (amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount != 2)
{
deviceConnectionStatusAsExpected = false;
}
// The connection status should be "Disabled", with connection status change reason "Client_close"
Assert.AreEqual(
ConnectionStatus.Disabled,
amqpConnectionStatuses[i].LastConnectionStatus,
$"The actual connection status is = {amqpConnectionStatuses[i].LastConnectionStatus}");
Assert.AreEqual(
ConnectionStatusChangeReason.Client_Close,
amqpConnectionStatuses[i].LastConnectionStatusChangeReason,
$"The actual connection status change reason is = {amqpConnectionStatuses[i].LastConnectionStatusChangeReason}");
}
}
if (deviceConnectionStatusAsExpected)
{
successfulRuns++;
}
currentSuccessRate = (int)((double)successfulRuns / totalRuns * 100);
reRunTest = currentSuccessRate < TestSuccessRate;
}
finally
{
// Close the service-side components and dispose the device client instances.
if (cleanupOperation != null)
{
await cleanupOperation().ConfigureAwait(false);
}
foreach (DeviceClient deviceClient in deviceClients)
{
deviceClient.Dispose();
}
// Clean up the local lists
testDevices.Clear();
deviceClients.Clear();
amqpConnectionStatuses.Clear();
}
} while (reRunTest && totalRuns < MaxTestRunCount);
Assert.IsFalse(reRunTest, $"Device client instances got disconnected in {totalRuns - successfulRuns} runs out of {totalRuns}; current testSuccessRate = {currentSuccessRate}%.");
}
private async Task SendMethodAndRespondMuxedOverAmqp(
ConnectionStringAuthScope authScope,
Client.TransportType transport,
int poolSize,
int devicesCount,
Func <DeviceClient, Task <Task> > setDeviceReceiveMethod
)
{
var transportSettings = new ITransportSettings[]
{
new AmqpTransportSettings(transport)
{
AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
{
MaxPoolSize = unchecked ((uint)poolSize),
Pooling = true
}
}
};
ICollection <DeviceClient> deviceClients = new List <DeviceClient>();
Dictionary <DeviceClient, int> deviceClientConnectionStatusChangeCount = new Dictionary <DeviceClient, int>();
try
{
for (int i = 0; i < devicesCount; i++)
{
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int setConnectionStatusChangesHandlerCount = 0;
TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{DevicePrefix}_{i}_").ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
deviceClients.Add(deviceClient);
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
setConnectionStatusChangesHandlerCount++;
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
_log.WriteLine($"{nameof(MethodE2EMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}");
deviceClientConnectionStatusChangeCount[deviceClient] = setConnectionStatusChangesHandlerCount;
});
Task methodReceivedTask = await setDeviceReceiveMethod(deviceClient).ConfigureAwait(false);
await Task.WhenAll(
MethodOperation.ServiceSendMethodAndVerifyResponse(testDevice.Id),
methodReceivedTask).ConfigureAwait(false);
}
}
finally
{
// Close and dispose all of the device client instances here
foreach (DeviceClient deviceClient in deviceClients)
{
await deviceClient.CloseAsync().ConfigureAwait(false);
// The connection status change count should be 2: connect (open) and disabled (close)
Assert.IsTrue(deviceClientConnectionStatusChangeCount[deviceClient] == 2, $"Connection status change count for deviceClient {TestLogging.GetHashCode(deviceClient)} is {deviceClientConnectionStatusChangeCount[deviceClient]}");
_log.WriteLine($"{nameof(MethodE2EMultiplexingTests)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
deviceClient.Dispose();
}
}
}
// Error injection template method.
public static async Task TestErrorInjectionAsync(
string devicePrefix,
TestDeviceType type,
Client.TransportType transport,
string faultType,
string reason,
int delayInSec,
int durationInSec,
Func <DeviceClient, TestDevice, Task> initOperation,
Func <DeviceClient, TestDevice, Task> testOperation,
Func <Task> cleanupOperation)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(devicePrefix, type).ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transport);
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int setConnectionStatusChangesHandlerCount = 0;
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
s_log.WriteLine($"{nameof(FaultInjection)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}");
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
setConnectionStatusChangesHandlerCount++;
});
var watch = new Stopwatch();
try
{
await deviceClient.OpenAsync().ConfigureAwait(false);
if (transport != Client.TransportType.Http1)
{
Assert.AreEqual(1, setConnectionStatusChangesHandlerCount);
Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason);
}
await initOperation(deviceClient, testDevice).ConfigureAwait(false);
s_log.WriteLine($">>> {nameof(FaultInjection)} Testing baseline");
await testOperation(deviceClient, testDevice).ConfigureAwait(false);
await ActivateFaultInjection(transport, faultType, reason, delayInSec, durationInSec, deviceClient).ConfigureAwait(false);
s_log.WriteLine($">>> {nameof(FaultInjection)} Testing fault handling");
watch.Start();
s_log.WriteLine($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {FaultInjection.WaitForDisconnectMilliseconds}ms");
await Task.Delay(FaultInjection.WaitForDisconnectMilliseconds).ConfigureAwait(false);
await testOperation(deviceClient, testDevice).ConfigureAwait(false);
await deviceClient.CloseAsync().ConfigureAwait(false);
if (transport == Client.TransportType.Mqtt ||
transport == Client.TransportType.Mqtt_Tcp_Only ||
transport == Client.TransportType.Mqtt_WebSocket_Only)
{
// Our fault injection is only terminating the connection for MQTT. (HTTP is not connection-oriented, AMQP is not actually terminating the TCP layer.)
Assert.IsTrue(setConnectionStatusChangesHandlerCount >= 4);
Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason);
}
}
finally
{
await cleanupOperation().ConfigureAwait(false);
s_log.WriteLine($"{nameof(FaultInjection)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
deviceClient.Dispose();
watch.Stop();
int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds;
if (timeToFinishFaultInjection > 0)
{
s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed.");
await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false);
}
}
}
public static async Task TestFaultInjectionPoolAmqpAsync(
string devicePrefix,
Client.TransportType transport,
int poolSize,
int devicesCount,
string faultType,
string reason,
int delayInSec,
int durationInSec,
Func <DeviceClient, TestDevice, Task> initOperation,
Func <DeviceClient, TestDevice, Task> testOperation,
Func <IList <DeviceClient>, Task> cleanupOperation,
ConnectionStringAuthScope authScope)
{
var transportSettings = new ITransportSettings[]
{
new AmqpTransportSettings(transport)
{
AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
{
MaxPoolSize = unchecked ((uint)poolSize),
Pooling = true
}
}
};
IList <TestDevice> testDevices = new List <TestDevice>();
IList <DeviceClient> deviceClients = new List <DeviceClient>();
IList <AmqpConnectionStatusChange> amqpConnectionStatuses = new List <AmqpConnectionStatusChange>();
IList <Task> operations = new List <Task>();
// Arrange
// Initialize the test device client instances
// Set the device client connection status change handler
s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)} Initializing Device Clients for multiplexing test.");
for (int i = 0; i < devicesCount; i++)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{devicePrefix}_{i}_").ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
var amqpConnectionStatusChange = new AmqpConnectionStatusChange(testDevice.Id);
deviceClient.SetConnectionStatusChangesHandler(amqpConnectionStatusChange.ConnectionStatusChangesHandler);
testDevices.Add(testDevice);
deviceClients.Add(deviceClient);
amqpConnectionStatuses.Add(amqpConnectionStatusChange);
operations.Add(initOperation(deviceClient, testDevice));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
var watch = new Stopwatch();
try
{
// Act-Assert
// Perform the test operation and verify the operation is successful
// Perform baseline test operation
for (int i = 0; i < devicesCount; i++)
{
s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing baseline operation for device {i}.");
operations.Add(testOperation(deviceClients[i], testDevices[i]));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
// Inject the fault into device 0
watch.Start();
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: {testDevices[0].Id} Requesting fault injection type={faultType} reason={reason}, delay={delayInSec}s, duration={durationInSec}s");
var faultInjectionMessage = FaultInjection.ComposeErrorInjectionProperties(faultType, reason, delayInSec, durationInSec);
await deviceClients[0].SendEventAsync(faultInjectionMessage).ConfigureAwait(false);
s_log.WriteLine($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {delayInSec} seconds.");
await Task.Delay(TimeSpan.FromSeconds(delayInSec)).ConfigureAwait(false);
// For disconnect type faults, the faulted device should disconnect and all devices should recover.
if (FaultInjection.FaultShouldDisconnect(faultType))
{
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirming fault injection has been actived.");
// Check that service issued the fault to the faulting device [device 0]
bool isFaulted = false;
for (int i = 0; i < FaultInjection.LatencyTimeBufferInSec; i++)
{
if (amqpConnectionStatuses[0].ConnectionStatusChangeCount >= 2)
{
isFaulted = true;
break;
}
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
Assert.IsTrue(isFaulted, $"The device {testDevices[0].Id} did not get faulted with fault type: {faultType}");
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirmed fault injection has been actived.");
// Check all devices are back online
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirming all devices back online.");
bool notRecovered = true;
int j = 0;
for (int i = 0; notRecovered && i < durationInSec + FaultInjection.LatencyTimeBufferInSec; i++)
{
notRecovered = false;
for (j = 0; j < devicesCount; j++)
{
if (amqpConnectionStatuses[j].LastConnectionStatus != ConnectionStatus.Connected)
{
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
notRecovered = true;
break;
}
}
}
if (notRecovered)
{
Assert.Fail($"{testDevices[j].Id} did not reconnect.");
}
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Confirmed all devices back online.");
// Perform the test operation for all devices
for (int i = 0; i < devicesCount; i++)
{
s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation for device {i}.");
operations.Add(testOperation(deviceClients[i], testDevices[i]));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
}
else
{
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation while fault injection is being activated.");
// Perform the test operation for the faulted device multi times.
for (int i = 0; i < FaultInjection.LatencyTimeBufferInSec; i++)
{
s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation for device 0 - Run {i}.");
await testOperation(deviceClients[0], testDevices[0]).ConfigureAwait(false);
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
}
// Close the device client instances
for (int i = 0; i < devicesCount; i++)
{
operations.Add(deviceClients[i].CloseAsync());
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
// Verify the connection status change checks.
// For all of the devices - last connection status should be "Disabled", with reason "Client_close"
for (int i = 0; i < devicesCount; i++)
{
// For the faulted device [device 0] - verify the connection status change count.
if (i == 0)
{
if (FaultInjection.FaultShouldDisconnect(faultType))
{
// 4 is the minimum notification count: connect, fault, reconnect, disable.
Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangeCount >= 4, $"The expected connection status change count for {testDevices[i].Id} should equals or greater than 4 but was {amqpConnectionStatuses[i].ConnectionStatusChangeCount}");
}
else
{
// 2 is the minimum notification count: connect, disable.
Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangeCount == 2, $"The expected connection status change count for {testDevices[i].Id} should be 2 but was {amqpConnectionStatuses[i].ConnectionStatusChangeCount}");
}
}
Assert.AreEqual(ConnectionStatus.Disabled, amqpConnectionStatuses[i].LastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Disabled} but was {amqpConnectionStatuses[i].LastConnectionStatus}");
Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, amqpConnectionStatuses[i].LastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Client_Close} but was {amqpConnectionStatuses[i].LastConnectionStatusChangeReason}");
}
}
finally
{
// Close the service-side components and dispose the device client instances.
await cleanupOperation(deviceClients).ConfigureAwait(false);
watch.Stop();
int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds;
if (timeToFinishFaultInjection > 0)
{
s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed.");
await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false);
}
}
}
private async Task DeviceCombinedClientOperations(
Client.TransportType transport,
int poolSize,
int devicesCount,
ConnectionStringAuthScope authScope
)
{
var transportSettings = new ITransportSettings[]
{
new AmqpTransportSettings(transport)
{
AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
{
MaxPoolSize = unchecked ((uint)poolSize),
Pooling = true
}
}
};
// Initialize service client for service-side operations
ServiceClient serviceClient = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);
ICollection <DeviceClient> deviceClients = new List <DeviceClient>();
Dictionary <DeviceClient, int> deviceClientConnectionStatusChangeCount = new Dictionary <DeviceClient, int>();
try
{
_log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Starting the test execution for {devicesCount} devices");
for (int i = 0; i < devicesCount; i++)
{
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int setConnectionStatusChangesHandlerCount = 0;
TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{DevicePrefix}_{i}_").ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
deviceClients.Add(deviceClient);
// Set the connection status change handler
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
setConnectionStatusChangesHandlerCount++;
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
_log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}");
deviceClientConnectionStatusChangeCount[deviceClient] = setConnectionStatusChangesHandlerCount;
});
// Perform D2C Operation
_log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Preparing to send message for device {i}");
await deviceClient.OpenAsync().ConfigureAwait(false);
await MessageSend.SendSingleMessageAndVerifyAsync(deviceClient, testDevice.Id).ConfigureAwait(false);
// Perform C2D Operation
_log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Setting the device {i} to receive C2D message.");
string payload, messageId, p1Value;
Message msg = MessageReceive.ComposeC2DTestMessage(out payload, out messageId, out p1Value);
await serviceClient.SendAsync(testDevice.Id, msg).ConfigureAwait(false);
await MessageReceive.VerifyReceivedC2DMessageAsync(transport, deviceClient, payload, p1Value).ConfigureAwait(false);
// Invoke direct methods
_log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Testing direct methods for device {i}");
Task methodReceivedTask = await MethodOperation.SetDeviceReceiveMethod(deviceClient).ConfigureAwait(false);
await Task.WhenAll(
MethodOperation.ServiceSendMethodAndVerifyResponse(testDevice.Id),
methodReceivedTask).ConfigureAwait(false);
// Set reported twin properties
_log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Setting reported Twin properties for device {i}");
await TwinOperation.Twin_DeviceSetsReportedPropertyAndGetsItBack(deviceClient).ConfigureAwait(false);
// Receive set desired twin properties
_log.WriteLine($"{nameof(CombinedClientOperationsMultiplexingTests)}: Received set desired tein properties for device {i}");
var propName = Guid.NewGuid().ToString();
var propValue = Guid.NewGuid().ToString();
Task updateReceivedTask = await TwinOperation.SetTwinPropertyUpdateCallbackHandlerAsync(deviceClient, propName, propValue).ConfigureAwait(false);
await Task.WhenAll(
TwinOperation.RegistryManagerUpdateDesiredPropertyAsync(testDevice.Id, propName, propValue),
updateReceivedTask).ConfigureAwait(false);
}
}
finally
{
// Close and dispose all of the device client instances here
foreach (DeviceClient deviceClient in deviceClients)
{
await deviceClient.CloseAsync().ConfigureAwait(false);
// The connection status change count should be 2: connect (open) and disabled (close)
Assert.IsTrue(deviceClientConnectionStatusChangeCount[deviceClient] == 2);
_log.WriteLine($"{nameof(MessageSendE2EMultiplexingTests)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
deviceClient.Dispose();
}
}
}
private async Task Twin_ServiceSetsDesiredPropertyAndDeviceReceivesEventMuxedOverAmqp(
ConnectionStringAuthScope authScope,
Client.TransportType transport,
int poolSize,
int devicesCount,
Func <DeviceClient, string, string, Task <Task> > setTwinPropertyUpdateCallbackAsync
)
{
var propName = Guid.NewGuid().ToString();
var propValue = Guid.NewGuid().ToString();
var transportSettings = new ITransportSettings[]
{
new AmqpTransportSettings(transport)
{
AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
{
MaxPoolSize = unchecked ((uint)poolSize),
Pooling = true
}
}
};
ICollection <DeviceClient> deviceClients = new List <DeviceClient>();
Dictionary <DeviceClient, int> deviceClientConnectionStatusChangeCount = new Dictionary <DeviceClient, int>();
try
{
_log.WriteLine($"{nameof(TwinE2EMultiplexingTests)}: Starting the test execution for {devicesCount} devices");
for (int i = 0; i < devicesCount; i++)
{
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int setConnectionStatusChangesHandlerCount = 0;
TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{DevicePrefix}_{i}_").ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
deviceClients.Add(deviceClient);
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
setConnectionStatusChangesHandlerCount++;
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
_log.WriteLine($"{nameof(TwinE2EMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}");
deviceClientConnectionStatusChangeCount[deviceClient] = setConnectionStatusChangesHandlerCount;
});
_log.WriteLine($"{nameof(Twin_ServiceSetsDesiredPropertyAndDeviceReceivesEventMuxedOverAmqp)}: name={propName}, value={propValue}");
Task updateReceivedTask = await setTwinPropertyUpdateCallbackAsync(deviceClient, propName, propValue).ConfigureAwait(false);
await Task.WhenAll(
TwinOperation.RegistryManagerUpdateDesiredPropertyAsync(testDevice.Id, propName, propValue),
updateReceivedTask).ConfigureAwait(false);
}
}
finally
{
// Close and dispose all of the device client instances here
foreach (DeviceClient deviceClient in deviceClients)
{
await deviceClient.CloseAsync().ConfigureAwait(false);
// The connection status change count should be 2: connect (open) and disabled (close)
Assert.IsTrue(deviceClientConnectionStatusChangeCount[deviceClient] == 2);
_log.WriteLine($"{nameof(TwinE2EMultiplexingTests)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
deviceClient.Dispose();
}
}
}
public async Task DuplicateDevice_NoRetry_NoPingpong_OpenAsync()
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(Logger, _devicePrefix, TestDeviceType.Sasl).ConfigureAwait(false);
Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: 2 device client instances with the same deviceId={testDevice.Id}.");
using DeviceClient deviceClient1 = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only);
using DeviceClient deviceClient2 = testDevice.CreateDeviceClient(Client.TransportType.Amqp_Tcp_Only);
Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: set device client instance 1 to no retry.");
deviceClient1.SetRetryPolicy(new NoRetry());
ConnectionStatus?lastConnectionStatusDevice1 = null;
var connectionStatusChangesDevice1 = new Dictionary <ConnectionStatus, int>();
deviceClient1.SetConnectionStatusChangesHandler((status, reason) =>
{
connectionStatusChangesDevice1.TryGetValue(status, out int count);
count++;
connectionStatusChangesDevice1[status] = count;
lastConnectionStatusDevice1 = status;
});
ConnectionStatus?lastConnectionStatusDevice2 = null;
var connectionStatusChangesDevice2 = new Dictionary <ConnectionStatus, int>();
deviceClient2.SetConnectionStatusChangesHandler((status, reason) =>
{
connectionStatusChangesDevice2.TryGetValue(status, out int count);
count++;
connectionStatusChangesDevice2[status] = count;
lastConnectionStatusDevice2 = status;
});
Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: device client instance 1 calling OpenAsync...");
await deviceClient1.OpenAsync().ConfigureAwait(false);
await deviceClient1
.SetMethodHandlerAsync(
"empty_method",
(methodRequest, userContext) => Task.FromResult(new MethodResponse(200)),
deviceClient1)
.ConfigureAwait(false);
Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: device client instance 2 calling OpenAsync...");
await deviceClient2.OpenAsync().ConfigureAwait(false);
await deviceClient2
.SetMethodHandlerAsync(
"empty_method",
(methodRequest, userContext) => Task.FromResult(new MethodResponse(200)),
deviceClient2)
.ConfigureAwait(false);
Logger.Trace($"{nameof(DuplicateDevice_NoRetry_NoPingpong_OpenAsync)}: waiting device client instance 1 to be kicked off...");
var sw = Stopwatch.StartNew();
while (sw.Elapsed < FaultInjection.LatencyTimeBuffer)
{
if (connectionStatusChangesDevice1.ContainsKey(ConnectionStatus.Disconnected))
{
break;
}
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
sw.Reset();
lastConnectionStatusDevice1.Should().Be(ConnectionStatus.Disconnected, $"Excpected device 1 to be {ConnectionStatus.Disconnected} but was {lastConnectionStatusDevice1}.");
connectionStatusChangesDevice1.Should().NotContainKey(ConnectionStatus.Disconnected_Retrying, $"Shouldn't get {ConnectionStatus.Disconnected_Retrying} status change.");
int connected = connectionStatusChangesDevice1[ConnectionStatus.Connected];
connected.Should().Be(1, $"Should get {ConnectionStatus.Connected} once but got it {connected} times.");
int disconnected = connectionStatusChangesDevice1[ConnectionStatus.Disconnected];
disconnected.Should().Be(1, $"Should get {ConnectionStatus.Disconnected} once but got it {disconnected} times.");
lastConnectionStatusDevice2.Should().Be(ConnectionStatus.Connected, $"Expected device 2 to be {ConnectionStatus.Connected} but was {lastConnectionStatusDevice2}.");
}
private async Task ReceiveMessageMuxedOverAmqp(
Client.TransportType transport,
int poolSize,
int devicesCount,
ConnectionStringAuthScope authScope = ConnectionStringAuthScope.Device)
{
var transportSettings = new ITransportSettings[]
{
new AmqpTransportSettings(transport)
{
AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
{
MaxPoolSize = unchecked ((uint)poolSize),
Pooling = true
}
}
};
ICollection <DeviceClient> deviceClients = new List <DeviceClient>();
Dictionary <DeviceClient, int> deviceClientConnectionStatusChangeCount = new Dictionary <DeviceClient, int>();
ServiceClient serviceClient = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);
try
{
_log.WriteLine($"{nameof(MessageReceiveE2EMultiplexingTests)}: Starting the test execution for {devicesCount} devices");
await serviceClient.OpenAsync().ConfigureAwait(false);
for (int i = 0; i < devicesCount; i++)
{
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int setConnectionStatusChangesHandlerCount = 0;
TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{DevicePrefix}_{i}_").ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
deviceClients.Add(deviceClient);
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
setConnectionStatusChangesHandlerCount++;
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
_log.WriteLine($"{nameof(MessageReceiveE2EMultiplexingTests)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={setConnectionStatusChangesHandlerCount}");
deviceClientConnectionStatusChangeCount[deviceClient] = setConnectionStatusChangesHandlerCount;
});
_log.WriteLine($"{nameof(MessageReceiveE2EMultiplexingTests)}: Preparing to receive message for device {i}");
await deviceClient.OpenAsync().ConfigureAwait(false);
string payload, messageId, p1Value;
Message msg = MessageReceive.ComposeC2DTestMessage(out payload, out messageId, out p1Value);
await serviceClient.SendAsync(testDevice.Id, msg).ConfigureAwait(false);
await MessageReceive.VerifyReceivedC2DMessageAsync(transport, deviceClient, payload, p1Value).ConfigureAwait(false);
}
}
finally
{
await serviceClient.CloseAsync().ConfigureAwait(false);
serviceClient.Dispose();
// Close and dispose all of the device client instances here
foreach (DeviceClient deviceClient in deviceClients)
{
await deviceClient.CloseAsync().ConfigureAwait(false);
// The connection status change count should be 2: connect (open) and disabled (close)
Assert.IsTrue(deviceClientConnectionStatusChangeCount[deviceClient] == 2);
_log.WriteLine($"{nameof(MessageReceiveE2EMultiplexingTests)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
deviceClient.Dispose();
}
}
}
// Error injection template method.
public static async Task TestErrorInjectionAsync(
string devicePrefix,
TestDeviceType type,
Client.TransportType transport,
string faultType,
string reason,
int delayInSec,
int durationInSec,
Func <DeviceClient, TestDevice, Task> initOperation,
Func <DeviceClient, TestDevice, Task> testOperation,
Func <Task> cleanupOperation)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(devicePrefix, type).ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transport);
ConnectionStatus? lastConnectionStatus = null;
ConnectionStatusChangeReason?lastConnectionStatusChangeReason = null;
int connectionStatusChangeCount = 0;
deviceClient.SetConnectionStatusChangesHandler((status, statusChangeReason) =>
{
connectionStatusChangeCount++;
lastConnectionStatus = status;
lastConnectionStatusChangeReason = statusChangeReason;
s_log.WriteLine($"{nameof(FaultInjection)}.{nameof(ConnectionStatusChangesHandler)}: status={status} statusChangeReason={statusChangeReason} count={connectionStatusChangeCount}");
});
var watch = new Stopwatch();
try
{
await deviceClient.OpenAsync().ConfigureAwait(false);
if (transport != Client.TransportType.Http1)
{
Assert.IsTrue(connectionStatusChangeCount >= 1, $"The expected connection status change should be equal or greater than 1 but was {connectionStatusChangeCount}"); // Normally one connection but in some cases, due to network issues we might have already retried several times to connect.
Assert.AreEqual(ConnectionStatus.Connected, lastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Connected} but was {lastConnectionStatus}");
Assert.AreEqual(ConnectionStatusChangeReason.Connection_Ok, lastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Connection_Ok} but was {lastConnectionStatusChangeReason}");
}
await initOperation(deviceClient, testDevice).ConfigureAwait(false);
s_log.WriteLine($">>> {nameof(FaultInjection)} Testing baseline");
await testOperation(deviceClient, testDevice).ConfigureAwait(false);
watch.Start();
s_log.WriteLine($">>> {nameof(FaultInjection)} Testing fault handling");
await ActivateFaultInjection(transport, faultType, reason, delayInSec, durationInSec, deviceClient).ConfigureAwait(false);
s_log.WriteLine($"{nameof(FaultInjection)}: Waiting for fault injection to be active: {delayInSec} seconds.");
await Task.Delay(TimeSpan.FromSeconds(delayInSec)).ConfigureAwait(false);
// For disconnect type faults, the device should disconnect and recover.
if (FaultShouldDisconnect(faultType))
{
s_log.WriteLine($"{nameof(FaultInjection)}: Confirming fault injection has been actived.");
// Check that service issued the fault to the faulting device
bool isFaulted = false;
for (int i = 0; i < LatencyTimeBufferInSec; i++)
{
if (connectionStatusChangeCount >= 2)
{
isFaulted = true;
break;
}
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
Assert.IsTrue(isFaulted, $"The device {testDevice.Id} did not get faulted with fault type: {faultType}");
s_log.WriteLine($"{nameof(FaultInjection)}: Confirmed fault injection has been actived.");
// Check the device is back online
s_log.WriteLine($"{nameof(FaultInjection)}: Confirming device back online.");
for (int i = 0; lastConnectionStatus != ConnectionStatus.Connected && i < durationInSec + LatencyTimeBufferInSec; i++)
{
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
Assert.AreEqual(lastConnectionStatus, ConnectionStatus.Connected, $"{testDevice.Id} did not reconnect.");
s_log.WriteLine($"{nameof(FaultInjection)}: Confirmed device back online.");
// Perform the test operation.
s_log.WriteLine($">>> {nameof(FaultInjection)}: Performing test operation for device {testDevice.Id}.");
await testOperation(deviceClient, testDevice).ConfigureAwait(false);
}
else
{
s_log.WriteLine($"{nameof(FaultInjection)}: Performing test operation while fault injection is being activated.");
// Perform the test operation for the faulted device multi times.
for (int i = 0; i < LatencyTimeBufferInSec; i++)
{
s_log.WriteLine($">>> {nameof(FaultInjection)}: Performing test operation for device - Run {i}.");
await testOperation(deviceClient, testDevice).ConfigureAwait(false);
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
}
await deviceClient.CloseAsync().ConfigureAwait(false);
if (transport != Client.TransportType.Http1)
{
if (FaultInjection.FaultShouldDisconnect(faultType))
{
// 4 is the minimum notification count: connect, fault, reconnect, disable.
// There are cases where the retry must be timed out (i.e. very likely for MQTT where otherwise
// we would attempt to send the fault injection forever.)
Assert.IsTrue(connectionStatusChangeCount >= 4, $"The expected connection status change count for {testDevice.Id} should be equal or greater than 4 but was {connectionStatusChangeCount}");
}
else
{
// 2 is the minimum notification count: connect, disable.
// We will monitor the test environment real network stability and switch to >=2 if necessary to
// account for real network issues.
Assert.IsTrue(connectionStatusChangeCount == 2, $"The expected connection status change count for {testDevice.Id} should be 2 but was {connectionStatusChangeCount}");
}
Assert.AreEqual(ConnectionStatus.Disabled, lastConnectionStatus, $"The expected connection status should be {ConnectionStatus.Disabled} but was {lastConnectionStatus}");
Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, lastConnectionStatusChangeReason, $"The expected connection status change reason should be {ConnectionStatusChangeReason.Client_Close} but was {lastConnectionStatusChangeReason}");
}
}
finally
{
await cleanupOperation().ConfigureAwait(false);
s_log.WriteLine($"{nameof(FaultInjection)}: Disposing deviceClient {TestLogging.GetHashCode(deviceClient)}");
deviceClient.Dispose();
watch.Stop();
int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds;
if (timeToFinishFaultInjection > 0)
{
s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed.");
await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false);
}
}
}
public static async Task TestFaultInjectionPoolAmqpAsync(
string devicePrefix,
Client.TransportType transport,
int poolSize,
int devicesCount,
string faultType,
string reason,
int delayInSec,
int durationInSec,
Func <DeviceClient, TestDevice, Task> initOperation,
Func <DeviceClient, TestDevice, Task> testOperation,
Func <IList <DeviceClient>, Task> cleanupOperation,
ConnectionStringAuthScope authScope)
{
var transportSettings = new ITransportSettings[]
{
new AmqpTransportSettings(transport)
{
AmqpConnectionPoolSettings = new AmqpConnectionPoolSettings()
{
MaxPoolSize = unchecked ((uint)poolSize),
Pooling = true
}
}
};
IList <TestDevice> testDevices = new List <TestDevice>();
IList <DeviceClient> deviceClients = new List <DeviceClient>();
IList <AmqpConnectionStatusChange> amqpConnectionStatuses = new List <AmqpConnectionStatusChange>();
IList <Task> operations = new List <Task>();
// Arrange
// Initialize the test device client instances
// Set the device client connection status change handler
s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)} Initializing Device Clients for multiplexing test.");
for (int i = 0; i < devicesCount; i++)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync($"{devicePrefix}_{i}_").ConfigureAwait(false);
DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings, authScope);
var amqpConnectionStatusChange = new AmqpConnectionStatusChange(testDevice.Id);
deviceClient.SetConnectionStatusChangesHandler(amqpConnectionStatusChange.ConnectionStatusChangesHandler);
testDevices.Add(testDevice);
deviceClients.Add(deviceClient);
amqpConnectionStatuses.Add(amqpConnectionStatusChange);
operations.Add(initOperation(deviceClient, testDevice));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
var watch = new Stopwatch();
try
{
// Act-Assert
// Perform the test operation and verify the operation is successful
// Perform baseline test operation
for (int i = 0; i < devicesCount; i++)
{
s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing baseline operation for device {i}.");
operations.Add(testOperation(deviceClients[i], testDevices[i]));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
// Inject the fault into device 0
watch.Start();
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Device {0} Requesting fault injection type={faultType} reason={reason}, delay={delayInSec}s, duration={durationInSec}s");
var faultInjectionMessage = FaultInjection.ComposeErrorInjectionProperties(faultType, reason, delayInSec, durationInSec);
await deviceClients[0].SendEventAsync(faultInjectionMessage).ConfigureAwait(false);
int delay = FaultInjection.WaitForReconnectMilliseconds - (int)watch.ElapsedMilliseconds;
if (delay < 0)
{
delay = 0;
}
s_log.WriteLine($"{nameof(FaultInjectionPoolingOverAmqp)}: Waiting for fault injection to be active and device to be connected: {delay}ms");
await Task.Delay(delay).ConfigureAwait(false);
// Perform the test operation for all devices
for (int i = 0; i < devicesCount; i++)
{
s_log.WriteLine($">>> {nameof(FaultInjectionPoolingOverAmqp)}: Performing test operation for device {i}.");
operations.Add(testOperation(deviceClients[i], testDevices[i]));
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
// Close the device client instances
for (int i = 0; i < devicesCount; i++)
{
operations.Add(deviceClients[i].CloseAsync());
}
await Task.WhenAll(operations).ConfigureAwait(false);
operations.Clear();
// Verify the connection status change checks.
// For all of the devices - last connection status should be "Disabled", with reason "Client_close"
for (int i = 0; i < devicesCount; i++)
{
// For the faulted device [device 0] - verify the connection status change count.
if (i == 0)
{
if (FaultInjection.FaultShouldDisconnect(faultType))
{
// 4 is the minimum notification count: connect, fault, reconnect, disable.
Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount >= 4, $"The actual connection status change count for faulted device[0] is = {amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount}");
}
else
{
// 2 is the minimum notification count: connect, disable.
Assert.IsTrue(amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount == 2, $"The actual connection status change count for for faulted device[0] is = {amqpConnectionStatuses[i].ConnectionStatusChangesHandlerCount}");
}
}
Assert.AreEqual(ConnectionStatus.Disabled, amqpConnectionStatuses[i].LastConnectionStatus);
Assert.AreEqual(ConnectionStatusChangeReason.Client_Close, amqpConnectionStatuses[i].LastConnectionStatusChangeReason);
}
}
finally
{
// Close the service-side components and dispose the device client instances.
await cleanupOperation(deviceClients).ConfigureAwait(false);
watch.Stop();
int timeToFinishFaultInjection = durationInSec * 1000 - (int)watch.ElapsedMilliseconds;
if (timeToFinishFaultInjection > 0)
{
s_log.WriteLine($"{nameof(FaultInjection)}: Waiting {timeToFinishFaultInjection}ms to ensure that FaultInjection duration passed.");
await Task.Delay(timeToFinishFaultInjection).ConfigureAwait(false);
}
}
}
