private async Task Twin_DeviceDesiredPropertyUpdateRecoveryAsync(
Client.TransportType transport,
string faultType,
string reason,
int delayInSec)
{
TestDeviceCallbackHandler testDeviceCallbackHandler = null;
var registryManager = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);
using var cts = new CancellationTokenSource(FaultInjection.RecoveryTimeMilliseconds);
var propName = Guid.NewGuid().ToString();
var props = new TwinCollection();
// Configure the callback and start accepting twin changes.
Func <DeviceClient, TestDevice, Task> initOperation = async(deviceClient, testDevice) =>
{
testDeviceCallbackHandler = new TestDeviceCallbackHandler(deviceClient);
await testDeviceCallbackHandler.SetTwinPropertyUpdateCallbackHandlerAsync(propName).ConfigureAwait(false);
};
// Change the twin from the service side and verify the device received it.
Func <DeviceClient, TestDevice, Task> testOperation = async(deviceClient, testDevice) =>
{
var propValue = Guid.NewGuid().ToString();
testDeviceCallbackHandler.ExpectedTwinPropertyValue = propValue;
s_log.WriteLine($"{nameof(Twin_DeviceDesiredPropertyUpdateRecoveryAsync)}: name={propName}, value={propValue}");
Task serviceSendTask = RegistryManagerUpdateDesiredPropertyAsync(testDevice.Id, propName, propValue);
Task twinReceivedTask = testDeviceCallbackHandler.WaitForTwinCallbackAsync(cts.Token);
var tasks = new List <Task>()
{
serviceSendTask, twinReceivedTask
};
while (tasks.Count > 0)
{
Task completedTask = await Task.WhenAny(tasks).ConfigureAwait(false);
completedTask.GetAwaiter().GetResult();
tasks.Remove(completedTask);
}
};
await FaultInjection
.TestErrorInjectionAsync(
s_devicePrefix,
TestDeviceType.Sasl,
transport,
faultType,
reason,
delayInSec,
FaultInjection.DefaultDurationInSec,
initOperation,
testOperation,
() => { return(Task.FromResult(false)); })
.ConfigureAwait(false);
}
private async Task SendMethodAndRespondRecoveryAsync(Client.TransportType transport, string faultType, string reason, int delayInSec)
{
TestDeviceCallbackHandler testDeviceCallbackHandler = null;
using var cts = new CancellationTokenSource(FaultInjection.RecoveryTimeMilliseconds);
// Configure the callback and start accepting method calls.
Func <DeviceClient, TestDevice, Task> initOperation = async(deviceClient, testDevice) =>
{
testDeviceCallbackHandler = new TestDeviceCallbackHandler(deviceClient);
await testDeviceCallbackHandler
.SetDeviceReceiveMethodAsync(MethodName, DeviceResponseJson, ServiceRequestJson)
.ConfigureAwait(false);
};
// Call the method from the service side and verify the device received the call.
Func <DeviceClient, TestDevice, Task> testOperation = async(deviceClient, testDevice) =>
{
Task serviceSendTask = ServiceSendMethodAndVerifyResponseAsync(testDevice.Id, MethodName, DeviceResponseJson, ServiceRequestJson);
Task methodReceivedTask = testDeviceCallbackHandler.WaitForMethodCallbackAsync(cts.Token);
var tasks = new List <Task>()
{
serviceSendTask, methodReceivedTask
};
while (tasks.Count > 0)
{
Task completedTask = await Task.WhenAny(tasks).ConfigureAwait(false);
completedTask.GetAwaiter().GetResult();
tasks.Remove(completedTask);
}
};
await FaultInjection
.TestErrorInjectionAsync(
DevicePrefix,
TestDeviceType.Sasl,
transport,
faultType,
reason,
delayInSec,
FaultInjection.DefaultDelayInSec,
initOperation,
testOperation,
() => { return(Task.FromResult <bool>(false)); })
.ConfigureAwait(false);
}
private async Task SendMethodAndRespondRecovery(Client.TransportType transport, string faultType, string reason, int delayInSec)
{
TestDeviceCallbackHandler testDeviceCallbackHandler = null;
var cts = new CancellationTokenSource(FaultInjection.RecoveryTimeMilliseconds);
// Configure the callback and start accepting method calls.
Func <DeviceClient, TestDevice, Task> initOperation = async(deviceClient, testDevice) =>
{
testDeviceCallbackHandler = new TestDeviceCallbackHandler(deviceClient);
await testDeviceCallbackHandler.SetDeviceReceiveMethodAsync(MethodName, DeviceResponseJson, ServiceRequestJson).ConfigureAwait(false);
};
// Call the method from the service side and verify the device received the call.
Func <DeviceClient, TestDevice, Task> testOperation = async(deviceClient, testDevice) =>
{
Task serviceSendTask = ServiceSendMethodAndVerifyResponse(testDevice.Id, MethodName, DeviceResponseJson, ServiceRequestJson);
Task methodReceivedTask = testDeviceCallbackHandler.WaitForMethodCallbackAsync(cts.Token);
var tasks = new List <Task>()
{
serviceSendTask, methodReceivedTask
};
while (tasks.Count > 0)
{
Task completedTask = await Task.WhenAny(tasks).ConfigureAwait(false);
completedTask.GetAwaiter().GetResult();
tasks.Remove(completedTask);
}
};
// This is a simple hack to ensure that we're still exercising these tests while we address the race conditions causing the following issues:
// [Ignore] // TODO: #571
// [Ignore] // TODO: #558
int retryCount = 3;
while (retryCount-- > 0)
{
try
{
await FaultInjection.TestErrorInjectionAsync(
DevicePrefix,
TestDeviceType.Sasl,
transport,
faultType,
reason,
delayInSec,
FaultInjection.DefaultDelayInSec,
initOperation,
testOperation,
() => { return(Task.FromResult <bool>(false)); }).ConfigureAwait(false);
}
catch (DeviceNotFoundException e)
{
s_log.WriteLine($"Retrying fault injection test ({retryCount} left): {e}");
}
catch (TaskCanceledException e)
{
s_log.WriteLine($"Retrying fault injection test ({retryCount} left): {e}");
}
}
}
private async Task SendMethodAndRespondRecoveryPoolOverAmqpAsync(
TestDeviceType type,
Client.TransportType transport,
int poolSize,
int devicesCount,
Func <DeviceClient, string, Task <Task> > setDeviceReceiveMethod,
string faultType,
string reason,
int delayInSec = FaultInjection.DefaultDelayInSec,
int durationInSec = FaultInjection.DefaultDurationInSec,
ConnectionStringAuthScope authScope = ConnectionStringAuthScope.Device)
{
var testDevicesWithCallbackHandler = new Dictionary <string, TestDeviceCallbackHandler>();
Func <DeviceClient, TestDevice, Task> initOperation = async(deviceClient, testDevice) =>
{
var testDeviceCallbackHandler = new TestDeviceCallbackHandler(deviceClient);
testDevicesWithCallbackHandler.Add(testDevice.Id, testDeviceCallbackHandler);
_log.WriteLine($"{nameof(MethodE2EPoolAmqpTests)}: Setting method callback handler for device {testDevice.Id}");
await testDeviceCallbackHandler
.SetDeviceReceiveMethodAsync(MethodName, MethodE2ETests.DeviceResponseJson, MethodE2ETests.ServiceRequestJson)
.ConfigureAwait(false);
};
Func <DeviceClient, TestDevice, Task> testOperation = async(deviceClient, testDevice) =>
{
TestDeviceCallbackHandler testDeviceCallbackHandler = testDevicesWithCallbackHandler[testDevice.Id];
using var cts = new CancellationTokenSource(FaultInjection.RecoveryTimeMilliseconds);
_log.WriteLine($"{nameof(MethodE2EPoolAmqpTests)}: Preparing to receive method for device {testDevice.Id}");
Task serviceSendTask = MethodE2ETests.ServiceSendMethodAndVerifyResponseAsync(
testDevice.Id,
MethodName,
MethodE2ETests.DeviceResponseJson,
MethodE2ETests.ServiceRequestJson);
Task methodReceivedTask = testDeviceCallbackHandler.WaitForMethodCallbackAsync(cts.Token);
await Task.WhenAll(serviceSendTask, methodReceivedTask).ConfigureAwait(false);
};
Func <IList <DeviceClient>, Task> cleanupOperation = async(deviceClients) =>
{
foreach (DeviceClient deviceClient in deviceClients)
{
deviceClient.Dispose();
}
testDevicesWithCallbackHandler.Clear();
await Task.FromResult <bool>(false).ConfigureAwait(false);
};
await FaultInjectionPoolingOverAmqp
.TestFaultInjectionPoolAmqpAsync(
MethodDevicePrefix,
transport,
poolSize,
devicesCount,
faultType,
reason,
delayInSec,
durationInSec,
initOperation,
testOperation,
cleanupOperation,
authScope)
.ConfigureAwait(false);
}
private async Task Twin_DeviceDesiredPropertyUpdateRecovery(Client.TransportType transport, string faultType, string reason, int delayInSec)
{
TestDeviceCallbackHandler testDeviceCallbackHandler = null;
RegistryManager registryManager = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);
var cts = new CancellationTokenSource(FaultInjection.RecoveryTimeMilliseconds);
var propName = Guid.NewGuid().ToString();
var props = new TwinCollection();
// Configure the callback and start accepting twin changes.
Func <DeviceClient, TestDevice, Task> initOperation = async(deviceClient, testDevice) =>
{
testDeviceCallbackHandler = new TestDeviceCallbackHandler(deviceClient);
await testDeviceCallbackHandler.SetTwinPropertyUpdateCallbackHandlerAsync(propName).ConfigureAwait(false);
};
// Change the twin from the service side and verify the device received it.
Func <DeviceClient, TestDevice, Task> testOperation = async(deviceClient, testDevice) =>
{
var propValue = Guid.NewGuid().ToString();
testDeviceCallbackHandler.ExpectedTwinPropertyValue = propValue;
s_log.WriteLine($"{nameof(Twin_DeviceDesiredPropertyUpdateRecovery)}: name={propName}, value={propValue}");
Task serviceSendTask = RegistryManagerUpdateDesiredPropertyAsync(testDevice.Id, propName, propValue);
Task twinReceivedTask = testDeviceCallbackHandler.WaitForTwinCallbackAsync(cts.Token);
var tasks = new List <Task>()
{
serviceSendTask, twinReceivedTask
};
while (tasks.Count > 0)
{
Task completedTask = await Task.WhenAny(tasks).ConfigureAwait(false);
completedTask.GetAwaiter().GetResult();
tasks.Remove(completedTask);
}
};
// This is a simple hack to ensure that we're still exercising these tests while we address the race conditions causing the following issues:
// [Ignore] // TODO: #571
// [Ignore] // TODO: #558
int retryCount = 3;
while (retryCount-- > 0)
{
try
{
await FaultInjection.TestErrorInjectionAsync(
DevicePrefix,
TestDeviceType.Sasl,
transport,
faultType,
reason,
delayInSec,
FaultInjection.DefaultDurationInSec,
initOperation,
testOperation,
() => { return(Task.FromResult <bool>(false)); }).ConfigureAwait(false);
}
catch (DeviceNotFoundException e)
{
s_log.WriteLine($"Retrying fault injection test ({retryCount} left)");
}
}
}
private async Task SendMethodAndRespondRecoveryPoolOverAmqpAsync(
TestDeviceType type,
Client.TransportType transport,
int poolSize,
int devicesCount,
Func <DeviceClient, string, MsTestLogger, Task <Task> > setDeviceReceiveMethod,
string faultType,
string reason,
TimeSpan delayInSec = default,
TimeSpan durationInSec = default,
ConnectionStringAuthScope authScope = ConnectionStringAuthScope.Device,
string proxyAddress = null)
{
var testDevicesWithCallbackHandler = new Dictionary <string, TestDeviceCallbackHandler>();
async Task InitOperationAsync(DeviceClient deviceClient, TestDevice testDevice, TestDeviceCallbackHandler _)
{
var testDeviceCallbackHandler = new TestDeviceCallbackHandler(deviceClient, testDevice, Logger);
testDevicesWithCallbackHandler.Add(testDevice.Id, testDeviceCallbackHandler);
Logger.Trace($"{nameof(MethodE2EPoolAmqpTests)}: Setting method callback handler for device {testDevice.Id}");
await testDeviceCallbackHandler
.SetDeviceReceiveMethodAsync(MethodName, MethodE2ETests.DeviceResponseJson, MethodE2ETests.ServiceRequestJson)
.ConfigureAwait(false);
}
async Task TestOperationAsync(DeviceClient deviceClient, TestDevice testDevice, TestDeviceCallbackHandler _)
{
TestDeviceCallbackHandler testDeviceCallbackHandler = testDevicesWithCallbackHandler[testDevice.Id];
using var cts = new CancellationTokenSource(FaultInjection.RecoveryTime);
Logger.Trace($"{nameof(MethodE2EPoolAmqpTests)}: Preparing to receive method for device {testDevice.Id}");
Task serviceSendTask = MethodE2ETests
.ServiceSendMethodAndVerifyResponseAsync(
testDevice.Id,
MethodName,
MethodE2ETests.DeviceResponseJson,
MethodE2ETests.ServiceRequestJson,
Logger);
Task methodReceivedTask = testDeviceCallbackHandler.WaitForMethodCallbackAsync(cts.Token);
await Task.WhenAll(serviceSendTask, methodReceivedTask).ConfigureAwait(false);
}
async Task CleanupOperationAsync(IList <DeviceClient> deviceClients)
{
foreach (DeviceClient deviceClient in deviceClients)
{
deviceClient.Dispose();
}
foreach (KeyValuePair <string, TestDeviceCallbackHandler> entry in testDevicesWithCallbackHandler)
{
TestDeviceCallbackHandler testDeviceCallbackHandler = entry.Value;
testDeviceCallbackHandler?.Dispose();
}
testDevicesWithCallbackHandler.Clear();
await Task.FromResult <bool>(false).ConfigureAwait(false);
}
await FaultInjectionPoolingOverAmqp
.TestFaultInjectionPoolAmqpAsync(
MethodDevicePrefix,
transport,
proxyAddress,
poolSize,
devicesCount,
faultType,
reason,
delayInSec == TimeSpan.Zero?FaultInjection.DefaultFaultDelay : delayInSec,
durationInSec == TimeSpan.Zero?FaultInjection.DefaultFaultDuration : durationInSec,
InitOperationAsync,
TestOperationAsync,
CleanupOperationAsync,
authScope,
Logger)
.ConfigureAwait(false);
}
