private async Task <JobProperties> CreateAndWaitForJobAsync(
StorageAuthenticationType storageAuthenticationType,
string devicesFileName,
string configsFileName,
RegistryManager registryManager,
Uri containerUri,
ManagedIdentity identity)
{
int tryCount = 0;
JobProperties importJobResponse = null;
JobProperties jobProperties = JobProperties.CreateForImportJob(
containerUri.ToString(),
containerUri.ToString(),
devicesFileName,
storageAuthenticationType,
identity);
jobProperties.ConfigurationsBlobName = configsFileName;
jobProperties.IncludeConfigurations = true;
while (tryCount < MaxIterationWait)
{
try
{
importJobResponse = await registryManager.ImportDevicesAsync(jobProperties).ConfigureAwait(false);
if (!string.IsNullOrWhiteSpace(importJobResponse.FailureReason))
{
Logger.Trace($"Job failed due to {importJobResponse.FailureReason}");
}
break;
}
// Concurrent jobs can be rejected, so implement a retry mechanism to handle conflicts with other tests
catch (JobQuotaExceededException) when(++tryCount < MaxIterationWait)
{
Logger.Trace($"JobQuotaExceededException... waiting.");
await Task.Delay(s_waitDuration).ConfigureAwait(false);
continue;
}
}
// wait for job to complete
for (int i = 0; i < MaxIterationWait; ++i)
{
await Task.Delay(1000).ConfigureAwait(false);
importJobResponse = await registryManager.GetJobAsync(importJobResponse?.JobId).ConfigureAwait(false);
Logger.Trace($"Job {importJobResponse.JobId} is {importJobResponse.Status} with progress {importJobResponse.Progress}%");
if (!s_incompleteJobs.Contains(importJobResponse.Status))
{
break;
}
}
return(importJobResponse);
}
public async Task ImportDevicesAsync(string hubConnectionString,
string blobContainerUri,
string userDefinedManagedIdentityResourceId = null)
{
using RegistryManager destRegistryManager = RegistryManager.CreateFromConnectionString(hubConnectionString);
// If StorageAuthenticationType is set to IdentityBased and userAssignedIdentity property is
// not null, the jobs will use user defined managed identity. If the IoT hub is not
// configured with the user defined managed identity specified in userAssignedIdentity,
// the job will fail.
// If StorageAuthenticationType is set to IdentityBased and userAssignedIdentity property is
// null, the jobs will use system defined identity by default. If the IoT hub is configured with the
// system defined managed identity, the job will succeed but will not use the user defined managed identity.
// If the IoT hub is not configured with system defined managed identity, the job will fail.
// If StorageAuthenticationType is set to IdentityBased and neither user defined nor system defined
// managed identities are configured on the hub, the job will fail.
JobProperties jobProperties = JobProperties.CreateForImportJob(
inputBlobContainerUri: blobContainerUri,
outputBlobContainerUri: blobContainerUri,
storageAuthenticationType: StorageAuthenticationType.IdentityBased,
identity: new ManagedIdentity
{
userAssignedIdentity = userDefinedManagedIdentityResourceId
});
JobProperties jobResult = await destRegistryManager
.ImportDevicesAsync(jobProperties);
// Poll every 5 seconds to see if the job has finished executing.
while (true)
{
jobResult = await destRegistryManager.GetJobAsync(jobResult.JobId);
if (jobResult.Status == JobStatus.Completed)
{
break;
}
else if (jobResult.Status == JobStatus.Failed)
{
throw new Exception("Import job failed.");
}
else if (jobResult.Status == JobStatus.Cancelled)
{
throw new Exception("Import job was canceled.");
}
else
{
await Task.Delay(TimeSpan.FromSeconds(5));
}
}
}
public async Task RegistryManager_ImportDevices(StorageAuthenticationType storageAuthenticationType)
{
// arrange
StorageContainer storageContainer = null;
string deviceId = $"{nameof(RegistryManager_ImportDevices)}-{StorageContainer.GetRandomSuffix(4)}";
var registryManager = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);
Logger.Trace($"Using deviceId {deviceId}");
try
{
string containerName = StorageContainer.BuildContainerName(nameof(RegistryManager_ImportDevices));
storageContainer = await StorageContainer
.GetInstanceAsync(containerName)
.ConfigureAwait(false);
Logger.Trace($"Using container {storageContainer.Uri}");
Uri containerUri = storageAuthenticationType == StorageAuthenticationType.KeyBased
? storageContainer.SasUri
: storageContainer.Uri;
Stream devicesFile = ImportExportDevicesHelpers.BuildDevicesStream(
new List <ExportImportDevice>
{
new ExportImportDevice(
new Device(deviceId)
{
Authentication = new AuthenticationMechanism {
Type = AuthenticationType.Sas
}
},
ImportMode.Create),
});
await UploadFileAndConfirmAsync(storageContainer, devicesFile).ConfigureAwait(false);
// act
JobProperties importJobResponse = null;
int tryCount = 0;
while (true)
{
try
{
importJobResponse = await registryManager
.ImportDevicesAsync(
JobProperties.CreateForImportJob(
containerUri.ToString(),
containerUri.ToString(),
null,
storageAuthenticationType))
.ConfigureAwait(false);
break;
}
// Concurrent jobs can be rejected, so implement a retry mechanism to handle conflicts with other tests
catch (JobQuotaExceededException) when(++tryCount < MaxIterationWait)
{
Logger.Trace($"JobQuotaExceededException... waiting.");
await Task.Delay(s_waitDuration).ConfigureAwait(false);
continue;
}
}
// wait for job to complete
for (int i = 0; i < MaxIterationWait; ++i)
{
await Task.Delay(1000).ConfigureAwait(false);
importJobResponse = await registryManager.GetJobAsync(importJobResponse.JobId).ConfigureAwait(false);
Logger.Trace($"Job {importJobResponse.JobId} is {importJobResponse.Status} with progress {importJobResponse.Progress}%");
if (!s_incompleteJobs.Contains(importJobResponse.Status))
{
break;
}
}
// assert
importJobResponse.Status.Should().Be(JobStatus.Completed, "Otherwise import failed");
importJobResponse.FailureReason.Should().BeNullOrEmpty("Otherwise import failed");
// should not throw due to 404, but device may not immediately appear in registry
Device device = null;
for (int i = 0; i < MaxIterationWait; ++i)
{
await Task.Delay(s_waitDuration).ConfigureAwait(false);
try
{
device = await registryManager.GetDeviceAsync(deviceId).ConfigureAwait(false);
break;
}
catch (Exception ex)
{
Logger.Trace($"Could not find device on iteration {i} due to [{ex.Message}]");
}
}
if (device == null)
{
Assert.Fail($"Device {deviceId} not found in registry manager");
}
}
finally
{
try
{
storageContainer?.Dispose();
await registryManager.RemoveDeviceAsync(deviceId).ConfigureAwait(false);
}
catch { }
}
}
private async Task CleanupDevices()
{
Console.WriteLine($"Using storage container {_blobContainerClient.Name}" +
$" for exporting devices identities to and importing device identities from.");
// Retrieve the SAS Uri that will be used to grant access to the storage containers.
string storageAccountSasUri = GetStorageAccountSasUriForCleanupJob(_blobContainerClient).ToString();
// Step 1: Export all device identities.
JobProperties exportAllDevicesProperties = JobProperties
.CreateForExportJob(
outputBlobContainerUri: storageAccountSasUri,
excludeKeysInExport: true,
storageAuthenticationType: StorageAuthenticationType.KeyBased);
JobProperties exportAllDevicesJob = null;
int tryCount = 0;
while (true)
{
try
{
exportAllDevicesJob = await _registryManager.ExportDevicesAsync(exportAllDevicesProperties);
break;
}
// Wait for pending jobs to finish.
catch (JobQuotaExceededException) when(++tryCount < MaxIterationWait)
{
Console.WriteLine($"JobQuotaExceededException... waiting.");
await Task.Delay(WaitDuration);
}
}
if (exportAllDevicesJob == null)
{
throw new Exception("Export devices job failed.");
}
// Wait until the export job is finished.
while (true)
{
exportAllDevicesJob = await _registryManager.GetJobAsync(exportAllDevicesJob.JobId);
if (s_completedJobs.Contains(exportAllDevicesJob.Status))
{
// Job has finished executing.
break;
}
Console.WriteLine($"Job {exportAllDevicesJob.JobId} is {exportAllDevicesJob.Status} with progress {exportAllDevicesJob.Progress}%");
await Task.Delay(s_waitDuration);
}
Console.WriteLine($"Job {exportAllDevicesJob.JobId} is {exportAllDevicesJob.Status}.");
if (exportAllDevicesJob.Status != JobStatus.Completed)
{
throw new Exception("Exporting devices failed, exiting.");
}
// Step 2: Download the exported devices list from the blob create in Step 1.
BlobClient blobClient = _blobContainerClient.GetBlobClient(ImportExportDevicesFileName);
BlobDownloadInfo download = await blobClient.DownloadAsync();
IEnumerable <ExportImportDevice> exportedDevices = ImportExportDevicesHelpers.BuildExportImportDeviceFromStream(download.Content);
// Step 3: Collect the devices that need to be deleted and update their ImportMode to be Delete.
// Thie step will create an ExportImportDevice identity for each device/ module identity registered on hub.
// If you hub instance has IoT Hub module or Edge module instances registered, then they will be counted as separate entities
// from the corresponding IoT Hub device/ Edge device that they are associated with.
// As a result, the count of ExportImportDevice identities to be deleted might be greater than the
// count of IoT hub devices retrieved in PrintDeviceCountAsync().
var devicesToBeDeleted = new List <ExportImportDevice>();
foreach (var device in exportedDevices)
{
string deviceId = device.Id;
foreach (string prefix in _deleteDevicesWithPrefix)
{
if (deviceId.StartsWith(prefix, StringComparison.OrdinalIgnoreCase))
{
devicesToBeDeleted.Add(device);
}
}
}
devicesToBeDeleted
.ForEach(device => device.ImportMode = ImportMode.Delete);
Console.WriteLine($"Retrieved {devicesToBeDeleted.Count} devices for deletion.");
if (devicesToBeDeleted.Count > 0)
{
// Step 3a: Write the new import data back to the blob.
using Stream devicesFile = ImportExportDevicesHelpers.BuildDevicesStream(devicesToBeDeleted);
await blobClient.UploadAsync(devicesFile, overwrite : true);
// Step 3b: Call import using the same blob to delete all devices.
JobProperties importDevicesToBeDeletedProperties = JobProperties
.CreateForImportJob(
inputBlobContainerUri: storageAccountSasUri,
outputBlobContainerUri: storageAccountSasUri,
storageAuthenticationType: StorageAuthenticationType.KeyBased);
JobProperties importDevicesToBeDeletedJob = null;
tryCount = 0;
while (true)
{
try
{
importDevicesToBeDeletedJob = await _registryManager.ImportDevicesAsync(importDevicesToBeDeletedProperties);
break;
}
// Wait for pending jobs to finish.
catch (JobQuotaExceededException) when(++tryCount < MaxIterationWait)
{
Console.WriteLine($"JobQuotaExceededException... waiting.");
await Task.Delay(WaitDuration);
}
}
if (importDevicesToBeDeletedJob == null)
{
throw new Exception("Import devices job failed.");
}
// Wait until job is finished.
while (true)
{
importDevicesToBeDeletedJob = await _registryManager.GetJobAsync(importDevicesToBeDeletedJob.JobId);
if (s_completedJobs.Contains(importDevicesToBeDeletedJob.Status))
{
// Job has finished executing.
break;
}
Console.WriteLine($"Job {importDevicesToBeDeletedJob.JobId} is {importDevicesToBeDeletedJob.Status} with progress {importDevicesToBeDeletedJob.Progress}%");
await Task.Delay(s_waitDuration);
}
Console.WriteLine($"Job {importDevicesToBeDeletedJob.JobId} is {importDevicesToBeDeletedJob.Status}.");
}
// Step 4: Delete the storage container created.
await _blobContainerClient.DeleteAsync();
Console.WriteLine($"Storage container {_blobContainerClient.Name} deleted.");
}
private async Task CleanupDevices(int deviceCount)
{
Console.WriteLine($"Using storage container {_blobContainerClient.Name} for importing device delete requests.");
// Step 1: Collect the devices that need to be deleted.
IReadOnlyList <ExportImportDevice> devicesToBeDeleted = await GetDeviceIdsToDeleteAsync(deviceCount);
Console.WriteLine($"Discovered {devicesToBeDeleted.Count} devices for deletion.");
string currentJobId = null;
if (devicesToBeDeleted.Any())
{
try
{
// Step 2: Write the new import data back to the blob.
using Stream devicesFile = ImportExportDevicesHelpers.BuildDevicesStream(devicesToBeDeleted);
// Retrieve the SAS Uri that will be used to grant access to the storage containers.
BlobClient blobClient = _blobContainerClient.GetBlobClient(ImportExportDevicesFileName);
var uploadResult = await blobClient.UploadAsync(devicesFile, overwrite : true);
string storageAccountSasUri = GetStorageAccountSasUriForCleanupJob(_blobContainerClient).ToString();
// Step 3: Call import using the same blob to delete all devices.
JobProperties importDevicesToBeDeletedProperties = JobProperties
.CreateForImportJob(
inputBlobContainerUri: storageAccountSasUri,
outputBlobContainerUri: storageAccountSasUri,
inputBlobName: ImportExportDevicesFileName,
storageAuthenticationType: StorageAuthenticationType.KeyBased);
JobProperties importDevicesToBeDeletedJob = null;
Stopwatch jobTimer = Stopwatch.StartNew();
do
{
try
{
importDevicesToBeDeletedJob = await _registryManager.ImportDevicesAsync(importDevicesToBeDeletedProperties);
currentJobId = importDevicesToBeDeletedJob.JobId;
break;
}
// Wait for pending jobs to finish.
catch (JobQuotaExceededException)
{
Console.WriteLine($"JobQuotaExceededException... waiting.");
await Task.Delay(s_waitDuration);
}
} while (jobTimer.Elapsed < s_maxJobDuration);
// Wait until job is finished.
jobTimer.Restart();
while (importDevicesToBeDeletedJob != null &&
jobTimer.Elapsed < s_maxJobDuration)
{
importDevicesToBeDeletedJob = await _registryManager.GetJobAsync(importDevicesToBeDeletedJob.JobId);
if (s_completedJobs.Contains(importDevicesToBeDeletedJob.Status))
{
// Job has finished executing.
Console.WriteLine($"Job {importDevicesToBeDeletedJob.JobId} is {importDevicesToBeDeletedJob.Status}.");
currentJobId = null;
break;
}
Console.WriteLine($"Job {importDevicesToBeDeletedJob.JobId} is {importDevicesToBeDeletedJob.Status} after {jobTimer.Elapsed}.");
await Task.Delay(s_waitDuration);
}
if (importDevicesToBeDeletedJob?.Status != JobStatus.Completed)
{
throw new Exception("Importing devices job failed; exiting.");
}
}
finally
{
if (!String.IsNullOrWhiteSpace(currentJobId))
{
Console.WriteLine($"Cancelling job {currentJobId}");
await _registryManager.CancelJobAsync(currentJobId);
}
}
}
// Step 4: Delete the storage container created.
await _blobContainerClient.DeleteAsync();
Console.WriteLine($"Storage container {_blobContainerClient.Name} deleted.");
}
