private async Task <JobProperties> CreateAndWaitForJobAsync(
            StorageAuthenticationType storageAuthenticationType,
            string devicesFileName,
            string configsFileName,
            RegistryManager registryManager,
            Uri containerUri,
            ManagedIdentity identity)
        {
            int           tryCount          = 0;
            JobProperties importJobResponse = null;

            JobProperties jobProperties = JobProperties.CreateForImportJob(
                containerUri.ToString(),
                containerUri.ToString(),
                devicesFileName,
                storageAuthenticationType,
                identity);

            jobProperties.ConfigurationsBlobName = configsFileName;
            jobProperties.IncludeConfigurations  = true;

            while (tryCount < MaxIterationWait)
            {
                try
                {
                    importJobResponse = await registryManager.ImportDevicesAsync(jobProperties).ConfigureAwait(false);

                    if (!string.IsNullOrWhiteSpace(importJobResponse.FailureReason))
                    {
                        Logger.Trace($"Job failed due to {importJobResponse.FailureReason}");
                    }
                    break;
                }
                // Concurrent jobs can be rejected, so implement a retry mechanism to handle conflicts with other tests
                catch (JobQuotaExceededException) when(++tryCount < MaxIterationWait)
                {
                    Logger.Trace($"JobQuotaExceededException... waiting.");
                    await Task.Delay(s_waitDuration).ConfigureAwait(false);

                    continue;
                }
            }

            // wait for job to complete
            for (int i = 0; i < MaxIterationWait; ++i)
            {
                await Task.Delay(1000).ConfigureAwait(false);

                importJobResponse = await registryManager.GetJobAsync(importJobResponse?.JobId).ConfigureAwait(false);

                Logger.Trace($"Job {importJobResponse.JobId} is {importJobResponse.Status} with progress {importJobResponse.Progress}%");
                if (!s_incompleteJobs.Contains(importJobResponse.Status))
                {
                    break;
                }
            }

            return(importJobResponse);
        }
        public async Task ImportDevicesAsync(string hubConnectionString,
                                             string blobContainerUri,
                                             string userDefinedManagedIdentityResourceId = null)
        {
            using RegistryManager destRegistryManager = RegistryManager.CreateFromConnectionString(hubConnectionString);

            // If StorageAuthenticationType is set to IdentityBased and userAssignedIdentity property is
            // not null, the jobs will use user defined managed identity. If the IoT hub is not
            // configured with the user defined managed identity specified in userAssignedIdentity,
            // the job will fail.
            // If StorageAuthenticationType is set to IdentityBased and userAssignedIdentity property is
            // null, the jobs will use system defined identity by default. If the IoT hub is configured with the
            // system defined managed identity, the job will succeed but will not use the user defined managed identity.
            // If the IoT hub is not configured with system defined managed identity, the job will fail.
            // If StorageAuthenticationType is set to IdentityBased and neither user defined nor system defined
            // managed identities are configured on the hub, the job will fail.
            JobProperties jobProperties = JobProperties.CreateForImportJob(
                inputBlobContainerUri: blobContainerUri,
                outputBlobContainerUri: blobContainerUri,
                storageAuthenticationType: StorageAuthenticationType.IdentityBased,
                identity: new ManagedIdentity
            {
                userAssignedIdentity = userDefinedManagedIdentityResourceId
            });

            JobProperties jobResult = await destRegistryManager
                                      .ImportDevicesAsync(jobProperties);

            // Poll every 5 seconds to see if the job has finished executing.
            while (true)
            {
                jobResult = await destRegistryManager.GetJobAsync(jobResult.JobId);

                if (jobResult.Status == JobStatus.Completed)
                {
                    break;
                }
                else if (jobResult.Status == JobStatus.Failed)
                {
                    throw new Exception("Import job failed.");
                }
                else if (jobResult.Status == JobStatus.Cancelled)
                {
                    throw new Exception("Import job was canceled.");
                }
                else
                {
                    await Task.Delay(TimeSpan.FromSeconds(5));
                }
            }
        }
示例#3
0
        public async Task RegistryManager_ImportDevices(StorageAuthenticationType storageAuthenticationType)
        {
            // arrange

            StorageContainer storageContainer = null;
            string           deviceId         = $"{nameof(RegistryManager_ImportDevices)}-{StorageContainer.GetRandomSuffix(4)}";
            var registryManager = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);

            Logger.Trace($"Using deviceId {deviceId}");

            try
            {
                string containerName = StorageContainer.BuildContainerName(nameof(RegistryManager_ImportDevices));
                storageContainer = await StorageContainer
                                   .GetInstanceAsync(containerName)
                                   .ConfigureAwait(false);

                Logger.Trace($"Using container {storageContainer.Uri}");

                Uri containerUri = storageAuthenticationType == StorageAuthenticationType.KeyBased
                    ? storageContainer.SasUri
                    : storageContainer.Uri;

                Stream devicesFile = ImportExportDevicesHelpers.BuildDevicesStream(
                    new List <ExportImportDevice>
                {
                    new ExportImportDevice(
                        new Device(deviceId)
                    {
                        Authentication = new AuthenticationMechanism {
                            Type = AuthenticationType.Sas
                        }
                    },
                        ImportMode.Create),
                });
                await UploadFileAndConfirmAsync(storageContainer, devicesFile).ConfigureAwait(false);

                // act

                JobProperties importJobResponse = null;
                int           tryCount          = 0;
                while (true)
                {
                    try
                    {
                        importJobResponse = await registryManager
                                            .ImportDevicesAsync(
                            JobProperties.CreateForImportJob(
                                containerUri.ToString(),
                                containerUri.ToString(),
                                null,
                                storageAuthenticationType))
                                            .ConfigureAwait(false);

                        break;
                    }
                    // Concurrent jobs can be rejected, so implement a retry mechanism to handle conflicts with other tests
                    catch (JobQuotaExceededException) when(++tryCount < MaxIterationWait)
                    {
                        Logger.Trace($"JobQuotaExceededException... waiting.");
                        await Task.Delay(s_waitDuration).ConfigureAwait(false);

                        continue;
                    }
                }

                // wait for job to complete
                for (int i = 0; i < MaxIterationWait; ++i)
                {
                    await Task.Delay(1000).ConfigureAwait(false);

                    importJobResponse = await registryManager.GetJobAsync(importJobResponse.JobId).ConfigureAwait(false);

                    Logger.Trace($"Job {importJobResponse.JobId} is {importJobResponse.Status} with progress {importJobResponse.Progress}%");
                    if (!s_incompleteJobs.Contains(importJobResponse.Status))
                    {
                        break;
                    }
                }

                // assert

                importJobResponse.Status.Should().Be(JobStatus.Completed, "Otherwise import failed");
                importJobResponse.FailureReason.Should().BeNullOrEmpty("Otherwise import failed");

                // should not throw due to 404, but device may not immediately appear in registry
                Device device = null;
                for (int i = 0; i < MaxIterationWait; ++i)
                {
                    await Task.Delay(s_waitDuration).ConfigureAwait(false);

                    try
                    {
                        device = await registryManager.GetDeviceAsync(deviceId).ConfigureAwait(false);

                        break;
                    }
                    catch (Exception ex)
                    {
                        Logger.Trace($"Could not find device on iteration {i} due to [{ex.Message}]");
                    }
                }
                if (device == null)
                {
                    Assert.Fail($"Device {deviceId} not found in registry manager");
                }
            }
            finally
            {
                try
                {
                    storageContainer?.Dispose();

                    await registryManager.RemoveDeviceAsync(deviceId).ConfigureAwait(false);
                }
                catch { }
            }
        }
示例#4
0
        private async Task CleanupDevices()
        {
            Console.WriteLine($"Using storage container {_blobContainerClient.Name}" +
                              $" for exporting devices identities to and importing device identities from.");

            // Retrieve the SAS Uri that will be used to grant access to the storage containers.
            string storageAccountSasUri = GetStorageAccountSasUriForCleanupJob(_blobContainerClient).ToString();

            // Step 1: Export all device identities.
            JobProperties exportAllDevicesProperties = JobProperties
                                                       .CreateForExportJob(
                outputBlobContainerUri: storageAccountSasUri,
                excludeKeysInExport: true,
                storageAuthenticationType: StorageAuthenticationType.KeyBased);

            JobProperties exportAllDevicesJob = null;

            int tryCount = 0;

            while (true)
            {
                try
                {
                    exportAllDevicesJob = await _registryManager.ExportDevicesAsync(exportAllDevicesProperties);

                    break;
                }
                // Wait for pending jobs to finish.
                catch (JobQuotaExceededException) when(++tryCount < MaxIterationWait)
                {
                    Console.WriteLine($"JobQuotaExceededException... waiting.");
                    await Task.Delay(WaitDuration);
                }
            }

            if (exportAllDevicesJob == null)
            {
                throw new Exception("Export devices job failed.");
            }

            // Wait until the export job is finished.
            while (true)
            {
                exportAllDevicesJob = await _registryManager.GetJobAsync(exportAllDevicesJob.JobId);

                if (s_completedJobs.Contains(exportAllDevicesJob.Status))
                {
                    // Job has finished executing.
                    break;
                }

                Console.WriteLine($"Job {exportAllDevicesJob.JobId} is {exportAllDevicesJob.Status} with progress {exportAllDevicesJob.Progress}%");
                await Task.Delay(s_waitDuration);
            }
            Console.WriteLine($"Job {exportAllDevicesJob.JobId} is {exportAllDevicesJob.Status}.");

            if (exportAllDevicesJob.Status != JobStatus.Completed)
            {
                throw new Exception("Exporting devices failed, exiting.");
            }

            // Step 2: Download the exported devices list from the blob create in Step 1.
            BlobClient       blobClient = _blobContainerClient.GetBlobClient(ImportExportDevicesFileName);
            BlobDownloadInfo download   = await blobClient.DownloadAsync();

            IEnumerable <ExportImportDevice> exportedDevices = ImportExportDevicesHelpers.BuildExportImportDeviceFromStream(download.Content);

            // Step 3: Collect the devices that need to be deleted and update their ImportMode to be Delete.
            // Thie step will create an ExportImportDevice identity for each device/ module identity registered on hub.
            // If you hub instance has IoT Hub module or Edge module instances registered, then they will be counted as separate entities
            // from the corresponding IoT Hub device/ Edge device that they are associated with.
            // As a result, the count of ExportImportDevice identities to be deleted might be greater than the
            // count of IoT hub devices retrieved in PrintDeviceCountAsync().
            var devicesToBeDeleted = new List <ExportImportDevice>();

            foreach (var device in exportedDevices)
            {
                string deviceId = device.Id;
                foreach (string prefix in _deleteDevicesWithPrefix)
                {
                    if (deviceId.StartsWith(prefix, StringComparison.OrdinalIgnoreCase))
                    {
                        devicesToBeDeleted.Add(device);
                    }
                }
            }
            devicesToBeDeleted
            .ForEach(device => device.ImportMode = ImportMode.Delete);
            Console.WriteLine($"Retrieved {devicesToBeDeleted.Count} devices for deletion.");

            if (devicesToBeDeleted.Count > 0)
            {
                // Step 3a: Write the new import data back to the blob.
                using Stream devicesFile = ImportExportDevicesHelpers.BuildDevicesStream(devicesToBeDeleted);
                await blobClient.UploadAsync(devicesFile, overwrite : true);

                // Step 3b: Call import using the same blob to delete all devices.
                JobProperties importDevicesToBeDeletedProperties = JobProperties
                                                                   .CreateForImportJob(
                    inputBlobContainerUri: storageAccountSasUri,
                    outputBlobContainerUri: storageAccountSasUri,
                    storageAuthenticationType: StorageAuthenticationType.KeyBased);

                JobProperties importDevicesToBeDeletedJob = null;

                tryCount = 0;
                while (true)
                {
                    try
                    {
                        importDevicesToBeDeletedJob = await _registryManager.ImportDevicesAsync(importDevicesToBeDeletedProperties);

                        break;
                    }
                    // Wait for pending jobs to finish.
                    catch (JobQuotaExceededException) when(++tryCount < MaxIterationWait)
                    {
                        Console.WriteLine($"JobQuotaExceededException... waiting.");
                        await Task.Delay(WaitDuration);
                    }
                }

                if (importDevicesToBeDeletedJob == null)
                {
                    throw new Exception("Import devices job failed.");
                }

                // Wait until job is finished.
                while (true)
                {
                    importDevicesToBeDeletedJob = await _registryManager.GetJobAsync(importDevicesToBeDeletedJob.JobId);

                    if (s_completedJobs.Contains(importDevicesToBeDeletedJob.Status))
                    {
                        // Job has finished executing.
                        break;
                    }

                    Console.WriteLine($"Job {importDevicesToBeDeletedJob.JobId} is {importDevicesToBeDeletedJob.Status} with progress {importDevicesToBeDeletedJob.Progress}%");
                    await Task.Delay(s_waitDuration);
                }
                Console.WriteLine($"Job {importDevicesToBeDeletedJob.JobId} is {importDevicesToBeDeletedJob.Status}.");
            }

            // Step 4: Delete the storage container created.
            await _blobContainerClient.DeleteAsync();

            Console.WriteLine($"Storage container {_blobContainerClient.Name} deleted.");
        }
        private async Task CleanupDevices(int deviceCount)
        {
            Console.WriteLine($"Using storage container {_blobContainerClient.Name} for importing device delete requests.");

            // Step 1: Collect the devices that need to be deleted.
            IReadOnlyList <ExportImportDevice> devicesToBeDeleted = await GetDeviceIdsToDeleteAsync(deviceCount);

            Console.WriteLine($"Discovered {devicesToBeDeleted.Count} devices for deletion.");

            string currentJobId = null;

            if (devicesToBeDeleted.Any())
            {
                try
                {
                    // Step 2: Write the new import data back to the blob.
                    using Stream devicesFile = ImportExportDevicesHelpers.BuildDevicesStream(devicesToBeDeleted);

                    // Retrieve the SAS Uri that will be used to grant access to the storage containers.
                    BlobClient blobClient   = _blobContainerClient.GetBlobClient(ImportExportDevicesFileName);
                    var        uploadResult = await blobClient.UploadAsync(devicesFile, overwrite : true);

                    string storageAccountSasUri = GetStorageAccountSasUriForCleanupJob(_blobContainerClient).ToString();

                    // Step 3: Call import using the same blob to delete all devices.
                    JobProperties importDevicesToBeDeletedProperties = JobProperties
                                                                       .CreateForImportJob(
                        inputBlobContainerUri: storageAccountSasUri,
                        outputBlobContainerUri: storageAccountSasUri,
                        inputBlobName: ImportExportDevicesFileName,
                        storageAuthenticationType: StorageAuthenticationType.KeyBased);

                    JobProperties importDevicesToBeDeletedJob = null;

                    Stopwatch jobTimer = Stopwatch.StartNew();
                    do
                    {
                        try
                        {
                            importDevicesToBeDeletedJob = await _registryManager.ImportDevicesAsync(importDevicesToBeDeletedProperties);

                            currentJobId = importDevicesToBeDeletedJob.JobId;
                            break;
                        }
                        // Wait for pending jobs to finish.
                        catch (JobQuotaExceededException)
                        {
                            Console.WriteLine($"JobQuotaExceededException... waiting.");
                            await Task.Delay(s_waitDuration);
                        }
                    } while (jobTimer.Elapsed < s_maxJobDuration);

                    // Wait until job is finished.
                    jobTimer.Restart();
                    while (importDevicesToBeDeletedJob != null &&
                           jobTimer.Elapsed < s_maxJobDuration)
                    {
                        importDevicesToBeDeletedJob = await _registryManager.GetJobAsync(importDevicesToBeDeletedJob.JobId);

                        if (s_completedJobs.Contains(importDevicesToBeDeletedJob.Status))
                        {
                            // Job has finished executing.
                            Console.WriteLine($"Job {importDevicesToBeDeletedJob.JobId} is {importDevicesToBeDeletedJob.Status}.");
                            currentJobId = null;
                            break;
                        }

                        Console.WriteLine($"Job {importDevicesToBeDeletedJob.JobId} is {importDevicesToBeDeletedJob.Status} after {jobTimer.Elapsed}.");
                        await Task.Delay(s_waitDuration);
                    }

                    if (importDevicesToBeDeletedJob?.Status != JobStatus.Completed)
                    {
                        throw new Exception("Importing devices job failed; exiting.");
                    }
                }
                finally
                {
                    if (!String.IsNullOrWhiteSpace(currentJobId))
                    {
                        Console.WriteLine($"Cancelling job {currentJobId}");
                        await _registryManager.CancelJobAsync(currentJobId);
                    }
                }
            }

            // Step 4: Delete the storage container created.
            await _blobContainerClient.DeleteAsync();

            Console.WriteLine($"Storage container {_blobContainerClient.Name} deleted.");
        }