예제 #1
0
        /// Get the list of devices registered to the IoT Hub
        ///   and export it to a blob as deserialized objects.
        private async Task ExportDevices(string containerURI, string hubConnectionString)
        {
            try
            {
                Console.WriteLine("Creating and running registry manager job to retrieve the devices from the hub.");
                // Create an instance of the registry manager class.
                using RegistryManager registryManager = RegistryManager.CreateFromConnectionString(hubConnectionString);

                // Call an export job on the IoT Hub to retrieve all devices.
                // This writes them to devices.txt in the container.
                // The second parameter indicates whether to export the keys or not.
                JobProperties exportJob = await registryManager.ExportDevicesAsync(containerURI, false).ConfigureAwait(false);

                // Poll every 5 seconds to see if the job has finished executing.
                while (true)
                {
                    exportJob = await registryManager.GetJobAsync(exportJob.JobId).ConfigureAwait(false);

                    if (exportJob.Status == JobStatus.Completed ||
                        exportJob.Status == JobStatus.Failed ||
                        exportJob.Status == JobStatus.Cancelled)
                    {
                        // Job has finished executing
                        break;
                    }

                    await Task.Delay(TimeSpan.FromSeconds(5)).ConfigureAwait(false);
                }
            }
            catch (Exception ex)
            {
                Debug.Print("Error exporting devices to blob storage. Exception message = {0}", ex.Message);
            }
        }
예제 #2
0
        /// <summary>
        /// Requests the device list export.
        /// </summary>
        /// <param name="registryManager">The registry manager.</param>
        /// <param name="blobStorageUri">The BLOB storage URI including SaS container token.
        /// See Shared Access Signiture </param>
        /// <param name="ct">The ct.</param>
        /// <returns></returns>
        private static async Task <JobProperties> RequestDeviceListExport(RegistryManager registryManager, string blobStorageUri, CancellationToken ct)
        {
            var exportTask = await registryManager.ExportDevicesAsync(blobStorageUri, "foo", false, ct);

            Console.WriteLine("Job Id: {0} current status: {1}", exportTask.JobId, exportTask.Status);
            return(exportTask);
        }
예제 #3
0
        /// Get the list of devices registered to the IoT Hub
        ///   and export it to a blob as deserialized objects.
        public static async Task ExportDevices(string containerURI, string hubConnectionString)
        {
            // Create an instance of the registry manager class.
            RegistryManager registryManager =
                RegistryManager.CreateFromConnectionString(hubConnectionString);

            // Call an export job on the IoT Hub to retrieve all devices.
            // This writes them to devices.txt in the container.
            // The second parameter indicates whether to export the keys or not.
            JobProperties exportJob = await
                                      registryManager.ExportDevicesAsync(containerURI, false);

            // Poll every 5 seconds to see if the job has finished executing.
            while (true)
            {
                exportJob = await registryManager.GetJobAsync(exportJob.JobId);

                if (exportJob.Status == JobStatus.Completed ||
                    exportJob.Status == JobStatus.Failed ||
                    exportJob.Status == JobStatus.Cancelled)
                {
                    // Job has finished executing
                    break;
                }

                await Task.Delay(TimeSpan.FromSeconds(5));
            }

            // Note: could add twin data here if you want to export it.
        }
예제 #4
0
        public async Task <bool> ExportAllDevices(string connString, string containerSasUri)
        {
            bool bret = false;
            // Call an export job on the IoT Hub to retrieve all devices
            RegistryManager registryManager = RegistryManager.CreateFromConnectionString(connString);
            JobProperties   exportJob       = await registryManager.ExportDevicesAsync(containerSasUri, false);

            // Wait until job is finished
            while (true)
            {
                exportJob = await registryManager.GetJobAsync(exportJob.JobId);

                if (exportJob.Status == JobStatus.Completed ||
                    exportJob.Status == JobStatus.Failed ||
                    exportJob.Status == JobStatus.Cancelled)
                {
                    if (exportJob.Status == JobStatus.Completed)
                    {
                        bret = true;
                    }
                    break;
                }

                await Task.Delay(TimeSpan.FromSeconds(5));
            }
            return(bret);
        }
예제 #5
0
        static async System.Threading.Tasks.Task Main(string[] args)
        {
            var configuration = new ConfigurationBuilder()
                                .AddJsonFile("appsettings.json")
                                .AddEnvironmentVariables()
                                .Build();

            Console.WriteLine(configuration.GetConnectionString("DestinationIotHubConnectionString"));
            Console.WriteLine(configuration.GetConnectionString("BlobSasToken"));

            RegistryManager sourceRegistryManager      = RegistryManager.CreateFromConnectionString(configuration.GetConnectionString("SourceIotHubConnectionString"));
            RegistryManager destinationRegistryManager = RegistryManager.CreateFromConnectionString(configuration.GetConnectionString("DestinationIotHubConnectionString"));

            string containerSasUri = configuration.GetConnectionString("BlobSasToken");

            Console.WriteLine($"Start export of IoT Hub devices to blob");
            // Call an export job on the IoT Hub to retrieve all devices

            JobProperties exportJob =
                await sourceRegistryManager.ExportDevicesAsync(containerSasUri, false);

            var exportJobResult = await WaitForJobCompletion(sourceRegistryManager, exportJob);

            Console.WriteLine($"Start import of IoT Hub devices to blob");

            JobProperties importJob =
                await destinationRegistryManager.ImportDevicesAsync(containerSasUri, containerSasUri);

            var importJobResult = await WaitForJobCompletion(destinationRegistryManager, importJob);
        }
        public async Task ExportDevicesAsync(string hubConnectionString,
                                             string blobContainerUri,
                                             string userDefinedManagedIdentityResourceId = null)
        {
            using RegistryManager srcRegistryManager = RegistryManager.CreateFromConnectionString(hubConnectionString);

            // If StorageAuthenticationType is set to IdentityBased and userAssignedIdentity property is
            // not null, the jobs will use user defined managed identity. If the IoT hub is not
            // configured with the user defined managed identity specified in userAssignedIdentity,
            // the job will fail.
            // If StorageAuthenticationType is set to IdentityBased and userAssignedIdentity property is
            // null, the jobs will use system defined identity by default. If the IoT hub is configured with the
            // system defined managed identity, the job will succeed but will not use the user defined managed identity.
            // If the IoT hub is not configured with system defined managed identity, the job will fail.
            // If StorageAuthenticationType is set to IdentityBased and neither user defined nor system defined
            // managed identities are configured on the hub, the job will fail.
            JobProperties jobProperties = JobProperties.CreateForExportJob(
                outputBlobContainerUri: blobContainerUri,
                excludeKeysInExport: false,
                storageAuthenticationType: StorageAuthenticationType.IdentityBased,
                identity: new ManagedIdentity
            {
                userAssignedIdentity = userDefinedManagedIdentityResourceId
            });

            JobProperties jobResult = await srcRegistryManager
                                      .ExportDevicesAsync(jobProperties);

            // Poll every 5 seconds to see if the job has finished executing.
            while (true)
            {
                jobResult = await srcRegistryManager.GetJobAsync(jobResult.JobId);

                if (jobResult.Status == JobStatus.Completed)
                {
                    break;
                }
                else if (jobResult.Status == JobStatus.Failed)
                {
                    throw new Exception("Export job failed.");
                }
                else if (jobResult.Status == JobStatus.Cancelled)
                {
                    throw new Exception("Export job was canceled.");
                }
                else
                {
                    await Task.Delay(TimeSpan.FromSeconds(5));
                }
            }
        }
        public static async void RunAsync([TimerTrigger("0 */2 * * * *")] TimerInfo myTimer, ILogger log, ExecutionContext context)
        {
            log.LogInformation($"C# Timer trigger function executed at: {DateTime.Now}");

            var config = new ConfigurationBuilder()
                         .SetBasePath(context.FunctionAppDirectory)
                         .AddJsonFile("local.settings.json", optional: true, reloadOnChange: true)
                         .AddEnvironmentVariables()
                         .Build();

            var BlobConStr    = config["blobConStr"];
            var BlobContainer = config["blobContainer"];
            var BlobURL       = config["blobURL"];
            var IotHubConStr  = config["iotHubConStr"];

            CloudStorageAccount storageAccount = CloudStorageAccount.Parse(BlobConStr);
            CloudBlobClient     blobClient     = storageAccount.CreateCloudBlobClient();
            CloudBlobContainer  container      = blobClient.GetContainerReference(BlobContainer);

            string storedPolicyName = null;

            string sasContainerToken;

            if (storedPolicyName == null)
            {
                SharedAccessBlobPolicy adHocPolicy = new SharedAccessBlobPolicy()
                {
                    SharedAccessExpiryTime = DateTime.UtcNow.AddHours(1),
                    Permissions            = SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.Write | SharedAccessBlobPermissions.List | SharedAccessBlobPermissions.Delete
                };

                sasContainerToken = container.GetSharedAccessSignature(adHocPolicy, null);

                Console.WriteLine("SAS for blob container (ad hoc): {0}", sasContainerToken);
            }
            else
            {
                sasContainerToken = container.GetSharedAccessSignature(null, storedPolicyName);

                Console.WriteLine("SAS for blob container (stored access policy): {0}", sasContainerToken);
            }

            string containerSasUri = container.Uri + sasContainerToken;


            Console.WriteLine("Blob SAS URI:  {0}", containerSasUri);

            RegistryManager registryManager = RegistryManager.CreateFromConnectionString(IotHubConStr);
            JobProperties   exportJob       = await registryManager.ExportDevicesAsync(containerSasUri, false);
        }
        private static async Task ExportDevices(RegistryManager registryManager)
        {
            try
            {
                ConsoleWriteLine();
                ConsoleWriteLine($"Will export all devices...", ConsoleColor.Yellow);

                var blobContainerName = "jobdatafromcode";
                var blobName          = $"device-export-{DateTime.UtcNow:MM-dd-HH-mm-ss}";
                var blobContainerUri  = await GetBlobContainerUriWithSas(blobContainerName);

                ConsoleWriteLine($"Using blob container URI: {blobContainerUri}", ConsoleColor.Yellow);

                var job = await registryManager.ExportDevicesAsync(
                    exportBlobContainerUri : blobContainerUri,
                    outputBlobName : blobName,
                    excludeKeys : false);

                while (true)
                {
                    job = await registryManager.GetJobAsync(job.JobId);

                    ConsoleWriteLine($"Export job status: {job.Status}, progress: {job.Progress}%", ConsoleColor.Yellow);

                    if (job.Status == JobStatus.Completed ||
                        job.Status == JobStatus.Failed ||
                        job.Status == JobStatus.Cancelled)
                    {
                        // job has stopped
                        break;
                    }

                    await Task.Delay(1000);
                }

                ConsoleWriteLine($"Job has stopped. Status: {job.Status}", ConsoleColor.Yellow);

                if (job.Status == JobStatus.Completed)
                {
                    ConsoleWriteLine($"Devices were exported to blob {blobContainerName}\\{blobName}", ConsoleColor.Yellow);
                }
            }
            catch (Exception ex)
            {
                ConsoleWriteLine($"* ERROR * {ex.Message}", ConsoleColor.Red);
            }
        }
        private static async Task export(string connStrFrom, string sasUri)
        {
            Console.WriteLine("Started exporting...");
            RegistryManager regMgr = RegistryManager.CreateFromConnectionString(connStrFrom);

            var jobs = regMgr.GetJobsAsync().Result;

            foreach (var item in jobs)
            {
                try
                {
                    Debug.WriteLine(item.Status);

                    if (item.Status == JobStatus.Running)
                    {
                        regMgr.CancelJobAsync(item.JobId).Wait();
                    }
                }
                catch (Exception ex)
                {
                }
            }

            var exportJob = await regMgr.ExportDevicesAsync(sasUri, false);

            while (true)
            {
                if (exportJob.Status == JobStatus.Completed ||
                    exportJob.Status == JobStatus.Failed ||
                    exportJob.Status == JobStatus.Cancelled)
                {
                    if (exportJob.Status != JobStatus.Completed)
                    {
                        throw new Exception($"Export failed with status: {exportJob.Status}");
                    }
                    break;
                }

                await Task.Delay(TimeSpan.FromSeconds(5));
            }

            Console.WriteLine("Exporting completed.");
        }
예제 #10
0
        /// <summary>
        ///
        /// </summary>
        public async void ExportIoTDevices()
        {
            // Create a blob client which is used to connect to the blob storage.
            var storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=iothubcookbook;AccountKey=pnVtxmokrbpl6OSMCYcLx9FfNWhxC8xkYx/sU8oPL0Gbrw5ka/yvP1bbak+sZrD2+Qejs8zWH1AiI0CEJ129AQ==;EndpointSuffix=core.windows.net");
            var blobClient     = storageAccount.CreateCloudBlobClient();

            string Containername = "iothubdevices";

            //Get a reference to a container to use for the sample code, and create it if it does not exist.
            var container = blobClient.GetContainerReference(Containername);

            container.CreateIfNotExists();

            //Generate a SAS token and assign it to the current job.
            var storageUri       = GetContainerSasUri(container);
            var outputStorageUri = GetContainerSasUri(container);
            await _registryManager.ExportDevicesAsync(storageUri, "devices1.txt", false);



            var job = await _registryManager.ImportDevicesAsync(storageUri, outputStorageUri);
        }
예제 #11
0
        private async Task CleanupDevices()
        {
            Console.WriteLine($"Using storage container {_blobContainerClient.Name}" +
                              $" for exporting devices identities to and importing device identities from.");

            // Retrieve the SAS Uri that will be used to grant access to the storage containers.
            string storageAccountSasUri = GetStorageAccountSasUriForCleanupJob(_blobContainerClient).ToString();

            // Step 1: Export all device identities.
            JobProperties exportAllDevicesProperties = JobProperties
                                                       .CreateForExportJob(
                outputBlobContainerUri: storageAccountSasUri,
                excludeKeysInExport: true,
                storageAuthenticationType: StorageAuthenticationType.KeyBased);

            JobProperties exportAllDevicesJob = null;

            int tryCount = 0;

            while (true)
            {
                try
                {
                    exportAllDevicesJob = await _registryManager.ExportDevicesAsync(exportAllDevicesProperties);

                    break;
                }
                // Wait for pending jobs to finish.
                catch (JobQuotaExceededException) when(++tryCount < MaxIterationWait)
                {
                    Console.WriteLine($"JobQuotaExceededException... waiting.");
                    await Task.Delay(WaitDuration);
                }
            }

            if (exportAllDevicesJob == null)
            {
                throw new Exception("Export devices job failed.");
            }

            // Wait until the export job is finished.
            while (true)
            {
                exportAllDevicesJob = await _registryManager.GetJobAsync(exportAllDevicesJob.JobId);

                if (s_completedJobs.Contains(exportAllDevicesJob.Status))
                {
                    // Job has finished executing.
                    break;
                }

                Console.WriteLine($"Job {exportAllDevicesJob.JobId} is {exportAllDevicesJob.Status} with progress {exportAllDevicesJob.Progress}%");
                await Task.Delay(s_waitDuration);
            }
            Console.WriteLine($"Job {exportAllDevicesJob.JobId} is {exportAllDevicesJob.Status}.");

            if (exportAllDevicesJob.Status != JobStatus.Completed)
            {
                throw new Exception("Exporting devices failed, exiting.");
            }

            // Step 2: Download the exported devices list from the blob create in Step 1.
            BlobClient       blobClient = _blobContainerClient.GetBlobClient(ImportExportDevicesFileName);
            BlobDownloadInfo download   = await blobClient.DownloadAsync();

            IEnumerable <ExportImportDevice> exportedDevices = ImportExportDevicesHelpers.BuildExportImportDeviceFromStream(download.Content);

            // Step 3: Collect the devices that need to be deleted and update their ImportMode to be Delete.
            // Thie step will create an ExportImportDevice identity for each device/ module identity registered on hub.
            // If you hub instance has IoT Hub module or Edge module instances registered, then they will be counted as separate entities
            // from the corresponding IoT Hub device/ Edge device that they are associated with.
            // As a result, the count of ExportImportDevice identities to be deleted might be greater than the
            // count of IoT hub devices retrieved in PrintDeviceCountAsync().
            var devicesToBeDeleted = new List <ExportImportDevice>();

            foreach (var device in exportedDevices)
            {
                string deviceId = device.Id;
                foreach (string prefix in _deleteDevicesWithPrefix)
                {
                    if (deviceId.StartsWith(prefix, StringComparison.OrdinalIgnoreCase))
                    {
                        devicesToBeDeleted.Add(device);
                    }
                }
            }
            devicesToBeDeleted
            .ForEach(device => device.ImportMode = ImportMode.Delete);
            Console.WriteLine($"Retrieved {devicesToBeDeleted.Count} devices for deletion.");

            if (devicesToBeDeleted.Count > 0)
            {
                // Step 3a: Write the new import data back to the blob.
                using Stream devicesFile = ImportExportDevicesHelpers.BuildDevicesStream(devicesToBeDeleted);
                await blobClient.UploadAsync(devicesFile, overwrite : true);

                // Step 3b: Call import using the same blob to delete all devices.
                JobProperties importDevicesToBeDeletedProperties = JobProperties
                                                                   .CreateForImportJob(
                    inputBlobContainerUri: storageAccountSasUri,
                    outputBlobContainerUri: storageAccountSasUri,
                    storageAuthenticationType: StorageAuthenticationType.KeyBased);

                JobProperties importDevicesToBeDeletedJob = null;

                tryCount = 0;
                while (true)
                {
                    try
                    {
                        importDevicesToBeDeletedJob = await _registryManager.ImportDevicesAsync(importDevicesToBeDeletedProperties);

                        break;
                    }
                    // Wait for pending jobs to finish.
                    catch (JobQuotaExceededException) when(++tryCount < MaxIterationWait)
                    {
                        Console.WriteLine($"JobQuotaExceededException... waiting.");
                        await Task.Delay(WaitDuration);
                    }
                }

                if (importDevicesToBeDeletedJob == null)
                {
                    throw new Exception("Import devices job failed.");
                }

                // Wait until job is finished.
                while (true)
                {
                    importDevicesToBeDeletedJob = await _registryManager.GetJobAsync(importDevicesToBeDeletedJob.JobId);

                    if (s_completedJobs.Contains(importDevicesToBeDeletedJob.Status))
                    {
                        // Job has finished executing.
                        break;
                    }

                    Console.WriteLine($"Job {importDevicesToBeDeletedJob.JobId} is {importDevicesToBeDeletedJob.Status} with progress {importDevicesToBeDeletedJob.Progress}%");
                    await Task.Delay(s_waitDuration);
                }
                Console.WriteLine($"Job {importDevicesToBeDeletedJob.JobId} is {importDevicesToBeDeletedJob.Status}.");
            }

            // Step 4: Delete the storage container created.
            await _blobContainerClient.DeleteAsync();

            Console.WriteLine($"Storage container {_blobContainerClient.Name} deleted.");
        }
예제 #12
0
        private async Task <IDictionary <string, string> > CreateIoTHubDevicesAndGetConnectionStringsAsync(
            IDictionary <string, List <DeviceDescription> > allDevices)
        {
            var result = new ConcurrentDictionary <string, string>();

            string[] deviceIds = allDevices.Keys.ToArray();

            Console.WriteLine($"{_actionMessage} devices...");
            if (RemoveDevices)
            {
                await RemoveDevicesAsync(deviceIds);
            }
            else
            {
                const int maxDevicesPerCall     = 100;
                int       numberOfCallsRequired = (int)Math.Ceiling(deviceIds.Length / (double)maxDevicesPerCall);
                for (int i = 0; i < numberOfCallsRequired; i++)
                {
                    string[] deviceIdsForCall = deviceIds.Skip(i * maxDevicesPerCall).Take(maxDevicesPerCall).ToArray();
                    BulkRegistryOperationResult bulkCreationResult =
                        await _registryManager.AddDevices2Async(deviceIdsForCall.Select(id => new Device(id)));

                    if (!bulkCreationResult.IsSuccessful)
                    {
                        Console.WriteLine("Failed creating devices...");
                        foreach (DeviceRegistryOperationError creationError in bulkCreationResult.Errors)
                        {
                            Console.WriteLine($"Device {creationError.DeviceId}: {creationError.ErrorStatus}");
                        }

                        throw new Exception();
                    }
                }
            }

            Console.WriteLine($"{_actionMessage} devices completed.");


            if (!RemoveDevices)
            {
                Console.WriteLine("Retrieving connection strings...");
                await InitializeBlobContainerAsync();

                string sasToken = _deviceExportContainer.GetSharedAccessSignature(new SharedAccessBlobPolicy(), "saspolicy");

                var containerSasUri = $"{_deviceExportContainer.Uri}{sasToken}";

                var job = await _registryManager.ExportDevicesAsync(containerSasUri, false);

                while (true)
                {
                    job = await _registryManager.GetJobAsync(job.JobId);

                    if (job.Status == JobStatus.Completed ||
                        job.Status == JobStatus.Failed ||
                        job.Status == JobStatus.Cancelled)
                    {
                        // Job has finished executing

                        break;
                    }

                    await Task.Delay(TimeSpan.FromSeconds(5));
                }

                var exportedDevices = new List <ExportImportDevice>();
                var blob            = _deviceExportContainer.GetBlobReference("devices.txt");
                using (var streamReader = new StreamReader(await blob.OpenReadAsync(), Encoding.UTF8))
                {
                    while (streamReader.Peek() != -1)
                    {
                        string line = await streamReader.ReadLineAsync();

                        var device = JsonConvert.DeserializeObject <ExportImportDevice>(line);
                        exportedDevices.Add(device);
                    }
                }

                await _deviceExportContainer.DeleteIfExistsAsync();

                await File.WriteAllTextAsync(CreatedDevicesJsonFileName, JsonConvert.SerializeObject(exportedDevices));

                foreach (ExportImportDevice device in exportedDevices.OrderBy(d => d.Id))
                {
                    result[device.Id] =
                        $"HostName={_iotHubConnectionStringBuilder.HostName};DeviceId={device.Id};SharedAccessKey={device.Authentication.SymmetricKey.PrimaryKey}";
                }

                Console.WriteLine("Retrieval complete.");
            }

            return(result);
        }
예제 #13
0
        static async Task ExportDeviceIdentities(RegistryManager registryManager, string containerUri)
        {
            var job = await registryManager.ExportDevicesAsync(containerUri, false);

            await WaitForJobToComplete(registryManager, job);
        }
        private async Task <JobProperties> CreateAndWaitForJobAsync(
            StorageAuthenticationType storageAuthenticationType,
            bool isUserAssignedMsi,
            string devicesFileName,
            string configsFileName,
            RegistryManager registryManager,
            Uri containerUri)
        {
            int tryCount = 0;

            ManagedIdentity identity = isUserAssignedMsi
                ? new ManagedIdentity
            {
                UserAssignedIdentity = TestConfiguration.IoTHub.UserAssignedMsiResourceId
            }
                : null;

            JobProperties exportJobResponse = JobProperties.CreateForExportJob(
                containerUri.ToString(),
                true,
                devicesFileName,
                storageAuthenticationType,
                identity);

            exportJobResponse.IncludeConfigurations  = true;
            exportJobResponse.ConfigurationsBlobName = configsFileName;

            while (tryCount < MaxIterationWait)
            {
                try
                {
                    exportJobResponse = await registryManager.ExportDevicesAsync(exportJobResponse).ConfigureAwait(false);

                    break;
                }
                // Concurrent jobs can be rejected, so implement a retry mechanism to handle conflicts with other tests
                catch (JobQuotaExceededException) when(++tryCount < MaxIterationWait)
                {
                    Logger.Trace($"JobQuotaExceededException... waiting.");
                    await Task.Delay(s_waitDuration).ConfigureAwait(false);

                    continue;
                }
            }

            for (int i = 0; i < MaxIterationWait; ++i)
            {
                await Task.Delay(s_waitDuration).ConfigureAwait(false);

                exportJobResponse = await registryManager.GetJobAsync(exportJobResponse.JobId).ConfigureAwait(false);

                Logger.Trace($"Job {exportJobResponse.JobId} is {exportJobResponse.Status} with progress {exportJobResponse.Progress}%");
                if (!s_incompleteJobs.Contains(exportJobResponse.Status))
                {
                    break;
                }
            }

            exportJobResponse.Status.Should().Be(JobStatus.Completed, "Otherwise import failed");
            exportJobResponse.FailureReason.Should().BeNullOrEmpty("Otherwise import failed");

            return(exportJobResponse);
        }