/// Get the list of devices registered to the IoT Hub /// and export it to a blob as deserialized objects. private async Task ExportDevices(string containerURI, string hubConnectionString) { try { Console.WriteLine("Creating and running registry manager job to retrieve the devices from the hub."); // Create an instance of the registry manager class. using RegistryManager registryManager = RegistryManager.CreateFromConnectionString(hubConnectionString); // Call an export job on the IoT Hub to retrieve all devices. // This writes them to devices.txt in the container. // The second parameter indicates whether to export the keys or not. JobProperties exportJob = await registryManager.ExportDevicesAsync(containerURI, false).ConfigureAwait(false); // Poll every 5 seconds to see if the job has finished executing. while (true) { exportJob = await registryManager.GetJobAsync(exportJob.JobId).ConfigureAwait(false); if (exportJob.Status == JobStatus.Completed || exportJob.Status == JobStatus.Failed || exportJob.Status == JobStatus.Cancelled) { // Job has finished executing break; } await Task.Delay(TimeSpan.FromSeconds(5)).ConfigureAwait(false); } } catch (Exception ex) { Debug.Print("Error exporting devices to blob storage. Exception message = {0}", ex.Message); } }
/// <summary> /// Requests the device list export. /// </summary> /// <param name="registryManager">The registry manager.</param> /// <param name="blobStorageUri">The BLOB storage URI including SaS container token. /// See Shared Access Signiture </param> /// <param name="ct">The ct.</param> /// <returns></returns> private static async Task <JobProperties> RequestDeviceListExport(RegistryManager registryManager, string blobStorageUri, CancellationToken ct) { var exportTask = await registryManager.ExportDevicesAsync(blobStorageUri, "foo", false, ct); Console.WriteLine("Job Id: {0} current status: {1}", exportTask.JobId, exportTask.Status); return(exportTask); }
/// Get the list of devices registered to the IoT Hub /// and export it to a blob as deserialized objects. public static async Task ExportDevices(string containerURI, string hubConnectionString) { // Create an instance of the registry manager class. RegistryManager registryManager = RegistryManager.CreateFromConnectionString(hubConnectionString); // Call an export job on the IoT Hub to retrieve all devices. // This writes them to devices.txt in the container. // The second parameter indicates whether to export the keys or not. JobProperties exportJob = await registryManager.ExportDevicesAsync(containerURI, false); // Poll every 5 seconds to see if the job has finished executing. while (true) { exportJob = await registryManager.GetJobAsync(exportJob.JobId); if (exportJob.Status == JobStatus.Completed || exportJob.Status == JobStatus.Failed || exportJob.Status == JobStatus.Cancelled) { // Job has finished executing break; } await Task.Delay(TimeSpan.FromSeconds(5)); } // Note: could add twin data here if you want to export it. }
public async Task <bool> ExportAllDevices(string connString, string containerSasUri) { bool bret = false; // Call an export job on the IoT Hub to retrieve all devices RegistryManager registryManager = RegistryManager.CreateFromConnectionString(connString); JobProperties exportJob = await registryManager.ExportDevicesAsync(containerSasUri, false); // Wait until job is finished while (true) { exportJob = await registryManager.GetJobAsync(exportJob.JobId); if (exportJob.Status == JobStatus.Completed || exportJob.Status == JobStatus.Failed || exportJob.Status == JobStatus.Cancelled) { if (exportJob.Status == JobStatus.Completed) { bret = true; } break; } await Task.Delay(TimeSpan.FromSeconds(5)); } return(bret); }
static async System.Threading.Tasks.Task Main(string[] args) { var configuration = new ConfigurationBuilder() .AddJsonFile("appsettings.json") .AddEnvironmentVariables() .Build(); Console.WriteLine(configuration.GetConnectionString("DestinationIotHubConnectionString")); Console.WriteLine(configuration.GetConnectionString("BlobSasToken")); RegistryManager sourceRegistryManager = RegistryManager.CreateFromConnectionString(configuration.GetConnectionString("SourceIotHubConnectionString")); RegistryManager destinationRegistryManager = RegistryManager.CreateFromConnectionString(configuration.GetConnectionString("DestinationIotHubConnectionString")); string containerSasUri = configuration.GetConnectionString("BlobSasToken"); Console.WriteLine($"Start export of IoT Hub devices to blob"); // Call an export job on the IoT Hub to retrieve all devices JobProperties exportJob = await sourceRegistryManager.ExportDevicesAsync(containerSasUri, false); var exportJobResult = await WaitForJobCompletion(sourceRegistryManager, exportJob); Console.WriteLine($"Start import of IoT Hub devices to blob"); JobProperties importJob = await destinationRegistryManager.ImportDevicesAsync(containerSasUri, containerSasUri); var importJobResult = await WaitForJobCompletion(destinationRegistryManager, importJob); }
public async Task ExportDevicesAsync(string hubConnectionString, string blobContainerUri, string userDefinedManagedIdentityResourceId = null) { using RegistryManager srcRegistryManager = RegistryManager.CreateFromConnectionString(hubConnectionString); // If StorageAuthenticationType is set to IdentityBased and userAssignedIdentity property is // not null, the jobs will use user defined managed identity. If the IoT hub is not // configured with the user defined managed identity specified in userAssignedIdentity, // the job will fail. // If StorageAuthenticationType is set to IdentityBased and userAssignedIdentity property is // null, the jobs will use system defined identity by default. If the IoT hub is configured with the // system defined managed identity, the job will succeed but will not use the user defined managed identity. // If the IoT hub is not configured with system defined managed identity, the job will fail. // If StorageAuthenticationType is set to IdentityBased and neither user defined nor system defined // managed identities are configured on the hub, the job will fail. JobProperties jobProperties = JobProperties.CreateForExportJob( outputBlobContainerUri: blobContainerUri, excludeKeysInExport: false, storageAuthenticationType: StorageAuthenticationType.IdentityBased, identity: new ManagedIdentity { userAssignedIdentity = userDefinedManagedIdentityResourceId }); JobProperties jobResult = await srcRegistryManager .ExportDevicesAsync(jobProperties); // Poll every 5 seconds to see if the job has finished executing. while (true) { jobResult = await srcRegistryManager.GetJobAsync(jobResult.JobId); if (jobResult.Status == JobStatus.Completed) { break; } else if (jobResult.Status == JobStatus.Failed) { throw new Exception("Export job failed."); } else if (jobResult.Status == JobStatus.Cancelled) { throw new Exception("Export job was canceled."); } else { await Task.Delay(TimeSpan.FromSeconds(5)); } } }
public static async void RunAsync([TimerTrigger("0 */2 * * * *")] TimerInfo myTimer, ILogger log, ExecutionContext context) { log.LogInformation($"C# Timer trigger function executed at: {DateTime.Now}"); var config = new ConfigurationBuilder() .SetBasePath(context.FunctionAppDirectory) .AddJsonFile("local.settings.json", optional: true, reloadOnChange: true) .AddEnvironmentVariables() .Build(); var BlobConStr = config["blobConStr"]; var BlobContainer = config["blobContainer"]; var BlobURL = config["blobURL"]; var IotHubConStr = config["iotHubConStr"]; CloudStorageAccount storageAccount = CloudStorageAccount.Parse(BlobConStr); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer container = blobClient.GetContainerReference(BlobContainer); string storedPolicyName = null; string sasContainerToken; if (storedPolicyName == null) { SharedAccessBlobPolicy adHocPolicy = new SharedAccessBlobPolicy() { SharedAccessExpiryTime = DateTime.UtcNow.AddHours(1), Permissions = SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.Write | SharedAccessBlobPermissions.List | SharedAccessBlobPermissions.Delete }; sasContainerToken = container.GetSharedAccessSignature(adHocPolicy, null); Console.WriteLine("SAS for blob container (ad hoc): {0}", sasContainerToken); } else { sasContainerToken = container.GetSharedAccessSignature(null, storedPolicyName); Console.WriteLine("SAS for blob container (stored access policy): {0}", sasContainerToken); } string containerSasUri = container.Uri + sasContainerToken; Console.WriteLine("Blob SAS URI: {0}", containerSasUri); RegistryManager registryManager = RegistryManager.CreateFromConnectionString(IotHubConStr); JobProperties exportJob = await registryManager.ExportDevicesAsync(containerSasUri, false); }
private static async Task ExportDevices(RegistryManager registryManager) { try { ConsoleWriteLine(); ConsoleWriteLine($"Will export all devices...", ConsoleColor.Yellow); var blobContainerName = "jobdatafromcode"; var blobName = $"device-export-{DateTime.UtcNow:MM-dd-HH-mm-ss}"; var blobContainerUri = await GetBlobContainerUriWithSas(blobContainerName); ConsoleWriteLine($"Using blob container URI: {blobContainerUri}", ConsoleColor.Yellow); var job = await registryManager.ExportDevicesAsync( exportBlobContainerUri : blobContainerUri, outputBlobName : blobName, excludeKeys : false); while (true) { job = await registryManager.GetJobAsync(job.JobId); ConsoleWriteLine($"Export job status: {job.Status}, progress: {job.Progress}%", ConsoleColor.Yellow); if (job.Status == JobStatus.Completed || job.Status == JobStatus.Failed || job.Status == JobStatus.Cancelled) { // job has stopped break; } await Task.Delay(1000); } ConsoleWriteLine($"Job has stopped. Status: {job.Status}", ConsoleColor.Yellow); if (job.Status == JobStatus.Completed) { ConsoleWriteLine($"Devices were exported to blob {blobContainerName}\\{blobName}", ConsoleColor.Yellow); } } catch (Exception ex) { ConsoleWriteLine($"* ERROR * {ex.Message}", ConsoleColor.Red); } }
private static async Task export(string connStrFrom, string sasUri) { Console.WriteLine("Started exporting..."); RegistryManager regMgr = RegistryManager.CreateFromConnectionString(connStrFrom); var jobs = regMgr.GetJobsAsync().Result; foreach (var item in jobs) { try { Debug.WriteLine(item.Status); if (item.Status == JobStatus.Running) { regMgr.CancelJobAsync(item.JobId).Wait(); } } catch (Exception ex) { } } var exportJob = await regMgr.ExportDevicesAsync(sasUri, false); while (true) { if (exportJob.Status == JobStatus.Completed || exportJob.Status == JobStatus.Failed || exportJob.Status == JobStatus.Cancelled) { if (exportJob.Status != JobStatus.Completed) { throw new Exception($"Export failed with status: {exportJob.Status}"); } break; } await Task.Delay(TimeSpan.FromSeconds(5)); } Console.WriteLine("Exporting completed."); }
/// <summary> /// /// </summary> public async void ExportIoTDevices() { // Create a blob client which is used to connect to the blob storage. var storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=iothubcookbook;AccountKey=pnVtxmokrbpl6OSMCYcLx9FfNWhxC8xkYx/sU8oPL0Gbrw5ka/yvP1bbak+sZrD2+Qejs8zWH1AiI0CEJ129AQ==;EndpointSuffix=core.windows.net"); var blobClient = storageAccount.CreateCloudBlobClient(); string Containername = "iothubdevices"; //Get a reference to a container to use for the sample code, and create it if it does not exist. var container = blobClient.GetContainerReference(Containername); container.CreateIfNotExists(); //Generate a SAS token and assign it to the current job. var storageUri = GetContainerSasUri(container); var outputStorageUri = GetContainerSasUri(container); await _registryManager.ExportDevicesAsync(storageUri, "devices1.txt", false); var job = await _registryManager.ImportDevicesAsync(storageUri, outputStorageUri); }
private async Task CleanupDevices() { Console.WriteLine($"Using storage container {_blobContainerClient.Name}" + $" for exporting devices identities to and importing device identities from."); // Retrieve the SAS Uri that will be used to grant access to the storage containers. string storageAccountSasUri = GetStorageAccountSasUriForCleanupJob(_blobContainerClient).ToString(); // Step 1: Export all device identities. JobProperties exportAllDevicesProperties = JobProperties .CreateForExportJob( outputBlobContainerUri: storageAccountSasUri, excludeKeysInExport: true, storageAuthenticationType: StorageAuthenticationType.KeyBased); JobProperties exportAllDevicesJob = null; int tryCount = 0; while (true) { try { exportAllDevicesJob = await _registryManager.ExportDevicesAsync(exportAllDevicesProperties); break; } // Wait for pending jobs to finish. catch (JobQuotaExceededException) when(++tryCount < MaxIterationWait) { Console.WriteLine($"JobQuotaExceededException... waiting."); await Task.Delay(WaitDuration); } } if (exportAllDevicesJob == null) { throw new Exception("Export devices job failed."); } // Wait until the export job is finished. while (true) { exportAllDevicesJob = await _registryManager.GetJobAsync(exportAllDevicesJob.JobId); if (s_completedJobs.Contains(exportAllDevicesJob.Status)) { // Job has finished executing. break; } Console.WriteLine($"Job {exportAllDevicesJob.JobId} is {exportAllDevicesJob.Status} with progress {exportAllDevicesJob.Progress}%"); await Task.Delay(s_waitDuration); } Console.WriteLine($"Job {exportAllDevicesJob.JobId} is {exportAllDevicesJob.Status}."); if (exportAllDevicesJob.Status != JobStatus.Completed) { throw new Exception("Exporting devices failed, exiting."); } // Step 2: Download the exported devices list from the blob create in Step 1. BlobClient blobClient = _blobContainerClient.GetBlobClient(ImportExportDevicesFileName); BlobDownloadInfo download = await blobClient.DownloadAsync(); IEnumerable <ExportImportDevice> exportedDevices = ImportExportDevicesHelpers.BuildExportImportDeviceFromStream(download.Content); // Step 3: Collect the devices that need to be deleted and update their ImportMode to be Delete. // Thie step will create an ExportImportDevice identity for each device/ module identity registered on hub. // If you hub instance has IoT Hub module or Edge module instances registered, then they will be counted as separate entities // from the corresponding IoT Hub device/ Edge device that they are associated with. // As a result, the count of ExportImportDevice identities to be deleted might be greater than the // count of IoT hub devices retrieved in PrintDeviceCountAsync(). var devicesToBeDeleted = new List <ExportImportDevice>(); foreach (var device in exportedDevices) { string deviceId = device.Id; foreach (string prefix in _deleteDevicesWithPrefix) { if (deviceId.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) { devicesToBeDeleted.Add(device); } } } devicesToBeDeleted .ForEach(device => device.ImportMode = ImportMode.Delete); Console.WriteLine($"Retrieved {devicesToBeDeleted.Count} devices for deletion."); if (devicesToBeDeleted.Count > 0) { // Step 3a: Write the new import data back to the blob. using Stream devicesFile = ImportExportDevicesHelpers.BuildDevicesStream(devicesToBeDeleted); await blobClient.UploadAsync(devicesFile, overwrite : true); // Step 3b: Call import using the same blob to delete all devices. JobProperties importDevicesToBeDeletedProperties = JobProperties .CreateForImportJob( inputBlobContainerUri: storageAccountSasUri, outputBlobContainerUri: storageAccountSasUri, storageAuthenticationType: StorageAuthenticationType.KeyBased); JobProperties importDevicesToBeDeletedJob = null; tryCount = 0; while (true) { try { importDevicesToBeDeletedJob = await _registryManager.ImportDevicesAsync(importDevicesToBeDeletedProperties); break; } // Wait for pending jobs to finish. catch (JobQuotaExceededException) when(++tryCount < MaxIterationWait) { Console.WriteLine($"JobQuotaExceededException... waiting."); await Task.Delay(WaitDuration); } } if (importDevicesToBeDeletedJob == null) { throw new Exception("Import devices job failed."); } // Wait until job is finished. while (true) { importDevicesToBeDeletedJob = await _registryManager.GetJobAsync(importDevicesToBeDeletedJob.JobId); if (s_completedJobs.Contains(importDevicesToBeDeletedJob.Status)) { // Job has finished executing. break; } Console.WriteLine($"Job {importDevicesToBeDeletedJob.JobId} is {importDevicesToBeDeletedJob.Status} with progress {importDevicesToBeDeletedJob.Progress}%"); await Task.Delay(s_waitDuration); } Console.WriteLine($"Job {importDevicesToBeDeletedJob.JobId} is {importDevicesToBeDeletedJob.Status}."); } // Step 4: Delete the storage container created. await _blobContainerClient.DeleteAsync(); Console.WriteLine($"Storage container {_blobContainerClient.Name} deleted."); }
private async Task <IDictionary <string, string> > CreateIoTHubDevicesAndGetConnectionStringsAsync( IDictionary <string, List <DeviceDescription> > allDevices) { var result = new ConcurrentDictionary <string, string>(); string[] deviceIds = allDevices.Keys.ToArray(); Console.WriteLine($"{_actionMessage} devices..."); if (RemoveDevices) { await RemoveDevicesAsync(deviceIds); } else { const int maxDevicesPerCall = 100; int numberOfCallsRequired = (int)Math.Ceiling(deviceIds.Length / (double)maxDevicesPerCall); for (int i = 0; i < numberOfCallsRequired; i++) { string[] deviceIdsForCall = deviceIds.Skip(i * maxDevicesPerCall).Take(maxDevicesPerCall).ToArray(); BulkRegistryOperationResult bulkCreationResult = await _registryManager.AddDevices2Async(deviceIdsForCall.Select(id => new Device(id))); if (!bulkCreationResult.IsSuccessful) { Console.WriteLine("Failed creating devices..."); foreach (DeviceRegistryOperationError creationError in bulkCreationResult.Errors) { Console.WriteLine($"Device {creationError.DeviceId}: {creationError.ErrorStatus}"); } throw new Exception(); } } } Console.WriteLine($"{_actionMessage} devices completed."); if (!RemoveDevices) { Console.WriteLine("Retrieving connection strings..."); await InitializeBlobContainerAsync(); string sasToken = _deviceExportContainer.GetSharedAccessSignature(new SharedAccessBlobPolicy(), "saspolicy"); var containerSasUri = $"{_deviceExportContainer.Uri}{sasToken}"; var job = await _registryManager.ExportDevicesAsync(containerSasUri, false); while (true) { job = await _registryManager.GetJobAsync(job.JobId); if (job.Status == JobStatus.Completed || job.Status == JobStatus.Failed || job.Status == JobStatus.Cancelled) { // Job has finished executing break; } await Task.Delay(TimeSpan.FromSeconds(5)); } var exportedDevices = new List <ExportImportDevice>(); var blob = _deviceExportContainer.GetBlobReference("devices.txt"); using (var streamReader = new StreamReader(await blob.OpenReadAsync(), Encoding.UTF8)) { while (streamReader.Peek() != -1) { string line = await streamReader.ReadLineAsync(); var device = JsonConvert.DeserializeObject <ExportImportDevice>(line); exportedDevices.Add(device); } } await _deviceExportContainer.DeleteIfExistsAsync(); await File.WriteAllTextAsync(CreatedDevicesJsonFileName, JsonConvert.SerializeObject(exportedDevices)); foreach (ExportImportDevice device in exportedDevices.OrderBy(d => d.Id)) { result[device.Id] = $"HostName={_iotHubConnectionStringBuilder.HostName};DeviceId={device.Id};SharedAccessKey={device.Authentication.SymmetricKey.PrimaryKey}"; } Console.WriteLine("Retrieval complete."); } return(result); }
static async Task ExportDeviceIdentities(RegistryManager registryManager, string containerUri) { var job = await registryManager.ExportDevicesAsync(containerUri, false); await WaitForJobToComplete(registryManager, job); }
private async Task <JobProperties> CreateAndWaitForJobAsync( StorageAuthenticationType storageAuthenticationType, bool isUserAssignedMsi, string devicesFileName, string configsFileName, RegistryManager registryManager, Uri containerUri) { int tryCount = 0; ManagedIdentity identity = isUserAssignedMsi ? new ManagedIdentity { UserAssignedIdentity = TestConfiguration.IoTHub.UserAssignedMsiResourceId } : null; JobProperties exportJobResponse = JobProperties.CreateForExportJob( containerUri.ToString(), true, devicesFileName, storageAuthenticationType, identity); exportJobResponse.IncludeConfigurations = true; exportJobResponse.ConfigurationsBlobName = configsFileName; while (tryCount < MaxIterationWait) { try { exportJobResponse = await registryManager.ExportDevicesAsync(exportJobResponse).ConfigureAwait(false); break; } // Concurrent jobs can be rejected, so implement a retry mechanism to handle conflicts with other tests catch (JobQuotaExceededException) when(++tryCount < MaxIterationWait) { Logger.Trace($"JobQuotaExceededException... waiting."); await Task.Delay(s_waitDuration).ConfigureAwait(false); continue; } } for (int i = 0; i < MaxIterationWait; ++i) { await Task.Delay(s_waitDuration).ConfigureAwait(false); exportJobResponse = await registryManager.GetJobAsync(exportJobResponse.JobId).ConfigureAwait(false); Logger.Trace($"Job {exportJobResponse.JobId} is {exportJobResponse.Status} with progress {exportJobResponse.Progress}%"); if (!s_incompleteJobs.Contains(exportJobResponse.Status)) { break; } } exportJobResponse.Status.Should().Be(JobStatus.Completed, "Otherwise import failed"); exportJobResponse.FailureReason.Should().BeNullOrEmpty("Otherwise import failed"); return(exportJobResponse); }