/// Get the list of devices registered to the IoT Hub
/// and export it to a blob as deserialized objects.
private async Task ExportDevices(string containerURI, string hubConnectionString)
{
try
{
Console.WriteLine("Creating and running registry manager job to retrieve the devices from the hub.");
// Create an instance of the registry manager class.
using RegistryManager registryManager = RegistryManager.CreateFromConnectionString(hubConnectionString);
// Call an export job on the IoT Hub to retrieve all devices.
// This writes them to devices.txt in the container.
// The second parameter indicates whether to export the keys or not.
JobProperties exportJob = await registryManager.ExportDevicesAsync(containerURI, false).ConfigureAwait(false);
// Poll every 5 seconds to see if the job has finished executing.
while (true)
{
exportJob = await registryManager.GetJobAsync(exportJob.JobId).ConfigureAwait(false);
if (exportJob.Status == JobStatus.Completed ||
exportJob.Status == JobStatus.Failed ||
exportJob.Status == JobStatus.Cancelled)
{
// Job has finished executing
break;
}
await Task.Delay(TimeSpan.FromSeconds(5)).ConfigureAwait(false);
}
}
catch (Exception ex)
{
Debug.Print("Error exporting devices to blob storage. Exception message = {0}", ex.Message);
}
}
/// <summary>
/// Requests the device list export.
/// </summary>
/// <param name="registryManager">The registry manager.</param>
/// <param name="blobStorageUri">The BLOB storage URI including SaS container token.
/// See Shared Access Signiture </param>
/// <param name="ct">The ct.</param>
/// <returns></returns>
private static async Task <JobProperties> RequestDeviceListExport(RegistryManager registryManager, string blobStorageUri, CancellationToken ct)
{
var exportTask = await registryManager.ExportDevicesAsync(blobStorageUri, "foo", false, ct);
Console.WriteLine("Job Id: {0} current status: {1}", exportTask.JobId, exportTask.Status);
return(exportTask);
}
/// Get the list of devices registered to the IoT Hub
/// and export it to a blob as deserialized objects.
public static async Task ExportDevices(string containerURI, string hubConnectionString)
{
// Create an instance of the registry manager class.
RegistryManager registryManager =
RegistryManager.CreateFromConnectionString(hubConnectionString);
// Call an export job on the IoT Hub to retrieve all devices.
// This writes them to devices.txt in the container.
// The second parameter indicates whether to export the keys or not.
JobProperties exportJob = await
registryManager.ExportDevicesAsync(containerURI, false);
// Poll every 5 seconds to see if the job has finished executing.
while (true)
{
exportJob = await registryManager.GetJobAsync(exportJob.JobId);
if (exportJob.Status == JobStatus.Completed ||
exportJob.Status == JobStatus.Failed ||
exportJob.Status == JobStatus.Cancelled)
{
// Job has finished executing
break;
}
await Task.Delay(TimeSpan.FromSeconds(5));
}
// Note: could add twin data here if you want to export it.
}
public async Task <bool> ExportAllDevices(string connString, string containerSasUri)
{
bool bret = false;
// Call an export job on the IoT Hub to retrieve all devices
RegistryManager registryManager = RegistryManager.CreateFromConnectionString(connString);
JobProperties exportJob = await registryManager.ExportDevicesAsync(containerSasUri, false);
// Wait until job is finished
while (true)
{
exportJob = await registryManager.GetJobAsync(exportJob.JobId);
if (exportJob.Status == JobStatus.Completed ||
exportJob.Status == JobStatus.Failed ||
exportJob.Status == JobStatus.Cancelled)
{
if (exportJob.Status == JobStatus.Completed)
{
bret = true;
}
break;
}
await Task.Delay(TimeSpan.FromSeconds(5));
}
return(bret);
}
static async System.Threading.Tasks.Task Main(string[] args)
{
var configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.AddEnvironmentVariables()
.Build();
Console.WriteLine(configuration.GetConnectionString("DestinationIotHubConnectionString"));
Console.WriteLine(configuration.GetConnectionString("BlobSasToken"));
RegistryManager sourceRegistryManager = RegistryManager.CreateFromConnectionString(configuration.GetConnectionString("SourceIotHubConnectionString"));
RegistryManager destinationRegistryManager = RegistryManager.CreateFromConnectionString(configuration.GetConnectionString("DestinationIotHubConnectionString"));
string containerSasUri = configuration.GetConnectionString("BlobSasToken");
Console.WriteLine($"Start export of IoT Hub devices to blob");
// Call an export job on the IoT Hub to retrieve all devices
JobProperties exportJob =
await sourceRegistryManager.ExportDevicesAsync(containerSasUri, false);
var exportJobResult = await WaitForJobCompletion(sourceRegistryManager, exportJob);
Console.WriteLine($"Start import of IoT Hub devices to blob");
JobProperties importJob =
await destinationRegistryManager.ImportDevicesAsync(containerSasUri, containerSasUri);
var importJobResult = await WaitForJobCompletion(destinationRegistryManager, importJob);
}
public async Task ExportDevicesAsync(string hubConnectionString,
string blobContainerUri,
string userDefinedManagedIdentityResourceId = null)
{
using RegistryManager srcRegistryManager = RegistryManager.CreateFromConnectionString(hubConnectionString);
// If StorageAuthenticationType is set to IdentityBased and userAssignedIdentity property is
// not null, the jobs will use user defined managed identity. If the IoT hub is not
// configured with the user defined managed identity specified in userAssignedIdentity,
// the job will fail.
// If StorageAuthenticationType is set to IdentityBased and userAssignedIdentity property is
// null, the jobs will use system defined identity by default. If the IoT hub is configured with the
// system defined managed identity, the job will succeed but will not use the user defined managed identity.
// If the IoT hub is not configured with system defined managed identity, the job will fail.
// If StorageAuthenticationType is set to IdentityBased and neither user defined nor system defined
// managed identities are configured on the hub, the job will fail.
JobProperties jobProperties = JobProperties.CreateForExportJob(
outputBlobContainerUri: blobContainerUri,
excludeKeysInExport: false,
storageAuthenticationType: StorageAuthenticationType.IdentityBased,
identity: new ManagedIdentity
{
userAssignedIdentity = userDefinedManagedIdentityResourceId
});
JobProperties jobResult = await srcRegistryManager
.ExportDevicesAsync(jobProperties);
// Poll every 5 seconds to see if the job has finished executing.
while (true)
{
jobResult = await srcRegistryManager.GetJobAsync(jobResult.JobId);
if (jobResult.Status == JobStatus.Completed)
{
break;
}
else if (jobResult.Status == JobStatus.Failed)
{
throw new Exception("Export job failed.");
}
else if (jobResult.Status == JobStatus.Cancelled)
{
throw new Exception("Export job was canceled.");
}
else
{
await Task.Delay(TimeSpan.FromSeconds(5));
}
}
}
public static async void RunAsync([TimerTrigger("0 */2 * * * *")] TimerInfo myTimer, ILogger log, ExecutionContext context)
{
log.LogInformation($"C# Timer trigger function executed at: {DateTime.Now}");
var config = new ConfigurationBuilder()
.SetBasePath(context.FunctionAppDirectory)
.AddJsonFile("local.settings.json", optional: true, reloadOnChange: true)
.AddEnvironmentVariables()
.Build();
var BlobConStr = config["blobConStr"];
var BlobContainer = config["blobContainer"];
var BlobURL = config["blobURL"];
var IotHubConStr = config["iotHubConStr"];
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(BlobConStr);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(BlobContainer);
string storedPolicyName = null;
string sasContainerToken;
if (storedPolicyName == null)
{
SharedAccessBlobPolicy adHocPolicy = new SharedAccessBlobPolicy()
{
SharedAccessExpiryTime = DateTime.UtcNow.AddHours(1),
Permissions = SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.Write | SharedAccessBlobPermissions.List | SharedAccessBlobPermissions.Delete
};
sasContainerToken = container.GetSharedAccessSignature(adHocPolicy, null);
Console.WriteLine("SAS for blob container (ad hoc): {0}", sasContainerToken);
}
else
{
sasContainerToken = container.GetSharedAccessSignature(null, storedPolicyName);
Console.WriteLine("SAS for blob container (stored access policy): {0}", sasContainerToken);
}
string containerSasUri = container.Uri + sasContainerToken;
Console.WriteLine("Blob SAS URI: {0}", containerSasUri);
RegistryManager registryManager = RegistryManager.CreateFromConnectionString(IotHubConStr);
JobProperties exportJob = await registryManager.ExportDevicesAsync(containerSasUri, false);
}
private static async Task ExportDevices(RegistryManager registryManager)
{
try
{
ConsoleWriteLine();
ConsoleWriteLine($"Will export all devices...", ConsoleColor.Yellow);
var blobContainerName = "jobdatafromcode";
var blobName = $"device-export-{DateTime.UtcNow:MM-dd-HH-mm-ss}";
var blobContainerUri = await GetBlobContainerUriWithSas(blobContainerName);
ConsoleWriteLine($"Using blob container URI: {blobContainerUri}", ConsoleColor.Yellow);
var job = await registryManager.ExportDevicesAsync(
exportBlobContainerUri : blobContainerUri,
outputBlobName : blobName,
excludeKeys : false);
while (true)
{
job = await registryManager.GetJobAsync(job.JobId);
ConsoleWriteLine($"Export job status: {job.Status}, progress: {job.Progress}%", ConsoleColor.Yellow);
if (job.Status == JobStatus.Completed ||
job.Status == JobStatus.Failed ||
job.Status == JobStatus.Cancelled)
{
// job has stopped
break;
}
await Task.Delay(1000);
}
ConsoleWriteLine($"Job has stopped. Status: {job.Status}", ConsoleColor.Yellow);
if (job.Status == JobStatus.Completed)
{
ConsoleWriteLine($"Devices were exported to blob {blobContainerName}\\{blobName}", ConsoleColor.Yellow);
}
}
catch (Exception ex)
{
ConsoleWriteLine($"* ERROR * {ex.Message}", ConsoleColor.Red);
}
}
private static async Task export(string connStrFrom, string sasUri)
{
Console.WriteLine("Started exporting...");
RegistryManager regMgr = RegistryManager.CreateFromConnectionString(connStrFrom);
var jobs = regMgr.GetJobsAsync().Result;
foreach (var item in jobs)
{
try
{
Debug.WriteLine(item.Status);
if (item.Status == JobStatus.Running)
{
regMgr.CancelJobAsync(item.JobId).Wait();
}
}
catch (Exception ex)
{
}
}
var exportJob = await regMgr.ExportDevicesAsync(sasUri, false);
while (true)
{
if (exportJob.Status == JobStatus.Completed ||
exportJob.Status == JobStatus.Failed ||
exportJob.Status == JobStatus.Cancelled)
{
if (exportJob.Status != JobStatus.Completed)
{
throw new Exception($"Export failed with status: {exportJob.Status}");
}
break;
}
await Task.Delay(TimeSpan.FromSeconds(5));
}
Console.WriteLine("Exporting completed.");
}
/// <summary>
///
/// </summary>
public async void ExportIoTDevices()
{
// Create a blob client which is used to connect to the blob storage.
var storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=iothubcookbook;AccountKey=pnVtxmokrbpl6OSMCYcLx9FfNWhxC8xkYx/sU8oPL0Gbrw5ka/yvP1bbak+sZrD2+Qejs8zWH1AiI0CEJ129AQ==;EndpointSuffix=core.windows.net");
var blobClient = storageAccount.CreateCloudBlobClient();
string Containername = "iothubdevices";
//Get a reference to a container to use for the sample code, and create it if it does not exist.
var container = blobClient.GetContainerReference(Containername);
container.CreateIfNotExists();
//Generate a SAS token and assign it to the current job.
var storageUri = GetContainerSasUri(container);
var outputStorageUri = GetContainerSasUri(container);
await _registryManager.ExportDevicesAsync(storageUri, "devices1.txt", false);
var job = await _registryManager.ImportDevicesAsync(storageUri, outputStorageUri);
}
private async Task CleanupDevices()
{
Console.WriteLine($"Using storage container {_blobContainerClient.Name}" +
$" for exporting devices identities to and importing device identities from.");
// Retrieve the SAS Uri that will be used to grant access to the storage containers.
string storageAccountSasUri = GetStorageAccountSasUriForCleanupJob(_blobContainerClient).ToString();
// Step 1: Export all device identities.
JobProperties exportAllDevicesProperties = JobProperties
.CreateForExportJob(
outputBlobContainerUri: storageAccountSasUri,
excludeKeysInExport: true,
storageAuthenticationType: StorageAuthenticationType.KeyBased);
JobProperties exportAllDevicesJob = null;
int tryCount = 0;
while (true)
{
try
{
exportAllDevicesJob = await _registryManager.ExportDevicesAsync(exportAllDevicesProperties);
break;
}
// Wait for pending jobs to finish.
catch (JobQuotaExceededException) when(++tryCount < MaxIterationWait)
{
Console.WriteLine($"JobQuotaExceededException... waiting.");
await Task.Delay(WaitDuration);
}
}
if (exportAllDevicesJob == null)
{
throw new Exception("Export devices job failed.");
}
// Wait until the export job is finished.
while (true)
{
exportAllDevicesJob = await _registryManager.GetJobAsync(exportAllDevicesJob.JobId);
if (s_completedJobs.Contains(exportAllDevicesJob.Status))
{
// Job has finished executing.
break;
}
Console.WriteLine($"Job {exportAllDevicesJob.JobId} is {exportAllDevicesJob.Status} with progress {exportAllDevicesJob.Progress}%");
await Task.Delay(s_waitDuration);
}
Console.WriteLine($"Job {exportAllDevicesJob.JobId} is {exportAllDevicesJob.Status}.");
if (exportAllDevicesJob.Status != JobStatus.Completed)
{
throw new Exception("Exporting devices failed, exiting.");
}
// Step 2: Download the exported devices list from the blob create in Step 1.
BlobClient blobClient = _blobContainerClient.GetBlobClient(ImportExportDevicesFileName);
BlobDownloadInfo download = await blobClient.DownloadAsync();
IEnumerable <ExportImportDevice> exportedDevices = ImportExportDevicesHelpers.BuildExportImportDeviceFromStream(download.Content);
// Step 3: Collect the devices that need to be deleted and update their ImportMode to be Delete.
// Thie step will create an ExportImportDevice identity for each device/ module identity registered on hub.
// If you hub instance has IoT Hub module or Edge module instances registered, then they will be counted as separate entities
// from the corresponding IoT Hub device/ Edge device that they are associated with.
// As a result, the count of ExportImportDevice identities to be deleted might be greater than the
// count of IoT hub devices retrieved in PrintDeviceCountAsync().
var devicesToBeDeleted = new List <ExportImportDevice>();
foreach (var device in exportedDevices)
{
string deviceId = device.Id;
foreach (string prefix in _deleteDevicesWithPrefix)
{
if (deviceId.StartsWith(prefix, StringComparison.OrdinalIgnoreCase))
{
devicesToBeDeleted.Add(device);
}
}
}
devicesToBeDeleted
.ForEach(device => device.ImportMode = ImportMode.Delete);
Console.WriteLine($"Retrieved {devicesToBeDeleted.Count} devices for deletion.");
if (devicesToBeDeleted.Count > 0)
{
// Step 3a: Write the new import data back to the blob.
using Stream devicesFile = ImportExportDevicesHelpers.BuildDevicesStream(devicesToBeDeleted);
await blobClient.UploadAsync(devicesFile, overwrite : true);
// Step 3b: Call import using the same blob to delete all devices.
JobProperties importDevicesToBeDeletedProperties = JobProperties
.CreateForImportJob(
inputBlobContainerUri: storageAccountSasUri,
outputBlobContainerUri: storageAccountSasUri,
storageAuthenticationType: StorageAuthenticationType.KeyBased);
JobProperties importDevicesToBeDeletedJob = null;
tryCount = 0;
while (true)
{
try
{
importDevicesToBeDeletedJob = await _registryManager.ImportDevicesAsync(importDevicesToBeDeletedProperties);
break;
}
// Wait for pending jobs to finish.
catch (JobQuotaExceededException) when(++tryCount < MaxIterationWait)
{
Console.WriteLine($"JobQuotaExceededException... waiting.");
await Task.Delay(WaitDuration);
}
}
if (importDevicesToBeDeletedJob == null)
{
throw new Exception("Import devices job failed.");
}
// Wait until job is finished.
while (true)
{
importDevicesToBeDeletedJob = await _registryManager.GetJobAsync(importDevicesToBeDeletedJob.JobId);
if (s_completedJobs.Contains(importDevicesToBeDeletedJob.Status))
{
// Job has finished executing.
break;
}
Console.WriteLine($"Job {importDevicesToBeDeletedJob.JobId} is {importDevicesToBeDeletedJob.Status} with progress {importDevicesToBeDeletedJob.Progress}%");
await Task.Delay(s_waitDuration);
}
Console.WriteLine($"Job {importDevicesToBeDeletedJob.JobId} is {importDevicesToBeDeletedJob.Status}.");
}
// Step 4: Delete the storage container created.
await _blobContainerClient.DeleteAsync();
Console.WriteLine($"Storage container {_blobContainerClient.Name} deleted.");
}
private async Task <IDictionary <string, string> > CreateIoTHubDevicesAndGetConnectionStringsAsync(
IDictionary <string, List <DeviceDescription> > allDevices)
{
var result = new ConcurrentDictionary <string, string>();
string[] deviceIds = allDevices.Keys.ToArray();
Console.WriteLine($"{_actionMessage} devices...");
if (RemoveDevices)
{
await RemoveDevicesAsync(deviceIds);
}
else
{
const int maxDevicesPerCall = 100;
int numberOfCallsRequired = (int)Math.Ceiling(deviceIds.Length / (double)maxDevicesPerCall);
for (int i = 0; i < numberOfCallsRequired; i++)
{
string[] deviceIdsForCall = deviceIds.Skip(i * maxDevicesPerCall).Take(maxDevicesPerCall).ToArray();
BulkRegistryOperationResult bulkCreationResult =
await _registryManager.AddDevices2Async(deviceIdsForCall.Select(id => new Device(id)));
if (!bulkCreationResult.IsSuccessful)
{
Console.WriteLine("Failed creating devices...");
foreach (DeviceRegistryOperationError creationError in bulkCreationResult.Errors)
{
Console.WriteLine($"Device {creationError.DeviceId}: {creationError.ErrorStatus}");
}
throw new Exception();
}
}
}
Console.WriteLine($"{_actionMessage} devices completed.");
if (!RemoveDevices)
{
Console.WriteLine("Retrieving connection strings...");
await InitializeBlobContainerAsync();
string sasToken = _deviceExportContainer.GetSharedAccessSignature(new SharedAccessBlobPolicy(), "saspolicy");
var containerSasUri = $"{_deviceExportContainer.Uri}{sasToken}";
var job = await _registryManager.ExportDevicesAsync(containerSasUri, false);
while (true)
{
job = await _registryManager.GetJobAsync(job.JobId);
if (job.Status == JobStatus.Completed ||
job.Status == JobStatus.Failed ||
job.Status == JobStatus.Cancelled)
{
// Job has finished executing
break;
}
await Task.Delay(TimeSpan.FromSeconds(5));
}
var exportedDevices = new List <ExportImportDevice>();
var blob = _deviceExportContainer.GetBlobReference("devices.txt");
using (var streamReader = new StreamReader(await blob.OpenReadAsync(), Encoding.UTF8))
{
while (streamReader.Peek() != -1)
{
string line = await streamReader.ReadLineAsync();
var device = JsonConvert.DeserializeObject <ExportImportDevice>(line);
exportedDevices.Add(device);
}
}
await _deviceExportContainer.DeleteIfExistsAsync();
await File.WriteAllTextAsync(CreatedDevicesJsonFileName, JsonConvert.SerializeObject(exportedDevices));
foreach (ExportImportDevice device in exportedDevices.OrderBy(d => d.Id))
{
result[device.Id] =
$"HostName={_iotHubConnectionStringBuilder.HostName};DeviceId={device.Id};SharedAccessKey={device.Authentication.SymmetricKey.PrimaryKey}";
}
Console.WriteLine("Retrieval complete.");
}
return(result);
}
static async Task ExportDeviceIdentities(RegistryManager registryManager, string containerUri)
{
var job = await registryManager.ExportDevicesAsync(containerUri, false);
await WaitForJobToComplete(registryManager, job);
}
private async Task <JobProperties> CreateAndWaitForJobAsync(
StorageAuthenticationType storageAuthenticationType,
bool isUserAssignedMsi,
string devicesFileName,
string configsFileName,
RegistryManager registryManager,
Uri containerUri)
{
int tryCount = 0;
ManagedIdentity identity = isUserAssignedMsi
? new ManagedIdentity
{
UserAssignedIdentity = TestConfiguration.IoTHub.UserAssignedMsiResourceId
}
: null;
JobProperties exportJobResponse = JobProperties.CreateForExportJob(
containerUri.ToString(),
true,
devicesFileName,
storageAuthenticationType,
identity);
exportJobResponse.IncludeConfigurations = true;
exportJobResponse.ConfigurationsBlobName = configsFileName;
while (tryCount < MaxIterationWait)
{
try
{
exportJobResponse = await registryManager.ExportDevicesAsync(exportJobResponse).ConfigureAwait(false);
break;
}
// Concurrent jobs can be rejected, so implement a retry mechanism to handle conflicts with other tests
catch (JobQuotaExceededException) when(++tryCount < MaxIterationWait)
{
Logger.Trace($"JobQuotaExceededException... waiting.");
await Task.Delay(s_waitDuration).ConfigureAwait(false);
continue;
}
}
for (int i = 0; i < MaxIterationWait; ++i)
{
await Task.Delay(s_waitDuration).ConfigureAwait(false);
exportJobResponse = await registryManager.GetJobAsync(exportJobResponse.JobId).ConfigureAwait(false);
Logger.Trace($"Job {exportJobResponse.JobId} is {exportJobResponse.Status} with progress {exportJobResponse.Progress}%");
if (!s_incompleteJobs.Contains(exportJobResponse.Status))
{
break;
}
}
exportJobResponse.Status.Should().Be(JobStatus.Completed, "Otherwise import failed");
exportJobResponse.FailureReason.Should().BeNullOrEmpty("Otherwise import failed");
return(exportJobResponse);
}
