/// <summary> /// Generate NumToAdd devices and add them to the hub. /// To do this, generate each identity. /// Include authentication keys. /// Write the device info to a block blob. /// Import the devices into the identity registry by calling the import job. /// </summary> private async Task GenerateAndAddDevices(string hubConnectionString, string containerURI, int NumToAdd, string devicesToAdd) { int interimProgressCount = 0; int displayProgressCount = 1000; int totalProgressCount = 0; // generate reference for list of new devices we're going to add, will write list to this blob CloudBlockBlob generatedListBlob = _cloudBlobContainer.GetBlockBlobReference(devicesToAdd); // define serializedDevices as a generic list<string> List <string> serializedDevices = new List <string>(NumToAdd); for (var i = 1; i <= NumToAdd; i++) { // Create device name with this format: Hub_00000000 + a new guid. // This should be large enough to display the largest number (1 million). string deviceName = $"Hub_{i:D8}_{Guid.NewGuid()}"; Debug.Print($"device = '{deviceName}'\n"); // Create a new ExportImportDevice. // CryptoKeyGenerator is in the Microsoft.Azure.Devices.Common namespace. var deviceToAdd = new ExportImportDevice { Id = deviceName, Status = DeviceStatus.Enabled, Authentication = new AuthenticationMechanism { SymmetricKey = new SymmetricKey { PrimaryKey = CryptoKeyGenerator.GenerateKey(32), SecondaryKey = CryptoKeyGenerator.GenerateKey(32), } }, // This indicates that the entry should be added as a new device. ImportMode = ImportMode.Create, }; // Add device to the list as a serialized object. serializedDevices.Add(JsonConvert.SerializeObject(deviceToAdd)); // Not real progress as you write the new devices, but will at least show *some* progress. interimProgressCount++; totalProgressCount++; if (interimProgressCount >= displayProgressCount) { Console.WriteLine("Added {0} messages.", totalProgressCount); interimProgressCount = 0; } } // Now have a list of devices to be added, each one has been serialized. // Write the list to the blob. var sb = new StringBuilder(); serializedDevices.ForEach(serializedDevice => sb.AppendLine(serializedDevice)); // Before writing the new file, make sure there's not already one there. await generatedListBlob.DeleteIfExistsAsync().ConfigureAwait(false); // Write list of serialized objects to the blob. using CloudBlobStream stream = await generatedListBlob.OpenWriteAsync().ConfigureAwait(false); byte[] bytes = Encoding.UTF8.GetBytes(sb.ToString()); for (var i = 0; i < bytes.Length; i += 500) { int length = Math.Min(bytes.Length - i, 500); await stream.WriteAsync(bytes, i, length).ConfigureAwait(false); } stream.Commit(); Console.WriteLine("Creating and running registry manager job to write the new devices."); // Should now have a file with all the new devices in it as serialized objects in blob storage. // generatedListBlob has the list of devices to be added as serialized objects. // Call import using the blob to add the new devices. // Log information related to the job is written to the same container. // This normally takes 1 minute per 100 devices (according to the docs). // First, initiate an import job. // This reads in the rows from the text file and writes them to IoT Devices. // If you want to add devices from a file, you can create a file and use this to import it. // They have to be in the exact right format. RegistryManager registryManager = RegistryManager.CreateFromConnectionString(hubConnectionString); try { // The first URL is the container to import from; the file must be called devices.txt // The second URL points to the container to write errors to as a block blob. // This lets you import the devices from any file name. Since we wrote the new // devices to [devicesToAdd], need to read the list from there as well. JobProperties importJob = await registryManager .ImportDevicesAsync(containerURI, containerURI, devicesToAdd) .ConfigureAwait(false); // This will catch any errors if something bad happens to interrupt the job. while (true) { importJob = await registryManager.GetJobAsync(importJob.JobId).ConfigureAwait(false); if (importJob.Status == JobStatus.Completed || importJob.Status == JobStatus.Failed || importJob.Status == JobStatus.Cancelled) { // Job has finished executing break; } await Task.Delay(TimeSpan.FromSeconds(5)).ConfigureAwait(false); } } catch (Exception ex) { Debug.Print("exception message {0}", ex.Message); } }
// Create a list of devices using bulk import via storage account public async Task <string> CreateListUsingJobsAsync(IEnumerable <string> deviceIds) { this.instance.InitRequired(); this.log.Info("Starting bulk device creation"); // List of devices var serializedDevices = new List <string>(); foreach (var deviceId in deviceIds) { var device = new ExportImportDevice { Id = deviceId, ImportMode = ImportMode.CreateOrUpdate, Authentication = new AuthenticationMechanism { Type = AuthenticationType.Sas, SymmetricKey = new SymmetricKey { PrimaryKey = this.fixedDeviceKey, SecondaryKey = this.fixedDeviceKey } }, Status = DeviceStatus.Enabled, Tags = new TwinCollection { [SIMULATED_TAG_KEY] = SIMULATED_TAG_VALUE } }; serializedDevices.Add(JsonConvert.SerializeObject(device)); } CloudBlockBlob blob; try { blob = await this.WriteDevicesToBlobAsync(serializedDevices); } catch (Exception e) { this.log.Error("Failed to create blob file required for the device bulk creation job", e); throw new ExternalDependencyException("Failed to create blob file", e); } // Create import job JobProperties job; try { var sasToken = this.GetSasTokenForImportExport(); this.log.Info("Creating job to import devices for bulk creation"); job = await this.registry.ImportDevicesAsync(blob.Container.StorageUri.PrimaryUri.AbsoluteUri + sasToken, blob.Name); this.log.Info("Job to import devices created for bulk creation"); } catch (JobQuotaExceededException e) { this.log.Error("Job quota exceeded, retry later", e); throw new ExternalDependencyException("Job quota exceeded, retry later", e); } catch (Exception e) { this.log.Error("Failed to create device import job for bulk creation", e); throw new ExternalDependencyException("Failed to create device import job for bulk creation", e); } return(job.JobId); }
public async Task RegistryManager_ExportDevices(StorageAuthenticationType storageAuthenticationType) { // arrange StorageContainer storageContainer = null; string deviceId = $"{nameof(RegistryManager_ExportDevices)}-{StorageContainer.GetRandomSuffix(4)}"; var registryManager = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString); _log.WriteLine($"Using deviceId {deviceId}"); try { string containerName = StorageContainer.BuildContainerName(nameof(RegistryManager_ExportDevices)); storageContainer = await StorageContainer .GetInstanceAsync(containerName) .ConfigureAwait(false); _log.WriteLine($"Using container {storageContainer.Uri}"); Uri containerUri = storageAuthenticationType == StorageAuthenticationType.KeyBased ? storageContainer.SasUri : storageContainer.Uri; await registryManager .AddDeviceAsync( new Device(deviceId) { Authentication = new AuthenticationMechanism { Type = AuthenticationType.Sas }, }) .ConfigureAwait(false); // act JobProperties exportJobResponse = null; int tryCount = 0; while (true) { try { exportJobResponse = await registryManager .ExportDevicesAsync( JobProperties.CreateForExportJob( containerUri.ToString(), true, null, storageAuthenticationType)) .ConfigureAwait(false); break; } // Concurrent jobs can be rejected, so implement a retry mechanism to handle conflicts with other tests catch (JobQuotaExceededException) when(++tryCount < MaxIterationWait) { _log.WriteLine($"JobQuotaExceededException... waiting."); await Task.Delay(s_waitDuration).ConfigureAwait(false); continue; } } // wait for job to complete for (int i = 0; i < MaxIterationWait; ++i) { await Task.Delay(s_waitDuration).ConfigureAwait(false); exportJobResponse = await registryManager.GetJobAsync(exportJobResponse.JobId).ConfigureAwait(false); _log.WriteLine($"Job {exportJobResponse.JobId} is {exportJobResponse.Status} with progress {exportJobResponse.Progress}%"); if (!s_incompleteJobs.Contains(exportJobResponse.Status)) { break; } } // assert exportJobResponse.Status.Should().Be(JobStatus.Completed, "Otherwise import failed"); exportJobResponse.FailureReason.Should().BeNullOrEmpty("Otherwise import failed"); string content = await DownloadFileAsync(storageContainer).ConfigureAwait(false); string[] serializedDevices = content.Split(s_newlines, StringSplitOptions.RemoveEmptyEntries); bool foundDeviceInExport = false; foreach (string serializedDeivce in serializedDevices) { // The first line may be a comment to the user, so skip any lines that don't start with a json object initial character: curly brace if (serializedDeivce[0] != '{') { continue; } ExportImportDevice device = JsonConvert.DeserializeObject <ExportImportDevice>(serializedDeivce); if (StringComparer.Ordinal.Equals(device.Id, deviceId)) { _log.WriteLine($"Found device in export as [{serializedDeivce}]"); foundDeviceInExport = true; break; } } foundDeviceInExport.Should().BeTrue("Expected device did not appear in the export"); } finally { try { storageContainer?.Dispose(); await registryManager.RemoveDeviceAsync(deviceId).ConfigureAwait(false); } catch { } } }
private static async Task <bool> HandleDevice(ExportImportDevice device, RegistryManager registryManager, TraceWriter log) { if (device.ImportMode == ImportMode.Delete) { try { await registryManager.RemoveDeviceAsync(device.Id); log.Info($"Device {device.Id} deleted"); return(true); } catch (Exception ex) { log.Error($"Failed to delete device {device.Id}", ex); } } else { // For now we try to create the device without checking if it already exists // add additional logic to update an existing device try { var newDevice = new Device(device.Id) { Status = DeviceStatus.Enabled }; var twin = new Twin { Tags = device.Tags }; if (device.Properties != null) { twin.Properties = new TwinProperties(); if (device.Properties.DesiredProperties != null) { twin.Properties.Desired = new TwinCollection(device.Properties.DesiredProperties.ToJson()); } if (device.Properties.ReportedProperties != null) { twin.Properties.Reported = new TwinCollection(device.Properties.ReportedProperties.ToJson()); } } await registryManager.AddDeviceWithTwinAsync(newDevice, twin); log.Info($"Device {device.Id} created"); return(true); } catch (DeviceAlreadyExistsException) { log.Info($"Device {device.Id} already exists, skipping"); } } return(true); }
private async Task ValidateDevicesAsync( string devicesFileName, StorageContainer storageContainer, Device edge1, Device edge2, Device device) { string devicesContent = await DownloadFileAsync(storageContainer, devicesFileName).ConfigureAwait(false); string[] serializedDevices = devicesContent.Split(s_newlines, StringSplitOptions.RemoveEmptyEntries); bool foundEdge1InExport = false; bool foundEdge2InExport = false; bool foundDeviceInExport = false; foreach (string serializedDevice in serializedDevices) { // The first line may be a comment to the user, so skip any lines that don't start with a json object initial character: curly brace if (serializedDevice[0] != '{') { continue; } if (foundEdge1InExport && foundEdge2InExport && foundDeviceInExport) { // we're done break; } ExportImportDevice exportedDevice = JsonConvert.DeserializeObject <ExportImportDevice>(serializedDevice); if (StringComparer.Ordinal.Equals(exportedDevice.Id, edge1.Id) && exportedDevice.Capabilities.IotEdge) { Logger.Trace($"Found edge1 in export as [{serializedDevice}]"); foundEdge1InExport = true; exportedDevice.DeviceScope.Should().Be(edge1.Scope, "Edges retain their own scope"); continue; } if (StringComparer.Ordinal.Equals(exportedDevice.Id, edge2.Id) && exportedDevice.Capabilities.IotEdge) { Logger.Trace($"Found edge2 in export as [{serializedDevice}]"); foundEdge2InExport = true; exportedDevice.DeviceScope.Should().Be(edge2.Scope, "Edges retain their own scope"); continue; } if (StringComparer.Ordinal.Equals(exportedDevice.Id, device.Id)) { Logger.Trace($"Found device in export as [{serializedDevice}]"); foundDeviceInExport = true; exportedDevice.DeviceScope.Should().Be(edge1.Scope); continue; } } foundEdge1InExport.Should().BeTrue("Expected edge did not appear in the export"); foundEdge2InExport.Should().BeTrue("Expected edge did not appear in the export"); foundDeviceInExport.Should().BeTrue("Expected device did not appear in the export"); }