/// <summary>
/// Generate NumToAdd devices and add them to the hub.
/// To do this, generate each identity.
/// Include authentication keys.
/// Write the device info to a block blob.
/// Import the devices into the identity registry by calling the import job.
/// </summary>
private async Task GenerateAndAddDevices(string hubConnectionString, string containerURI, int NumToAdd, string devicesToAdd)
{
int interimProgressCount = 0;
int displayProgressCount = 1000;
int totalProgressCount = 0;
// generate reference for list of new devices we're going to add, will write list to this blob
CloudBlockBlob generatedListBlob = _cloudBlobContainer.GetBlockBlobReference(devicesToAdd);
// define serializedDevices as a generic list<string>
List <string> serializedDevices = new List <string>(NumToAdd);
for (var i = 1; i <= NumToAdd; i++)
{
// Create device name with this format: Hub_00000000 + a new guid.
// This should be large enough to display the largest number (1 million).
string deviceName = $"Hub_{i:D8}_{Guid.NewGuid()}";
Debug.Print($"device = '{deviceName}'\n");
// Create a new ExportImportDevice.
// CryptoKeyGenerator is in the Microsoft.Azure.Devices.Common namespace.
var deviceToAdd = new ExportImportDevice
{
Id = deviceName,
Status = DeviceStatus.Enabled,
Authentication = new AuthenticationMechanism
{
SymmetricKey = new SymmetricKey
{
PrimaryKey = CryptoKeyGenerator.GenerateKey(32),
SecondaryKey = CryptoKeyGenerator.GenerateKey(32),
}
},
// This indicates that the entry should be added as a new device.
ImportMode = ImportMode.Create,
};
// Add device to the list as a serialized object.
serializedDevices.Add(JsonConvert.SerializeObject(deviceToAdd));
// Not real progress as you write the new devices, but will at least show *some* progress.
interimProgressCount++;
totalProgressCount++;
if (interimProgressCount >= displayProgressCount)
{
Console.WriteLine("Added {0} messages.", totalProgressCount);
interimProgressCount = 0;
}
}
// Now have a list of devices to be added, each one has been serialized.
// Write the list to the blob.
var sb = new StringBuilder();
serializedDevices.ForEach(serializedDevice => sb.AppendLine(serializedDevice));
// Before writing the new file, make sure there's not already one there.
await generatedListBlob.DeleteIfExistsAsync().ConfigureAwait(false);
// Write list of serialized objects to the blob.
using CloudBlobStream stream = await generatedListBlob.OpenWriteAsync().ConfigureAwait(false);
byte[] bytes = Encoding.UTF8.GetBytes(sb.ToString());
for (var i = 0; i < bytes.Length; i += 500)
{
int length = Math.Min(bytes.Length - i, 500);
await stream.WriteAsync(bytes, i, length).ConfigureAwait(false);
}
stream.Commit();
Console.WriteLine("Creating and running registry manager job to write the new devices.");
// Should now have a file with all the new devices in it as serialized objects in blob storage.
// generatedListBlob has the list of devices to be added as serialized objects.
// Call import using the blob to add the new devices.
// Log information related to the job is written to the same container.
// This normally takes 1 minute per 100 devices (according to the docs).
// First, initiate an import job.
// This reads in the rows from the text file and writes them to IoT Devices.
// If you want to add devices from a file, you can create a file and use this to import it.
// They have to be in the exact right format.
RegistryManager registryManager = RegistryManager.CreateFromConnectionString(hubConnectionString);
try
{
// The first URL is the container to import from; the file must be called devices.txt
// The second URL points to the container to write errors to as a block blob.
// This lets you import the devices from any file name. Since we wrote the new
// devices to [devicesToAdd], need to read the list from there as well.
JobProperties importJob = await registryManager
.ImportDevicesAsync(containerURI, containerURI, devicesToAdd)
.ConfigureAwait(false);
// This will catch any errors if something bad happens to interrupt the job.
while (true)
{
importJob = await registryManager.GetJobAsync(importJob.JobId).ConfigureAwait(false);
if (importJob.Status == JobStatus.Completed ||
importJob.Status == JobStatus.Failed ||
importJob.Status == JobStatus.Cancelled)
{
// Job has finished executing
break;
}
await Task.Delay(TimeSpan.FromSeconds(5)).ConfigureAwait(false);
}
}
catch (Exception ex)
{
Debug.Print("exception message {0}", ex.Message);
}
}
// Create a list of devices using bulk import via storage account
public async Task <string> CreateListUsingJobsAsync(IEnumerable <string> deviceIds)
{
this.instance.InitRequired();
this.log.Info("Starting bulk device creation");
// List of devices
var serializedDevices = new List <string>();
foreach (var deviceId in deviceIds)
{
var device = new ExportImportDevice
{
Id = deviceId,
ImportMode = ImportMode.CreateOrUpdate,
Authentication = new AuthenticationMechanism
{
Type = AuthenticationType.Sas,
SymmetricKey = new SymmetricKey
{
PrimaryKey = this.fixedDeviceKey,
SecondaryKey = this.fixedDeviceKey
}
},
Status = DeviceStatus.Enabled,
Tags = new TwinCollection {
[SIMULATED_TAG_KEY] = SIMULATED_TAG_VALUE
}
};
serializedDevices.Add(JsonConvert.SerializeObject(device));
}
CloudBlockBlob blob;
try
{
blob = await this.WriteDevicesToBlobAsync(serializedDevices);
}
catch (Exception e)
{
this.log.Error("Failed to create blob file required for the device bulk creation job", e);
throw new ExternalDependencyException("Failed to create blob file", e);
}
// Create import job
JobProperties job;
try
{
var sasToken = this.GetSasTokenForImportExport();
this.log.Info("Creating job to import devices for bulk creation");
job = await this.registry.ImportDevicesAsync(blob.Container.StorageUri.PrimaryUri.AbsoluteUri + sasToken, blob.Name);
this.log.Info("Job to import devices created for bulk creation");
}
catch (JobQuotaExceededException e)
{
this.log.Error("Job quota exceeded, retry later", e);
throw new ExternalDependencyException("Job quota exceeded, retry later", e);
}
catch (Exception e)
{
this.log.Error("Failed to create device import job for bulk creation", e);
throw new ExternalDependencyException("Failed to create device import job for bulk creation", e);
}
return(job.JobId);
}
public async Task RegistryManager_ExportDevices(StorageAuthenticationType storageAuthenticationType)
{
// arrange
StorageContainer storageContainer = null;
string deviceId = $"{nameof(RegistryManager_ExportDevices)}-{StorageContainer.GetRandomSuffix(4)}";
var registryManager = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);
_log.WriteLine($"Using deviceId {deviceId}");
try
{
string containerName = StorageContainer.BuildContainerName(nameof(RegistryManager_ExportDevices));
storageContainer = await StorageContainer
.GetInstanceAsync(containerName)
.ConfigureAwait(false);
_log.WriteLine($"Using container {storageContainer.Uri}");
Uri containerUri = storageAuthenticationType == StorageAuthenticationType.KeyBased
? storageContainer.SasUri
: storageContainer.Uri;
await registryManager
.AddDeviceAsync(
new Device(deviceId)
{
Authentication = new AuthenticationMechanism {
Type = AuthenticationType.Sas
},
})
.ConfigureAwait(false);
// act
JobProperties exportJobResponse = null;
int tryCount = 0;
while (true)
{
try
{
exportJobResponse = await registryManager
.ExportDevicesAsync(
JobProperties.CreateForExportJob(
containerUri.ToString(),
true,
null,
storageAuthenticationType))
.ConfigureAwait(false);
break;
}
// Concurrent jobs can be rejected, so implement a retry mechanism to handle conflicts with other tests
catch (JobQuotaExceededException) when(++tryCount < MaxIterationWait)
{
_log.WriteLine($"JobQuotaExceededException... waiting.");
await Task.Delay(s_waitDuration).ConfigureAwait(false);
continue;
}
}
// wait for job to complete
for (int i = 0; i < MaxIterationWait; ++i)
{
await Task.Delay(s_waitDuration).ConfigureAwait(false);
exportJobResponse = await registryManager.GetJobAsync(exportJobResponse.JobId).ConfigureAwait(false);
_log.WriteLine($"Job {exportJobResponse.JobId} is {exportJobResponse.Status} with progress {exportJobResponse.Progress}%");
if (!s_incompleteJobs.Contains(exportJobResponse.Status))
{
break;
}
}
// assert
exportJobResponse.Status.Should().Be(JobStatus.Completed, "Otherwise import failed");
exportJobResponse.FailureReason.Should().BeNullOrEmpty("Otherwise import failed");
string content = await DownloadFileAsync(storageContainer).ConfigureAwait(false);
string[] serializedDevices = content.Split(s_newlines, StringSplitOptions.RemoveEmptyEntries);
bool foundDeviceInExport = false;
foreach (string serializedDeivce in serializedDevices)
{
// The first line may be a comment to the user, so skip any lines that don't start with a json object initial character: curly brace
if (serializedDeivce[0] != '{')
{
continue;
}
ExportImportDevice device = JsonConvert.DeserializeObject <ExportImportDevice>(serializedDeivce);
if (StringComparer.Ordinal.Equals(device.Id, deviceId))
{
_log.WriteLine($"Found device in export as [{serializedDeivce}]");
foundDeviceInExport = true;
break;
}
}
foundDeviceInExport.Should().BeTrue("Expected device did not appear in the export");
}
finally
{
try
{
storageContainer?.Dispose();
await registryManager.RemoveDeviceAsync(deviceId).ConfigureAwait(false);
}
catch { }
}
}
private static async Task <bool> HandleDevice(ExportImportDevice device, RegistryManager registryManager, TraceWriter log)
{
if (device.ImportMode == ImportMode.Delete)
{
try
{
await registryManager.RemoveDeviceAsync(device.Id);
log.Info($"Device {device.Id} deleted");
return(true);
}
catch (Exception ex)
{
log.Error($"Failed to delete device {device.Id}", ex);
}
}
else
{
// For now we try to create the device without checking if it already exists
// add additional logic to update an existing device
try
{
var newDevice = new Device(device.Id)
{
Status = DeviceStatus.Enabled
};
var twin = new Twin
{
Tags = device.Tags
};
if (device.Properties != null)
{
twin.Properties = new TwinProperties();
if (device.Properties.DesiredProperties != null)
{
twin.Properties.Desired = new TwinCollection(device.Properties.DesiredProperties.ToJson());
}
if (device.Properties.ReportedProperties != null)
{
twin.Properties.Reported = new TwinCollection(device.Properties.ReportedProperties.ToJson());
}
}
await registryManager.AddDeviceWithTwinAsync(newDevice, twin);
log.Info($"Device {device.Id} created");
return(true);
}
catch (DeviceAlreadyExistsException)
{
log.Info($"Device {device.Id} already exists, skipping");
}
}
return(true);
}
private async Task ValidateDevicesAsync(
string devicesFileName,
StorageContainer storageContainer,
Device edge1,
Device edge2,
Device device)
{
string devicesContent = await DownloadFileAsync(storageContainer, devicesFileName).ConfigureAwait(false);
string[] serializedDevices = devicesContent.Split(s_newlines, StringSplitOptions.RemoveEmptyEntries);
bool foundEdge1InExport = false;
bool foundEdge2InExport = false;
bool foundDeviceInExport = false;
foreach (string serializedDevice in serializedDevices)
{
// The first line may be a comment to the user, so skip any lines that don't start with a json object initial character: curly brace
if (serializedDevice[0] != '{')
{
continue;
}
if (foundEdge1InExport &&
foundEdge2InExport &&
foundDeviceInExport)
{
// we're done
break;
}
ExportImportDevice exportedDevice = JsonConvert.DeserializeObject <ExportImportDevice>(serializedDevice);
if (StringComparer.Ordinal.Equals(exportedDevice.Id, edge1.Id) && exportedDevice.Capabilities.IotEdge)
{
Logger.Trace($"Found edge1 in export as [{serializedDevice}]");
foundEdge1InExport = true;
exportedDevice.DeviceScope.Should().Be(edge1.Scope, "Edges retain their own scope");
continue;
}
if (StringComparer.Ordinal.Equals(exportedDevice.Id, edge2.Id) && exportedDevice.Capabilities.IotEdge)
{
Logger.Trace($"Found edge2 in export as [{serializedDevice}]");
foundEdge2InExport = true;
exportedDevice.DeviceScope.Should().Be(edge2.Scope, "Edges retain their own scope");
continue;
}
if (StringComparer.Ordinal.Equals(exportedDevice.Id, device.Id))
{
Logger.Trace($"Found device in export as [{serializedDevice}]");
foundDeviceInExport = true;
exportedDevice.DeviceScope.Should().Be(edge1.Scope);
continue;
}
}
foundEdge1InExport.Should().BeTrue("Expected edge did not appear in the export");
foundEdge2InExport.Should().BeTrue("Expected edge did not appear in the export");
foundDeviceInExport.Should().BeTrue("Expected device did not appear in the export");
}