private HashSet <string> GetDevicesInQuery(string hubQuery, string deploymentId, string tenantId)
{
var query = string.Format(hubQuery, deploymentId);
var queryResponse = TenantConnectionHelper.GetRegistry(tenantId).CreateQuery(query);
var deviceIds = new HashSet <string>();
try
{
while (queryResponse.HasMoreResults)
{
// TODO: Add pagination with queryOptions
var resultSet = queryResponse.GetNextAsJsonAsync();
foreach (var result in resultSet.Result)
{
var deviceId = JToken.Parse(result)[DeviceIdKey];
deviceIds.Add(deviceId.ToString());
}
}
}
catch (Exception)
{
// this.logger.LogError(ex, "Error getting status of devices in query {query}", query);
}
return(deviceIds);
}
private async Task <ResultWithContinuationToken <List <Twin> > > GetTwinByQueryAsync(
string tenantId,
string queryPrefix,
string query,
string continuationToken,
int numberOfResult)
{
query = string.IsNullOrEmpty(query) ? queryPrefix : $"{queryPrefix} where {query}";
var twins = new List <Twin>();
var twinQuery = TenantConnectionHelper.GetRegistry(tenantId).CreateQuery(query);
QueryOptions options = new QueryOptions();
options.ContinuationToken = continuationToken;
while (twinQuery.HasMoreResults && twins.Count < numberOfResult)
{
var response = await twinQuery.GetNextAsTwinAsync(options);
options.ContinuationToken = response.ContinuationToken;
twins.AddRange(response);
}
return(new ResultWithContinuationToken <List <Twin> >(twins, options.ContinuationToken));
}
public DeviceGroupMigration(TableStorageHelper tableStorageClient, TenantConnectionHelper tenantConnectionHelper, ILogger logger, AppConfig appConfig)
{
this.tableStorageClient = tableStorageClient;
this.tenantConnectionHelper = tenantConnectionHelper;
this.logger = logger;
this.appConfig = appConfig;
}
private static async Task Main(string[] args)
{
var appConfig = InitializeAppConfig();
ILogger logger = LoggerSetup();
try
{
AppConfigHelper appConfigHelper = new AppConfigHelper(appConfig.AppConfigurationConnectionString);
TableStorageHelper tableStorageHelper = new TableStorageHelper(appConfig);
TenantConnectionHelper tenantConnectionHelper = new TenantConnectionHelper(appConfigHelper);
DeviceGroupMigration deviceGroupMigration = new DeviceGroupMigration(tableStorageHelper, tenantConnectionHelper, logger, appConfig);
logger.LogInformation("Device Group Migration Started");
await deviceGroupMigration.Start();
logger.LogInformation("Device Group Migration Completed");
DeviceTwinMigration deviceTwinMigration = new DeviceTwinMigration(tableStorageHelper, tenantConnectionHelper, logger);
logger.LogInformation("Device Twin Migration Started");
await deviceTwinMigration.Start();
logger.LogInformation("Device Twin Migration Completed");
}
catch (System.Exception ex)
{
logger.LogError(ex, "Migration Failed");
}
}
public async Task ProcessDeviceTwin(EventData[] source, ILogger log, string tenant, string deviceId)
{
foreach (EventData message in source)
{
try
{
if (tenant != null)
{
var connectionString = TenantConnectionHelper.GetEventHubConnectionString(Convert.ToString(tenant));
EventHubHelper eventHubHelper = new EventHubHelper(connectionString);
string eventData = Encoding.UTF8.GetString(message.Body.Array);
message.Properties.TryGetValue("opType", out object operationType);
message.SystemProperties.TryGetValue(DeviceTelemetryKeyConstants.IotHubEnqueuedTime, out object dateTimeReceived);
var timeStamp = new TelemetryTimestamp(Convert.ToDateTime(dateTimeReceived));
DeviceService deviceService = new DeviceService();
await deviceService.SaveDeviceTwinOperationAsync(eventData, log, Convert.ToString(tenant), deviceId.ToString(), operationType.ToString(), timeStamp, eventHubHelper);
}
}
catch (Exception ex)
{
log.LogError($"Error occurrred in for loop: {ex.Message} StackTrace: {ex.StackTrace} Inner Exception: {(string.IsNullOrEmpty(ex.StackTrace) ? string.Empty : ex.StackTrace)}");
}
}
}
public static async Task Run([EventHubTrigger(eventHubName: "twin-change", Connection = "TwinChangeEventHubConnectionString", ConsumerGroup = "%DeviceDeploymentHistoryConsumerGroup%")] EventData[] events, ILogger log)
{
bool exceptionOccurred = false;
List <Task> list = new List <Task>();
foreach (EventData message in events)
{
try
{
message.Properties.TryGetValue("tenant", out object tenant);
if (tenant != null)
{
string eventData = Encoding.UTF8.GetString(message.Body.Array);
Twin twin = JsonConvert.DeserializeObject <Twin>(eventData);
TwinServiceModel twinServiceModel = new TwinServiceModel(twin);
Dictionary <string, JToken> reportedProperties = twinServiceModel.ReportedProperties;
var json = JToken.Parse(JsonConvert.SerializeObject(reportedProperties));
var fieldsCollector = new JsonFieldsCollector(json);
var fields = fieldsCollector.GetAllFields();
bool isFirmWareUpdate = false;
foreach (var field in fields)
{
isFirmWareUpdate = field.Key.Contains(FIRMWARE, StringComparison.OrdinalIgnoreCase);
if (isFirmWareUpdate)
{
break;
}
}
if (isFirmWareUpdate)
{
message.SystemProperties.TryGetValue("iothub-connection-device-id", out object deviceId);
var newTwin = await TenantConnectionHelper.GetRegistry(Convert.ToString(tenant)).GetTwinAsync(deviceId.ToString());
var appliedConfigurations = newTwin.Configurations.Where(c => c.Value.Status.Equals(ConfigurationStatus.Applied));
if (appliedConfigurations != null && appliedConfigurations.Count() > 0)
{
DeploymentSyncService service = new DeploymentSyncService();
var appliedDeploymentFromStorage = service.GetDeploymentsByIdFromStorage(Convert.ToString(tenant), appliedConfigurations.Select(ac => ac.Key).ToArray()).Result.FirstOrDefault();
await service.SaveDeploymentHistory(Convert.ToString(tenant), appliedDeploymentFromStorage, newTwin);
}
}
}
}
catch (Exception ex)
{
log.LogError($"Error occurrred : {ex.Message} StackTrace: {ex.StackTrace} Inner Exception: {(string.IsNullOrEmpty(ex.StackTrace) ? string.Empty : ex.StackTrace)}");
exceptionOccurred = true;
}
}
if (exceptionOccurred)
{
throw new Exception("Function Failed with exception");
}
}
public static async Task Run([QueueTrigger("tenantstosync", Connection = "AzureStorageConnectionString")] string myQueueItem, ILogger log)
{
log.LogInformation($"C# Queue trigger function processed: {myQueueItem}");
if (!string.IsNullOrEmpty(myQueueItem))
{
TenantQueueItem tenant = JsonConvert.DeserializeObject <TenantQueueItem>(myQueueItem);
if (tenant != null && !string.IsNullOrWhiteSpace(tenant.TenantId))
{
List <DeploymentServiceModel> deploymentsToSync = new List <DeploymentServiceModel>();
IEnumerable <Configuration> deploymentsFromHub = null;
try
{
deploymentsFromHub = await TenantConnectionHelper.GetRegistry(tenant.TenantId).GetConfigurationsAsync(100);
DeploymentSyncService service = new DeploymentSyncService();
deploymentsToSync.AddRange(await service.GetDeploymentsToSync(tenant.TenantId, deploymentsFromHub));
}
catch (Exception)
{
log.LogError($"Error occurrred while fetching deployments");
throw;
}
if (deploymentsToSync != null && deploymentsToSync.Count > 0)
{
// Get the connection string from app settings
string connectionString = Environment.GetEnvironmentVariable("AzureStorageConnectionString", EnvironmentVariableTarget.Process);
// Instantiate a QueueClient which will be used to create and manipulate the queue
QueueClient queueClient = new QueueClient(connectionString, "deploymentstosync");
await queueClient.CreateIfNotExistsAsync();
if (queueClient.Exists())
{
foreach (var deploymentToSync in deploymentsToSync)
{
DeploymentModel deployment = new DeploymentModel();
deployment.TenantId = tenant.TenantId;
deployment.Deployment = deploymentToSync;
deployment.Configuration = deploymentsFromHub.FirstOrDefault(d => d.Id == deploymentToSync.Id);
var deploymentToSyncString = JsonConvert.SerializeObject(deployment);
queueClient.SendMessage(Base64Encode(deploymentToSyncString));
}
}
}
}
}
}
public Storage(
IStorageAdapterClient client,
IAsaManagerClient asaManager,
AppConfig config,
IPackageEventLog packageLog,
ILogger <Storage> logger,
IHttpContextAccessor httpContextAccessor,
IAppConfigurationClient appConfigurationClient)
{
this.client = client;
this.asaManager = asaManager;
this.config = config;
this.packageLog = packageLog;
this.logger = logger;
this.tenantConnectionHelper = new TenantConnectionHelper(appConfigurationClient);
this.httpContextAccessor = httpContextAccessor;
}
public DeviceTwinMigration(TableStorageHelper tableStorageClient, TenantConnectionHelper tenantConnectionHelper, ILogger logger)
{
this.tableStorageClient = tableStorageClient;
this.tenantConnectionHelper = tenantConnectionHelper;
this.logger = logger;
}
public async Task SaveDeviceTwinOperationAsync(string eventData, ILogger log, string tenant, string deviceId, string operationType, TelemetryTimestamp timeStamp, EventHubHelper eventHubHelper)
{
try
{
string deviceTwin = eventData;
Twin twin = null;
JObject deviceTwinJson = new JObject();
deviceTwinJson.Add(DeviceTelemetryKeyConstants.DeviceId, deviceId.ToString());
deviceTwinJson.Add(DeviceTelemetryKeyConstants.TimeStamp, timeStamp.DateTime);
deviceTwinJson.Add(DeviceTelemetryKeyConstants.TimeReceived, timeStamp.EpochTimestamp);
deviceTwinJson.Add(DeviceTelemetryKeyConstants.EventOpType, operationType);
deviceTwinJson.Add(DeviceTelemetryKeyConstants.IsDeleted, false);
if (operationType.ToString().Equals("createDeviceIdentity"))
{
deviceTwinJson.Add(DeviceTelemetryKeyConstants.DeviceCreatedDate, timeStamp.DateTime);
try
{
twin = await TenantConnectionHelper.GetRegistry(Convert.ToString(tenant)).GetTwinAsync(deviceId.ToString());
deviceTwin = twin.ToJson();
}
catch (Exception e)
{
log.LogError($"Unable to fetch DeviceId: {deviceId} device twin from iothub of tenant: {tenant}.", e);
}
}
else
{
JObject previousTwin = null;
KustoOperations kustoClient = await KustoOperations.GetClientAsync();
string kustoQuery = $"DeviceTwin | where DeviceId == \"{deviceId}\" | summarize arg_max(TimeStamp, *) by DeviceId | where IsDeleted == false";
var deviceTwinList = await kustoClient.QueryAsync <DeviceTwinModel>($"IoT-{tenant}", kustoQuery, null);
DeviceTwinModel preDeviceTwin = deviceTwinList.FirstOrDefault();
if (preDeviceTwin != null)
{
previousTwin = preDeviceTwin.Twin;
deviceTwinJson.Add(DeviceTelemetryKeyConstants.DeviceCreatedDate, preDeviceTwin.DeviceCreatedDate);
}
else
{
deviceTwinJson.Add(DeviceTelemetryKeyConstants.DeviceCreatedDate, default(DateTime)); // Set Device Created Date to Default if twin is not present in storage.
try
{
twin = await TenantConnectionHelper.GetRegistry(Convert.ToString(tenant)).GetTwinAsync(deviceId.ToString());
if (twin != null)
{
previousTwin = JObject.Parse(twin.ToJson());
}
}
catch (Exception e)
{
log.LogError($"Unable to fetch DeviceId: {deviceId} device twin from iothub of tenant: {tenant}.", e);
}
}
switch (operationType)
{
case "deviceConnected":
if (preDeviceTwin != null)
{
previousTwin["connectionState"] = "Connected";
previousTwin["lastActivityTime"] = timeStamp.DateTime;
}
break;
case "deviceDisconnected":
if (preDeviceTwin != null)
{
previousTwin["connectionState"] = "Disconnected";
previousTwin["lastActivityTime"] = timeStamp.DateTime;
}
break;
case "updateTwin":
if (preDeviceTwin != null)
{
JObject twinFragment = JObject.Parse(eventData);
previousTwin = previousTwin.UpdateJson(twinFragment);
}
else
{
previousTwin = JObject.Parse(eventData);
}
break;
case "deleteDeviceIdentity":
deviceTwinJson[DeviceTelemetryKeyConstants.IsDeleted] = true;
break;
default:
break;
}
deviceTwin = previousTwin?.ToString();
}
deviceTwinJson.Add(DeviceTelemetryKeyConstants.Data, deviceTwin);
// Save the Device Twin Data to EventHub for further processing
var byteMessage = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(deviceTwinJson));
var eventDeviceTwinData = new Azure.Messaging.EventHubs.EventData(byteMessage);
eventDeviceTwinData.Properties.Add("deviceid", deviceId.ToString());
await eventHubHelper.SendMessageToEventHub($"{tenant}-devicetwin", new Azure.Messaging.EventHubs.EventData[] { eventDeviceTwinData });
}
catch (Exception exception)
{
throw new ApplicationException($"Save Device Twin operation failed: {exception}, operation type: {operationType}, tenant: {tenant}, deviceId {deviceId}");
}
}
