private TelemetryTimestamp ComputeBatchedEnqueuedTime( TelemetryTimestamp reportedTimestamp, TelemetryTimestamp currentTimestamp, TelemetryTimestamp finalTimestamp) { // compute the delay from the final timestamp vs the iothub reported timestamp (IotHubEnqueuedTime) long offset = reportedTimestamp.EpochTimestamp - finalTimestamp.EpochTimestamp; // the computed time stamp takes the expected delay from the reportedTimestamp plus the actual timestamp from data return(new TelemetryTimestamp( currentTimestamp.EpochTimestamp + offset, currentTimestamp.EpochTimestampMeasurement)); }
private async Task ProcessBatchedTelemetryMessagesAsync( EventData[] batchedSource, EventHubHelper eventHubHelper, ILogger log, object tenant, object deviceId, TelemetryTimestamp telemetryTimestamp, int batchThreshold, int batchWriteDelay) { // get the final data point for timestamp calculation // if batchedSource is empty set to null, as it's value will never be used. var finalDataPoint = batchedSource.Length > 0 ? batchedSource.Last() : null; var finalTelemetry = finalDataPoint != null?this.ConvertEventDataToJObject(finalDataPoint, log) : null; var finalTimestamp = this.GetBatchedDataPointTimestamp(finalTelemetry, telemetryTimestamp); await Task.WhenAll( batchedSource.Select((dataPoint, idx) => { return(Task.Run(async() => { // apply some delay as the batch size grows await Task.Delay(idx / batchThreshold *batchWriteDelay); var telemetry = this.ConvertEventDataToJObject(dataPoint, log); var dataPointTimestamp = this.GetBatchedDataPointTimestamp( telemetry, telemetryTimestamp); var computedOffsetTimestamp = this.ComputeBatchedEnqueuedTime( telemetryTimestamp, dataPointTimestamp, finalTimestamp); await this.ProcessTelemetryMessageAndWriteToEventHub( telemetry, eventHubHelper, log, tenant, deviceId, computedOffsetTimestamp); })); })); }
private async Task ProcessTelemetryMessageAndWriteToEventHub( JObject telemetry, EventHubHelper eventHubHelper, ILogger log, object tenant, object deviceId, TelemetryTimestamp telemetryTimestamp) { try { if (telemetry == null || !telemetry.HasValues) { throw new Exception("Telemetry message had no values to write to cosmos"); } /* * telemetry[DeviceTelemetryKeyConstants.DeviceId] = deviceId.ToString(); * * telemetry.Add(DeviceTelemetryKeyConstants.DateTimeReceived, telemetryTimestamp.DateTime.ToString()); * telemetry.Add(DeviceTelemetryKeyConstants.TimeReceived, telemetryTimestamp.EpochTimestamp); * telemetry.Add(DeviceTelemetryKeyConstants.Schema, DeviceTelemetryKeyConstants.MessageSchema); */ JObject telemetryData = new JObject(); telemetryData.Add(DeviceTelemetryKeyConstants.Data, telemetry); telemetryData.Add(DeviceTelemetryKeyConstants.DeviceId, deviceId.ToString()); telemetryData.Add(DeviceTelemetryKeyConstants.DateTimeReceived, telemetryTimestamp.DateTime); telemetryData.Add(DeviceTelemetryKeyConstants.TimeReceived, telemetryTimestamp.EpochTimestamp); telemetryData.Add(DeviceTelemetryKeyConstants.Schema, DeviceTelemetryKeyConstants.MessageSchema); // Save the telemetry message to EventHub for further processing var byteMessage = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(telemetryData)); var eventData = new Azure.Messaging.EventHubs.EventData(byteMessage); eventData.Properties.Add("deviceid", deviceId.ToString()); await eventHubHelper.SendMessageToEventHub($"{tenant}-telemetry", new Azure.Messaging.EventHubs.EventData[] { eventData }); } catch (Exception ex) { this.success = false; log.LogError($"Error occurrred : {ex.Message}"); } }
private TelemetryTimestamp GetBatchedDataPointTimestamp(JObject dataPoint, TelemetryTimestamp defaultTimestamp) { bool dataPointHasTimestamp = dataPoint.TryGetValue( DeviceTelemetryKeyConstants.BatchedDataPointTimestampKey, StringComparison.InvariantCultureIgnoreCase, out JToken dataPointTimestampToken); bool dataPointHasTimestampMeasurement = dataPoint.TryGetValue( DeviceTelemetryKeyConstants.BatchedDataPointTimestampMeasurementKey, StringComparison.InvariantCultureIgnoreCase, out JToken dataPointTimestampMeasurementToken); // take the value given in the dataPoint, otherwise default to seconds if the dataPoint has a value var timestampMeasurement = dataPointHasTimestampMeasurement ? dataPointTimestampMeasurementToken.Value <TimestampMeasurement>() : dataPointHasTimestamp ? TimestampMeasurement.S : defaultTimestamp.EpochTimestampMeasurement; return(dataPointHasTimestamp ? new TelemetryTimestamp( dataPointTimestampToken?.Value <long>() ?? defaultTimestamp.EpochTimestamp, timestampMeasurement) : defaultTimestamp); }
public async Task ProcessTelemetryAsync(EventData[] source, ILogger log, AppConfigHelper configHelper, int batchThreshold = 12, int batchWriteDelay = 85) { if (batchThreshold <= 0) { throw new ArgumentOutOfRangeException("batchThreshold must be greater than 0"); } List <Task> messagingTasks = new List <Task>(); // iterate over the eventData array and add a task to process each element foreach (EventData eventData in source) { // get the iothub-enqueuedtime, iothub-connection-device-id, and tenent from the eventData system properties // these values are to be added to the telemetry message before being stored in cosmos (tenent is used to save telemetry to cosmos) eventData.SystemProperties.TryGetValue(DeviceTelemetryKeyConstants.IotHubEnqueuedTime, out object dateTimeReceived); eventData.SystemProperties.TryGetValue(DeviceTelemetryKeyConstants.IotHubConnectionDeviceId, out object deviceId); eventData.Properties.TryGetValue(DeviceTelemetryKeyConstants.Tenant, out object tenant); if (tenant == null || dateTimeReceived == null || deviceId == null) { this.success = false; log.LogError($"EventData did not contain one or more of the following: {DeviceTelemetryKeyConstants.Tenant}, {DeviceTelemetryKeyConstants.IotHubEnqueuedTime}, {DeviceTelemetryKeyConstants.IotHubEnqueuedTime}."); continue; // go on to the next event data - this one cannot be written } var telemetryTimestamp = new TelemetryTimestamp(Convert.ToDateTime(dateTimeReceived)); var connectionString = await configHelper.GetValueByKey($"tenant:{tenant}:eventHubConn"); EventHubHelper eventHubHelper = new EventHubHelper(connectionString); bool isBatchedTelemetry = false; if (eventData.Properties.TryGetValue(DeviceTelemetryKeyConstants.BatchedTelemetry, out object isBatchedValue)) { bool.TryParse(isBatchedValue.ToString(), out isBatchedTelemetry); } try { if (isBatchedTelemetry) { messagingTasks.Add( Task.Run(async() => { var batchedEventData = this.ConvertBatchedDataToKeyValueData(eventData, log); await this.ProcessBatchedTelemetryMessagesAsync( batchedEventData, eventHubHelper, log, tenant, deviceId, telemetryTimestamp, batchThreshold, batchWriteDelay); })); } else { messagingTasks.Add(Task.Run(async() => { var telemetry = this.ConvertEventDataToJObject(eventData, log); await this.ProcessTelemetryMessageAndWriteToEventHub( telemetry, eventHubHelper, log, tenant, deviceId, telemetryTimestamp); })); } } catch (Exception) { this.success = false; log.LogError($"An error occurred while writing some eventData to cosmos"); } } await Task.WhenAll(messagingTasks.ToArray()); if (!this.success) { throw new Exception("Failed to process one or more telemetry messages."); } }