public static async Task RunAsync(
[EventHubTrigger("%EventHubName%", Connection = "EventHubConnectionString", ConsumerGroup = "%ConsumerGroup%")]
EventData[] messages,
ExecutionContext context,
TraceWriter log)
{
CustomTelemetry.TrackMetric(context, "IoTHubMessagesReceived", messages.Length);
await Task.Delay(0);
var ticksUTCNow = DateTimeOffset.UtcNow;
var cutoffTime = DateTimeOffset.UtcNow.AddMinutes(-5);
// Track whether messages are arriving at the function late.
DateTime?firstMsgEnqueuedTicksUtc = messages[0]?.EnqueuedTimeUtc;
if (firstMsgEnqueuedTicksUtc.HasValue)
{
CustomTelemetry.TrackMetric(
context,
"IoTHubMessagesReceivedFreshnessMsec",
(ticksUTCNow - firstMsgEnqueuedTicksUtc.Value).TotalMilliseconds);
}
int count = 0;
int droppedMessages = 0;
var batchStatement = new BatchStatement();
batchStatement.SetBatchType(BatchType.Unlogged);
foreach (var message in messages)
{
// Drop stale messages,
if (message.EnqueuedTimeUtc < cutoffTime)
{
log.Info($"Dropping late message batch. Enqueued time = {message.EnqueuedTimeUtc}, Cutoff = {cutoffTime}");
droppedMessages++;
continue;
}
var text = Encoding.UTF8.GetString(message.GetBytes());
log.Info($"Process message: {text}");
try
{
dynamic telemetry = JObject.Parse(text);
if (telemetry.sensorType == DroneSensorEventType)
{
string position = telemetry.position;
var(latitude, longitude) = DroneTelemetryConverter.ConvertPosition(position);
string deviceId = telemetry.deviceId;
var statementAdd = new SimpleStatement($"INSERT INTO {tableName} (device_id, location, event_time) VALUES (?, ?, ?) USING TTL 259200",
deviceId, new Point(longitude, latitude), new DateTimeOffset(message.EnqueuedTimeUtc));
batchStatement.Add(statementAdd);
count++;
}
}
catch (Exception ex)
{
log.Error("Error processing message", ex);
}
}
try
{
await session.ExecuteAsync(batchStatement);
log.Info("Successfully written batch to cassandra");
CustomTelemetry.TrackMetric(
context,
"IoTHubMessagesDropped",
droppedMessages);
CustomTelemetry.TrackMetric(
context,
"CassandraDocumentsCreated",
count);
}
catch (Exception ex)
{
log.Error("Error processing batch of messages", ex);
}
}
private async Task UpsertDocuments(
int taskId,
IEnumerable <EventData> docsToUpsert,
DateTimeOffset cutoffTime,
TraceWriter log)
{
var cosmosDbLatency = new Stopwatch();
int count = 0;
int droppedMessages = 0;
foreach (var message in docsToUpsert)
{
// Drop stale messages,
if (message.EnqueuedTimeUtc < cutoffTime)
{
log.Info($"Dropping late message batch. Enqueued time = {message.EnqueuedTimeUtc}, Cutoff = {cutoffTime}");
droppedMessages++;
continue;
}
var text = Encoding.UTF8.GetString(message.GetBytes());
log.Info($"Process message: {text}");
try
{
dynamic telemetry = JObject.Parse(text);
if (telemetry.sensorType == DroneSensorEventType)
{
string position = telemetry.position;
var(latitude, longitude) = DroneTelemetryConverter
.ConvertPosition(position);
cosmosDbLatency.Start();
await Client.UpsertDocumentAsync(
CollectionLink,
new
{
id = telemetry.deviceId,
deviceId = telemetry.deviceId,
Location = new Point(longitude, latitude),
Timestamp = message.EnqueuedTimeUtc
});
cosmosDbLatency.Stop();
count++;
}
}
catch (Exception e)
{
if (e is DocumentClientException documentClientEx)
{
log.Error($"Error processing message with status code {documentClientEx.StatusCode}. Exception was {documentClientEx.Message}");
}
else
{
log.Error($"Error processing message. Exception was {e.Message}");
}
}
finally
{
Interlocked.Add(ref this.UpsertedDocuments, count);
Interlocked.Add(ref this.DroppedMessages, droppedMessages);
Interlocked.Add(ref this.CosmosDbTotalMilliseconds,
(long)cosmosDbLatency
.Elapsed
.TotalMilliseconds);
}
}
}
private static async Task <long> BulkLoadEvents(
IEnumerable <EventData> docsToUpsert,
TraceWriter log)
{
// Define retry logic for reliable database operations
RetryStrategy retryStrategy = new FixedInterval(3, TimeSpan.FromSeconds(10));
RetryPolicy retryPolicy = new RetryPolicy <SqlDatabaseTransientErrorDetectionStrategy>(retryStrategy);
// Define data structure that will load events into database
DataTable dt = new DataTable();
dt.Columns.Add("deviceid", typeof(string));
dt.Columns.Add("timestamp", typeof(DateTime));
dt.Columns.Add("geo", typeof(string));
dt.Columns.Add("json", typeof(string));
var sqlDbLatency = new Stopwatch();
// for each message read from IoTHub
foreach (var message in docsToUpsert)
{
var text = Encoding.UTF8.GetString(message.GetBytes());
// Create a new row
DataRow dr = dt.NewRow();
// Parse telemetry message
dynamic telemetry = JObject.Parse(text);
if (telemetry.sensorType == DroneSensorEventType)
{
// Convert position
string position = telemetry.position;
var(latitude, longitude) = DroneTelemetryConverter.ConvertPosition(position);
// Conver to WKT format
string geo = string.Format($"POINT ({longitude} {latitude})");
dr["deviceid"] = telemetry.deviceId;
dr["timestamp"] = message.EnqueuedTimeUtc;
dr["geo"] = geo;
dr["json"] = text;
dt.Rows.Add(dr);
}
}
try
{
sqlDbLatency.Start();
await retryPolicy.ExecuteAsync(async() =>
{
using (SqlConnection cnn = new SqlConnection(SQLDBConnectionString))
{
cnn.Open();
SqlBulkCopy bc = new SqlBulkCopy(cnn);
bc.BatchSize = 10000;
bc.DestinationTableName = "events";
await bc.WriteToServerAsync(dt);
}
});
sqlDbLatency.Stop();
}
catch (SqlException sqlEx)
{
log.Error($"Error processing message with err number {sqlEx.Number}. Exception was {sqlEx.ToString()}");
}
catch (Exception e)
{
log.Error($"Error processing message. Exception was {e.ToString()}");
}
return((long)sqlDbLatency
.Elapsed
.TotalMilliseconds);
}
public static async Task RunAsync(
[EventHubTrigger("%EventHubName%", Connection = "EventHubConnectionString", ConsumerGroup = "%ConsumerGroup%")]
EventData[] messages,
[DocumentDB("%CosmosDBDataBase%", "%CosmosDBCollection%", ConnectionStringSetting = "CosmosDBConnectionString", CreateIfNotExists = false)]
IAsyncCollector <dynamic> documents,
ExecutionContext context,
TraceWriter log)
{
CustomTelemetry.TrackMetric(context, "IoTHubMessagesReceived", messages.Length);
var ticksUTCNow = DateTimeOffset.UtcNow;
var cutoffTime = DateTimeOffset.UtcNow.AddMinutes(-5);
// Track whether messages are arriving at the function late.
DateTime?firstMsgEnqueuedTicksUtc = messages[0]?.EnqueuedTimeUtc;
if (firstMsgEnqueuedTicksUtc.HasValue)
{
CustomTelemetry.TrackMetric(
context,
"IoTHubMessagesReceivedFreshnessMsec",
(ticksUTCNow - firstMsgEnqueuedTicksUtc.Value).TotalMilliseconds);
}
int count = 0;
int droppedMessages = 0;
foreach (var message in messages)
{
// Drop stale messages,
if (message.EnqueuedTimeUtc < cutoffTime)
{
log.Info($"Dropping late message batch. Enqueued time = {message.EnqueuedTimeUtc}, Cutoff = {cutoffTime}");
droppedMessages++;
continue;
}
var text = Encoding.UTF8.GetString(message.GetBytes());
log.Info($"Process message: {text}");
try
{
dynamic telemetry = JObject.Parse(text);
if (telemetry.sensorType == DroneSensorEventType)
{
string position = telemetry.position;
var(latitude, longitude) = DroneTelemetryConverter.ConvertPosition(position);
await documents.AddAsync(new
{
id = telemetry.deviceId,
deviceId = telemetry.deviceId,
Location = new Point(longitude, latitude),
Timestamp = message.EnqueuedTimeUtc
});
count++;
}
}
catch (Exception ex)
{
log.Error("Error processing message", ex);
}
}
CustomTelemetry.TrackMetric(
context,
"IoTHubMessagesDropped",
droppedMessages);
CustomTelemetry.TrackMetric(
context,
"CosmosDbDocumentsCreated",
count);
}