public static async Task RunAsync(
[EventHubTrigger(
"%EventHubName%",
Connection = "EventHubConnectionString",
ConsumerGroup = "%ConsumerGroup%")]
EventData[] messages,
Microsoft.Azure.WebJobs.ExecutionContext context,
TraceWriter log)
{
DateTimeOffset ticksUTCNow = DateTimeOffset.UtcNow;
CustomTelemetry.TrackMetric(
context,
"IoTHubMessagesReceived",
messages.Length);
// Track whether messages are arriving at the function late.
DateTime?firstMsgEnqueuedTicksUtc = messages[0]?.EnqueuedTimeUtc;
if (firstMsgEnqueuedTicksUtc.HasValue)
{
CustomTelemetry.TrackMetric(
context,
"IoTHubMessagesReceivedFreshnessMsec",
(ticksUTCNow - firstMsgEnqueuedTicksUtc.Value)
.TotalMilliseconds);
}
var length = (double)messages.Length;
log.Info($"Starting load | Docs to bulk load {messages.Length} | Docs to bulk load per task {length}");
// Bulk load events
long sqlDbTotalMilliseconds = await BulkLoadEvents(messages, log);
CustomTelemetry.TrackMetric(
context,
"IoTHubMessagesDropped",
messages.Length);
CustomTelemetry.TrackMetric(
context,
"SqlDbDocumentsCreated",
messages.Length);
var latency = messages.Length > 0
? sqlDbTotalMilliseconds / messages.Length
: 0;
CustomTelemetry.TrackMetric(
context,
"SqlDbLatencyMsec",
latency);
}
public static async Task RunAsync(
[EventHubTrigger("%EventHubName%", Connection = "EventHubConnectionString", ConsumerGroup = "%ConsumerGroup%")]
EventData[] messages,
ExecutionContext context,
TraceWriter log)
{
CustomTelemetry.TrackMetric(context, "IoTHubMessagesReceived", messages.Length);
await Task.Delay(0);
var ticksUTCNow = DateTimeOffset.UtcNow;
var cutoffTime = DateTimeOffset.UtcNow.AddMinutes(-5);
// Track whether messages are arriving at the function late.
DateTime?firstMsgEnqueuedTicksUtc = messages[0]?.EnqueuedTimeUtc;
if (firstMsgEnqueuedTicksUtc.HasValue)
{
CustomTelemetry.TrackMetric(
context,
"IoTHubMessagesReceivedFreshnessMsec",
(ticksUTCNow - firstMsgEnqueuedTicksUtc.Value).TotalMilliseconds);
}
int count = 0;
int droppedMessages = 0;
var batchStatement = new BatchStatement();
batchStatement.SetBatchType(BatchType.Unlogged);
foreach (var message in messages)
{
// Drop stale messages,
if (message.EnqueuedTimeUtc < cutoffTime)
{
log.Info($"Dropping late message batch. Enqueued time = {message.EnqueuedTimeUtc}, Cutoff = {cutoffTime}");
droppedMessages++;
continue;
}
var text = Encoding.UTF8.GetString(message.GetBytes());
log.Info($"Process message: {text}");
try
{
dynamic telemetry = JObject.Parse(text);
if (telemetry.sensorType == DroneSensorEventType)
{
string position = telemetry.position;
var(latitude, longitude) = DroneTelemetryConverter.ConvertPosition(position);
string deviceId = telemetry.deviceId;
var statementAdd = new SimpleStatement($"INSERT INTO {tableName} (device_id, location, event_time) VALUES (?, ?, ?) USING TTL 259200",
deviceId, new Point(longitude, latitude), new DateTimeOffset(message.EnqueuedTimeUtc));
batchStatement.Add(statementAdd);
count++;
}
}
catch (Exception ex)
{
log.Error("Error processing message", ex);
}
}
try
{
await session.ExecuteAsync(batchStatement);
log.Info("Successfully written batch to cassandra");
CustomTelemetry.TrackMetric(
context,
"IoTHubMessagesDropped",
droppedMessages);
CustomTelemetry.TrackMetric(
context,
"CassandraDocumentsCreated",
count);
}
catch (Exception ex)
{
log.Error("Error processing batch of messages", ex);
}
}
public static async Task RunAsync(
[EventHubTrigger(
"%EventHubName%",
Connection = "EventHubConnectionString",
ConsumerGroup = "%ConsumerGroup%")]
EventData[] messages,
Microsoft.Azure.WebJobs.ExecutionContext context,
TraceWriter log)
{
DateTimeOffset ticksUTCNow = DateTimeOffset.UtcNow;
ThreadPool.SetMinThreads(MinThreadPoolSize, MinThreadPoolSize);
CustomTelemetry.TrackMetric(
context,
"IoTHubMessagesReceived",
messages.Length);
// Track whether messages are arriving at the function late.
DateTime?firstMsgEnqueuedTicksUtc = messages[0]?.EnqueuedTimeUtc;
if (firstMsgEnqueuedTicksUtc.HasValue)
{
CustomTelemetry.TrackMetric(
context,
"IoTHubMessagesReceivedFreshnessMsec",
(ticksUTCNow - firstMsgEnqueuedTicksUtc.Value)
.TotalMilliseconds);
}
var length = (double)messages.Length;
var numberOfDocumentsToUpsertPerTask = (int)Math.Ceiling(length / MinThreadPoolSize);
var taskCount = (int)Math.Ceiling(length / numberOfDocumentsToUpsertPerTask);
log.Info($"Starting upserts with {taskCount} tasks | Docs to upsert {messages.Length} | Docs to upsert per task {numberOfDocumentsToUpsertPerTask}");
var(documentsUpserted, droppedMessages, cosmosDbTotalMilliseconds) =
await new WarmPathFunction().ProcessMessagesFromEventHub(
taskCount,
numberOfDocumentsToUpsertPerTask,
messages,
log);
CustomTelemetry.TrackMetric(
context,
"IoTHubMessagesDropped",
droppedMessages);
CustomTelemetry.TrackMetric(
context,
"CosmosDbDocumentsCreated",
documentsUpserted);
// some telemetry could be not stored in WarmPath, so it gets
// filtered and documents are not upserted.
var latency = documentsUpserted > 0
? cosmosDbTotalMilliseconds / documentsUpserted
: 0;
CustomTelemetry.TrackMetric(
context,
"CosmosDbLatencyMsec",
latency);
}
public static async Task RunAsync(
[EventHubTrigger("%EventHubName%", Connection = "EventHubConnectionString", ConsumerGroup = "%ConsumerGroup%")]
EventData[] messages,
[DocumentDB("%CosmosDBDataBase%", "%CosmosDBCollection%", ConnectionStringSetting = "CosmosDBConnectionString", CreateIfNotExists = false)]
IAsyncCollector <dynamic> documents,
ExecutionContext context,
TraceWriter log)
{
CustomTelemetry.TrackMetric(context, "IoTHubMessagesReceived", messages.Length);
var ticksUTCNow = DateTimeOffset.UtcNow;
var cutoffTime = DateTimeOffset.UtcNow.AddMinutes(-5);
// Track whether messages are arriving at the function late.
DateTime?firstMsgEnqueuedTicksUtc = messages[0]?.EnqueuedTimeUtc;
if (firstMsgEnqueuedTicksUtc.HasValue)
{
CustomTelemetry.TrackMetric(
context,
"IoTHubMessagesReceivedFreshnessMsec",
(ticksUTCNow - firstMsgEnqueuedTicksUtc.Value).TotalMilliseconds);
}
int count = 0;
int droppedMessages = 0;
foreach (var message in messages)
{
// Drop stale messages,
if (message.EnqueuedTimeUtc < cutoffTime)
{
log.Info($"Dropping late message batch. Enqueued time = {message.EnqueuedTimeUtc}, Cutoff = {cutoffTime}");
droppedMessages++;
continue;
}
var text = Encoding.UTF8.GetString(message.GetBytes());
log.Info($"Process message: {text}");
try
{
dynamic telemetry = JObject.Parse(text);
if (telemetry.sensorType == DroneSensorEventType)
{
string position = telemetry.position;
var(latitude, longitude) = DroneTelemetryConverter.ConvertPosition(position);
await documents.AddAsync(new
{
id = telemetry.deviceId,
deviceId = telemetry.deviceId,
Location = new Point(longitude, latitude),
Timestamp = message.EnqueuedTimeUtc
});
count++;
}
}
catch (Exception ex)
{
log.Error("Error processing message", ex);
}
}
CustomTelemetry.TrackMetric(
context,
"IoTHubMessagesDropped",
droppedMessages);
CustomTelemetry.TrackMetric(
context,
"CosmosDbDocumentsCreated",
count);
}