コード例 #1
0
        public static async Task RunAsync(
            [EventHubTrigger(
                 "%EventHubName%",
                 Connection = "EventHubConnectionString",
                 ConsumerGroup = "%ConsumerGroup%")]
            EventData[] messages,
            Microsoft.Azure.WebJobs.ExecutionContext context,
            TraceWriter log)
        {
            DateTimeOffset ticksUTCNow = DateTimeOffset.UtcNow;

            CustomTelemetry.TrackMetric(
                context,
                "IoTHubMessagesReceived",
                messages.Length);

            // Track whether messages are arriving at the function late.
            DateTime?firstMsgEnqueuedTicksUtc = messages[0]?.EnqueuedTimeUtc;

            if (firstMsgEnqueuedTicksUtc.HasValue)
            {
                CustomTelemetry.TrackMetric(
                    context,
                    "IoTHubMessagesReceivedFreshnessMsec",
                    (ticksUTCNow - firstMsgEnqueuedTicksUtc.Value)
                    .TotalMilliseconds);
            }

            var length = (double)messages.Length;

            log.Info($"Starting load | Docs to bulk load {messages.Length} | Docs to bulk load per task {length}");

            // Bulk load events
            long sqlDbTotalMilliseconds = await BulkLoadEvents(messages, log);

            CustomTelemetry.TrackMetric(
                context,
                "IoTHubMessagesDropped",
                messages.Length);
            CustomTelemetry.TrackMetric(
                context,
                "SqlDbDocumentsCreated",
                messages.Length);
            var latency = messages.Length > 0
                                ? sqlDbTotalMilliseconds / messages.Length
                                : 0;

            CustomTelemetry.TrackMetric(
                context,
                "SqlDbLatencyMsec",
                latency);
        }
コード例 #2
0
        public static async Task RunAsync(
            [EventHubTrigger("%EventHubName%", Connection = "EventHubConnectionString", ConsumerGroup = "%ConsumerGroup%")]
            EventData[] messages,
            ExecutionContext context,
            TraceWriter log)
        {
            CustomTelemetry.TrackMetric(context, "IoTHubMessagesReceived", messages.Length);
            await Task.Delay(0);

            var ticksUTCNow = DateTimeOffset.UtcNow;
            var cutoffTime  = DateTimeOffset.UtcNow.AddMinutes(-5);

            // Track whether messages are arriving at the function late.
            DateTime?firstMsgEnqueuedTicksUtc = messages[0]?.EnqueuedTimeUtc;

            if (firstMsgEnqueuedTicksUtc.HasValue)
            {
                CustomTelemetry.TrackMetric(
                    context,
                    "IoTHubMessagesReceivedFreshnessMsec",
                    (ticksUTCNow - firstMsgEnqueuedTicksUtc.Value).TotalMilliseconds);
            }

            int count           = 0;
            int droppedMessages = 0;

            var batchStatement = new BatchStatement();

            batchStatement.SetBatchType(BatchType.Unlogged);

            foreach (var message in messages)
            {
                // Drop stale messages,
                if (message.EnqueuedTimeUtc < cutoffTime)
                {
                    log.Info($"Dropping late message batch. Enqueued time = {message.EnqueuedTimeUtc}, Cutoff = {cutoffTime}");
                    droppedMessages++;
                    continue;
                }

                var text = Encoding.UTF8.GetString(message.GetBytes());
                log.Info($"Process message: {text}");

                try
                {
                    dynamic telemetry = JObject.Parse(text);
                    if (telemetry.sensorType == DroneSensorEventType)
                    {
                        string position = telemetry.position;
                        var(latitude, longitude) = DroneTelemetryConverter.ConvertPosition(position);

                        string deviceId = telemetry.deviceId;

                        var statementAdd = new SimpleStatement($"INSERT INTO {tableName} (device_id, location, event_time) VALUES (?, ?, ?) USING TTL 259200",
                                                               deviceId, new Point(longitude, latitude), new DateTimeOffset(message.EnqueuedTimeUtc));
                        batchStatement.Add(statementAdd);

                        count++;
                    }
                }
                catch (Exception ex)
                {
                    log.Error("Error processing message", ex);
                }
            }

            try
            {
                await session.ExecuteAsync(batchStatement);

                log.Info("Successfully written batch to cassandra");

                CustomTelemetry.TrackMetric(
                    context,
                    "IoTHubMessagesDropped",
                    droppedMessages);
                CustomTelemetry.TrackMetric(
                    context,
                    "CassandraDocumentsCreated",
                    count);
            }
            catch (Exception ex)
            {
                log.Error("Error processing batch of messages", ex);
            }
        }
コード例 #3
0
        public static async Task RunAsync(
            [EventHubTrigger(
                 "%EventHubName%",
                 Connection = "EventHubConnectionString",
                 ConsumerGroup = "%ConsumerGroup%")]
            EventData[] messages,
            Microsoft.Azure.WebJobs.ExecutionContext context,
            TraceWriter log)
        {
            DateTimeOffset ticksUTCNow = DateTimeOffset.UtcNow;

            ThreadPool.SetMinThreads(MinThreadPoolSize, MinThreadPoolSize);

            CustomTelemetry.TrackMetric(
                context,
                "IoTHubMessagesReceived",
                messages.Length);

            // Track whether messages are arriving at the function late.
            DateTime?firstMsgEnqueuedTicksUtc = messages[0]?.EnqueuedTimeUtc;

            if (firstMsgEnqueuedTicksUtc.HasValue)
            {
                CustomTelemetry.TrackMetric(
                    context,
                    "IoTHubMessagesReceivedFreshnessMsec",
                    (ticksUTCNow - firstMsgEnqueuedTicksUtc.Value)
                    .TotalMilliseconds);
            }

            var length = (double)messages.Length;
            var numberOfDocumentsToUpsertPerTask = (int)Math.Ceiling(length / MinThreadPoolSize);
            var taskCount = (int)Math.Ceiling(length / numberOfDocumentsToUpsertPerTask);

            log.Info($"Starting upserts with {taskCount} tasks | Docs to upsert {messages.Length} | Docs to upsert per task {numberOfDocumentsToUpsertPerTask}");

            var(documentsUpserted, droppedMessages, cosmosDbTotalMilliseconds) =
                await new WarmPathFunction().ProcessMessagesFromEventHub(
                    taskCount,
                    numberOfDocumentsToUpsertPerTask,
                    messages,
                    log);

            CustomTelemetry.TrackMetric(
                context,
                "IoTHubMessagesDropped",
                droppedMessages);
            CustomTelemetry.TrackMetric(
                context,
                "CosmosDbDocumentsCreated",
                documentsUpserted);
            // some telemetry could be not stored in WarmPath, so it gets
            // filtered and documents are not upserted.
            var latency = documentsUpserted > 0
                                ? cosmosDbTotalMilliseconds / documentsUpserted
                                : 0;

            CustomTelemetry.TrackMetric(
                context,
                "CosmosDbLatencyMsec",
                latency);
        }
コード例 #4
0
        public static async Task RunAsync(
            [EventHubTrigger("%EventHubName%", Connection = "EventHubConnectionString", ConsumerGroup = "%ConsumerGroup%")]
            EventData[] messages,
            [DocumentDB("%CosmosDBDataBase%", "%CosmosDBCollection%", ConnectionStringSetting = "CosmosDBConnectionString", CreateIfNotExists = false)]
            IAsyncCollector <dynamic> documents,
            ExecutionContext context,
            TraceWriter log)
        {
            CustomTelemetry.TrackMetric(context, "IoTHubMessagesReceived", messages.Length);

            var ticksUTCNow = DateTimeOffset.UtcNow;
            var cutoffTime  = DateTimeOffset.UtcNow.AddMinutes(-5);

            // Track whether messages are arriving at the function late.
            DateTime?firstMsgEnqueuedTicksUtc = messages[0]?.EnqueuedTimeUtc;

            if (firstMsgEnqueuedTicksUtc.HasValue)
            {
                CustomTelemetry.TrackMetric(
                    context,
                    "IoTHubMessagesReceivedFreshnessMsec",
                    (ticksUTCNow - firstMsgEnqueuedTicksUtc.Value).TotalMilliseconds);
            }

            int count           = 0;
            int droppedMessages = 0;

            foreach (var message in messages)
            {
                // Drop stale messages,
                if (message.EnqueuedTimeUtc < cutoffTime)
                {
                    log.Info($"Dropping late message batch. Enqueued time = {message.EnqueuedTimeUtc}, Cutoff = {cutoffTime}");
                    droppedMessages++;
                    continue;
                }

                var text = Encoding.UTF8.GetString(message.GetBytes());
                log.Info($"Process message: {text}");

                try
                {
                    dynamic telemetry = JObject.Parse(text);
                    if (telemetry.sensorType == DroneSensorEventType)
                    {
                        string position = telemetry.position;
                        var(latitude, longitude) = DroneTelemetryConverter.ConvertPosition(position);

                        await documents.AddAsync(new
                        {
                            id        = telemetry.deviceId,
                            deviceId  = telemetry.deviceId,
                            Location  = new Point(longitude, latitude),
                            Timestamp = message.EnqueuedTimeUtc
                        });

                        count++;
                    }
                }
                catch (Exception ex)
                {
                    log.Error("Error processing message", ex);
                }
            }

            CustomTelemetry.TrackMetric(
                context,
                "IoTHubMessagesDropped",
                droppedMessages);
            CustomTelemetry.TrackMetric(
                context,
                "CosmosDbDocumentsCreated",
                count);
        }