Beispiel #1
0
        public static async Task RunAsync(
            [EventHubTrigger("%EventHubName%", Connection = "EventHubConnectionString", ConsumerGroup = "%ConsumerGroup%")]
            EventData[] messages,
            ExecutionContext context,
            TraceWriter log)
        {
            CustomTelemetry.TrackMetric(context, "IoTHubMessagesReceived", messages.Length);
            await Task.Delay(0);

            var ticksUTCNow = DateTimeOffset.UtcNow;
            var cutoffTime  = DateTimeOffset.UtcNow.AddMinutes(-5);

            // Track whether messages are arriving at the function late.
            DateTime?firstMsgEnqueuedTicksUtc = messages[0]?.EnqueuedTimeUtc;

            if (firstMsgEnqueuedTicksUtc.HasValue)
            {
                CustomTelemetry.TrackMetric(
                    context,
                    "IoTHubMessagesReceivedFreshnessMsec",
                    (ticksUTCNow - firstMsgEnqueuedTicksUtc.Value).TotalMilliseconds);
            }

            int count           = 0;
            int droppedMessages = 0;

            var batchStatement = new BatchStatement();

            batchStatement.SetBatchType(BatchType.Unlogged);

            foreach (var message in messages)
            {
                // Drop stale messages,
                if (message.EnqueuedTimeUtc < cutoffTime)
                {
                    log.Info($"Dropping late message batch. Enqueued time = {message.EnqueuedTimeUtc}, Cutoff = {cutoffTime}");
                    droppedMessages++;
                    continue;
                }

                var text = Encoding.UTF8.GetString(message.GetBytes());
                log.Info($"Process message: {text}");

                try
                {
                    dynamic telemetry = JObject.Parse(text);
                    if (telemetry.sensorType == DroneSensorEventType)
                    {
                        string position = telemetry.position;
                        var(latitude, longitude) = DroneTelemetryConverter.ConvertPosition(position);

                        string deviceId = telemetry.deviceId;

                        var statementAdd = new SimpleStatement($"INSERT INTO {tableName} (device_id, location, event_time) VALUES (?, ?, ?) USING TTL 259200",
                                                               deviceId, new Point(longitude, latitude), new DateTimeOffset(message.EnqueuedTimeUtc));
                        batchStatement.Add(statementAdd);

                        count++;
                    }
                }
                catch (Exception ex)
                {
                    log.Error("Error processing message", ex);
                }
            }

            try
            {
                await session.ExecuteAsync(batchStatement);

                log.Info("Successfully written batch to cassandra");

                CustomTelemetry.TrackMetric(
                    context,
                    "IoTHubMessagesDropped",
                    droppedMessages);
                CustomTelemetry.TrackMetric(
                    context,
                    "CassandraDocumentsCreated",
                    count);
            }
            catch (Exception ex)
            {
                log.Error("Error processing batch of messages", ex);
            }
        }
Beispiel #2
0
        private async Task UpsertDocuments(
            int taskId,
            IEnumerable <EventData> docsToUpsert,
            DateTimeOffset cutoffTime,
            TraceWriter log)
        {
            var cosmosDbLatency = new Stopwatch();
            int count           = 0;
            int droppedMessages = 0;

            foreach (var message in docsToUpsert)
            {
                // Drop stale messages,
                if (message.EnqueuedTimeUtc < cutoffTime)
                {
                    log.Info($"Dropping late message batch. Enqueued time = {message.EnqueuedTimeUtc}, Cutoff = {cutoffTime}");
                    droppedMessages++;
                    continue;
                }

                var text = Encoding.UTF8.GetString(message.GetBytes());
                log.Info($"Process message: {text}");

                try
                {
                    dynamic telemetry = JObject.Parse(text);
                    if (telemetry.sensorType == DroneSensorEventType)
                    {
                        string position = telemetry.position;
                        var(latitude, longitude) = DroneTelemetryConverter
                                                   .ConvertPosition(position);

                        cosmosDbLatency.Start();
                        await Client.UpsertDocumentAsync(
                            CollectionLink,
                            new
                        {
                            id        = telemetry.deviceId,
                            deviceId  = telemetry.deviceId,
                            Location  = new Point(longitude, latitude),
                            Timestamp = message.EnqueuedTimeUtc
                        });

                        cosmosDbLatency.Stop();

                        count++;
                    }
                }
                catch (Exception e)
                {
                    if (e is DocumentClientException documentClientEx)
                    {
                        log.Error($"Error processing message with status code {documentClientEx.StatusCode}. Exception was {documentClientEx.Message}");
                    }
                    else
                    {
                        log.Error($"Error processing message. Exception was {e.Message}");
                    }
                }
                finally
                {
                    Interlocked.Add(ref this.UpsertedDocuments, count);
                    Interlocked.Add(ref this.DroppedMessages, droppedMessages);
                    Interlocked.Add(ref this.CosmosDbTotalMilliseconds,
                                    (long)cosmosDbLatency
                                    .Elapsed
                                    .TotalMilliseconds);
                }
            }
        }
        private static async Task <long> BulkLoadEvents(
            IEnumerable <EventData> docsToUpsert,
            TraceWriter log)
        {
            // Define retry logic for reliable database operations
            RetryStrategy retryStrategy = new FixedInterval(3, TimeSpan.FromSeconds(10));
            RetryPolicy   retryPolicy   = new RetryPolicy <SqlDatabaseTransientErrorDetectionStrategy>(retryStrategy);

            // Define data structure that will load events into database
            DataTable dt = new DataTable();

            dt.Columns.Add("deviceid", typeof(string));
            dt.Columns.Add("timestamp", typeof(DateTime));
            dt.Columns.Add("geo", typeof(string));
            dt.Columns.Add("json", typeof(string));

            var sqlDbLatency = new Stopwatch();

            // for each message read from IoTHub
            foreach (var message in docsToUpsert)
            {
                var text = Encoding.UTF8.GetString(message.GetBytes());
                // Create a new row
                DataRow dr = dt.NewRow();
                // Parse telemetry message
                dynamic telemetry = JObject.Parse(text);
                if (telemetry.sensorType == DroneSensorEventType)
                {
                    // Convert position
                    string position = telemetry.position;
                    var(latitude, longitude) = DroneTelemetryConverter.ConvertPosition(position);
                    // Conver to WKT format
                    string geo = string.Format($"POINT ({longitude} {latitude})");

                    dr["deviceid"]  = telemetry.deviceId;
                    dr["timestamp"] = message.EnqueuedTimeUtc;
                    dr["geo"]       = geo;
                    dr["json"]      = text;

                    dt.Rows.Add(dr);
                }
            }

            try
            {
                sqlDbLatency.Start();

                await retryPolicy.ExecuteAsync(async() =>
                {
                    using (SqlConnection cnn = new SqlConnection(SQLDBConnectionString))
                    {
                        cnn.Open();
                        SqlBulkCopy bc          = new SqlBulkCopy(cnn);
                        bc.BatchSize            = 10000;
                        bc.DestinationTableName = "events";
                        await bc.WriteToServerAsync(dt);
                    }
                });

                sqlDbLatency.Stop();
            }
            catch (SqlException sqlEx)
            {
                log.Error($"Error processing message with err number {sqlEx.Number}. Exception was {sqlEx.ToString()}");
            }
            catch (Exception e)
            {
                log.Error($"Error processing message. Exception was {e.ToString()}");
            }

            return((long)sqlDbLatency
                   .Elapsed
                   .TotalMilliseconds);
        }
Beispiel #4
0
        public static async Task RunAsync(
            [EventHubTrigger("%EventHubName%", Connection = "EventHubConnectionString", ConsumerGroup = "%ConsumerGroup%")]
            EventData[] messages,
            [DocumentDB("%CosmosDBDataBase%", "%CosmosDBCollection%", ConnectionStringSetting = "CosmosDBConnectionString", CreateIfNotExists = false)]
            IAsyncCollector <dynamic> documents,
            ExecutionContext context,
            TraceWriter log)
        {
            CustomTelemetry.TrackMetric(context, "IoTHubMessagesReceived", messages.Length);

            var ticksUTCNow = DateTimeOffset.UtcNow;
            var cutoffTime  = DateTimeOffset.UtcNow.AddMinutes(-5);

            // Track whether messages are arriving at the function late.
            DateTime?firstMsgEnqueuedTicksUtc = messages[0]?.EnqueuedTimeUtc;

            if (firstMsgEnqueuedTicksUtc.HasValue)
            {
                CustomTelemetry.TrackMetric(
                    context,
                    "IoTHubMessagesReceivedFreshnessMsec",
                    (ticksUTCNow - firstMsgEnqueuedTicksUtc.Value).TotalMilliseconds);
            }

            int count           = 0;
            int droppedMessages = 0;

            foreach (var message in messages)
            {
                // Drop stale messages,
                if (message.EnqueuedTimeUtc < cutoffTime)
                {
                    log.Info($"Dropping late message batch. Enqueued time = {message.EnqueuedTimeUtc}, Cutoff = {cutoffTime}");
                    droppedMessages++;
                    continue;
                }

                var text = Encoding.UTF8.GetString(message.GetBytes());
                log.Info($"Process message: {text}");

                try
                {
                    dynamic telemetry = JObject.Parse(text);
                    if (telemetry.sensorType == DroneSensorEventType)
                    {
                        string position = telemetry.position;
                        var(latitude, longitude) = DroneTelemetryConverter.ConvertPosition(position);

                        await documents.AddAsync(new
                        {
                            id        = telemetry.deviceId,
                            deviceId  = telemetry.deviceId,
                            Location  = new Point(longitude, latitude),
                            Timestamp = message.EnqueuedTimeUtc
                        });

                        count++;
                    }
                }
                catch (Exception ex)
                {
                    log.Error("Error processing message", ex);
                }
            }

            CustomTelemetry.TrackMetric(
                context,
                "IoTHubMessagesDropped",
                droppedMessages);
            CustomTelemetry.TrackMetric(
                context,
                "CosmosDbDocumentsCreated",
                count);
        }