public async Task ProcessEventsAsync(PartitionContext context, IEnumerable <EventData> events) { try { var now = DateTime.UtcNow; foreach (var eventData in events) { // We don't care about messages that are older than bufferTimeInterval if ((eventData.EnqueuedTimeUtc + bufferTimeInterval) >= now) { // Get message from the eventData body and convert JSON string into message object string eventBodyAsString = Encoding.UTF8.GetString(eventData.GetBytes()); // There can be several messages in one IList <IDictionary <string, object> > messagePayloads; try { // Attempt to deserialze event body as single JSON message messagePayloads = new List <IDictionary <string, object> > { JsonConvert.DeserializeObject <IDictionary <string, object> >(eventBodyAsString) }; } catch { // Not a single JSON message: attempt to deserialize as array of messages // Azure Stream Analytics Preview generates invalid JSON for some multi-values queries // Workaround: turn concatenated json objects (ivalid JSON) into array of json objects (valid JSON) if (eventBodyAsString.IndexOf("}{") >= 0) { eventBodyAsString = eventBodyAsString.Replace("}{", "},{"); } if (eventBodyAsString.IndexOf("}\r\n{") >= 0) { eventBodyAsString = eventBodyAsString.Replace("}\r\n{", "},{"); } if (!eventBodyAsString.EndsWith("]")) { eventBodyAsString = eventBodyAsString + "]"; } if (!eventBodyAsString.StartsWith("[")) { eventBodyAsString = "[" + eventBodyAsString.Substring(eventBodyAsString.IndexOf("{")); } messagePayloads = JsonConvert.DeserializeObject <IList <IDictionary <string, object> > >(eventBodyAsString); } var rnd = new Random(); foreach (var messagePayload in messagePayloads) { // Read time value if (messagePayload.ContainsKey("timecreated")) { messagePayload["time"] = messagePayload["timecreated"]; } if (messagePayload.ContainsKey("timearrived")) { messagePayload["time"] = messagePayload["timearrived"]; } // process an alert if (messagePayload.ContainsKey("alerttype") && messagePayload.ContainsKey("timecreated")) { Debug.Print("Alert message received!"); // If the IoTHub service client has not yet been created, go create it. if (IoTHub == null) { IoTHub = new IoTHubHelper(Microsoft.Azure.CloudConfigurationManager.GetSetting("Azure.IoT.IoTHub.ConnectionString")); } // Send alert to device if (IoTHub != null) { var alertMessage = JsonConvert.SerializeObject(messagePayload).ToString(); IoTHub.SendMessage(messagePayload["guid"].ToString(), alertMessage); } DateTime time = DateTime.Parse(messagePayload["timecreated"].ToString()); // find the nearest point lock (sortedDataBuffer) { int idx = SearchHelper.FindFirstIndexGreaterThanOrEqualTo(sortedDataBuffer, time); bool found = false; string alertType = messagePayload["alerttype"] as string; if (idx >= sortedDataBuffer.Values.Count) { idx = sortedDataBuffer.Values.Count - 1; } while (idx >= 0) { List <IDictionary <string, object> > dictList = sortedDataBuffer.Values[idx]; foreach (IDictionary <string, object> dict in dictList) { if ( (dict.ContainsKey("guid") && messagePayload.ContainsKey("guid") && messagePayload["guid"].ToString() == dict["guid"].ToString()) && (dict.ContainsKey("measurename") && messagePayload.ContainsKey("measurename") && messagePayload["measurename"].ToString() == dict["measurename"].ToString()) && (!messagePayload.ContainsKey("displayname") || dict.ContainsKey("displayname") && messagePayload["measurename"].ToString() == dict["measurename"].ToString()) ) { // fill anomaly message if (!messagePayload.ContainsKey("value")) { messagePayload["value"] = dict["value"]; } if (!messagePayload.ContainsKey("displayname") && dict.ContainsKey("displayname")) { messagePayload["displayname"] = dict["displayname"]; } if (!messagePayload.ContainsKey("time")) { messagePayload["time"] = messagePayload["timecreated"]; } found = true; break; } } if (found) { break; } idx--; } } } if (messagePayload.ContainsKey("guid")) { var guid = messagePayload["guid"].ToString(); double val = Convert.ToDouble(messagePayload["value"]); if (!MinMaxValue.ContainsKey(guid)) { MinMaxValue.Add(guid, new MinMax { min = val, max = val }); } MinMax tmp = MinMaxValue[messagePayload["guid"].ToString()]; if (tmp.min > val) { tmp.min = val; } if (tmp.max < val) { tmp.max = val; } } // We want to read the time value from the message itself. // If none is found we will use the enqueued time DateTime messageTimeStamp = new DateTime(); if (messagePayload.ContainsKey("time")) { messageTimeStamp = DateTime.Parse(messagePayload["time"].ToString()); //if (GenerateAnomalies && rnd.Next(100) >= 95) //{ // messagePayload.Add("alerttype", "testType"); // messagePayload.Add("dsplalert", "testAlert"); // messagePayload.Add("message", "Anomaly detected by Azure ML model."); // messagePayload.Add("timestart", messagePayload["time"]); // // correct value // if (rnd.Next(2) == 1) // messagePayload["value"] = MinMaxValue[messagePayload["guid"].ToString()].max * (1.01 + 0.05 * rnd.Next(100) / 100); // else // messagePayload["value"] = MinMaxValue[messagePayload["guid"].ToString()].min * (0.99 - 0.05 * rnd.Next(100) / 100); //} } else if (messagePayload.ContainsKey("timestart")) { messageTimeStamp = DateTime.Parse(messagePayload["timestart"].ToString()); } else { messageTimeStamp = eventData.EnqueuedTimeUtc; } // Build up the list of devices seen so far (in lieu of a formal device repository) // Also keep the last message received per device (not currently used in the sample) if (messagePayload.ContainsKey("guid") && !messagePayload.ContainsKey("valueAvg")) { string guid = messagePayload["guid"].ToString(); if (guid != null) { WebSocketEventProcessor.g_devices.TryAdd(guid, messagePayload); } } // Notify clients MyWebSocketHandler.SendToClients(messagePayload); // Buffer messages so we can resend them to clients that connect later // or when a client requests data for a different device // Lock to guard against concurrent reads from client resend lock (sortedDataBuffer) { if (!sortedDataBuffer.ContainsKey(messageTimeStamp)) { sortedDataBuffer.Add(messageTimeStamp, new List <IDictionary <string, object> >()); } sortedDataBuffer[messageTimeStamp].Add(messagePayload); } } } else { Debug.Print("Received old message timestamped:" + eventData.EnqueuedTimeUtc.ToString()); } } //Call checkpoint every minute if (this.checkpointStopWatch.Elapsed > TimeSpan.FromMinutes(1)) { await context.CheckpointAsync(); lock (this) { this.checkpointStopWatch.Restart(); } // trim data buffer to keep only last 10 minutes of data lock (sortedDataBuffer) { DateTime oldDataPoint = now - bufferTimeInterval; // find the closest point int idx = SearchHelper.FindFirstIndexGreaterThanOrEqualTo(sortedDataBuffer, oldDataPoint); // trim while (idx > 0 && sortedDataBuffer.Count > 0 && sortedDataBuffer.Keys[0] <= oldDataPoint) { sortedDataBuffer.RemoveAt(0); } } } } catch (Exception e) { Trace.TraceError("Error processing events in EH {0}, partition {1}: {0}", context.EventHubPath, context.Lease.PartitionId, e.Message); } }
public async Task ProcessEventsAsync(PartitionContext context, IEnumerable <EventData> events) { try { var now = DateTime.UtcNow; foreach (var eventData in events) { // Get message from the eventData body and convert JSON string into message object string eventBodyAsString = Encoding.UTF8.GetString(eventData.GetBytes()); IList <IDictionary <string, object> > messagePayloads; try { // Attempt to deserialze event body as single JSON message messagePayloads = new List <IDictionary <string, object> > { JsonConvert.DeserializeObject <IDictionary <string, object> >(eventBodyAsString) }; } catch { // Not a single JSON message: attempt to deserialize as array of messages // Azure Stream Analytics Preview generates invalid JSON for some multi-values queries // Workaround: turn concatenated json objects (ivalid JSON) into array of json objects (valid JSON) if (eventBodyAsString.IndexOf("}{") >= 0) { eventBodyAsString = eventBodyAsString.Replace("}{", "},{"); } if (!eventBodyAsString.EndsWith("]")) { eventBodyAsString = eventBodyAsString + "]"; } if (!eventBodyAsString.StartsWith("[")) { eventBodyAsString = "[" + eventBodyAsString; } messagePayloads = JsonConvert.DeserializeObject <IList <IDictionary <string, object> > >(eventBodyAsString); } // Only send messages within the display/buffer interval to clients, to speed up recovery after downtime if ((eventData.EnqueuedTimeUtc + bufferTimeInterval).AddMinutes(1) > now) { foreach (var messagePayload in messagePayloads) { // Build up the list of devices seen so far (in lieu of a formal device repository) // Also keep the last message received per device (not currently used in the sample) string deviceName = null; if (messagePayload.ContainsKey("dspl")) { deviceName = messagePayload["dspl"] as string; if (deviceName != null) { WebSocketEventProcessor.g_devices.TryAdd(deviceName, messagePayload); } } // Notify clients MyWebSocketHandler.SendToClients(messagePayload); // Buffer messages so we can resend them to clients that connect later // or when a client requests data for a different device // Lock to guard against concurrent reads from client resend // Note that the Add operations are not contentious with each other // because EH processor host serializes per partition, and we use one buffer per partition lock (bufferedMessages) { bufferedMessages.Add(messagePayload); if (messagePayload.ContainsKey("tempavg")) { bufferedMessagesAvg.Add(messagePayload); } } } } else { Debug.WriteLine("Received event older than {0} in EH {1}, partition {2}: {3} - Sequence Number {4}", bufferTimeInterval, context.EventHubPath, context.Lease.PartitionId, eventData.EnqueuedTimeUtc, eventData.SequenceNumber); eventForNextCheckpoint = eventData; } // Remember first event to checkpoint to later if (eventForNextCheckpoint == null) { eventForNextCheckpoint = eventData; } } // Checkpoint to an event before the buffer/display time period, so we can recover the events on VM restart if (eventForNextCheckpoint != null && eventForNextCheckpoint.EnqueuedTimeUtc + bufferTimeInterval < now && lastCheckPoint + maxCheckpointFrequency < now) // Don't checkpoint too often, as every checkpoint incurs at least one blob storage roundtrip { await context.CheckpointAsync(eventForNextCheckpoint); Trace.TraceInformation("Checkpointed EH {0}, partition {1}: offset {2}, Sequence Number {3}, time {4}", context.EventHubPath, context.Lease.PartitionId, eventForNextCheckpoint.Offset, eventForNextCheckpoint.SequenceNumber, eventForNextCheckpoint.EnqueuedTimeUtc); // Remove all older messages from the resend buffer lock (bufferedMessages) { if (this.indexOfLastCheckpoint >= 0) { bufferedMessages.RemoveRange(0, this.indexOfLastCheckpoint); } indexOfLastCheckpoint = bufferedMessages.Count - 1; } // Get ready for next checkpoint lastCheckPoint = now; eventForNextCheckpoint = events.Last <EventData>(); } } catch (Exception e) { Trace.TraceError("Error processing events in EH {0}, partition {1}: {0}", context.EventHubPath, context.Lease.PartitionId, e.Message); } }
public async Task ProcessEventsAsync(PartitionContext context, IEnumerable <EventData> events) { try { var now = DateTime.UtcNow; foreach (var eventData in events) { // We don't care about messages that are older than bufferTimeInterval if ((eventData.EnqueuedTimeUtc + bufferTimeInterval) >= now) { // Get message from the eventData body and convert JSON string into message object string eventBodyAsString = Encoding.UTF8.GetString(eventData.GetBytes()); // There can be several messages in one IList <IDictionary <string, object> > messagePayloads; try { // Attempt to deserialze event body as single JSON message messagePayloads = new List <IDictionary <string, object> > { JsonConvert.DeserializeObject <IDictionary <string, object> >(eventBodyAsString) }; } catch { // Not a single JSON message: attempt to deserialize as array of messages // Azure Stream Analytics Preview generates invalid JSON for some multi-values queries // Workaround: turn concatenated json objects (ivalid JSON) into array of json objects (valid JSON) if (eventBodyAsString.IndexOf("}{") >= 0) { eventBodyAsString = eventBodyAsString.Replace("}{", "},{"); } if (!eventBodyAsString.EndsWith("]")) { eventBodyAsString = eventBodyAsString + "]"; } if (!eventBodyAsString.StartsWith("[")) { eventBodyAsString = "[" + eventBodyAsString.Substring(eventBodyAsString.IndexOf("{")); } messagePayloads = JsonConvert.DeserializeObject <IList <IDictionary <string, object> > >(eventBodyAsString); } foreach (var messagePayload in messagePayloads) { // We want to read the time value from the message itself. // If none is found we will use the enqueued time DateTime messageTimeStamp = new DateTime(); if (messagePayload.ContainsKey("time")) { messageTimeStamp = DateTime.Parse(messagePayload["time"].ToString()); } else if (messagePayload.ContainsKey("timestart")) { messageTimeStamp = DateTime.Parse(messagePayload["timestart"].ToString()); } else { messageTimeStamp = eventData.EnqueuedTimeUtc; } // Build up the list of devices seen so far (in lieu of a formal device repository) // Also keep the last message received per device (not currently used in the sample) string deviceName = null; if (messagePayload.ContainsKey("dspl")) { deviceName = messagePayload["dspl"] as string; if (deviceName != null) { WebSocketEventProcessor.g_devices.TryAdd(deviceName, messagePayload); } } // Notify clients MyWebSocketHandler.SendToClients(messagePayload); // Buffer messages so we can resend them to clients that connect later // or when a client requests data for a different device // Lock to guard against concurrent reads from client resend lock (sortedDataBuffer) { if (!sortedDataBuffer.ContainsKey(messageTimeStamp)) { sortedDataBuffer.Add(messageTimeStamp, messagePayload); } } } } } //Call checkpoint every minute if (this.checkpointStopWatch.Elapsed > TimeSpan.FromMinutes(1)) { await context.CheckpointAsync(); lock (this) { this.checkpointStopWatch.Reset(); } // trim data buffer to keep only last 10 minutes of data lock (sortedDataBuffer) { SortedList <DateTime, IDictionary <string, object> > tempBuffer = new SortedList <DateTime, IDictionary <string, object> >(); foreach (var item in sortedDataBuffer) { if (item.Key + bufferTimeInterval >= now) { tempBuffer.Add(item.Key, item.Value); } } sortedDataBuffer.Clear(); foreach (var item in tempBuffer) { sortedDataBuffer.Add(item.Key, item.Value); } } } } catch (Exception e) { Trace.TraceError("Error processing events in EH {0}, partition {1}: {0}", context.EventHubPath, context.Lease.PartitionId, e.Message); } }
public async Task ProcessEventsAsync(PartitionContext context, IEnumerable <EventData> events) { try { var now = DateTime.UtcNow; foreach (var eventData in events) { // We don't care about messages that are older than bufferTimeInterval if ((eventData.EnqueuedTimeUtc + bufferTimeInterval) >= now) { // Get message from the eventData body and convert JSON string into message object string eventBodyAsString = Encoding.UTF8.GetString(eventData.GetBytes()); // There can be several messages in one IList <IDictionary <string, object> > messagePayloads; try { // Attempt to deserialze event body as single JSON message messagePayloads = new List <IDictionary <string, object> > { JsonConvert.DeserializeObject <IDictionary <string, object> >(eventBodyAsString) }; } catch { // Not a single JSON message: attempt to deserialize as array of messages // Azure Stream Analytics Preview generates invalid JSON for some multi-values queries // Workaround: turn concatenated json objects (ivalid JSON) into array of json objects (valid JSON) if (eventBodyAsString.IndexOf("}{") >= 0) { eventBodyAsString = eventBodyAsString.Replace("}{", "},{"); } if (!eventBodyAsString.EndsWith("]")) { eventBodyAsString = eventBodyAsString + "]"; } if (!eventBodyAsString.StartsWith("[")) { eventBodyAsString = "[" + eventBodyAsString.Substring(eventBodyAsString.IndexOf("{")); } messagePayloads = JsonConvert.DeserializeObject <IList <IDictionary <string, object> > >(eventBodyAsString); } var rnd = new Random(); foreach (var messagePayload in messagePayloads) { // Read time value if (messagePayload.ContainsKey("timecreated")) { messagePayload["time"] = messagePayload["timecreated"]; } if (messagePayload.ContainsKey("timearrived")) { messagePayload["time"] = messagePayload["timearrived"]; } // process an alert if (messagePayload.ContainsKey("alerttype") && messagePayload.ContainsKey("timecreated")) { Debug.Print("Alert message received!"); DateTime time = DateTime.Parse(messagePayload["timecreated"].ToString()); // find the nearest point lock (sortedDataBuffer) { int idx = SearchHelper.FindFirstIndexGreaterThanOrEqualTo(sortedDataBuffer, time); bool found = false; string alertType = messagePayload["alerttype"] as string; if (idx >= sortedDataBuffer.Values.Count) { idx = sortedDataBuffer.Values.Count - 1; } while (idx >= 0) { List <IDictionary <string, object> > dictList = sortedDataBuffer.Values[idx]; foreach (IDictionary <string, object> dict in dictList) { if ( (dict.ContainsKey("guid") && messagePayload.ContainsKey("guid") && messagePayload["guid"].ToString() == dict["guid"].ToString()) && (dict.ContainsKey("measurename") && messagePayload.ContainsKey("measurename") && messagePayload["measurename"].ToString() == dict["measurename"].ToString()) && (!messagePayload.ContainsKey("displayname") || dict.ContainsKey("displayname") && messagePayload["measurename"].ToString() == dict["measurename"].ToString()) ) { // fill anomaly message if (!messagePayload.ContainsKey("value")) { messagePayload["value"] = dict["value"]; } if (!messagePayload.ContainsKey("displayname") && dict.ContainsKey("displayname")) { messagePayload["displayname"] = dict["displayname"]; } if (!messagePayload.ContainsKey("time")) { messagePayload["time"] = messagePayload["timecreated"]; } found = true; break; } } if (found) { break; } idx--; } } } if (messagePayload.ContainsKey("guid")) { var guid = messagePayload["guid"].ToString(); double val = Convert.ToDouble(messagePayload["value"]); if (!MinMaxValue.ContainsKey(guid)) { MinMaxValue.Add(guid, new MinMax { min = val, max = val }); } MinMax tmp = MinMaxValue[messagePayload["guid"].ToString()]; if (tmp.min > val) { tmp.min = val; } if (tmp.max < val) { tmp.max = val; } } // We want to read the time value from the message itself. // If none is found we will use the enqueued time DateTime messageTimeStamp = new DateTime(); if (messagePayload.ContainsKey("time")) { messageTimeStamp = DateTime.Parse(messagePayload["time"].ToString()); //data.Timestamp = messagePayload["time"].ToString(); //if (GenerateAnomalies && rnd.Next(100) >= 95) //{ // messagePayload.Add("alerttype", "testType"); // messagePayload.Add("dsplalert", "testAlert"); // messagePayload.Add("message", "Anomaly detected by Azure ML model."); // messagePayload.Add("timestart", messagePayload["time"]); // // correct value // if (rnd.Next(2) == 1) // messagePayload["value"] = MinMaxValue[messagePayload["guid"].ToString()].max * (1.01 + 0.05 * rnd.Next(100) / 100); // else // messagePayload["value"] = MinMaxValue[messagePayload["guid"].ToString()].min * (0.99 - 0.05 * rnd.Next(100) / 100); //} } else if (messagePayload.ContainsKey("timestart")) { messageTimeStamp = DateTime.Parse(messagePayload["timestart"].ToString()); } else { messageTimeStamp = eventData.EnqueuedTimeUtc; } // Build up the list of devices seen so far (in lieu of a formal device repository) // Also keep the last message received per device (not currently used in the sample) if (messagePayload.ContainsKey("guid") && !messagePayload.ContainsKey("valueAvg")) { string guid = messagePayload["guid"].ToString(); if (guid != null) { WebSocketEventProcessor.g_devices.TryAdd(guid, messagePayload); } } if (messagePayload["measurename"].ToString().ToLower().Equals("temperature")) { if (!String.IsNullOrEmpty(messagePayload["value"].ToString())) { CloudStorageAccount storageAccount = CloudStorageAccount.Parse("Storage Connection String"); // Create the table client. CloudTableClient tableClient = storageAccount.CreateCloudTableClient(); // Create the CloudTable object that represents the "people" table. CloudTable table = tableClient.GetTableReference("Table Storage Name"); // Create a new customer entity. DeloreanData data = new DeloreanData("Table Storage Primary Key", ""); if (messagePayload.ContainsKey("measurename")) { data.measurename = messagePayload["measurename"].ToString(); } if (messagePayload.ContainsKey("value")) { data.value = messagePayload["value"].ToString(); } if (messagePayload.ContainsKey("unitofmeasure")) { data.unitofmeasure = messagePayload["unitofmeasure"].ToString(); } if (messagePayload.ContainsKey("displayname")) { data.displayname = messagePayload["displayname"].ToString(); } if (messagePayload.ContainsKey("organization")) { data.organization = messagePayload["organization"].ToString(); } if (messagePayload.ContainsKey("location")) { data.location = messagePayload["location"].ToString(); } data.RowKey = DateTime.Parse(messagePayload["timecreated"].ToString()).Ticks.ToString() + "-" + data.measurename; TableOperation insertOperation = TableOperation.Insert(data); //var temperature = double.Parse(data.value); //if (temperature > 75) // { //NotificationHubClient hub = NotificationHubClient // .CreateClientFromConnectionString("Endpoint=sb://deloreantesthub-ns.servicebus.windows.net/;SharedAccessKeyName=DefaultFullSharedAccessSignature;SharedAccessKey=wSpPlbYOksiifQGSa7qT1lXeWzxOKatCbHoGSqZOKfY=", "deloreantesthub"); //var toast = @"<toast><visual><binding template=""ToastText01""><text id=""1"">Hello from a .NET App!</text></binding></visual></toast>"; //await hub.SendWindowsNativeNotificationAsync(toast); table.Execute(insertOperation); //} } } // Create the TableOperation that inserts the customer entity. //table.Execute(insertOperation); // Notify clients MyWebSocketHandler.SendToClients(messagePayload); // Buffer messages so we can resend them to clients that connect later // or when a client requests data for a different device // Lock to guard against concurrent reads from client resend lock (sortedDataBuffer) { if (!sortedDataBuffer.ContainsKey(messageTimeStamp)) { sortedDataBuffer.Add(messageTimeStamp, new List <IDictionary <string, object> >()); } sortedDataBuffer[messageTimeStamp].Add(messagePayload); } } } else { Debug.Print("Received old message timestamped:" + eventData.EnqueuedTimeUtc.ToString()); } } //Call checkpoint every minute if (this.checkpointStopWatch.Elapsed > TimeSpan.FromMinutes(1)) { await context.CheckpointAsync(); lock (this) { this.checkpointStopWatch.Restart(); } // trim data buffer to keep only last 10 minutes of data lock (sortedDataBuffer) { DateTime oldDataPoint = now - bufferTimeInterval; // find the closest point int idx = SearchHelper.FindFirstIndexGreaterThanOrEqualTo(sortedDataBuffer, oldDataPoint); // trim while (idx > 0 && sortedDataBuffer.Count > 0 && sortedDataBuffer.Keys[0] <= oldDataPoint) { sortedDataBuffer.RemoveAt(0); } } } } catch (Exception e) { Trace.TraceError("Error processing events in EH {0}, partition {1}: {0}", context.EventHubPath, context.Lease.PartitionId, e.Message); } }