private static string GetTimeseriesData(SensorDataContract[] data)
{
var step = 1;
var prevTime = DateTime.MinValue;
var prevVal = 0d;
List<SensorDataContract> newData = new List<SensorDataContract>();
foreach (var d in data.OrderBy(dd => dd.TimeCreated))
{
d.TimeCreated = d.TimeCreated.AddTicks(-(d.TimeCreated.Ticks % TimeSpan.TicksPerSecond)); // round off the millisecs
if (prevTime != DateTime.MinValue)
{
for (; prevTime.AddSeconds(step) < d.TimeCreated; )
{
prevTime = prevTime.AddSeconds(step);
newData.Add(new SensorDataContract { TimeCreated = prevTime, Value = prevVal });
}
}
newData.Add(d);
prevTime = d.TimeCreated;
prevVal = d.Value;
}
var sb = new StringBuilder();
foreach (var d in newData)
{
sb.Append(string.Format("{0}={1};", d.TimeCreated.ToString("O"), d.Value));
}
return sb.ToString();
}
//void ProcessNoData()
//{
// lock (_lockNoData)
// {
// DateTime now = DateTime.UtcNow;
// if (_receiversLastUpdate.All(d => now - d > TimeSpan.FromMinutes(3)))
// {
// Trace.TraceError("No data for the last 3 minutes. Reinitializing");
// FailureEvent.Set();
// }
// }
//}
void Process(int iPart, bool firstReport, IEnumerable <EventData> batch)
{
UTF8Encoding enc = new UTF8Encoding();
foreach (EventData e in batch)
{
string body = enc.GetString(e.GetBytes());
string[] lines = body.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var line in lines)
{
try
{
var payload = JsonConvert.DeserializeObject <IDictionary <string, object> >(line);
var sensorData = new SensorDataContract
{
DisplayName = (string)payload["displayname"],
Guid = (string)payload["guid"],
Location = (string)payload["location"],
MeasureName = (string)payload["measurename"],
Organization = (string)payload["organization"],
TimeCreated = (DateTime)payload["timecreated"],
UnitOfMeasure = (string)payload["unitofmeasure"],
Value = (double)payload["value"]
};
var from = sensorData.UniqueId();
lock (_lock)
{
CircularBuffer <SensorDataContract> buffer;
if (!_buffers.TryGetValue(from, out buffer))
{
buffer = new CircularBuffer <SensorDataContract>(_bufferSize);
_buffers.Add(from, buffer);
}
buffer.Add(sensorData);
#if DEBUG_LOG
Console.WriteLine("Data from device {0}, Total count: {1}", from, buffer.Count);
#endif
}
}
catch (Exception)
{
#if DEBUG_LOG
Trace.TraceError("Ignored invalid event data: {0}", line);
#endif
}
}
}
//lock (_lockNoData)
//{
// _receiversLastUpdate[iPart] = DateTime.UtcNow;
//}
}
public Task<AnomalyRecord[]> Analyze(SensorDataContract[] data)
{
var timeSeriesData = GetTimeseriesData(data);
#if DEBUG_LOG
Trace.TraceInformation("AzureML request: {0}", timeSeriesData);
#endif
if (_useMarketApi)
{
return Task.Run(() => GetAlertsFromAnomalyDetectionAPI(timeSeriesData));
}
return GetAlertsFromRRS(timeSeriesData);
}
private static string OutputResults(string from, SensorDataContract sensorMeta, AnomalyRecord alert)
{
return(JsonConvert.SerializeObject(
new
{
guid = sensorMeta.Guid,
displayname = sensorMeta.DisplayName,
measurename = sensorMeta.MeasureName,
unitofmeasure = sensorMeta.UnitOfMeasure,
location = sensorMeta.Location,
organization = sensorMeta.Organization,
timecreated = alert.Time.ToLocalTime(),
value = alert.Data,
alerttype = "MLModelAlert",
message = "Anomaly detected by Machine Learning model."
}));
}
//void ProcessNoData()
//{
// lock (_lockNoData)
// {
// DateTime now = DateTime.UtcNow;
// if (_receiversLastUpdate.All(d => now - d > TimeSpan.FromMinutes(3)))
// {
// Trace.TraceError("No data for the last 3 minutes. Reinitializing");
// FailureEvent.Set();
// }
// }
//}
void Process(int iPart, bool firstReport, IEnumerable<EventData> batch)
{
UTF8Encoding enc = new UTF8Encoding();
foreach (EventData e in batch)
{
string body = enc.GetString(e.GetBytes());
string[] lines = body.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var line in lines)
{
try
{
var payload = JsonConvert.DeserializeObject<IDictionary<string, object>>(line);
var sensorData = new SensorDataContract
{
DisplayName = (string) payload["displayname"],
Guid = (string) payload["guid"],
Location = (string) payload["location"],
MeasureName = (string) payload["measurename"],
Organization = (string) payload["organization"],
TimeCreated = (DateTime)payload["timecreated"],
UnitOfMeasure = (string) payload["unitofmeasure"],
Value = (double) payload["value"]
};
var from = sensorData.UniqueId();
lock (_lock)
{
CircularBuffer<SensorDataContract> buffer;
if (!_buffers.TryGetValue(from, out buffer))
{
buffer = new CircularBuffer<SensorDataContract>(_bufferSize);
_buffers.Add(from, buffer);
}
buffer.Add(sensorData);
#if DEBUG_LOG
Console.WriteLine("Data from device {0}, Total count: {1}", from, buffer.Count);
#endif
}
}
catch (Exception)
{
#if DEBUG_LOG
Trace.TraceError("Ignored invalid event data: {0}", line);
#endif
}
}
}
//lock (_lockNoData)
//{
// _receiversLastUpdate[iPart] = DateTime.UtcNow;
//}
}
// adds measurement data from IoTHub to circular buffer
public void Process(IDictionary<string, object> messagePayload)
{
// Filter on MeasureName and time
if ((messagePayload.ContainsKey(_measureNameFilter)) &&
(messagePayload.ContainsKey("time")))
{
DateTime eventTime;
if (DateTime.TryParse(messagePayload["time"].ToString(), out eventTime))
{
// the measure is an array of values.
// array position maps to joint in device
IEnumerable<object> measures = messagePayload[_measureNameFilter] as IEnumerable<object>;
if (measures != null)
{
int index = 0;
foreach (var ovalue in measures)
{
try
{
var val = (Newtonsoft.Json.Linq.JValue)ovalue;
double value = (double) val.Value;
var sensorData = new SensorDataContract
{
MeasureName = _measureNameFilter,
TimeCreated = eventTime,
Index = index,
Value = value
};
// UniqueId treats each joint as a separate device for anomaly detection
var from = sensorData.UniqueId();
lock (_lock)
{
CircularBuffer<SensorDataContract> buffer;
if (!_buffers.TryGetValue(from, out buffer))
{
buffer = new CircularBuffer<SensorDataContract>(_bufferSize);
_buffers.Add(from, buffer);
}
buffer.Add(sensorData);
#if DEBUG_LOG
Console.WriteLine("Data from device {0}, Total count: {1}", from, buffer.Count);
#endif
}
}
catch (Exception)
{
#if DEBUG_LOG
Trace.TraceError("Ignored invalid event data: {0}", line);
#endif
}
index++;
}
}
}
}
}
