/// <summary>
/// Find the timestamp of the most recent record for each device.
/// </summary>
/// <returns>The high water mark.</returns>
private DateTime getWeatherHighWaterMark()
{
Utils.Log("Getting high water mark for weather...");
var mostRecent = ElasticUtils.getHighWaterMark <WeatherReading>(EsClient, settings.weatherUnderground.IndexName, null);
if (mostRecent != null)
{
return(mostRecent.timestamp.AddSeconds(1));
}
return(new DateTime(2014, 1, 1, 0, 0, 0, DateTimeKind.Utc));
}
/// <summary>
/// Stores the sensor readings.
/// </summary>
/// <param name="readings">Readings.</param>
private void StoreReadings(ICollection <Reading> readings, string indexName)
{
if (readings.Any())
{
var uniqueReadings = readings.Distinct().ToList();
if (uniqueReadings.Count() < readings.Count())
{
Utils.Log("Conflated duplicate readings from {0} to {1}", readings.Count(), uniqueReadings.Count());
}
Utils.Log("Indexing {0} sensor readings from {1} to {2}", uniqueReadings.Count(), uniqueReadings.First().timestamp, uniqueReadings.Last().timestamp);
var years = uniqueReadings.GroupBy(x => x.timestamp.Year, (year, values) =>
new { year, values = values.OrderBy(x => x.timestamp).ToList() })
.ToList();
var now = DateTime.UtcNow;
foreach (var kvp in years)
{
if (kvp.values.Any())
{
string yearIndex = string.Format("{0}-{1}", indexName, kvp.year);
foreach (var x in kvp.values)
{
x.ingestionTimeStamp = now;
}
ElasticUtils.BulkInsert(EsClient, yearIndex, kvp.values);
}
}
ElasticUtils.createDateBasedAliases(EsClient, indexName);
}
else
{
Utils.Log("No readings to ingest.");
}
}
private void DeleteWeatherDateRange()
{
var q = new DateRangeQuery
{
Name = "named_query",
Boost = 1.1,
GreaterThanOrEqualTo = DateTime.Now.AddDays(-14),
LessThanOrEqualTo = DateTime.Now,
Format = "dd/MM/yyyy||yyyy"
};
var allDocs = new List <WeatherSummary>();
ElasticUtils.ScanAllDocs <WeatherSummary>(EsClient, settings.weatherUnderground.IndexName,
"timestamp", (doc, id) => { allDocs.Add(new WeatherSummary {
Id = id
}); }, q);
var bulkResponse = EsClient.DeleteMany(allDocs);
}
/// <summary>
/// Find the timestamp of the most recent record for each device.
/// </summary>
/// <returns>The high water mark.</returns>
/// <param name="uuid">Device UUID</param>
private DateTime getDeviceHighWaterMark(string uuid)
{
var query = new TermQuery {
Field = "device.uuid.keyword", Value = uuid
};
Utils.Log("Getting high water mark for {0}...", uuid);
var mostRecent = ElasticUtils.getHighWaterMark <SensorReading>(EsClient, settings.indexname, query);
if (mostRecent != null)
{
if (mostRecent.device.uuid != uuid)
{
throw new ArgumentException("High watermark returned for incorrect UUID!");
}
return(mostRecent.timestamp.AddSeconds(1));
}
return(new DateTime(2017, 1, 1, 0, 0, 0, DateTimeKind.Utc));
}
/// <summary>
/// Main work method
/// </summary>
/// <param name="settings">Settings.</param>
public void ProcessTags(Settings settings)
{
this.settings = settings;
Uri esPath = new UriBuilder
{
Host = settings.elasticserver,
Port = 9200
}.Uri;
EsClient = ElasticUtils.getElasticClient(esPath, settings.indexname, false);
var alerts = new List <Alert>();
var allReadings = new List <SensorReading>();
var allDevices = new List <SensorDevice>();
try
{
if (settings.wirelesstag != null)
{
WirelessSensorTagAPI tagService = new WirelessSensorTagAPI(settings.wirelesstag.WirelessTagServiceUrl);
if (tagService.SignIn(settings.wirelesstag.username, settings.wirelesstag.password))
{
var wirelessTagReadings = QuerySensorTags(tagService, allDevices);
allReadings.AddRange(wirelessTagReadings);
}
else
{
alerts.Add(new Alert {
deviceName = "Wireless Tags", alertText = "Sign-in failed."
});
}
}
}
catch (Exception ex)
{
Utils.Log("Exception querying SensorTag data. {0}", ex);
}
QueryHiveData(allReadings, alerts);
QueryWeatherData(settings.weatherUnderground);
if (allReadings.Any())
{
try
{
StoreReadings(allReadings, settings.indexname);
}
catch (Exception ex)
{
Utils.Log("Exception ingesting data in ES. {0}", ex);
}
// See if any of the data we got back indicated a drained battery.
CheckBatteryStatus(allReadings, settings, alerts);
}
try
{
// Now check for any missing data - i.e., long gaps since we last saw anything
CheckMissingData(allDevices, alerts);
// And send any alerts we saw.
if (alerts.Any())
{
Utils.Log("Sending {0} alerts.", alerts.Count);
if (settings.email != null)
{
Utils.SendAlertEmail(settings.email, alerts);
}
}
}
catch (Exception ex)
{
Utils.Log("Exception checking missing data. {0}", ex);
}
Utils.Log("Run complete.");
}
/// <summary>
/// Main work method
/// </summary>
/// <param name="settings">Settings.</param>
public void ProcessTags(Settings settings)
{
this.settings = settings;
Uri esPath = new UriBuilder
{
Host = settings.elasticserver,
Port = 9200
}.Uri;
EsClient = ElasticUtils.getElasticClient(esPath, settings.indexname, false);
try
{
var weatherReadings = QueryWeatherData(settings.weatherUnderground);
StoreReadings(weatherReadings, settings.weatherUnderground.IndexName);
}
catch (Exception ex)
{
Utils.Log("Exception querying Weather data. {0}", ex);
}
List <Reading> allReadings = new List <Reading>();
List <SensorDevice> allDevices = new List <SensorDevice>();
try
{
if (settings.hive != null)
{
HiveService service = new HiveService();
if (service.SignIn(settings.hive.username, settings.hive.password))
{
var hiveReadings = QueryHiveData(service, allDevices);
allReadings.AddRange(hiveReadings);
}
}
}
catch (Exception ex)
{
Utils.Log("Exception querying Hive data. {0}", ex);
}
try
{
if (settings.wirelesstag != null)
{
WirelessSensorTagAPI tagService = new WirelessSensorTagAPI(settings.wirelesstag.WirelessTagServiceUrl);
if (tagService.SignIn(settings.wirelesstag.username, settings.wirelesstag.password))
{
var wirelessTagReadings = QuerySensorTags(tagService, allDevices);
allReadings.AddRange(wirelessTagReadings);
}
}
}
catch (Exception ex)
{
Utils.Log("Exception querying SensorTag data. {0}", ex);
}
if (allReadings.Any() && settings.email != null)
{
try
{
StoreReadings(allReadings, settings.indexname);
ElasticUtils.DeleteDuplicates(EsClient, settings.indexname);
}
catch (Exception ex)
{
Utils.Log("Exception ingesting data in ES. {0}", ex);
}
try
{
List <Alert> alerts = new List <Alert>();
// See if any of the data we got back indicated a drained battery.
CheckBatteryStatus(allReadings, settings.lowBatteryThreshold, alerts);
CheckMissingData(allDevices, alerts);
if (alerts.Any())
{
Utils.SendAlertEmail(settings.email, alerts);
}
}
catch (Exception ex)
{
Utils.Log("Exception checking missing data. {0}", ex);
}
}
Utils.Log("Run complete.");
}
