/// <summary> /// Find the timestamp of the most recent record for each device. /// </summary> /// <returns>The high water mark.</returns> private DateTime getWeatherHighWaterMark() { Utils.Log("Getting high water mark for weather..."); var mostRecent = ElasticUtils.getHighWaterMark <WeatherReading>(EsClient, settings.weatherUnderground.IndexName, null); if (mostRecent != null) { return(mostRecent.timestamp.AddSeconds(1)); } return(new DateTime(2014, 1, 1, 0, 0, 0, DateTimeKind.Utc)); }
/// <summary> /// Stores the sensor readings. /// </summary> /// <param name="readings">Readings.</param> private void StoreReadings(ICollection <Reading> readings, string indexName) { if (readings.Any()) { var uniqueReadings = readings.Distinct().ToList(); if (uniqueReadings.Count() < readings.Count()) { Utils.Log("Conflated duplicate readings from {0} to {1}", readings.Count(), uniqueReadings.Count()); } Utils.Log("Indexing {0} sensor readings from {1} to {2}", uniqueReadings.Count(), uniqueReadings.First().timestamp, uniqueReadings.Last().timestamp); var years = uniqueReadings.GroupBy(x => x.timestamp.Year, (year, values) => new { year, values = values.OrderBy(x => x.timestamp).ToList() }) .ToList(); var now = DateTime.UtcNow; foreach (var kvp in years) { if (kvp.values.Any()) { string yearIndex = string.Format("{0}-{1}", indexName, kvp.year); foreach (var x in kvp.values) { x.ingestionTimeStamp = now; } ElasticUtils.BulkInsert(EsClient, yearIndex, kvp.values); } } ElasticUtils.createDateBasedAliases(EsClient, indexName); } else { Utils.Log("No readings to ingest."); } }
private void DeleteWeatherDateRange() { var q = new DateRangeQuery { Name = "named_query", Boost = 1.1, GreaterThanOrEqualTo = DateTime.Now.AddDays(-14), LessThanOrEqualTo = DateTime.Now, Format = "dd/MM/yyyy||yyyy" }; var allDocs = new List <WeatherSummary>(); ElasticUtils.ScanAllDocs <WeatherSummary>(EsClient, settings.weatherUnderground.IndexName, "timestamp", (doc, id) => { allDocs.Add(new WeatherSummary { Id = id }); }, q); var bulkResponse = EsClient.DeleteMany(allDocs); }
/// <summary> /// Find the timestamp of the most recent record for each device. /// </summary> /// <returns>The high water mark.</returns> /// <param name="uuid">Device UUID</param> private DateTime getDeviceHighWaterMark(string uuid) { var query = new TermQuery { Field = "device.uuid.keyword", Value = uuid }; Utils.Log("Getting high water mark for {0}...", uuid); var mostRecent = ElasticUtils.getHighWaterMark <SensorReading>(EsClient, settings.indexname, query); if (mostRecent != null) { if (mostRecent.device.uuid != uuid) { throw new ArgumentException("High watermark returned for incorrect UUID!"); } return(mostRecent.timestamp.AddSeconds(1)); } return(new DateTime(2017, 1, 1, 0, 0, 0, DateTimeKind.Utc)); }
/// <summary> /// Main work method /// </summary> /// <param name="settings">Settings.</param> public void ProcessTags(Settings settings) { this.settings = settings; Uri esPath = new UriBuilder { Host = settings.elasticserver, Port = 9200 }.Uri; EsClient = ElasticUtils.getElasticClient(esPath, settings.indexname, false); var alerts = new List <Alert>(); var allReadings = new List <SensorReading>(); var allDevices = new List <SensorDevice>(); try { if (settings.wirelesstag != null) { WirelessSensorTagAPI tagService = new WirelessSensorTagAPI(settings.wirelesstag.WirelessTagServiceUrl); if (tagService.SignIn(settings.wirelesstag.username, settings.wirelesstag.password)) { var wirelessTagReadings = QuerySensorTags(tagService, allDevices); allReadings.AddRange(wirelessTagReadings); } else { alerts.Add(new Alert { deviceName = "Wireless Tags", alertText = "Sign-in failed." }); } } } catch (Exception ex) { Utils.Log("Exception querying SensorTag data. {0}", ex); } QueryHiveData(allReadings, alerts); QueryWeatherData(settings.weatherUnderground); if (allReadings.Any()) { try { StoreReadings(allReadings, settings.indexname); } catch (Exception ex) { Utils.Log("Exception ingesting data in ES. {0}", ex); } // See if any of the data we got back indicated a drained battery. CheckBatteryStatus(allReadings, settings, alerts); } try { // Now check for any missing data - i.e., long gaps since we last saw anything CheckMissingData(allDevices, alerts); // And send any alerts we saw. if (alerts.Any()) { Utils.Log("Sending {0} alerts.", alerts.Count); if (settings.email != null) { Utils.SendAlertEmail(settings.email, alerts); } } } catch (Exception ex) { Utils.Log("Exception checking missing data. {0}", ex); } Utils.Log("Run complete."); }
/// <summary> /// Main work method /// </summary> /// <param name="settings">Settings.</param> public void ProcessTags(Settings settings) { this.settings = settings; Uri esPath = new UriBuilder { Host = settings.elasticserver, Port = 9200 }.Uri; EsClient = ElasticUtils.getElasticClient(esPath, settings.indexname, false); try { var weatherReadings = QueryWeatherData(settings.weatherUnderground); StoreReadings(weatherReadings, settings.weatherUnderground.IndexName); } catch (Exception ex) { Utils.Log("Exception querying Weather data. {0}", ex); } List <Reading> allReadings = new List <Reading>(); List <SensorDevice> allDevices = new List <SensorDevice>(); try { if (settings.hive != null) { HiveService service = new HiveService(); if (service.SignIn(settings.hive.username, settings.hive.password)) { var hiveReadings = QueryHiveData(service, allDevices); allReadings.AddRange(hiveReadings); } } } catch (Exception ex) { Utils.Log("Exception querying Hive data. {0}", ex); } try { if (settings.wirelesstag != null) { WirelessSensorTagAPI tagService = new WirelessSensorTagAPI(settings.wirelesstag.WirelessTagServiceUrl); if (tagService.SignIn(settings.wirelesstag.username, settings.wirelesstag.password)) { var wirelessTagReadings = QuerySensorTags(tagService, allDevices); allReadings.AddRange(wirelessTagReadings); } } } catch (Exception ex) { Utils.Log("Exception querying SensorTag data. {0}", ex); } if (allReadings.Any() && settings.email != null) { try { StoreReadings(allReadings, settings.indexname); ElasticUtils.DeleteDuplicates(EsClient, settings.indexname); } catch (Exception ex) { Utils.Log("Exception ingesting data in ES. {0}", ex); } try { List <Alert> alerts = new List <Alert>(); // See if any of the data we got back indicated a drained battery. CheckBatteryStatus(allReadings, settings.lowBatteryThreshold, alerts); CheckMissingData(allDevices, alerts); if (alerts.Any()) { Utils.SendAlertEmail(settings.email, alerts); } } catch (Exception ex) { Utils.Log("Exception checking missing data. {0}", ex); } } Utils.Log("Run complete."); }