public static ComputationPeriod InferPeriodFromRecentPoints(TimeSeriesDataServiceResponse timeSeries) { var recentPoints = timeSeries.Points .Skip(timeSeries.Points.Count - MinimumPointCount) .ToList(); if (recentPoints.Count < 2) { return(ComputationPeriod.Unknown); } var periodFrequencyBins = new Dictionary <ComputationPeriod, int>(); for (var i = 0; i < recentPoints.Count - 1; ++i) { var timeSpan = recentPoints[i + 1].Timestamp.DateTimeOffset .Subtract(recentPoints[i].Timestamp.DateTimeOffset); var period = FindClosestPeriod(timeSpan); if (!periodFrequencyBins.ContainsKey(period)) { periodFrequencyBins.Add(period, 0); } periodFrequencyBins[period] += 1; } var mostCommonPeriod = periodFrequencyBins .Select(kvp => kvp) .OrderByDescending(kvp => kvp.Value) .First().Key; return(mostCommonPeriod); }
public void DeleteSensor(TimeSeriesDataServiceResponse timeSeries) { var xml = TransformXmlTemplate(@"XmlTemplates\DeleteSensor.xml", timeSeries); var procedureUniqueId = CreateProcedureUniqueId(timeSeries); try { Log.Info($"Deleting sensor for '{procedureUniqueId}' ..."); PostPox(xml); } catch (WebServiceException exception) { var fieldError = exception.GetFieldErrors().FirstOrDefault(); if (fieldError?.ErrorCode == "InvalidParameterValue" && fieldError.FieldName == "procedure") { // Silently ignore "Time-series not found" when deleting return; } throw; } var existingSensor = FindExistingSensor(procedureUniqueId); if (existingSensor != null) { // Keep the sensor cache up to date Capabilities.Contents.Remove(existingSensor); } }
private void TrimEarlyPoints( TimeSeriesDescription timeSeriesDescription, TimeSeriesDataServiceResponse timeSeries, ComputationPeriod period) { var maximumDaysToExport = Context.Config.MaximumPointDays[period]; if (maximumDaysToExport <= 0 || !timeSeries.Points.Any()) { return; } var earliestDayToUpload = SubtractTimeSpan( timeSeries.Points.Last().Timestamp.DateTimeOffset, TimeSpan.FromDays(maximumDaysToExport)); var remainingPoints = timeSeries.Points .Where(p => p.Timestamp.DateTimeOffset >= earliestDayToUpload) .ToList(); var trimmedPointCount = timeSeries.NumPoints - remainingPoints.Count; Log.Info( $"Trimming '{timeSeriesDescription.Identifier}' {trimmedPointCount} points before {earliestDayToUpload:O} with {remainingPoints.Count} points remaining with Frequency={period}"); timeSeries.Points = remainingPoints; timeSeries.NumPoints = timeSeries.Points.Count; }
private static Dictionary <string, string> CreateSubstitutions(TimeSeriesDataServiceResponse timeSeries) { var timeSeriesIdentifier = CreateProcedureUniqueId(timeSeries); return(new Dictionary <string, string> { { ProcedureUniqueIdKey, timeSeriesIdentifier }, { "{__featureOfInterestId__}", SanitizeIdentifier(timeSeries.LocationIdentifier) }, { "{__observablePropertyName__}", SanitizeIdentifier($"{timeSeries.Parameter}_{timeSeries.Label}") }, }); }
public void FilterTimeSeriesPoints(TimeSeriesDataServiceResponse timeSeries) { if (!Context.Config.Approvals.Any() && !Context.Config.Grades.Any() && !Context.Config.Qualifiers.Any()) { return; } var approvalFilter = new Filter <ApprovalFilter>(Context.Config.Approvals); var gradeFilter = new Filter <GradeFilter>(Context.Config.Grades); var qualifierFilter = new Filter <QualifierFilter>(Context.Config.Qualifiers); var filteredPoints = new List <TimeSeriesPoint>(); foreach (var point in timeSeries.Points) { var approval = timeSeries.Approvals.Single(a => IsPointWithinTimeRange(point, a)); var grade = timeSeries.Grades.Single(g => IsPointWithinTimeRange(point, g)); var qualifiers = timeSeries.Qualifiers.Where(q => IsPointWithinTimeRange(point, q)).ToList(); if (approvalFilter.IsFiltered(f => IsApprovalFiltered(approval, f))) { continue; } if (gradeFilter.IsFiltered(f => IsGradeFiltered(grade, f))) { continue; } if (qualifierFilter.IsFiltered(f => qualifiers.Any(q => IsQualifierFiltered(q, f)))) { continue; } filteredPoints.Add(point); } if (timeSeries.NumPoints == filteredPoints.Count) { return; } var appliedFilters = new[] { approvalFilter.Count > 0 ? "approval" : string.Empty, gradeFilter.Count > 0 ? "grade" : string.Empty, qualifierFilter.Count > 0 ? "qualifier" : string.Empty, } .Where(s => !string.IsNullOrEmpty(s)); Log.Info($"Excluded {timeSeries.NumPoints - filteredPoints.Count} of {timeSeries.NumPoints} points due to {string.Join(" and ", appliedFilters)} filters."); timeSeries.NumPoints = filteredPoints.Count; timeSeries.Points = filteredPoints; }
private void TrimExcludedPoints( TimeSeriesDescription timeSeriesDescription, TimeSeriesDataServiceResponse timeSeries, DateTime?nextChangesSinceToken) { var(exportDuration, exportLabel) = GetExportDuration(timeSeriesDescription); if (exportDuration <= TimeSpan.Zero || !timeSeries.Points.Any()) { return; } var firstTimeToInclude = timeSeries.Points.Last().Timestamp.DateTimeOffset - exportDuration; var firstTimeToExclude = nextChangesSinceToken.HasValue ? new DateTimeOffset(nextChangesSinceToken.Value) : (DateTimeOffset?)null; var nonFuturePoints = timeSeries.Points .Where(p => p.Timestamp.DateTimeOffset < firstTimeToExclude) .ToList(); var futurePointCount = timeSeries.Points.Count - nonFuturePoints.Count; var remainingPoints = nonFuturePoints .Where(p => p.Timestamp.DateTimeOffset >= firstTimeToInclude) .ToList(); if (remainingPoints.Count > Context.MaximumPointsPerSensor) { remainingPoints = remainingPoints .Skip(remainingPoints.Count - Context.MaximumPointsPerSensor) .ToList(); } var earlyPointCount = timeSeries.Points.Count - remainingPoints.Count; var excludedPointCount = futurePointCount + earlyPointCount; if (excludedPointCount <= 0) { return; } Log.Info($"Trimming '{timeSeriesDescription.Identifier}' {"point".ToQuantity(earlyPointCount)} before {firstTimeToInclude:O} and {"point".ToQuantity(futurePointCount)} after {firstTimeToExclude:O} with {remainingPoints.Count} points remaining with ExportDuration={exportLabel}"); timeSeries.Points = remainingPoints; timeSeries.NumPoints = timeSeries.Points.Count; }
private void TrimEarlyPoints( TimeSeriesDescription timeSeriesDescription, TimeSeriesDataServiceResponse timeSeries, ComputationPeriod period) { var maximumDaysToExport = Context.Config.MaximumPointDays[period]; if (maximumDaysToExport <= 0 || !timeSeries.Points.Any()) { return; } var earliestDayToUpload = SubtractTimeSpan( timeSeries.Points.Last().Timestamp.DateTimeOffset, TimeSpan.FromDays(maximumDaysToExport)); var remainingPoints = timeSeries.Points .Where(p => p.Timestamp.DateTimeOffset >= earliestDayToUpload) .ToList(); if (!RoughDailyPointCount.TryGetValue(period, out var expectedDailyPointCount)) { expectedDailyPointCount = 1.0; } var roughPointLimit = Convert.ToInt32(maximumDaysToExport * expectedDailyPointCount * 1.5); if (remainingPoints.Count > roughPointLimit) { var limitExceededCount = remainingPoints.Count - roughPointLimit; Log.Warn($"Upper limit of {roughPointLimit} points exceeded by {limitExceededCount} points for Frequency={period} and MaximumPointDays={maximumDaysToExport} in '{timeSeriesDescription.Identifier}'."); remainingPoints = remainingPoints .Skip(limitExceededCount) .ToList(); } var trimmedPointCount = timeSeries.NumPoints - remainingPoints.Count; Log.Info( $"Trimming '{timeSeriesDescription.Identifier}' {trimmedPointCount} points before {earliestDayToUpload:O} with {remainingPoints.Count} points remaining with Frequency={period}"); timeSeries.Points = remainingPoints; timeSeries.NumPoints = timeSeries.Points.Count; }
public InsertSensorResponse InsertSensor(TimeSeriesDataServiceResponse timeSeries) { var xml = TransformXmlTemplate(@"XmlTemplates\InsertSensor.xml", timeSeries); Log.Info($"Inserting sensor for '{CreateProcedureUniqueId(timeSeries)}' ..."); var responseXml = PostPox(xml); var insertedSensor = FromXml <InsertSensorResponse>(responseXml); // Insert a SensorInfo to the capabilities cache, so that any future calls to FindExistingSensor will find something Capabilities.Contents.Add(new SensorInfo { Procedure = new List <string> { insertedSensor.AssignedProcedure }, Identifier = insertedSensor.AssignedOffering, PhenomenonTime = new List <DateTimeOffset>() }); return(insertedSensor); }
private void TrimExcludedPoints( TimeSeriesDescription timeSeriesDescription, TimeSeriesDataServiceResponse timeSeries) { var(exportDuration, exportLabel) = GetExportDuration(timeSeriesDescription); if (exportDuration <= TimeSpan.Zero || !timeSeries.Points.Any()) { return; } var firstTimeToInclude = timeSeries.Points.Last().Timestamp.DateTimeOffset - exportDuration; var remainingPoints = timeSeries.Points .Where(p => p.Timestamp.DateTimeOffset >= firstTimeToInclude) .ToList(); if (remainingPoints.Count > Context.MaximumPointsPerSensor) { remainingPoints = remainingPoints .Skip(remainingPoints.Count - Context.MaximumPointsPerSensor) .ToList(); } var earlyPointCount = timeSeries.Points.Count - remainingPoints.Count; if (earlyPointCount <= 0) { return; } Log.Info($"Trimming '{timeSeriesDescription.Identifier}' {"point".ToQuantity(earlyPointCount)} before {firstTimeToInclude:O} with {remainingPoints.Count} points remaining with ExportDuration={exportLabel}"); timeSeries.Points = remainingPoints; timeSeries.NumPoints = timeSeries.Points.Count; }
private static string CreateProcedureUniqueId(TimeSeriesDataServiceResponse timeSeries) { return(SanitizeIdentifier($"{ComposeProcedureUniqueId(timeSeries.Parameter, timeSeries.Label, timeSeries.LocationIdentifier)}{InterpolationTypeSuffix[timeSeries.InterpolationTypes.First().Type]}")); }
private static string TransformXmlTemplate(string path, TimeSeriesDataServiceResponse timeSeries) { return(TransformXmlTemplate(path, CreateSubstitutions(timeSeries))); }
public void InsertObservation(string assignedOffering, LocationDataServiceResponse location, LocationDescription locationDescription, TimeSeriesDataServiceResponse timeSeries, TimeSeriesDescription timeSeriesDescription) { var xmlTemplatePath = IsSpatialLocationDefined(location) ? @"XmlTemplates\InsertObservation.xml" : @"XmlTemplates\InsertObservationWithNoGeoSpatial.xml"; const string pointTokenSeparator = ";"; const string pointBlockSeparator = "@"; var observablePropertyFieldName = SanitizeIdentifier($" {timeSeries.Parameter}_{timeSeries.Label}".Replace(" ", "_")); // TODO: Figure this mapping out var substitutions = CreateSubstitutions(timeSeries) .Concat(new Dictionary <string, string> { { "{__offeringUri__}", assignedOffering }, { "{__featureOfInterestName__}", location.LocationName }, { "{__featureOfInterestLatitude__}", $"{location.Latitude}" }, { "{__featureOfInterestLongitude__}", $"{location.Longitude}" }, { "{__observablePropertyUnit__}", SanitizeUnitSymbol(timeSeries.Unit) }, { "{__observablePropertyFieldName__}", observablePropertyFieldName }, { "{__resultTime__}", $"{FixedResultTime:O}" }, { "{__pointTokenSeparator__}", pointTokenSeparator }, { "{__pointBlockSeparator__}", pointBlockSeparator }, }) .ToDictionary(kvp => kvp.Key, kvp => kvp.Value); var procedureUniqueId = substitutions[ProcedureUniqueIdKey]; var existingSensor = FindExistingSensor(procedureUniqueId); for (var insertedPoints = 0; insertedPoints < timeSeries.Points.Count;) { var points = timeSeries.Points .Skip(insertedPoints) .Take(MaximumPointsPerObservation) .ToList(); substitutions["{__phenomenonStartTime__}"] = $"{points.First().Timestamp.DateTimeOffset:O}"; substitutions["{__phenomenonEndTime__}"] = $"{points.Last().Timestamp.DateTimeOffset:O}"; substitutions["{__pointCount__}"] = $"{points.Count}"; substitutions["{__pointValues__}"] = string.Join(pointBlockSeparator, points.Select(p => $"{p.Timestamp.DateTimeOffset:O}{pointTokenSeparator}{p.Value.Display}")); existingSensor.PhenomenonTime = existingSensor.PhenomenonTime .Concat(new[] { points.Last().Timestamp.DateTimeOffset }) .OrderBy(x => x) .ToList(); insertedPoints += points.Count; var xml = TransformXmlTemplate(xmlTemplatePath, substitutions); Log.Info($"Posting {points.Count} data points to '{procedureUniqueId}' ..."); PostPox(xml); } }