public static ComputationPeriod InferPeriodFromRecentPoints(TimeSeriesDataServiceResponse timeSeries)
{
var recentPoints = timeSeries.Points
.Skip(timeSeries.Points.Count - MinimumPointCount)
.ToList();
if (recentPoints.Count < 2)
{
return(ComputationPeriod.Unknown);
}
var periodFrequencyBins = new Dictionary <ComputationPeriod, int>();
for (var i = 0; i < recentPoints.Count - 1; ++i)
{
var timeSpan = recentPoints[i + 1].Timestamp.DateTimeOffset
.Subtract(recentPoints[i].Timestamp.DateTimeOffset);
var period = FindClosestPeriod(timeSpan);
if (!periodFrequencyBins.ContainsKey(period))
{
periodFrequencyBins.Add(period, 0);
}
periodFrequencyBins[period] += 1;
}
var mostCommonPeriod = periodFrequencyBins
.Select(kvp => kvp)
.OrderByDescending(kvp => kvp.Value)
.First().Key;
return(mostCommonPeriod);
}
public void DeleteSensor(TimeSeriesDataServiceResponse timeSeries)
{
var xml = TransformXmlTemplate(@"XmlTemplates\DeleteSensor.xml", timeSeries);
var procedureUniqueId = CreateProcedureUniqueId(timeSeries);
try
{
Log.Info($"Deleting sensor for '{procedureUniqueId}' ...");
PostPox(xml);
}
catch (WebServiceException exception)
{
var fieldError = exception.GetFieldErrors().FirstOrDefault();
if (fieldError?.ErrorCode == "InvalidParameterValue" && fieldError.FieldName == "procedure")
{
// Silently ignore "Time-series not found" when deleting
return;
}
throw;
}
var existingSensor = FindExistingSensor(procedureUniqueId);
if (existingSensor != null)
{
// Keep the sensor cache up to date
Capabilities.Contents.Remove(existingSensor);
}
}
private void TrimEarlyPoints(
TimeSeriesDescription timeSeriesDescription,
TimeSeriesDataServiceResponse timeSeries,
ComputationPeriod period)
{
var maximumDaysToExport = Context.Config.MaximumPointDays[period];
if (maximumDaysToExport <= 0 || !timeSeries.Points.Any())
{
return;
}
var earliestDayToUpload = SubtractTimeSpan(
timeSeries.Points.Last().Timestamp.DateTimeOffset,
TimeSpan.FromDays(maximumDaysToExport));
var remainingPoints = timeSeries.Points
.Where(p => p.Timestamp.DateTimeOffset >= earliestDayToUpload)
.ToList();
var trimmedPointCount = timeSeries.NumPoints - remainingPoints.Count;
Log.Info(
$"Trimming '{timeSeriesDescription.Identifier}' {trimmedPointCount} points before {earliestDayToUpload:O} with {remainingPoints.Count} points remaining with Frequency={period}");
timeSeries.Points = remainingPoints;
timeSeries.NumPoints = timeSeries.Points.Count;
}
private static Dictionary <string, string> CreateSubstitutions(TimeSeriesDataServiceResponse timeSeries)
{
var timeSeriesIdentifier = CreateProcedureUniqueId(timeSeries);
return(new Dictionary <string, string>
{
{ ProcedureUniqueIdKey, timeSeriesIdentifier },
{ "{__featureOfInterestId__}", SanitizeIdentifier(timeSeries.LocationIdentifier) },
{ "{__observablePropertyName__}", SanitizeIdentifier($"{timeSeries.Parameter}_{timeSeries.Label}") },
});
}
public void FilterTimeSeriesPoints(TimeSeriesDataServiceResponse timeSeries)
{
if (!Context.Config.Approvals.Any() && !Context.Config.Grades.Any() && !Context.Config.Qualifiers.Any())
{
return;
}
var approvalFilter = new Filter <ApprovalFilter>(Context.Config.Approvals);
var gradeFilter = new Filter <GradeFilter>(Context.Config.Grades);
var qualifierFilter = new Filter <QualifierFilter>(Context.Config.Qualifiers);
var filteredPoints = new List <TimeSeriesPoint>();
foreach (var point in timeSeries.Points)
{
var approval = timeSeries.Approvals.Single(a => IsPointWithinTimeRange(point, a));
var grade = timeSeries.Grades.Single(g => IsPointWithinTimeRange(point, g));
var qualifiers = timeSeries.Qualifiers.Where(q => IsPointWithinTimeRange(point, q)).ToList();
if (approvalFilter.IsFiltered(f => IsApprovalFiltered(approval, f)))
{
continue;
}
if (gradeFilter.IsFiltered(f => IsGradeFiltered(grade, f)))
{
continue;
}
if (qualifierFilter.IsFiltered(f => qualifiers.Any(q => IsQualifierFiltered(q, f))))
{
continue;
}
filteredPoints.Add(point);
}
if (timeSeries.NumPoints == filteredPoints.Count)
{
return;
}
var appliedFilters = new[]
{
approvalFilter.Count > 0 ? "approval" : string.Empty,
gradeFilter.Count > 0 ? "grade" : string.Empty,
qualifierFilter.Count > 0 ? "qualifier" : string.Empty,
}
.Where(s => !string.IsNullOrEmpty(s));
Log.Info($"Excluded {timeSeries.NumPoints - filteredPoints.Count} of {timeSeries.NumPoints} points due to {string.Join(" and ", appliedFilters)} filters.");
timeSeries.NumPoints = filteredPoints.Count;
timeSeries.Points = filteredPoints;
}
private void TrimExcludedPoints(
TimeSeriesDescription timeSeriesDescription,
TimeSeriesDataServiceResponse timeSeries,
DateTime?nextChangesSinceToken)
{
var(exportDuration, exportLabel) = GetExportDuration(timeSeriesDescription);
if (exportDuration <= TimeSpan.Zero || !timeSeries.Points.Any())
{
return;
}
var firstTimeToInclude = timeSeries.Points.Last().Timestamp.DateTimeOffset - exportDuration;
var firstTimeToExclude = nextChangesSinceToken.HasValue
? new DateTimeOffset(nextChangesSinceToken.Value)
: (DateTimeOffset?)null;
var nonFuturePoints = timeSeries.Points
.Where(p => p.Timestamp.DateTimeOffset < firstTimeToExclude)
.ToList();
var futurePointCount = timeSeries.Points.Count - nonFuturePoints.Count;
var remainingPoints = nonFuturePoints
.Where(p => p.Timestamp.DateTimeOffset >= firstTimeToInclude)
.ToList();
if (remainingPoints.Count > Context.MaximumPointsPerSensor)
{
remainingPoints = remainingPoints
.Skip(remainingPoints.Count - Context.MaximumPointsPerSensor)
.ToList();
}
var earlyPointCount = timeSeries.Points.Count - remainingPoints.Count;
var excludedPointCount = futurePointCount + earlyPointCount;
if (excludedPointCount <= 0)
{
return;
}
Log.Info($"Trimming '{timeSeriesDescription.Identifier}' {"point".ToQuantity(earlyPointCount)} before {firstTimeToInclude:O} and {"point".ToQuantity(futurePointCount)} after {firstTimeToExclude:O} with {remainingPoints.Count} points remaining with ExportDuration={exportLabel}");
timeSeries.Points = remainingPoints;
timeSeries.NumPoints = timeSeries.Points.Count;
}
private void TrimEarlyPoints(
TimeSeriesDescription timeSeriesDescription,
TimeSeriesDataServiceResponse timeSeries,
ComputationPeriod period)
{
var maximumDaysToExport = Context.Config.MaximumPointDays[period];
if (maximumDaysToExport <= 0 || !timeSeries.Points.Any())
{
return;
}
var earliestDayToUpload = SubtractTimeSpan(
timeSeries.Points.Last().Timestamp.DateTimeOffset,
TimeSpan.FromDays(maximumDaysToExport));
var remainingPoints = timeSeries.Points
.Where(p => p.Timestamp.DateTimeOffset >= earliestDayToUpload)
.ToList();
if (!RoughDailyPointCount.TryGetValue(period, out var expectedDailyPointCount))
{
expectedDailyPointCount = 1.0;
}
var roughPointLimit = Convert.ToInt32(maximumDaysToExport * expectedDailyPointCount * 1.5);
if (remainingPoints.Count > roughPointLimit)
{
var limitExceededCount = remainingPoints.Count - roughPointLimit;
Log.Warn($"Upper limit of {roughPointLimit} points exceeded by {limitExceededCount} points for Frequency={period} and MaximumPointDays={maximumDaysToExport} in '{timeSeriesDescription.Identifier}'.");
remainingPoints = remainingPoints
.Skip(limitExceededCount)
.ToList();
}
var trimmedPointCount = timeSeries.NumPoints - remainingPoints.Count;
Log.Info(
$"Trimming '{timeSeriesDescription.Identifier}' {trimmedPointCount} points before {earliestDayToUpload:O} with {remainingPoints.Count} points remaining with Frequency={period}");
timeSeries.Points = remainingPoints;
timeSeries.NumPoints = timeSeries.Points.Count;
}
public InsertSensorResponse InsertSensor(TimeSeriesDataServiceResponse timeSeries)
{
var xml = TransformXmlTemplate(@"XmlTemplates\InsertSensor.xml", timeSeries);
Log.Info($"Inserting sensor for '{CreateProcedureUniqueId(timeSeries)}' ...");
var responseXml = PostPox(xml);
var insertedSensor = FromXml <InsertSensorResponse>(responseXml);
// Insert a SensorInfo to the capabilities cache, so that any future calls to FindExistingSensor will find something
Capabilities.Contents.Add(new SensorInfo
{
Procedure = new List <string> {
insertedSensor.AssignedProcedure
},
Identifier = insertedSensor.AssignedOffering,
PhenomenonTime = new List <DateTimeOffset>()
});
return(insertedSensor);
}
private void TrimExcludedPoints(
TimeSeriesDescription timeSeriesDescription,
TimeSeriesDataServiceResponse timeSeries)
{
var(exportDuration, exportLabel) = GetExportDuration(timeSeriesDescription);
if (exportDuration <= TimeSpan.Zero || !timeSeries.Points.Any())
{
return;
}
var firstTimeToInclude = timeSeries.Points.Last().Timestamp.DateTimeOffset - exportDuration;
var remainingPoints = timeSeries.Points
.Where(p => p.Timestamp.DateTimeOffset >= firstTimeToInclude)
.ToList();
if (remainingPoints.Count > Context.MaximumPointsPerSensor)
{
remainingPoints = remainingPoints
.Skip(remainingPoints.Count - Context.MaximumPointsPerSensor)
.ToList();
}
var earlyPointCount = timeSeries.Points.Count - remainingPoints.Count;
if (earlyPointCount <= 0)
{
return;
}
Log.Info($"Trimming '{timeSeriesDescription.Identifier}' {"point".ToQuantity(earlyPointCount)} before {firstTimeToInclude:O} with {remainingPoints.Count} points remaining with ExportDuration={exportLabel}");
timeSeries.Points = remainingPoints;
timeSeries.NumPoints = timeSeries.Points.Count;
}
private static string CreateProcedureUniqueId(TimeSeriesDataServiceResponse timeSeries)
{
return(SanitizeIdentifier($"{ComposeProcedureUniqueId(timeSeries.Parameter, timeSeries.Label, timeSeries.LocationIdentifier)}{InterpolationTypeSuffix[timeSeries.InterpolationTypes.First().Type]}"));
}
private static string TransformXmlTemplate(string path, TimeSeriesDataServiceResponse timeSeries)
{
return(TransformXmlTemplate(path, CreateSubstitutions(timeSeries)));
}
public void InsertObservation(string assignedOffering, LocationDataServiceResponse location, LocationDescription locationDescription, TimeSeriesDataServiceResponse timeSeries, TimeSeriesDescription timeSeriesDescription)
{
var xmlTemplatePath = IsSpatialLocationDefined(location)
? @"XmlTemplates\InsertObservation.xml"
: @"XmlTemplates\InsertObservationWithNoGeoSpatial.xml";
const string pointTokenSeparator = ";";
const string pointBlockSeparator = "@";
var observablePropertyFieldName = SanitizeIdentifier($" {timeSeries.Parameter}_{timeSeries.Label}".Replace(" ", "_")); // TODO: Figure this mapping out
var substitutions = CreateSubstitutions(timeSeries)
.Concat(new Dictionary <string, string>
{
{ "{__offeringUri__}", assignedOffering },
{ "{__featureOfInterestName__}", location.LocationName },
{ "{__featureOfInterestLatitude__}", $"{location.Latitude}" },
{ "{__featureOfInterestLongitude__}", $"{location.Longitude}" },
{ "{__observablePropertyUnit__}", SanitizeUnitSymbol(timeSeries.Unit) },
{ "{__observablePropertyFieldName__}", observablePropertyFieldName },
{ "{__resultTime__}", $"{FixedResultTime:O}" },
{ "{__pointTokenSeparator__}", pointTokenSeparator },
{ "{__pointBlockSeparator__}", pointBlockSeparator },
})
.ToDictionary(kvp => kvp.Key, kvp => kvp.Value);
var procedureUniqueId = substitutions[ProcedureUniqueIdKey];
var existingSensor = FindExistingSensor(procedureUniqueId);
for (var insertedPoints = 0; insertedPoints < timeSeries.Points.Count;)
{
var points = timeSeries.Points
.Skip(insertedPoints)
.Take(MaximumPointsPerObservation)
.ToList();
substitutions["{__phenomenonStartTime__}"] = $"{points.First().Timestamp.DateTimeOffset:O}";
substitutions["{__phenomenonEndTime__}"] = $"{points.Last().Timestamp.DateTimeOffset:O}";
substitutions["{__pointCount__}"] = $"{points.Count}";
substitutions["{__pointValues__}"] = string.Join(pointBlockSeparator, points.Select(p => $"{p.Timestamp.DateTimeOffset:O}{pointTokenSeparator}{p.Value.Display}"));
existingSensor.PhenomenonTime = existingSensor.PhenomenonTime
.Concat(new[] { points.Last().Timestamp.DateTimeOffset })
.OrderBy(x => x)
.ToList();
insertedPoints += points.Count;
var xml = TransformXmlTemplate(xmlTemplatePath, substitutions);
Log.Info($"Posting {points.Count} data points to '{procedureUniqueId}' ...");
PostPox(xml);
}
}
