private void TrimEarlyPoints( TimeSeriesDescription timeSeriesDescription, TimeSeriesDataServiceResponse timeSeries, ComputationPeriod period) { var maximumDaysToExport = Context.Config.MaximumPointDays[period]; if (maximumDaysToExport <= 0 || !timeSeries.Points.Any()) { return; } var earliestDayToUpload = SubtractTimeSpan( timeSeries.Points.Last().Timestamp.DateTimeOffset, TimeSpan.FromDays(maximumDaysToExport)); var remainingPoints = timeSeries.Points .Where(p => p.Timestamp.DateTimeOffset >= earliestDayToUpload) .ToList(); var trimmedPointCount = timeSeries.NumPoints - remainingPoints.Count; Log.Info( $"Trimming '{timeSeriesDescription.Identifier}' {trimmedPointCount} points before {earliestDayToUpload:O} with {remainingPoints.Count} points remaining with Frequency={period}"); timeSeries.Points = remainingPoints; timeSeries.NumPoints = timeSeries.Points.Count; }
private (TimeSpan ExportDuration, string ExportLabel) GetExportDuration(TimeSeriesDescription timeSeriesDescription) { var durationAttribute = timeSeriesDescription .ExtendedAttributes .FirstOrDefault(a => a.Name.Equals(Context.Config.ExportDurationAttributeName, StringComparison.InvariantCultureIgnoreCase)); var attributeValue = durationAttribute?.Value as string; return(ParseHumanDuration(attributeValue) ?? TimeSpan.FromDays(Context.Config.DefaultExportDurationDays), !string.IsNullOrWhiteSpace(attributeValue) ? attributeValue : nameof(Context.Config.DefaultExportDurationDays)); }
private TimeSeriesDataServiceResponse FetchMinimumTimeSeries( TimeSeriesChangeEvent detectedChange, TimeSeriesDescription timeSeriesDescription, SensorInfo existingSensor, TimeSeriesDataCorrectedServiceRequest dataRequest, ref bool deleteExistingSensor, ref ComputationPeriod period) { TimeSeriesDataServiceResponse timeSeries; if (!deleteExistingSensor && GetLastSensorTime(existingSensor) < dataRequest.QueryFrom) { // All the changed points have occurred after the last sensor point which exists in the SOS server. // This is the preferred code path, since we only need to export the new points. timeSeries = Aquarius.Publish.Get(dataRequest); if (period == ComputationPeriod.Unknown) { // We may have just fetched enough recent points to determine the time-series frequency period = ComputationPeriodEstimator.InferPeriodFromRecentPoints(timeSeries); } TrimEarlyPoints(timeSeriesDescription, timeSeries, period); return(timeSeries); } if (GetLastSensorTime(existingSensor) >= detectedChange.FirstPointChanged) { // A point has changed before the last known observation, so we'll need to throw out the entire sensor deleteExistingSensor = true; // We'll also need to fetch more data again dataRequest.QueryFrom = null; } timeSeries = FetchRecentSignal(timeSeriesDescription, dataRequest, ref period); if (GetLastSensorTime(existingSensor) >= detectedChange.FirstPointChanged) { // A point has changed before the last known observation, so we'll need to throw out the entire sensor deleteExistingSensor = true; // We'll also need to fetch more data again dataRequest.QueryFrom = null; timeSeries = FetchRecentSignal(timeSeriesDescription, dataRequest, ref period); } TrimEarlyPoints(timeSeriesDescription, timeSeries, period); return(timeSeries); }
private void TrimExcludedPoints( TimeSeriesDescription timeSeriesDescription, TimeSeriesDataServiceResponse timeSeries, DateTime?nextChangesSinceToken) { var(exportDuration, exportLabel) = GetExportDuration(timeSeriesDescription); if (exportDuration <= TimeSpan.Zero || !timeSeries.Points.Any()) { return; } var firstTimeToInclude = timeSeries.Points.Last().Timestamp.DateTimeOffset - exportDuration; var firstTimeToExclude = nextChangesSinceToken.HasValue ? new DateTimeOffset(nextChangesSinceToken.Value) : (DateTimeOffset?)null; var nonFuturePoints = timeSeries.Points .Where(p => p.Timestamp.DateTimeOffset < firstTimeToExclude) .ToList(); var futurePointCount = timeSeries.Points.Count - nonFuturePoints.Count; var remainingPoints = nonFuturePoints .Where(p => p.Timestamp.DateTimeOffset >= firstTimeToInclude) .ToList(); if (remainingPoints.Count > Context.MaximumPointsPerSensor) { remainingPoints = remainingPoints .Skip(remainingPoints.Count - Context.MaximumPointsPerSensor) .ToList(); } var earlyPointCount = timeSeries.Points.Count - remainingPoints.Count; var excludedPointCount = futurePointCount + earlyPointCount; if (excludedPointCount <= 0) { return; } Log.Info($"Trimming '{timeSeriesDescription.Identifier}' {"point".ToQuantity(earlyPointCount)} before {firstTimeToInclude:O} and {"point".ToQuantity(futurePointCount)} after {firstTimeToExclude:O} with {remainingPoints.Count} points remaining with ExportDuration={exportLabel}"); timeSeries.Points = remainingPoints; timeSeries.NumPoints = timeSeries.Points.Count; }
private void TrimEarlyPoints( TimeSeriesDescription timeSeriesDescription, TimeSeriesDataServiceResponse timeSeries, ComputationPeriod period) { var maximumDaysToExport = Context.Config.MaximumPointDays[period]; if (maximumDaysToExport <= 0 || !timeSeries.Points.Any()) { return; } var earliestDayToUpload = SubtractTimeSpan( timeSeries.Points.Last().Timestamp.DateTimeOffset, TimeSpan.FromDays(maximumDaysToExport)); var remainingPoints = timeSeries.Points .Where(p => p.Timestamp.DateTimeOffset >= earliestDayToUpload) .ToList(); if (!RoughDailyPointCount.TryGetValue(period, out var expectedDailyPointCount)) { expectedDailyPointCount = 1.0; } var roughPointLimit = Convert.ToInt32(maximumDaysToExport * expectedDailyPointCount * 1.5); if (remainingPoints.Count > roughPointLimit) { var limitExceededCount = remainingPoints.Count - roughPointLimit; Log.Warn($"Upper limit of {roughPointLimit} points exceeded by {limitExceededCount} points for Frequency={period} and MaximumPointDays={maximumDaysToExport} in '{timeSeriesDescription.Identifier}'."); remainingPoints = remainingPoints .Skip(limitExceededCount) .ToList(); } var trimmedPointCount = timeSeries.NumPoints - remainingPoints.Count; Log.Info( $"Trimming '{timeSeriesDescription.Identifier}' {trimmedPointCount} points before {earliestDayToUpload:O} with {remainingPoints.Count} points remaining with Frequency={period}"); timeSeries.Points = remainingPoints; timeSeries.NumPoints = timeSeries.Points.Count; }
private void TrimExcludedPoints( TimeSeriesDescription timeSeriesDescription, TimeSeriesDataServiceResponse timeSeries) { var(exportDuration, exportLabel) = GetExportDuration(timeSeriesDescription); if (exportDuration <= TimeSpan.Zero || !timeSeries.Points.Any()) { return; } var firstTimeToInclude = timeSeries.Points.Last().Timestamp.DateTimeOffset - exportDuration; var remainingPoints = timeSeries.Points .Where(p => p.Timestamp.DateTimeOffset >= firstTimeToInclude) .ToList(); if (remainingPoints.Count > Context.MaximumPointsPerSensor) { remainingPoints = remainingPoints .Skip(remainingPoints.Count - Context.MaximumPointsPerSensor) .ToList(); } var earlyPointCount = timeSeries.Points.Count - remainingPoints.Count; if (earlyPointCount <= 0) { return; } Log.Info($"Trimming '{timeSeriesDescription.Identifier}' {"point".ToQuantity(earlyPointCount)} before {firstTimeToInclude:O} with {remainingPoints.Count} points remaining with ExportDuration={exportLabel}"); timeSeries.Points = remainingPoints; timeSeries.NumPoints = timeSeries.Points.Count; }
private bool HaveExistingSosPointsChanged( TimeSeriesDataCorrectedServiceRequest dataRequest, DateTimeOffset?lastSensorTime, TimeSeriesChangeEvent detectedChange, TimeSeriesDescription timeSeriesDescription) { if (detectedChange.HasAttributeChange ?? false) { return(true); } if (!detectedChange.FirstPointChanged.HasValue || !lastSensorTime.HasValue) { return(false); } if (lastSensorTime < detectedChange.FirstPointChanged) { return(false); } var timeSeriesIdentifier = timeSeriesDescription.Identifier; var sosPoints = new Queue <TimeSeriesPoint>(Sos.GetObservations(timeSeriesDescription, AddMilliseconds(detectedChange.FirstPointChanged.Value, -1), AddMilliseconds(lastSensorTime.Value, 1))); var aqtsPoints = new Queue <TimeSeriesPoint>(Aquarius.Publish.Get(dataRequest).Points); var sosCount = sosPoints.Count; var aqtsCount = aqtsPoints.Count; Log.Info($"Fetched {sosCount} SOS points and {aqtsCount} AQUARIUS points for '{timeSeriesIdentifier}' from {dataRequest.QueryFrom:O} ..."); while (sosPoints.Any() || aqtsPoints.Any()) { var sosPoint = sosPoints.FirstOrDefault(); var aqtsPoint = aqtsPoints.FirstOrDefault(); if (aqtsPoint == null) { Log.Warn($"'{timeSeriesIdentifier}': AQUARIUS now has fewer points than SOS@{sosPoint?.Timestamp.DateTimeOffset:O}"); return(true); } if (sosPoint == null) { break; } var aqtsValue = (dataRequest.ApplyRounding ?? false) ? double.Parse(aqtsPoint.Value.Display) : aqtsPoint.Value.Numeric; var sosValue = sosPoint.Value.Numeric; if (sosPoint.Timestamp.DateTimeOffset != aqtsPoint.Timestamp.DateTimeOffset) { Log.Warn($"'{timeSeriesIdentifier}': Different timestamps: AQUARIUS={aqtsValue}@{aqtsPoint.Timestamp.DateTimeOffset:O} vs SOS={sosValue}@{sosPoint.Timestamp.DateTimeOffset:O}"); return(true); } if (!DoubleHelper.AreSame(aqtsValue, sosValue)) { Log.Warn($"'{timeSeriesIdentifier}': Different values @ {aqtsPoint.Timestamp.DateTimeOffset:O}: AQUARIUS={aqtsValue} vs SOS={sosValue}"); return(true); } sosPoints.Dequeue(); aqtsPoints.Dequeue(); } Log.Info($"'{timeSeriesDescription.Identifier}': All {sosCount} SOS points match between SOS and AQUARIUS."); dataRequest.QueryFrom = lastSensorTime.Value.AddTicks(1); return(false); }
private void ExportTimeSeries(bool clearExportedData, TimeSeriesChangeEvent detectedChange, TimeSeriesDescription timeSeriesDescription) { var locationInfo = GetLocationInfo(timeSeriesDescription.LocationIdentifier); var(exportDuration, exportLabel) = GetExportDuration(timeSeriesDescription); var dataRequest = new TimeSeriesDataCorrectedServiceRequest { TimeSeriesUniqueId = timeSeriesDescription.UniqueId, QueryFrom = GetInitialQueryFrom(detectedChange), ApplyRounding = Context.ApplyRounding, }; var existingSensor = Sos.FindExistingSensor(timeSeriesDescription); var deleteExistingSensor = clearExportedData && existingSensor != null; var assignedOffering = existingSensor?.Identifier; var lastSensorTime = GetLastSensorTime(existingSensor); if (HaveExistingSosPointsChanged(dataRequest, lastSensorTime, detectedChange, timeSeriesDescription)) { Log.Warn($"FirstPointChanged={detectedChange.FirstPointChanged:O} AttributeChange={detectedChange.HasAttributeChange} of '{timeSeriesDescription.Identifier}' precedes LastSensorTime={lastSensorTime:O} of '{existingSensor?.Identifier}'. Forcing delete of existing sensor."); // A point has changed before the last known observation, so we'll need to throw out the entire sensor deleteExistingSensor = true; // We'll also need to fetch more data again dataRequest.QueryFrom = null; } if (dataRequest.QueryFrom == null) { // Get the full extraction var endPoint = dataRequest.QueryTo ?? DateTimeOffset.Now; var startOfToday = new DateTimeOffset(endPoint.Year, endPoint.Month, endPoint.Day, 0, 0, 0, timeSeriesDescription.UtcOffsetIsoDuration.ToTimeSpan()); dataRequest.QueryFrom = startOfToday - exportDuration; } Log.Info($"Fetching changes from '{timeSeriesDescription.Identifier}' FirstPointChanged={detectedChange.FirstPointChanged:O} HasAttributeChanged={detectedChange.HasAttributeChange} QueryFrom={dataRequest.QueryFrom:O} ..."); var timeSeries = Aquarius.Publish.Get(dataRequest); TrimExcludedPoints(timeSeriesDescription, timeSeries); var createSensor = existingSensor == null || deleteExistingSensor; TimeSeriesPointFilter.FilterTimeSeriesPoints(timeSeries); var exportSummary = $"{timeSeries.NumPoints} points [{timeSeries.Points.FirstOrDefault()?.Timestamp.DateTimeOffset:O} to {timeSeries.Points.LastOrDefault()?.Timestamp.DateTimeOffset:O}] from '{timeSeriesDescription.Identifier}' with ExportDuration={exportLabel}"; ExportedTimeSeriesCount += 1; ExportedPointCount += timeSeries.NumPoints ?? 0; if (Context.DryRun) { if (deleteExistingSensor) { LogDryRun($"Would delete existing sensor '{existingSensor?.Identifier}'"); } if (createSensor) { LogDryRun($"Would create new sensor for '{timeSeriesDescription.Identifier}'"); } LogDryRun($"Would export {exportSummary}."); return; } Log.Info($"Exporting {exportSummary} ..."); if (deleteExistingSensor) { Sos.DeleteSensor(timeSeries); Sos.DeleteDeletedObservations(); } if (createSensor) { var sensor = Sos.InsertSensor(timeSeries); assignedOffering = sensor.AssignedOffering; } Sos.InsertObservation(assignedOffering, locationInfo.LocationData, locationInfo.LocationDescription, timeSeries); }
private void ExportTimeSeries( bool clearExportedData, TimeSeriesChangeEvent detectedChange, TimeSeriesDescription timeSeriesDescription) { Log.Info($"Fetching changes from '{timeSeriesDescription.Identifier}' FirstPointChanged={detectedChange.FirstPointChanged:O} HasAttributeChanged={detectedChange.HasAttributeChange} ..."); var locationInfo = GetLocationInfo(timeSeriesDescription.LocationIdentifier); var period = GetTimeSeriesPeriod(timeSeriesDescription); var dataRequest = new TimeSeriesDataCorrectedServiceRequest { TimeSeriesUniqueId = timeSeriesDescription.UniqueId, QueryFrom = GetInitialQueryFrom(detectedChange), ApplyRounding = Context.ApplyRounding, }; var existingSensor = Sos.FindExistingSensor(timeSeriesDescription); var deleteExistingSensor = clearExportedData && existingSensor != null; var assignedOffering = existingSensor?.Identifier; var timeSeries = FetchMinimumTimeSeries(detectedChange, timeSeriesDescription, existingSensor, dataRequest, ref deleteExistingSensor, ref period); var createSensor = existingSensor == null || deleteExistingSensor; TimeSeriesPointFilter.FilterTimeSeriesPoints(timeSeries); var exportSummary = $"{timeSeries.NumPoints} points [{timeSeries.Points.FirstOrDefault()?.Timestamp.DateTimeOffset:O} to {timeSeries.Points.LastOrDefault()?.Timestamp.DateTimeOffset:O}] from '{timeSeriesDescription.Identifier}' with Frequency={period}"; ExportedTimeSeriesCount += 1; ExportedPointCount += timeSeries.NumPoints ?? 0; if (Context.DryRun) { if (deleteExistingSensor) { LogDryRun($"Would delete existing sensor '{existingSensor?.Identifier}'"); } if (createSensor) { LogDryRun($"Would create new sensor for '{timeSeriesDescription.Identifier}'"); } LogDryRun($"Would export {exportSummary}."); return; } Log.Info($"Exporting {exportSummary} ..."); if (deleteExistingSensor) { Sos.DeleteSensor(timeSeries); Sos.DeleteDeletedObservations(); } if (createSensor) { var sensor = Sos.InsertSensor(timeSeries); assignedOffering = sensor.AssignedOffering; } Sos.InsertObservation(assignedOffering, locationInfo.LocationData, locationInfo.LocationDescription, timeSeries, timeSeriesDescription); }