private void TrimEarlyPoints(
TimeSeriesDescription timeSeriesDescription,
TimeSeriesDataServiceResponse timeSeries,
ComputationPeriod period)
{
var maximumDaysToExport = Context.Config.MaximumPointDays[period];
if (maximumDaysToExport <= 0 || !timeSeries.Points.Any())
{
return;
}
var earliestDayToUpload = SubtractTimeSpan(
timeSeries.Points.Last().Timestamp.DateTimeOffset,
TimeSpan.FromDays(maximumDaysToExport));
var remainingPoints = timeSeries.Points
.Where(p => p.Timestamp.DateTimeOffset >= earliestDayToUpload)
.ToList();
var trimmedPointCount = timeSeries.NumPoints - remainingPoints.Count;
Log.Info(
$"Trimming '{timeSeriesDescription.Identifier}' {trimmedPointCount} points before {earliestDayToUpload:O} with {remainingPoints.Count} points remaining with Frequency={period}");
timeSeries.Points = remainingPoints;
timeSeries.NumPoints = timeSeries.Points.Count;
}
private (TimeSpan ExportDuration, string ExportLabel) GetExportDuration(TimeSeriesDescription timeSeriesDescription)
{
var durationAttribute = timeSeriesDescription
.ExtendedAttributes
.FirstOrDefault(a => a.Name.Equals(Context.Config.ExportDurationAttributeName,
StringComparison.InvariantCultureIgnoreCase));
var attributeValue = durationAttribute?.Value as string;
return(ParseHumanDuration(attributeValue) ?? TimeSpan.FromDays(Context.Config.DefaultExportDurationDays),
!string.IsNullOrWhiteSpace(attributeValue) ? attributeValue : nameof(Context.Config.DefaultExportDurationDays));
}
private TimeSeriesDataServiceResponse FetchMinimumTimeSeries(
TimeSeriesChangeEvent detectedChange,
TimeSeriesDescription timeSeriesDescription,
SensorInfo existingSensor,
TimeSeriesDataCorrectedServiceRequest dataRequest,
ref bool deleteExistingSensor,
ref ComputationPeriod period)
{
TimeSeriesDataServiceResponse timeSeries;
if (!deleteExistingSensor && GetLastSensorTime(existingSensor) < dataRequest.QueryFrom)
{
// All the changed points have occurred after the last sensor point which exists in the SOS server.
// This is the preferred code path, since we only need to export the new points.
timeSeries = Aquarius.Publish.Get(dataRequest);
if (period == ComputationPeriod.Unknown)
{
// We may have just fetched enough recent points to determine the time-series frequency
period = ComputationPeriodEstimator.InferPeriodFromRecentPoints(timeSeries);
}
TrimEarlyPoints(timeSeriesDescription, timeSeries, period);
return(timeSeries);
}
if (GetLastSensorTime(existingSensor) >= detectedChange.FirstPointChanged)
{
// A point has changed before the last known observation, so we'll need to throw out the entire sensor
deleteExistingSensor = true;
// We'll also need to fetch more data again
dataRequest.QueryFrom = null;
}
timeSeries = FetchRecentSignal(timeSeriesDescription, dataRequest, ref period);
if (GetLastSensorTime(existingSensor) >= detectedChange.FirstPointChanged)
{
// A point has changed before the last known observation, so we'll need to throw out the entire sensor
deleteExistingSensor = true;
// We'll also need to fetch more data again
dataRequest.QueryFrom = null;
timeSeries = FetchRecentSignal(timeSeriesDescription, dataRequest, ref period);
}
TrimEarlyPoints(timeSeriesDescription, timeSeries, period);
return(timeSeries);
}
private void TrimExcludedPoints(
TimeSeriesDescription timeSeriesDescription,
TimeSeriesDataServiceResponse timeSeries,
DateTime?nextChangesSinceToken)
{
var(exportDuration, exportLabel) = GetExportDuration(timeSeriesDescription);
if (exportDuration <= TimeSpan.Zero || !timeSeries.Points.Any())
{
return;
}
var firstTimeToInclude = timeSeries.Points.Last().Timestamp.DateTimeOffset - exportDuration;
var firstTimeToExclude = nextChangesSinceToken.HasValue
? new DateTimeOffset(nextChangesSinceToken.Value)
: (DateTimeOffset?)null;
var nonFuturePoints = timeSeries.Points
.Where(p => p.Timestamp.DateTimeOffset < firstTimeToExclude)
.ToList();
var futurePointCount = timeSeries.Points.Count - nonFuturePoints.Count;
var remainingPoints = nonFuturePoints
.Where(p => p.Timestamp.DateTimeOffset >= firstTimeToInclude)
.ToList();
if (remainingPoints.Count > Context.MaximumPointsPerSensor)
{
remainingPoints = remainingPoints
.Skip(remainingPoints.Count - Context.MaximumPointsPerSensor)
.ToList();
}
var earlyPointCount = timeSeries.Points.Count - remainingPoints.Count;
var excludedPointCount = futurePointCount + earlyPointCount;
if (excludedPointCount <= 0)
{
return;
}
Log.Info($"Trimming '{timeSeriesDescription.Identifier}' {"point".ToQuantity(earlyPointCount)} before {firstTimeToInclude:O} and {"point".ToQuantity(futurePointCount)} after {firstTimeToExclude:O} with {remainingPoints.Count} points remaining with ExportDuration={exportLabel}");
timeSeries.Points = remainingPoints;
timeSeries.NumPoints = timeSeries.Points.Count;
}
private void TrimEarlyPoints(
TimeSeriesDescription timeSeriesDescription,
TimeSeriesDataServiceResponse timeSeries,
ComputationPeriod period)
{
var maximumDaysToExport = Context.Config.MaximumPointDays[period];
if (maximumDaysToExport <= 0 || !timeSeries.Points.Any())
{
return;
}
var earliestDayToUpload = SubtractTimeSpan(
timeSeries.Points.Last().Timestamp.DateTimeOffset,
TimeSpan.FromDays(maximumDaysToExport));
var remainingPoints = timeSeries.Points
.Where(p => p.Timestamp.DateTimeOffset >= earliestDayToUpload)
.ToList();
if (!RoughDailyPointCount.TryGetValue(period, out var expectedDailyPointCount))
{
expectedDailyPointCount = 1.0;
}
var roughPointLimit = Convert.ToInt32(maximumDaysToExport * expectedDailyPointCount * 1.5);
if (remainingPoints.Count > roughPointLimit)
{
var limitExceededCount = remainingPoints.Count - roughPointLimit;
Log.Warn($"Upper limit of {roughPointLimit} points exceeded by {limitExceededCount} points for Frequency={period} and MaximumPointDays={maximumDaysToExport} in '{timeSeriesDescription.Identifier}'.");
remainingPoints = remainingPoints
.Skip(limitExceededCount)
.ToList();
}
var trimmedPointCount = timeSeries.NumPoints - remainingPoints.Count;
Log.Info(
$"Trimming '{timeSeriesDescription.Identifier}' {trimmedPointCount} points before {earliestDayToUpload:O} with {remainingPoints.Count} points remaining with Frequency={period}");
timeSeries.Points = remainingPoints;
timeSeries.NumPoints = timeSeries.Points.Count;
}
private void TrimExcludedPoints(
TimeSeriesDescription timeSeriesDescription,
TimeSeriesDataServiceResponse timeSeries)
{
var(exportDuration, exportLabel) = GetExportDuration(timeSeriesDescription);
if (exportDuration <= TimeSpan.Zero || !timeSeries.Points.Any())
{
return;
}
var firstTimeToInclude = timeSeries.Points.Last().Timestamp.DateTimeOffset - exportDuration;
var remainingPoints = timeSeries.Points
.Where(p => p.Timestamp.DateTimeOffset >= firstTimeToInclude)
.ToList();
if (remainingPoints.Count > Context.MaximumPointsPerSensor)
{
remainingPoints = remainingPoints
.Skip(remainingPoints.Count - Context.MaximumPointsPerSensor)
.ToList();
}
var earlyPointCount = timeSeries.Points.Count - remainingPoints.Count;
if (earlyPointCount <= 0)
{
return;
}
Log.Info($"Trimming '{timeSeriesDescription.Identifier}' {"point".ToQuantity(earlyPointCount)} before {firstTimeToInclude:O} with {remainingPoints.Count} points remaining with ExportDuration={exportLabel}");
timeSeries.Points = remainingPoints;
timeSeries.NumPoints = timeSeries.Points.Count;
}
private bool HaveExistingSosPointsChanged(
TimeSeriesDataCorrectedServiceRequest dataRequest,
DateTimeOffset?lastSensorTime,
TimeSeriesChangeEvent detectedChange,
TimeSeriesDescription timeSeriesDescription)
{
if (detectedChange.HasAttributeChange ?? false)
{
return(true);
}
if (!detectedChange.FirstPointChanged.HasValue || !lastSensorTime.HasValue)
{
return(false);
}
if (lastSensorTime < detectedChange.FirstPointChanged)
{
return(false);
}
var timeSeriesIdentifier = timeSeriesDescription.Identifier;
var sosPoints = new Queue <TimeSeriesPoint>(Sos.GetObservations(timeSeriesDescription, AddMilliseconds(detectedChange.FirstPointChanged.Value, -1), AddMilliseconds(lastSensorTime.Value, 1)));
var aqtsPoints = new Queue <TimeSeriesPoint>(Aquarius.Publish.Get(dataRequest).Points);
var sosCount = sosPoints.Count;
var aqtsCount = aqtsPoints.Count;
Log.Info($"Fetched {sosCount} SOS points and {aqtsCount} AQUARIUS points for '{timeSeriesIdentifier}' from {dataRequest.QueryFrom:O} ...");
while (sosPoints.Any() || aqtsPoints.Any())
{
var sosPoint = sosPoints.FirstOrDefault();
var aqtsPoint = aqtsPoints.FirstOrDefault();
if (aqtsPoint == null)
{
Log.Warn($"'{timeSeriesIdentifier}': AQUARIUS now has fewer points than SOS@{sosPoint?.Timestamp.DateTimeOffset:O}");
return(true);
}
if (sosPoint == null)
{
break;
}
var aqtsValue = (dataRequest.ApplyRounding ?? false)
? double.Parse(aqtsPoint.Value.Display)
: aqtsPoint.Value.Numeric;
var sosValue = sosPoint.Value.Numeric;
if (sosPoint.Timestamp.DateTimeOffset != aqtsPoint.Timestamp.DateTimeOffset)
{
Log.Warn($"'{timeSeriesIdentifier}': Different timestamps: AQUARIUS={aqtsValue}@{aqtsPoint.Timestamp.DateTimeOffset:O} vs SOS={sosValue}@{sosPoint.Timestamp.DateTimeOffset:O}");
return(true);
}
if (!DoubleHelper.AreSame(aqtsValue, sosValue))
{
Log.Warn($"'{timeSeriesIdentifier}': Different values @ {aqtsPoint.Timestamp.DateTimeOffset:O}: AQUARIUS={aqtsValue} vs SOS={sosValue}");
return(true);
}
sosPoints.Dequeue();
aqtsPoints.Dequeue();
}
Log.Info($"'{timeSeriesDescription.Identifier}': All {sosCount} SOS points match between SOS and AQUARIUS.");
dataRequest.QueryFrom = lastSensorTime.Value.AddTicks(1);
return(false);
}
private void ExportTimeSeries(bool clearExportedData,
TimeSeriesChangeEvent detectedChange,
TimeSeriesDescription timeSeriesDescription)
{
var locationInfo = GetLocationInfo(timeSeriesDescription.LocationIdentifier);
var(exportDuration, exportLabel) = GetExportDuration(timeSeriesDescription);
var dataRequest = new TimeSeriesDataCorrectedServiceRequest
{
TimeSeriesUniqueId = timeSeriesDescription.UniqueId,
QueryFrom = GetInitialQueryFrom(detectedChange),
ApplyRounding = Context.ApplyRounding,
};
var existingSensor = Sos.FindExistingSensor(timeSeriesDescription);
var deleteExistingSensor = clearExportedData && existingSensor != null;
var assignedOffering = existingSensor?.Identifier;
var lastSensorTime = GetLastSensorTime(existingSensor);
if (HaveExistingSosPointsChanged(dataRequest, lastSensorTime, detectedChange, timeSeriesDescription))
{
Log.Warn($"FirstPointChanged={detectedChange.FirstPointChanged:O} AttributeChange={detectedChange.HasAttributeChange} of '{timeSeriesDescription.Identifier}' precedes LastSensorTime={lastSensorTime:O} of '{existingSensor?.Identifier}'. Forcing delete of existing sensor.");
// A point has changed before the last known observation, so we'll need to throw out the entire sensor
deleteExistingSensor = true;
// We'll also need to fetch more data again
dataRequest.QueryFrom = null;
}
if (dataRequest.QueryFrom == null)
{
// Get the full extraction
var endPoint = dataRequest.QueryTo ?? DateTimeOffset.Now;
var startOfToday = new DateTimeOffset(endPoint.Year, endPoint.Month, endPoint.Day, 0, 0, 0,
timeSeriesDescription.UtcOffsetIsoDuration.ToTimeSpan());
dataRequest.QueryFrom = startOfToday - exportDuration;
}
Log.Info($"Fetching changes from '{timeSeriesDescription.Identifier}' FirstPointChanged={detectedChange.FirstPointChanged:O} HasAttributeChanged={detectedChange.HasAttributeChange} QueryFrom={dataRequest.QueryFrom:O} ...");
var timeSeries = Aquarius.Publish.Get(dataRequest);
TrimExcludedPoints(timeSeriesDescription, timeSeries);
var createSensor = existingSensor == null || deleteExistingSensor;
TimeSeriesPointFilter.FilterTimeSeriesPoints(timeSeries);
var exportSummary = $"{timeSeries.NumPoints} points [{timeSeries.Points.FirstOrDefault()?.Timestamp.DateTimeOffset:O} to {timeSeries.Points.LastOrDefault()?.Timestamp.DateTimeOffset:O}] from '{timeSeriesDescription.Identifier}' with ExportDuration={exportLabel}";
ExportedTimeSeriesCount += 1;
ExportedPointCount += timeSeries.NumPoints ?? 0;
if (Context.DryRun)
{
if (deleteExistingSensor)
{
LogDryRun($"Would delete existing sensor '{existingSensor?.Identifier}'");
}
if (createSensor)
{
LogDryRun($"Would create new sensor for '{timeSeriesDescription.Identifier}'");
}
LogDryRun($"Would export {exportSummary}.");
return;
}
Log.Info($"Exporting {exportSummary} ...");
if (deleteExistingSensor)
{
Sos.DeleteSensor(timeSeries);
Sos.DeleteDeletedObservations();
}
if (createSensor)
{
var sensor = Sos.InsertSensor(timeSeries);
assignedOffering = sensor.AssignedOffering;
}
Sos.InsertObservation(assignedOffering, locationInfo.LocationData, locationInfo.LocationDescription, timeSeries);
}
private void ExportTimeSeries(
bool clearExportedData,
TimeSeriesChangeEvent detectedChange,
TimeSeriesDescription timeSeriesDescription)
{
Log.Info($"Fetching changes from '{timeSeriesDescription.Identifier}' FirstPointChanged={detectedChange.FirstPointChanged:O} HasAttributeChanged={detectedChange.HasAttributeChange} ...");
var locationInfo = GetLocationInfo(timeSeriesDescription.LocationIdentifier);
var period = GetTimeSeriesPeriod(timeSeriesDescription);
var dataRequest = new TimeSeriesDataCorrectedServiceRequest
{
TimeSeriesUniqueId = timeSeriesDescription.UniqueId,
QueryFrom = GetInitialQueryFrom(detectedChange),
ApplyRounding = Context.ApplyRounding,
};
var existingSensor = Sos.FindExistingSensor(timeSeriesDescription);
var deleteExistingSensor = clearExportedData && existingSensor != null;
var assignedOffering = existingSensor?.Identifier;
var timeSeries = FetchMinimumTimeSeries(detectedChange, timeSeriesDescription, existingSensor, dataRequest, ref deleteExistingSensor, ref period);
var createSensor = existingSensor == null || deleteExistingSensor;
TimeSeriesPointFilter.FilterTimeSeriesPoints(timeSeries);
var exportSummary = $"{timeSeries.NumPoints} points [{timeSeries.Points.FirstOrDefault()?.Timestamp.DateTimeOffset:O} to {timeSeries.Points.LastOrDefault()?.Timestamp.DateTimeOffset:O}] from '{timeSeriesDescription.Identifier}' with Frequency={period}";
ExportedTimeSeriesCount += 1;
ExportedPointCount += timeSeries.NumPoints ?? 0;
if (Context.DryRun)
{
if (deleteExistingSensor)
{
LogDryRun($"Would delete existing sensor '{existingSensor?.Identifier}'");
}
if (createSensor)
{
LogDryRun($"Would create new sensor for '{timeSeriesDescription.Identifier}'");
}
LogDryRun($"Would export {exportSummary}.");
return;
}
Log.Info($"Exporting {exportSummary} ...");
if (deleteExistingSensor)
{
Sos.DeleteSensor(timeSeries);
Sos.DeleteDeletedObservations();
}
if (createSensor)
{
var sensor = Sos.InsertSensor(timeSeries);
assignedOffering = sensor.AssignedOffering;
}
Sos.InsertObservation(assignedOffering, locationInfo.LocationData, locationInfo.LocationDescription, timeSeries, timeSeriesDescription);
}
