private static DateTimeOffset?GetInitialQueryFrom(TimeSeriesChangeEvent detectedChange)
{
// When a derived time-series is reported as changed, the first point changed is always the beginning of time.
// Rather than always pull the whole signal (which can be expensive with rounded values), only to trim most points before exporting,
// just treat a re-derived series event like an initial sync, so we'll "walk backwards" from the current time.
//
// If and when partial-re-derivation is implemented, this condition will no longer be triggered.
return(detectedChange.FirstPointChanged == DateTimeOffset.MinValue
? null
: detectedChange.FirstPointChanged);
}
private TimeSeriesDataServiceResponse FetchMinimumTimeSeries(
TimeSeriesChangeEvent detectedChange,
TimeSeriesDescription timeSeriesDescription,
SensorInfo existingSensor,
TimeSeriesDataCorrectedServiceRequest dataRequest,
ref bool deleteExistingSensor,
ref ComputationPeriod period)
{
TimeSeriesDataServiceResponse timeSeries;
if (!deleteExistingSensor && GetLastSensorTime(existingSensor) < dataRequest.QueryFrom)
{
// All the changed points have occurred after the last sensor point which exists in the SOS server.
// This is the preferred code path, since we only need to export the new points.
timeSeries = Aquarius.Publish.Get(dataRequest);
if (period == ComputationPeriod.Unknown)
{
// We may have just fetched enough recent points to determine the time-series frequency
period = ComputationPeriodEstimator.InferPeriodFromRecentPoints(timeSeries);
}
TrimEarlyPoints(timeSeriesDescription, timeSeries, period);
return(timeSeries);
}
if (GetLastSensorTime(existingSensor) >= detectedChange.FirstPointChanged)
{
// A point has changed before the last known observation, so we'll need to throw out the entire sensor
deleteExistingSensor = true;
// We'll also need to fetch more data again
dataRequest.QueryFrom = null;
}
timeSeries = FetchRecentSignal(timeSeriesDescription, dataRequest, ref period);
if (GetLastSensorTime(existingSensor) >= detectedChange.FirstPointChanged)
{
// A point has changed before the last known observation, so we'll need to throw out the entire sensor
deleteExistingSensor = true;
// We'll also need to fetch more data again
dataRequest.QueryFrom = null;
timeSeries = FetchRecentSignal(timeSeriesDescription, dataRequest, ref period);
}
TrimEarlyPoints(timeSeriesDescription, timeSeries, period);
return(timeSeries);
}
private static void MergeTimeSeriesChangeEvent(TimeSeriesChangeEvent existingEvent, TimeSeriesChangeEvent newEvent)
{
if (existingEvent.HasAttributeChange.HasValue && newEvent.HasAttributeChange.HasValue)
{
existingEvent.HasAttributeChange = existingEvent.HasAttributeChange.Value || newEvent.HasAttributeChange.Value;
}
else if (newEvent.HasAttributeChange.HasValue)
{
existingEvent.HasAttributeChange = newEvent.HasAttributeChange;
}
if (existingEvent.FirstPointChanged.HasValue && newEvent.FirstPointChanged.HasValue)
{
if (newEvent.FirstPointChanged < existingEvent.FirstPointChanged)
{
existingEvent.FirstPointChanged = newEvent.FirstPointChanged;
}
}
else if (newEvent.FirstPointChanged.HasValue)
{
existingEvent.FirstPointChanged = newEvent.FirstPointChanged;
}
}
private bool HaveExistingSosPointsChanged(
TimeSeriesDataCorrectedServiceRequest dataRequest,
DateTimeOffset?lastSensorTime,
TimeSeriesChangeEvent detectedChange,
TimeSeriesDescription timeSeriesDescription)
{
if (detectedChange.HasAttributeChange ?? false)
{
return(true);
}
if (!detectedChange.FirstPointChanged.HasValue || !lastSensorTime.HasValue)
{
return(false);
}
if (lastSensorTime < detectedChange.FirstPointChanged)
{
return(false);
}
var timeSeriesIdentifier = timeSeriesDescription.Identifier;
var sosPoints = new Queue <TimeSeriesPoint>(Sos.GetObservations(timeSeriesDescription, AddMilliseconds(detectedChange.FirstPointChanged.Value, -1), AddMilliseconds(lastSensorTime.Value, 1)));
var aqtsPoints = new Queue <TimeSeriesPoint>(Aquarius.Publish.Get(dataRequest).Points);
var sosCount = sosPoints.Count;
var aqtsCount = aqtsPoints.Count;
Log.Info($"Fetched {sosCount} SOS points and {aqtsCount} AQUARIUS points for '{timeSeriesIdentifier}' from {dataRequest.QueryFrom:O} ...");
while (sosPoints.Any() || aqtsPoints.Any())
{
var sosPoint = sosPoints.FirstOrDefault();
var aqtsPoint = aqtsPoints.FirstOrDefault();
if (aqtsPoint == null)
{
Log.Warn($"'{timeSeriesIdentifier}': AQUARIUS now has fewer points than SOS@{sosPoint?.Timestamp.DateTimeOffset:O}");
return(true);
}
if (sosPoint == null)
{
break;
}
var aqtsValue = (dataRequest.ApplyRounding ?? false)
? double.Parse(aqtsPoint.Value.Display)
: aqtsPoint.Value.Numeric;
var sosValue = sosPoint.Value.Numeric;
if (sosPoint.Timestamp.DateTimeOffset != aqtsPoint.Timestamp.DateTimeOffset)
{
Log.Warn($"'{timeSeriesIdentifier}': Different timestamps: AQUARIUS={aqtsValue}@{aqtsPoint.Timestamp.DateTimeOffset:O} vs SOS={sosValue}@{sosPoint.Timestamp.DateTimeOffset:O}");
return(true);
}
if (!DoubleHelper.AreSame(aqtsValue, sosValue))
{
Log.Warn($"'{timeSeriesIdentifier}': Different values @ {aqtsPoint.Timestamp.DateTimeOffset:O}: AQUARIUS={aqtsValue} vs SOS={sosValue}");
return(true);
}
sosPoints.Dequeue();
aqtsPoints.Dequeue();
}
Log.Info($"'{timeSeriesDescription.Identifier}': All {sosCount} SOS points match between SOS and AQUARIUS.");
dataRequest.QueryFrom = lastSensorTime.Value.AddTicks(1);
return(false);
}
private void ExportTimeSeries(bool clearExportedData,
TimeSeriesChangeEvent detectedChange,
TimeSeriesDescription timeSeriesDescription)
{
var locationInfo = GetLocationInfo(timeSeriesDescription.LocationIdentifier);
var(exportDuration, exportLabel) = GetExportDuration(timeSeriesDescription);
var dataRequest = new TimeSeriesDataCorrectedServiceRequest
{
TimeSeriesUniqueId = timeSeriesDescription.UniqueId,
QueryFrom = GetInitialQueryFrom(detectedChange),
ApplyRounding = Context.ApplyRounding,
};
var existingSensor = Sos.FindExistingSensor(timeSeriesDescription);
var deleteExistingSensor = clearExportedData && existingSensor != null;
var assignedOffering = existingSensor?.Identifier;
var lastSensorTime = GetLastSensorTime(existingSensor);
if (HaveExistingSosPointsChanged(dataRequest, lastSensorTime, detectedChange, timeSeriesDescription))
{
Log.Warn($"FirstPointChanged={detectedChange.FirstPointChanged:O} AttributeChange={detectedChange.HasAttributeChange} of '{timeSeriesDescription.Identifier}' precedes LastSensorTime={lastSensorTime:O} of '{existingSensor?.Identifier}'. Forcing delete of existing sensor.");
// A point has changed before the last known observation, so we'll need to throw out the entire sensor
deleteExistingSensor = true;
// We'll also need to fetch more data again
dataRequest.QueryFrom = null;
}
if (dataRequest.QueryFrom == null)
{
// Get the full extraction
var endPoint = dataRequest.QueryTo ?? DateTimeOffset.Now;
var startOfToday = new DateTimeOffset(endPoint.Year, endPoint.Month, endPoint.Day, 0, 0, 0,
timeSeriesDescription.UtcOffsetIsoDuration.ToTimeSpan());
dataRequest.QueryFrom = startOfToday - exportDuration;
}
Log.Info($"Fetching changes from '{timeSeriesDescription.Identifier}' FirstPointChanged={detectedChange.FirstPointChanged:O} HasAttributeChanged={detectedChange.HasAttributeChange} QueryFrom={dataRequest.QueryFrom:O} ...");
var timeSeries = Aquarius.Publish.Get(dataRequest);
TrimExcludedPoints(timeSeriesDescription, timeSeries);
var createSensor = existingSensor == null || deleteExistingSensor;
TimeSeriesPointFilter.FilterTimeSeriesPoints(timeSeries);
var exportSummary = $"{timeSeries.NumPoints} points [{timeSeries.Points.FirstOrDefault()?.Timestamp.DateTimeOffset:O} to {timeSeries.Points.LastOrDefault()?.Timestamp.DateTimeOffset:O}] from '{timeSeriesDescription.Identifier}' with ExportDuration={exportLabel}";
ExportedTimeSeriesCount += 1;
ExportedPointCount += timeSeries.NumPoints ?? 0;
if (Context.DryRun)
{
if (deleteExistingSensor)
{
LogDryRun($"Would delete existing sensor '{existingSensor?.Identifier}'");
}
if (createSensor)
{
LogDryRun($"Would create new sensor for '{timeSeriesDescription.Identifier}'");
}
LogDryRun($"Would export {exportSummary}.");
return;
}
Log.Info($"Exporting {exportSummary} ...");
if (deleteExistingSensor)
{
Sos.DeleteSensor(timeSeries);
Sos.DeleteDeletedObservations();
}
if (createSensor)
{
var sensor = Sos.InsertSensor(timeSeries);
assignedOffering = sensor.AssignedOffering;
}
Sos.InsertObservation(assignedOffering, locationInfo.LocationData, locationInfo.LocationDescription, timeSeries);
}
private void ExportTimeSeries(
bool clearExportedData,
TimeSeriesChangeEvent detectedChange,
TimeSeriesDescription timeSeriesDescription)
{
Log.Info($"Fetching changes from '{timeSeriesDescription.Identifier}' FirstPointChanged={detectedChange.FirstPointChanged:O} HasAttributeChanged={detectedChange.HasAttributeChange} ...");
var locationInfo = GetLocationInfo(timeSeriesDescription.LocationIdentifier);
var period = GetTimeSeriesPeriod(timeSeriesDescription);
var dataRequest = new TimeSeriesDataCorrectedServiceRequest
{
TimeSeriesUniqueId = timeSeriesDescription.UniqueId,
QueryFrom = GetInitialQueryFrom(detectedChange),
ApplyRounding = Context.ApplyRounding,
};
var existingSensor = Sos.FindExistingSensor(timeSeriesDescription);
var deleteExistingSensor = clearExportedData && existingSensor != null;
var assignedOffering = existingSensor?.Identifier;
var timeSeries = FetchMinimumTimeSeries(detectedChange, timeSeriesDescription, existingSensor, dataRequest, ref deleteExistingSensor, ref period);
var createSensor = existingSensor == null || deleteExistingSensor;
TimeSeriesPointFilter.FilterTimeSeriesPoints(timeSeries);
var exportSummary = $"{timeSeries.NumPoints} points [{timeSeries.Points.FirstOrDefault()?.Timestamp.DateTimeOffset:O} to {timeSeries.Points.LastOrDefault()?.Timestamp.DateTimeOffset:O}] from '{timeSeriesDescription.Identifier}' with Frequency={period}";
ExportedTimeSeriesCount += 1;
ExportedPointCount += timeSeries.NumPoints ?? 0;
if (Context.DryRun)
{
if (deleteExistingSensor)
{
LogDryRun($"Would delete existing sensor '{existingSensor?.Identifier}'");
}
if (createSensor)
{
LogDryRun($"Would create new sensor for '{timeSeriesDescription.Identifier}'");
}
LogDryRun($"Would export {exportSummary}.");
return;
}
Log.Info($"Exporting {exportSummary} ...");
if (deleteExistingSensor)
{
Sos.DeleteSensor(timeSeries);
Sos.DeleteDeletedObservations();
}
if (createSensor)
{
var sensor = Sos.InsertSensor(timeSeries);
assignedOffering = sensor.AssignedOffering;
}
Sos.InsertObservation(assignedOffering, locationInfo.LocationData, locationInfo.LocationDescription, timeSeries, timeSeriesDescription);
}