private void TrimEarlyPoints(
TimeSeriesDescription timeSeriesDescription,
TimeSeriesDataServiceResponse timeSeries,
ComputationPeriod period)
{
var maximumDaysToExport = Context.Config.MaximumPointDays[period];
if (maximumDaysToExport <= 0 || !timeSeries.Points.Any())
{
return;
}
var earliestDayToUpload = SubtractTimeSpan(
timeSeries.Points.Last().Timestamp.DateTimeOffset,
TimeSpan.FromDays(maximumDaysToExport));
var remainingPoints = timeSeries.Points
.Where(p => p.Timestamp.DateTimeOffset >= earliestDayToUpload)
.ToList();
var trimmedPointCount = timeSeries.NumPoints - remainingPoints.Count;
Log.Info(
$"Trimming '{timeSeriesDescription.Identifier}' {trimmedPointCount} points before {earliestDayToUpload:O} with {remainingPoints.Count} points remaining with Frequency={period}");
timeSeries.Points = remainingPoints;
timeSeries.NumPoints = timeSeries.Points.Count;
}
private TimeSeriesDataServiceResponse FetchMinimumTimeSeries(
TimeSeriesChangeEvent detectedChange,
TimeSeriesDescription timeSeriesDescription,
SensorInfo existingSensor,
TimeSeriesDataCorrectedServiceRequest dataRequest,
ref bool deleteExistingSensor,
ref ComputationPeriod period)
{
TimeSeriesDataServiceResponse timeSeries;
if (!deleteExistingSensor && GetLastSensorTime(existingSensor) < dataRequest.QueryFrom)
{
// All the changed points have occurred after the last sensor point which exists in the SOS server.
// This is the preferred code path, since we only need to export the new points.
timeSeries = Aquarius.Publish.Get(dataRequest);
if (period == ComputationPeriod.Unknown)
{
// We may have just fetched enough recent points to determine the time-series frequency
period = ComputationPeriodEstimator.InferPeriodFromRecentPoints(timeSeries);
}
TrimEarlyPoints(timeSeriesDescription, timeSeries, period);
return(timeSeries);
}
if (GetLastSensorTime(existingSensor) >= detectedChange.FirstPointChanged)
{
// A point has changed before the last known observation, so we'll need to throw out the entire sensor
deleteExistingSensor = true;
// We'll also need to fetch more data again
dataRequest.QueryFrom = null;
}
timeSeries = FetchRecentSignal(timeSeriesDescription, dataRequest, ref period);
if (GetLastSensorTime(existingSensor) >= detectedChange.FirstPointChanged)
{
// A point has changed before the last known observation, so we'll need to throw out the entire sensor
deleteExistingSensor = true;
// We'll also need to fetch more data again
dataRequest.QueryFrom = null;
timeSeries = FetchRecentSignal(timeSeriesDescription, dataRequest, ref period);
}
TrimEarlyPoints(timeSeriesDescription, timeSeries, period);
return(timeSeries);
}
private void TrimEarlyPoints(
TimeSeriesDescription timeSeriesDescription,
TimeSeriesDataServiceResponse timeSeries,
ComputationPeriod period)
{
var maximumDaysToExport = Context.Config.MaximumPointDays[period];
if (maximumDaysToExport <= 0 || !timeSeries.Points.Any())
{
return;
}
var earliestDayToUpload = SubtractTimeSpan(
timeSeries.Points.Last().Timestamp.DateTimeOffset,
TimeSpan.FromDays(maximumDaysToExport));
var remainingPoints = timeSeries.Points
.Where(p => p.Timestamp.DateTimeOffset >= earliestDayToUpload)
.ToList();
if (!RoughDailyPointCount.TryGetValue(period, out var expectedDailyPointCount))
{
expectedDailyPointCount = 1.0;
}
var roughPointLimit = Convert.ToInt32(maximumDaysToExport * expectedDailyPointCount * 1.5);
if (remainingPoints.Count > roughPointLimit)
{
var limitExceededCount = remainingPoints.Count - roughPointLimit;
Log.Warn($"Upper limit of {roughPointLimit} points exceeded by {limitExceededCount} points for Frequency={period} and MaximumPointDays={maximumDaysToExport} in '{timeSeriesDescription.Identifier}'.");
remainingPoints = remainingPoints
.Skip(limitExceededCount)
.ToList();
}
var trimmedPointCount = timeSeries.NumPoints - remainingPoints.Count;
Log.Info(
$"Trimming '{timeSeriesDescription.Identifier}' {trimmedPointCount} points before {earliestDayToUpload:O} with {remainingPoints.Count} points remaining with Frequency={period}");
timeSeries.Points = remainingPoints;
timeSeries.NumPoints = timeSeries.Points.Count;
}
