Exemple #1
0
        private void ExportTimeSeries(bool clearExportedData,
                                      TimeSeriesChangeEvent detectedChange,
                                      TimeSeriesDescription timeSeriesDescription)
        {
            var locationInfo = GetLocationInfo(timeSeriesDescription.LocationIdentifier);

            var(exportDuration, exportLabel) = GetExportDuration(timeSeriesDescription);

            var dataRequest = new TimeSeriesDataCorrectedServiceRequest
            {
                TimeSeriesUniqueId = timeSeriesDescription.UniqueId,
                QueryFrom          = GetInitialQueryFrom(detectedChange),
                ApplyRounding      = Context.ApplyRounding,
            };

            var existingSensor       = Sos.FindExistingSensor(timeSeriesDescription);
            var deleteExistingSensor = clearExportedData && existingSensor != null;
            var assignedOffering     = existingSensor?.Identifier;

            var lastSensorTime = GetLastSensorTime(existingSensor);

            if (HaveExistingSosPointsChanged(dataRequest, lastSensorTime, detectedChange, timeSeriesDescription))
            {
                Log.Warn($"FirstPointChanged={detectedChange.FirstPointChanged:O} AttributeChange={detectedChange.HasAttributeChange} of '{timeSeriesDescription.Identifier}' precedes LastSensorTime={lastSensorTime:O} of '{existingSensor?.Identifier}'. Forcing delete of existing sensor.");

                // A point has changed before the last known observation, so we'll need to throw out the entire sensor
                deleteExistingSensor = true;

                // We'll also need to fetch more data again
                dataRequest.QueryFrom = null;
            }

            if (dataRequest.QueryFrom == null)
            {
                // Get the full extraction
                var endPoint     = dataRequest.QueryTo ?? DateTimeOffset.Now;
                var startOfToday = new DateTimeOffset(endPoint.Year, endPoint.Month, endPoint.Day, 0, 0, 0,
                                                      timeSeriesDescription.UtcOffsetIsoDuration.ToTimeSpan());

                dataRequest.QueryFrom = startOfToday - exportDuration;
            }

            Log.Info($"Fetching changes from '{timeSeriesDescription.Identifier}' FirstPointChanged={detectedChange.FirstPointChanged:O} HasAttributeChanged={detectedChange.HasAttributeChange} QueryFrom={dataRequest.QueryFrom:O} ...");

            var timeSeries = Aquarius.Publish.Get(dataRequest);

            TrimExcludedPoints(timeSeriesDescription, timeSeries);

            var createSensor = existingSensor == null || deleteExistingSensor;

            TimeSeriesPointFilter.FilterTimeSeriesPoints(timeSeries);

            var exportSummary = $"{timeSeries.NumPoints} points [{timeSeries.Points.FirstOrDefault()?.Timestamp.DateTimeOffset:O} to {timeSeries.Points.LastOrDefault()?.Timestamp.DateTimeOffset:O}] from '{timeSeriesDescription.Identifier}' with ExportDuration={exportLabel}";

            ExportedTimeSeriesCount += 1;
            ExportedPointCount      += timeSeries.NumPoints ?? 0;

            if (Context.DryRun)
            {
                if (deleteExistingSensor)
                {
                    LogDryRun($"Would delete existing sensor '{existingSensor?.Identifier}'");
                }

                if (createSensor)
                {
                    LogDryRun($"Would create new sensor for '{timeSeriesDescription.Identifier}'");
                }

                LogDryRun($"Would export {exportSummary}.");
                return;
            }

            Log.Info($"Exporting {exportSummary} ...");

            if (deleteExistingSensor)
            {
                Sos.DeleteSensor(timeSeries);
                Sos.DeleteDeletedObservations();
            }

            if (createSensor)
            {
                var sensor = Sos.InsertSensor(timeSeries);

                assignedOffering = sensor.AssignedOffering;
            }

            Sos.InsertObservation(assignedOffering, locationInfo.LocationData, locationInfo.LocationDescription, timeSeries);
        }
Exemple #2
0
        private void RunOnce()
        {
            SyncStatus = new SyncStatus(Aquarius)
            {
                Context = Context
            };
            TimeSeriesPointFilter = new TimeSeriesPointFilter {
                Context = Context
            };

            ValidateFilters();

            MaximumExportDuration = Context.MaximumPollDuration
                                    ?? SyncStatus.GetMaximumChangeEventDuration()
                                    .Subtract(TimeSpan.FromHours(1));

            var request = CreateFilterRequest();

            if (Context.ForceResync)
            {
                Log.Warn("Forcing a full time-series resync.");
                request.ChangesSinceToken = null;
            }
            else if (Context.ChangesSince.HasValue)
            {
                Log.Warn($"Overriding current ChangesSinceToken='{request.ChangesSinceToken:O}' with '{Context.ChangesSince:O}'");
                request.ChangesSinceToken = Context.ChangesSince.Value.UtcDateTime;
            }

            Log.Info($"Checking {GetFilterSummary(request)} ...");

            var stopwatch = Stopwatch.StartNew();

            var response = Aquarius.Publish.Get(request);

            if (response.TokenExpired ?? false)
            {
                if (Context.NeverResync)
                {
                    Log.Warn("Skipping a recommended resync.");
                }
                else
                {
                    Log.Warn($"The ChangesSinceToken of {request.ChangesSinceToken:O} has expired. Forcing a full resync. You may need to run the exporter more frequently.");
                    request.ChangesSinceToken = null;

                    response = Aquarius.Publish.Get(request);
                }
            }

            var bootstrapToken = response.ResponseTime
                                 .Subtract(stopwatch.Elapsed)
                                 .Subtract(TimeSpan.FromMinutes(1))
                                 .UtcDateTime;

            var nextChangesSinceToken = response.NextToken ?? bootstrapToken;

            var timeSeriesDescriptions = FetchChangedTimeSeriesDescriptions(response);

            Log.Info($"Connecting to {Context.Config.SosServer} as {Context.Config.SosUsername} ...");

            var clearExportedData = !request.ChangesSinceToken.HasValue;

            request.ChangesSinceToken = nextChangesSinceToken;

            ExportToSos(request, response, timeSeriesDescriptions, clearExportedData);

            RemoveDeletedTimeSeries(response);

            SyncStatus.SaveConfiguration(request.ChangesSinceToken.Value);
        }
        private void ExportTimeSeries(
            bool clearExportedData,
            TimeSeriesChangeEvent detectedChange,
            TimeSeriesDescription timeSeriesDescription)
        {
            Log.Info($"Fetching changes from '{timeSeriesDescription.Identifier}' FirstPointChanged={detectedChange.FirstPointChanged:O} HasAttributeChanged={detectedChange.HasAttributeChange} ...");

            var locationInfo = GetLocationInfo(timeSeriesDescription.LocationIdentifier);

            var period = GetTimeSeriesPeriod(timeSeriesDescription);

            var dataRequest = new TimeSeriesDataCorrectedServiceRequest
            {
                TimeSeriesUniqueId = timeSeriesDescription.UniqueId,
                QueryFrom          = GetInitialQueryFrom(detectedChange),
                ApplyRounding      = Context.ApplyRounding,
            };

            var existingSensor       = Sos.FindExistingSensor(timeSeriesDescription);
            var deleteExistingSensor = clearExportedData && existingSensor != null;
            var assignedOffering     = existingSensor?.Identifier;

            var timeSeries = FetchMinimumTimeSeries(detectedChange, timeSeriesDescription, existingSensor, dataRequest, ref deleteExistingSensor, ref period);

            var createSensor = existingSensor == null || deleteExistingSensor;

            TimeSeriesPointFilter.FilterTimeSeriesPoints(timeSeries);

            var exportSummary = $"{timeSeries.NumPoints} points [{timeSeries.Points.FirstOrDefault()?.Timestamp.DateTimeOffset:O} to {timeSeries.Points.LastOrDefault()?.Timestamp.DateTimeOffset:O}] from '{timeSeriesDescription.Identifier}' with Frequency={period}";

            ExportedTimeSeriesCount += 1;
            ExportedPointCount      += timeSeries.NumPoints ?? 0;

            if (Context.DryRun)
            {
                if (deleteExistingSensor)
                {
                    LogDryRun($"Would delete existing sensor '{existingSensor?.Identifier}'");
                }

                if (createSensor)
                {
                    LogDryRun($"Would create new sensor for '{timeSeriesDescription.Identifier}'");
                }

                LogDryRun($"Would export {exportSummary}.");
                return;
            }

            Log.Info($"Exporting {exportSummary} ...");

            if (deleteExistingSensor)
            {
                Sos.DeleteSensor(timeSeries);
                Sos.DeleteDeletedObservations();
            }

            if (createSensor)
            {
                var sensor = Sos.InsertSensor(timeSeries);

                assignedOffering = sensor.AssignedOffering;
            }

            Sos.InsertObservation(assignedOffering, locationInfo.LocationData, locationInfo.LocationDescription, timeSeries, timeSeriesDescription);
        }