private void ExportTimeSeries(bool clearExportedData, TimeSeriesChangeEvent detectedChange, TimeSeriesDescription timeSeriesDescription) { var locationInfo = GetLocationInfo(timeSeriesDescription.LocationIdentifier); var(exportDuration, exportLabel) = GetExportDuration(timeSeriesDescription); var dataRequest = new TimeSeriesDataCorrectedServiceRequest { TimeSeriesUniqueId = timeSeriesDescription.UniqueId, QueryFrom = GetInitialQueryFrom(detectedChange), ApplyRounding = Context.ApplyRounding, }; var existingSensor = Sos.FindExistingSensor(timeSeriesDescription); var deleteExistingSensor = clearExportedData && existingSensor != null; var assignedOffering = existingSensor?.Identifier; var lastSensorTime = GetLastSensorTime(existingSensor); if (HaveExistingSosPointsChanged(dataRequest, lastSensorTime, detectedChange, timeSeriesDescription)) { Log.Warn($"FirstPointChanged={detectedChange.FirstPointChanged:O} AttributeChange={detectedChange.HasAttributeChange} of '{timeSeriesDescription.Identifier}' precedes LastSensorTime={lastSensorTime:O} of '{existingSensor?.Identifier}'. Forcing delete of existing sensor."); // A point has changed before the last known observation, so we'll need to throw out the entire sensor deleteExistingSensor = true; // We'll also need to fetch more data again dataRequest.QueryFrom = null; } if (dataRequest.QueryFrom == null) { // Get the full extraction var endPoint = dataRequest.QueryTo ?? DateTimeOffset.Now; var startOfToday = new DateTimeOffset(endPoint.Year, endPoint.Month, endPoint.Day, 0, 0, 0, timeSeriesDescription.UtcOffsetIsoDuration.ToTimeSpan()); dataRequest.QueryFrom = startOfToday - exportDuration; } Log.Info($"Fetching changes from '{timeSeriesDescription.Identifier}' FirstPointChanged={detectedChange.FirstPointChanged:O} HasAttributeChanged={detectedChange.HasAttributeChange} QueryFrom={dataRequest.QueryFrom:O} ..."); var timeSeries = Aquarius.Publish.Get(dataRequest); TrimExcludedPoints(timeSeriesDescription, timeSeries); var createSensor = existingSensor == null || deleteExistingSensor; TimeSeriesPointFilter.FilterTimeSeriesPoints(timeSeries); var exportSummary = $"{timeSeries.NumPoints} points [{timeSeries.Points.FirstOrDefault()?.Timestamp.DateTimeOffset:O} to {timeSeries.Points.LastOrDefault()?.Timestamp.DateTimeOffset:O}] from '{timeSeriesDescription.Identifier}' with ExportDuration={exportLabel}"; ExportedTimeSeriesCount += 1; ExportedPointCount += timeSeries.NumPoints ?? 0; if (Context.DryRun) { if (deleteExistingSensor) { LogDryRun($"Would delete existing sensor '{existingSensor?.Identifier}'"); } if (createSensor) { LogDryRun($"Would create new sensor for '{timeSeriesDescription.Identifier}'"); } LogDryRun($"Would export {exportSummary}."); return; } Log.Info($"Exporting {exportSummary} ..."); if (deleteExistingSensor) { Sos.DeleteSensor(timeSeries); Sos.DeleteDeletedObservations(); } if (createSensor) { var sensor = Sos.InsertSensor(timeSeries); assignedOffering = sensor.AssignedOffering; } Sos.InsertObservation(assignedOffering, locationInfo.LocationData, locationInfo.LocationDescription, timeSeries); }
private void RunOnce() { SyncStatus = new SyncStatus(Aquarius) { Context = Context }; TimeSeriesPointFilter = new TimeSeriesPointFilter { Context = Context }; ValidateFilters(); MaximumExportDuration = Context.MaximumPollDuration ?? SyncStatus.GetMaximumChangeEventDuration() .Subtract(TimeSpan.FromHours(1)); var request = CreateFilterRequest(); if (Context.ForceResync) { Log.Warn("Forcing a full time-series resync."); request.ChangesSinceToken = null; } else if (Context.ChangesSince.HasValue) { Log.Warn($"Overriding current ChangesSinceToken='{request.ChangesSinceToken:O}' with '{Context.ChangesSince:O}'"); request.ChangesSinceToken = Context.ChangesSince.Value.UtcDateTime; } Log.Info($"Checking {GetFilterSummary(request)} ..."); var stopwatch = Stopwatch.StartNew(); var response = Aquarius.Publish.Get(request); if (response.TokenExpired ?? false) { if (Context.NeverResync) { Log.Warn("Skipping a recommended resync."); } else { Log.Warn($"The ChangesSinceToken of {request.ChangesSinceToken:O} has expired. Forcing a full resync. You may need to run the exporter more frequently."); request.ChangesSinceToken = null; response = Aquarius.Publish.Get(request); } } var bootstrapToken = response.ResponseTime .Subtract(stopwatch.Elapsed) .Subtract(TimeSpan.FromMinutes(1)) .UtcDateTime; var nextChangesSinceToken = response.NextToken ?? bootstrapToken; var timeSeriesDescriptions = FetchChangedTimeSeriesDescriptions(response); Log.Info($"Connecting to {Context.Config.SosServer} as {Context.Config.SosUsername} ..."); var clearExportedData = !request.ChangesSinceToken.HasValue; request.ChangesSinceToken = nextChangesSinceToken; ExportToSos(request, response, timeSeriesDescriptions, clearExportedData); RemoveDeletedTimeSeries(response); SyncStatus.SaveConfiguration(request.ChangesSinceToken.Value); }
private void ExportTimeSeries( bool clearExportedData, TimeSeriesChangeEvent detectedChange, TimeSeriesDescription timeSeriesDescription) { Log.Info($"Fetching changes from '{timeSeriesDescription.Identifier}' FirstPointChanged={detectedChange.FirstPointChanged:O} HasAttributeChanged={detectedChange.HasAttributeChange} ..."); var locationInfo = GetLocationInfo(timeSeriesDescription.LocationIdentifier); var period = GetTimeSeriesPeriod(timeSeriesDescription); var dataRequest = new TimeSeriesDataCorrectedServiceRequest { TimeSeriesUniqueId = timeSeriesDescription.UniqueId, QueryFrom = GetInitialQueryFrom(detectedChange), ApplyRounding = Context.ApplyRounding, }; var existingSensor = Sos.FindExistingSensor(timeSeriesDescription); var deleteExistingSensor = clearExportedData && existingSensor != null; var assignedOffering = existingSensor?.Identifier; var timeSeries = FetchMinimumTimeSeries(detectedChange, timeSeriesDescription, existingSensor, dataRequest, ref deleteExistingSensor, ref period); var createSensor = existingSensor == null || deleteExistingSensor; TimeSeriesPointFilter.FilterTimeSeriesPoints(timeSeries); var exportSummary = $"{timeSeries.NumPoints} points [{timeSeries.Points.FirstOrDefault()?.Timestamp.DateTimeOffset:O} to {timeSeries.Points.LastOrDefault()?.Timestamp.DateTimeOffset:O}] from '{timeSeriesDescription.Identifier}' with Frequency={period}"; ExportedTimeSeriesCount += 1; ExportedPointCount += timeSeries.NumPoints ?? 0; if (Context.DryRun) { if (deleteExistingSensor) { LogDryRun($"Would delete existing sensor '{existingSensor?.Identifier}'"); } if (createSensor) { LogDryRun($"Would create new sensor for '{timeSeriesDescription.Identifier}'"); } LogDryRun($"Would export {exportSummary}."); return; } Log.Info($"Exporting {exportSummary} ..."); if (deleteExistingSensor) { Sos.DeleteSensor(timeSeries); Sos.DeleteDeletedObservations(); } if (createSensor) { var sensor = Sos.InsertSensor(timeSeries); assignedOffering = sensor.AssignedOffering; } Sos.InsertObservation(assignedOffering, locationInfo.LocationData, locationInfo.LocationDescription, timeSeries, timeSeriesDescription); }