private void ExportTimeSeries(bool clearExportedData,
TimeSeriesChangeEvent detectedChange,
TimeSeriesDescription timeSeriesDescription)
{
var locationInfo = GetLocationInfo(timeSeriesDescription.LocationIdentifier);
var(exportDuration, exportLabel) = GetExportDuration(timeSeriesDescription);
var dataRequest = new TimeSeriesDataCorrectedServiceRequest
{
TimeSeriesUniqueId = timeSeriesDescription.UniqueId,
QueryFrom = GetInitialQueryFrom(detectedChange),
ApplyRounding = Context.ApplyRounding,
};
var existingSensor = Sos.FindExistingSensor(timeSeriesDescription);
var deleteExistingSensor = clearExportedData && existingSensor != null;
var assignedOffering = existingSensor?.Identifier;
var lastSensorTime = GetLastSensorTime(existingSensor);
if (HaveExistingSosPointsChanged(dataRequest, lastSensorTime, detectedChange, timeSeriesDescription))
{
Log.Warn($"FirstPointChanged={detectedChange.FirstPointChanged:O} AttributeChange={detectedChange.HasAttributeChange} of '{timeSeriesDescription.Identifier}' precedes LastSensorTime={lastSensorTime:O} of '{existingSensor?.Identifier}'. Forcing delete of existing sensor.");
// A point has changed before the last known observation, so we'll need to throw out the entire sensor
deleteExistingSensor = true;
// We'll also need to fetch more data again
dataRequest.QueryFrom = null;
}
if (dataRequest.QueryFrom == null)
{
// Get the full extraction
var endPoint = dataRequest.QueryTo ?? DateTimeOffset.Now;
var startOfToday = new DateTimeOffset(endPoint.Year, endPoint.Month, endPoint.Day, 0, 0, 0,
timeSeriesDescription.UtcOffsetIsoDuration.ToTimeSpan());
dataRequest.QueryFrom = startOfToday - exportDuration;
}
Log.Info($"Fetching changes from '{timeSeriesDescription.Identifier}' FirstPointChanged={detectedChange.FirstPointChanged:O} HasAttributeChanged={detectedChange.HasAttributeChange} QueryFrom={dataRequest.QueryFrom:O} ...");
var timeSeries = Aquarius.Publish.Get(dataRequest);
TrimExcludedPoints(timeSeriesDescription, timeSeries);
var createSensor = existingSensor == null || deleteExistingSensor;
TimeSeriesPointFilter.FilterTimeSeriesPoints(timeSeries);
var exportSummary = $"{timeSeries.NumPoints} points [{timeSeries.Points.FirstOrDefault()?.Timestamp.DateTimeOffset:O} to {timeSeries.Points.LastOrDefault()?.Timestamp.DateTimeOffset:O}] from '{timeSeriesDescription.Identifier}' with ExportDuration={exportLabel}";
ExportedTimeSeriesCount += 1;
ExportedPointCount += timeSeries.NumPoints ?? 0;
if (Context.DryRun)
{
if (deleteExistingSensor)
{
LogDryRun($"Would delete existing sensor '{existingSensor?.Identifier}'");
}
if (createSensor)
{
LogDryRun($"Would create new sensor for '{timeSeriesDescription.Identifier}'");
}
LogDryRun($"Would export {exportSummary}.");
return;
}
Log.Info($"Exporting {exportSummary} ...");
if (deleteExistingSensor)
{
Sos.DeleteSensor(timeSeries);
Sos.DeleteDeletedObservations();
}
if (createSensor)
{
var sensor = Sos.InsertSensor(timeSeries);
assignedOffering = sensor.AssignedOffering;
}
Sos.InsertObservation(assignedOffering, locationInfo.LocationData, locationInfo.LocationDescription, timeSeries);
}
private void RunOnce()
{
SyncStatus = new SyncStatus(Aquarius)
{
Context = Context
};
TimeSeriesPointFilter = new TimeSeriesPointFilter {
Context = Context
};
ValidateFilters();
MaximumExportDuration = Context.MaximumPollDuration
?? SyncStatus.GetMaximumChangeEventDuration()
.Subtract(TimeSpan.FromHours(1));
var request = CreateFilterRequest();
if (Context.ForceResync)
{
Log.Warn("Forcing a full time-series resync.");
request.ChangesSinceToken = null;
}
else if (Context.ChangesSince.HasValue)
{
Log.Warn($"Overriding current ChangesSinceToken='{request.ChangesSinceToken:O}' with '{Context.ChangesSince:O}'");
request.ChangesSinceToken = Context.ChangesSince.Value.UtcDateTime;
}
Log.Info($"Checking {GetFilterSummary(request)} ...");
var stopwatch = Stopwatch.StartNew();
var response = Aquarius.Publish.Get(request);
if (response.TokenExpired ?? false)
{
if (Context.NeverResync)
{
Log.Warn("Skipping a recommended resync.");
}
else
{
Log.Warn($"The ChangesSinceToken of {request.ChangesSinceToken:O} has expired. Forcing a full resync. You may need to run the exporter more frequently.");
request.ChangesSinceToken = null;
response = Aquarius.Publish.Get(request);
}
}
var bootstrapToken = response.ResponseTime
.Subtract(stopwatch.Elapsed)
.Subtract(TimeSpan.FromMinutes(1))
.UtcDateTime;
var nextChangesSinceToken = response.NextToken ?? bootstrapToken;
var timeSeriesDescriptions = FetchChangedTimeSeriesDescriptions(response);
Log.Info($"Connecting to {Context.Config.SosServer} as {Context.Config.SosUsername} ...");
var clearExportedData = !request.ChangesSinceToken.HasValue;
request.ChangesSinceToken = nextChangesSinceToken;
ExportToSos(request, response, timeSeriesDescriptions, clearExportedData);
RemoveDeletedTimeSeries(response);
SyncStatus.SaveConfiguration(request.ChangesSinceToken.Value);
}
private void ExportTimeSeries(
bool clearExportedData,
TimeSeriesChangeEvent detectedChange,
TimeSeriesDescription timeSeriesDescription)
{
Log.Info($"Fetching changes from '{timeSeriesDescription.Identifier}' FirstPointChanged={detectedChange.FirstPointChanged:O} HasAttributeChanged={detectedChange.HasAttributeChange} ...");
var locationInfo = GetLocationInfo(timeSeriesDescription.LocationIdentifier);
var period = GetTimeSeriesPeriod(timeSeriesDescription);
var dataRequest = new TimeSeriesDataCorrectedServiceRequest
{
TimeSeriesUniqueId = timeSeriesDescription.UniqueId,
QueryFrom = GetInitialQueryFrom(detectedChange),
ApplyRounding = Context.ApplyRounding,
};
var existingSensor = Sos.FindExistingSensor(timeSeriesDescription);
var deleteExistingSensor = clearExportedData && existingSensor != null;
var assignedOffering = existingSensor?.Identifier;
var timeSeries = FetchMinimumTimeSeries(detectedChange, timeSeriesDescription, existingSensor, dataRequest, ref deleteExistingSensor, ref period);
var createSensor = existingSensor == null || deleteExistingSensor;
TimeSeriesPointFilter.FilterTimeSeriesPoints(timeSeries);
var exportSummary = $"{timeSeries.NumPoints} points [{timeSeries.Points.FirstOrDefault()?.Timestamp.DateTimeOffset:O} to {timeSeries.Points.LastOrDefault()?.Timestamp.DateTimeOffset:O}] from '{timeSeriesDescription.Identifier}' with Frequency={period}";
ExportedTimeSeriesCount += 1;
ExportedPointCount += timeSeries.NumPoints ?? 0;
if (Context.DryRun)
{
if (deleteExistingSensor)
{
LogDryRun($"Would delete existing sensor '{existingSensor?.Identifier}'");
}
if (createSensor)
{
LogDryRun($"Would create new sensor for '{timeSeriesDescription.Identifier}'");
}
LogDryRun($"Would export {exportSummary}.");
return;
}
Log.Info($"Exporting {exportSummary} ...");
if (deleteExistingSensor)
{
Sos.DeleteSensor(timeSeries);
Sos.DeleteDeletedObservations();
}
if (createSensor)
{
var sensor = Sos.InsertSensor(timeSeries);
assignedOffering = sensor.AssignedOffering;
}
Sos.InsertObservation(assignedOffering, locationInfo.LocationData, locationInfo.LocationDescription, timeSeries, timeSeriesDescription);
}