private void SaveAllNewSeries(TimeSeriesUniqueIdListServiceRequest request)
{
if (string.IsNullOrEmpty(Context.DetectedChangesCsv))
{
return;
}
// Fetch all of the time-series that match the filter
request.ChangesSinceToken = null;
Log.Info($"Fetching all time-series for initial sync ...");
var response = Client.Publish.Get(request);
var newSeriesUniqueIds = response
.TimeSeriesUniqueIds
.Where(ts => !(ts.IsDeleted ?? false))
.Select(ts => ts.UniqueId)
.ToList();
FetchAllUnknownSeriesDescriptions(newSeriesUniqueIds);
var changedSeries = GetChangedTimeSeries(response.TimeSeriesUniqueIds);
SaveDetectedChanges(changedSeries, response, true);
}
private string GetFilterSummary(TimeSeriesUniqueIdListServiceRequest request)
{
var sb = new StringBuilder();
sb.Append(string.IsNullOrEmpty(request.LocationIdentifier)
? "all locations"
: $"location '{request.LocationIdentifier}'");
var filters = new List <string>();
if (request.Publish.HasValue)
{
filters.Add($"Publish={request.Publish}");
}
if (!string.IsNullOrEmpty(request.Parameter))
{
filters.Add($"Parameter={request.Parameter}");
}
if (!string.IsNullOrEmpty(request.ComputationIdentifier))
{
filters.Add($"ComputationIdentifier={request.ComputationIdentifier}");
}
if (!string.IsNullOrEmpty(request.ComputationPeriodIdentifier))
{
filters.Add($"ComputationPeriodIdentifier={request.ComputationPeriodIdentifier}");
}
if (!string.IsNullOrEmpty(request.ChangeEventType))
{
filters.Add($"ChangeEventType={request.ChangeEventType}");
}
if (request.ExtendedFilters != null && request.ExtendedFilters.Any())
{
filters.Add($"ExtendedFilters={string.Join(", ", request.ExtendedFilters.Select(f => $"{f.FilterName}={f.FilterValue}"))}");
}
if (filters.Any())
{
sb.Append($" with {string.Join(" and ", filters)}");
}
sb.Append(" for ");
sb.Append(Context.MaximumChangeCount > 0
? $"up to {"change".ToQuantity(Context.MaximumChangeCount)}"
: "changes");
sb.Append(TimeSeries.Any() ? $" in {TimeSeries.Count}" : " in any");
sb.Append(" time-series");
return(sb.ToString());
}
private string GetFilterSummary(TimeSeriesUniqueIdListServiceRequest request)
{
var sb = new StringBuilder();
sb.Append(string.IsNullOrEmpty(request.LocationIdentifier)
? "all locations"
: $"location '{request.LocationIdentifier}'");
var filters = new List <string>();
if (request.Publish.HasValue)
{
filters.Add($"Publish={request.Publish}");
}
if (!string.IsNullOrEmpty(request.Parameter))
{
filters.Add($"Parameter={request.Parameter}");
}
if (!string.IsNullOrEmpty(request.ComputationIdentifier))
{
filters.Add($"ComputationIdentifier={request.ComputationIdentifier}");
}
if (!string.IsNullOrEmpty(request.ComputationPeriodIdentifier))
{
filters.Add($"ComputationPeriodIdentifier={request.ComputationPeriodIdentifier}");
}
if (!string.IsNullOrEmpty(request.ChangeEventType))
{
filters.Add($"ChangeEventType={request.ChangeEventType}");
}
if (request.ExtendedFilters != null && request.ExtendedFilters.Any())
{
filters.Add($"ExtendedFilters={string.Join(", ", request.ExtendedFilters.Select(f => $"{f.FilterName}={f.FilterValue}"))}");
}
if (filters.Any())
{
sb.Append($" with {string.Join(" and ", filters)}");
}
sb.Append(" for time-series");
if (request.ChangesSinceToken.HasValue)
{
sb.Append($" change since {request.ChangesSinceToken:O}");
}
return(sb.ToString());
}
private void PollForChangedPoints()
{
_sourceTimeSeriesUniqueId = GetTimeSeriesUniqueId(_sourceTimeSeriesIdentifier);
_reflectedTimeSeriesUniqueId = GetTimeSeriesUniqueId(_reflectedTimeSeriesIdentifier);
var timeSeriesType = GetTimeSeriesType(_reflectedTimeSeriesUniqueId);
if (timeSeriesType != TimeSeriesType.Reflected)
{
throw new ArgumentException($"{_reflectedTimeSeriesIdentifier} is not a reflected time-series (actual={timeSeriesType})");
}
var sourceLocation = ParseLocationIdentifier(_sourceTimeSeriesIdentifier);
while (true)
{
Console.WriteLine($"Polling for changes to {_sourceTimeSeriesIdentifier} since {_changesSinceTime} ...");
var request = new TimeSeriesUniqueIdListServiceRequest
{
ChangesSinceToken = _changesSinceTime.ToDateTimeUtc(),
LocationIdentifier = sourceLocation
};
var response = _client.Publish.Get(request);
var nextToken = Instant.FromDateTimeUtc(response.NextToken ?? DateTime.UtcNow.AddHours(-1));
_changesSinceTime = nextToken;
var hasTokenExpired = response.TokenExpired ?? false;
if (hasTokenExpired)
{
Console.WriteLine($"The changes since token expired. Some data may have been missed. Resetting to '{response.NextToken}'");
continue;
}
var sourceTimeSeries =
response.TimeSeriesUniqueIds.SingleOrDefault(t => t.UniqueId == _sourceTimeSeriesUniqueId);
if (sourceTimeSeries?.FirstPointChanged != null)
{
AppendRecalculatedPoints(Instant.FromDateTimeOffset(sourceTimeSeries.FirstPointChanged.Value));
}
Console.WriteLine($"Sleeping for {_pollInterval} ...");
Thread.Sleep(_pollInterval.ToTimeSpan());
}
}
private void ExportToSos(
TimeSeriesUniqueIdListServiceRequest request,
TimeSeriesUniqueIdListServiceResponse response,
List <TimeSeriesDescription> timeSeriesDescriptions,
bool clearExportedData)
{
var filteredTimeSeriesDescriptions = FilterTimeSeriesDescriptions(timeSeriesDescriptions);
var changeEvents = response.TimeSeriesUniqueIds;
Log.Info($"Exporting {filteredTimeSeriesDescriptions.Count} time-series ...");
if (clearExportedData)
{
ClearExportedData();
}
var stopwatch = Stopwatch.StartNew();
foreach (var timeSeriesDescription in filteredTimeSeriesDescriptions)
{
using (Sos = SosClient.CreateConnectedClient(Context))
{
// Create a separate SOS client connection to ensure that the transactions are committed after each export
var description = timeSeriesDescription;
ExportTimeSeries(
clearExportedData,
request.ChangesSinceToken,
changeEvents.Single(t => t.UniqueId == description.UniqueId),
timeSeriesDescription);
}
if (stopwatch.Elapsed <= MaximumExportDuration)
{
continue;
}
Log.Info($"Maximum export duration has elapsed. Checking {GetFilterSummary(request)} ...");
stopwatch.Restart();
FetchNewChanges(request, filteredTimeSeriesDescriptions, changeEvents);
}
}
private void FetchNewChanges(TimeSeriesUniqueIdListServiceRequest request, List <TimeSeriesDescription> timeSeriesToExport, List <TimeSeriesChangeEvent> timeSeriesChangeEvents)
{
var response = Aquarius.Publish.Get(request);
if (response.TokenExpired ?? !response.NextToken.HasValue)
{
throw new ExpectedException($"Logic-error: A secondary changes-since response should always have an updated token.");
}
request.ChangesSinceToken = response.NextToken;
var newTimeSeriesDescriptions = FilterTimeSeriesDescriptions(FetchChangedTimeSeriesDescriptions(response));
RemoveDeletedTimeSeries(response);
if (!newTimeSeriesDescriptions.Any())
{
return;
}
Log.Info($"Merging {newTimeSeriesDescriptions.Count} changed time-series into the export queue ...");
timeSeriesToExport.AddRange(newTimeSeriesDescriptions);
foreach (var newTimeSeriesDescription in newTimeSeriesDescriptions)
{
var newEvent = response.TimeSeriesUniqueIds.Single(e => e.UniqueId == newTimeSeriesDescription.UniqueId);
var existingEvent = timeSeriesChangeEvents.SingleOrDefault(e => e.UniqueId == newEvent.UniqueId);
if (existingEvent == null)
{
timeSeriesChangeEvents.Add(newEvent);
continue;
}
MergeTimeSeriesChangeEvent(existingEvent, newEvent);
}
}
