private void MeasurePollResponseTime(long elapsedMilliseconds, TimeSeriesUniqueIdListServiceResponse response)
{
foreach (var timingBucket in TimingBuckets)
{
if (timingBucket.UpperBoundMilliseconds <= elapsedMilliseconds)
{
continue;
}
timingBucket.TotalMilliseconds += elapsedMilliseconds;
timingBucket.ResponseCount += 1;
timingBucket.DetectedChangeCount += response.TimeSeriesUniqueIds.Count;
break;
}
}
private void RemoveDeletedTimeSeries(TimeSeriesUniqueIdListServiceResponse response)
{
var deletedTimeSeriesUniqueIds = response
.TimeSeriesUniqueIds
.Where(ts => ts.IsDeleted ?? false)
.ToList();
if (!deletedTimeSeriesUniqueIds.Any())
{
return;
}
// TODO: At this point, we only have a time-series GUID, no Identifier.
// If the SOS sensor could be configured with some property that would store the GUID, then we could find it and delete it.
Log.Warn($"{"deleted time-series".ToQuantity(deletedTimeSeriesUniqueIds.Count)} detected. A full resync may be required!");
}
private void ExportToSos(
TimeSeriesUniqueIdListServiceRequest request,
TimeSeriesUniqueIdListServiceResponse response,
List <TimeSeriesDescription> timeSeriesDescriptions,
bool clearExportedData)
{
var filteredTimeSeriesDescriptions = FilterTimeSeriesDescriptions(timeSeriesDescriptions);
var changeEvents = response.TimeSeriesUniqueIds;
Log.Info($"Exporting {filteredTimeSeriesDescriptions.Count} time-series ...");
if (clearExportedData)
{
ClearExportedData();
}
var stopwatch = Stopwatch.StartNew();
foreach (var timeSeriesDescription in filteredTimeSeriesDescriptions)
{
using (Sos = SosClient.CreateConnectedClient(Context))
{
// Create a separate SOS client connection to ensure that the transactions are committed after each export
var description = timeSeriesDescription;
ExportTimeSeries(
clearExportedData,
request.ChangesSinceToken,
changeEvents.Single(t => t.UniqueId == description.UniqueId),
timeSeriesDescription);
}
if (stopwatch.Elapsed <= MaximumExportDuration)
{
continue;
}
Log.Info($"Maximum export duration has elapsed. Checking {GetFilterSummary(request)} ...");
stopwatch.Restart();
FetchNewChanges(request, filteredTimeSeriesDescriptions, changeEvents);
}
}
private void SaveDetectedChanges(List <TimeSeriesUniqueIds> changedSeries, TimeSeriesUniqueIdListServiceResponse response, bool isFullSync = false)
{
if (string.IsNullOrEmpty(Context.DetectedChangesCsv))
{
return;
}
if (!response.NextToken.HasValue)
{
isFullSync = true;
}
var detectedChanges = changedSeries
.Select(ts => Convert(ts, isFullSync))
.ToList();
var csvOutput = CsvSerializer.SerializeToCsv(detectedChanges);
File.WriteAllText(Context.DetectedChangesCsv, csvOutput);
}
private List <TimeSeriesDescription> FetchChangedTimeSeriesDescriptions(TimeSeriesUniqueIdListServiceResponse response)
{
var timeSeriesUniqueIdsToFetch = response.TimeSeriesUniqueIds
.Where(ts => !(ts.IsDeleted ?? false))
.Select(ts => ts.UniqueId)
.ToList();
Log.Info($"Fetching descriptions of {timeSeriesUniqueIdsToFetch.Count} changed time-series ...");
var timeSeriesDescriptions = new List <TimeSeriesDescription>();
using (var batchClient = CreatePublishClientWithPostMethodOverride())
{
while (timeSeriesUniqueIdsToFetch.Any())
{
const int batchSize = 400;
var batchList = timeSeriesUniqueIdsToFetch.Take(batchSize).ToList();
timeSeriesUniqueIdsToFetch = timeSeriesUniqueIdsToFetch.Skip(batchSize).ToList();
var request = new TimeSeriesDescriptionListByUniqueIdServiceRequest();
// We need to resolve the URL without any unique IDs on the GET command line
var requestUrl = RemoveQueryFromUrl(request.ToGetUrl());
request.TimeSeriesUniqueIds = batchList;
var batchResponse =
batchClient.Send <TimeSeriesDescriptionListByUniqueIdServiceResponse>(HttpMethods.Post,
requestUrl, request);
timeSeriesDescriptions.AddRange(batchResponse.TimeSeriesDescriptions);
}
}
return(timeSeriesDescriptions
.OrderBy(ts => ts.LocationIdentifier)
.ThenBy(ts => ts.Identifier)
.ToList());
}