private void MeasurePollResponseTime(long elapsedMilliseconds, TimeSeriesUniqueIdListServiceResponse response) { foreach (var timingBucket in TimingBuckets) { if (timingBucket.UpperBoundMilliseconds <= elapsedMilliseconds) { continue; } timingBucket.TotalMilliseconds += elapsedMilliseconds; timingBucket.ResponseCount += 1; timingBucket.DetectedChangeCount += response.TimeSeriesUniqueIds.Count; break; } }
private void RemoveDeletedTimeSeries(TimeSeriesUniqueIdListServiceResponse response) { var deletedTimeSeriesUniqueIds = response .TimeSeriesUniqueIds .Where(ts => ts.IsDeleted ?? false) .ToList(); if (!deletedTimeSeriesUniqueIds.Any()) { return; } // TODO: At this point, we only have a time-series GUID, no Identifier. // If the SOS sensor could be configured with some property that would store the GUID, then we could find it and delete it. Log.Warn($"{"deleted time-series".ToQuantity(deletedTimeSeriesUniqueIds.Count)} detected. A full resync may be required!"); }
private void ExportToSos( TimeSeriesUniqueIdListServiceRequest request, TimeSeriesUniqueIdListServiceResponse response, List <TimeSeriesDescription> timeSeriesDescriptions, bool clearExportedData) { var filteredTimeSeriesDescriptions = FilterTimeSeriesDescriptions(timeSeriesDescriptions); var changeEvents = response.TimeSeriesUniqueIds; Log.Info($"Exporting {filteredTimeSeriesDescriptions.Count} time-series ..."); if (clearExportedData) { ClearExportedData(); } var stopwatch = Stopwatch.StartNew(); foreach (var timeSeriesDescription in filteredTimeSeriesDescriptions) { using (Sos = SosClient.CreateConnectedClient(Context)) { // Create a separate SOS client connection to ensure that the transactions are committed after each export var description = timeSeriesDescription; ExportTimeSeries( clearExportedData, request.ChangesSinceToken, changeEvents.Single(t => t.UniqueId == description.UniqueId), timeSeriesDescription); } if (stopwatch.Elapsed <= MaximumExportDuration) { continue; } Log.Info($"Maximum export duration has elapsed. Checking {GetFilterSummary(request)} ..."); stopwatch.Restart(); FetchNewChanges(request, filteredTimeSeriesDescriptions, changeEvents); } }
private void SaveDetectedChanges(List <TimeSeriesUniqueIds> changedSeries, TimeSeriesUniqueIdListServiceResponse response, bool isFullSync = false) { if (string.IsNullOrEmpty(Context.DetectedChangesCsv)) { return; } if (!response.NextToken.HasValue) { isFullSync = true; } var detectedChanges = changedSeries .Select(ts => Convert(ts, isFullSync)) .ToList(); var csvOutput = CsvSerializer.SerializeToCsv(detectedChanges); File.WriteAllText(Context.DetectedChangesCsv, csvOutput); }
private List <TimeSeriesDescription> FetchChangedTimeSeriesDescriptions(TimeSeriesUniqueIdListServiceResponse response) { var timeSeriesUniqueIdsToFetch = response.TimeSeriesUniqueIds .Where(ts => !(ts.IsDeleted ?? false)) .Select(ts => ts.UniqueId) .ToList(); Log.Info($"Fetching descriptions of {timeSeriesUniqueIdsToFetch.Count} changed time-series ..."); var timeSeriesDescriptions = new List <TimeSeriesDescription>(); using (var batchClient = CreatePublishClientWithPostMethodOverride()) { while (timeSeriesUniqueIdsToFetch.Any()) { const int batchSize = 400; var batchList = timeSeriesUniqueIdsToFetch.Take(batchSize).ToList(); timeSeriesUniqueIdsToFetch = timeSeriesUniqueIdsToFetch.Skip(batchSize).ToList(); var request = new TimeSeriesDescriptionListByUniqueIdServiceRequest(); // We need to resolve the URL without any unique IDs on the GET command line var requestUrl = RemoveQueryFromUrl(request.ToGetUrl()); request.TimeSeriesUniqueIds = batchList; var batchResponse = batchClient.Send <TimeSeriesDescriptionListByUniqueIdServiceResponse>(HttpMethods.Post, requestUrl, request); timeSeriesDescriptions.AddRange(batchResponse.TimeSeriesDescriptions); } } return(timeSeriesDescriptions .OrderBy(ts => ts.LocationIdentifier) .ThenBy(ts => ts.Identifier) .ToList()); }