private async Task UpdateTsmInstancesAsync(List <Station> stations) { List <TimeSeriesInstance> instances = await ConvertStationsToTimeSeriesInstancesAsync(stations); TsiDataClient.BatchResult[] putTimeSeriesInstances = await Retry.RetryWebCallAsync( () => _tsiDataClient.PutTimeSeriesInstancesAsync(_tsiEnvironmentFqdn, instances.ToArray()), $"UpdateTsm({_tsiEnvironmentFqdn})/Instances", // If failed, skip it during this pass - it will be retried during the next pass. numberOfAttempts : 1, waitMilliseconds : 0, rethrowWebException : false); // If failed, skip it during this pass - it will be retried during the next pass. if (putTimeSeriesInstances == null) { Logger.TraceLine($"UpdateTsm({_tsiEnvironmentFqdn})/Instances: Update for the following stations failed and will be skipped during this pass: "******", ", stations.Select(s => s.ShortId))); } else if (putTimeSeriesInstances.Any(e => !String.IsNullOrWhiteSpace(e.Error))) { Logger.TraceLine($"UpdateTsm({_tsiEnvironmentFqdn})/Instances: Update failed and will be skipped during this pass. Errors: " + String.Join(", ", putTimeSeriesInstances.Where(e => !String.IsNullOrWhiteSpace(e.Error)) .Select(e => $"instanceId({e.ItemId}) -> {e.Error}"))); } else { Logger.TraceLine($"UpdateTsm({_tsiEnvironmentFqdn})/Instances: Model for the following stations has been updated: " + String.Join(", ", stations.Select(s => s.ShortId))); } }
public async Task ProsessStationObservationsAsync() { if (DateTime.Now < GoodNextTimeToProcess) { Logger.TraceLine($"ProcessObservations({StationShortId}): {(GoodNextTimeToProcess - DateTime.Now).TotalMinutes:N} minutes remain till the next pull"); return; } DateTimeOffset pullFromTimestamp = _lastCommittedTimestamp ?? await _checkpointing.TryGetLastCommittedTimestampAsync(StationShortId) // For brand-new stations start with one day back. ?? DateTimeOffset.UtcNow - TimeSpan.FromDays(1); StationObservation[] observations = await Retry.RetryWebCallAsync( () => _noaaClient.GetStationObservationsAsync(StationShortId, pullFromTimestamp), $"ProcessObservations({StationShortId})", // Sometimes it fails with timeout, try it second time with some random delay, otherwise let it fail and skip during this pass. numberOfAttempts : 2, waitMilliseconds : RetryDelayRandom.Next(500, 1500), rethrowWebException : false); if (observations == null) { // If failed for any reason, skip it during this pass - it will be retried during the next pass. Logger.TraceLine($"ProcessObservations({StationShortId}): Pull failed and skipped during this pass"); return; } observations = observations.Where(o => o.Timestamp > pullFromTimestamp).OrderBy(o => o.Timestamp).ToArray(); if (observations.Length > 0) { TimeSpan sleepInterval = (DateTime.Now - _lastSuccessfulPullTime) / 2; sleepInterval = DefaultSleepInterval < sleepInterval ? DefaultSleepInterval : sleepInterval; GoodNextTimeToProcess = DateTime.Now + sleepInterval; _lastSuccessfulPullTime = DateTime.Now; await SendDataToEventHub(observations); _lastCommittedTimestamp = observations.Max(o => o.Timestamp); await _checkpointing.SetLastCommittedTimestampAsync(StationShortId, _lastCommittedTimestamp.Value); Logger.TraceLine( $"ProcessObservations({StationShortId}): {observations.Length} observations during the last " + $"{(DateTime.Now - pullFromTimestamp).TotalMinutes:N} minutes, " + $"available for the next pull in {(GoodNextTimeToProcess - DateTime.Now).TotalMinutes:N} minutes"); } else { // We've go here because there is no new data. _lastCommittedTimestamp = pullFromTimestamp; await _checkpointing.SetLastCommittedTimestampAsync(StationShortId, _lastCommittedTimestamp.Value); GoodNextTimeToProcess = DateTimeOffset.Now + DefaultSleepInterval / 2; Logger.TraceLine($"ProcessObservations({StationShortId}): No new data, available for the next pull in {(GoodNextTimeToProcess - DateTime.Now).TotalMinutes:N} minutes"); } }
public async Task <bool> UpdateTsmAsync() { if (Stations.Length == 0) { Logger.TraceLine($"UpdateTsm({_tsiEnvironmentFqdn}): Stations not loaded, model update skipped during this pass"); return(false); } TsiDataClient.BatchResult[] putTypesResult = await Retry.RetryWebCallAsync( () => _tsiDataClient.PutTimeSeriesTypesAsync(_tsiEnvironmentFqdn, new [] { TsmMapping.StationObservationsType }), "UpdateTsm({_tsiEnvironmentFqdn})/Type", // If failed, skip it during this pass - it will be retried during the next pass. numberOfAttempts : 1, waitMilliseconds : 0, rethrowWebException : false); // If failed, skip it during this pass - it will be retried during the next pass. if (putTypesResult == null) { Logger.TraceLine($"UpdateTsm({_tsiEnvironmentFqdn})/Type: Update failed, model update skipped during this pass"); return(false); } else if (putTypesResult.Any(e => !String.IsNullOrWhiteSpace(e.Error))) { Logger.TraceLine($"UpdateTsm({_tsiEnvironmentFqdn})/Type: Update failed, model update skipped during this pass. Errors: " + String.Join(", ", putTypesResult.Where(e => !String.IsNullOrWhiteSpace(e.Error)) .Select(e => $"typeId({e.ItemId}) -> {e.Error}"))); return(false); } TsiDataClient.BatchResult[] putHierarchiesResult = await Retry.RetryWebCallAsync( () => _tsiDataClient.PutTimeSeriesHierarchiesAsync(_tsiEnvironmentFqdn, new [] { TsmMapping.GeoLocationMetadata.GeoLocationsHierarchy }), "UpdateTsm({_tsiEnvironmentFqdn})/Hierarchy", // If failed, skip it during this pass - it will be retried during the next pass. numberOfAttempts : 1, waitMilliseconds : 0, rethrowWebException : false); // If failed, skip it during this pass - it will be retried during the next pass. if (putHierarchiesResult == null) { Logger.TraceLine($"UpdateTsm({_tsiEnvironmentFqdn})/Hierarchy: Update failed, model update skipped during this pass"); return(false); } else if (putHierarchiesResult.Any(e => !String.IsNullOrWhiteSpace(e.Error))) { Logger.TraceLine($"UpdateTsm({_tsiEnvironmentFqdn})/Hierarchy: Update failed, model update skipped during this pass. Errors: " + String.Join(", ", putHierarchiesResult.Where(e => !String.IsNullOrWhiteSpace(e.Error)) .Select(e => $"hierarchyId({e.ItemId}) -> {e.Error}"))); return(false); } await ProcessStationsInBatches(UpdateTsmInstancesAsync); return(true); }
public async Task ReloadStationsAsync() { Station[] stations = await Retry.RetryWebCallAsync( () => _noaaClient.GetStationsAsync(), "ReloadStations", // Keep retrying until we get the stations. numberOfAttempts : -1, waitMilliseconds : 5000, rethrowWebException : false); if (stations != null && stations.Length > 0) { Stations = stations; } }
private async Task <TimeSeriesInstance> ConvertStationToTimeSeriesInstanceAsync(Station station) { var instanceFields = new Dictionary <string, string>() { { "FullName", station.Name }, { "TimeZone", station.TimeZone } }; AzureMapsClient.Address address = station.Latitude != null && station.Longitude != null ? await Retry.RetryWebCallAsync( () => _azureMapsClient.SearchAddressReverseAsync(station.Latitude.Value, station.Longitude.Value), $"UpdateTsm({_tsiEnvironmentFqdn})/ResolveStationAddress({station.ShortId})", // Try few times. If failed, skip it - this station will have no address. numberOfAttempts : 3, waitMilliseconds : 500, rethrowWebException : false) : null; if (address != null) { instanceFields.Add(TsmMapping.GeoLocationMetadata.InstanceFieldName_Country, address.Country); instanceFields.Add(TsmMapping.GeoLocationMetadata.InstanceFieldName_CountrySubdivisionName, address.CountrySubdivisionName); instanceFields.Add(TsmMapping.GeoLocationMetadata.InstanceFieldName_CountrySecondarySubdivision, address.CountrySecondarySubdivision); instanceFields.Add(TsmMapping.GeoLocationMetadata.InstanceFieldName_Municipality, address.Municipality); instanceFields.Add(TsmMapping.GeoLocationMetadata.InstanceFieldName_PostalCode, address.PostalCode); } return(new TimeSeriesInstance( timeSeriesId: new object[] { station.Id }, typeId: TsmMapping.StationObservationsType.Id, name: $"Observations {station.ShortId} ({station.Name})", description: "Weather observations for " + station.Name, instanceFields: instanceFields, hierarchyIds: new [] { TsmMapping.GeoLocationMetadata.GeoLocationsHierarchy.Id })); }