private async Task UpdateTsmInstancesAsync(List <Station> stations)
{
List <TimeSeriesInstance> instances = await ConvertStationsToTimeSeriesInstancesAsync(stations);
TsiDataClient.BatchResult[] putTimeSeriesInstances = await Retry.RetryWebCallAsync(
() => _tsiDataClient.PutTimeSeriesInstancesAsync(_tsiEnvironmentFqdn, instances.ToArray()),
$"UpdateTsm({_tsiEnvironmentFqdn})/Instances",
// If failed, skip it during this pass - it will be retried during the next pass.
numberOfAttempts : 1, waitMilliseconds : 0, rethrowWebException : false);
// If failed, skip it during this pass - it will be retried during the next pass.
if (putTimeSeriesInstances == null)
{
Logger.TraceLine($"UpdateTsm({_tsiEnvironmentFqdn})/Instances: Update for the following stations failed and will be skipped during this pass: "******", ", stations.Select(s => s.ShortId)));
}
else if (putTimeSeriesInstances.Any(e => !String.IsNullOrWhiteSpace(e.Error)))
{
Logger.TraceLine($"UpdateTsm({_tsiEnvironmentFqdn})/Instances: Update failed and will be skipped during this pass. Errors: " +
String.Join(", ", putTimeSeriesInstances.Where(e => !String.IsNullOrWhiteSpace(e.Error))
.Select(e => $"instanceId({e.ItemId}) -> {e.Error}")));
}
else
{
Logger.TraceLine($"UpdateTsm({_tsiEnvironmentFqdn})/Instances: Model for the following stations has been updated: " +
String.Join(", ", stations.Select(s => s.ShortId)));
}
}
public async Task ProsessStationObservationsAsync()
{
if (DateTime.Now < GoodNextTimeToProcess)
{
Logger.TraceLine($"ProcessObservations({StationShortId}): {(GoodNextTimeToProcess - DateTime.Now).TotalMinutes:N} minutes remain till the next pull");
return;
}
DateTimeOffset pullFromTimestamp =
_lastCommittedTimestamp
?? await _checkpointing.TryGetLastCommittedTimestampAsync(StationShortId)
// For brand-new stations start with one day back.
?? DateTimeOffset.UtcNow - TimeSpan.FromDays(1);
StationObservation[] observations = await Retry.RetryWebCallAsync(
() => _noaaClient.GetStationObservationsAsync(StationShortId, pullFromTimestamp),
$"ProcessObservations({StationShortId})",
// Sometimes it fails with timeout, try it second time with some random delay, otherwise let it fail and skip during this pass.
numberOfAttempts : 2, waitMilliseconds : RetryDelayRandom.Next(500, 1500), rethrowWebException : false);
if (observations == null)
{
// If failed for any reason, skip it during this pass - it will be retried during the next pass.
Logger.TraceLine($"ProcessObservations({StationShortId}): Pull failed and skipped during this pass");
return;
}
observations = observations.Where(o => o.Timestamp > pullFromTimestamp).OrderBy(o => o.Timestamp).ToArray();
if (observations.Length > 0)
{
TimeSpan sleepInterval = (DateTime.Now - _lastSuccessfulPullTime) / 2;
sleepInterval = DefaultSleepInterval < sleepInterval ? DefaultSleepInterval : sleepInterval;
GoodNextTimeToProcess = DateTime.Now + sleepInterval;
_lastSuccessfulPullTime = DateTime.Now;
await SendDataToEventHub(observations);
_lastCommittedTimestamp = observations.Max(o => o.Timestamp);
await _checkpointing.SetLastCommittedTimestampAsync(StationShortId, _lastCommittedTimestamp.Value);
Logger.TraceLine(
$"ProcessObservations({StationShortId}): {observations.Length} observations during the last " +
$"{(DateTime.Now - pullFromTimestamp).TotalMinutes:N} minutes, " +
$"available for the next pull in {(GoodNextTimeToProcess - DateTime.Now).TotalMinutes:N} minutes");
}
else
{
// We've go here because there is no new data.
_lastCommittedTimestamp = pullFromTimestamp;
await _checkpointing.SetLastCommittedTimestampAsync(StationShortId, _lastCommittedTimestamp.Value);
GoodNextTimeToProcess = DateTimeOffset.Now + DefaultSleepInterval / 2;
Logger.TraceLine($"ProcessObservations({StationShortId}): No new data, available for the next pull in {(GoodNextTimeToProcess - DateTime.Now).TotalMinutes:N} minutes");
}
}
public async Task <bool> UpdateTsmAsync()
{
if (Stations.Length == 0)
{
Logger.TraceLine($"UpdateTsm({_tsiEnvironmentFqdn}): Stations not loaded, model update skipped during this pass");
return(false);
}
TsiDataClient.BatchResult[] putTypesResult = await Retry.RetryWebCallAsync(
() => _tsiDataClient.PutTimeSeriesTypesAsync(_tsiEnvironmentFqdn, new [] { TsmMapping.StationObservationsType }),
"UpdateTsm({_tsiEnvironmentFqdn})/Type",
// If failed, skip it during this pass - it will be retried during the next pass.
numberOfAttempts : 1, waitMilliseconds : 0, rethrowWebException : false);
// If failed, skip it during this pass - it will be retried during the next pass.
if (putTypesResult == null)
{
Logger.TraceLine($"UpdateTsm({_tsiEnvironmentFqdn})/Type: Update failed, model update skipped during this pass");
return(false);
}
else if (putTypesResult.Any(e => !String.IsNullOrWhiteSpace(e.Error)))
{
Logger.TraceLine($"UpdateTsm({_tsiEnvironmentFqdn})/Type: Update failed, model update skipped during this pass. Errors: " +
String.Join(", ", putTypesResult.Where(e => !String.IsNullOrWhiteSpace(e.Error))
.Select(e => $"typeId({e.ItemId}) -> {e.Error}")));
return(false);
}
TsiDataClient.BatchResult[] putHierarchiesResult = await Retry.RetryWebCallAsync(
() => _tsiDataClient.PutTimeSeriesHierarchiesAsync(_tsiEnvironmentFqdn, new [] { TsmMapping.GeoLocationMetadata.GeoLocationsHierarchy }),
"UpdateTsm({_tsiEnvironmentFqdn})/Hierarchy",
// If failed, skip it during this pass - it will be retried during the next pass.
numberOfAttempts : 1, waitMilliseconds : 0, rethrowWebException : false);
// If failed, skip it during this pass - it will be retried during the next pass.
if (putHierarchiesResult == null)
{
Logger.TraceLine($"UpdateTsm({_tsiEnvironmentFqdn})/Hierarchy: Update failed, model update skipped during this pass");
return(false);
}
else if (putHierarchiesResult.Any(e => !String.IsNullOrWhiteSpace(e.Error)))
{
Logger.TraceLine($"UpdateTsm({_tsiEnvironmentFqdn})/Hierarchy: Update failed, model update skipped during this pass. Errors: " +
String.Join(", ", putHierarchiesResult.Where(e => !String.IsNullOrWhiteSpace(e.Error))
.Select(e => $"hierarchyId({e.ItemId}) -> {e.Error}")));
return(false);
}
await ProcessStationsInBatches(UpdateTsmInstancesAsync);
return(true);
}
public async Task ReloadStationsAsync()
{
Station[] stations = await Retry.RetryWebCallAsync(
() => _noaaClient.GetStationsAsync(),
"ReloadStations",
// Keep retrying until we get the stations.
numberOfAttempts : -1, waitMilliseconds : 5000, rethrowWebException : false);
if (stations != null && stations.Length > 0)
{
Stations = stations;
}
}
private async Task <TimeSeriesInstance> ConvertStationToTimeSeriesInstanceAsync(Station station)
{
var instanceFields = new Dictionary <string, string>()
{
{ "FullName", station.Name },
{ "TimeZone", station.TimeZone }
};
AzureMapsClient.Address address = station.Latitude != null && station.Longitude != null ? await Retry.RetryWebCallAsync(
() => _azureMapsClient.SearchAddressReverseAsync(station.Latitude.Value, station.Longitude.Value),
$"UpdateTsm({_tsiEnvironmentFqdn})/ResolveStationAddress({station.ShortId})",
// Try few times. If failed, skip it - this station will have no address.
numberOfAttempts : 3, waitMilliseconds : 500, rethrowWebException : false) : null;
if (address != null)
{
instanceFields.Add(TsmMapping.GeoLocationMetadata.InstanceFieldName_Country, address.Country);
instanceFields.Add(TsmMapping.GeoLocationMetadata.InstanceFieldName_CountrySubdivisionName, address.CountrySubdivisionName);
instanceFields.Add(TsmMapping.GeoLocationMetadata.InstanceFieldName_CountrySecondarySubdivision, address.CountrySecondarySubdivision);
instanceFields.Add(TsmMapping.GeoLocationMetadata.InstanceFieldName_Municipality, address.Municipality);
instanceFields.Add(TsmMapping.GeoLocationMetadata.InstanceFieldName_PostalCode, address.PostalCode);
}
return(new TimeSeriesInstance(
timeSeriesId: new object[] { station.Id },
typeId: TsmMapping.StationObservationsType.Id,
name: $"Observations {station.ShortId} ({station.Name})",
description: "Weather observations for " + station.Name,
instanceFields: instanceFields,
hierarchyIds: new [] { TsmMapping.GeoLocationMetadata.GeoLocationsHierarchy.Id }));
}
