private async Task <TimeSeriesInstance> ConvertStationToTimeSeriesInstanceAsync(Station station)
{
var instanceFields = new Dictionary <string, string>()
{
{ "FullName", station.Name },
{ "TimeZone", station.TimeZone }
};
AzureMapsClient.Address address = station.Latitude != null && station.Longitude != null ? await Retry.RetryWebCallAsync(
() => _azureMapsClient.SearchAddressReverseAsync(station.Latitude.Value, station.Longitude.Value),
$"UpdateTsm({_tsiEnvironmentFqdn})/ResolveStationAddress({station.ShortId})",
// Try few times. If failed, skip it - this station will have no address.
numberOfAttempts : 3, waitMilliseconds : 500, rethrowWebException : false) : null;
if (address != null)
{
instanceFields.Add(TsmMapping.GeoLocationMetadata.InstanceFieldName_Country, address.Country);
instanceFields.Add(TsmMapping.GeoLocationMetadata.InstanceFieldName_CountrySubdivisionName, address.CountrySubdivisionName);
instanceFields.Add(TsmMapping.GeoLocationMetadata.InstanceFieldName_CountrySecondarySubdivision, address.CountrySecondarySubdivision);
instanceFields.Add(TsmMapping.GeoLocationMetadata.InstanceFieldName_Municipality, address.Municipality);
instanceFields.Add(TsmMapping.GeoLocationMetadata.InstanceFieldName_PostalCode, address.PostalCode);
}
return(new TimeSeriesInstance(
timeSeriesId: new object[] { station.Id },
typeId: TsmMapping.StationObservationsType.Id,
name: $"Observations {station.ShortId} ({station.Name})",
description: "Weather observations for " + station.Name,
instanceFields: instanceFields,
hierarchyIds: new [] { TsmMapping.GeoLocationMetadata.GeoLocationsHierarchy.Id }));
}
public async Task Run()
{
DateTime?updateStationsTimestamp = null;
var tasks = new List <Task>(HttpUtils.DefaultConnectionLimit);
int passCount = 1;
while (true)
{
// Make a pass thru all stations.
Logger.TraceLine($"PASS {passCount} START");
Stopwatch stopwatch = Stopwatch.StartNew();
if (updateStationsTimestamp == null || DateTime.Now - updateStationsTimestamp > UpdateStationsInterval)
{
await ReloadStationsAndUpdateTsmAsync();
updateStationsTimestamp = DateTime.Now;
}
List <StationObservationsProcessor> stationObservationsProcessors = _stationObservationProcessors.Values.ToList();
bool noaaThrottlingDetected = false;
while (stationObservationsProcessors.Count > 0 || tasks.Count > 0)
{
while (stationObservationsProcessors.Count > 0 &&
tasks.Count < System.Net.ServicePointManager.DefaultConnectionLimit)
{
StationObservationsProcessor stationObservationsProcessor = stationObservationsProcessors[0];
tasks.Add(stationObservationsProcessor.ProsessStationObservationsAsync());
stationObservationsProcessors.RemoveAt(0);
}
Task completedTask = await Task.WhenAny(tasks);
tasks.Remove(completedTask);
try
{
await completedTask;
}
catch (Exception e)
{
noaaThrottlingDetected |= Retry.UnwrapAggregateException <NoaaThrottlingDetected>(e) != null;
if (noaaThrottlingDetected)
{
// Skip remaining work on this pass.
stationObservationsProcessors.Clear();
}
else
{
throw;
}
}
}
TimeSpan delay;
if (noaaThrottlingDetected)
{
Logger.TraceLine($"PASS {passCount} CANCELLED: Detected NOAA throttling, waiting until {DateTime.Now + DelayOnNoaaThrottling}");
delay = DelayOnNoaaThrottling;
}
else
{
Logger.TraceLine($"PASS {passCount} DONE: Elapsed {stopwatch.Elapsed}");
delay = (_stationObservationProcessors.Values.Min(sop => sop.GoodNextTimeToProcess) - DateTime.Now) + TimeSpan.FromMinutes(1);
delay = NormalPassDelay > delay ? NormalPassDelay : delay;
}
++passCount;
Logger.Flush();
await Task.Delay(delay);
}
}
