public async Task Process(ConfigurationItems configuration)
{
var sw = Stopwatch.StartNew();
try
{
_logger.Information("try to read weatherinformations");
var locationList =
(await _locationFileReader.ReadLocationsAsync(configuration.PathToLocationsMap)).ValueOr(
new List <string>());
_logger.Information($"read the list of locations with {locationList.Count} entries");
var rootTasksOption = _processingBaseImplementations
.ConvertToParallelQuery(locationList, configuration.Parallelism)
.Select(async location =>
{
_logger.Information($"get weather information for configured {location}");
var url =
$"https://api.openweathermap.org/data/2.5/weather?{location}&APPID={configuration.OwmApiKey}&units=metric";
return(await _owmApiReader.ReadDataFromLocationAsync(url, location));
});
var rootOptions = (await Task.WhenAll(rootTasksOption)).Values();
var rootStrings = _processingBaseImplementations
.ConvertToParallelQuery(rootOptions, configuration.Parallelism)
.Select(async rootString =>
await _processingBaseImplementations.DeserializeObjectAsync <CurrentWeatherBase>(rootString));
var readTime = DateTime.Now;
_logger.Information($"define document timestamp for elastic is {readTime}");
var toElastic = (await Task.WhenAll(rootStrings))
.Values()
.Select(item =>
{
item.ReadTime = readTime;
item.Guid = Guid.NewGuid();
return(item);
});
var concurrentBag = new ConcurrentBag <CurrentWeatherBase>(toElastic);
await _processingUtils.WriteFilesToDirectory(configuration.FileStorageTemplate, concurrentBag);
var elasticDocsTasks = _processingBaseImplementations
.ConvertToParallelQuery(concurrentBag, configuration.Parallelism)
.Select(async rootDoc => await _owmToElasticDocumentConverter.ConvertAsync(rootDoc));
var elasticDocs = (await Task.WhenAll(elasticDocsTasks))
.Values();
var indexName = _elasticConnection.BuildIndexName(configuration.ElasticIndexName, readTime);
_logger.Information($"write weather data to index {indexName}");
if (!await _elasticConnection.IndexExistsAsync(indexName))
{
await _elasticConnection.CreateIndexAsync <WeatherLocationDocument>(indexName);
await _elasticConnection.RefreshIndexAsync(indexName);
await _elasticConnection.FlushIndexAsync(indexName);
}
await _elasticConnection.BulkWriteDocumentsAsync(elasticDocs, indexName);
}
finally
{
sw.Stop();
_logger.Information("Processed {MethodName} in {ElapsedMs:000} ms", "ProcessingBaseCurrentWeatherImpl.Execute",
sw.ElapsedMilliseconds);
}
}
public async Task Process(ConfigurationItems configuration)
{
var sw = Stopwatch.StartNew();
try
{
var locationsList =
(await _locationFileReader.ReadLocationsAsync(configuration.AirPollutionLocationsFile)).ValueOr(
new List <string>());
_logger.Information($"read the list of locations with {locationsList.Count} entries");
var splitLocationList = locationsList.Select(element =>
{
var splt = element.Split(";");
return(splt[0], splt[1]);
});
var rootTasks = _processingBaseImplementations
.ConvertToParallelQuery(splitLocationList, configuration.Parallelism)
.Select(async location =>
{
_logger.Information($"get airpollution information for configured {location}");
var uri =
$"https://api.openweathermap.org/data/2.5/air_pollution?{location.Item1}&appid={configuration.OwmApiKey}";
var resultOpt = await _owmApiReader.ReadDataFromLocationAsync(uri, location.Item1);
return(resultOpt.Map(result => (location.Item2, result)));
});
var rootOptions = (await Task.WhenAll(rootTasks)).Values();
var readTime = DateTime.Now;
_logger.Information($"define document timestamp for elastic is {readTime}");
var rootStrings = _processingBaseImplementations
.ConvertToParallelQuery(rootOptions, configuration.Parallelism)
.Select(async rootElement =>
{
var elementOpt =
await _processingBaseImplementations
.DeserializeObjectAsync <AirPollutionBase>(rootElement.Item2);
return(elementOpt.Map(element =>
{
element.LocationName = rootElement.Item1;
element.ReadTime = readTime;
element.Guid = Guid.NewGuid();
return element;
}));
});
var toElastic = (await Task.WhenAll(rootStrings)).Values();
var concurrentBag = new ConcurrentBag <AirPollutionBase>(toElastic);
await _processingUtils.WriteFilesToDirectory(configuration.AirPollutionFileStoragePath, concurrentBag);
var elasticDocTasks = _processingBaseImplementations
.ConvertToParallelQuery(concurrentBag, configuration.Parallelism)
.Select(async apDoc => await _owmToElasticDocumentConverter.ConvertAsync(apDoc));
var elasticDocs = (await Task.WhenAll(elasticDocTasks)).Values();
var indexName = _elasticConnection.BuildIndexName(configuration.AirPollutionIndexName, readTime);
_logger.Information($"write airpollution data to index {indexName}");
if (!await _elasticConnection.IndexExistsAsync(indexName))
{
await _elasticConnection.CreateIndexAsync <AirPollutionDocument>(indexName);
await _elasticConnection.RefreshIndexAsync(indexName);
await _elasticConnection.FlushIndexAsync(indexName);
}
await _elasticConnection.BulkWriteDocumentsAsync(elasticDocs, indexName);
}
finally
{
sw.Stop();
_logger.Information("Processed {MethodName} in {ElapsedMs:000} ms", "ProcessingBaseAirPollutionImpl.Execute",
sw.ElapsedMilliseconds);
}
}