public async Task Process(ConfigurationItems configuration) { var sw = Stopwatch.StartNew(); try { _logger.Information("try to read weatherinformations"); var locationList = (await _locationFileReader.ReadLocationsAsync(configuration.PathToLocationsMap)).ValueOr( new List <string>()); _logger.Information($"read the list of locations with {locationList.Count} entries"); var rootTasksOption = _processingBaseImplementations .ConvertToParallelQuery(locationList, configuration.Parallelism) .Select(async location => { _logger.Information($"get weather information for configured {location}"); var url = $"https://api.openweathermap.org/data/2.5/weather?{location}&APPID={configuration.OwmApiKey}&units=metric"; return(await _owmApiReader.ReadDataFromLocationAsync(url, location)); }); var rootOptions = (await Task.WhenAll(rootTasksOption)).Values(); var rootStrings = _processingBaseImplementations .ConvertToParallelQuery(rootOptions, configuration.Parallelism) .Select(async rootString => await _processingBaseImplementations.DeserializeObjectAsync <CurrentWeatherBase>(rootString)); var readTime = DateTime.Now; _logger.Information($"define document timestamp for elastic is {readTime}"); var toElastic = (await Task.WhenAll(rootStrings)) .Values() .Select(item => { item.ReadTime = readTime; item.Guid = Guid.NewGuid(); return(item); }); var concurrentBag = new ConcurrentBag <CurrentWeatherBase>(toElastic); await _processingUtils.WriteFilesToDirectory(configuration.FileStorageTemplate, concurrentBag); var elasticDocsTasks = _processingBaseImplementations .ConvertToParallelQuery(concurrentBag, configuration.Parallelism) .Select(async rootDoc => await _owmToElasticDocumentConverter.ConvertAsync(rootDoc)); var elasticDocs = (await Task.WhenAll(elasticDocsTasks)) .Values(); var indexName = _elasticConnection.BuildIndexName(configuration.ElasticIndexName, readTime); _logger.Information($"write weather data to index {indexName}"); if (!await _elasticConnection.IndexExistsAsync(indexName)) { await _elasticConnection.CreateIndexAsync <WeatherLocationDocument>(indexName); await _elasticConnection.RefreshIndexAsync(indexName); await _elasticConnection.FlushIndexAsync(indexName); } await _elasticConnection.BulkWriteDocumentsAsync(elasticDocs, indexName); } finally { sw.Stop(); _logger.Information("Processed {MethodName} in {ElapsedMs:000} ms", "ProcessingBaseCurrentWeatherImpl.Execute", sw.ElapsedMilliseconds); } }
public async Task Process(ConfigurationItems configuration) { var sw = Stopwatch.StartNew(); try { var locationsList = (await _locationFileReader.ReadLocationsAsync(configuration.AirPollutionLocationsFile)).ValueOr( new List <string>()); _logger.Information($"read the list of locations with {locationsList.Count} entries"); var splitLocationList = locationsList.Select(element => { var splt = element.Split(";"); return(splt[0], splt[1]); }); var rootTasks = _processingBaseImplementations .ConvertToParallelQuery(splitLocationList, configuration.Parallelism) .Select(async location => { _logger.Information($"get airpollution information for configured {location}"); var uri = $"https://api.openweathermap.org/data/2.5/air_pollution?{location.Item1}&appid={configuration.OwmApiKey}"; var resultOpt = await _owmApiReader.ReadDataFromLocationAsync(uri, location.Item1); return(resultOpt.Map(result => (location.Item2, result))); }); var rootOptions = (await Task.WhenAll(rootTasks)).Values(); var readTime = DateTime.Now; _logger.Information($"define document timestamp for elastic is {readTime}"); var rootStrings = _processingBaseImplementations .ConvertToParallelQuery(rootOptions, configuration.Parallelism) .Select(async rootElement => { var elementOpt = await _processingBaseImplementations .DeserializeObjectAsync <AirPollutionBase>(rootElement.Item2); return(elementOpt.Map(element => { element.LocationName = rootElement.Item1; element.ReadTime = readTime; element.Guid = Guid.NewGuid(); return element; })); }); var toElastic = (await Task.WhenAll(rootStrings)).Values(); var concurrentBag = new ConcurrentBag <AirPollutionBase>(toElastic); await _processingUtils.WriteFilesToDirectory(configuration.AirPollutionFileStoragePath, concurrentBag); var elasticDocTasks = _processingBaseImplementations .ConvertToParallelQuery(concurrentBag, configuration.Parallelism) .Select(async apDoc => await _owmToElasticDocumentConverter.ConvertAsync(apDoc)); var elasticDocs = (await Task.WhenAll(elasticDocTasks)).Values(); var indexName = _elasticConnection.BuildIndexName(configuration.AirPollutionIndexName, readTime); _logger.Information($"write airpollution data to index {indexName}"); if (!await _elasticConnection.IndexExistsAsync(indexName)) { await _elasticConnection.CreateIndexAsync <AirPollutionDocument>(indexName); await _elasticConnection.RefreshIndexAsync(indexName); await _elasticConnection.FlushIndexAsync(indexName); } await _elasticConnection.BulkWriteDocumentsAsync(elasticDocs, indexName); } finally { sw.Stop(); _logger.Information("Processed {MethodName} in {ElapsedMs:000} ms", "ProcessingBaseAirPollutionImpl.Execute", sw.ElapsedMilliseconds); } }