Пример #1
0
        public async Task Process(ConfigurationItems configuration)
        {
            var sw = Stopwatch.StartNew();

            try
            {
                _logger.Information("try to read weatherinformations");
                var locationList =
                    (await _locationFileReader.ReadLocationsAsync(configuration.PathToLocationsMap)).ValueOr(
                        new List <string>());
                _logger.Information($"read the list of locations with {locationList.Count} entries");
                var rootTasksOption = _processingBaseImplementations
                                      .ConvertToParallelQuery(locationList, configuration.Parallelism)
                                      .Select(async location =>
                {
                    _logger.Information($"get weather information for configured {location}");
                    var url =
                        $"https://api.openweathermap.org/data/2.5/weather?{location}&APPID={configuration.OwmApiKey}&units=metric";
                    return(await _owmApiReader.ReadDataFromLocationAsync(url, location));
                });

                var rootOptions = (await Task.WhenAll(rootTasksOption)).Values();

                var rootStrings = _processingBaseImplementations
                                  .ConvertToParallelQuery(rootOptions, configuration.Parallelism)
                                  .Select(async rootString =>
                                          await _processingBaseImplementations.DeserializeObjectAsync <CurrentWeatherBase>(rootString));


                var readTime = DateTime.Now;
                _logger.Information($"define document timestamp for elastic is {readTime}");
                var toElastic = (await Task.WhenAll(rootStrings))
                                .Values()
                                .Select(item =>
                {
                    item.ReadTime = readTime;
                    item.Guid     = Guid.NewGuid();
                    return(item);
                });

                var concurrentBag = new ConcurrentBag <CurrentWeatherBase>(toElastic);

                await _processingUtils.WriteFilesToDirectory(configuration.FileStorageTemplate, concurrentBag);

                var elasticDocsTasks = _processingBaseImplementations
                                       .ConvertToParallelQuery(concurrentBag, configuration.Parallelism)
                                       .Select(async rootDoc => await _owmToElasticDocumentConverter.ConvertAsync(rootDoc));

                var elasticDocs = (await Task.WhenAll(elasticDocsTasks))
                                  .Values();

                var indexName = _elasticConnection.BuildIndexName(configuration.ElasticIndexName, readTime);
                _logger.Information($"write weather data to index {indexName}");

                if (!await _elasticConnection.IndexExistsAsync(indexName))
                {
                    await _elasticConnection.CreateIndexAsync <WeatherLocationDocument>(indexName);

                    await _elasticConnection.RefreshIndexAsync(indexName);

                    await _elasticConnection.FlushIndexAsync(indexName);
                }

                await _elasticConnection.BulkWriteDocumentsAsync(elasticDocs, indexName);
            }
            finally
            {
                sw.Stop();
                _logger.Information("Processed {MethodName} in {ElapsedMs:000} ms", "ProcessingBaseCurrentWeatherImpl.Execute",
                                    sw.ElapsedMilliseconds);
            }
        }
        public async Task Process(ConfigurationItems configuration)
        {
            var sw = Stopwatch.StartNew();

            try
            {
                var locationsList =
                    (await _locationFileReader.ReadLocationsAsync(configuration.AirPollutionLocationsFile)).ValueOr(
                        new List <string>());
                _logger.Information($"read the list of locations with {locationsList.Count} entries");
                var splitLocationList = locationsList.Select(element =>
                {
                    var splt = element.Split(";");
                    return(splt[0], splt[1]);
                });

                var rootTasks = _processingBaseImplementations
                                .ConvertToParallelQuery(splitLocationList, configuration.Parallelism)
                                .Select(async location =>
                {
                    _logger.Information($"get airpollution information for configured {location}");
                    var uri =
                        $"https://api.openweathermap.org/data/2.5/air_pollution?{location.Item1}&appid={configuration.OwmApiKey}";
                    var resultOpt = await _owmApiReader.ReadDataFromLocationAsync(uri, location.Item1);
                    return(resultOpt.Map(result => (location.Item2, result)));
                });

                var rootOptions = (await Task.WhenAll(rootTasks)).Values();

                var readTime = DateTime.Now;
                _logger.Information($"define document timestamp for elastic is {readTime}");
                var rootStrings = _processingBaseImplementations
                                  .ConvertToParallelQuery(rootOptions, configuration.Parallelism)
                                  .Select(async rootElement =>
                {
                    var elementOpt =
                        await _processingBaseImplementations
                        .DeserializeObjectAsync <AirPollutionBase>(rootElement.Item2);

                    return(elementOpt.Map(element =>
                    {
                        element.LocationName = rootElement.Item1;
                        element.ReadTime = readTime;
                        element.Guid = Guid.NewGuid();
                        return element;
                    }));
                });

                var toElastic = (await Task.WhenAll(rootStrings)).Values();

                var concurrentBag = new ConcurrentBag <AirPollutionBase>(toElastic);

                await _processingUtils.WriteFilesToDirectory(configuration.AirPollutionFileStoragePath, concurrentBag);

                var elasticDocTasks = _processingBaseImplementations
                                      .ConvertToParallelQuery(concurrentBag, configuration.Parallelism)
                                      .Select(async apDoc => await _owmToElasticDocumentConverter.ConvertAsync(apDoc));

                var elasticDocs = (await Task.WhenAll(elasticDocTasks)).Values();


                var indexName = _elasticConnection.BuildIndexName(configuration.AirPollutionIndexName, readTime);
                _logger.Information($"write airpollution data to index {indexName}");
                if (!await _elasticConnection.IndexExistsAsync(indexName))
                {
                    await _elasticConnection.CreateIndexAsync <AirPollutionDocument>(indexName);

                    await _elasticConnection.RefreshIndexAsync(indexName);

                    await _elasticConnection.FlushIndexAsync(indexName);
                }

                await _elasticConnection.BulkWriteDocumentsAsync(elasticDocs, indexName);
            }
            finally
            {
                sw.Stop();
                _logger.Information("Processed {MethodName} in {ElapsedMs:000} ms", "ProcessingBaseAirPollutionImpl.Execute",
                                    sw.ElapsedMilliseconds);
            }
        }