public async Task <IActionResult> PostRebuildSource(string source) { var adapter = _adaptersFactory.GetBySource(source); if (adapter == null) { return(NotFound($"Source {source} does not exist")); } _logger.LogInformation($"Starting rebuilding {source}, getting points..."); var points = await adapter.GetPointsForIndexing(); _logger.LogInformation($"Got {points.Count} points for {source}"); var fullPoints = new ConcurrentBag <FeatureCollection>(); await _elasticSearchGateway.DeleteExternalPoisBySource(source); var counter = 0; Parallel.For(0, points.Count, new ParallelOptions { MaxDegreeOfParallelism = 10 }, (index) => { var featureCollection = adapter.GetRawPointOfInterestById(points[index].Attributes[FeatureAttributes.ID].ToString()).Result; _elasticSearchGateway.AddExternalPoi(featureCollection); Interlocked.Increment(ref counter); if (counter % 100 == 0) { _logger.LogInformation($"Indexed {counter} points of {points.Count} for {source}"); } }); _logger.LogInformation($"Finished rebuilding {source}, indexed {points.Count} points."); // HM TODO: set rebuild date for source somehow... return(Ok()); }
public async Task <IActionResult> PostRebuildSource(string source) { if (!GetSources().Contains(source) && source != "all") { return(NotFound($"Source {source} does not exist")); } var sources = source == "all" ? GetSources() : new[] { source }; foreach (var currentSource in sources) { _logger.LogInformation($"Starting rebuilding {currentSource}, getting points..."); var adapter = _adaptersFactory.GetBySource(currentSource); var points = await adapter.GetPointsForIndexing(); _logger.LogInformation($"Got {points.Count} points for {currentSource}"); var fullPoints = new ConcurrentBag <FeatureCollection>(); await _elasticSearchGateway.DeleteExternalPoisBySource(currentSource); var counter = 0; Parallel.For(0, points.Count, new ParallelOptions { MaxDegreeOfParallelism = 10 }, (index) => { try { var feature = adapter.GetRawPointOfInterestById(points[index].Attributes[FeatureAttributes.ID].ToString()).Result; var geoLocation = feature.Attributes[FeatureAttributes.POI_GEOLOCATION] as AttributesTable; var geoLocationCoordinate = new Coordinate((double)geoLocation[FeatureAttributes.LON], (double)geoLocation[FeatureAttributes.LAT]); feature.Attributes.AddOrUpdate(FeatureAttributes.POI_ALT, _elevationDataStorage.GetElevation(geoLocationCoordinate).Result); var northEast = _wgs84ItmTransform.Transform(geoLocationCoordinate.X, geoLocationCoordinate.Y); feature.Attributes.AddOrUpdate(FeatureAttributes.POI_ITM_EAST, (int)northEast.x); feature.Attributes.AddOrUpdate(FeatureAttributes.POI_ITM_NORTH, (int)northEast.y); _elasticSearchGateway.AddExternalPoi(feature); Interlocked.Increment(ref counter); if (counter % 100 == 0) { _logger.LogInformation($"Indexed {counter} points of {points.Count} for {currentSource}"); } } catch (Exception ex) { _logger.LogError($"failed to index point with title: {points[index].GetTitle(Languages.ALL)} for {currentSource} with exception: {ex.ToString()}"); } }); _logger.LogInformation($"Finished rebuilding {currentSource}, indexed {points.Count} points."); // HM TODO: set rebuild date for source somehow... } return(Ok()); }
private async Task RebuildPointsOfInterest() { _logger.LogInformation("Starting rebuilding POIs database."); var osmSource = _pointsOfInterestAdapterFactory.GetBySource(Sources.OSM); var osmFeaturesTask = osmSource.GetPointsForIndexing(); var sources = _pointsOfInterestAdapterFactory.GetAll().Where(s => s.Source != Sources.OSM).Select(s => s.Source); var otherTasks = sources.Select(s => _elasticSearchGateway.GetExternalPoisBySource(s)); await Task.WhenAll(new Task[] { osmFeaturesTask }.Concat(otherTasks)); var features = _featuresMergeExecutor.Merge(osmFeaturesTask.Result.Concat(otherTasks.SelectMany(t => t.Result)).ToList()); foreach (var feature in features) { var geoLocation = feature.Attributes[FeatureAttributes.POI_GEOLOCATION] as AttributesTable; feature.Attributes.AddOrUpdate(FeatureAttributes.POI_ALT, await _elevationDataStorage.GetElevation( new Coordinate((double)geoLocation[FeatureAttributes.LON], (double)geoLocation[FeatureAttributes.LAT])) ); var northEast = _mathTransform.Transform((double)geoLocation[FeatureAttributes.LON], (double)geoLocation[FeatureAttributes.LAT]); feature.Attributes.AddOrUpdate(FeatureAttributes.POI_ITM_EAST, (int)northEast.x); feature.Attributes.AddOrUpdate(FeatureAttributes.POI_ITM_NORTH, (int)northEast.y); } await _elasticSearchGateway.UpdatePointsOfInterestZeroDownTime(features); _logger.LogInformation("Finished rebuilding POIs database."); }
/// <inheritdoc /> public async Task <PointOfInterestExtended> Get(string source, string id, string language = "") { var adapter = _pointsOfInterestAdapterFactory.GetBySource(source); if (adapter == null) { return(null); } var poiItem = await adapter.GetPointOfInterestById(id, language); if (poiItem == null) { return(null); } if (poiItem.CombinedIds == null) { return(poiItem); } foreach (var poiItemCombinedIdKey in poiItem.CombinedIds.Keys) { adapter = _pointsOfInterestAdapterFactory.GetBySource(poiItemCombinedIdKey); foreach (var currentId in poiItem.CombinedIds[poiItemCombinedIdKey]) { var currentPoiItem = await adapter.GetPointOfInterestById(currentId, language); if (currentPoiItem == null) { continue; } if (string.IsNullOrWhiteSpace(poiItem.Description)) { poiItem.Description = currentPoiItem.Description; poiItem.Contribution = currentPoiItem.Contribution; } poiItem.ImagesUrls = poiItem.ImagesUrls.Concat(currentPoiItem.ImagesUrls) .Distinct() .ToArray(); poiItem.References = poiItem.References.Concat(currentPoiItem.References) .GroupBy(r => r.Url) .Select(r => r.FirstOrDefault()) .ToArray(); } } return(poiItem); }
private async Task RebuildPointsOfInterest() { _logger.LogInformation("Starting rebuilding POIs database."); var osmSource = _pointsOfInterestAdapterFactory.GetBySource(Sources.OSM); var osmFeaturesTask = osmSource.GetPointsForIndexing(); var sources = _pointsOfInterestAdapterFactory.GetAll().Where(s => s.Source != Sources.OSM).Select(s => s.Source); var otherTasks = sources.Select(s => _elasticSearchGateway.GetExternalPoisBySource(s)).ToArray(); await Task.WhenAll(new Task[] { osmFeaturesTask }.Concat(otherTasks)); var features = _featuresMergeExecutor.Merge(osmFeaturesTask.Result.Concat(otherTasks.SelectMany(t => t.Result)).ToList()); await _elasticSearchGateway.UpdatePointsOfInterestZeroDownTime(features); _logger.LogInformation("Finished rebuilding POIs database."); }
public void TestRebuild_ShouldRebuildHighwaysAndPoints() { var adapter = Substitute.For <IPointsOfInterestAdapter>(); adapter.GetPointsForIndexing().Returns(new List <Feature>()); _pointsOfInterestAdapterFactory.GetBySource(Arg.Any <string>()).Returns(adapter); _elasticSearchGateway.GetExternalPoisBySource(Arg.Any <string>()).Returns(new List <Feature>()); _featuresMergeExecutor.Merge(Arg.Any <List <Feature> >()).Returns(new List <Feature>()); _service.Rebuild(new UpdateRequest { Highways = true, PointsOfInterest = true, SiteMap = true }).Wait(); _elasticSearchGateway.Received(1).UpdateHighwaysZeroDownTime(Arg.Any <List <Feature> >()); _elasticSearchGateway.Received(1).UpdatePointsOfInterestZeroDownTime(Arg.Any <List <Feature> >()); _pointsOfInterestFilesCreatorExecutor.Received(1).Create(Arg.Any <List <Feature> >()); }