public async Task <IActionResult> PostRebuildSource(string source)
{
var adapter = _adaptersFactory.GetBySource(source);
if (adapter == null)
{
return(NotFound($"Source {source} does not exist"));
}
_logger.LogInformation($"Starting rebuilding {source}, getting points...");
var points = await adapter.GetPointsForIndexing();
_logger.LogInformation($"Got {points.Count} points for {source}");
var fullPoints = new ConcurrentBag <FeatureCollection>();
await _elasticSearchGateway.DeleteExternalPoisBySource(source);
var counter = 0;
Parallel.For(0, points.Count, new ParallelOptions {
MaxDegreeOfParallelism = 10
}, (index) =>
{
var featureCollection = adapter.GetRawPointOfInterestById(points[index].Attributes[FeatureAttributes.ID].ToString()).Result;
_elasticSearchGateway.AddExternalPoi(featureCollection);
Interlocked.Increment(ref counter);
if (counter % 100 == 0)
{
_logger.LogInformation($"Indexed {counter} points of {points.Count} for {source}");
}
});
_logger.LogInformation($"Finished rebuilding {source}, indexed {points.Count} points.");
// HM TODO: set rebuild date for source somehow...
return(Ok());
}
public async Task <IActionResult> PostRebuildSource(string source)
{
if (!GetSources().Contains(source) && source != "all")
{
return(NotFound($"Source {source} does not exist"));
}
var sources = source == "all" ? GetSources() : new[] { source };
foreach (var currentSource in sources)
{
_logger.LogInformation($"Starting rebuilding {currentSource}, getting points...");
var adapter = _adaptersFactory.GetBySource(currentSource);
var points = await adapter.GetPointsForIndexing();
_logger.LogInformation($"Got {points.Count} points for {currentSource}");
var fullPoints = new ConcurrentBag <FeatureCollection>();
await _elasticSearchGateway.DeleteExternalPoisBySource(currentSource);
var counter = 0;
Parallel.For(0, points.Count, new ParallelOptions {
MaxDegreeOfParallelism = 10
}, (index) =>
{
try
{
var feature = adapter.GetRawPointOfInterestById(points[index].Attributes[FeatureAttributes.ID].ToString()).Result;
var geoLocation = feature.Attributes[FeatureAttributes.POI_GEOLOCATION] as AttributesTable;
var geoLocationCoordinate = new Coordinate((double)geoLocation[FeatureAttributes.LON], (double)geoLocation[FeatureAttributes.LAT]);
feature.Attributes.AddOrUpdate(FeatureAttributes.POI_ALT, _elevationDataStorage.GetElevation(geoLocationCoordinate).Result);
var northEast = _wgs84ItmTransform.Transform(geoLocationCoordinate.X, geoLocationCoordinate.Y);
feature.Attributes.AddOrUpdate(FeatureAttributes.POI_ITM_EAST, (int)northEast.x);
feature.Attributes.AddOrUpdate(FeatureAttributes.POI_ITM_NORTH, (int)northEast.y);
_elasticSearchGateway.AddExternalPoi(feature);
Interlocked.Increment(ref counter);
if (counter % 100 == 0)
{
_logger.LogInformation($"Indexed {counter} points of {points.Count} for {currentSource}");
}
}
catch (Exception ex)
{
_logger.LogError($"failed to index point with title: {points[index].GetTitle(Languages.ALL)} for {currentSource} with exception: {ex.ToString()}");
}
});
_logger.LogInformation($"Finished rebuilding {currentSource}, indexed {points.Count} points.");
// HM TODO: set rebuild date for source somehow...
}
return(Ok());
}
private async Task RebuildPointsOfInterest()
{
_logger.LogInformation("Starting rebuilding POIs database.");
var osmSource = _pointsOfInterestAdapterFactory.GetBySource(Sources.OSM);
var osmFeaturesTask = osmSource.GetPointsForIndexing();
var sources = _pointsOfInterestAdapterFactory.GetAll().Where(s => s.Source != Sources.OSM).Select(s => s.Source);
var otherTasks = sources.Select(s => _elasticSearchGateway.GetExternalPoisBySource(s));
await Task.WhenAll(new Task[] { osmFeaturesTask }.Concat(otherTasks));
var features = _featuresMergeExecutor.Merge(osmFeaturesTask.Result.Concat(otherTasks.SelectMany(t => t.Result)).ToList());
foreach (var feature in features)
{
var geoLocation = feature.Attributes[FeatureAttributes.POI_GEOLOCATION] as AttributesTable;
feature.Attributes.AddOrUpdate(FeatureAttributes.POI_ALT, await _elevationDataStorage.GetElevation(
new Coordinate((double)geoLocation[FeatureAttributes.LON], (double)geoLocation[FeatureAttributes.LAT]))
);
var northEast = _mathTransform.Transform((double)geoLocation[FeatureAttributes.LON], (double)geoLocation[FeatureAttributes.LAT]);
feature.Attributes.AddOrUpdate(FeatureAttributes.POI_ITM_EAST, (int)northEast.x);
feature.Attributes.AddOrUpdate(FeatureAttributes.POI_ITM_NORTH, (int)northEast.y);
}
await _elasticSearchGateway.UpdatePointsOfInterestZeroDownTime(features);
_logger.LogInformation("Finished rebuilding POIs database.");
}
/// <inheritdoc />
public async Task <PointOfInterestExtended> Get(string source, string id, string language = "")
{
var adapter = _pointsOfInterestAdapterFactory.GetBySource(source);
if (adapter == null)
{
return(null);
}
var poiItem = await adapter.GetPointOfInterestById(id, language);
if (poiItem == null)
{
return(null);
}
if (poiItem.CombinedIds == null)
{
return(poiItem);
}
foreach (var poiItemCombinedIdKey in poiItem.CombinedIds.Keys)
{
adapter = _pointsOfInterestAdapterFactory.GetBySource(poiItemCombinedIdKey);
foreach (var currentId in poiItem.CombinedIds[poiItemCombinedIdKey])
{
var currentPoiItem = await adapter.GetPointOfInterestById(currentId, language);
if (currentPoiItem == null)
{
continue;
}
if (string.IsNullOrWhiteSpace(poiItem.Description))
{
poiItem.Description = currentPoiItem.Description;
poiItem.Contribution = currentPoiItem.Contribution;
}
poiItem.ImagesUrls = poiItem.ImagesUrls.Concat(currentPoiItem.ImagesUrls)
.Distinct()
.ToArray();
poiItem.References = poiItem.References.Concat(currentPoiItem.References)
.GroupBy(r => r.Url)
.Select(r => r.FirstOrDefault())
.ToArray();
}
}
return(poiItem);
}
private async Task RebuildPointsOfInterest()
{
_logger.LogInformation("Starting rebuilding POIs database.");
var osmSource = _pointsOfInterestAdapterFactory.GetBySource(Sources.OSM);
var osmFeaturesTask = osmSource.GetPointsForIndexing();
var sources = _pointsOfInterestAdapterFactory.GetAll().Where(s => s.Source != Sources.OSM).Select(s => s.Source);
var otherTasks = sources.Select(s => _elasticSearchGateway.GetExternalPoisBySource(s)).ToArray();
await Task.WhenAll(new Task[] { osmFeaturesTask }.Concat(otherTasks));
var features = _featuresMergeExecutor.Merge(osmFeaturesTask.Result.Concat(otherTasks.SelectMany(t => t.Result)).ToList());
await _elasticSearchGateway.UpdatePointsOfInterestZeroDownTime(features);
_logger.LogInformation("Finished rebuilding POIs database.");
}
public void TestRebuild_ShouldRebuildHighwaysAndPoints()
{
var adapter = Substitute.For <IPointsOfInterestAdapter>();
adapter.GetPointsForIndexing().Returns(new List <Feature>());
_pointsOfInterestAdapterFactory.GetBySource(Arg.Any <string>()).Returns(adapter);
_elasticSearchGateway.GetExternalPoisBySource(Arg.Any <string>()).Returns(new List <Feature>());
_featuresMergeExecutor.Merge(Arg.Any <List <Feature> >()).Returns(new List <Feature>());
_service.Rebuild(new UpdateRequest {
Highways = true, PointsOfInterest = true, SiteMap = true
}).Wait();
_elasticSearchGateway.Received(1).UpdateHighwaysZeroDownTime(Arg.Any <List <Feature> >());
_elasticSearchGateway.Received(1).UpdatePointsOfInterestZeroDownTime(Arg.Any <List <Feature> >());
_pointsOfInterestFilesCreatorExecutor.Received(1).Create(Arg.Any <List <Feature> >());
}
