private async Task Updatehighways(OsmChange changes)
{
var deleteTasks = new List <Task>();
foreach (var highwaysToRemove in changes.Delete.OfType <Way>())
{
var task = _elasticSearchGateway.DeleteHighwaysById(highwaysToRemove.Id.ToString());
deleteTasks.Add(task);
}
await Task.WhenAll(deleteTasks);
var updateTasks = new List <Task <CompleteWay> >();
foreach (var highwaysToUpdate in changes.Modify
.Concat(changes.Create)
.OfType <Way>()
.Where(w => w.Tags != null && w.Tags.ContainsKey("highway")))
{
var task = _osmGateway.GetCompleteWay(highwaysToUpdate.Id.ToString());
updateTasks.Add(task);
}
var updatedWays = await Task.WhenAll(updateTasks);
var geoJsonHighways = _osmGeoJsonPreprocessorExecutor.Preprocess(updatedWays.ToList());
await _elasticSearchGateway.UpdateHighwaysData(geoJsonHighways);
}
/// <inheritdoc />
public async Task <List <Feature> > GetPointsForIndexing(Stream memoryStream)
{
var osmNamesDictionary = await _osmRepository.GetElementsWithName(memoryStream);
var geoJsonNamesDictionary = _osmGeoJsonPreprocessorExecutor.Preprocess(osmNamesDictionary);
return(geoJsonNamesDictionary.Values.SelectMany(v => v).ToList());
}
/// <inheritdoc />
public async Task <List <Feature> > GetPointsForIndexing(Stream memoryStream)
{
var osmNamesDictionary = await _osmRepository.GetElementsWithName(memoryStream);
var relevantTagsDictionary = _tagsHelper.GetAllTags();
var namelessNodes = await _osmRepository.GetPointsWithNoNameByTags(memoryStream, relevantTagsDictionary);
osmNamesDictionary.Add(string.Empty, namelessNodes.Cast <ICompleteOsmGeo>().ToList());
RemoveKklRoutes(osmNamesDictionary);
var geoJsonNamesDictionary = _osmGeoJsonPreprocessorExecutor.Preprocess(osmNamesDictionary);
return(geoJsonNamesDictionary.Values.SelectMany(v => v).ToList());
}
/// <inheritdoc />
public override async Task <List <Feature> > GetPointsForIndexing()
{
_logger.LogInformation("Starting getting OSM points of interest");
using (var stream = _latestFileFetcherExecutor.Get())
{
var osmNamesDictionary = await _osmRepository.GetElementsWithName(stream);
var relevantTagsDictionary = _tagsHelper.GetAllTags();
var namelessNodes = await _osmRepository.GetPointsWithNoNameByTags(stream, relevantTagsDictionary);
osmNamesDictionary.Add(string.Empty, namelessNodes.Cast <ICompleteOsmGeo>().ToList());
var features = _osmGeoJsonPreprocessorExecutor.Preprocess(osmNamesDictionary);
_logger.LogInformation("Finished getting OSM points of interest: " + features.Count);
return(features);
}
}
public async Task <IActionResult> PostUpdateData(IFormFile file)
{
if (HttpContext.Connection.LocalIpAddress.Equals(HttpContext.Connection.RemoteIpAddress) == false &&
IPAddress.IsLoopback(HttpContext.Connection.RemoteIpAddress) == false &&
HttpContext.Connection.RemoteIpAddress.Equals(IPAddress.Parse("10.10.10.10")) == false)
{
return(BadRequest($"This operation can't be done from a remote client, please run this from the server: \n {HttpContext.Connection.LocalIpAddress}, {HttpContext.Connection.RemoteIpAddress}, {IPAddress.Parse("10.10.10.10")}"));
}
if (file == null)
{
return(BadRequest("No file was provided"));
}
_logger.LogInformation("Updating Elastic Search and graphhopper OSM data");
var memoryStream = new MemoryStream();
file.OpenReadStream().CopyTo(memoryStream);
_logger.LogInformation("Copy uploaded data completed.");
var elasticSearchTask = Task.Run(async() =>
{
var osmHighways = await _osmRepository.GetAllHighways(memoryStream);
var geoJsonHighways = _osmGeoJsonPreprocessorExecutor.Preprocess(osmHighways);
var fetchTask = _adapters.Select(a => a.GetPointsForIndexing(memoryStream)).ToArray();
var features = await Task.WhenAll(fetchTask);
await _elasticSearchGateway.UpdateDataZeroDownTime(features.SelectMany(v => v).ToList(), geoJsonHighways);
});
var graphHopperTask = _graphHopperGateway.Rebuild(memoryStream, file.FileName);
await Task.WhenAll(elasticSearchTask, graphHopperTask);
return(Ok());
}
private async Task AddWaysToElasticSearch(List <long?> wayIds)
{
var tasksList = wayIds.Select(wayId => _osmGateway.GetCompleteWay(wayId.Value)).ToList();
var newlyaddedWays = await Task.WhenAll(tasksList);
var newlyHighwaysFeatures = _geoJsonPreprocessorExecutor.Preprocess(newlyaddedWays.ToList());
await _elasticSearchGateway.UpdateHighwaysData(newlyHighwaysFeatures);
}
public void TestUpdate_EmptyOsmChangeFile_ShouldNotUpdateAnything()
{
var changes = new OsmChange {
Create = new OsmGeo[0], Modify = new OsmGeo[0], Delete = new OsmGeo[0]
};
_geoJsonPreprocessorExecutor
.Preprocess(Arg.Is <Dictionary <string, List <ICompleteOsmGeo> > >(x => x.Values.Count == 0))
.Returns(new List <Feature>());
_geoJsonPreprocessorExecutor
.Preprocess(Arg.Is <List <CompleteWay> >(x => x.Count == 0))
.Returns(new List <Feature>());
_service.Update(changes).Wait();
_elasticSearchGateway.Received(1).UpdatePointsOfInterestData(Arg.Is <List <Feature> >(x => x.Count == 0));
_elasticSearchGateway.Received(1).UpdateHighwaysData(Arg.Is <List <Feature> >(x => x.Count == 0));
}
public void PreprocessOneNode_ShouldNotDoAnyManipulation()
{
var node = CreateNode(1);
var osmElements = new List <ICompleteOsmGeo> {
node
};
var dictionary = new Dictionary <string, List <ICompleteOsmGeo> > {
{ FeatureAttributes.NAME, osmElements }
};
var results = _preprocessorExecutor.Preprocess(dictionary);
Assert.AreEqual(1, results.Count);
}
public void PostUpdateData_RemoteIs10101010_ShouldUpdateGateways()
{
SetupContext(IPAddress.Parse("1.2.3.4"), IPAddress.Parse("10.10.10.10"));
var file = Substitute.For <IFormFile>();
file.FileName.Returns("somefile.pbf");
file.OpenReadStream().Returns(new MemoryStream(new byte[] { 1 }));
_geoJsonPreprocessorExecutor.Preprocess(Arg.Any <Dictionary <string, List <ICompleteOsmGeo> > >()).Returns(new Dictionary <string, List <Feature> >());
_updateController.PostUpdateData(file).Wait();
_graphHopperGateway.Received(1).Rebuild(Arg.Any <MemoryStream>(), Arg.Any <string>());
_elasticSearchGateway.Received(1).UpdateDataZeroDownTime(Arg.Any <List <Feature> >(), Arg.Any <List <Feature> >());
}