public void GetPointsForIndexing_ShouldGetAllPointsFromGateway()
{
_wikipediaGateway.GetByBoundingBox(Arg.Any <Coordinate>(), Arg.Any <Coordinate>(), Arg.Any <string>()).Returns(new List <Feature> {
GetValidFeature("1", Sources.WIKIPEDIA)
});
var points = _adapter.GetPointsForIndexing().Result;
_wikipediaGateway.Received(952).GetByBoundingBox(Arg.Any <Coordinate>(), Arg.Any <Coordinate>(), Arg.Any <string>());
Assert.AreEqual(1, points.Count); // only 1 distinct
}
/// <inheritdoc />
public override async Task <List <Feature> > GetPointsForIndexing()
{
_logger.LogInformation("Start getting Wikipedia pages for indexing.");
var startCoordinate = new Coordinate(34, 29);
var endCoordinate = new Coordinate(36, 34);
double step = 0.15; // bigger step causes wiki toobig exception...
var coordinatesList = new List <Coordinate>();
var currentCoordinate = new Coordinate();
for (
currentCoordinate.X = startCoordinate.X;
currentCoordinate.X < endCoordinate.X;
currentCoordinate.X += step
)
{
for (
currentCoordinate.Y = startCoordinate.Y;
currentCoordinate.Y < endCoordinate.Y;
currentCoordinate.Y += step
)
{
coordinatesList.Add(currentCoordinate.Copy());
}
}
_logger.LogInformation($"Created {coordinatesList.Count} coordinates centers to fetch Wikipedia data.");
var lists = new ConcurrentBag <List <Feature> >();
await Task.Run(() =>
{
Parallel.ForEach(coordinatesList, new ParallelOptions {
MaxDegreeOfParallelism = 10
}, (coordinate) =>
{
foreach (var language in Languages.Array)
{
lists.Add(_wikipediaGateway.GetByBoundingBox(coordinate, new Coordinate(coordinate.X + step, coordinate.Y + step), language).Result);
}
});
}).ConfigureAwait(false);
var wikiFeatures = lists.SelectMany(l => l)
.GroupBy(f => f.Attributes[FeatureAttributes.ID])
.Select(g => g.First())
.ToList();
_logger.LogInformation($"Finished getting Wikipedia pages for indexing, got {wikiFeatures.Count} pages.");
return(wikiFeatures);
}
