private void SetupRemoteFileFetcherWithBlankTile()
{
var bitmap = new Image <Rgba32>(256, 256);
var stream = new MemoryStream();
bitmap.SaveAsPng(stream);
_remoteFileFetcherGateway.GetFileContent(Arg.Any <string>())
.Returns(new RemoteFileFetcherGatewayResponse
{
FileName = "file.png",
Content = stream.ToArray()
});
}
private void SetupRemoteFileFetcherWithBlankTile()
{
var bitmap = new Bitmap(256, 256);
var stream = new MemoryStream();
bitmap.Save(stream, ImageFormat.Png);
_remoteFileFetcherGateway.GetFileContent(Arg.Any <string>())
.Returns(
Task.FromResult(new RemoteFileFetcherGatewayResponse
{
FileName = "file.png",
Content = stream.ToArray()
}));
}
/// <inheritdoc />
public override async Task <PointOfInterestExtended> GetPointOfInterestById(string id, string language)
{
var featureCollection = await GetFromCacheIfExists(id);
if (featureCollection == null)
{
var feature = GetRecords().Where(r => r.Id == id).Select(ConvertCsvRowToFeature).First();
if (feature.Attributes.Exists(FeatureAttributes.POI_SHARE_REFERENCE) &&
!string.IsNullOrWhiteSpace(feature.Attributes[FeatureAttributes.POI_SHARE_REFERENCE].ToString()))
{
var content = await _remoteFileFetcherGateway.GetFileContent(feature.Attributes[FeatureAttributes.POI_SHARE_REFERENCE].ToString());
var convertedBytes = await _dataContainerConverterService.Convert(content.Content, content.FileName, FlowFormats.GEOJSON);
feature.Geometry = convertedBytes.ToFeatureCollection().Features.FirstOrDefault()?.Geometry ?? feature.Geometry;
}
featureCollection = SetToCache(feature);
}
var poiItem = await ConvertToPoiExtended(featureCollection, language);
poiItem.IsEditable = false;
poiItem.IsArea = false;
poiItem.IsRoute = !(featureCollection.Features.First().Geometry is Point);
return(poiItem);
}
public void UploadCsv_ShouldConvertItAndAddMissingFields()
{
IFormFile file = Substitute.For <IFormFile>();
file.OpenReadStream()
.Returns(new MemoryStream(Encoding.UTF8.GetBytes(
"Title,Description,Website,ImageUrl,FileUrl\r\ntitle,description,website?id=42,image,file")));
_remoteFileFetcherGateway.GetFileContent("file").Returns(new RemoteFileFetcherGatewayResponse());
var featureCollection = new FeatureCollection {
new Feature(new Point(new Coordinate(11, 12)), new AttributesTable())
};
_dataContainerConverterService.Convert(Arg.Any <byte[]>(), Arg.Any <string>(), FlowFormats.GEOJSON)
.Returns(featureCollection.ToBytes());
var resutls = _controller.UploadCsv(file, "\\?id=(.*)", "http://sourceImageUrl/1.png", "icon", "icon-color", Categories.ROUTE_HIKE).Result as FileStreamResult;
Assert.IsNotNull(resutls);
var memoryStream = new MemoryStream();
resutls.FileStream.CopyTo(memoryStream);
var resutlsString = Encoding.UTF8.GetString(memoryStream.ToArray());
Assert.IsTrue(resutlsString.Contains("42"));
Assert.IsTrue(resutlsString.Contains("11"));
Assert.IsTrue(resutlsString.Contains("12"));
Assert.IsTrue(resutlsString.Contains("icon"));
Assert.IsTrue(resutlsString.Contains("icon-color"));
Assert.IsTrue(resutlsString.Contains("http://sourceImageUrl/1.png"));
Assert.IsTrue(resutlsString.Contains("Hiking"));
}
private async Task <ImageWithOffset> GetTileImage(Point topLeft, Point offset, int zoom, AddressAndOpacity addressTemplate)
{
var file = addressTemplate.Address.Replace("{z}", zoom.ToString())
.Replace("{zoom}", zoom.ToString())
.Replace("{x}", (topLeft.X + offset.X).ToString())
.Replace("{y}", (topLeft.Y + offset.Y).ToString());
var fileResponse = await _remoteFileFetcherGateway.GetFileContent(file);
if (!fileResponse.Content.Any())
{
return(new ImageWithOffset
{
Image = new Bitmap(TILE_SIZE, TILE_SIZE),
Offset = offset
});
}
var image = Image.FromStream(new MemoryStream(fileResponse.Content), true);
if (addressTemplate.Opacity < 1.0)
{
image = ChangeOpacity(image, addressTemplate.Opacity);
}
return(new ImageWithOffset
{
Image = image,
Offset = offset
});
}
public void GetPointById_WithUrl_ShouldGetItFromCsvFileAndFetchFile()
{
var id = "1";
var source = "csv";
var fileUrl = "fileUrl";
var dataContainer = new DataContainer {
Routes = new List <RouteData> {
new RouteData {
Name = "name"
}
}
};
var feature = GetValidFeature(id, source);
feature.Attributes.AddAttribute(FeatureAttributes.POI_SHARE_REFERENCE, fileUrl);
_elasticSearchGateway.GetCachedItemById(id, source).Returns(new FeatureCollection(new Collection <IFeature> {
feature
}));
_remoteFileFetcherGateway.GetFileContent(Arg.Any <string>()).Returns(new RemoteFileFetcherGatewayResponse {
FileName = fileUrl, Content = new byte[0]
});
_dataContainerConverterService.Convert(Arg.Any <byte[]>(), fileUrl, FlowFormats.GEOJSON).Returns(new FeatureCollection(new Collection <IFeature> {
feature
}).ToBytes());
_dataContainerConverterService.ToDataContainer(Arg.Any <byte[]>(), Arg.Any <string>()).Returns(dataContainer);
SetupFileStream();
var point = _adapter.GetPointOfInterestById(id, Languages.HEBREW).Result;
Assert.IsNotNull(point);
Assert.IsNotNull(point.DataContainer);
Assert.AreEqual(dataContainer.Routes.Count, point.DataContainer.Routes.Count);
_elasticSearchGateway.Received(1).CacheItem(Arg.Any <FeatureCollection>());
}
// GET api/files?url=http://jeeptrip.co.il/routes/pd6bccre.twl
public async Task <DataContainer> GetRemoteFile(string url)
{
var response = await _remoteFileFetcherGateway.GetFileContent(url);
var dataContainer = await ConvertToDataContainer(response.Content, response.FileName);
return(dataContainer);
}
private async Task <Image> GetTileImage(int x, int y, int zoom, string addressTemplate)
{
var file = addressTemplate.Replace("{z}", zoom.ToString())
.Replace("{zoom}", zoom.ToString())
.Replace("{x}", x.ToString())
.Replace("{y}", y.ToString());
var fileResponse = await _remoteFileFetcherGateway.GetFileContent(file);
return(fileResponse.Content.Any() ? Image.FromStream(new MemoryStream(fileResponse.Content), true) : new Bitmap(TILE_SIZE, TILE_SIZE));
}
/// <inheritdoc />
public override async Task <Feature> GetRawPointOfInterestById(string id)
{
var feature = GetRecords().Where(r => r.Id == id).Select(ConvertCsvRowToFeature).First();
if (feature.Attributes.Exists(FeatureAttributes.POI_SHARE_REFERENCE) &&
!string.IsNullOrWhiteSpace(feature.Attributes[FeatureAttributes.POI_SHARE_REFERENCE].ToString()))
{
var content = await _remoteFileFetcherGateway.GetFileContent(feature.Attributes[FeatureAttributes.POI_SHARE_REFERENCE].ToString());
var convertedBytes = await _dataContainerConverterService.Convert(content.Content, content.FileName, FlowFormats.GEOJSON);
feature.Geometry = convertedBytes.ToFeatureCollection().FirstOrDefault()?.Geometry ?? feature.Geometry;
}
return(feature);
}
public void GetRemoteFile_ConvertKmlToGeoJson_ShouldReturnOnePointAndOneLineString()
{
var url = "someurl";
byte[] bytes = Encoding.ASCII.GetBytes(GPX_DATA);
_remoteFileFetcherGateway.GetFileContent(url).Returns(new RemoteFileFetcherGatewayResponse {
Content = bytes, FileName = "file.KML"
});
_gpsBabelGateway.ConvertFileFromat(bytes, Arg.Is <string>(x => x.Contains("kml")), Arg.Is <string>(x => x.Contains("gpx"))).Returns(bytes);
_controller.SetupIdentity();
var dataContainer = _controller.GetRemoteFile(url).Result;
Assert.AreEqual(1, dataContainer.Routes.Count);
Assert.AreEqual(1, dataContainer.Routes.First().Markers.Count);
}
public async Task <IActionResult> UploadCsv(IFormFile file, [FromQuery] string idRegExPattern, [FromQuery] string sourceImageUrl, [FromQuery] string icon = "icon-bike", [FromQuery] string iconColor = "black", [FromQuery] string category = Categories.ROUTE_BIKE)
{
var reader = new StreamReader(file.OpenReadStream());
var csvReader = new CsvReader(reader);
csvReader.Configuration.HeaderValidated = null;
csvReader.Configuration.MissingFieldFound = null;
var pointsOfInterest = csvReader.GetRecords <CsvPointOfInterestRow>().ToList();
var stream = new MemoryStream();
using (TextWriter writer = new StreamWriter(stream, Encoding.UTF8, 1024, true))
{
var csvWriter = new CsvWriter(writer);
csvWriter.Configuration.HasHeaderRecord = true;
csvWriter.WriteHeader <CsvPointOfInterestRow>();
csvWriter.NextRecord();
foreach (var csvRow in pointsOfInterest)
{
if (!string.IsNullOrWhiteSpace(csvRow.FileUrl))
{
var response = await _remoteFileFetcher.GetFileContent(csvRow.FileUrl);
var geojsonBytes = await _dataContainerConverterService.Convert(response.Content,
response.FileName, FlowFormats.GEOJSON);
var geoJson = geojsonBytes.ToFeatureCollection();
var coordinate = geoJson.First().Geometry.Coordinate;
csvRow.Latitude = coordinate.Y;
csvRow.Longitude = coordinate.X;
}
csvRow.SourceImageUrl = sourceImageUrl;
csvRow.Website = csvRow.Website;
csvRow.Icon = icon;
csvRow.IconColor = iconColor;
csvRow.Category = category;
csvRow.Id = Regex.Match(csvRow.Website, idRegExPattern).Groups[1].Value;
csvWriter.WriteRecord(csvRow);
csvWriter.NextRecord();
}
csvWriter.Flush();
writer.Flush();
stream.Seek(0, SeekOrigin.Begin);
return(File(stream, "text/csv"));
}
}
/// <summary>
/// This method will fetch the relevant image
/// If the required zoom is too big it will fetch and image from a lower zoom and split the relevant part of it
/// This allow the other parts of the algorithm to be ignorat to the max zoom .
/// </summary>
/// <param name="topLeft">Top left corner</param>
/// <param name="offset">Offset from corner</param>
/// <param name="zoom">required zoom level</param>
/// <param name="addressTemplate">The address template to fetch the file from</param>
/// <returns></returns>
private async Task <ImageWithOffset> GetTileImage(Point topLeft, Point offset, int zoom, AddressAndOpacity addressTemplate)
{
var xY = new Point(topLeft.X + offset.X, topLeft.Y + offset.Y);
var translatedXy = xY;
var zoomDifference = Math.Pow(2, zoom - MAX_ZOOM);
if (zoomDifference > 1)
{
// zoom is above max native zoom
zoom = MAX_ZOOM;
translatedXy = new Point
{
X = (int)(xY.X / zoomDifference),
Y = (int)(xY.Y / zoomDifference),
};
}
var file = addressTemplate.Address.Replace("{z}", "{zoom}")
.Replace("{zoom}", zoom.ToString())
.Replace("{x}", translatedXy.X.ToString())
.Replace("{y}", translatedXy.Y.ToString());
var fileResponse = await _remoteFileFetcherGateway.GetFileContent(file);
if (!fileResponse.Content.Any())
{
return(new ImageWithOffset
{
Image = new Bitmap(TILE_SIZE, TILE_SIZE),
Offset = offset
});
}
var image = Image.FromStream(new MemoryStream(fileResponse.Content), true);
if (addressTemplate.Opacity < 1.0)
{
image = ChangeOpacity(image, addressTemplate.Opacity);
}
if (zoomDifference > 1)
{
image = MagnifyImagePart(image, zoomDifference, xY, translatedXy);
}
return(new ImageWithOffset
{
Image = image,
Offset = offset
});
}
private void CreateImagesJsonFiles(List <Feature> features)
{
_logger.LogInformation("Staring Image file creation: " + features.Count + " features");
var items = new ConcurrentBag <ImageItem>();
var size = 200;
Parallel.ForEach(features, new ParallelOptions {
MaxDegreeOfParallelism = 10
}, (feature) =>
{
var urls = feature.Attributes.GetNames()
.Where(n => n.StartsWith(FeatureAttributes.IMAGE_URL)).Select(n => feature.Attributes[n].ToString())
.Where(u => !string.IsNullOrWhiteSpace(u));
foreach (var url in urls)
{
var needResize = true;
var updatedUrl = url;
var pattern = @"(http.*\/\/upload\.wikimedia\.org\/wikipedia\/(commons|he|en)\/)(.*\/)(.*)";
if (Regex.Match(url, pattern).Success)
{
updatedUrl = Regex.Replace(url, pattern, $"$1thumb/$3$4/{size}px-$4");
updatedUrl = url.EndsWith(".svg") ? updatedUrl + ".png" : updatedUrl;
needResize = false;
}
try
{
var content = _remoteFileFetcherGateway.GetFileContent(updatedUrl).Result.Content;
if (content.Length == 0)
{
_logger.LogWarning("The following image does not exist: " + url + " feature: " + feature.GetId());
continue;
}
var image = Image.FromStream(new MemoryStream(content));
var format = image.RawFormat.ToString().ToLowerInvariant();
if (!needResize)
{
items.Add(new ImageItem {
ImageUrl = url, Data = $"data:image/{format};base64," + Convert.ToBase64String(content)
});
}
else
{
content = ResizeImage(image, size);
items.Add(new ImageItem {
ImageUrl = url, Data = $"data:image/jpeg;base64," + Convert.ToBase64String(content)
});
}
}
catch (Exception)
{
_logger.LogWarning("The following image is not an image: " + url + " feature: " + feature.GetId());
}
}
});
var list = items.ToList();
var index = 0;
while (list.Count > 0)
{
var imageItemsString = JsonConvert.SerializeObject(list.Take(1000).ToList(), new JsonSerializerSettings
{
ContractResolver = new CamelCasePropertyNamesContractResolver()
});
_fileSystemHelper.WriteAllText($"images{index.ToString("000")}.json", imageItemsString);
list = list.Skip(1000).ToList();
index++;
}
_logger.LogInformation("Finished Image file creation: " + items.Count());
}
public void TestGateway_JeepologAttachmentGpx()
{
var response = _gateway.GetFileContent("http://www.jeepolog.com/forums/attachment.php?attachmentid=103471").Result;
Assert.AreEqual("yehuda-2015.GPX", response.FileName);
}
