public async Task <CrawlResult> CrawlDimensionContentAsync(EntityFramework.Shared.Entities.Dimension dimension)
{
var webPath = $"{dimension.Zoom}/{dimension.X}/{dimension.Y}.png";
var directoryPath = $"{_directoryBaseAddress}\\{dimension.Zoom}\\{dimension.X}";
var directoryInfo = Directory.CreateDirectory(directoryPath);
var filePath = directoryInfo.FullName + $"\\{dimension.Y}.png";
try
{
var client = _httpClientFactory.GetHttpClient(_webBaseAddress);
var item = await client.GetAsync(webPath);
var contentStream = await item.Content.ReadAsStreamAsync(); // get the actual content stream
await using (var stream = new FileStream(filePath, FileMode.Create))
{
await contentStream.CopyToAsync(stream);
}
_logger.LogInformation($"Crawl Tile Succeeded : {_webBaseAddress}/{webPath}");
return(new CrawlResult()
{
Succeeded = true
});
}
catch (Exception e)
{
_logger.LogError(e, $"Crawl Tile Exception : {_webBaseAddress}/{webPath}");
return(new CrawlResult()
{
Succeeded = false, Exception = e
});
}
}
public FileStream GetDimensionContentAsync(EntityFramework.Shared.Entities.Dimension dimension)
{
var webPath = $"{dimension.Zoom}/{dimension.X}/{dimension.Y}.png";
var filePath = $"{_directoryBaseAddress}\\{dimension.Zoom}\\{dimension.X}\\{dimension.Y}.png";
if (!File.Exists(filePath))
{
return(null);
}
return(File.OpenRead(filePath));
}
public async Task <CrawlResult> CrawlDimensionContentAsync(EntityFramework.Shared.Entities.Dimension dimension)
{
var webPath = $"{dimension.Zoom}/{dimension.X}/{dimension.Y}.json";
try
{
var client = _httpClientFactory.GetHttpClient(_webBaseAddress);
var item = await client.GetAsync(webPath);
var contentString = await item.Content.ReadAsStringAsync(); // get the actual content stream
var cityRecords = JsonConvert.DeserializeObject <List <List <object> > >(contentString);
foreach (var record in cityRecords)
{
try
{
var label = new EntityFramework.Shared.Entities.Label()
{
DimensionId = dimension.Id,
RegisterDate = DateTime.Now,
O = record[0].ToString(),
Name = record[1].ToString(),
Type = record[2].ToString(),
X = Convert.ToDouble(record[3].ToString()),
Y = Convert.ToDouble(record[4].ToString()),
ExtraField1 = Convert.ToInt32(record[5].ToString()),
ExtraField2 = Convert.ToInt32(record[6].ToString()),
};
await _labelRepository.Add(label);
}
catch (Exception e)
{
_logger.LogError(e, $"Crawl Label Exception : {contentString}");
}
}
return(new CrawlResult()
{
Succeeded = true
});
}
catch (Exception e)
{
_logger.LogError(e, $"Crawl Label Exception : {_webBaseAddress}/{webPath}");
return(new CrawlResult()
{
Succeeded = false, Exception = e
});
}
}
public async Task <string> GetDimensionContentAsync(EntityFramework.Shared.Entities.Dimension dimension)
{
var returnValue = "[";
var labels = await _labelRepository.GetLabelsByDimensionId(dimension.Id);
if (labels.Count == 0)
{
await CrawlDimensionContentAsync(dimension);
labels = await _labelRepository.GetLabelsByDimensionId(dimension.Id);
}
foreach (var label in labels)
{
if (returnValue != "[")
{
returnValue += ",";
}
returnValue +=
$"[\"{label.O}\", \"{label.Name}\", \"{label.Type}\", {label.X}, {label.Y}, {label.ExtraField1}, {label.ExtraField2}]";
}
returnValue += "]";
return(returnValue);
}