private async Task DownloadBatch(IEnumerable <Tile> batch, DownloadTilesProgress progress, HttpClient httpClient)
{
var tasks = batch.Select(x => x.Download(downloadFolder: this.options.Path, imageFormat: this.options.ImageFormat, httpClient: httpClient, skipIfExist: this.options.SkipExisting));
var result = await Task.WhenAll(tasks);
progress.SetTilesDownloaded(result.Where(x => x.Item2).Select(x => x.Item1).ToList());
foreach (var tileResult in result)
{
if (tileResult.Item2)
{
errorTiles.Remove(tileResult.Item1);
}
else
{
if (!errorTiles.Contains(tileResult.Item1))
{
errorTiles.Add(tileResult.Item1);
}
}
}
if (this.progressReporter != null)
{
this.progressReporter.Report(progress.Report);
}
}
private async Task RetryDownloadOfFailedTiles(HttpClient client, DownloadTilesProgress progress)
{
int retryCount = this.options.RetryCount;
while (errorTiles.Count() > 0 && retryCount > 0)
{
retryCount--;
foreach (var batch in errorTiles.Chunkify(this.options.ParallellTasks))
{
await DownloadBatch(batch, progress, client);
}
}
}
/**
* Start download
*/
public async Task Start()
{
LogManager.GetCurrentClassLogger().Info($"Starting download tiles from {this.options.TileSourceUrls}, options: {Newtonsoft.Json.JsonConvert.SerializeObject(this.options)}");
var tilesInPolygon = GetTilesToDownloadFromPolygon();
var tilesToDownload = GetTilesToDownloadForAllSources(tilesInPolygon);
var client = new HttpClient();
var progress = new DownloadTilesProgress(tilesToDownload);
progress.Start();
foreach (var batch in tilesToDownload.Chunkify(this.options.ParallellTasks))
{
await DownloadBatch(batch, progress, client);
}
await RetryDownloadOfFailedTiles(client, progress);
if (errorTiles.Count > 0)
{
LogManager.GetCurrentClassLogger().Error($"Tiles left to download: ${string.Join(", ", errorTiles)}");
throw new ApplicationException("Could not download all tiles!");
}
if (this.options.WriteMetadata)
{
var folders = tilesToDownload.GroupBy(x => x.GroupName).Select((g) => new { key = g.Key, metadata = Newtonsoft.Json.JsonConvert.SerializeObject(g.Select(t => $"{t.TileName}/{t.Z}/tile_{t.X}_{t.Y}")) });
foreach (var folder in folders)
{
var path = this.options.Path + "//" + folder.key + "//" + "tiles.json";
File.WriteAllText(path, folder.metadata);
}
}
if (this.options.WriteReport)
{
var report = $"<html><body><table><tr><td>Last run</td><td>{DateTime.Now}</td></tr><tr><td>Tiles downloaded</td><td>{progress.Report.Complete}</td></tr><tr><td>Downloaded to path</td><td>{this.options.Path}</td></tr><tr><td>Time used</td><td>{progress.Elapsed}</td></tr></table></body></html>";
File.WriteAllText("report.html", report);
}
}
