public async Task DownloadImageTiles(OrthophotoSource orthophotoSource, List <ImageTile> imageTiles, IProgress <DownloadThreadProgress> threadProgress, string downloadDirectory)
{
// Reset cancellation token status
cancellationTokenSource = new CancellationTokenSource();
if (imageTiles.Count > 0)
{
log.InfoFormat("Beginning download of {0} image tiles from {1}", imageTiles.Count, orthophotoSource.ToString());
int downloadsPerThread = imageTiles.Count / this.downloadThreads;
int downloadsPerThreadMod = imageTiles.Count % this.downloadThreads;
var tasks = new List <Task>();
// Spawn the required number of threads
for (int i = 0; i < this.downloadThreads; i++)
{
var threadNumber = i;
tasks.Add(Task.Run(async() =>
{
var xmlSerializer = new XmlSerializer(typeof(ImageTile));
var maxWait = AeroSceneryManager.Instance.Settings.DownloadWaitMs.Value + AeroSceneryManager.Instance.Settings.DownloadWaitRandomMs.Value;
var minWait = AeroSceneryManager.Instance.Settings.DownloadWaitMs.Value - AeroSceneryManager.Instance.Settings.DownloadWaitRandomMs.Value;
Random random = new Random();
var downloadThreadProgress = new DownloadThreadProgress();
downloadThreadProgress.TotalFiles = downloadsPerThread;
downloadThreadProgress.DownloadThreadNumber = threadNumber;
if (threadNumber == this.downloadThreads - 1)
{
downloadThreadProgress.TotalFiles += downloadsPerThreadMod;
}
//Debug.WriteLine("Thread " + threadNumber.ToString());
var cookieContainer = new CookieContainer();
using (var handler = new HttpClientHandler()
{
CookieContainer = cookieContainer
})
using (HttpClient httpClient = new HttpClient(handler))
{
long lastDownloadTimestamp = DateTime.UtcNow.Ticks / TimeSpan.TicksPerMillisecond;
// Work through this threads share of downloads
for (int j = 0 + (threadNumber * downloadsPerThread); j < (threadNumber + 1) * downloadsPerThread; j++)
{
if (this.cancellationTokenSource.Token.IsCancellationRequested)
{
break;
}
int waitTime = random.Next(minWait, maxWait);
var waitTimeSpan = new TimeSpan(waitTime * TimeSpan.TicksPerMillisecond);
await Task.Delay(waitTimeSpan);
// We nee to re-eval this. If the user has cancelled, image tiles will be cleared
if (imageTiles != null && imageTiles.Count > 0)
{
this.DownloadFile(httpClient, cookieContainer, imageTiles[j], downloadDirectory);
}
// We nee to re-eval this. If the user has cancelled, image tiles will be cleared
if (imageTiles != null && imageTiles.Count > 0)
{
this.SaveImageTileAeroFile(xmlSerializer, imageTiles[j], downloadDirectory);
}
downloadThreadProgress.FilesDownloaded++;
threadProgress.Report(downloadThreadProgress);
//Debug.WriteLine("Thread " + threadNumber.ToString() + " Index " + j.ToString());
}
// If this is the 'last' thread, also work through the remainder
if (threadNumber == this.downloadThreads - 1)
{
for (int k = 0; k < downloadsPerThreadMod; k++)
{
if (this.cancellationTokenSource.Token.IsCancellationRequested)
{
break;
}
int waitTime = random.Next(minWait, maxWait);
var waitTimeSpan = new TimeSpan(waitTime * TimeSpan.TicksPerMillisecond);
await Task.Delay(waitTimeSpan);
var index = k + (downloadsPerThread * this.downloadThreads);
// We nee to re-eval this. If the user has cancelled, image tiles will be cleared
if (imageTiles != null && imageTiles.Count > 0)
{
this.DownloadFile(httpClient, cookieContainer, imageTiles[index], downloadDirectory);
}
// We nee to re-eval this. If the user has cancelled, image tiles will be cleared
if (imageTiles != null && imageTiles.Count > 0)
{
this.SaveImageTileAeroFile(xmlSerializer, imageTiles[index], downloadDirectory);
}
downloadThreadProgress.FilesDownloaded++;
threadProgress.Report(downloadThreadProgress);
//Debug.WriteLine("Thread " + threadNumber.ToString() + "Index " + k.ToString());
}
}
}
xmlSerializer = null;
}, this.cancellationTokenSource.Token));
}
await Task.WhenAll(tasks);
log.InfoFormat("Finished download of {0} image tiles from {1}", imageTiles.Count, orthophotoSource.ToString());
}
}
private void DownloadFile(HttpClient httpClient, CookieContainer cookieContainer, ImageTile imageTile, string path, OrthophotoSource orthophotoSource)
{
string fullFilePath = path + imageTile.FileName + "." + imageTile.ImageExtension;
cookieContainer = new CookieContainer();
cookieContainer.Add(new Uri(imageTile.URL), new Cookie("APISID", Guid.NewGuid().ToString()));
cookieContainer.Add(new Uri(imageTile.URL), new Cookie("NID", "119=" + Guid.NewGuid().ToString()));
cookieContainer.Add(new Uri(imageTile.URL), new Cookie("NID", "129=" + Guid.NewGuid().ToString()));
httpClient.DefaultRequestHeaders.Clear();
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(AeroSceneryManager.Instance.Settings.UserAgent);
httpClient.DefaultRequestHeaders.Referrer = new Uri("http://google.com/");
httpClient.DefaultRequestHeaders.Add("Accept-Language", "en-US,en;q=0.5");
httpClient.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate");
httpClient.DefaultRequestHeaders.Add("Accept", "application/json, text/javascript, */*; q=0.01");
try
{
var responseResult = httpClient.GetAsync(imageTile.URL);
bool saveFile = true;
// If we are Bing we might be served a valid image but there is really no tile available
if (orthophotoSource == OrthophotoSource.Bing)
{
// Check the Bing tile info header
if (responseResult.Result.Headers.Contains("X-VE-Tile-Info"))
{
var tileInfoHeaderValue = responseResult.Result.Headers.GetValues("X-VE-Tile-Info").FirstOrDefault();
// If there is really no file, the header value will be no-tile
// In this case we shouldn't save the image
if (tileInfoHeaderValue == "no-tile")
{
saveFile = false;
}
}
}
if (saveFile)
{
using (var memStream = responseResult.Result.Content.ReadAsStreamAsync().Result)
{
using (var fileStream = File.Create(fullFilePath))
{
var fileSize = memStream.Length;
if (fileSize == 1033)
{
var afd = "asdf";
}
memStream.CopyTo(fileStream);
}
}
}
}
catch (Exception ex)
{
log.Error("There was an error downloading " + imageTile.URL, ex);
}
finally
{
}
}
