public async Task <string> StartDownload(DownloadRequest downloadRequest, AppSettings appSettings)
{
string fileName = null;
SetClientRequestHeaders(downloadRequest, appSettings);
using (var response = await Client.GetAsync(downloadRequest.DownloadUrl, HttpCompletionOption.ResponseHeadersRead))
{
if (response.StatusCode == System.Net.HttpStatusCode.Unauthorized)
{
Log.Error("Download failed: You need to authorize access before downloading file");
}
else if (!response.IsSuccessStatusCode)
{
var message = $"Download failed for url: {downloadRequest.DownloadUrl}, - response from server was: {response.StatusCode} - {response.ReasonPhrase}";
Log.Error(message);
Console.WriteLine(message);
}
else
{
using (var contentStream = await response.Content.ReadAsStreamAsync())
{
string destinationFilePath = downloadRequest.GetDestinationFilePath(response);
fileName = downloadRequest.GetDestinationFileName(response);
var totalBytes = response.Content.Headers.ContentLength;
await ProcessContentStream(totalBytes, contentStream, destinationFilePath);
}
}
}
return(fileName);
}
public async Task <string> StartDownload(DownloadRequest downloadRequest, AppSettings appSettings)
{
string fileName = null;
SetClientRequestHeaders(downloadRequest, appSettings);
using (var response = await Client.GetAsync(downloadRequest.DownloadUrl, HttpCompletionOption.ResponseHeadersRead))
{
if (!response.IsSuccessStatusCode)
{
Console.WriteLine("Download failed - response from server was: " + response.StatusCode + " - " + response.ReasonPhrase);
}
else
{
using (var contentStream = await response.Content.ReadAsStreamAsync())
{
string destinationFilePath = downloadRequest.GetDestinationFilePath(response);
fileName = downloadRequest.GetDestinationFileName(response);
var totalBytes = response.Content.Headers.ContentLength;
await ProcessContentStream(totalBytes, contentStream, destinationFilePath);
}
}
}
return(fileName);
}
private static void SetClientRequestHeaders(DownloadRequest downloadRequest, AppSettings appSettings)
{
if (downloadRequest.MustAuthenticate)
{
var byteArray = Encoding.ASCII.GetBytes(appSettings.Username + ":" + ProtectionService.GetUnprotectedPassword(appSettings.Password));
Client.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray));
}
}
private static void SetClientRequestHeaders(DownloadRequest downloadRequest, AppSettings appSettings)
{
if (!Client.DefaultRequestHeaders.Any())
{
Client.DefaultRequestHeaders.UserAgent.ParseAdd($"GeonorgeNedlastingsklient/{Assembly.GetExecutingAssembly().GetName().Version.ToString()}");
}
if (downloadRequest.MustAuthenticate)
{
var byteArray = Encoding.ASCII.GetBytes(appSettings.Username + ":" + ProtectionService.GetUnprotectedPassword(appSettings.Password));
Client.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray));
}
}
private static async Task StartDownloadAsync()
{
DatasetService datasetService;
if (userpath.Length > 1)
{
datasetService = new DatasetService(userpath);
}
else
{
datasetService = new DatasetService();
}
List <DatasetFile> datasetToDownload = datasetService.GetSelectedFiles();
List <DatasetFile> updatedDatasetToDownload = new List <DatasetFile>();
DownloadLog downloadLog = new DownloadLog();
downloadLog.TotalDatasetsToDownload = datasetToDownload.Count;
var appSettings = applicationService.GetAppSettings();
long totalSizeUpdatedFiles = 0;
var downloader = new FileDownloader();
foreach (var localDataset in datasetToDownload)
{
var fileLog = new DatasetFileLog(localDataset);
try
{
Console.WriteLine(localDataset.DatasetId + " - " + localDataset.Title);
DirectoryInfo downloadDirectory = GetDownloadDirectory(appSettings, localDataset);
DatasetFile datasetFromFeed = datasetService.GetDatasetFile(localDataset);
DownloadHistory downloadHistory = datasetService.GetFileDownloaHistory(datasetFromFeed.Url);
bool newDatasetAvailable = NewDatasetAvailable(downloadHistory, datasetFromFeed, downloadDirectory);
if (newDatasetAvailable)
{
Console.WriteLine("Updated version of dataset is available.");
}
if (newDatasetAvailable)
{
Console.WriteLine("Starting download process.");
downloader.ProgressChanged += (totalFileSize, totalBytesDownloaded, progressPercentage) =>
{
fileLog.HumanReadableSize = HumanReadableBytes(totalFileSize.Value);
totalSizeUpdatedFiles += totalFileSize.Value;
Console.CursorLeft = 0;
Console.Write($"{progressPercentage}% ({HumanReadableBytes(totalBytesDownloaded)}/{HumanReadableBytes(totalFileSize.Value)}) "); // add som extra whitespace to blank out previous updates
};
var downloadRequest = new DownloadRequest(localDataset.Url, downloadDirectory, localDataset.IsRestricted());
localDataset.FilePath = await downloader.StartDownload(downloadRequest, appSettings);
Console.WriteLine();
downloadLog.Updated.Add(fileLog);
updatedDatasetToDownload.Add(localDataset);
}
else
{
fileLog.Message = "Not necessary to download dataset.";
downloadLog.NotUpdated.Add(fileLog);
Console.WriteLine("Not necessary to download dataset.");
localDataset.FilePath = downloadHistory.FilePath;
updatedDatasetToDownload.Add(localDataset);
}
}
catch (Exception e)
{
updatedDatasetToDownload.Add(localDataset);
fileLog.Message = "Error while downloading dataset: " + e.Message;
downloadLog.Faild.Add(fileLog);
Console.WriteLine("Error while downloading dataset: " + e.Message);
}
Console.WriteLine("-------------");
}
downloadLog.TotalSizeOfDownloadedFiles = HumanReadableBytes(totalSizeUpdatedFiles);
datasetService.WriteToDownloadLogFile(downloadLog);
datasetService.WriteToDownloadFile(updatedDatasetToDownload);
datasetService.WriteToDownloadHistoryFile(updatedDatasetToDownload);
}
private static async Task StartDownloadAsync(ConfigFile config)
{
var appSettings = ApplicationService.GetAppSettings();
var datasetService = new DatasetService(config);
var updatedDatasetToDownload = new List <Download>();
var downloadLog = new DownloadLog();
var downloader = new FileDownloader();
var datasetToDownload = datasetService.GetSelectedFilesToDownload();
var downloadUsage = config.DownloadUsage;
downloadLog.TotalDatasetsToDownload = datasetToDownload.Count;
foreach (var localDataset in datasetToDownload)
{
var updatedDatasetFileToDownload = new List <DatasetFile>();
if (localDataset.Subscribe)
{
Log.Information("Subscribe to Dataset files");
var datasetFilesFromFeed = datasetService.GetDatasetFiles(localDataset);
var filterDatasetFromFeed = datasetFilesFromFeed.Where(p => localDataset.Projections.Where(s => s.Selected == false).All(p2 => p2.Epsg != p.Projection)).ToList();
if (localDataset.AutoDeleteFiles)
{
Log.Debug("Delete files");
localDataset.Files = RemoveFiles(filterDatasetFromFeed, localDataset.Files, config);
}
if (localDataset.AutoAddFiles)
{
Log.Debug("Add new files");
localDataset.Files = AddFiles(filterDatasetFromFeed, localDataset.Files);
}
}
foreach (var datasetFile in localDataset.Files)
{
var fileLog = new DatasetFileLog(datasetFile);
try
{
Console.WriteLine(datasetFile.DatasetId + " - " + datasetFile.Title);
DirectoryInfo downloadDirectory = GetDownloadDirectory(config, datasetFile);
DatasetFile datasetFromFeed = datasetService.GetDatasetFile(datasetFile);
DownloadHistory downloadHistory = datasetService.GetFileDownloaHistory(datasetFile.Url);
bool newDatasetAvailable = NewDatasetAvailable(downloadHistory, datasetFromFeed, downloadDirectory);
if (newDatasetAvailable)
{
Console.WriteLine("Updated version of dataset is available.");
Console.WriteLine("Starting download process.");
downloader.ProgressChanged += (totalFileSize, totalBytesDownloaded, progressPercentage) =>
{
Console.CursorLeft = 0;
Console.Write($"{progressPercentage}% ({HumanReadableBytes(totalBytesDownloaded)}/{HumanReadableBytes(totalFileSize.Value)}) "); // add som extra whitespace to blank out previous updates
};
var downloadRequest = new DownloadRequest(datasetFile.Url, downloadDirectory, datasetFile.IsRestricted());
datasetFile.FilePath = await downloader.StartDownload(downloadRequest, appSettings);
downloadLog.Updated.Add(fileLog);
Console.WriteLine();
downloadUsage?.Entries.Add(new DownloadUsageEntries(datasetFile));
updatedDatasetFileToDownload.Add(datasetFile);
}
else
{
fileLog.Message = "Not necessary to download dataset." + datasetFromFeed.LastUpdated;
downloadLog.NotUpdated.Add(fileLog);
Console.WriteLine("Not necessary to download dataset.");
datasetFile.FilePath = downloadHistory.FilePath;
updatedDatasetFileToDownload.Add(datasetFile);
}
datasetFile.DownloadSuccess = true;
}
catch (Exception e)
{
Log.Error(e, "Error while downloading file " + datasetFile.Title);
updatedDatasetFileToDownload.Add(datasetFile);
fileLog.Message = "Error while downloading dataset: " + e.Message;
downloadLog.Faild.Add(fileLog);
Console.WriteLine("Error while downloading dataset: " + e.Message);
datasetFile.DownloadSuccess = true;
}
Console.WriteLine("-------------");
}
updatedDatasetToDownload.Add(localDataset);
}
Log.Information("Send download usage");
datasetService.SendDownloadUsage(downloadUsage);
Log.Information("Write to config file");
datasetService.WriteToConfigFile(updatedDatasetToDownload);
Log.Information("Write to download history file");
datasetService.WriteToDownloadHistoryFile(updatedDatasetToDownload);
Log.Information("Write to download log file");
datasetService.WriteToDownloadLogFile(downloadLog);
}
