private static async Task StartDownloadAsync()
{
DatasetService datasetService;
if (userpath.Length > 1)
{
datasetService = new DatasetService(userpath);
}
else
{
datasetService = new DatasetService();
}
List <DatasetFile> datasetToDownload = datasetService.GetSelectedFiles();
List <DatasetFile> updatedDatasetToDownload = new List <DatasetFile>();
DownloadLog downloadLog = new DownloadLog();
downloadLog.TotalDatasetsToDownload = datasetToDownload.Count;
var appSettings = applicationService.GetAppSettings();
long totalSizeUpdatedFiles = 0;
var downloader = new FileDownloader();
foreach (var localDataset in datasetToDownload)
{
var fileLog = new DatasetFileLog(localDataset);
try
{
Console.WriteLine(localDataset.DatasetId + " - " + localDataset.Title);
DirectoryInfo downloadDirectory = GetDownloadDirectory(appSettings, localDataset);
DatasetFile datasetFromFeed = datasetService.GetDatasetFile(localDataset);
DownloadHistory downloadHistory = datasetService.GetFileDownloaHistory(datasetFromFeed.Url);
bool newDatasetAvailable = NewDatasetAvailable(downloadHistory, datasetFromFeed, downloadDirectory);
if (newDatasetAvailable)
{
Console.WriteLine("Updated version of dataset is available.");
}
if (newDatasetAvailable)
{
Console.WriteLine("Starting download process.");
downloader.ProgressChanged += (totalFileSize, totalBytesDownloaded, progressPercentage) =>
{
fileLog.HumanReadableSize = HumanReadableBytes(totalFileSize.Value);
totalSizeUpdatedFiles += totalFileSize.Value;
Console.CursorLeft = 0;
Console.Write($"{progressPercentage}% ({HumanReadableBytes(totalBytesDownloaded)}/{HumanReadableBytes(totalFileSize.Value)}) "); // add som extra whitespace to blank out previous updates
};
var downloadRequest = new DownloadRequest(localDataset.Url, downloadDirectory, localDataset.IsRestricted());
localDataset.FilePath = await downloader.StartDownload(downloadRequest, appSettings);
Console.WriteLine();
downloadLog.Updated.Add(fileLog);
updatedDatasetToDownload.Add(localDataset);
}
else
{
fileLog.Message = "Not necessary to download dataset.";
downloadLog.NotUpdated.Add(fileLog);
Console.WriteLine("Not necessary to download dataset.");
localDataset.FilePath = downloadHistory.FilePath;
updatedDatasetToDownload.Add(localDataset);
}
}
catch (Exception e)
{
updatedDatasetToDownload.Add(localDataset);
fileLog.Message = "Error while downloading dataset: " + e.Message;
downloadLog.Faild.Add(fileLog);
Console.WriteLine("Error while downloading dataset: " + e.Message);
}
Console.WriteLine("-------------");
}
downloadLog.TotalSizeOfDownloadedFiles = HumanReadableBytes(totalSizeUpdatedFiles);
datasetService.WriteToDownloadLogFile(downloadLog);
datasetService.WriteToDownloadFile(updatedDatasetToDownload);
datasetService.WriteToDownloadHistoryFile(updatedDatasetToDownload);
}
private static async Task StartDownloadAsync(ConfigFile config)
{
var appSettings = ApplicationService.GetAppSettings();
var datasetService = new DatasetService(config);
var updatedDatasetToDownload = new List <Download>();
var downloadLog = new DownloadLog();
var downloader = new FileDownloader();
var datasetToDownload = datasetService.GetSelectedFilesToDownload();
var downloadUsage = config.DownloadUsage;
downloadLog.TotalDatasetsToDownload = datasetToDownload.Count;
foreach (var localDataset in datasetToDownload)
{
var updatedDatasetFileToDownload = new List <DatasetFile>();
if (localDataset.Subscribe)
{
Log.Information("Subscribe to Dataset files");
var datasetFilesFromFeed = datasetService.GetDatasetFiles(localDataset);
var filterDatasetFromFeed = datasetFilesFromFeed.Where(p => localDataset.Projections.Where(s => s.Selected == false).All(p2 => p2.Epsg != p.Projection)).ToList();
if (localDataset.AutoDeleteFiles)
{
Log.Debug("Delete files");
localDataset.Files = RemoveFiles(filterDatasetFromFeed, localDataset.Files, config);
}
if (localDataset.AutoAddFiles)
{
Log.Debug("Add new files");
localDataset.Files = AddFiles(filterDatasetFromFeed, localDataset.Files);
}
}
foreach (var datasetFile in localDataset.Files)
{
var fileLog = new DatasetFileLog(datasetFile);
try
{
Console.WriteLine(datasetFile.DatasetId + " - " + datasetFile.Title);
DirectoryInfo downloadDirectory = GetDownloadDirectory(config, datasetFile);
DatasetFile datasetFromFeed = datasetService.GetDatasetFile(datasetFile);
DownloadHistory downloadHistory = datasetService.GetFileDownloaHistory(datasetFile.Url);
bool newDatasetAvailable = NewDatasetAvailable(downloadHistory, datasetFromFeed, downloadDirectory);
if (newDatasetAvailable)
{
Console.WriteLine("Updated version of dataset is available.");
Console.WriteLine("Starting download process.");
downloader.ProgressChanged += (totalFileSize, totalBytesDownloaded, progressPercentage) =>
{
Console.CursorLeft = 0;
Console.Write($"{progressPercentage}% ({HumanReadableBytes(totalBytesDownloaded)}/{HumanReadableBytes(totalFileSize.Value)}) "); // add som extra whitespace to blank out previous updates
};
var downloadRequest = new DownloadRequest(datasetFile.Url, downloadDirectory, datasetFile.IsRestricted());
datasetFile.FilePath = await downloader.StartDownload(downloadRequest, appSettings);
downloadLog.Updated.Add(fileLog);
Console.WriteLine();
downloadUsage?.Entries.Add(new DownloadUsageEntries(datasetFile));
updatedDatasetFileToDownload.Add(datasetFile);
}
else
{
fileLog.Message = "Not necessary to download dataset." + datasetFromFeed.LastUpdated;
downloadLog.NotUpdated.Add(fileLog);
Console.WriteLine("Not necessary to download dataset.");
datasetFile.FilePath = downloadHistory.FilePath;
updatedDatasetFileToDownload.Add(datasetFile);
}
datasetFile.DownloadSuccess = true;
}
catch (Exception e)
{
Log.Error(e, "Error while downloading file " + datasetFile.Title);
updatedDatasetFileToDownload.Add(datasetFile);
fileLog.Message = "Error while downloading dataset: " + e.Message;
downloadLog.Faild.Add(fileLog);
Console.WriteLine("Error while downloading dataset: " + e.Message);
datasetFile.DownloadSuccess = true;
}
Console.WriteLine("-------------");
}
updatedDatasetToDownload.Add(localDataset);
}
Log.Information("Send download usage");
datasetService.SendDownloadUsage(downloadUsage);
Log.Information("Write to config file");
datasetService.WriteToConfigFile(updatedDatasetToDownload);
Log.Information("Write to download history file");
datasetService.WriteToDownloadHistoryFile(updatedDatasetToDownload);
Log.Information("Write to download log file");
datasetService.WriteToDownloadLogFile(downloadLog);
}
