public async Task <string> StartDownload(DownloadRequest downloadRequest, AppSettings appSettings) { string fileName = null; SetClientRequestHeaders(downloadRequest, appSettings); using (var response = await Client.GetAsync(downloadRequest.DownloadUrl, HttpCompletionOption.ResponseHeadersRead)) { if (response.StatusCode == System.Net.HttpStatusCode.Unauthorized) { Log.Error("Download failed: You need to authorize access before downloading file"); } else if (!response.IsSuccessStatusCode) { var message = $"Download failed for url: {downloadRequest.DownloadUrl}, - response from server was: {response.StatusCode} - {response.ReasonPhrase}"; Log.Error(message); Console.WriteLine(message); } else { using (var contentStream = await response.Content.ReadAsStreamAsync()) { string destinationFilePath = downloadRequest.GetDestinationFilePath(response); fileName = downloadRequest.GetDestinationFileName(response); var totalBytes = response.Content.Headers.ContentLength; await ProcessContentStream(totalBytes, contentStream, destinationFilePath); } } } return(fileName); }
public async Task <string> StartDownload(DownloadRequest downloadRequest, AppSettings appSettings) { string fileName = null; SetClientRequestHeaders(downloadRequest, appSettings); using (var response = await Client.GetAsync(downloadRequest.DownloadUrl, HttpCompletionOption.ResponseHeadersRead)) { if (!response.IsSuccessStatusCode) { Console.WriteLine("Download failed - response from server was: " + response.StatusCode + " - " + response.ReasonPhrase); } else { using (var contentStream = await response.Content.ReadAsStreamAsync()) { string destinationFilePath = downloadRequest.GetDestinationFilePath(response); fileName = downloadRequest.GetDestinationFileName(response); var totalBytes = response.Content.Headers.ContentLength; await ProcessContentStream(totalBytes, contentStream, destinationFilePath); } } } return(fileName); }
private static void SetClientRequestHeaders(DownloadRequest downloadRequest, AppSettings appSettings) { if (downloadRequest.MustAuthenticate) { var byteArray = Encoding.ASCII.GetBytes(appSettings.Username + ":" + ProtectionService.GetUnprotectedPassword(appSettings.Password)); Client.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray)); } }
private static void SetClientRequestHeaders(DownloadRequest downloadRequest, AppSettings appSettings) { if (!Client.DefaultRequestHeaders.Any()) { Client.DefaultRequestHeaders.UserAgent.ParseAdd($"GeonorgeNedlastingsklient/{Assembly.GetExecutingAssembly().GetName().Version.ToString()}"); } if (downloadRequest.MustAuthenticate) { var byteArray = Encoding.ASCII.GetBytes(appSettings.Username + ":" + ProtectionService.GetUnprotectedPassword(appSettings.Password)); Client.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray)); } }
private static async Task StartDownloadAsync() { DatasetService datasetService; if (userpath.Length > 1) { datasetService = new DatasetService(userpath); } else { datasetService = new DatasetService(); } List <DatasetFile> datasetToDownload = datasetService.GetSelectedFiles(); List <DatasetFile> updatedDatasetToDownload = new List <DatasetFile>(); DownloadLog downloadLog = new DownloadLog(); downloadLog.TotalDatasetsToDownload = datasetToDownload.Count; var appSettings = applicationService.GetAppSettings(); long totalSizeUpdatedFiles = 0; var downloader = new FileDownloader(); foreach (var localDataset in datasetToDownload) { var fileLog = new DatasetFileLog(localDataset); try { Console.WriteLine(localDataset.DatasetId + " - " + localDataset.Title); DirectoryInfo downloadDirectory = GetDownloadDirectory(appSettings, localDataset); DatasetFile datasetFromFeed = datasetService.GetDatasetFile(localDataset); DownloadHistory downloadHistory = datasetService.GetFileDownloaHistory(datasetFromFeed.Url); bool newDatasetAvailable = NewDatasetAvailable(downloadHistory, datasetFromFeed, downloadDirectory); if (newDatasetAvailable) { Console.WriteLine("Updated version of dataset is available."); } if (newDatasetAvailable) { Console.WriteLine("Starting download process."); downloader.ProgressChanged += (totalFileSize, totalBytesDownloaded, progressPercentage) => { fileLog.HumanReadableSize = HumanReadableBytes(totalFileSize.Value); totalSizeUpdatedFiles += totalFileSize.Value; Console.CursorLeft = 0; Console.Write($"{progressPercentage}% ({HumanReadableBytes(totalBytesDownloaded)}/{HumanReadableBytes(totalFileSize.Value)}) "); // add som extra whitespace to blank out previous updates }; var downloadRequest = new DownloadRequest(localDataset.Url, downloadDirectory, localDataset.IsRestricted()); localDataset.FilePath = await downloader.StartDownload(downloadRequest, appSettings); Console.WriteLine(); downloadLog.Updated.Add(fileLog); updatedDatasetToDownload.Add(localDataset); } else { fileLog.Message = "Not necessary to download dataset."; downloadLog.NotUpdated.Add(fileLog); Console.WriteLine("Not necessary to download dataset."); localDataset.FilePath = downloadHistory.FilePath; updatedDatasetToDownload.Add(localDataset); } } catch (Exception e) { updatedDatasetToDownload.Add(localDataset); fileLog.Message = "Error while downloading dataset: " + e.Message; downloadLog.Faild.Add(fileLog); Console.WriteLine("Error while downloading dataset: " + e.Message); } Console.WriteLine("-------------"); } downloadLog.TotalSizeOfDownloadedFiles = HumanReadableBytes(totalSizeUpdatedFiles); datasetService.WriteToDownloadLogFile(downloadLog); datasetService.WriteToDownloadFile(updatedDatasetToDownload); datasetService.WriteToDownloadHistoryFile(updatedDatasetToDownload); }
private static async Task StartDownloadAsync(ConfigFile config) { var appSettings = ApplicationService.GetAppSettings(); var datasetService = new DatasetService(config); var updatedDatasetToDownload = new List <Download>(); var downloadLog = new DownloadLog(); var downloader = new FileDownloader(); var datasetToDownload = datasetService.GetSelectedFilesToDownload(); var downloadUsage = config.DownloadUsage; downloadLog.TotalDatasetsToDownload = datasetToDownload.Count; foreach (var localDataset in datasetToDownload) { var updatedDatasetFileToDownload = new List <DatasetFile>(); if (localDataset.Subscribe) { Log.Information("Subscribe to Dataset files"); var datasetFilesFromFeed = datasetService.GetDatasetFiles(localDataset); var filterDatasetFromFeed = datasetFilesFromFeed.Where(p => localDataset.Projections.Where(s => s.Selected == false).All(p2 => p2.Epsg != p.Projection)).ToList(); if (localDataset.AutoDeleteFiles) { Log.Debug("Delete files"); localDataset.Files = RemoveFiles(filterDatasetFromFeed, localDataset.Files, config); } if (localDataset.AutoAddFiles) { Log.Debug("Add new files"); localDataset.Files = AddFiles(filterDatasetFromFeed, localDataset.Files); } } foreach (var datasetFile in localDataset.Files) { var fileLog = new DatasetFileLog(datasetFile); try { Console.WriteLine(datasetFile.DatasetId + " - " + datasetFile.Title); DirectoryInfo downloadDirectory = GetDownloadDirectory(config, datasetFile); DatasetFile datasetFromFeed = datasetService.GetDatasetFile(datasetFile); DownloadHistory downloadHistory = datasetService.GetFileDownloaHistory(datasetFile.Url); bool newDatasetAvailable = NewDatasetAvailable(downloadHistory, datasetFromFeed, downloadDirectory); if (newDatasetAvailable) { Console.WriteLine("Updated version of dataset is available."); Console.WriteLine("Starting download process."); downloader.ProgressChanged += (totalFileSize, totalBytesDownloaded, progressPercentage) => { Console.CursorLeft = 0; Console.Write($"{progressPercentage}% ({HumanReadableBytes(totalBytesDownloaded)}/{HumanReadableBytes(totalFileSize.Value)}) "); // add som extra whitespace to blank out previous updates }; var downloadRequest = new DownloadRequest(datasetFile.Url, downloadDirectory, datasetFile.IsRestricted()); datasetFile.FilePath = await downloader.StartDownload(downloadRequest, appSettings); downloadLog.Updated.Add(fileLog); Console.WriteLine(); downloadUsage?.Entries.Add(new DownloadUsageEntries(datasetFile)); updatedDatasetFileToDownload.Add(datasetFile); } else { fileLog.Message = "Not necessary to download dataset." + datasetFromFeed.LastUpdated; downloadLog.NotUpdated.Add(fileLog); Console.WriteLine("Not necessary to download dataset."); datasetFile.FilePath = downloadHistory.FilePath; updatedDatasetFileToDownload.Add(datasetFile); } datasetFile.DownloadSuccess = true; } catch (Exception e) { Log.Error(e, "Error while downloading file " + datasetFile.Title); updatedDatasetFileToDownload.Add(datasetFile); fileLog.Message = "Error while downloading dataset: " + e.Message; downloadLog.Faild.Add(fileLog); Console.WriteLine("Error while downloading dataset: " + e.Message); datasetFile.DownloadSuccess = true; } Console.WriteLine("-------------"); } updatedDatasetToDownload.Add(localDataset); } Log.Information("Send download usage"); datasetService.SendDownloadUsage(downloadUsage); Log.Information("Write to config file"); datasetService.WriteToConfigFile(updatedDatasetToDownload); Log.Information("Write to download history file"); datasetService.WriteToDownloadHistoryFile(updatedDatasetToDownload); Log.Information("Write to download log file"); datasetService.WriteToDownloadLogFile(downloadLog); }