private static bool NewDatasetAvailable(DownloadHistory downloadHistory, DatasetFile datasetFromFeed, DirectoryInfo downloadDirectory)
{
Console.WriteLine(datasetFromFeed.DatasetUrl);
if (string.IsNullOrEmpty(datasetFromFeed.LastUpdated))
{
return(true);
}
if (downloadHistory == null)
{
Console.WriteLine("Download history is null, NewDatasetAvailable");
return(true);
}
if (!LocalFileExists(downloadHistory, downloadDirectory, datasetFromFeed))
{
Console.WriteLine("LocalFile does not exist, NewDatasetAvailable");
return(true);
}
var originalDatasetLastUpdated = DateTime.Parse(downloadHistory.LastUpdated, System.Globalization.CultureInfo.InvariantCulture);
var datasetFromFeedLastUpdated = DateTime.Parse(datasetFromFeed.LastUpdated, System.Globalization.CultureInfo.InvariantCulture);
Console.WriteLine($"originalDatasetLastUpdated = {originalDatasetLastUpdated} , datasetFromFeedLastUpdated = {datasetFromFeedLastUpdated}");
var updatedDatasetAvailable = originalDatasetLastUpdated < datasetFromFeedLastUpdated;
if (updatedDatasetAvailable)
{
Console.WriteLine($"updatedDatasetAvailable = {updatedDatasetAvailable}");
}
return(updatedDatasetAvailable);
}
public List <DatasetFile> ParseDatasetFiles(string xml, Dataset dataset)
{
var datasetFiles = new List <DatasetFile>();
XmlDocument xmlDoc = new XmlDocument();
xmlDoc.LoadXml(xml);
string xpath = "//a:feed/a:entry";
XmlNamespaceManager nsmgr = new XmlNamespaceManager(xmlDoc.NameTable);
nsmgr.AddNamespace("a", "http://www.w3.org/2005/Atom");
nsmgr.AddNamespace("inspire_dls", "http://inspire.ec.europa.eu/schemas/inspire_dls/1.0");
nsmgr.AddNamespace("gn", "http://geonorge.no/Atom");
var nodes = xmlDoc.SelectNodes(xpath, nsmgr);
foreach (XmlNode childrenNode in nodes)
{
var datasetFile = new DatasetFile();
datasetFile.Title = childrenNode.SelectSingleNode("a:title", nsmgr).InnerXml;
datasetFile.Description = GetDescription(childrenNode, nsmgr);
datasetFile.Url = GetUrl(childrenNode, nsmgr);
datasetFile.LastUpdated = GetLastUpdated(childrenNode, nsmgr);
datasetFile.Projection = GetProjection(childrenNode.SelectNodes("a:category", nsmgr));
datasetFile.Format = GetFormat(childrenNode.SelectSingleNode("a:title", nsmgr), childrenNode.SelectNodes("a:category", nsmgr));
datasetFile.AreaCode = GetAreaCode(childrenNode.SelectNodes("a:category", nsmgr));
datasetFile.AreaLabel = GetAreaLabel(childrenNode.SelectNodes("a:category", nsmgr));
datasetFile.Organization = GetOrganization(childrenNode, nsmgr, dataset);
datasetFiles.Add(datasetFile);
}
return(datasetFiles);
}
private static bool LocalFileExists(DirectoryInfo downloadDirectory, DatasetFile dataset)
{
if (!dataset.HasLocalFileName())
{
return(false);
}
var filePath = new FileInfo(Path.Combine(downloadDirectory.FullName, dataset.LocalFileName()));
return(filePath.Exists);
}
private static DirectoryInfo GetDownloadDirectory(AppSettings appSettings, DatasetFile dataset)
{
var downloadDirectory = new DirectoryInfo(Path.Combine(appSettings.DownloadDirectory, dataset.DatasetId));
if (!downloadDirectory.Exists)
{
Console.WriteLine($"Creating directory: {downloadDirectory}");
downloadDirectory.Create();
}
return(downloadDirectory);
}
public static DatasetFile NewDatasetFile(string title = null)
{
var datasetFile = new DatasetFile();
datasetFile.Title = title ?? "GML-format, 3001, Halden";
datasetFile.Description = "Dataset description";
datasetFile.Url = "https://nedlasting.geonorge.no/geonorge/KystOgFiskeri/AkvakulturLokaliteter/GML/KystOgFiskeri_0101_Halden_25833_AkvakulturLokaliteter_GML.zip";
datasetFile.LastUpdated = "2018-10-31T01:43:35";
datasetFile.Organization = "Kartverket";
datasetFile.Projection = "EPSG:25833";
datasetFile.DatasetId = "Akvakultur - lokaliteter GML-format";
datasetFile.DatasetUrl = "http://nedlasting.geonorge.no/geonorge/ATOM-feeds/AkvakulturLokaliteter_AtomFeedGML.xml";
return(datasetFile);
}
private static bool NewDatasetAvailable(DownloadHistory downloadHistory, DatasetFile datasetFromFeed, DirectoryInfo downloadDirectory)
{
if (downloadHistory == null)
{
return(true);
}
if (!LocalFileExists(downloadHistory, downloadDirectory, datasetFromFeed))
{
return(true);
}
var originalDatasetLastUpdated = DateTime.Parse(downloadHistory.Downloaded);
var datasetFromFeedLastUpdated = DateTime.Parse(datasetFromFeed.LastUpdated);
var updatedDatasetAvailable = originalDatasetLastUpdated < datasetFromFeedLastUpdated;
return(updatedDatasetAvailable);
}
private static DirectoryInfo GetDownloadDirectory(ConfigFile configFile, DatasetFile dataset)
{
try
{
var downloadDirectory = new DirectoryInfo(Path.Combine(configFile.DownloadDirectory, dataset.DatasetId));
if (!downloadDirectory.Exists)
{
Console.WriteLine($"Creating directory: {downloadDirectory}");
downloadDirectory.Create();
}
return(downloadDirectory);
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
private static async Task StartDownloadAsync()
{
DatasetService datasetService;
if (userpath.Length > 1)
{
datasetService = new DatasetService(userpath);
}
else
{
datasetService = new DatasetService();
}
List <DatasetFile> datasetToDownload = datasetService.GetSelectedFiles();
List <DatasetFile> updatedDatasetToDownload = new List <DatasetFile>();
DownloadLog downloadLog = new DownloadLog();
downloadLog.TotalDatasetsToDownload = datasetToDownload.Count;
var appSettings = applicationService.GetAppSettings();
long totalSizeUpdatedFiles = 0;
var downloader = new FileDownloader();
foreach (var localDataset in datasetToDownload)
{
var fileLog = new DatasetFileLog(localDataset);
try
{
Console.WriteLine(localDataset.DatasetId + " - " + localDataset.Title);
DirectoryInfo downloadDirectory = GetDownloadDirectory(appSettings, localDataset);
DatasetFile datasetFromFeed = datasetService.GetDatasetFile(localDataset);
DownloadHistory downloadHistory = datasetService.GetFileDownloaHistory(datasetFromFeed.Url);
bool newDatasetAvailable = NewDatasetAvailable(downloadHistory, datasetFromFeed, downloadDirectory);
if (newDatasetAvailable)
{
Console.WriteLine("Updated version of dataset is available.");
}
if (newDatasetAvailable)
{
Console.WriteLine("Starting download process.");
downloader.ProgressChanged += (totalFileSize, totalBytesDownloaded, progressPercentage) =>
{
fileLog.HumanReadableSize = HumanReadableBytes(totalFileSize.Value);
totalSizeUpdatedFiles += totalFileSize.Value;
Console.CursorLeft = 0;
Console.Write($"{progressPercentage}% ({HumanReadableBytes(totalBytesDownloaded)}/{HumanReadableBytes(totalFileSize.Value)}) "); // add som extra whitespace to blank out previous updates
};
var downloadRequest = new DownloadRequest(localDataset.Url, downloadDirectory, localDataset.IsRestricted());
localDataset.FilePath = await downloader.StartDownload(downloadRequest, appSettings);
Console.WriteLine();
downloadLog.Updated.Add(fileLog);
updatedDatasetToDownload.Add(localDataset);
}
else
{
fileLog.Message = "Not necessary to download dataset.";
downloadLog.NotUpdated.Add(fileLog);
Console.WriteLine("Not necessary to download dataset.");
localDataset.FilePath = downloadHistory.FilePath;
updatedDatasetToDownload.Add(localDataset);
}
}
catch (Exception e)
{
updatedDatasetToDownload.Add(localDataset);
fileLog.Message = "Error while downloading dataset: " + e.Message;
downloadLog.Faild.Add(fileLog);
Console.WriteLine("Error while downloading dataset: " + e.Message);
}
Console.WriteLine("-------------");
}
downloadLog.TotalSizeOfDownloadedFiles = HumanReadableBytes(totalSizeUpdatedFiles);
datasetService.WriteToDownloadLogFile(downloadLog);
datasetService.WriteToDownloadFile(updatedDatasetToDownload);
datasetService.WriteToDownloadHistoryFile(updatedDatasetToDownload);
}
private static bool LocalFileExists(DownloadHistory downloadHistory, DirectoryInfo downloadDirectory, DatasetFile dataset)
{
if (downloadHistory.FilePath != null)
{
var filePath = new FileInfo(Path.Combine(downloadDirectory.FullName, downloadHistory.FilePath));
return(filePath.Exists);
}
else
{
return(LocalFileExists(downloadDirectory, dataset));
}
}
public DatasetFileLog(DatasetFile localDataset)
{
DatasetId = localDataset.DatasetId;
Name = localDataset.Title;
Projection = localDataset.Projection;
}
private static async Task StartDownloadAsync(ConfigFile config)
{
var appSettings = ApplicationService.GetAppSettings();
var datasetService = new DatasetService(config);
var updatedDatasetToDownload = new List <Download>();
var downloadLog = new DownloadLog();
var downloader = new FileDownloader();
var datasetToDownload = datasetService.GetSelectedFilesToDownload();
var downloadUsage = config.DownloadUsage;
downloadLog.TotalDatasetsToDownload = datasetToDownload.Count;
foreach (var localDataset in datasetToDownload)
{
var updatedDatasetFileToDownload = new List <DatasetFile>();
if (localDataset.Subscribe)
{
Log.Information("Subscribe to Dataset files");
var datasetFilesFromFeed = datasetService.GetDatasetFiles(localDataset);
var filterDatasetFromFeed = datasetFilesFromFeed.Where(p => localDataset.Projections.Where(s => s.Selected == false).All(p2 => p2.Epsg != p.Projection)).ToList();
if (localDataset.AutoDeleteFiles)
{
Log.Debug("Delete files");
localDataset.Files = RemoveFiles(filterDatasetFromFeed, localDataset.Files, config);
}
if (localDataset.AutoAddFiles)
{
Log.Debug("Add new files");
localDataset.Files = AddFiles(filterDatasetFromFeed, localDataset.Files);
}
}
foreach (var datasetFile in localDataset.Files)
{
var fileLog = new DatasetFileLog(datasetFile);
try
{
Console.WriteLine(datasetFile.DatasetId + " - " + datasetFile.Title);
DirectoryInfo downloadDirectory = GetDownloadDirectory(config, datasetFile);
DatasetFile datasetFromFeed = datasetService.GetDatasetFile(datasetFile);
DownloadHistory downloadHistory = datasetService.GetFileDownloaHistory(datasetFile.Url);
bool newDatasetAvailable = NewDatasetAvailable(downloadHistory, datasetFromFeed, downloadDirectory);
if (newDatasetAvailable)
{
Console.WriteLine("Updated version of dataset is available.");
Console.WriteLine("Starting download process.");
downloader.ProgressChanged += (totalFileSize, totalBytesDownloaded, progressPercentage) =>
{
Console.CursorLeft = 0;
Console.Write($"{progressPercentage}% ({HumanReadableBytes(totalBytesDownloaded)}/{HumanReadableBytes(totalFileSize.Value)}) "); // add som extra whitespace to blank out previous updates
};
var downloadRequest = new DownloadRequest(datasetFile.Url, downloadDirectory, datasetFile.IsRestricted());
datasetFile.FilePath = await downloader.StartDownload(downloadRequest, appSettings);
downloadLog.Updated.Add(fileLog);
Console.WriteLine();
downloadUsage?.Entries.Add(new DownloadUsageEntries(datasetFile));
updatedDatasetFileToDownload.Add(datasetFile);
}
else
{
fileLog.Message = "Not necessary to download dataset." + datasetFromFeed.LastUpdated;
downloadLog.NotUpdated.Add(fileLog);
Console.WriteLine("Not necessary to download dataset.");
datasetFile.FilePath = downloadHistory.FilePath;
updatedDatasetFileToDownload.Add(datasetFile);
}
datasetFile.DownloadSuccess = true;
}
catch (Exception e)
{
Log.Error(e, "Error while downloading file " + datasetFile.Title);
updatedDatasetFileToDownload.Add(datasetFile);
fileLog.Message = "Error while downloading dataset: " + e.Message;
downloadLog.Faild.Add(fileLog);
Console.WriteLine("Error while downloading dataset: " + e.Message);
datasetFile.DownloadSuccess = true;
}
Console.WriteLine("-------------");
}
updatedDatasetToDownload.Add(localDataset);
}
Log.Information("Send download usage");
datasetService.SendDownloadUsage(downloadUsage);
Log.Information("Write to config file");
datasetService.WriteToConfigFile(updatedDatasetToDownload);
Log.Information("Write to download history file");
datasetService.WriteToDownloadHistoryFile(updatedDatasetToDownload);
Log.Information("Write to download log file");
datasetService.WriteToDownloadLogFile(downloadLog);
}
