/// <summary>
/// Gets files to download from RSS feed
/// </summary>
/// <param name="xmlDoc">RSS feed</param>
/// <returns>List of files to download</returns>
public List <IliasFile> GetFilesFromXml(XmlDocument xmlDoc)
{
var rssFiles = new List <IliasFile>();
XmlNodeList items = xmlDoc.SelectNodes("//item");
string subfolderPattern = @"\[(.*?)\]";
string fileNamePattern = @"]\s(.*): Die Datei";
string fileIdPattern = @"file_(\d*)";
IliasFile[] filesArr = Files.ToArray();
foreach (XmlNode item in items)
{
if (item.SelectSingleNode(".//link") != null)
{
string link = item.SelectSingleNode(".//link").InnerText;
if (link.Contains("target=file"))
{
string title = item.SelectSingleNode(".//title").InnerText;
var subfoldersMatch = Regex.Match(title, subfolderPattern);
string[] subfolders = subfoldersMatch.Groups[1].Value.Split(" > ");
subfolders = subfolders.Select(sub => Util.ReplaceInvalidChars(sub)).ToArray();
string fileName = Util.ReplaceInvalidChars(Regex.Match(title, fileNamePattern).Groups[1].Value);
int fileId = int.Parse(Regex.Match(link, fileIdPattern).Groups[1].Value);
var fileUrl = new Uri($"https://ilias.uni-konstanz.de/ilias/goto_ilias_uni_file_{fileId}_download.html");
DateTime date = DateTime.Parse(item.SelectSingleNode(".//pubDate").InnerText);
IliasFile fileToDownload = new IliasFile(subfolders, fileName, fileId, fileUrl, date);
// Skip file if it exists on disk and either is ignored, extension is ignored, or file is not newer
if (Array.Exists(ExistingFiles, f => f == fileName) &&
(
Array.Exists(Builder.Config.IgnoreFiles, f => f == fileName) ||
Array.Exists(Builder.Config.IgnoreExtensions, e => e == fileToDownload.Extension) ||
Array.Exists(filesArr, f => f.Equals(fileToDownload))
))
{
continue;
}
rssFiles.Add(fileToDownload);
}
}
}
return(rssFiles);
}
/// <summary>
/// Downloads file from <c>fileUrl</c> to <c>path</c>
/// </summary>
/// <param name="path">Path to download directory</param>
/// <param name="file">File to download</param>
/// <param name="retrying">Retry download on failure</param>
/// <returns>The task object representing integer 0 on success and -1 on failure</returns>
public async Task <int> DownloadFile(string path, IliasFile file, bool retrying = true)
{
try
{
HttpResponseMessage response = await Client.GetAsync(file.Url);
if (response.IsSuccessStatusCode)
{
DateTime lastModified = new DateTime();
if (response.Content.Headers.LastModified != null)
{
lastModified = DateTime.Parse(response.Content.Headers.LastModified.ToString());
}
file.LastModified = lastModified;
bool updated = false;
if (Files.Count > 0)
{
IliasFile currentFile = Files.Find(f => f.Matches(file));
var idx = Files.FindIndex(f => f.Matches(file));
bool isOnDisk = Array.Exists(ExistingFiles, f => f == file.Name);
if (currentFile != null)
{
if (isOnDisk)
{
if (currentFile.LastModified.CompareTo(file.LastModified) >= 0 && currentFile.Date.CompareTo(file.Date) >= 0)
{
return(0);
}
}
updated = true;
Files[idx] = file;
}
}
Task <byte[]> fileArray = response.Content.ReadAsByteArrayAsync();
Directory.CreateDirectory(path);
try
{
File.WriteAllBytes(Path.Combine(path, file.Name), await fileArray);
if (!updated)
{
Files.Add(file);
}
Console.WriteLine($"{ILIAS_TAG} Downloaded { file.Name }");
return(0);
}
catch (IOException)
{
if (!retrying)
{
Console.WriteLine($"{ILIAS_TAG} An error occurred while writing { file.Name }. Please check manually if the file is correct.");
return(-1);
}
Console.WriteLine($"{ILIAS_TAG} An error occurred while writing { file.Name }. Retrying.");
return(await DownloadFile(path, file, false));
}
}
return(-1);
}
catch (TaskCanceledException e)
{
if (!retrying)
{
Console.WriteLine($"{ILIAS_TAG} Downloading { file.Name } failed.");
Console.WriteLine(e.InnerException.Message);
return(-1);
}
Console.WriteLine($"{ILIAS_TAG} An error occurred while downloading { file.Name }. Retrying.");
Console.WriteLine(e);
return(await DownloadFile(path, file, false));
}
}
/// <summary>
/// Gets URL to single exercises from <c>exercisePageUrl</c>
/// </summary>
/// <param name="exercisePageUrl">URL to page where the exercises are listed</param>
/// <returns>The task object representing an array of lists where each element is an IliasFile of a single exercise file.
/// The index 0 of the array contains exercise files and index 1 contains feedback files. Returns <c>null</c> on failure</returns>
public async Task <List <IliasFile>[]> GetExerciseFileUrls(Uri exercisePageUrl)
{
try
{
var exerciseFiles = new List <IliasFile> [2];
exerciseFiles[0] = new List <IliasFile>();
exerciseFiles[1] = new List <IliasFile>();
HttpResponseMessage response = await Client.GetAsync(exercisePageUrl);
HtmlDocument pageDocument = await Util.LoadHtmlDocument(response);
HtmlNodeCollection rows = pageDocument.DocumentNode.SelectNodes("//div[@class='ilInfoScreenSec form-horizontal']");
HtmlNodeCollection navigation = pageDocument.DocumentNode.SelectSingleNode("//ol[@class='breadcrumb hidden-print']").SelectNodes(".//li");
var navigationSplit = new List <string>();
/*
* TODO: Determine the exercise subfolder in a better way
* The trigger word determines when to start adding the categories/subfolders from Ilias to the list.
* This is used to determine which subfolder the exercise files belong to.
* An example of navigation is:
* Magazin, Mathematisch-Naturwissenschaftliche Sektion, Informatik und Informationswissenschaft, Lehrveranstaltungen WS 20/21, Data Visualization: Advanced Topics, Assignments, Assignment 01
*/
string triggerWord = "Lehrveranstaltungen";
var foundTriggerWord = false;
foreach (HtmlNode li in navigation)
{
string text = li.InnerText;
if (foundTriggerWord)
{
navigationSplit.Add(text);
}
else if (text.StartsWith(triggerWord))
{
foundTriggerWord = true;
}
}
var tempSubfolders = navigationSplit.Select(sub => Util.ReplaceInvalidChars(sub)).ToArray();
foreach (HtmlNode row in rows)
{
HtmlNodeCollection links = row.SelectNodes(".//a");
if (links != null)
{
foreach (HtmlNode link in links)
{
HtmlAttribute hrefAttribute = link.Attributes["href"];
if (hrefAttribute != null)
{
string href = WebUtility.HtmlDecode(hrefAttribute.Value);
if (href.Contains("file="))
{
var url = new Uri(ILIAS_BASE_URL + href);
var query = HttpUtility.ParseQueryString(url.Query);
string fileName = query["file"];
int fileId = int.Parse(query["ref_id"]);
var subfolderList = tempSubfolders.ToList();
if (href.Contains("cmd=downloadFile"))
{
var file = new IliasFile(tempSubfolders, fileName, fileId, url);
exerciseFiles[0].Add(file);
}
else if (href.Contains("cmd=downloadFeedbackFile"))
{
var file = new IliasFile(tempSubfolders, "feedback_" + fileName, fileId, url);
exerciseFiles[1].Add(file);
}
}
}
}
}
}
return(exerciseFiles);
}
catch (TaskCanceledException e)
{
Console.WriteLine($"{ILIAS_TAG} Fetching exercise file URLs timed out.");
Console.WriteLine(e.InnerException.Message);
return(null);
}
}
