public static async Task <List <ICourseLink> > ExtractCourses() { await CoursesClient.LazyRefresh(); var coursesPageText = await CoursesClient.SessionClient.GetStringAsyncHttp(CoursesProfileAllCoursesUrl); CoursesClient.FindSessKey(coursesPageText); var doc = new HtmlDocument(); doc.LoadHtml(coursesPageText); var coursesLinksNodes = doc.DocumentNode.SelectNodes(XPathFilterProfileCoursesLinks); var coursesLinksTasks = coursesLinksNodes .Where(l => Regex.IsMatch(l.InnerText, CurrentSemesterCourseLongFormNameRegex)) .Select(async(l, _) => { var longName = ExtractName(l); var url = ExtractHref(l); string shortName; using (var courseHtml = await CoursesClient.SessionClient.GetStreamAsyncHttp(url)) { shortName = LazyHtmlParser.FindShortNameInHtml(courseHtml); shortName = CleanName(shortName, true); } return(new CourseLink($"{shortName,-8}{longName}", url)); }); var coursesLinks = await Task.WhenAll(coursesLinksTasks); SharedVars.Courses = coursesLinks.ToList <ICourseLink>(); return(SharedVars.Courses); }
public static async Task <List <ISection> > ExtractSectionsForCourse(ICourseLink courseLink) { await CoursesClient.LazyRefresh(); var coursePageText = await CoursesClient.SessionClient.GetStringAsyncHttp(courseLink.Url); CoursesClient.FindSessKey(coursePageText); var doc = new HtmlDocument(); doc.LoadHtml(coursePageText); var headersLinks = doc.DocumentNode.SelectNodes(XPathFilterLinksHeadersFolders); SharedVars.Sections = new List <ISection>(); var currentSection = new Section(); SharedVars.Sections.Add(currentSection); foreach (var headerLink in headersLinks) { var itemType = TryGetItemType(headerLink); string innerText = null; string href = null; if (itemType != ItemType.Header) { innerText = headerLink.Descendants().First(d => d.Name == "#text").InnerText.DecodeHtml(); href = headerLink.Attributes.First(l => l.Name == "href").Value; } switch (itemType) { case ItemType.Header: var headerName = headerLink.InnerText.DecodeHtml(); var headerTag = headerLink.OriginalName; var headerId = FindIdFromAncestors(headerLink); currentSection = new Section(new Header(headerName, headerTag, headerId), courseLink); SharedVars.Sections.Add(currentSection); break; case ItemType.File: currentSection.Links.Add(new FileLink(innerText, href, currentSection)); break; case ItemType.Folder: currentSection.Links.Add(new FolderLink(innerText, href, currentSection)); break; case ItemType.Url: currentSection.Links.Add(new ExternalLink(innerText, href, currentSection)); break; case ItemType.Page: currentSection.Links.Add(new PageLink(innerText, href, currentSection)); break; default: throw new ArgumentOutOfRangeException(); } } SharedVars.Sections = SharedVars.Sections.Where(s => s.Links.Any()).ToList(); return(SharedVars.Sections); }