private void extractAllPages() { List <Page> pageUrls = new List <Page>(); var divContainers = document.DocumentNode.Descendants("div").Where(div => div.Id == "genreMoreList").Take(1).ToList(); if (divContainers.Count > 0) { divContainers[0].Descendants("ul").Where(ul => ul.Attributes.Contains("class") && ul.Attributes["class"].Value.Contains("img-link")) .Take(1).ToList()[0].Descendants("li").ToList().ForEach(li => { li.Descendants("a").ToList().ForEach(a => { Page page = new Page(); if (a.Attributes.Contains("href")) { page.Url = a.Attributes["href"].Value; page.Title = a.InnerText; pageUrls.Add(page); } }); }); } ; if (pageUrls.Count > 0) { for (int i = 0; i < pageUrls.Count; i++) { PagalWorldPage page = new PagalWorldPage(pageUrls[i].Url, pageUrls[i].Title, _hapScraper, this); //extract contents of child pages page.ExtractContents(); this.ChildPages.Add(page); } } else { this.HasChildren = true; } }
public void Run() { PagalWorldPage pagalWorldPage = new PagalWorldPage(MasterUrl, "ExtractedMidis", this); pagalWorldPage.ExtractContents(); }