public void CrawlStepParseLinkListTest() { // Arrange var stepper = new CrawlStepper(); stepper.StepSequence.Add(new CrawlStepGetLinkList( 1, CrawlSetpType.Single, new AngleSharp.Url("http://gate.ahram.org.eg/index.aspx"), "#ContentPlaceHolder1_divUrgent .blok_file > a" )); stepper.StepSequence.Add(new CrawlStepParseLinkList { Id = 2, CrawlItems = new List <CrawlItem> { new CrawlItem { Name = "Title", Selector = "Title" }, new CrawlItem { Name = "Image", Selector = "#ContentPlaceHolder1_divMainImage > img", Attr = "src" } } }); // Act var res = stepper.Crawl(); // Assert Assert.True(res.Count > 0, "This test needs an implementation"); }
public void CrawlStepGetRssLinksTest() { // Arrange var stepper = new CrawlStepper(); stepper.StepSequence.Add(new CrawlStepGetRssLinks() { Id = 1, CrawlStepType = CrawlSetpType.Single, Url = new AngleSharp.Url("https://www.almasryalyoum.com/rss/rssfeeds"), Selector = "//rss/channel/item/link/text()" }); stepper.StepSequence.Add(new CrawlStepParseLinkList { Id = 2, CrawlItems = new List <CrawlItem> { new CrawlItem { Name = "Title", Selector = "Title" }, new CrawlItem { Name = "Image", Selector = ".article .articleimg > img", Attr = "src" } } }); // Act var res = stepper.Crawl(); // Assert //Assert.True(String.IsNullOrEmpty(res[0]?.CrawlItems[0]?.Value) , "Title Crawling fail for El masry Elyom "); //Assert.True(String.IsNullOrEmpty(res[0]?.CrawlItems[1]?.Value), "Image Crawling fail for El masry Elyom "); Assert.True(res.Count > 0, "This test needs an implementation"); }
public void CrawlStepGetLinkListTest() { // Arrange var stepper = new CrawlStepper(); stepper.StepSequence.Add(new CrawlStepGetLinkList(1, CrawlSetpType.Single, new AngleSharp.Url("http://gate.ahram.org.eg/index.aspx"), "#ContentPlaceHolder1_divUrgent .blok_file > a" )); // Act var res = stepper.Crawl(); // Assert Assert.True(res.Count > 0, "This test needs an implementation"); }