public async Task <ActionResult <CrawlStepper> > PostCrawlStepper([FromForm] CrawlStepper crawlStepper)
{
_context.CrawlSteppers.Add(crawlStepper);
await _context.SaveChangesAsync();
return(CreatedAtAction("GetCrawlStepper", new { id = crawlStepper.Id }, crawlStepper));
}
public async Task <IActionResult> PutCrawlStepper([FromForm] int id, [FromForm] CrawlStepper crawlStepper)
{
if (id != crawlStepper.Id)
{
return(BadRequest());
}
_context.Entry(crawlStepper).State = EntityState.Modified;
try
{
await _context.SaveChangesAsync();
}
catch (DbUpdateConcurrencyException)
{
if (!CrawlStepperExists(id))
{
return(NotFound());
}
else
{
throw;
}
}
return(NoContent());
}
public void CrawlStepParseLinkListTest()
{
// Arrange
var stepper = new CrawlStepper();
stepper.StepSequence.Add(new CrawlStepGetLinkList(
1, CrawlSetpType.Single, new AngleSharp.Url("http://gate.ahram.org.eg/index.aspx"), "#ContentPlaceHolder1_divUrgent .blok_file > a"
));
stepper.StepSequence.Add(new CrawlStepParseLinkList {
Id = 2,
CrawlItems = new List <CrawlItem>
{
new CrawlItem {
Name = "Title", Selector = "Title"
},
new CrawlItem {
Name = "Image", Selector = "#ContentPlaceHolder1_divMainImage > img", Attr = "src"
}
}
});
// Act
var res = stepper.Crawl();
// Assert
Assert.True(res.Count > 0, "This test needs an implementation");
}
public void CrawlStepGetRssLinksTest()
{
// Arrange
var stepper = new CrawlStepper();
stepper.StepSequence.Add(new CrawlStepGetRssLinks()
{
Id = 1,
CrawlStepType = CrawlSetpType.Single,
Url = new AngleSharp.Url("https://www.almasryalyoum.com/rss/rssfeeds"),
Selector = "//rss/channel/item/link/text()"
});
stepper.StepSequence.Add(new CrawlStepParseLinkList
{
Id = 2,
CrawlItems = new List <CrawlItem>
{
new CrawlItem {
Name = "Title", Selector = "Title"
},
new CrawlItem {
Name = "Image", Selector = ".article .articleimg > img", Attr = "src"
}
}
});
// Act
var res = stepper.Crawl();
// Assert
//Assert.True(String.IsNullOrEmpty(res[0]?.CrawlItems[0]?.Value) , "Title Crawling fail for El masry Elyom ");
//Assert.True(String.IsNullOrEmpty(res[0]?.CrawlItems[1]?.Value), "Image Crawling fail for El masry Elyom ");
Assert.True(res.Count > 0, "This test needs an implementation");
}
public void CrawlStepGetLinkListTest()
{
// Arrange
var stepper = new CrawlStepper();
stepper.StepSequence.Add(new CrawlStepGetLinkList(1, CrawlSetpType.Single,
new AngleSharp.Url("http://gate.ahram.org.eg/index.aspx"),
"#ContentPlaceHolder1_divUrgent .blok_file > a"
));
// Act
var res = stepper.Crawl();
// Assert
Assert.True(res.Count > 0, "This test needs an implementation");
}
