static void Main(string[] args)
{
var scrapeJob = new ScrapeJob();
var user = new User();
var scrapeService = new ScrapeService();
scrapeService.StartAScrapeJob("1111197");
Console.ReadLine(); // catch debug before exiting
}
// SUBMETHOD
private static void AddProductsToTheScrapingList(HtmlDocument htmlDocument, ScrapeJob scrapeJob)
{
var ProductsHtml = htmlDocument.DocumentNode.Descendants("a")
.Where(node => node.GetAttributeValue("class", "")
.Equals("product-title")).ToList();
//here starts the scan for each productHref on a PLP
foreach (var ProductListItem in ProductsHtml)
{
var ProductHref = ProductListItem
.Attributes
.Where(attribute => attribute.Name == "href")
.FirstOrDefault()
.Value
.Replace("?suggestionType=browse", "")
.Insert(0, "https://www.bol.com");
Console.WriteLine(ProductHref);
scrapeJob.PagesToScrape.Add(ProductHref);
}
Console.WriteLine("done looping over current page, total urls : " + scrapeJob.AmountOfPagesToScrape);
}