示例#1
0
        static void Main(string[] args)
        {
            var scrapeJob     = new ScrapeJob();
            var user          = new User();
            var scrapeService = new ScrapeService();

            scrapeService.StartAScrapeJob("1111197");

            Console.ReadLine(); // catch debug before exiting
        }
示例#2
0
        // SUBMETHOD
        private static void AddProductsToTheScrapingList(HtmlDocument htmlDocument, ScrapeJob scrapeJob)
        {
            var ProductsHtml = htmlDocument.DocumentNode.Descendants("a")
                               .Where(node => node.GetAttributeValue("class", "")
                                      .Equals("product-title")).ToList();

            //here starts the scan for each productHref on a PLP
            foreach (var ProductListItem in ProductsHtml)
            {
                var ProductHref = ProductListItem
                                  .Attributes
                                  .Where(attribute => attribute.Name == "href")
                                  .FirstOrDefault()
                                  .Value
                                  .Replace("?suggestionType=browse", "")
                                  .Insert(0, "https://www.bol.com");
                Console.WriteLine(ProductHref);
                scrapeJob.PagesToScrape.Add(ProductHref);
            }
            Console.WriteLine("done looping over current page, total urls : " + scrapeJob.AmountOfPagesToScrape);
        }