public async Task StartScraping(ParamsDTO queryParams)
{
try
{
HtmlWeb web = new HtmlWeb();
var htmlDoc = web.Load(queryParams.HomeUrl);
var products = htmlDoc.DocumentNode.SelectNodes(queryParams.ProductList);
level++;
var links = products.Descendants("a").Select(a => a.Attributes["href"].Value).ToList();
if (level == 1)
{
lastLink = links.Last();
sitename = UrlConverter.GetSiteHostWithProtocol(queryParams.HomeUrl);
SetFilename();
}
await ScrapProducts(links, queryParams);
}
catch (Exception ex)
{
await _hubContext.Clients.All.SendAsync("Send", $"Error due parsing website or product list");
throw (ex);
}
}