public async Task <List <SearchResultViewDto> > GetSearchResultListAsync(string searchTerm, int maxResultCount) { var resultList = new List <SearchResultViewDto>(); try { Searching = true; var resultCount = 0; var helper = new WebScraperHelper(); while (resultCount < maxResultCount && Searching) { var pageDoc = await helper.GetPageAsync(string.Format(GoogleSearchURL, searchTerm, resultCount)); var resultColumn = pageDoc.GetElementbyId(@"main"); var nodesList = resultColumn.SelectNodes("*/div"); if (nodesList.Count == 0) { Searching = false; } foreach (var searchelement in nodesList) { if (searchelement.GetAttributeValue("class", "").Contains("ZINbbc") && !searchelement.GetAttributeValue("class", "").Contains("Pg70bf") && !searchelement.InnerText.StartsWith("Related searches")) { var resultItem = new SearchResultViewDto() { ResultId = Guid.NewGuid(), ResultTitle = searchelement.ChildNodes[0].ChildNodes[0].ChildNodes[0].InnerText, LinkURL = HttpUtility.HtmlDecode(@"https://google.co.uk" + searchelement.ChildNodes[0].ChildNodes[0].Attributes["href"].Value), ResultText = searchelement.ChildNodes[2].InnerText }; resultList.Add(resultItem); resultCount++; } if (resultCount >= maxResultCount) { break; } } } } catch (Exception ex) { } return(resultList); }
public async Task <List <SearchResultViewDto> > GetSearchResultListAsync(string searchTerm, int maxResultCount) { var resultList = new List <SearchResultViewDto>(); try { Searching = true; var resultCount = 0; var helper = new WebScraperHelper(); while (resultCount < maxResultCount && Searching) { var pageDoc = await helper.GetPageAsync(string.Format(BingSearchURL, searchTerm, resultCount)); var nodesList = pageDoc.DocumentNode.SelectNodes("//li[contains(@class, 'b_algo')]"); foreach (var searchelement in nodesList) { var resultItem = new SearchResultViewDto() { ResultId = Guid.NewGuid(), ResultTitle = searchelement.ChildNodes[0].ChildNodes[0].InnerText, LinkURL = searchelement.ChildNodes[0].ChildNodes[0].Attributes["href"].Value, ResultText = searchelement.ChildNodes[1].ChildNodes[1].InnerText }; resultList.Add(resultItem); resultCount++; if (resultCount >= maxResultCount) { break; } } } } catch (Exception ex) { } return(resultList); }