public async Task <List <SearchResultViewDto> > GetSearchResultListAsync(string searchTerm, int maxResultCount)
{
var resultList = new List <SearchResultViewDto>();
try
{
Searching = true;
var resultCount = 0;
var helper = new WebScraperHelper();
while (resultCount < maxResultCount && Searching)
{
var pageDoc = await helper.GetPageAsync(string.Format(GoogleSearchURL, searchTerm, resultCount));
var resultColumn = pageDoc.GetElementbyId(@"main");
var nodesList = resultColumn.SelectNodes("*/div");
if (nodesList.Count == 0)
{
Searching = false;
}
foreach (var searchelement in nodesList)
{
if (searchelement.GetAttributeValue("class", "").Contains("ZINbbc") && !searchelement.GetAttributeValue("class", "").Contains("Pg70bf") && !searchelement.InnerText.StartsWith("Related searches"))
{
var resultItem = new SearchResultViewDto()
{
ResultId = Guid.NewGuid(),
ResultTitle = searchelement.ChildNodes[0].ChildNodes[0].ChildNodes[0].InnerText,
LinkURL = HttpUtility.HtmlDecode(@"https://google.co.uk" + searchelement.ChildNodes[0].ChildNodes[0].Attributes["href"].Value),
ResultText = searchelement.ChildNodes[2].InnerText
};
resultList.Add(resultItem);
resultCount++;
}
if (resultCount >= maxResultCount)
{
break;
}
}
}
}
catch (Exception ex)
{
}
return(resultList);
}
public async Task <List <SearchResultViewDto> > GetSearchResultListAsync(string searchTerm, int maxResultCount)
{
var resultList = new List <SearchResultViewDto>();
try
{
Searching = true;
var resultCount = 0;
var helper = new WebScraperHelper();
while (resultCount < maxResultCount && Searching)
{
var pageDoc = await helper.GetPageAsync(string.Format(BingSearchURL, searchTerm, resultCount));
var nodesList = pageDoc.DocumentNode.SelectNodes("//li[contains(@class, 'b_algo')]");
foreach (var searchelement in nodesList)
{
var resultItem = new SearchResultViewDto()
{
ResultId = Guid.NewGuid(),
ResultTitle = searchelement.ChildNodes[0].ChildNodes[0].InnerText,
LinkURL = searchelement.ChildNodes[0].ChildNodes[0].Attributes["href"].Value,
ResultText = searchelement.ChildNodes[1].ChildNodes[1].InnerText
};
resultList.Add(resultItem);
resultCount++;
if (resultCount >= maxResultCount)
{
break;
}
}
}
}
catch (Exception ex)
{
}
return(resultList);
}
