protected override async Task <CrawlingResult> CrawleInnerAsync(CrawlingRequest request)
{
CrawlingResult result = null;
string url = string.Format("{0}?q={1}", BaseUrl, request.QueryWord);
string resultHtml = null;
using (var httpClient = new HttpClient())
{
var response = await httpClient.GetAsync(url);
resultHtml = await response.Content.ReadAsStringAsync();
var document = new HtmlDocument();
document.LoadHtml(resultHtml);
var htmlNodes = document.DocumentNode.CssSelect("#b_results li h2 a");
result = new CrawlingResult();
var hodeArray = htmlNodes.ToArray();
for (int i = 0; i < request.Take; i++)
{
var htmlNode = hodeArray[i];
var item = new SearchResultItem();
item.SearchEngine = ESearchEngine.Bing;
item.Title = htmlNode.InnerText;
result.SearchResults.Add(item);
}
}
return(result);
}
protected override async Task <CrawlingResult> CrawleInnerAsync(CrawlingRequest request)
{
string url = string.Format("{0}?q={1}", BaseUrl, request.QueryWord);
var httpRequest = createHttpRequest(url);
var httpResponse = (HttpWebResponse)httpRequest.GetResponse();
var streamResponse = httpResponse.GetResponseStream();
var streamRead = new StreamReader(streamResponse);
var htmlString = await streamRead.ReadToEndAsync();
var document = new HtmlDocument();
document.LoadHtml(htmlString);
var htmlNodes = document.DocumentNode.CssSelect(".g h3").ToArray();
var result = new CrawlingResult();
for (int i = 0; i < request.Take; i++)
{
var htmlNode = htmlNodes[i];
var item = new SearchResultItem();
item.SearchEngine = ESearchEngine.Google;
item.Title = htmlNode.InnerText;
result.SearchResults.Add(item);
}
return(result);
}
