コード例 #1
0
        protected override async Task <CrawlingResult> CrawleInnerAsync(CrawlingRequest request)
        {
            CrawlingResult result = null;
            string         url    = string.Format("{0}?q={1}", BaseUrl, request.QueryWord);

            string resultHtml = null;

            using (var httpClient = new HttpClient())
            {
                var response = await httpClient.GetAsync(url);

                resultHtml = await response.Content.ReadAsStringAsync();

                var document = new HtmlDocument();
                document.LoadHtml(resultHtml);

                var htmlNodes = document.DocumentNode.CssSelect("#b_results li h2 a");

                result = new CrawlingResult();

                var hodeArray = htmlNodes.ToArray();
                for (int i = 0; i < request.Take; i++)
                {
                    var htmlNode = hodeArray[i];
                    var item     = new SearchResultItem();
                    item.SearchEngine = ESearchEngine.Bing;
                    item.Title        = htmlNode.InnerText;
                    result.SearchResults.Add(item);
                }
            }
            return(result);
        }
コード例 #2
0
        protected override async Task <CrawlingResult> CrawleInnerAsync(CrawlingRequest request)
        {
            string url = string.Format("{0}?q={1}", BaseUrl, request.QueryWord);

            var httpRequest = createHttpRequest(url);

            var httpResponse   = (HttpWebResponse)httpRequest.GetResponse();
            var streamResponse = httpResponse.GetResponseStream();
            var streamRead     = new StreamReader(streamResponse);
            var htmlString     = await streamRead.ReadToEndAsync();

            var document = new HtmlDocument();

            document.LoadHtml(htmlString);

            var htmlNodes = document.DocumentNode.CssSelect(".g h3").ToArray();

            var result = new CrawlingResult();

            for (int i = 0; i < request.Take; i++)
            {
                var htmlNode = htmlNodes[i];
                var item     = new SearchResultItem();
                item.SearchEngine = ESearchEngine.Google;
                item.Title        = htmlNode.InnerText;
                result.SearchResults.Add(item);
            }

            return(result);
        }