Exemple #1
0
        public async Task Perform()
        {
            int len = _links.Count;

            await Notify("Finished:", true);

            await Task.Run(() => {
                Parallel.For(0, 5, i => {
                    while (_links.Exists(x => !x.Checked))
                    {
                        Link link;
                        lock (_locker)
                        {
                            link = _links.FirstOrDefault(x => !x.Checked);
                            if (link != null)
                            {
                                link.Checked = true;
                            }
                        }
                        string content = HttpClientX.Find(link.Url).Result;
                        string result  = string.Empty;
                        try
                        {
                            HtmlAgilityPack.HtmlDocument html = new HtmlAgilityPack.HtmlDocument();
                            html.LoadHtml(content);
                            var hs = html.DocumentNode.SelectNodes(".//*");
                            foreach (var h in hs)
                            {
                                string tag = h.Name;
                                if (tag.Equals("h1") || tag.Equals("h2") || tag.Equals("h3") || tag.Equals("h4") || tag.Equals("h5") || tag.Equals("h6"))
                                {
                                    result += "\r\n" + h.OuterHtml + "\r\n";
                                }
                            }
                        }
                        catch
                        {
                        }

                        Notify(link.Number.ToString() + " ; ", false).Wait();

                        Application.Current.Dispatcher.Invoke(() => { link.Content = link.Url + "\r\n" + result; });
                    }
                });
            });
        }
Exemple #2
0
        private async Task <string> GetPages(string key)
        {
            await Notify("getting google pages", true);

            string searchurl = "http://google.com/search?q=" + key;

            string resp = await HttpClientX.Find(searchurl);

            var html = new HtmlAgilityPack.HtmlDocument();

            html.LoadHtml(resp);

            var table = html.GetElementbyId("foot");

            foreach (var td in table.QuerySelectorAll("a"))
            {
                string href = td.GetAttributeValue("href", "");

                _pages.Add("http://google.com" + href);
            }
            return(resp);
        }
Exemple #3
0
        public async Task <List <string> > GetTop(string key)
        {
            string resp = await GetPages(key);


            List <string> result = new List <string>();

            string searchurl = "http://google.com/search?q=" + key;

            int page = 0;

            while (true)
            {
                await Notify("getting website links", true);

                HtmlAgilityPack.HtmlDocument html;
                try
                {
                    if (page != 0)
                    {
                        resp = await HttpClientX.Find(_pages[page]);
                    }

                    html = new HtmlAgilityPack.HtmlDocument();
                    html.LoadHtml(resp);
                    var hs = html.DocumentNode.QuerySelectorAll("#search div[class=\"g\"] div[class=\"r\"]  a");

                    foreach (var h in hs)
                    {
                        if (h.Name == "a" && h.QuerySelectorAll("h3").Count != 0)
                        {
                            string href = h.GetAttributeValue("href", "");
                            if (href != "")
                            {
                                result.Add(href);
                            }
                        }
                    }


                    if (_pages.Count > page && result.Count < Depth)
                    {
                        page++;
                        continue;
                    }
                    else
                    {
                        break;
                    }
                }
                catch
                {
                }
                finally
                {
                }
            }
            await Notify("list created", true);

            return(result.Where(x => x.Contains("http:") || x.Contains("https:")).ToList());
        }