示例#1
0
        private List <CaoQunMainItem> GetCaoQunMainItems()
        {
            List <CaoQunMainItem> res = new List <CaoQunMainItem>();
            HttpCore hc = new HttpCore();

            hc.SetUrl(CaoQunUrl);
            string             mainHtml  = hc.GetHtml();
            HtmlNodeCollection mainItems = mainHtml.SelectNodes("//*[@id='ajaxtable']/tbody[1]/tr[@class='tr3 t_one']");

            if (mainItems == null)
            {
                return(new List <CaoQunMainItem>());
            }
            foreach (HtmlNode mainItem in mainItems)
            {
                var an = mainItem.SelectSingleNode("/td[2]/h3/a");

                string infoUrl = "http://ac168.info/bt/" + an.Attributes["href"].Value;
                string title   = an.InnerText;

                CaoQunMainItem temp = new CaoQunMainItem
                {
                    CaoQunDetailItem = new CaoQunDetailItem(),
                    InfoUrl          = infoUrl,
                    Title            = title,
                    Url = CaoQunUrl
                };
                res.Add(temp);
            }
            return(res);
        }
示例#2
0
        private IEnumerable<MainItem> GetCaoQunMainItems()
        {
            List<MainItem> res = new List<MainItem>();
            HttpCore hc = new HttpCore();
            hc.SetUrl(CaoQunUrl);
            hc.CurrentHttpItem.Allowautoredirect = true;
            string mainHtml = hc.GetHtml();

            HtmlNodeCollection mainItems = mainHtml.SelectNodes("//*[@id='ajaxtable']/tbody[1]/tr");
            if (mainItems == null)
            {
                return new List<MainItem>();
            }
            foreach (HtmlNode mainItem in mainItems)
            {
                var an = mainItem.SelectSingleNode(mainItem.XPath + "/td[2]/h3/a");
                if (an == null)
                {
                    continue;
                }
                string infoUrl = "http://5.yao.cl/" + an.Attributes["href"].Value;
                string title = an.InnerText;

                MainItem temp = new MainItem
                {
                    HPLDetailItem = new HPLDetailItem(),
                    InfoUrl = infoUrl,
                    Title = title,
                    Url = CaoQunUrl
                };
                res.Add(temp);
            }
            return res;
        }
示例#3
0
        private List <MainItem> GetCaoQunMainItems()
        {
            List <MainItem> res = new List <MainItem>();
            HttpCore        hc  = new HttpCore();

            hc.SetUrl(CaoQunUrl);
            string mainHtml = hc.GetHtml();

            //MSScriptControl.ScriptControlClass scc = new MSScriptControl.ScriptControlClass { Language = "javascript" };
            //mainHtml = scc.Eval(mainHtml).ToString();

            ThreadWebBrowser(CaoQunUrl);
            while (kM == null)
            {
                Thread.Sleep(1000);
            }
            HtmlNodeCollection mainItems = mainHtml.SelectNodes("/html/body/div[8]/div[2]/div[@class='list-pianyuan-box']");

            if (mainItems == null)
            {
                return(new List <MainItem>());
            }
            foreach (HtmlNode mainItem in mainItems)
            {
                var an = mainItem.SelectSingleNode("/div[1]/div[2]/div[1]/a");

                string infoUrl = "http://www.haipilu.net/" + an.Attributes["href"].Value;
                string title   = an.InnerText;

                MainItem temp = new MainItem
                {
                    HPLDetailItem = new HPLDetailItem(),
                    InfoUrl       = infoUrl,
                    Title         = title,
                    Url           = CaoQunUrl
                };
                res.Add(temp);
            }
            return(res);
        }
示例#4
0
        public void ExecCrawler()
        {
            List <CaoQunMainItem> mainItems = GetCaoQunMainItems();

            Parallel.ForEach(mainItems, caoQunMainItem =>
            {
                HttpCore hc1 = new HttpCore();
                hc1.SetUrl(caoQunMainItem.InfoUrl);
                string infoHtml             = hc1.GetHtml();
                HtmlNodeCollection imgNodes = infoHtml.SelectNodes("//*[@id='read_tpc']/img");
                if (imgNodes == null)
                {
                    return;
                }

                string remark         = infoHtml.SelectNodes("//*[@id='read_tpc']")[0].InnerText;
                caoQunMainItem.Remark = remark;
                Parallel.ForEach(imgNodes, htmlNode =>
                {
                    string imgUrl = htmlNode.Attributes["src"].Value;
                    HttpCore hc   = new HttpCore {
                        CurrentHttpItem = { ResultType = ResultType.Byte }
                    };
                    hc.SetUrl(imgUrl);
                    HttpResult temp = hc.GetHttpResult();
                    //把得到的Byte转成图片
                    Image img = temp.ResultByte.ByteArrayToImage();
                    if (img == null)
                    {
                        return;
                    }
                    SaveImg("img2", caoQunMainItem, img, imgUrl);
                });

                //数据入库
                StoreDB(caoQunMainItem);
            });
        }
示例#5
0
        public void ExecCrawler()
        {
            List<MainItem> mainItems = GetCaoQunMainItems();

            Parallel.ForEach(mainItems, caoQunMainItem =>
            {
                HttpCore hc1 = new HttpCore();
                hc1.SetUrl(caoQunMainItem.InfoUrl);
                string infoHtml = hc1.GetHtml();
                HtmlNodeCollection imgNodes = infoHtml.SelectNodes("/html/body/div[7]/div/div[4]/div[2]/div/img");
                if (imgNodes == null)
                {
                    return;
                }

                string remark = infoHtml.SelectNodes("/html/body/div[7]/div/div[4]/div[2]/div/img")[0].InnerText;
                caoQunMainItem.Remark = remark;
                Parallel.ForEach(imgNodes, htmlNode =>
                {
                    string imgUrl = htmlNode.Attributes["src"].Value;
                    HttpCore hc = new HttpCore { CurrentHttpItem = { ResultType = ResultType.Byte } };
                    hc.SetUrl(imgUrl);
                    HttpResult temp = hc.GetHttpResult();
                    //把得到的Byte转成图片
                    Image img = temp.ResultByte.ByteArrayToImage();
                    if (img == null)
                    {
                        return;
                    }
                    SaveImg("img5", caoQunMainItem, img, imgUrl);
                });

                //数据入库
                StoreDB(caoQunMainItem);
            });
        }
示例#6
0
        private IEnumerable <MainItem> GetCaoQunMainItems()
        {
            List <MainItem> res = new List <MainItem>();
            HttpCore        hc  = new HttpCore();

            hc.SetUrl(CaoQunUrl);
            hc.CurrentHttpItem.Allowautoredirect = true;
            string mainHtml = hc.GetHtml();

            HtmlNodeCollection mainItems = mainHtml.SelectNodes("//*[@id='ajaxtable']/tbody[1]/tr");

            if (mainItems == null)
            {
                return(new List <MainItem>());
            }
            foreach (HtmlNode mainItem in mainItems)
            {
                var an = mainItem.SelectSingleNode(mainItem.XPath + "/td[2]/h3/a");
                if (an == null)
                {
                    continue;
                }
                string infoUrl = "http://5.yao.cl/" + an.Attributes["href"].Value;
                string title   = an.InnerText;

                MainItem temp = new MainItem
                {
                    HPLDetailItem = new HPLDetailItem(),
                    InfoUrl       = infoUrl,
                    Title         = title,
                    Url           = CaoQunUrl
                };
                res.Add(temp);
            }
            return(res);
        }
示例#7
0
        private List<MainItem> GetCaoQunMainItems()
        {
            List<MainItem> res = new List<MainItem>();
            HttpCore hc = new HttpCore();
            hc.SetUrl(CaoQunUrl);
            string mainHtml = hc.GetHtml();

            //MSScriptControl.ScriptControlClass scc = new MSScriptControl.ScriptControlClass { Language = "javascript" };
            //mainHtml = scc.Eval(mainHtml).ToString();

            ThreadWebBrowser(CaoQunUrl);
            while (kM == null)
            {
                Thread.Sleep(1000);
            }
            HtmlNodeCollection mainItems = mainHtml.SelectNodes("/html/body/div[8]/div[2]/div[@class='list-pianyuan-box']");
            if (mainItems == null)
            {
                return new List<MainItem>();
            }
            foreach (HtmlNode mainItem in mainItems)
            {
                var an = mainItem.SelectSingleNode("/div[1]/div[2]/div[1]/a");

                string infoUrl = "http://www.haipilu.net/" + an.Attributes["href"].Value;
                string title = an.InnerText;

                MainItem temp = new MainItem
                {
                    HPLDetailItem = new HPLDetailItem(),
                    InfoUrl = infoUrl,
                    Title = title,
                    Url = CaoQunUrl
                };
                res.Add(temp);
            }
            return res;
        }
示例#8
0
        public void Test()
        {
            Parallel.For(0, 100, i =>
            {
                DateTime dt = SysTimeRecord.Get(i.ToString());
                SysTimeRecord.Set(i.ToString(), dt == DateTime.MinValue ? DateTime.Now.AddDays(-i) : dt.AddHours(1));
            });

            for (int i = 0; i < 100; i++)
            {
                DateTime dt = SysTimeRecord.Get(i.ToString());
                SysTimeRecord.Set(i.ToString(), dt == DateTime.MinValue ? DateTime.Now.AddDays(i) : dt.AddHours(-1));
            }
            RedisTest.Test();


            AsyncWorker<int> aw = new AsyncWorker<int>(3, Console.WriteLine);
            for (int i = 0; i < 100; i++)
            {
                aw.AddItem(i);
            }
            aw.StopAndWait();


            Thread.Sleep(10000000);

            Parallel.Invoke(
                () => ExecCrawler(1),
                () => ExecCrawler(2),
                () => ExecCrawler(3),
                () => ExecCrawler(4),
                () => ExecCrawler(5)
                );



            HttpCore hc = new HttpCore();
            hc.SetUrl("http://ac168.info/bt/thread.php?fid=16&page={0}", 2);
            var res = hc.GetHtml();
            var tres = res.SelectNodes("//*[@id='ajaxtable']/tbody[1]/tr[@class='tr3 t_one']");

            HtmlNode node = tres[10];
            var an = node.SelectSingleNode("/td[2]/h3/a");

            string aHref = "http://ac168.info/bt/" + an.Attributes["href"].Value;
            string aText = an.InnerHtml;


            hc.SetUrl(aHref);
            res = hc.GetHtml();
            tres = res.SelectNodes("//*[@id='read_tpc']/img");

            string imgUrl = tres[0].Attributes["src"].Value;

            HttpCore hcImg = new HttpCore();
            hcImg.SetUrl(imgUrl);
            hcImg.CurrentHttpItem.ResultType = ResultType.Byte;
            //得到HTML代码
            HttpResult result = hcImg.CurrentHttpHelper.GetHtml(hcImg.CurrentHttpItem);
            if (result.StatusCode == System.Net.HttpStatusCode.OK)
            {
                //表示访问成功,具体的大家就参考HttpStatusCode类
            }
            //表示StatusCode的文字说明与描述
            string statusCodeDescription = result.StatusDescription;
            //把得到的Byte转成图片
            Image img = result.ResultByte.ByteArrayToImage();
            img.Save("f://imgs/" + imgUrl.Split('/').Last());
            Console.WriteLine(tres);
            Console.ReadLine();
            RequestBase rb = new RequestBase();
            rb.BindLocalCache("intc", s => GetSourceCacheInt(s[0].TryChangeType(0)), 3);
            rb.BindLocalCache("person", s => GetSourceCachePerson(), null);

            Stopwatch st0 = new Stopwatch();
            st0.Start();
            rb.BindLocalCache("ps", s => GetSourceCacheListPerson(), null);
            st0.Stop();
            Console.WriteLine("实例化1千万个对象,耗时【{0}】毫秒", st0.ElapsedMilliseconds);

            Stopwatch st1 = new Stopwatch();
            st1.Start();
            for (int i = 0; i < 10000000; i++)
            {
                rb.BindLocalCache("ps", s => GetSourceCacheListPerson(), null);
            }
            st1.Stop();
            Console.WriteLine("绑定1千万次本地缓存,耗时【{0}】毫秒", st1.ElapsedMilliseconds);

            var t1 = rb.GetLocalCache("intc");
            var t2 = rb.GetLocalCache("person");
            var t3 = rb.GetLocalCache<Person>("person");
            var t4 = rb.LocalCache.person;
            var t5 = rb.LocalCache.intc;

            Stopwatch st = new Stopwatch();
            st.Start();
            for (int i = 0; i < 10000000; i++)
            {
                var t = rb.LocalCache.ps;
                var tt = rb.GetLocalCache<List<Person>>("ps");
            }
            st.Stop();
            Console.WriteLine("读取2千万次本地缓存,耗时【{0}】毫秒", st.ElapsedMilliseconds);
        }
示例#9
0
        private List<CaoQunMainItem> GetCaoQunMainItems()
        {
            List<CaoQunMainItem> res = new List<CaoQunMainItem>();
            HttpCore hc = new HttpCore();
            hc.SetUrl(CaoQunUrl);
            string mainHtml = hc.GetHtml();
            HtmlNodeCollection mainItems = mainHtml.SelectNodes("//*[@id='ajaxtable']/tbody[1]/tr[@class='tr3 t_one']");
            if (mainItems == null)
            {
                return new List<CaoQunMainItem>();
            }
            foreach (HtmlNode mainItem in mainItems)
            {
                var an = mainItem.SelectSingleNode("/td[2]/h3/a");

                string infoUrl = "http://ac168.info/bt/" + an.Attributes["href"].Value;
                string title = an.InnerText;

                CaoQunMainItem temp = new CaoQunMainItem
                {
                    CaoQunDetailItem = new CaoQunDetailItem(),
                    InfoUrl = infoUrl,
                    Title = title,
                    Url = CaoQunUrl
                };
                res.Add(temp);
            }
            return res;
        }