Exemple #1
0
        private LandingsRecord ParseRequest(string htmlReturn, string nNumber)
        {
            HtmlDocument doc = new HtmlDocument();

            doc.LoadHtml(htmlReturn);
            HtmlNodeCollection bTags = doc.DocumentNode.SelectNodes("//b");
            int startIndex           = bTags.IndexOf(bTags.FirstOrDefault(record => record.InnerText.Contains(nNumber.ToUpper())));
            int currentIndex         = startIndex;

            LandingsRecord landingsRecord = new LandingsRecord();

            landingsRecord.N_Number          = bTags[currentIndex++].InnerText;
            landingsRecord.Serial_Number     = bTags[currentIndex++].InnerText;
            landingsRecord.Mfr               = bTags[currentIndex++].InnerText;
            landingsRecord.Model             = bTags[currentIndex++].InnerText;
            landingsRecord.Eng_Mfr           = bTags[currentIndex++].InnerText;
            landingsRecord.Type_Engine       = bTags[currentIndex++].InnerText;
            landingsRecord.Year_Mfr          = bTags[currentIndex++].InnerText;
            landingsRecord.Name              = bTags[currentIndex++].InnerText;
            landingsRecord.Street            = bTags[currentIndex++].InnerText;
            landingsRecord.City              = bTags[currentIndex++].InnerText;
            landingsRecord.Type_Aircraft     = bTags[currentIndex++].InnerText;
            landingsRecord.Cert_Issue_Date   = bTags[currentIndex++].InnerText;
            landingsRecord.Classification    = bTags[currentIndex++].InnerText;
            landingsRecord.Aircraft_Category = bTags[currentIndex++].InnerText;

            landingsRecord = FixAddressFields(landingsRecord);

            return(landingsRecord);
        }
Exemple #2
0
 public int IndexOf(HtmlNode item)
 {
     HtmlNode[] items = this._items;
     if (items == null)
     {
         return(-1);
     }
     return(HtmlNodeCollection.IndexOf(items, item, this._parentnode._childNodesCount));
 }
Exemple #3
0
        private List <HeadingModel> ExtractHeadingIndexes(HtmlNodeCollection headingNodes)
        {
            List <HeadingModel> headingIndexes = new List <HeadingModel>();

            foreach (HtmlNode headingNode in headingNodes)
            {
                string heading = headingNode.InnerText;
                int    index   = headingNodes.IndexOf(headingNode);

                HeadingModel headingModel = new HeadingModel(heading, index);
                headingIndexes.Add(headingModel);
            }

            return(headingIndexes);
        }
Exemple #4
0
        public bool Remove(HtmlNode item)
        {
            HtmlNode[] items = this._items;
            if (items == null)
            {
                return(false);
            }
            int num = HtmlNodeCollection.IndexOf(items, item, this._parentnode._childNodesCount);

            if (num == -1)
            {
                return(false);
            }
            this.RemoveAt(num);
            return(true);
        }
Exemple #5
0
        //Updated July 30th - Combined with WriteEpisodes
        public static void RefreshEpisodes()
        {
            try
            {
                MongoCollection mongoCollection = MongoUtils.GetMongoCollection
                                                  (
                    @"mongodb://" + ConfigurationManager.AppSettings["mongoHost"] + @"/",
                    ConfigurationManager.AppSettings["port"],
                    ConfigurationManager.AppSettings["db"],
                    ConfigurationManager.AppSettings["show_collection"]
                                                  );

                foreach (var show in mongoCollection.FindAllAs <TvShow>().ToList <TvShow>())
                {
                    if (!ConfigurationManager.AppSettings["show_excl"].Split(',').Any(s => show.Name.Contains(s)))
                    {
                        HtmlNodeCollection collection =
                            DownloadMgmtUtils.GetHtmlNodeCollection(
                                ConfigurationManager.AppSettings["show_url"] + show.Path, "//a");

                        if (collection != null)
                        {
                            foreach (var link in collection.Where(link =>
                                                                  !ConfigurationManager.AppSettings["episode_innerhtml_excl"].Split(',')
                                                                  .Any(s => link.InnerHtml.Contains(s)) &&
                                                                  link.Attributes.Count > 0 && !link.Attributes[0].Value.Contains("tvnews") &&
                                                                  link.Attributes[HrefIndex(link)].Value.Contains("/ep")))
                            {
                                UpdateShowFromHtml(show, link.InnerHtml, collection, collection.IndexOf(link));
                            }
                        }

                        mongoCollection.Save(show);
                    }
                }
            }
            catch (Exception e)
            {
                Log.AppendToLog("Error : FATAL Refresh Episodes Issue : " + e,
                                ConfigurationManager.AppSettings["log_file"]);
            }
        }
        public static string GetPropertyValueByLabel(string key, HtmlNodeCollection labels, HtmlNodeCollection infos, string def = "")
        {
            var currentDesc = labels.FirstOrDefault(l => l.InnerText == key);

            return(currentDesc != null ? infos[labels.IndexOf(currentDesc)]?.InnerText ?? def : def);
        }
Exemple #7
0
    private void ProcessCompare()
    {
        JObject jo = new JObject();

        try
        {
            DataTable dt = new DataTable();
            dt.Columns.Add("t", typeof(int));
            dt.Columns.Add("o1", typeof(double));
            dt.Columns.Add("o2", typeof(double));
            dt.Columns.Add("o3", typeof(double));

            HtmlWeb            web     = new HtmlWeb();
            HtmlDocument       doc     = web.Load("http://live1.nowscore.com/odds/match.aspx?id=" + Request["id"]);
            HtmlNodeCollection zdnodes = doc.DocumentNode.SelectNodes("//span[@id='odds']//span[text()='走地']/../..");
            foreach (HtmlNode zdnode in zdnodes)
            {
                for (int i = 1; i <= 3; i++)
                {
                    DataRow dr = dt.NewRow();
                    dr["t"] = i;
                    if (string.IsNullOrEmpty(zdnode.SelectSingleNode("td[" + (i * 6 - 1) + "]").InnerText.Trim()))
                    {
                        continue;
                    }
                    dr["o1"] = zdnode.SelectSingleNode("td[" + (i * 6 - 1) + "]").InnerText;
                    dr["o3"] = zdnode.SelectSingleNode("td[" + (i * 6 + 1) + "]").InnerText;
                    string o2text = zdnode.SelectSingleNode("td[" + (i * 6) + "]").InnerText;
                    if (i == 1)
                    {
                        dr["o2"] = CommonHelper.GoalCnToGoal(o2text);
                    }
                    else if (i == 2)
                    {
                        dr["o2"] = o2text;
                    }
                    else if (i == 3)
                    {
                        dr["o2"] = CommonHelper.BallSizeToBall(o2text);
                    }
                    dt.Rows.Add(dr);
                }
            }

            doc = web.Load("https://mobile.bet365.com/premium/V6/sport/coupon/coupon.aspx?zone=0&isocode=HK&tzi=1&key=" + Request["key"] + "&ip=1&gn=0&lng=2&ct=198&cts=43&clt=9996&ot=2");

            DataTable bdt = dt.Clone();
            zdnodes = doc.DocumentNode.SelectNodes("//section[position()<3] | //h1[starts-with(text(),'全場賽果')]/../../../..");
            foreach (HtmlNode zdnode in zdnodes)
            {
                try
                {
                    DataRow            dr        = bdt.NewRow();
                    HtmlNodeCollection oddsNodes = zdnode.SelectNodes("div//td[@class='cols2'] | div//td[@class='cols3']");

                    if (oddsNodes.Count == 2)
                    {
                        dr["t"] = zdnodes.IndexOf(zdnode) == 0 ? 1 : 3;
                        string        pankou = oddsNodes[0].SelectSingleNode("div/text()").InnerText;
                        List <double> pkList = new List <double>();
                        foreach (string item in pankou.Split(','))
                        {
                            pkList.Add(Convert.ToDouble(item.Replace("超過", "")));
                        }
                        dr["o2"] = Convert.ToInt32(dr["t"]) == 1 ? (0 - pkList.Average()) : pkList.Average();
                        dr["o1"] = Convert.ToDouble(oddsNodes[0].SelectSingleNode("div/span").InnerText) - 1;
                        dr["o3"] = Convert.ToDouble(oddsNodes[oddsNodes.Count - 1].SelectSingleNode("div/span").InnerText) - 1;
                    }
                    else
                    {
                        dr["t"]  = 2;
                        dr["o2"] = oddsNodes[1].SelectSingleNode("div/span").InnerText;
                        dr["o1"] = Convert.ToDouble(oddsNodes[0].SelectSingleNode("div/span").InnerText);
                        dr["o3"] = Convert.ToDouble(oddsNodes[oddsNodes.Count - 1].SelectSingleNode("div/span").InnerText);
                    }
                    bdt.Rows.Add(dr);
                }
                catch (Exception)
                {
                    continue;
                }
            }
            string s = "";
            if (Convert.ToInt32(bdt.Compute("count(t)", "t=1")) > 0)
            {
                DataRow dr    = bdt.Select("t=1")[0];
                int     count = Convert.ToInt32(dt.Compute("count(t)", string.Format("t=1 and o2={0}", new object[] { dr["o2"] })));
                if (count > 0)
                {
                    if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=1 and o1<={0} and o2={1}", new object[] { dr["o1"], dr["o2"] }))) == 0)
                    {
                        s += "亚盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看好赢盘信心" + count + "<br>";
                    }
                    if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=1 and o1>={0} and o2={1}", new object[] { dr["o1"], dr["o2"] }))) == 0)
                    {
                        s += "亚盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看衰赢盘信心" + count + "<br>";
                    }
                    if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=1 and o3<={0} and o2={1}", new object[] { dr["o3"], dr["o2"] }))) == 0)
                    {
                        s += "亚盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看好输盘信心" + count + "<br>";
                    }
                    if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=1 and o3>={0} and o2={1}", new object[] { dr["o3"], dr["o2"] }))) == 0)
                    {
                        s += "亚盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看衰输盘信心" + count + "<br>";
                    }
                }
            }
            if (Convert.ToInt32(bdt.Compute("count(t)", "t=2")) > 0)
            {
                DataRow dr = bdt.Select("t=2")[0];
                if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=2 and o1<={0}", new object[] { dr["o1"] }))) == 0)
                {
                    s += "欧盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看好胜<br>";
                }
                if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=2 and o1>={0}", new object[] { dr["o1"] }))) == 0)
                {
                    s += "欧盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看衰胜<br>";
                }
                if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=2 and o2<={0}", new object[] { dr["o2"] }))) == 0)
                {
                    s += "欧盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看好平<br>";
                }
                if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=2 and o2>={0}", new object[] { dr["o2"] }))) == 0)
                {
                    s += "欧盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看衰平<br>";
                }
                if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=2 and o3<={0}", new object[] { dr["o3"] }))) == 0)
                {
                    s += "欧盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看好负<br>";
                }
                if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=2 and o3>={0}", new object[] { dr["o3"] }))) == 0)
                {
                    s += "欧盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看衰负<br>";
                }
            }
            if (Convert.ToInt32(bdt.Compute("count(t)", "t=3")) > 0)
            {
                DataRow dr    = bdt.Select("t=3")[0];
                int     count = Convert.ToInt32(dt.Compute("count(t)", string.Format("t=3 and o2={0}", new object[] { dr["o2"] })));
                if (count > 0)
                {
                    if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=3 and o1<={0} and o2={1}", new object[] { dr["o1"], dr["o2"] }))) == 0)
                    {
                        s += "大小盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看好大球信心" + count + "<br>";
                    }
                    if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=3 and o1>={0} and o2={1}", new object[] { dr["o1"], dr["o2"] }))) == 0)
                    {
                        s += "大小盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看衰大球信心" + count + "<br>";
                    }
                    if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=3 and o3<={0} and o2={1}", new object[] { dr["o3"], dr["o2"] }))) == 0)
                    {
                        s += "大小盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看好小球信心" + count + "<br>";
                    }
                    if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=3 and o3>={0} and o2={1}", new object[] { dr["o3"], dr["o2"] }))) == 0)
                    {
                        s += "大小盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看衰小球信心" + count + "<br>";
                    }
                }
            }
            if (Convert.ToInt32(bdt.Compute("count(t)", "t=1")) > 0)
            {
                DataRow dr    = bdt.Select("t=1")[0];
                int     count = Convert.ToInt32(dt.Compute("count(t)", string.Format("t=1 and o2={0}", new object[] { dr["o2"] })));
                if (count > 0)
                {
                    s += "--------逆向亚盘---------<br>";
                    if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=1 and o1<={0} and o2={1}", new object[] { dr["o3"], dr["o2"] }))) == 0)
                    {
                        s += "亚盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看好赢盘信心" + count + "<br>";
                    }
                    if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=1 and o1>={0} and o2={1}", new object[] { dr["o3"], dr["o2"] }))) == 0)
                    {
                        s += "亚盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看衰赢盘信心" + count + "<br>";
                    }
                    if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=1 and o3<={0} and o2={1}", new object[] { dr["o1"], dr["o2"] }))) == 0)
                    {
                        s += "亚盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看好输盘信心" + count + "<br>";
                    }
                    if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=1 and o3>={0} and o2={1}", new object[] { dr["o1"], dr["o2"] }))) == 0)
                    {
                        s += "亚盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看衰输盘信心" + count + "<br>";
                    }
                }
            }
            jo.Add("success", true);
            jo.Add("message", s);
        }
        catch (Exception e)
        {
            jo.Add("success", false);
            jo.Add("message", e.Message);
        }
        finally {
            json = jo.ToString();
        }
    }
Exemple #8
0
        public void DownloadCouncilPdfFiles()
        {
            List <Documents>   docs    = this.LoadDocumentsDoneSQL();
            List <QueryResult> queries = this.LoadQueriesDoneSQL();
            WebClient          c       = new WebClient();
            ChromeDriver       cd      = new ChromeDriver();
            Regex dateReg = new Regex("[a-zA-Z]+[\\s]{0,2}[0-9]{1,2},[\\s]{0,2}[0-9]{4}");

            foreach (string url in this.docUrls)
            {
                cd.Navigate().GoToUrl(url);
                System.Threading.Thread.Sleep(10000);
                HtmlDocument doc = new HtmlDocument();
                doc.LoadHtml(cd.PageSource);
                HtmlNodeCollection fileNodes = null;
                string             category  = string.Empty;
                if (!url.Contains("planning-commission"))
                {
                    category  = "City Council";
                    fileNodes = doc.DocumentNode.SelectNodes("//a[@class='agendasminutesitem']");
                }
                else
                {
                    fileNodes = doc.DocumentNode.SelectNodes("//div[@class='interiorcontenteditabletext']/table");
                }

                foreach (HtmlNode entryNode in fileNodes)
                {
                    if (category == "City Council")
                    {
                        string   meetingUrl      = entryNode.Attributes["href"].Value;
                        string   meetingDateText = dateReg.Match(entryNode.InnerText).ToString();
                        DateTime meetingDate     = DateTime.Parse(meetingDateText);

                        if (meetingDate < this.dtStartFrom)
                        {
                            Console.WriteLine("Too early, skip...");
                            continue;
                        }

                        this.ExtractADoc(c, meetingUrl, category, "pdf", meetingDate, ref docs, ref queries);
                    }
                    else
                    {
                        if (fileNodes.IndexOf(entryNode) == 0)
                        {
                            category = "Planning Commission";
                        }
                        else
                        {
                            category = "Zoning Board of Appeals";
                        }

                        HtmlNodeCollection meetingNodes = entryNode.SelectNodes(".//a[@class='pdf']");

                        if (meetingNodes != null)
                        {
                            foreach (HtmlNode meetingNode in meetingNodes)
                            {
                                string   meetingDocUrl   = meetingNode.Attributes["href"].Value;
                                string   meetingDateText = dateReg.Match(meetingNode.InnerText).Value;
                                DateTime meetingDate     = DateTime.Parse(meetingDateText);

                                if (meetingDate < this.dtStartFrom)
                                {
                                    Console.WriteLine("Too early, skip...");
                                    continue;
                                }

                                this.ExtractADoc(c, meetingDocUrl, category, "pdf", meetingDate, ref docs, ref queries);
                            }
                        }
                    }
                }
            }

            cd.Quit();
        }
        public void InsertTest()
        {
            HtmlElement root = new HtmlElement("root");
            HtmlNodeCollection target = new HtmlNodeCollection(root);
            HtmlElement child = new HtmlElement("child");
            target.Add(child);

            child = new HtmlElement("second");
            target.Insert(0, child);
            Assert.AreEqual(root, child.Parent);
            Assert.AreEqual(target.IndexOf(child), 0);

            target.Insert(0, null);
        }
Exemple #10
0
        public bool getIndex(int novelid)
        {
            this.novelid = novelid;
            setPrompt("下载首页中……", Color.Orange);
            HttpUtil hu    = getHTTPUtil();
            string   index = hu.Get(getNovelURL(), getReferer());

            //StreamReader sFile = new StreamReader("z://1.htm", Encoding.GetEncoding("gb2312"));
            //string index = sFile.ReadToEnd();
            HtmlAgilityPack.HtmlDocument hd = new HtmlAgilityPack.HtmlDocument();
            hd.LoadHtml(index);
            //setPrompt("分析中……",Color.Orange);
            HtmlNode           hn;
            HtmlNodeCollection hnc;
            HtmlNode           root = hd.DocumentNode;

            if (useMobileEdition)
            {
                //book title
                hn        = root.SelectSingleNode("//h2");
                this.name = hn.InnerText.Substring(3);
                //descriptions
                hnc = root.SelectNodes("//div[@class='b module']/ul/li");
                foreach (HtmlNode n in hnc)
                {
                    if (n.SelectSingleNode("a") != null && this.author == "")
                    {
                        this.author = hnc[0].SelectSingleNode("a").InnerText;
                        continue;
                    }
                    this.descriptions += n.InnerText.Replace("\n", "").Replace("\r", "").
                                         Replace(" ", "").Replace(":", ":") + "\n";
                }
                //chapters
                HtmlNodeCollection chaps = root.SelectNodes("//div[@style='padding-left:10px']/a");
                this.chapterCount = chaps.Count;
                foreach (HtmlNode c in chaps)
                {
                    if (c.OuterHtml.IndexOf("vip") != -1)
                    {
                        vipChapters.Add(chaps.IndexOf(c) + 1);
                    }
                }
            }
            else
            {
                this.name   = root.SelectSingleNode("//span[@itemprop='articleSection']").InnerText;
                this.author = root.SelectSingleNode("//span[@itemprop='author']").InnerText;
                hnc         = root.SelectNodes("//ul[@class='rightul']/li");
                for (int idx = 0; idx < 6; idx++)
                {
                    HtmlNode n = hnc[idx];
                    this.descriptions += HtmlEntity.DeEntitize(n.InnerText.Replace("\n", "").Replace("\r", "").
                                                               Replace(" ", "").Replace(":", ":")) + "\n";
                }
                this.descriptions += root.SelectSingleNode("//div[@class='smallreadbody']/font").InnerText;
                HtmlNodeCollection chaps = root.SelectNodes("//table[@class='cytable']/tbody/tr[@itemtype='http://schema.org/Chapter']");
                this.chapterCount = chaps.Count;
                foreach (HtmlNode c in chaps)
                {
                    if (c.InnerHtml.IndexOf("onebook_vip") != -1)
                    {
                        vipChapters.Add(int.Parse(c.SelectSingleNode("./td[1]").InnerText.Trim()));
                    }
                }
                this.chapterCount = chaps.Count;
            }
            isFinnished = descriptions.IndexOf("连载中") == -1;
            setPrompt("首页分析完成,可以开始了");
            readProgress();
            return(true);
        }
Exemple #11
0
        public string ExportacionCAP(string HtmlDoc, bool Soar)
        {
            var    random = new Random();
            string token  = new string(Enumerable.Repeat("ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", 30).Select(s => s[random.Next(s.Length)]).ToArray());

            XSSFWorkbook xwb = new XSSFWorkbook();
            XSSFSheet    xs  = (XSSFSheet)xwb.CreateSheet("CAP");
            XSSFRow      xcr = (XSSFRow)xs.CreateRow(0);
            XSSFCell     xcc;

            XSSFCellStyle cstl;
            XSSFFont      cfnt;
            XSSFColor     ccol;

            int iCell = 0;
            int iRow  = 1;

            HtmlDocument ht = new HtmlDocument();

            ht.LoadHtml(HtmlDoc);

            HtmlNodeCollection Tables = ht.DocumentNode.SelectNodes("//body/table");
            HtmlNodeCollection HeaderRows;
            HtmlNodeCollection Rows;
            HtmlNodeCollection ValueRows;

            for (int t = 0; t < Tables.Count; t++)
            {
                if ((Soar && t == 0) || !Soar)
                {
                    iCell = 0;
                    if (Tables.IndexOf(Tables[t]) != 0)
                    {
                        iRow += Soar?1:2;
                    }
                    xcr = (XSSFRow)xs.CreateRow(iRow - 1);
                    xcc = (XSSFCell)xcr.CreateCell(iCell);

                    HeaderRows = Tables[t].SelectNodes("./thead/tr/th");

                    foreach (HtmlNode TableHeader in HeaderRows)
                    {
                        xcc = (XSSFCell)xcr.CreateCell(iCell);
                        string[] arr = TableHeader.Attributes["style"].Value.ToString().Split(';');

                        int[]    BackRgb = new int[3];
                        string[] BackRgbStr;
                        int[]    FrontRgb = new int[3];
                        string[] FrontRgbStr;

                        foreach (string style in arr)
                        {
                            if (style.Trim().StartsWith("background-color:"))
                            {
                                BackRgbStr = style.Split(':')[1].Split(',');
                                for (int i = 0; i < BackRgbStr.Length; i++)
                                {
                                    BackRgb[i] = int.Parse(BackRgbStr[i].Replace("rgb(", "").Replace(")", ""));
                                }
                            }

                            if (style.Trim().StartsWith("color:"))
                            {
                                FrontRgbStr = style.Split(':')[1].Split(',');
                                for (int i = 0; i < FrontRgbStr.Length; i++)
                                {
                                    FrontRgb[i] = int.Parse(FrontRgbStr[i].Replace("rgb(", "").Replace(")", ""));
                                }
                            }

                            for (int i = 0; i < 3; i++)
                            {
                                if (BackRgb[i] == 0)
                                {
                                    BackRgb[i] = 255;
                                }
                            }

                            cstl = (XSSFCellStyle)xwb.CreateCellStyle();
                            cfnt = (XSSFFont)xwb.CreateFont();
                            ccol = (!Soar? new XSSFColor(Color.FromArgb(BackRgb[0], BackRgb[1], BackRgb[2])): new XSSFColor((iCell != 0 && iCell != 1?Color.FromArgb(253, 234, 218): Color.FromArgb(255, 255, 255))));
                            cstl.SetFillForegroundColor(ccol);
                            cstl.FillPattern = FillPattern.SolidForeground;
                            ccol             = (!Soar? new XSSFColor(Color.FromArgb(FrontRgb[0], FrontRgb[1], FrontRgb[2])): new XSSFColor(Color.FromArgb(0, 0, 255)));
                            cfnt.SetColor(ccol);
                            cfnt.IsBold             = true;
                            cfnt.FontHeightInPoints = (short)(!Soar?14:10);
                            cstl.SetFont(cfnt);
                            if (Soar)
                            {
                                cstl.BorderBottom = BorderStyle.Thin;
                            }
                            if (!Soar)
                            {
                                cstl.Alignment = HorizontalAlignment.Center;
                            }

                            xcc.CellStyle = cstl;

                            xs.AutoSizeColumn(xcc.ColumnIndex);
                        }
                        xcc.SetCellValue(Soar ? TableHeader.InnerHtml.Split('(')[0] : TableHeader.InnerHtml);

                        iCell++;
                    }
                }
                Rows  = Tables[t].SelectNodes("./tbody/tr");
                iCell = 0;

                foreach (HtmlNode Row in Rows)
                {
                    xcr       = (XSSFRow)xs.CreateRow(iRow++);
                    ValueRows = Row.SelectNodes("./td");
                    iCell     = 0;
                    foreach (HtmlNode RowData in ValueRows)
                    {
                        string[] arr = RowData.Attributes["style"].Value.ToString().Split(';');

                        int[]    BackRgb = new int[3];
                        string[] BackRgbStr;
                        int[]    FrontRgb = new int[3];
                        string[] FrontRgbStr;
                        bool     bRightBorder = false;

                        xcc = (XSSFCell)xcr.CreateCell(iCell++);

                        foreach (string style in arr)
                        {
                            if (style.Trim().StartsWith("background-color:"))
                            {
                                BackRgbStr = style.Split(':')[1].Split(',');
                                for (int i = 0; i < BackRgbStr.Length; i++)
                                {
                                    BackRgb[i] = int.Parse(BackRgbStr[i].Replace("rgb(", "").Replace(")", ""));
                                }
                            }

                            if (style.Trim().StartsWith("color:"))
                            {
                                FrontRgbStr = style.Split(':')[1].Split(',');
                                for (int i = 0; i < FrontRgbStr.Length; i++)
                                {
                                    FrontRgb[i] = int.Parse(FrontRgbStr[i].Replace("rgb(", "").Replace(")", ""));
                                }
                            }

                            if (style.Trim().StartsWith("border-right: 1px dashed black"))
                            {
                                bRightBorder = true;
                            }

                            for (int i = 0; i < 3; i++)
                            {
                                if (BackRgb[i] == 0)
                                {
                                    BackRgb[i] = 255;
                                }
                            }

                            cstl = (XSSFCellStyle)xwb.CreateCellStyle();
                            cfnt = (XSSFFont)xwb.CreateFont();
                            ccol = new XSSFColor(Color.FromArgb(BackRgb[0], BackRgb[1], BackRgb[2]));
                            cstl.SetFillForegroundColor(ccol);
                            cstl.FillPattern = FillPattern.SolidForeground;
                            if (bRightBorder)
                            {
                                cstl.BorderRight = BorderStyle.DashDot;
                            }
                            ccol = new XSSFColor(Color.FromArgb(FrontRgb[0], FrontRgb[1], FrontRgb[2]));
                            cfnt.SetColor(ccol);
                            cfnt.FontHeightInPoints = (short)(!Soar ? 12 : 11);
                            cstl.SetFont(cfnt);
                            if (!Soar)
                            {
                                cstl.Alignment = HorizontalAlignment.Center;
                            }

                            xcc.CellStyle = cstl;

                            xs.AutoSizeColumn(xcc.ColumnIndex);
                        }

                        int  val  = int.MinValue;
                        bool bVal = int.TryParse(RowData.InnerHtml, out val);
                        if (bVal)
                        {
                            xcc.SetCellType(CellType.Numeric);
                            xcc.SetCellValue(Convert.ToInt32(RowData.InnerHtml));
                        }
                        else
                        {
                            xcc.SetCellValue(RowData.InnerHtml.ToUpper());
                        }
                    }
                }
            }


            FileStream File = new FileStream(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Temp", token + ".xlsx"), FileMode.Create, FileAccess.Write);

            xwb.Write(File);
            File.Close();

            return(token);
        }
Exemple #12
0
 public bool Contains(HtmlNode item)
 {
     HtmlNode[] items = this._items;
     return(items != null && HtmlNodeCollection.IndexOf(items, item, this._parentnode._childNodesCount) != -1);
 }
Exemple #13
0
        /// <summary>
        /// 从Web抓取天气数据,并存储在SQLite中
        /// </summary>
        public static async Task <bool> GetDataFormHtmlAndSaveToDB(string url, int cityID)
        {
            try
            {
                using (HttpClient client = new HttpClient())
                {
                    HttpResponseMessage resp = await client.GetAsync(url);

                    if (resp.IsSuccessStatusCode)
                    {
                        byte[] bytes = await resp.Content.ReadAsByteArrayAsync();

                        //注册RegisterProvider
                        Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
                        //声明GB2312
                        var GbEncoding = Encoding.GetEncoding("GB2312");
                        //byte[]转html字符串
                        var htmlStr = GbEncoding.GetString(bytes, 0, bytes.Length);
                        //从字符串解析html文档
                        HtmlDocument doc = new HtmlDocument();
                        doc.LoadHtml(htmlStr);
                        HtmlNode content = doc.DocumentNode.SelectSingleNode("//body//div//div[@id='bd']//div[@class='hd']//div[@id='content']");
                        string   title   = content.SelectSingleNode("h1").InnerHtml.Replace("\r\n", "").Replace(" ", "").Trim();
                        Tool.WriteLog("消息", title);
                        HtmlNodeCollection trs = content.SelectSingleNode("table").SelectNodes("tr");
                        if (trs != null && trs.Count > 0)
                        {
                            foreach (var tr in trs)
                            {
                                HtmlNodeCollection tds = tr.SelectNodes("td");
                                //跳过标题
                                if (trs.IndexOf(tr) == 0)
                                {
                                    continue;
                                }
                                Weather weather = new Weather()
                                {
                                    ID = Guid.NewGuid().ToString().ToLower(), CityID = cityID
                                };
                                foreach (var td in tds)
                                {
                                    HtmlNode a     = td.SelectSingleNode("a");
                                    string   value = string.Empty;
                                    value = a != null ? a.InnerHtml : td.InnerHtml;
                                    value = value.Replace("\r\n", "").Replace(" ", "").Trim();
                                    if (string.IsNullOrWhiteSpace(value))
                                    {
                                        continue;
                                    }
                                    // 日期 天气状况 气温 风力风向
                                    switch (tds.IndexOf(td))
                                    {
                                    case 0:
                                        value = value.Replace("年", "/").Replace("月", "/").Replace("日", "");
                                        if (DateTime.TryParse(value, out DateTime date))
                                        {
                                            weather.Date = date;
                                        }
                                        break;

                                    case 1:
                                        weather.WeatherDetail = value;
                                        break;

                                    case 2:
                                        weather.Temperature = value;
                                        break;

                                    case 3:
                                        weather.Wind = value;
                                        break;
                                    }
                                }
                                if (db == null)
                                {
                                    db = new SQLiteDB();
                                }
                                db.Weathers.Add(weather);
                                await db.SaveChangesAsync();
                            }
                        }
                    }
                }
                Tool.WriteLog("消息", "本次数据抓取成功");
                return(true);
            }
            catch (Exception e)
            {
                Tool.WriteLog("错误", e.Message);

                return(false);
            }
        }
Exemple #14
0
        static void Main(string[] args)
        {
            //SqlConnection conn = new SqlConnection("server=localhost;database=test;Trusted_Connection=SSPI");
            //try
            //{
            //    conn.Open();
            //}
            //catch
            //{
            //    Console.WriteLine("数据库链接失败");
            //}
            int          n       = 0;
            string       url     = "http://www.mahua.com/newjokes/index_2.htm";
            WebRequest   request = WebRequest.Create(url);
            Stream       stream  = WebRequest.Create(url).GetResponse().GetResponseStream();
            StreamReader reader  = new StreamReader(stream, Encoding.UTF8);
            //获取到源码
            string str = reader.ReadToEnd();

            stream.Close();
            stream.Dispose();
            //声明HtmlDocument对象
            HtmlDocument doc  = new HtmlDocument();
            HtmlDocument doc1 = new HtmlDocument();

            //加载html文档
            doc.LoadHtml(str);
            //通过xPath获取到html中指定元素
            HtmlNodeCollection collection = doc.DocumentNode.SelectNodes(@"/html/body/div[2]/div[1]/dl");
            string             b          = string.Empty;
            string             a          = string.Empty; //接受地址
            string             fileName   = string.Empty;
            string             c          = string.Empty; //图片种类
            SqlCommand         cmd;
            int k;
            int m;

            foreach (var item in collection)
            {
                k = collection.IndexOf(item);
                b = item.SelectNodes(@"//span[@class='joke-title']/a")[k].InnerText;  // title
                //图片地址//*[@id="\"joke_1641076\""]/dd[1]/img;
                a = item.SelectNodes(@"./dd[1]/img")[0].GetAttributeValue("src", "");
                //foreach (var item1 in hn)
                //{
                //    string q = item1.GetAttributeValue("mahuaimg", "");
                //    HtmlAttributeCollection hac;
                //    hac = item1.Attributes;
                //    IEnumerable<HtmlAttribute> ha = hac.AttributesWithName("src");

                //    Console.WriteLine("标题{0}---网址{1}", b, q);
                //}
                //a = item.SelectNodes(@"//dd[1]/img")[0].Attributes["mahuaImg"].Value;    //图片地址mahuaImg mahuaimg  mahuaImg
                Console.WriteLine("标题{0}---网址{1}", b, a);
                //if (!String.IsNullOrEmpty(a))
                //{//   [^\/]*$
                //    Regex reg = new Regex(@"[^\/]*$");
                //    fileName = reg.Match(a).ToString();  //获取URL 尾部设置为文件名
                //    WebClient mywebclient = new WebClient();
                //    string filepath = @"D:\img\" + fileName;
                //    try
                //    {
                //        mywebclient.DownloadFile(a, filepath);
                //        cmd = new SqlCommand("insert into article values("+c+",1,'" + b + "','" + fileName + "',0,0,getdate(),1)", conn);
                //        //执行SQL语句
                //        if (cmd.ExecuteNonQuery() > 0)
                //        {
                //            n++;
                //        }
                //        Console.WriteLine("抓取了" + n + "条消息");
                //    }
                //    catch (Exception ex)
                //    {
                //        Console.WriteLine(ex.ToString());
                //        Console.ReadKey();
                //    }
                //}
            }
            Console.WriteLine("完成");
            Console.ReadKey();
        }