private LandingsRecord ParseRequest(string htmlReturn, string nNumber)
{
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(htmlReturn);
HtmlNodeCollection bTags = doc.DocumentNode.SelectNodes("//b");
int startIndex = bTags.IndexOf(bTags.FirstOrDefault(record => record.InnerText.Contains(nNumber.ToUpper())));
int currentIndex = startIndex;
LandingsRecord landingsRecord = new LandingsRecord();
landingsRecord.N_Number = bTags[currentIndex++].InnerText;
landingsRecord.Serial_Number = bTags[currentIndex++].InnerText;
landingsRecord.Mfr = bTags[currentIndex++].InnerText;
landingsRecord.Model = bTags[currentIndex++].InnerText;
landingsRecord.Eng_Mfr = bTags[currentIndex++].InnerText;
landingsRecord.Type_Engine = bTags[currentIndex++].InnerText;
landingsRecord.Year_Mfr = bTags[currentIndex++].InnerText;
landingsRecord.Name = bTags[currentIndex++].InnerText;
landingsRecord.Street = bTags[currentIndex++].InnerText;
landingsRecord.City = bTags[currentIndex++].InnerText;
landingsRecord.Type_Aircraft = bTags[currentIndex++].InnerText;
landingsRecord.Cert_Issue_Date = bTags[currentIndex++].InnerText;
landingsRecord.Classification = bTags[currentIndex++].InnerText;
landingsRecord.Aircraft_Category = bTags[currentIndex++].InnerText;
landingsRecord = FixAddressFields(landingsRecord);
return(landingsRecord);
}
public int IndexOf(HtmlNode item)
{
HtmlNode[] items = this._items;
if (items == null)
{
return(-1);
}
return(HtmlNodeCollection.IndexOf(items, item, this._parentnode._childNodesCount));
}
private List <HeadingModel> ExtractHeadingIndexes(HtmlNodeCollection headingNodes)
{
List <HeadingModel> headingIndexes = new List <HeadingModel>();
foreach (HtmlNode headingNode in headingNodes)
{
string heading = headingNode.InnerText;
int index = headingNodes.IndexOf(headingNode);
HeadingModel headingModel = new HeadingModel(heading, index);
headingIndexes.Add(headingModel);
}
return(headingIndexes);
}
public bool Remove(HtmlNode item)
{
HtmlNode[] items = this._items;
if (items == null)
{
return(false);
}
int num = HtmlNodeCollection.IndexOf(items, item, this._parentnode._childNodesCount);
if (num == -1)
{
return(false);
}
this.RemoveAt(num);
return(true);
}
//Updated July 30th - Combined with WriteEpisodes
public static void RefreshEpisodes()
{
try
{
MongoCollection mongoCollection = MongoUtils.GetMongoCollection
(
@"mongodb://" + ConfigurationManager.AppSettings["mongoHost"] + @"/",
ConfigurationManager.AppSettings["port"],
ConfigurationManager.AppSettings["db"],
ConfigurationManager.AppSettings["show_collection"]
);
foreach (var show in mongoCollection.FindAllAs <TvShow>().ToList <TvShow>())
{
if (!ConfigurationManager.AppSettings["show_excl"].Split(',').Any(s => show.Name.Contains(s)))
{
HtmlNodeCollection collection =
DownloadMgmtUtils.GetHtmlNodeCollection(
ConfigurationManager.AppSettings["show_url"] + show.Path, "//a");
if (collection != null)
{
foreach (var link in collection.Where(link =>
!ConfigurationManager.AppSettings["episode_innerhtml_excl"].Split(',')
.Any(s => link.InnerHtml.Contains(s)) &&
link.Attributes.Count > 0 && !link.Attributes[0].Value.Contains("tvnews") &&
link.Attributes[HrefIndex(link)].Value.Contains("/ep")))
{
UpdateShowFromHtml(show, link.InnerHtml, collection, collection.IndexOf(link));
}
}
mongoCollection.Save(show);
}
}
}
catch (Exception e)
{
Log.AppendToLog("Error : FATAL Refresh Episodes Issue : " + e,
ConfigurationManager.AppSettings["log_file"]);
}
}
public static string GetPropertyValueByLabel(string key, HtmlNodeCollection labels, HtmlNodeCollection infos, string def = "")
{
var currentDesc = labels.FirstOrDefault(l => l.InnerText == key);
return(currentDesc != null ? infos[labels.IndexOf(currentDesc)]?.InnerText ?? def : def);
}
private void ProcessCompare()
{
JObject jo = new JObject();
try
{
DataTable dt = new DataTable();
dt.Columns.Add("t", typeof(int));
dt.Columns.Add("o1", typeof(double));
dt.Columns.Add("o2", typeof(double));
dt.Columns.Add("o3", typeof(double));
HtmlWeb web = new HtmlWeb();
HtmlDocument doc = web.Load("http://live1.nowscore.com/odds/match.aspx?id=" + Request["id"]);
HtmlNodeCollection zdnodes = doc.DocumentNode.SelectNodes("//span[@id='odds']//span[text()='走地']/../..");
foreach (HtmlNode zdnode in zdnodes)
{
for (int i = 1; i <= 3; i++)
{
DataRow dr = dt.NewRow();
dr["t"] = i;
if (string.IsNullOrEmpty(zdnode.SelectSingleNode("td[" + (i * 6 - 1) + "]").InnerText.Trim()))
{
continue;
}
dr["o1"] = zdnode.SelectSingleNode("td[" + (i * 6 - 1) + "]").InnerText;
dr["o3"] = zdnode.SelectSingleNode("td[" + (i * 6 + 1) + "]").InnerText;
string o2text = zdnode.SelectSingleNode("td[" + (i * 6) + "]").InnerText;
if (i == 1)
{
dr["o2"] = CommonHelper.GoalCnToGoal(o2text);
}
else if (i == 2)
{
dr["o2"] = o2text;
}
else if (i == 3)
{
dr["o2"] = CommonHelper.BallSizeToBall(o2text);
}
dt.Rows.Add(dr);
}
}
doc = web.Load("https://mobile.bet365.com/premium/V6/sport/coupon/coupon.aspx?zone=0&isocode=HK&tzi=1&key=" + Request["key"] + "&ip=1&gn=0&lng=2&ct=198&cts=43&clt=9996&ot=2");
DataTable bdt = dt.Clone();
zdnodes = doc.DocumentNode.SelectNodes("//section[position()<3] | //h1[starts-with(text(),'全場賽果')]/../../../..");
foreach (HtmlNode zdnode in zdnodes)
{
try
{
DataRow dr = bdt.NewRow();
HtmlNodeCollection oddsNodes = zdnode.SelectNodes("div//td[@class='cols2'] | div//td[@class='cols3']");
if (oddsNodes.Count == 2)
{
dr["t"] = zdnodes.IndexOf(zdnode) == 0 ? 1 : 3;
string pankou = oddsNodes[0].SelectSingleNode("div/text()").InnerText;
List <double> pkList = new List <double>();
foreach (string item in pankou.Split(','))
{
pkList.Add(Convert.ToDouble(item.Replace("超過", "")));
}
dr["o2"] = Convert.ToInt32(dr["t"]) == 1 ? (0 - pkList.Average()) : pkList.Average();
dr["o1"] = Convert.ToDouble(oddsNodes[0].SelectSingleNode("div/span").InnerText) - 1;
dr["o3"] = Convert.ToDouble(oddsNodes[oddsNodes.Count - 1].SelectSingleNode("div/span").InnerText) - 1;
}
else
{
dr["t"] = 2;
dr["o2"] = oddsNodes[1].SelectSingleNode("div/span").InnerText;
dr["o1"] = Convert.ToDouble(oddsNodes[0].SelectSingleNode("div/span").InnerText);
dr["o3"] = Convert.ToDouble(oddsNodes[oddsNodes.Count - 1].SelectSingleNode("div/span").InnerText);
}
bdt.Rows.Add(dr);
}
catch (Exception)
{
continue;
}
}
string s = "";
if (Convert.ToInt32(bdt.Compute("count(t)", "t=1")) > 0)
{
DataRow dr = bdt.Select("t=1")[0];
int count = Convert.ToInt32(dt.Compute("count(t)", string.Format("t=1 and o2={0}", new object[] { dr["o2"] })));
if (count > 0)
{
if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=1 and o1<={0} and o2={1}", new object[] { dr["o1"], dr["o2"] }))) == 0)
{
s += "亚盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看好赢盘信心" + count + "<br>";
}
if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=1 and o1>={0} and o2={1}", new object[] { dr["o1"], dr["o2"] }))) == 0)
{
s += "亚盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看衰赢盘信心" + count + "<br>";
}
if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=1 and o3<={0} and o2={1}", new object[] { dr["o3"], dr["o2"] }))) == 0)
{
s += "亚盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看好输盘信心" + count + "<br>";
}
if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=1 and o3>={0} and o2={1}", new object[] { dr["o3"], dr["o2"] }))) == 0)
{
s += "亚盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看衰输盘信心" + count + "<br>";
}
}
}
if (Convert.ToInt32(bdt.Compute("count(t)", "t=2")) > 0)
{
DataRow dr = bdt.Select("t=2")[0];
if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=2 and o1<={0}", new object[] { dr["o1"] }))) == 0)
{
s += "欧盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看好胜<br>";
}
if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=2 and o1>={0}", new object[] { dr["o1"] }))) == 0)
{
s += "欧盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看衰胜<br>";
}
if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=2 and o2<={0}", new object[] { dr["o2"] }))) == 0)
{
s += "欧盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看好平<br>";
}
if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=2 and o2>={0}", new object[] { dr["o2"] }))) == 0)
{
s += "欧盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看衰平<br>";
}
if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=2 and o3<={0}", new object[] { dr["o3"] }))) == 0)
{
s += "欧盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看好负<br>";
}
if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=2 and o3>={0}", new object[] { dr["o3"] }))) == 0)
{
s += "欧盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看衰负<br>";
}
}
if (Convert.ToInt32(bdt.Compute("count(t)", "t=3")) > 0)
{
DataRow dr = bdt.Select("t=3")[0];
int count = Convert.ToInt32(dt.Compute("count(t)", string.Format("t=3 and o2={0}", new object[] { dr["o2"] })));
if (count > 0)
{
if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=3 and o1<={0} and o2={1}", new object[] { dr["o1"], dr["o2"] }))) == 0)
{
s += "大小盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看好大球信心" + count + "<br>";
}
if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=3 and o1>={0} and o2={1}", new object[] { dr["o1"], dr["o2"] }))) == 0)
{
s += "大小盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看衰大球信心" + count + "<br>";
}
if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=3 and o3<={0} and o2={1}", new object[] { dr["o3"], dr["o2"] }))) == 0)
{
s += "大小盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看好小球信心" + count + "<br>";
}
if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=3 and o3>={0} and o2={1}", new object[] { dr["o3"], dr["o2"] }))) == 0)
{
s += "大小盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看衰小球信心" + count + "<br>";
}
}
}
if (Convert.ToInt32(bdt.Compute("count(t)", "t=1")) > 0)
{
DataRow dr = bdt.Select("t=1")[0];
int count = Convert.ToInt32(dt.Compute("count(t)", string.Format("t=1 and o2={0}", new object[] { dr["o2"] })));
if (count > 0)
{
s += "--------逆向亚盘---------<br>";
if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=1 and o1<={0} and o2={1}", new object[] { dr["o3"], dr["o2"] }))) == 0)
{
s += "亚盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看好赢盘信心" + count + "<br>";
}
if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=1 and o1>={0} and o2={1}", new object[] { dr["o3"], dr["o2"] }))) == 0)
{
s += "亚盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看衰赢盘信心" + count + "<br>";
}
if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=1 and o3<={0} and o2={1}", new object[] { dr["o1"], dr["o2"] }))) == 0)
{
s += "亚盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看好输盘信心" + count + "<br>";
}
if (Convert.ToInt32(dt.Compute("count(t)", string.Format("t=1 and o3>={0} and o2={1}", new object[] { dr["o1"], dr["o2"] }))) == 0)
{
s += "亚盘" + dr["o1"] + " " + dr["o2"] + " " + dr["o3"] + "看衰输盘信心" + count + "<br>";
}
}
}
jo.Add("success", true);
jo.Add("message", s);
}
catch (Exception e)
{
jo.Add("success", false);
jo.Add("message", e.Message);
}
finally {
json = jo.ToString();
}
}
public void DownloadCouncilPdfFiles()
{
List <Documents> docs = this.LoadDocumentsDoneSQL();
List <QueryResult> queries = this.LoadQueriesDoneSQL();
WebClient c = new WebClient();
ChromeDriver cd = new ChromeDriver();
Regex dateReg = new Regex("[a-zA-Z]+[\\s]{0,2}[0-9]{1,2},[\\s]{0,2}[0-9]{4}");
foreach (string url in this.docUrls)
{
cd.Navigate().GoToUrl(url);
System.Threading.Thread.Sleep(10000);
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(cd.PageSource);
HtmlNodeCollection fileNodes = null;
string category = string.Empty;
if (!url.Contains("planning-commission"))
{
category = "City Council";
fileNodes = doc.DocumentNode.SelectNodes("//a[@class='agendasminutesitem']");
}
else
{
fileNodes = doc.DocumentNode.SelectNodes("//div[@class='interiorcontenteditabletext']/table");
}
foreach (HtmlNode entryNode in fileNodes)
{
if (category == "City Council")
{
string meetingUrl = entryNode.Attributes["href"].Value;
string meetingDateText = dateReg.Match(entryNode.InnerText).ToString();
DateTime meetingDate = DateTime.Parse(meetingDateText);
if (meetingDate < this.dtStartFrom)
{
Console.WriteLine("Too early, skip...");
continue;
}
this.ExtractADoc(c, meetingUrl, category, "pdf", meetingDate, ref docs, ref queries);
}
else
{
if (fileNodes.IndexOf(entryNode) == 0)
{
category = "Planning Commission";
}
else
{
category = "Zoning Board of Appeals";
}
HtmlNodeCollection meetingNodes = entryNode.SelectNodes(".//a[@class='pdf']");
if (meetingNodes != null)
{
foreach (HtmlNode meetingNode in meetingNodes)
{
string meetingDocUrl = meetingNode.Attributes["href"].Value;
string meetingDateText = dateReg.Match(meetingNode.InnerText).Value;
DateTime meetingDate = DateTime.Parse(meetingDateText);
if (meetingDate < this.dtStartFrom)
{
Console.WriteLine("Too early, skip...");
continue;
}
this.ExtractADoc(c, meetingDocUrl, category, "pdf", meetingDate, ref docs, ref queries);
}
}
}
}
}
cd.Quit();
}
public void InsertTest()
{
HtmlElement root = new HtmlElement("root");
HtmlNodeCollection target = new HtmlNodeCollection(root);
HtmlElement child = new HtmlElement("child");
target.Add(child);
child = new HtmlElement("second");
target.Insert(0, child);
Assert.AreEqual(root, child.Parent);
Assert.AreEqual(target.IndexOf(child), 0);
target.Insert(0, null);
}
public bool getIndex(int novelid)
{
this.novelid = novelid;
setPrompt("下载首页中……", Color.Orange);
HttpUtil hu = getHTTPUtil();
string index = hu.Get(getNovelURL(), getReferer());
//StreamReader sFile = new StreamReader("z://1.htm", Encoding.GetEncoding("gb2312"));
//string index = sFile.ReadToEnd();
HtmlAgilityPack.HtmlDocument hd = new HtmlAgilityPack.HtmlDocument();
hd.LoadHtml(index);
//setPrompt("分析中……",Color.Orange);
HtmlNode hn;
HtmlNodeCollection hnc;
HtmlNode root = hd.DocumentNode;
if (useMobileEdition)
{
//book title
hn = root.SelectSingleNode("//h2");
this.name = hn.InnerText.Substring(3);
//descriptions
hnc = root.SelectNodes("//div[@class='b module']/ul/li");
foreach (HtmlNode n in hnc)
{
if (n.SelectSingleNode("a") != null && this.author == "")
{
this.author = hnc[0].SelectSingleNode("a").InnerText;
continue;
}
this.descriptions += n.InnerText.Replace("\n", "").Replace("\r", "").
Replace(" ", "").Replace(":", "：") + "\n";
}
//chapters
HtmlNodeCollection chaps = root.SelectNodes("//div[@style='padding-left:10px']/a");
this.chapterCount = chaps.Count;
foreach (HtmlNode c in chaps)
{
if (c.OuterHtml.IndexOf("vip") != -1)
{
vipChapters.Add(chaps.IndexOf(c) + 1);
}
}
}
else
{
this.name = root.SelectSingleNode("//span[@itemprop='articleSection']").InnerText;
this.author = root.SelectSingleNode("//span[@itemprop='author']").InnerText;
hnc = root.SelectNodes("//ul[@class='rightul']/li");
for (int idx = 0; idx < 6; idx++)
{
HtmlNode n = hnc[idx];
this.descriptions += HtmlEntity.DeEntitize(n.InnerText.Replace("\n", "").Replace("\r", "").
Replace(" ", "").Replace(":", "：")) + "\n";
}
this.descriptions += root.SelectSingleNode("//div[@class='smallreadbody']/font").InnerText;
HtmlNodeCollection chaps = root.SelectNodes("//table[@class='cytable']/tbody/tr[@itemtype='http://schema.org/Chapter']");
this.chapterCount = chaps.Count;
foreach (HtmlNode c in chaps)
{
if (c.InnerHtml.IndexOf("onebook_vip") != -1)
{
vipChapters.Add(int.Parse(c.SelectSingleNode("./td[1]").InnerText.Trim()));
}
}
this.chapterCount = chaps.Count;
}
isFinnished = descriptions.IndexOf("连载中") == -1;
setPrompt("首页分析完成，可以开始了");
readProgress();
return(true);
}
public string ExportacionCAP(string HtmlDoc, bool Soar)
{
var random = new Random();
string token = new string(Enumerable.Repeat("ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", 30).Select(s => s[random.Next(s.Length)]).ToArray());
XSSFWorkbook xwb = new XSSFWorkbook();
XSSFSheet xs = (XSSFSheet)xwb.CreateSheet("CAP");
XSSFRow xcr = (XSSFRow)xs.CreateRow(0);
XSSFCell xcc;
XSSFCellStyle cstl;
XSSFFont cfnt;
XSSFColor ccol;
int iCell = 0;
int iRow = 1;
HtmlDocument ht = new HtmlDocument();
ht.LoadHtml(HtmlDoc);
HtmlNodeCollection Tables = ht.DocumentNode.SelectNodes("//body/table");
HtmlNodeCollection HeaderRows;
HtmlNodeCollection Rows;
HtmlNodeCollection ValueRows;
for (int t = 0; t < Tables.Count; t++)
{
if ((Soar && t == 0) || !Soar)
{
iCell = 0;
if (Tables.IndexOf(Tables[t]) != 0)
{
iRow += Soar?1:2;
}
xcr = (XSSFRow)xs.CreateRow(iRow - 1);
xcc = (XSSFCell)xcr.CreateCell(iCell);
HeaderRows = Tables[t].SelectNodes("./thead/tr/th");
foreach (HtmlNode TableHeader in HeaderRows)
{
xcc = (XSSFCell)xcr.CreateCell(iCell);
string[] arr = TableHeader.Attributes["style"].Value.ToString().Split(';');
int[] BackRgb = new int[3];
string[] BackRgbStr;
int[] FrontRgb = new int[3];
string[] FrontRgbStr;
foreach (string style in arr)
{
if (style.Trim().StartsWith("background-color:"))
{
BackRgbStr = style.Split(':')[1].Split(',');
for (int i = 0; i < BackRgbStr.Length; i++)
{
BackRgb[i] = int.Parse(BackRgbStr[i].Replace("rgb(", "").Replace(")", ""));
}
}
if (style.Trim().StartsWith("color:"))
{
FrontRgbStr = style.Split(':')[1].Split(',');
for (int i = 0; i < FrontRgbStr.Length; i++)
{
FrontRgb[i] = int.Parse(FrontRgbStr[i].Replace("rgb(", "").Replace(")", ""));
}
}
for (int i = 0; i < 3; i++)
{
if (BackRgb[i] == 0)
{
BackRgb[i] = 255;
}
}
cstl = (XSSFCellStyle)xwb.CreateCellStyle();
cfnt = (XSSFFont)xwb.CreateFont();
ccol = (!Soar? new XSSFColor(Color.FromArgb(BackRgb[0], BackRgb[1], BackRgb[2])): new XSSFColor((iCell != 0 && iCell != 1?Color.FromArgb(253, 234, 218): Color.FromArgb(255, 255, 255))));
cstl.SetFillForegroundColor(ccol);
cstl.FillPattern = FillPattern.SolidForeground;
ccol = (!Soar? new XSSFColor(Color.FromArgb(FrontRgb[0], FrontRgb[1], FrontRgb[2])): new XSSFColor(Color.FromArgb(0, 0, 255)));
cfnt.SetColor(ccol);
cfnt.IsBold = true;
cfnt.FontHeightInPoints = (short)(!Soar?14:10);
cstl.SetFont(cfnt);
if (Soar)
{
cstl.BorderBottom = BorderStyle.Thin;
}
if (!Soar)
{
cstl.Alignment = HorizontalAlignment.Center;
}
xcc.CellStyle = cstl;
xs.AutoSizeColumn(xcc.ColumnIndex);
}
xcc.SetCellValue(Soar ? TableHeader.InnerHtml.Split('(')[0] : TableHeader.InnerHtml);
iCell++;
}
}
Rows = Tables[t].SelectNodes("./tbody/tr");
iCell = 0;
foreach (HtmlNode Row in Rows)
{
xcr = (XSSFRow)xs.CreateRow(iRow++);
ValueRows = Row.SelectNodes("./td");
iCell = 0;
foreach (HtmlNode RowData in ValueRows)
{
string[] arr = RowData.Attributes["style"].Value.ToString().Split(';');
int[] BackRgb = new int[3];
string[] BackRgbStr;
int[] FrontRgb = new int[3];
string[] FrontRgbStr;
bool bRightBorder = false;
xcc = (XSSFCell)xcr.CreateCell(iCell++);
foreach (string style in arr)
{
if (style.Trim().StartsWith("background-color:"))
{
BackRgbStr = style.Split(':')[1].Split(',');
for (int i = 0; i < BackRgbStr.Length; i++)
{
BackRgb[i] = int.Parse(BackRgbStr[i].Replace("rgb(", "").Replace(")", ""));
}
}
if (style.Trim().StartsWith("color:"))
{
FrontRgbStr = style.Split(':')[1].Split(',');
for (int i = 0; i < FrontRgbStr.Length; i++)
{
FrontRgb[i] = int.Parse(FrontRgbStr[i].Replace("rgb(", "").Replace(")", ""));
}
}
if (style.Trim().StartsWith("border-right: 1px dashed black"))
{
bRightBorder = true;
}
for (int i = 0; i < 3; i++)
{
if (BackRgb[i] == 0)
{
BackRgb[i] = 255;
}
}
cstl = (XSSFCellStyle)xwb.CreateCellStyle();
cfnt = (XSSFFont)xwb.CreateFont();
ccol = new XSSFColor(Color.FromArgb(BackRgb[0], BackRgb[1], BackRgb[2]));
cstl.SetFillForegroundColor(ccol);
cstl.FillPattern = FillPattern.SolidForeground;
if (bRightBorder)
{
cstl.BorderRight = BorderStyle.DashDot;
}
ccol = new XSSFColor(Color.FromArgb(FrontRgb[0], FrontRgb[1], FrontRgb[2]));
cfnt.SetColor(ccol);
cfnt.FontHeightInPoints = (short)(!Soar ? 12 : 11);
cstl.SetFont(cfnt);
if (!Soar)
{
cstl.Alignment = HorizontalAlignment.Center;
}
xcc.CellStyle = cstl;
xs.AutoSizeColumn(xcc.ColumnIndex);
}
int val = int.MinValue;
bool bVal = int.TryParse(RowData.InnerHtml, out val);
if (bVal)
{
xcc.SetCellType(CellType.Numeric);
xcc.SetCellValue(Convert.ToInt32(RowData.InnerHtml));
}
else
{
xcc.SetCellValue(RowData.InnerHtml.ToUpper());
}
}
}
}
FileStream File = new FileStream(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Temp", token + ".xlsx"), FileMode.Create, FileAccess.Write);
xwb.Write(File);
File.Close();
return(token);
}
public bool Contains(HtmlNode item)
{
HtmlNode[] items = this._items;
return(items != null && HtmlNodeCollection.IndexOf(items, item, this._parentnode._childNodesCount) != -1);
}
/// <summary>
/// 从Web抓取天气数据，并存储在SQLite中
/// </summary>
public static async Task <bool> GetDataFormHtmlAndSaveToDB(string url, int cityID)
{
try
{
using (HttpClient client = new HttpClient())
{
HttpResponseMessage resp = await client.GetAsync(url);
if (resp.IsSuccessStatusCode)
{
byte[] bytes = await resp.Content.ReadAsByteArrayAsync();
//注册RegisterProvider
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
//声明GB2312
var GbEncoding = Encoding.GetEncoding("GB2312");
//byte[]转html字符串
var htmlStr = GbEncoding.GetString(bytes, 0, bytes.Length);
//从字符串解析html文档
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(htmlStr);
HtmlNode content = doc.DocumentNode.SelectSingleNode("//body//div//div[@id='bd']//div[@class='hd']//div[@id='content']");
string title = content.SelectSingleNode("h1").InnerHtml.Replace("\r\n", "").Replace(" ", "").Trim();
Tool.WriteLog("消息", title);
HtmlNodeCollection trs = content.SelectSingleNode("table").SelectNodes("tr");
if (trs != null && trs.Count > 0)
{
foreach (var tr in trs)
{
HtmlNodeCollection tds = tr.SelectNodes("td");
//跳过标题
if (trs.IndexOf(tr) == 0)
{
continue;
}
Weather weather = new Weather()
{
ID = Guid.NewGuid().ToString().ToLower(), CityID = cityID
};
foreach (var td in tds)
{
HtmlNode a = td.SelectSingleNode("a");
string value = string.Empty;
value = a != null ? a.InnerHtml : td.InnerHtml;
value = value.Replace("\r\n", "").Replace(" ", "").Trim();
if (string.IsNullOrWhiteSpace(value))
{
continue;
}
// 日期 天气状况 气温 风力风向
switch (tds.IndexOf(td))
{
case 0:
value = value.Replace("年", "/").Replace("月", "/").Replace("日", "");
if (DateTime.TryParse(value, out DateTime date))
{
weather.Date = date;
}
break;
case 1:
weather.WeatherDetail = value;
break;
case 2:
weather.Temperature = value;
break;
case 3:
weather.Wind = value;
break;
}
}
if (db == null)
{
db = new SQLiteDB();
}
db.Weathers.Add(weather);
await db.SaveChangesAsync();
}
}
}
}
Tool.WriteLog("消息", "本次数据抓取成功");
return(true);
}
catch (Exception e)
{
Tool.WriteLog("错误", e.Message);
return(false);
}
}
static void Main(string[] args)
{
//SqlConnection conn = new SqlConnection("server=localhost;database=test;Trusted_Connection=SSPI");
//try
//{
// conn.Open();
//}
//catch
//{
// Console.WriteLine("数据库链接失败");
//}
int n = 0;
string url = "http://www.mahua.com/newjokes/index_2.htm";
WebRequest request = WebRequest.Create(url);
Stream stream = WebRequest.Create(url).GetResponse().GetResponseStream();
StreamReader reader = new StreamReader(stream, Encoding.UTF8);
//获取到源码
string str = reader.ReadToEnd();
stream.Close();
stream.Dispose();
//声明HtmlDocument对象
HtmlDocument doc = new HtmlDocument();
HtmlDocument doc1 = new HtmlDocument();
//加载html文档
doc.LoadHtml(str);
//通过xPath获取到html中指定元素
HtmlNodeCollection collection = doc.DocumentNode.SelectNodes(@"/html/body/div[2]/div[1]/dl");
string b = string.Empty;
string a = string.Empty; //接受地址
string fileName = string.Empty;
string c = string.Empty; //图片种类
SqlCommand cmd;
int k;
int m;
foreach (var item in collection)
{
k = collection.IndexOf(item);
b = item.SelectNodes(@"//span[@class='joke-title']/a")[k].InnerText; // title
//图片地址//*[@id="\"joke_1641076\""]/dd[1]/img;
a = item.SelectNodes(@"./dd[1]/img")[0].GetAttributeValue("src", "");
//foreach (var item1 in hn)
//{
// string q = item1.GetAttributeValue("mahuaimg", "");
// HtmlAttributeCollection hac;
// hac = item1.Attributes;
// IEnumerable<HtmlAttribute> ha = hac.AttributesWithName("src");
// Console.WriteLine("标题{0}---网址{1}", b, q);
//}
//a = item.SelectNodes(@"//dd[1]/img")[0].Attributes["mahuaImg"].Value; //图片地址mahuaImg mahuaimg mahuaImg
Console.WriteLine("标题{0}---网址{1}", b, a);
//if (!String.IsNullOrEmpty(a))
//{// [^\/]*$
// Regex reg = new Regex(@"[^\/]*$");
// fileName = reg.Match(a).ToString(); //获取URL 尾部设置为文件名
// WebClient mywebclient = new WebClient();
// string filepath = @"D:\img\" + fileName;
// try
// {
// mywebclient.DownloadFile(a, filepath);
// cmd = new SqlCommand("insert into article values("+c+",1,'" + b + "','" + fileName + "',0,0,getdate(),1)", conn);
// //执行SQL语句
// if (cmd.ExecuteNonQuery() > 0)
// {
// n++;
// }
// Console.WriteLine("抓取了" + n + "条消息");
// }
// catch (Exception ex)
// {
// Console.WriteLine(ex.ToString());
// Console.ReadKey();
// }
//}
}
Console.WriteLine("完成");
Console.ReadKey();
}
