private void button2_Click(object sender, EventArgs e) { EnumGloabParas.EnumEncodeType enocodeType = GetEndcode(); switch (enocodeType) { case EnumGloabParas.EnumEncodeType.AUTO: case EnumGloabParas.EnumEncodeType.UTF8: this.textBox2.Text = System.Web.HttpUtility.UrlDecode(this.textBox1.Text, Encoding.UTF8); break; case EnumGloabParas.EnumEncodeType.BIG5: this.textBox2.Text = System.Web.HttpUtility.UrlDecode(this.textBox1.Text, Encoding.GetEncoding("big5")); break; case EnumGloabParas.EnumEncodeType.GB2312: this.textBox2.Text = System.Web.HttpUtility.UrlDecode(this.textBox1.Text, Encoding.GetEncoding("gb2312")); break; case EnumGloabParas.EnumEncodeType.GBK: this.textBox2.Text = System.Web.HttpUtility.UrlDecode(this.textBox1.Text, Encoding.GetEncoding("gbk")); break; } }
public DataTable GetDataTable(DataTable dataTable, List <TaskColumnItem> columnItemList, cGatherUrlItem item, string webCookie, EnumGloabParas.EnumEncodeType webEncode, bool isAjax, ref string errMsg) { try { string html = this.GetHtml(item.Url, webCookie, webEncode, item.StartPos, item.EndPos, isAjax); if (string.IsNullOrEmpty(html)) { errMsg = "网页获取为空,有可能是采集范围设置错误或网络错误"; return(null); } dataTable = this.GetDataTable(dataTable, columnItemList, html, ref errMsg); } catch (Exception ex) { errMsg = ex.Message; } return(dataTable); }
public string GetHtml(string url, string cookie, EnumGloabParas.EnumEncodeType webCode, string startPos, string endPos, bool IsAjax) { Encoding wCode; string PostPara; CookieContainer CookieCon = new CookieContainer(); if (Regex.IsMatch(url, @"<POST>.*</POST>", RegexOptions.IgnoreCase)) { wReq = (HttpWebRequest)WebRequest.Create(@url.Substring(0, url.IndexOf("<POST>"))); } else { Uri uri = new Uri(url); wReq = (HttpWebRequest)WebRequest.Create(uri); } Match a = Regex.Match(url, @"(http://).[^/]*[?=/]", RegexOptions.IgnoreCase); string url1 = a.Groups[0].Value.ToString(); wReq.Referer = url1; if (cookie != "") { CookieCollection cl = GetWebCookies(cookie); CookieCon.Add(new Uri(url), cl); wReq.CookieContainer = CookieCon; } if (Regex.IsMatch(url, @"(?<=<POST>)[\S\s]*(?=</POST>)", RegexOptions.IgnoreCase)) { Match s = Regex.Match(url, @"(?<=<POST>).*(?=</POST>)", RegexOptions.IgnoreCase); PostPara = s.Groups[0].Value.ToString(); byte[] pPara = Encoding.ASCII.GetBytes(PostPara); wReq.ContentType = "application/x-www-form-urlencoded"; wReq.ContentLength = pPara.Length; wReq.Method = "POST"; System.IO.Stream reqStream = wReq.GetRequestStream(); reqStream.Write(pPara, 0, pPara.Length); reqStream.Close(); } else { wReq.Method = "GET"; wReq.ContentType = "text/html"; } this.Load(); HttpWebResponse wResp = GetResponse(wReq, 0, null); System.IO.Stream respStream = wResp.GetResponseStream(); string strWebData = ""; switch (webCode) { case EnumGloabParas.EnumEncodeType.AUTO: try { wCode = Encoding.Default; string cType = wResp.ContentType.ToLower(); Match charSetMatch = Regex.Match(cType, "(?<=charset=)([^<]*)*", RegexOptions.IgnoreCase | RegexOptions.Multiline); string webCharSet = charSetMatch.ToString(); wCode = System.Text.Encoding.GetEncoding(webCharSet); } catch { wCode = Encoding.Default; } break; case EnumGloabParas.EnumEncodeType.GB2312: wCode = Encoding.GetEncoding("gb2312"); break; case EnumGloabParas.EnumEncodeType.UTF8: wCode = Encoding.UTF8; break; case EnumGloabParas.EnumEncodeType.GBK: wCode = Encoding.GetEncoding("GBK"); break; default: wCode = Encoding.UTF8; break; } if (wResp.ContentEncoding == "gzip") { GZipStream myGZip = new GZipStream(respStream, CompressionMode.Decompress); System.IO.StreamReader reader; reader = new System.IO.StreamReader(myGZip, wCode); strWebData = reader.ReadToEnd(); reader.Close(); reader.Dispose(); } else { System.IO.StreamReader reader; reader = new System.IO.StreamReader(respStream, wCode); strWebData = reader.ReadToEnd(); reader.Close(); reader.Dispose(); } if (!string.IsNullOrEmpty(startPos) && !string.IsNullOrEmpty(endPos)) { string Splitstr = "(" + startPos + ").*?(" + endPos + ")"; Match aa = Regex.Match(strWebData, Splitstr, RegexOptions.IgnoreCase | RegexOptions.Multiline); if (aa.Success) { strWebData = aa.Groups[0].ToString(); } } if (IsAjax == true) { strWebData = System.Web.HttpUtility.UrlDecode(strWebData, Encoding.UTF8); } wResp.Close(); wReq.Abort(); return(strWebData); }
private EnumGloabParas.EnumEncodeType GetEndcode() { EnumGloabParas.EnumEncodeType enocodeType = (EnumGloabParas.EnumEncodeType)Enum.Parse(typeof(EnumGloabParas.EnumEncodeType), comboBoxExt1.SelectedValue.ToString()); return(enocodeType); }