/// <summary> /// 根据传入的参数,得到相应Http请求数据(唯一公开方法) /// </summary> /// <param name="item">参数类对象</param> /// <returns>返回HttpResult类型</returns> public HttpResult GetHttpRequestData(HttpRequestParam item) { ////返回参数 HttpResult result = new HttpResult(); try { try { ////准备参数 this.SetRequest(item); } catch (Exception ex) { // 注释掉,业务层捕获异常,可以根据网络异常重试 return new HttpResult() { Cookie = string.Empty, Header = null, Html = ex.Message, StatusDescription = "出现异常:" + ex.Message }; throw ex; } try { ////请求数据 using (this.response = (HttpWebResponse)this.request.GetResponse()) { this.GetData(item, result); } } catch (Exception ex) { result.Html = ex.ToString(); LogManager.Log.WriteAppException(new AppException("获取返回数据异常", ex, ExceptionLevel.Info)); throw ex; } } finally { if (item.IsWriteLog) { MInteractionParam param = new MInteractionParam() { DicContext = null, Module = "HttpHelper", Key1 = item.ProxyBusinessName, Key2 = item.Method.ToUpper(), SendAddress = item.URL, SendContent = this.GetRequestInfo(item), ReceiveContent = result.Html, Message = this.GetRequestProxy(item), }; LogManager.Log.InteractionLog(param); } this.AbortRequest(); } return result; }
/// <summary> /// 设置代理 /// </summary> /// <param name="item">参数对象</param> private void SetProxy(HttpRequestParam item) { if (!string.IsNullOrEmpty(item.ProxyBusinessName)) { WebProxy webProxy = null; if (item.ProxyIP.Contains(":")) { string[] ipAndPort = item.ProxyIP.Split(':'); webProxy = new WebProxy(ipAndPort[0].Trim(), Convert.ToInt32(ipAndPort[1].Trim())); } else { webProxy = new WebProxy(item.ProxyIP, false); } if (!string.IsNullOrEmpty(item.ProxyUserName)) { webProxy.Credentials = new NetworkCredential(item.ProxyUserName, item.ProxyPwd); this.request.UseDefaultCredentials = true; } this.request.Proxy = webProxy; } else { if (!item.IsIEProxy) { ////Proxy属性置为null后,即不使用IE代理,Fiddler无法抓包 this.request.Proxy = null; } } }
/// <summary> /// 为请求准备参数 /// </summary> /// <param name="item">参数列表</param> private void SetRequest(HttpRequestParam item) { ////设置最大连接数(初始化request对象前设置,则request默认连接数即为ServicePointManager.DefaultConnectionLimit) ServicePointManager.DefaultConnectionLimit = item.ConnectionLimit; ////https回调证书验证 if (item.URL.ToLower().IndexOf("https", System.StringComparison.Ordinal) > -1) { ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(this.CheckValidationResult); } ////初始化对像,并设置请求的URL地址 this.request = (HttpWebRequest)WebRequest.Create(item.URL); ////设置证书 this.SetCert(item); ////设置Header参数 if (item.Header != null && item.Header.Count > 0) { foreach (string key in item.Header.AllKeys) { this.request.Headers.Add(key, item.Header[key]); } } ////设置代理 this.SetProxy(item); if (item.ProtocolVersion != null) { this.request.ProtocolVersion = item.ProtocolVersion; } ////请求方式Get或者Post this.request.Method = item.Method; this.request.KeepAlive = item.KeepAlive; this.request.Timeout = item.Timeout; this.request.ReadWriteTimeout = item.ReadWriteTimeout; ////Accept this.request.Accept = item.Accept; ////ContentType返回类型 this.request.ContentType = item.ContentType; ////UserAgent客户端的访问类型,包括浏览器版本和操作系统信息 this.request.UserAgent = item.UserAgent; ////设置Cookie this.SetCookie(item); ////来源地址 this.request.Referer = item.Referer; ////是否执行跳转功能 this.request.AllowAutoRedirect = item.AllowAutoRedirect; ////100-Continue this.request.ServicePoint.Expect100Continue = item.Expect100Continue; ////设置Post数据 this.SetPostData(item); }
/// <summary> /// 设置编码 /// </summary> /// <param name="item">HttpItem</param> /// <param name="result">HttpResult</param> /// <param name="responseByte">byte[]</param> private void SetEncoding(HttpRequestParam item, HttpResult result, byte[] responseByte) { ////是否返回Byte类型数据 if (item.ResultType == ResultType.Byte) { result.ResultByte = responseByte; } if (item.Encoding == null) { Match meta = Regex.Match(Encoding.Default.GetString(responseByte), "<meta[^<]*charset=([^<]*)[\"']", RegexOptions.IgnoreCase); string c = string.Empty; if (meta != null && meta.Groups.Count > 0) { c = meta.Groups[1].Value.ToLower().Trim(); } if (c.Length > 2) { try { item.Encoding = Encoding.GetEncoding(c.Replace("\"", string.Empty).Replace("'", string.Empty).Replace(";", string.Empty).Replace("iso-8859-1", "gbk").Trim()); } catch { if (string.IsNullOrEmpty(this.response.CharacterSet)) { item.Encoding = Encoding.UTF8; } else { item.Encoding = Encoding.GetEncoding(this.response.CharacterSet); } } } else { if (string.IsNullOrEmpty(this.response.CharacterSet)) { item.Encoding = Encoding.UTF8; } else { item.Encoding = Encoding.GetEncoding(this.response.CharacterSet); } } } if (item.Encoding == null) { item.Encoding = Encoding.UTF8; } }
/// <summary> /// 设置Post数据 /// </summary> /// <param name="item">Http参数</param> private void SetPostData(HttpRequestParam item) { ////验证在得到结果时是否有传入数据 if (this.request.Method.Trim().ToLower().Contains("post")) { Encoding postEncoding = Encoding.Default; if (item.PostEncoding != null) { postEncoding = item.PostEncoding; } byte[] buffer = null; ////写入Byte类型 if (item.PostDataType == PostDataType.Byte && item.PostdataByte != null && item.PostdataByte.Length > 0) { ////验证在得到结果时是否有传入数据 buffer = item.PostdataByte; } else if (item.PostDataType == PostDataType.FilePath && !string.IsNullOrEmpty(item.Postdata)) { ////写入文件 using (StreamReader r = new StreamReader(item.Postdata, postEncoding)) { buffer = postEncoding.GetBytes(r.ReadToEnd()); } ////r.Close(); } else if (!string.IsNullOrEmpty(item.Postdata)) { ////写入字符串 buffer = postEncoding.GetBytes(item.Postdata); } if (buffer != null) { item.PostdataByte = buffer; this.request.ContentLength = buffer.Length; using (Stream requestStream = this.request.GetRequestStream()) { requestStream.Write(buffer, 0, buffer.Length); } } } }
/// <summary> /// 设置Cookie /// </summary> /// <param name="item">The item.</param> private void SetCookie(HttpRequestParam item) { if (!string.IsNullOrEmpty(item.Cookie)) { this.request.Headers[HttpRequestHeader.Cookie] = item.Cookie; } ////设置CookieCollection if (item.ResultCookieType == ResultCookieType.CookieCollection) { this.request.CookieContainer = new CookieContainer(); if (item.CookieCollection != null && item.CookieCollection.Count > 0) { this.request.CookieContainer.Add(item.CookieCollection); } } }
/// <summary> /// 设置证书 /// </summary> /// <param name="item">The item.</param> private void SetCert(HttpRequestParam item) { if (item.ClientCertificates != null && item.ClientCertificates.Count > 0) { foreach (X509Certificate c in item.ClientCertificates) { this.request.ClientCertificates.Add(c); } } }
/// <summary> /// 获取请求使用的代理IP /// </summary> /// <param name="item">请求参数</param> /// <returns>请求使用的代理IP</returns> private string GetRequestProxy(HttpRequestParam item) { try { if (item != null && !string.IsNullOrEmpty(item.ProxyIP)) { return item.ProxyIP; } else { return string.Empty; } } catch { return string.Empty; } }
/// <summary> /// 获取请求信息(记录交互日志用) /// </summary> /// <param name="request">请求对象</param> /// <returns>请求信息字符串</returns> private string GetRequestInfo(HttpRequestParam request) { try { StringBuilder sb = new StringBuilder(); string method = request.Method.Trim().ToUpper(); sb.Append(method + " " + request.URL + "\r\n"); if (method == "POST") { sb.Append("Content-Type: " + request.ContentType + "\r\n"); sb.Append("Content-Length: " + request.PostdataByte.Length + "\r\n"); sb.Append("Post-Data: " + request.Postdata + "\r\n"); } sb.Append("Cookie: " + request.Cookie); return sb.ToString(); } catch { return request.URL; } }
/// <summary> /// 获取数据的并解析的方法 /// </summary> /// <param name="item">The item.</param> /// <param name="result">The result.</param> private void GetData(HttpRequestParam item, HttpResult result) { ////获取StatusCode result.StatusCode = this.response.StatusCode; ////获取StatusDescription result.StatusDescription = this.response.StatusDescription; ////获取Headers result.Header = this.response.Headers; ////获取重定向路径 if (this.response.ResponseUri != null && this.response.ResponseUri.ToString() != item.URL) { result.ResponseUrl = this.response.ResponseUri.ToString(); } ////获取CookieCollection if (item.ResultCookieType == ResultCookieType.CookieCollection && this.request.CookieContainer != null) { result.CookieCollection = this.request.CookieContainer.GetCookies(this.request.RequestUri); } ////获取set-cookie if (this.response.Headers["set-cookie"] != null) { result.Cookie = this.ProcessCookies(item.Cookie, this.response.Headers["set-cookie"]); } ////处理网页Byte byte[] responseByte = this.GetByte(); if (responseByte != null && responseByte.Length > 0) { ////设置编码 this.SetEncoding(item, result, responseByte); ////得到返回的HTML result.Html = item.Encoding.GetString(responseByte); } else { ////没有返回任何Html代码 result.Html = string.Empty; } }