/// <summary>
/// 设置代理
/// </summary>
/// <param name="objHttpItems">参数对象</param>
private void SetProxy(HttpItems objHttpItems)
{
if (!string.IsNullOrEmpty(objHttpItems.ProxyIp))
{
WebProxy myProxy;
//设置代理服务器
if (objHttpItems.ProxyIp.Contains(":"))
{
string[] plist = objHttpItems.ProxyIp.Split(':');
myProxy = new WebProxy(plist[0].Trim(), Convert.ToInt32(plist[1].Trim()));
}
else
{
myProxy = new WebProxy(objHttpItems.ProxyIp, false);
}
if (!string.IsNullOrEmpty(objHttpItems.ProxyUserName) && !string.IsNullOrEmpty(objHttpItems.ProxyPwd))
{
myProxy.Credentials = new NetworkCredential(objHttpItems.ProxyUserName, objHttpItems.ProxyPwd);
}
_request.Proxy = myProxy;
//设置安全凭证
_request.Credentials = CredentialCache.DefaultNetworkCredentials;
}
else
{
_request.Proxy = null;
}
}
/// <summary>
/// 设置编码
/// </summary>
/// <param name="objHttpItems">Http参数</param>
private void SetEncoding(HttpItems objHttpItems)
{
if (!string.IsNullOrEmpty(objHttpItems.EncodingStr))
{
//读取数据时的编码方式
objHttpItems.Encoding = System.Text.Encoding.GetEncoding(objHttpItems.EncodingStr);
}
}
/// <summary>
/// 抓取页面
/// </summary>
/// <param name="obj"></param>
/// <param name="url"></param>
/// <returns></returns>
public static HttpHelpers SetUrl(this HttpHelpers obj, string url, CookieContainer cc = null)
{
cc = cc ?? new CookieContainer();
HttpItems item = new HttpItems();
item.Url = url;
item.Container = cc;
return(obj.AddHttpItem(item));
}
/// <summary>
/// 根据传入的数据，得到相应页面数据 异步
/// </summary>
/// <param name="objHttpItems">请求设置参数</param>
/// <returns>请求结果</returns>
private async Task <HttpResults> GetHttpRequestDataAsync(HttpItems objHttpItems)
{
try
{
#region 设置请求参数
SetRequest(objHttpItems);
#endregion
#region 获得应答结果
_response = (HttpWebResponse)await _request.GetResponseAsync();
return(GetResponesInfo(ref objHttpItems));
#endregion
}
catch (WebException ex)
{
#region 获取异常数据与结果
_result.Html = ex.Message;
_response = (HttpWebResponse)ex.Response;
if (_response != null)
{
_result.StatusCode = _response.StatusCode;
_result.StatusDescription = ex.Message;
try
{
return(GetResponesInfo(ref objHttpItems));
}
catch
{
return(_result);
}
}
else
{
_result.StatusCode = HttpStatusCode.NotFound;
_result.StatusDescription = ex.Message;
}
#endregion
}
return(_result);
}
/// <summary>
/// 设置Cookie
/// </summary>
/// <param name="objHttpItems">Http参数</param>
private void SetCookie(HttpItems objHttpItems)
{
if (!string.IsNullOrEmpty(objHttpItems.Cookie))
{
//字符串方式
_request.Headers[HttpRequestHeader.Cookie] = objHttpItems.Cookie;
}
if (objHttpItems.CookieCollection != null)
{
//CookieCollection方式
_request.CookieContainer = new CookieContainer();
_request.CookieContainer.Add(objHttpItems.CookieCollection);
}
if (objHttpItems.Container != null)
{
//CookieContainer方式
_request.CookieContainer = objHttpItems.Container;
}
}
/// <summary>
/// 设置证书
/// </summary>
/// <param name="objHttpItems">请求设置参数</param>
private void SetCer(HttpItems objHttpItems)
{
if (objHttpItems.IsEofError)
{
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls;
}
if (objHttpItems.IsSetSecurityProtocolType)
{
ServicePointManager.SecurityProtocol = (SecurityProtocolType)objHttpItems.SecProtocolTypeEx;
}
if (!string.IsNullOrEmpty(objHttpItems.CerPath))
{
ServicePointManager.ServerCertificateValidationCallback = new System.Net.Security.RemoteCertificateValidationCallback(CheckValidationResult);
//初始化对像，并设置请求的URL地址
_request = (HttpWebRequest)WebRequest.Create(GetUrl(objHttpItems.Url));
//创建证书文件
X509Certificate objx509;
if (!string.IsNullOrEmpty(objHttpItems.CerPass))
{
objx509 = new X509Certificate(objHttpItems.CerPath, objHttpItems.CerPass, X509KeyStorageFlags.PersistKeySet);
}
else
{
objx509 = new X509Certificate(objHttpItems.CerPath);
}
//添加到请求里
_request.ClientCertificates.Add(objx509);
}
else
{
//初始化对像，并设置请求的URL地址
try
{
_request = (HttpWebRequest)WebRequest.Create(GetUrl(objHttpItems.Url));
}
catch
{
throw new Exception("请检查输入的网址!");
}
}
}
/// <summary>
/// 设置Post数据
/// </summary>
/// <param name="objHttpItems">Http参数</param>
private void SetPostData(HttpItems objHttpItems)
{
//验证在得到结果时是否有传入数据
if (_request.Method.Trim().ToLower() == "post")
{
//写入Byte类型
if (objHttpItems.PostDataType == PostDataType.Byte)
{
//验证在得到结果时是否有传入数据
if (objHttpItems.PostdataByte != null && objHttpItems.PostdataByte.Length > 0)
{
_request.ContentLength = objHttpItems.PostdataByte.Length;
_request.GetRequestStream().Write(objHttpItems.PostdataByte, 0, objHttpItems.PostdataByte.Length);
}
}//写入文件
else if (objHttpItems.PostDataType == PostDataType.FilePath)
{
using (StreamReader r = new StreamReader(objHttpItems.Postdata, objHttpItems.Encoding))
{
byte[] buffer = objHttpItems.PostEncoding.GetBytes(r.ReadToEnd());
_request.ContentLength = buffer.Length;
_request.GetRequestStream().Write(buffer, 0, buffer.Length);
}
}
else
{
if (!string.IsNullOrEmpty(objHttpItems.Postdata))
{
//写入字符串数据.如果希望修改提交时的编码.请修改objHttpItems.PostEncoding
byte[] buffer = objHttpItems.PostEncoding.GetBytes(objHttpItems.Postdata);
_request.ContentLength = buffer.Length;
_request.GetRequestStream().Write(buffer, 0, buffer.Length);
}
}
}
}
/// <summary>
/// 设置请求参数
/// </summary>
/// <param name="items"></param>
public void SetHttpItems(HttpItems items)
{
_httpItems = items;
}
/*
* 迁移到ns2.0移除
* /// <summary>
* /// 反射修改useUnsafeHeaderParsing
* /// </summary>
* /// <param name="useUnsafe">设置的参数值</param>
* /// <returns></returns>
* private bool SetAllowUnsafeHeaderParsing20(bool useUnsafe)
* {
* System.Reflection.Assembly aNetAssembly = System.Reflection.Assembly.GetAssembly(typeof(System.Net.Configuration.SettingsSection));
* if (aNetAssembly != null)
* {
*
* Type aSettingsType = aNetAssembly.GetType("System.Net.Configuration.SettingsSectionInternal");
* if (aSettingsType != null)
* {
*
* object anInstance = aSettingsType.InvokeMember("Section",
* System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.GetProperty | System.Reflection.BindingFlags.NonPublic, null, null, new object[] { });
*
* if (anInstance != null)
* {
* System.Reflection.FieldInfo aUseUnsafeHeaderParsing = aSettingsType.GetField("useUnsafeHeaderParsing", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance);
* if (aUseUnsafeHeaderParsing != null)
* {
* aUseUnsafeHeaderParsing.SetValue(anInstance, useUnsafe);
* return true;
* }
* }
* }
* }
* return false;
* }*/
#endregion
/// <summary>
/// 为请求准备参数
/// </summary>
///<param name="objHttpItems">参数列表</param>
private void SetRequest(HttpItems objHttpItems)
{
#region 处理服务器提交协议错误 ns2.0移除
/*if (objHttpItems.UseUnsafe)
* {
* SetAllowUnsafeHeaderParsing20(objHttpItems.UseUnsafe);
* }*/
#endregion
#region 设置证书
SetCer(objHttpItems);
#endregion
#region 设置Header参数 如果有则添加,如果失败,则重置数据
if (objHttpItems.Header.Count > 0)
{
try
{
_request.Headers = objHttpItems.Header;
}
catch
{
_request.Headers = new WebHeaderCollection();
}
}
if (objHttpItems.AutoRedirectMax)
{
_request.MaximumAutomaticRedirections = 9999;
}
#endregion
#region 添加 Ajax数据头
if (objHttpItems.IsAjax)
{
_request.Headers.Add("x-requested-with: XMLHttpRequest");
}
#endregion
#region 设置代理
SetProxy(objHttpItems);
#endregion
#region 通用属性设置
//请求方式Get或者Post
_request.Method = objHttpItems.Method;
_request.Timeout = objHttpItems.Timeout;
_request.ReadWriteTimeout = objHttpItems.ReadWriteTimeout;
//Accept
_request.Accept = objHttpItems.Accept;
//ContentType返回类型
_request.ContentType = objHttpItems.ContentType;
//UserAgent客户端的访问类型，包括浏览器版本和操作系统信息
_request.UserAgent = objHttpItems.UserAgent;
// 如果设置编码,则从设置结果读取
SetEncoding(objHttpItems);
//设置Cookie
SetCookie(objHttpItems);
//来源地址
_request.Referer = objHttpItems.Referer;
//是否执行跳转功能
_request.AllowAutoRedirect = objHttpItems.Allowautoredirect;
//设置最大连接
if (objHttpItems.Connectionlimit > 0)
{
_request.ServicePoint.ConnectionLimit = objHttpItems.Connectionlimit;
System.Net.ServicePointManager.DefaultConnectionLimit = 1024;
}
//设置 post数据在大于1024时是否分包
_request.ServicePoint.Expect100Continue = objHttpItems.Expect100Continue;
#endregion
#region 如果当前为Post则设置并提交数据
SetPostData(objHttpItems);
#endregion
}
/// <summary>
/// 识别编码
/// </summary>
/// <param name="objHttpItems"></param>
private void GetEncodingNew(ref HttpItems objHttpItems)
{
string temp = Encoding.Default.GetString(_rawResponse, 0, _rawResponse.Length);
Match meta = Regex.Match(temp, "<meta([^<]*)charset=([^<]*)[\"']", RegexOptions.IgnoreCase | RegexOptions.Multiline);
string charter = (meta.Groups.Count > 2) ? meta.Groups[2].Value : string.Empty;
if (charter.IndexOf("\"") > 0)
{
charter = charter.Split('\"')[0];
}
if (charter.IndexOf(" ") > 0)
{
charter = charter.Split(' ')[0];
}
charter = charter.Replace("\"", string.Empty).Replace("'", string.Empty).Replace(";", string.Empty);
if (charter.Length > 0)
{
charter = charter.ToLower().Replace("iso-8859-1", "gbk");
if (string.IsNullOrEmpty(_response.CharacterSet.Trim()) || _response.CharacterSet.Trim().Contains("utf"))
{
objHttpItems.Encoding = Encoding.UTF8;
}
else
{
Encoding ed = StreamEncoder.GetEncodingFromBytes(_rawResponse);
if (ed != null)
{
objHttpItems.Encoding = ed;
}
else
{
//强制UTF8
objHttpItems.Encoding = Encoding.UTF8;
}
}
}
else
{
if (_response.CharacterSet != null)
{
if (_response.CharacterSet.ToLower().Trim() == "iso-8859-1")
{
// encoding = Encoding.GetEncoding("gbk"); 直接改为UTF8编码
objHttpItems.Encoding = Encoding.GetEncoding("gbk");//Encoding.UTF8;
}
else
{
if (string.IsNullOrEmpty(_response.CharacterSet.Trim()) || _response.CharacterSet.Trim().Contains("utf"))
{
objHttpItems.Encoding = Encoding.UTF8;
}
else
{
objHttpItems.Encoding = Encoding.GetEncoding(_response.CharacterSet);
}
}
}
else
{
Encoding ed = StreamEncoder.GetEncodingFromBytes(_rawResponse);
if (ed != null)
{
objHttpItems.Encoding = ed;
}
else
{
//强制UTF8
objHttpItems.Encoding = Encoding.UTF8;
}
}
}
}
/// <summary>
/// 处理响应结果
/// </summary>
/// <param name="objHttpItems">请求设置参数</param>
/// <returns></returns>
private HttpResults GetResponesInfo(ref HttpItems objHttpItems)
{
#region 设置返回结果
_result.StatusCode = _response.StatusCode;
_result.StatusDescription = _response.StatusDescription;
_result.Header = _response.Headers;
if (_response.Cookies != null)
{
_result.CookieCollection = _response.Cookies;
}
if (_response.ResponseUri != null)
{
_result.ResponseUrl = _response.ResponseUri.ToString();
}
if (_response.Headers["set-cookie"] != null)
{
//清理垃圾Cookie信息
_result.Cookie = new XJHTTP().ClearCookie(_response.Headers["set-cookie"]);
_result.RawCookie = _response.Headers["set-cookie"];
}
//处理返回值Container
_result.Container = objHttpItems.Container;
#endregion
#region 急速请求,不解析结果数据
if (objHttpItems.ResultType == ResultType.So)
{
return(_result);
}
#endregion
#region 应答结果转换为内存数据
using (_stream = new MemoryStream())
{
//GZIIP处理
if (_response.ContentEncoding != null && _response.ContentEncoding.Equals("gzip", StringComparison.InvariantCultureIgnoreCase))
{
_stream = GetMemoryStream(new GZipStream(_response.GetResponseStream(), CompressionMode.Decompress));
}
else
{
_stream = GetMemoryStream(_response.GetResponseStream());
}
//获取Byte
_rawResponse = _stream.ToArray();
}
#endregion
#region 如果设置返回结果类型则返回Byte数据
if (objHttpItems.ResultType == ResultType.Byte)
{
_result.ResultByte = _rawResponse;
return(_result);
}
#endregion
#region 处理编码结果
if (objHttpItems.Encoding == null)//如果用户没有指定编码格式
{
try
{
Encoding ed = StreamEncoder.GetEncodingFromBytes(_rawResponse);
if (ed != null)
{
objHttpItems.Encoding = ed;
}
else
{
GetEncodingNew(ref objHttpItems);
}
}
catch
{
//强制UTF8
objHttpItems.Encoding = Encoding.UTF8;
}
}
//得到返回的HTML
try
{
if (_rawResponse.Length > 0)
{
_result.Html = objHttpItems.Encoding.GetString(_rawResponse);
}
else
{
_result.Html = "HttpCode接收数据长度为0.请检查数据包.当前状态码:" + _result.StatusCodeNum.ToString() + _result.Html;
}
_response.Close();
}
catch
{
_result.Html = "HttpCode接收数据出现异常..请检查数据包.当前状态码:" + _result.StatusCodeNum.ToString() + _result.Html;
}
#endregion
return(_result);
}
/// <summary>
/// 拓展参数
/// </summary>
/// <param name="obj"></param>
/// <param name="item"></param>
/// <returns></returns>
public static HttpHelpers AddHttpItem(this HttpHelpers obj, HttpItems item)
{
obj.SetHttpItems(item);
return(obj);
}