/// <summary>
/// 发送WEB请求
/// </summary>
/// <param name="url">地址</param>
/// <param name="method">POST、GET</param>
/// <param name="data">POST数据</param>
/// <param name="config">配置信息</param>
/// <exception cref="System.InvalidOperationException">流正由上一个 System.Net.HttpWebRequest.BeginGetResponse(System.AsyncCallback,System.Object)调用使用。- 或 -System.Net.HttpWebRequest.TransferEncoding 被设置为一个值，并且 System.Net.HttpWebRequest.SendChunked为 false。</exception>
/// <exception cref="System.Net.ProtocolViolationException">System.Net.HttpWebRequest.Method 为 GET 或 HEAD，并且或者 System.Net.HttpWebRequest.ContentLength大于或等于零，或者 System.Net.HttpWebRequest.SendChunked 为 true。- 或 -System.Net.HttpWebRequest.KeepAlive为 true，System.Net.HttpWebRequest.AllowWriteStreamBuffering 为 false，System.Net.HttpWebRequest.ContentLength为 -1，System.Net.HttpWebRequest.SendChunked 为 false，System.Net.HttpWebRequest.Method为 POST 或 PUT。</exception>
/// <exception cref="System.NotSupportedException">请求缓存验证程序指示对此请求的响应可从缓存中提供；但是，此请求包含要发送到服务器的数据。发送数据的请求不得使用缓存。如果您正在使用错误实现的自定义缓存验证程序，则会发生此异常。</exception>
/// <exception cref="System.Net.WebException">System.Net.HttpWebRequest.Abort() 以前被调用过。- 或 -请求的超时期限到期。- 或 -处理请求时发生错误。</exception>
/// <returns>Response数据</returns>
public static string Send(string url, string method, string data, HttpConfig config)
{
if (config == null)
{
config = new HttpConfig();
}
string result;
using (HttpWebResponse response = GetResponse(url, method, data, config))
{
Stream stream = response.GetResponseStream();
if (!String.IsNullOrEmpty(response.ContentEncoding))
{
if (response.ContentEncoding.Contains("gzip"))
{
stream = new GZipStream(stream, CompressionMode.Decompress);
}
else if (response.ContentEncoding.Contains("deflate"))
{
stream = new DeflateStream(stream, CompressionMode.Decompress);
}
}
byte[] bytes = null;
using (MemoryStream ms = new MemoryStream())
{
int count;
byte[] buffer = new byte[4096];
while ((count = stream.Read(buffer, 0, buffer.Length)) > 0)
{
ms.Write(buffer, 0, count);
}
bytes = ms.ToArray();
}
#region 检测流编码
Encoding encoding;
//检测响应头是否返回了编码类型,若返回了编码类型则使用返回的编码
//注：有时响应头没有编码类型，CharacterSet经常设置为ISO-8859-1
if (!string.IsNullOrEmpty(response.CharacterSet) && response.CharacterSet.ToUpper() != "ISO-8859-1")
{
encoding = Encoding.GetEncoding(response.CharacterSet == "utf8" ? "utf-8" : response.CharacterSet);
}
else
{
//若没有在响应头找到编码，则去html找meta头的charset
result = Encoding.Default.GetString(bytes);
//在返回的html里使用正则匹配页面编码
Match match = Regex.Match(result, @"<meta.*charset=""?([\w-]+)""?.*>", RegexOptions.IgnoreCase);
if (match.Success)
{
encoding = Encoding.GetEncoding(match.Groups[1].Value);
}
else
{
//若html里面也找不到编码，默认使用utf-8
encoding = Encoding.GetEncoding(config.CharacterSet);
}
}
#endregion
result = encoding.GetString(bytes);
}
return(result);
}
private static HttpWebResponse GetResponse(string url, string method, string data, HttpConfig config)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Method = method;
request.Referer = config.Referer;
//有些页面不设置用户代理信息则会抓取不到内容
request.UserAgent = config.UserAgent;
request.Timeout = config.Timeout;
request.Accept = config.Accept;
request.Headers.Set("Accept-Encoding", config.AcceptEncoding);
request.ContentType = config.ContentType;
request.KeepAlive = config.KeepAlive;
if (url.ToLower().StartsWith("https"))
{
//这里加入解决生产环境访问https的问题--Could not establish trust relationship for the SSL/TLS secure channel
ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(RemoteCertificateValidate);
}
if (method.ToUpper() == "POST")
{
if (!string.IsNullOrEmpty(data))
{
byte[] bytes = Encoding.UTF8.GetBytes(data);
if (config.GZipCompress)
{
Console.WriteLine("压缩前:" + bytes.Length);
using (MemoryStream stream = new MemoryStream())
{
using (GZipStream gZipStream = new GZipStream(stream, CompressionMode.Compress))
{
gZipStream.Write(bytes, 0, bytes.Length);
}
bytes = stream.ToArray();
}
Console.WriteLine("压缩后:" + bytes.Length);
}
request.ContentLength = bytes.Length;
request.GetRequestStream().Write(bytes, 0, bytes.Length);
}
else
{
request.ContentLength = 0;
}
}
return (HttpWebResponse)request.GetResponse();
}
private static HttpWebResponse GetResponse(string url, string method, string data, HttpConfig config)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Method = method;
request.Referer = config.Referer;
//有些页面不设置用户代理信息则会抓取不到内容
request.UserAgent = config.UserAgent;
request.Timeout = config.Timeout;
request.Accept = config.Accept;
request.Headers.Set("Accept-Encoding", config.AcceptEncoding);
request.ContentType = config.ContentType;
request.KeepAlive = config.KeepAlive;
if (url.ToLower().StartsWith("https"))
{
//这里加入解决生产环境访问https的问题--Could not establish trust relationship for the SSL/TLS secure channel
ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(RemoteCertificateValidate);
}
if (method.ToUpper() == "POST")
{
if (!string.IsNullOrEmpty(data))
{
byte[] bytes = Encoding.UTF8.GetBytes(data);
if (config.GZipCompress)
{
Console.WriteLine("压缩前:" + bytes.Length);
using (MemoryStream stream = new MemoryStream())
{
using (GZipStream gZipStream = new GZipStream(stream, CompressionMode.Compress))
{
gZipStream.Write(bytes, 0, bytes.Length);
}
bytes = stream.ToArray();
}
Console.WriteLine("压缩后:" + bytes.Length);
}
request.ContentLength = bytes.Length;
request.GetRequestStream().Write(bytes, 0, bytes.Length);
}
else
{
request.ContentLength = 0;
}
}
return((HttpWebResponse)request.GetResponse());
}
/// <summary>
/// 发送WEB请求
/// </summary>
/// <param name="url">地址</param>
/// <param name="method">POST、GET</param>
/// <param name="data">POST数据</param>
/// <param name="config">配置信息</param>
/// <exception cref="System.InvalidOperationException">流正由上一个 System.Net.HttpWebRequest.BeginGetResponse(System.AsyncCallback,System.Object)调用使用。- 或 -System.Net.HttpWebRequest.TransferEncoding 被设置为一个值，并且 System.Net.HttpWebRequest.SendChunked为 false。</exception>
/// <exception cref="System.Net.ProtocolViolationException">System.Net.HttpWebRequest.Method 为 GET 或 HEAD，并且或者 System.Net.HttpWebRequest.ContentLength大于或等于零，或者 System.Net.HttpWebRequest.SendChunked 为 true。- 或 -System.Net.HttpWebRequest.KeepAlive为 true，System.Net.HttpWebRequest.AllowWriteStreamBuffering 为 false，System.Net.HttpWebRequest.ContentLength为 -1，System.Net.HttpWebRequest.SendChunked 为 false，System.Net.HttpWebRequest.Method为 POST 或 PUT。</exception>
/// <exception cref="System.NotSupportedException">请求缓存验证程序指示对此请求的响应可从缓存中提供；但是，此请求包含要发送到服务器的数据。发送数据的请求不得使用缓存。如果您正在使用错误实现的自定义缓存验证程序，则会发生此异常。</exception>
/// <exception cref="System.Net.WebException">System.Net.HttpWebRequest.Abort() 以前被调用过。- 或 -请求的超时期限到期。- 或 -处理请求时发生错误。</exception>
/// <returns>Response数据</returns>
public static string Send(string url, string method, string data, HttpConfig config)
{
if (config == null) config = new HttpConfig();
string result;
using (HttpWebResponse response = GetResponse(url, method, data, config))
{
Stream stream = response.GetResponseStream();
if (!String.IsNullOrEmpty(response.ContentEncoding))
{
if (response.ContentEncoding.Contains("gzip"))
{
stream = new GZipStream(stream, CompressionMode.Decompress);
}
else if (response.ContentEncoding.Contains("deflate"))
{
stream = new DeflateStream(stream, CompressionMode.Decompress);
}
}
byte[] bytes = null;
using (MemoryStream ms = new MemoryStream())
{
int count;
byte[] buffer = new byte[4096];
while ((count = stream.Read(buffer, 0, buffer.Length)) > 0)
{
ms.Write(buffer, 0, count);
}
bytes = ms.ToArray();
}
#region 检测流编码
Encoding encoding;
//检测响应头是否返回了编码类型,若返回了编码类型则使用返回的编码
//注：有时响应头没有编码类型，CharacterSet经常设置为ISO-8859-1
if (!string.IsNullOrEmpty(response.CharacterSet) && response.CharacterSet.ToUpper() != "ISO-8859-1")
{
encoding = Encoding.GetEncoding(response.CharacterSet == "utf8" ? "utf-8" : response.CharacterSet);
}
else
{
//若没有在响应头找到编码，则去html找meta头的charset
result = Encoding.Default.GetString(bytes);
//在返回的html里使用正则匹配页面编码
Match match = Regex.Match(result, @"<meta.*charset=""?([\w-]+)""?.*>", RegexOptions.IgnoreCase);
if (match.Success)
{
encoding = Encoding.GetEncoding(match.Groups[1].Value);
}
else
{
//若html里面也找不到编码，默认使用utf-8
encoding = Encoding.GetEncoding(config.CharacterSet);
}
}
#endregion
result = encoding.GetString(bytes);
}
return result;
}
