/// <summary> /// 发送WEB请求 /// </summary> /// <param name="url">地址</param> /// <param name="method">POST、GET</param> /// <param name="data">POST数据</param> /// <param name="config">配置信息</param> /// <exception cref="System.InvalidOperationException">流正由上一个 System.Net.HttpWebRequest.BeginGetResponse(System.AsyncCallback,System.Object)调用使用。- 或 -System.Net.HttpWebRequest.TransferEncoding 被设置为一个值,并且 System.Net.HttpWebRequest.SendChunked为 false。</exception> /// <exception cref="System.Net.ProtocolViolationException">System.Net.HttpWebRequest.Method 为 GET 或 HEAD,并且或者 System.Net.HttpWebRequest.ContentLength大于或等于零,或者 System.Net.HttpWebRequest.SendChunked 为 true。- 或 -System.Net.HttpWebRequest.KeepAlive为 true,System.Net.HttpWebRequest.AllowWriteStreamBuffering 为 false,System.Net.HttpWebRequest.ContentLength为 -1,System.Net.HttpWebRequest.SendChunked 为 false,System.Net.HttpWebRequest.Method为 POST 或 PUT。</exception> /// <exception cref="System.NotSupportedException">请求缓存验证程序指示对此请求的响应可从缓存中提供;但是,此请求包含要发送到服务器的数据。发送数据的请求不得使用缓存。如果您正在使用错误实现的自定义缓存验证程序,则会发生此异常。</exception> /// <exception cref="System.Net.WebException">System.Net.HttpWebRequest.Abort() 以前被调用过。- 或 -请求的超时期限到期。- 或 -处理请求时发生错误。</exception> /// <returns>Response数据</returns> public static string Send(string url, string method, string data, HttpConfig config) { if (config == null) { config = new HttpConfig(); } string result; using (HttpWebResponse response = GetResponse(url, method, data, config)) { Stream stream = response.GetResponseStream(); if (!String.IsNullOrEmpty(response.ContentEncoding)) { if (response.ContentEncoding.Contains("gzip")) { stream = new GZipStream(stream, CompressionMode.Decompress); } else if (response.ContentEncoding.Contains("deflate")) { stream = new DeflateStream(stream, CompressionMode.Decompress); } } byte[] bytes = null; using (MemoryStream ms = new MemoryStream()) { int count; byte[] buffer = new byte[4096]; while ((count = stream.Read(buffer, 0, buffer.Length)) > 0) { ms.Write(buffer, 0, count); } bytes = ms.ToArray(); } #region 检测流编码 Encoding encoding; //检测响应头是否返回了编码类型,若返回了编码类型则使用返回的编码 //注:有时响应头没有编码类型,CharacterSet经常设置为ISO-8859-1 if (!string.IsNullOrEmpty(response.CharacterSet) && response.CharacterSet.ToUpper() != "ISO-8859-1") { encoding = Encoding.GetEncoding(response.CharacterSet == "utf8" ? "utf-8" : response.CharacterSet); } else { //若没有在响应头找到编码,则去html找meta头的charset result = Encoding.Default.GetString(bytes); //在返回的html里使用正则匹配页面编码 Match match = Regex.Match(result, @"<meta.*charset=""?([\w-]+)""?.*>", RegexOptions.IgnoreCase); if (match.Success) { encoding = Encoding.GetEncoding(match.Groups[1].Value); } else { //若html里面也找不到编码,默认使用utf-8 encoding = Encoding.GetEncoding(config.CharacterSet); } } #endregion result = encoding.GetString(bytes); } return(result); }
private static HttpWebResponse GetResponse(string url, string method, string data, HttpConfig config) { HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url); request.Method = method; request.Referer = config.Referer; //有些页面不设置用户代理信息则会抓取不到内容 request.UserAgent = config.UserAgent; request.Timeout = config.Timeout; request.Accept = config.Accept; request.Headers.Set("Accept-Encoding", config.AcceptEncoding); request.ContentType = config.ContentType; request.KeepAlive = config.KeepAlive; if (url.ToLower().StartsWith("https")) { //这里加入解决生产环境访问https的问题--Could not establish trust relationship for the SSL/TLS secure channel ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(RemoteCertificateValidate); } if (method.ToUpper() == "POST") { if (!string.IsNullOrEmpty(data)) { byte[] bytes = Encoding.UTF8.GetBytes(data); if (config.GZipCompress) { Console.WriteLine("压缩前:" + bytes.Length); using (MemoryStream stream = new MemoryStream()) { using (GZipStream gZipStream = new GZipStream(stream, CompressionMode.Compress)) { gZipStream.Write(bytes, 0, bytes.Length); } bytes = stream.ToArray(); } Console.WriteLine("压缩后:" + bytes.Length); } request.ContentLength = bytes.Length; request.GetRequestStream().Write(bytes, 0, bytes.Length); } else { request.ContentLength = 0; } } return (HttpWebResponse)request.GetResponse(); }
private static HttpWebResponse GetResponse(string url, string method, string data, HttpConfig config) { HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url); request.Method = method; request.Referer = config.Referer; //有些页面不设置用户代理信息则会抓取不到内容 request.UserAgent = config.UserAgent; request.Timeout = config.Timeout; request.Accept = config.Accept; request.Headers.Set("Accept-Encoding", config.AcceptEncoding); request.ContentType = config.ContentType; request.KeepAlive = config.KeepAlive; if (url.ToLower().StartsWith("https")) { //这里加入解决生产环境访问https的问题--Could not establish trust relationship for the SSL/TLS secure channel ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(RemoteCertificateValidate); } if (method.ToUpper() == "POST") { if (!string.IsNullOrEmpty(data)) { byte[] bytes = Encoding.UTF8.GetBytes(data); if (config.GZipCompress) { Console.WriteLine("压缩前:" + bytes.Length); using (MemoryStream stream = new MemoryStream()) { using (GZipStream gZipStream = new GZipStream(stream, CompressionMode.Compress)) { gZipStream.Write(bytes, 0, bytes.Length); } bytes = stream.ToArray(); } Console.WriteLine("压缩后:" + bytes.Length); } request.ContentLength = bytes.Length; request.GetRequestStream().Write(bytes, 0, bytes.Length); } else { request.ContentLength = 0; } } return((HttpWebResponse)request.GetResponse()); }
/// <summary> /// 发送WEB请求 /// </summary> /// <param name="url">地址</param> /// <param name="method">POST、GET</param> /// <param name="data">POST数据</param> /// <param name="config">配置信息</param> /// <exception cref="System.InvalidOperationException">流正由上一个 System.Net.HttpWebRequest.BeginGetResponse(System.AsyncCallback,System.Object)调用使用。- 或 -System.Net.HttpWebRequest.TransferEncoding 被设置为一个值,并且 System.Net.HttpWebRequest.SendChunked为 false。</exception> /// <exception cref="System.Net.ProtocolViolationException">System.Net.HttpWebRequest.Method 为 GET 或 HEAD,并且或者 System.Net.HttpWebRequest.ContentLength大于或等于零,或者 System.Net.HttpWebRequest.SendChunked 为 true。- 或 -System.Net.HttpWebRequest.KeepAlive为 true,System.Net.HttpWebRequest.AllowWriteStreamBuffering 为 false,System.Net.HttpWebRequest.ContentLength为 -1,System.Net.HttpWebRequest.SendChunked 为 false,System.Net.HttpWebRequest.Method为 POST 或 PUT。</exception> /// <exception cref="System.NotSupportedException">请求缓存验证程序指示对此请求的响应可从缓存中提供;但是,此请求包含要发送到服务器的数据。发送数据的请求不得使用缓存。如果您正在使用错误实现的自定义缓存验证程序,则会发生此异常。</exception> /// <exception cref="System.Net.WebException">System.Net.HttpWebRequest.Abort() 以前被调用过。- 或 -请求的超时期限到期。- 或 -处理请求时发生错误。</exception> /// <returns>Response数据</returns> public static string Send(string url, string method, string data, HttpConfig config) { if (config == null) config = new HttpConfig(); string result; using (HttpWebResponse response = GetResponse(url, method, data, config)) { Stream stream = response.GetResponseStream(); if (!String.IsNullOrEmpty(response.ContentEncoding)) { if (response.ContentEncoding.Contains("gzip")) { stream = new GZipStream(stream, CompressionMode.Decompress); } else if (response.ContentEncoding.Contains("deflate")) { stream = new DeflateStream(stream, CompressionMode.Decompress); } } byte[] bytes = null; using (MemoryStream ms = new MemoryStream()) { int count; byte[] buffer = new byte[4096]; while ((count = stream.Read(buffer, 0, buffer.Length)) > 0) { ms.Write(buffer, 0, count); } bytes = ms.ToArray(); } #region 检测流编码 Encoding encoding; //检测响应头是否返回了编码类型,若返回了编码类型则使用返回的编码 //注:有时响应头没有编码类型,CharacterSet经常设置为ISO-8859-1 if (!string.IsNullOrEmpty(response.CharacterSet) && response.CharacterSet.ToUpper() != "ISO-8859-1") { encoding = Encoding.GetEncoding(response.CharacterSet == "utf8" ? "utf-8" : response.CharacterSet); } else { //若没有在响应头找到编码,则去html找meta头的charset result = Encoding.Default.GetString(bytes); //在返回的html里使用正则匹配页面编码 Match match = Regex.Match(result, @"<meta.*charset=""?([\w-]+)""?.*>", RegexOptions.IgnoreCase); if (match.Success) { encoding = Encoding.GetEncoding(match.Groups[1].Value); } else { //若html里面也找不到编码,默认使用utf-8 encoding = Encoding.GetEncoding(config.CharacterSet); } } #endregion result = encoding.GetString(bytes); } return result; }