public static void DownloadFile(string url, string target)
{
var config = new HttpConfig();
try
{
using (HttpWebResponse response = GetResponse(url, "GET", null, config))
{
Stream stream = response.GetResponseStream();
if (!string.IsNullOrEmpty(response.ContentEncoding))
{
if (response.ContentEncoding.Contains("gzip"))
{
stream = new GZipStream(stream, CompressionMode.Decompress);
}
else if (response.ContentEncoding.Contains("deflate"))
{
stream = new DeflateStream(stream, CompressionMode.Decompress);
}
}
using (var fileStream = new FileStream(target, FileMode.Create))
{
int count;
byte[] buffer = new byte[4096];
while ((count = stream.Read(buffer, 0, buffer.Length)) > 0)
{
fileStream.Write(buffer, 0, count);
}
}
}
}
catch
{
}
}
public static string Send(string url, string method, string data, HttpConfig config)
{
if (config == null)
{
config = new HttpConfig();
}
string result;
using (HttpWebResponse response = GetResponse(url, method, data, config))
{
Stream stream = response.GetResponseStream();
if (!string.IsNullOrEmpty(response.ContentEncoding))
{
if (response.ContentEncoding.Contains("gzip"))
{
stream = new GZipStream(stream, CompressionMode.Decompress);
}
else if (response.ContentEncoding.Contains("deflate"))
{
stream = new DeflateStream(stream, CompressionMode.Decompress);
}
}
byte[] bytes = null;
using (MemoryStream ms = new MemoryStream())
{
int count;
byte[] buffer = new byte[4096];
while ((count = stream.Read(buffer, 0, buffer.Length)) > 0)
{
ms.Write(buffer, 0, count);
}
bytes = ms.ToArray();
}
#region 检测流编码
Encoding encoding;
//检测响应头是否返回了编码类型,若返回了编码类型则使用返回的编码
//注：有时响应头没有编码类型，CharacterSet经常设置为ISO-8859-1
if (!string.IsNullOrEmpty(response.CharacterSet) && response.CharacterSet.ToUpper() != "ISO-8859-1")
{
encoding = Encoding.GetEncoding(response.CharacterSet == "utf8" ? "utf-8" : response.CharacterSet);
}
else
{
//若没有在响应头找到编码，则去html找meta头的charset
result = Encoding.Default.GetString(bytes);
//在返回的html里使用正则匹配页面编码
Match match = Regex.Match(result, @"<meta.*charset=""?([\w-]+)""?.*>", RegexOptions.IgnoreCase);
if (match.Success)
{
encoding = Encoding.GetEncoding(match.Groups[1].Value);
}
else
{
//若html里面也找不到编码，默认使用utf-8
encoding = Encoding.GetEncoding(config.CharacterSet);
}
}
#endregion
result = encoding.GetString(bytes);
}
return(result);
}
public static string GET(string url, HttpConfig config)
{
return(Send(url, "GET", null, config));
}
public static string POST(string url, string data, HttpConfig config)
{
return(Send(url, "POST", data, config));
}
private static HttpWebResponse GetResponse(string url, string method, string data, HttpConfig config)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Method = method;
request.Referer = config.Referer;
//有些页面不设置用户代理信息则会抓取不到内容
request.UserAgent = config.UserAgent;
request.Timeout = config.Timeout;
request.Accept = config.Accept;
request.Headers.Set("Accept-Encoding", config.AcceptEncoding);
request.ContentType = config.ContentType;
request.KeepAlive = config.KeepAlive;
if (url.ToLower().StartsWith("https"))
{
//这里加入解决生产环境访问https的问题--Could not establish trust relationship for the SSL/TLS secure channel
ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(RemoteCertificateValidate);
}
if (method.ToUpper() == "POST")
{
if (!string.IsNullOrEmpty(data))
{
byte[] bytes = Encoding.UTF8.GetBytes(data);
if (config.GZipCompress)
{
using (MemoryStream stream = new MemoryStream())
{
using (GZipStream gZipStream = new GZipStream(stream, CompressionMode.Compress))
{
gZipStream.Write(bytes, 0, bytes.Length);
}
bytes = stream.ToArray();
}
}
request.ContentLength = bytes.Length;
request.GetRequestStream().Write(bytes, 0, bytes.Length);
}
else
{
request.ContentLength = 0;
}
}
return((HttpWebResponse)request.GetResponse());
}