/// <summary>
/// 获取网页源代码
/// </summary>
public static HtmlDocument GetHtmlSourceCode(string url, ProxyTuple proxy = null)
{
if (string.IsNullOrWhiteSpace(url))
{
return(null);
}
HtmlDocument document = null;
Console.WriteLine("Get html source code from {0}", url);
var webClient = new HtmlWeb
{
UserAgent = "User-Agent:Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36"
};
try
{
document = proxy == null?webClient.Load(url) : webClient.Load(url, proxy.Host, proxy.Port, proxy.UserId, proxy.Password);
}
catch
{
Console.WriteLine("An error appear when getting source code of {0}", url);
return(null);
}
return(document);
}
/// <summary>
/// 添加代理
/// </summary>
public static void AddInto(ProxyTuple newProxy)
{
ProxyTuple oldProxy;
newProxy.Useful = true;
if ((oldProxy = bagProxys.FirstOrDefault(item => item.Host == newProxy.Host)) != null)
{
oldProxy.Port = newProxy.Port;
oldProxy.UserId = newProxy.UserId;
oldProxy.Password = newProxy.Password;
}
else
{
newProxy.Index = bagProxys.Count;
bagProxys.Add(newProxy);
}
}