private SpiderResult GetFromResponse(HttpWebResponse response)
{
var r = new SpiderResult();
try
{
using (var stream = response.GetResponseStream())
{
using (var read = new StreamReader(stream, _encoding))
{
r.HttpCode = response.StatusCode;
r.Content = read.ReadToEnd();
}
}
}
catch (Exception ex)
{
r.HttpCode = HttpStatusCode.InternalServerError;
r.Content = ex.Message;
}
return(r);
}
public SpiderResult HttpGet(string url, string refer, IWebProxy proxy = null)
{
SpiderResult spider = null;
try
{
ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckCertificate);
var request = (HttpWebRequest)HttpWebRequest.Create(url);
request.UserAgent = _userAgent;
request.Timeout = _timeout;
request.ContentType = _contentType;
request.Accept = _accept;
request.Method = "GET";
request.Referer = refer;
request.KeepAlive = true;
request.AllowAutoRedirect = true;
request.UnsafeAuthenticatedConnectionSharing = true;
request.CookieContainer = new CookieContainer();
//据说设为null可以提高性能
request.Proxy = proxy;
if (_cookiecollection != null)
{
foreach (Cookie c in _cookiecollection)
{
c.Domain = request.Host;
}
request.CookieContainer.Add(_cookiecollection);
}
foreach (KeyValuePair <String, String> hd in _headers)
{
request.Headers[hd.Key] = hd.Value;
}
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
spider = GetFromResponse(response);
if (request.CookieContainer != null)
{
response.Cookies = request.CookieContainer.GetCookies(request.RequestUri);
}
if (response.Cookies != null)
{
_cookiecollection = response.Cookies;
}
if (response.Headers["Set-Cookie"] != null)
{
var cookie = response.Headers["Set-Cookie"];
_cookiecollection.Add(ConvertCookie(cookie));
}
}
}
catch (Exception ex)
{
spider = new SpiderResult
{
HttpCode = HttpStatusCode.InternalServerError,
Content = ex.Message
};
}
return(spider);
}
