private SpiderResult GetFromResponse(HttpWebResponse response) { var r = new SpiderResult(); try { using (var stream = response.GetResponseStream()) { using (var read = new StreamReader(stream, _encoding)) { r.HttpCode = response.StatusCode; r.Content = read.ReadToEnd(); } } } catch (Exception ex) { r.HttpCode = HttpStatusCode.InternalServerError; r.Content = ex.Message; } return(r); }
public SpiderResult HttpGet(string url, string refer, IWebProxy proxy = null) { SpiderResult spider = null; try { ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckCertificate); var request = (HttpWebRequest)HttpWebRequest.Create(url); request.UserAgent = _userAgent; request.Timeout = _timeout; request.ContentType = _contentType; request.Accept = _accept; request.Method = "GET"; request.Referer = refer; request.KeepAlive = true; request.AllowAutoRedirect = true; request.UnsafeAuthenticatedConnectionSharing = true; request.CookieContainer = new CookieContainer(); //据说设为null可以提高性能 request.Proxy = proxy; if (_cookiecollection != null) { foreach (Cookie c in _cookiecollection) { c.Domain = request.Host; } request.CookieContainer.Add(_cookiecollection); } foreach (KeyValuePair <String, String> hd in _headers) { request.Headers[hd.Key] = hd.Value; } using (HttpWebResponse response = (HttpWebResponse)request.GetResponse()) { spider = GetFromResponse(response); if (request.CookieContainer != null) { response.Cookies = request.CookieContainer.GetCookies(request.RequestUri); } if (response.Cookies != null) { _cookiecollection = response.Cookies; } if (response.Headers["Set-Cookie"] != null) { var cookie = response.Headers["Set-Cookie"]; _cookiecollection.Add(ConvertCookie(cookie)); } } } catch (Exception ex) { spider = new SpiderResult { HttpCode = HttpStatusCode.InternalServerError, Content = ex.Message }; } return(spider); }