/// <summary> /// 根据相传入的数据,得到相应页面数据 /// </summary> /// <param name="strPostdata">传入的数据Post方式,get方式传NUll或者空字符串都可以</param> /// <param name="ContentType">返回的响应数据的类型</param> /// <returns>string类型的响应数据</returns> private async Task <HttpResponse> GetHttpRequestData(HttpWebRequest request, HttpItem objhttpitem) { var result = ""; #region 得到请求的response HttpResponse httpResponse = new HttpResponse(); var baseresponse = await request.GetResponseAsync(); using (var response = (HttpWebResponse)baseresponse) { MemoryStream stream; var docu = objhttpitem.GetHeaderParameter(); { if (response.Headers["set-cookie"] != null) { var cookieStr = response.Headers["set-cookie"]; //cookieStr = Regex.Replace(cookieStr, ",(?[^=;]=[^;];)", ";$1"); docu["Cookie"] = MergeCookie(docu["Cookie"].ToString(), cookieStr); } } httpResponse.ResponseHeaders = response.Headers; httpResponse.Code = response.StatusCode; objhttpitem.Parameters = HttpItem.HeaderToString(docu); //GZIIP处理 if (response.ContentEncoding != null && response.ContentEncoding.Equals("gzip", StringComparison.InvariantCultureIgnoreCase)) { stream = GetMemoryStream(new GZipStream(response.GetResponseStream(), CompressionMode.Decompress)); } else { stream = GetMemoryStream(response.GetResponseStream()); } //获取Byte var rawResponse = stream.ToArray(); //是否返回Byte类型数据 if (objhttpitem.Encoding == EncodingType.Unknown || encoding == null) { var temp = Encoding.Default.GetString(rawResponse, 0, rawResponse.Length); //<meta(.*?)charset([\s]?)=[^>](.*?)> var meta = Regex.Match(temp, "<meta([^<]*)charset=([^<]*)[\"']", RegexOptions.IgnoreCase | RegexOptions.Multiline); var charter = (meta.Groups.Count > 2) ? meta.Groups[2].Value : string.Empty; charter = charter.Replace("\"", string.Empty) .Replace("'", string.Empty) .Replace(";", string.Empty); if (charter.Length > 0) { charter = charter.ToLower().Replace("iso-8859-1", "gbk"); if (charter.Contains("utf-8") || charter.Contains("UTF-8")) { encoding = Encoding.UTF8; } else if (charter.Contains("gb")) { encoding = Encoding.GetEncoding("GB2312"); } else { encoding = Encoding.GetEncoding(charter); } } else { if (response.CharacterSet != null && response.CharacterSet.ToLower().Trim() == "iso-8859-1") { encoding = Encoding.GetEncoding("gbk"); } } } //得到返回的HTML httpResponse.Html = encoding.GetString(rawResponse); //最后释放流 stream.Close(); } return(httpResponse); }
public static void SetRequest(HttpItem item, HttpWebRequest request, string desturl = null, string post = null) { var docu = item.GetHeaderParameter(); // 设置代理 if (string.IsNullOrEmpty(item.ProxyIP)) { //不需要设置 } else { //设置代理服务器 string username, password, url; if (!string.IsNullOrWhiteSpace(item.ProxyIP)) { var myProxy = new WebProxy(item.ProxyIP, item.ProxyPort); //建议连接 if (!string.IsNullOrWhiteSpace(item.UserName)) { myProxy.Credentials = new NetworkCredential(item.UserName, item.Password); } //给当前请求对象 request.Proxy = myProxy; //设置安全凭证 request.Credentials = CredentialCache.DefaultNetworkCredentials; } } //请求方式Get或者Post request.Method = item.Method.ToString(); request.Timeout = item.Timeout; request.ReadWriteTimeout = item.ReadWriteTimeout; //Accept request.Headers = new WebHeaderCollection(); if (docu["Headers"].ToString() != "") { var str = docu["Headers"].ToString().Split('\n'); foreach (var s in str) { var ms = s.Split(':'); if (ms.Length != 2) { continue; } var key = ms[0].Trim(); var value = ms[1].Trim(); if (SetHeaderValue(request.Headers, key, value) == false) { request.Headers.Add(key, value); } } } request.Accept = docu["Accept"].ToString(); //ContentType返回类型 request.ContentType = docu["Content_Type"].ToString(); //UserAgent客户端的访问类型,包括浏览器版本和操作系统信息 request.UserAgent = docu["User-Agent"].ToString(); var host = docu["Host"].ToString(); //if (string.IsNullOrEmpty(host) == false) // request.Host = host; //设置Cookie var cookie = docu["Cookie"].ToString(); if (!string.IsNullOrEmpty(cookie)) { request.Headers[HttpRequestHeader.Cookie] = cookie; } //来源地址 request.Referer = docu["Referer"].ToString(); //是否执行跳转功能 request.AllowAutoRedirect = item.Allowautoredirect; //设置Post数据 string postdata = null; if (post == null) { postdata = item.Postdata; } else { postdata = post; } //验证在得到结果时是否有传入数据 if (!string.IsNullOrEmpty(postdata) && request.Method.Trim().ToLower().Contains("post")) { var buffer = Encoding.Default.GetBytes(postdata); request.ContentLength = buffer.Length; request.GetRequestStream().Write(buffer, 0, buffer.Length); } ////设置最大连接 //if (item.Connectionlimit > 0) //{ // request.ServicePoint.ConnectionLimit = item.Connectionlimit; //} }
public CookieAwareWebClient(HttpItem item) { Encoding = Encoding.UTF8; HttpItem = item; }