public static async Task <string> HttpClientGetStringAsync(string url, Encoding encoding = null, CookieCollection cookies = null) { if (cookies != null) { var cookiesStr = ""; foreach (System.Net.Cookie item in cookies) { cookiesStr += $"{item.Name}={item.Value};"; } var message = new System.Net.Http.HttpRequestMessage(System.Net.Http.HttpMethod.Get, url); message.Headers.Add("Cookie", cookiesStr); await httpClient.SendAsync(message); } var source = ""; var buffer = await httpClient.GetByteArrayAsync(url); if (encoding == null) { encoding = EncodingUtil.GetEncoding(buffer); } source = encoding.GetString(buffer); return(source); }
public static async Task <Tuple <string, CookieContainer> > GetHtmlSource(string url, CookieContainer cookieContainer, string accept = "", string userAgent = "", Encoding encoding = null) { //现在很多是共用的,直接复制过来了,以后再统一整理吧。 try { //不受信任的HTTPS ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback((a, b, c, d) => { return(true); }); HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url); request.Method = "GET"; //默认就是GET if (!string.IsNullOrEmpty(accept)) { request.Accept = accept; } if (!string.IsNullOrEmpty(userAgent)) { request.UserAgent = userAgent; } request.CookieContainer = cookieContainer; using (WebResponse response = await request.GetResponseAsync()) { Encoding tempEncoding = Encoding.Default; if (encoding == null) { tempEncoding = EncodingUtil.GetEncoding(url); } else { tempEncoding = encoding; } Stream stream = response.GetResponseStream(); //GZIP流 if (((HttpWebResponse)response).ContentEncoding.ToLower().Contains("gzip")) { stream = new GZipStream(stream, CompressionMode.Decompress); } using (StreamReader sr = new StreamReader(stream, tempEncoding)) { var html = sr.ReadToEnd(); CookieCollection responseCookieContainer = ((HttpWebResponse)response).Cookies; stream.Close(); return(new Tuple <string, CookieContainer>(html, cookieContainer)); } } } catch (Exception ex) { throw ex; } }
/// <summary> /// /// </summary> /// <param name="url"></param> /// <param name="accept"></param> /// <param name="userAgent"></param> /// <param name="encoding"></param> /// <remarks>以后再改</remarks> /// <returns></returns> public static async Task <string> GetHtmlSource(string url, string accept, string userAgent, Encoding encoding = null, CookieContainer cookieContainer = null) { try { //不受信任的HTTPS ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback((a, b, c, d) => { return(true); }); HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url); request.Method = "GET"; //默认就是GET request.Accept = accept; request.UserAgent = userAgent; if (cookieContainer != null) { request.CookieContainer = cookieContainer; } using (WebResponse response = await request.GetResponseAsync()) { Encoding tempEncoding = Encoding.Default; if (encoding == null) { tempEncoding = EncodingUtil.GetEncoding(url); } else { tempEncoding = encoding; } Stream stream = response.GetResponseStream(); //GZIP流 if (((HttpWebResponse)response).ContentEncoding.ToLower().Contains("gzip")) { stream = new GZipStream(stream, CompressionMode.Decompress); } using (StreamReader sr = new StreamReader(stream, tempEncoding)) { return(sr.ReadToEnd()); } } } catch (Exception ex) { throw ex; } }
public static string GetHtmlSourceWithSocket(string hostName, int port = 80) { Socket socket = ConnectSocket(hostName, port); if (socket != null) { string request = "GET / HTTP/1.1\r\n" + "Host:" + hostName + "\r\n" + "Connection:Close\r\n\r\n"; byte[] sentBuffer = Encoding.ASCII.GetBytes(request); byte[] receiveBuffer = new byte[1024]; socket.Send(sentBuffer); int bufferLength = 0; string htmlSource = ""; Encoding encoding = Encoding.Default; int times = 0; do { bufferLength = socket.Receive(receiveBuffer); if (times == 0) { encoding = EncodingUtil.GetEncoding(receiveBuffer); } times++; htmlSource += encoding.GetString(receiveBuffer, 0, bufferLength); } while (bufferLength > 0); return(htmlSource); } else { return(""); } }