public static async Task <string> HttpClientGetStringAsync(string url, Encoding encoding = null, CookieCollection cookies = null)
{
if (cookies != null)
{
var cookiesStr = "";
foreach (System.Net.Cookie item in cookies)
{
cookiesStr += $"{item.Name}={item.Value};";
}
var message = new System.Net.Http.HttpRequestMessage(System.Net.Http.HttpMethod.Get, url);
message.Headers.Add("Cookie", cookiesStr);
await httpClient.SendAsync(message);
}
var source = "";
var buffer = await httpClient.GetByteArrayAsync(url);
if (encoding == null)
{
encoding = EncodingUtil.GetEncoding(buffer);
}
source = encoding.GetString(buffer);
return(source);
}
public static async Task <Tuple <string, CookieContainer> > GetHtmlSource(string url, CookieContainer cookieContainer, string accept = "", string userAgent = "", Encoding encoding = null)
{
//现在很多是共用的，直接复制过来了，以后再统一整理吧。
try
{
//不受信任的HTTPS
ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback((a, b, c, d) => { return(true); });
HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);
request.Method = "GET"; //默认就是GET
if (!string.IsNullOrEmpty(accept))
{
request.Accept = accept;
}
if (!string.IsNullOrEmpty(userAgent))
{
request.UserAgent = userAgent;
}
request.CookieContainer = cookieContainer;
using (WebResponse response = await request.GetResponseAsync())
{
Encoding tempEncoding = Encoding.Default;
if (encoding == null)
{
tempEncoding = EncodingUtil.GetEncoding(url);
}
else
{
tempEncoding = encoding;
}
Stream stream = response.GetResponseStream();
//GZIP流
if (((HttpWebResponse)response).ContentEncoding.ToLower().Contains("gzip"))
{
stream = new GZipStream(stream, CompressionMode.Decompress);
}
using (StreamReader sr = new StreamReader(stream, tempEncoding))
{
var html = sr.ReadToEnd();
CookieCollection responseCookieContainer = ((HttpWebResponse)response).Cookies;
stream.Close();
return(new Tuple <string, CookieContainer>(html, cookieContainer));
}
}
}
catch (Exception ex)
{
throw ex;
}
}
/// <summary>
///
/// </summary>
/// <param name="url"></param>
/// <param name="accept"></param>
/// <param name="userAgent"></param>
/// <param name="encoding"></param>
/// <remarks>以后再改</remarks>
/// <returns></returns>
public static async Task <string> GetHtmlSource(string url, string accept, string userAgent, Encoding encoding = null, CookieContainer cookieContainer = null)
{
try
{
//不受信任的HTTPS
ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback((a, b, c, d) => { return(true); });
HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);
request.Method = "GET"; //默认就是GET
request.Accept = accept;
request.UserAgent = userAgent;
if (cookieContainer != null)
{
request.CookieContainer = cookieContainer;
}
using (WebResponse response = await request.GetResponseAsync())
{
Encoding tempEncoding = Encoding.Default;
if (encoding == null)
{
tempEncoding = EncodingUtil.GetEncoding(url);
}
else
{
tempEncoding = encoding;
}
Stream stream = response.GetResponseStream();
//GZIP流
if (((HttpWebResponse)response).ContentEncoding.ToLower().Contains("gzip"))
{
stream = new GZipStream(stream, CompressionMode.Decompress);
}
using (StreamReader sr = new StreamReader(stream, tempEncoding))
{
return(sr.ReadToEnd());
}
}
}
catch (Exception ex)
{
throw ex;
}
}
public static string GetHtmlSourceWithSocket(string hostName, int port = 80)
{
Socket socket = ConnectSocket(hostName, port);
if (socket != null)
{
string request = "GET / HTTP/1.1\r\n" +
"Host:" + hostName + "\r\n" +
"Connection:Close\r\n\r\n";
byte[] sentBuffer = Encoding.ASCII.GetBytes(request);
byte[] receiveBuffer = new byte[1024];
socket.Send(sentBuffer);
int bufferLength = 0;
string htmlSource = "";
Encoding encoding = Encoding.Default;
int times = 0;
do
{
bufferLength = socket.Receive(receiveBuffer);
if (times == 0)
{
encoding = EncodingUtil.GetEncoding(receiveBuffer);
}
times++;
htmlSource += encoding.GetString(receiveBuffer, 0, bufferLength);
} while (bufferLength > 0);
return(htmlSource);
}
else
{
return("");
}
}
