/// <summary>
/// Serializes the cookie collection to the stream.
/// </summary>
/// <param name="cookies">You can obtain the collection through your <see cref="CookieAwareWebClient">WebClient</see>'s <code>CookieContainer.GetCookies(Uri)</code>-method.</param>
/// <param name="address">The <see cref="Uri">Uri</see> that produced the cookies</param>
/// <param name="stream">The stream to which to serialize</param>
public static void Serialize(CookieCollection cookies, Uri address, Stream stream)
{
using (var writer = new StreamWriter(stream))
{
for (var enumerator = cookies.GetEnumerator(); enumerator.MoveNext();)
{
var cookie = enumerator.Current as Cookie;
if (cookie == null)
{
continue;
}
writer.WriteLine(address.AbsoluteUri);
writer.WriteLine(cookie.Comment);
writer.WriteLine(cookie.CommentUri == null ? null : cookie.CommentUri.AbsoluteUri);
writer.WriteLine(cookie.Discard);
writer.WriteLine(cookie.Domain);
writer.WriteLine(cookie.Expired);
writer.WriteLine(cookie.Expires);
writer.WriteLine(cookie.HttpOnly);
writer.WriteLine(cookie.Name);
writer.WriteLine(cookie.Path);
writer.WriteLine(cookie.Port);
writer.WriteLine(cookie.Secure);
writer.WriteLine(cookie.Value);
writer.WriteLine(cookie.Version);
}
}
}
public static CookieCollection SyncCookies(CookieCollection oldCookieCollection, CookieCollection addCookieCollection)
{
oldCookieCollection.Add(addCookieCollection);
CookieCollection cookieCollection = new CookieCollection();
try
{
IEnumerator enumerator = oldCookieCollection.GetEnumerator();
while (enumerator.MoveNext())
{
Cookie cookie = (Cookie)enumerator.Current;
if (DateTime.Compare(cookie.Expires, DateTime.MinValue) == 0 || !cookie.Expired)
{
cookieCollection.Add(cookie);
}
}
}
finally
{
IEnumerator enumerator = null;
if (enumerator is IDisposable)
{
(enumerator as IDisposable).Dispose();
}
}
return(cookieCollection);
}
public static void Enumerator_Version_Invalid()
{
CookieCollection cc = CreateCookieCollection1();
IEnumerator enumerator = cc.GetEnumerator();
enumerator.MoveNext();
cc.Add(new Cookie("name5", "value"));
Assert.NotNull(enumerator.Current);
Assert.Throws <InvalidOperationException>(() => enumerator.MoveNext()); // Enumerator out of sync
}
/// <summary>
/// Returns the session cookie string from the total available set of cookies
/// </summary>
public static string ReceiveSessionCookie(this CookieCollection responseCookies)
{
var enumerator = responseCookies.GetEnumerator();
string sessionCookie = string.Empty;
foreach (Cookie cookie in responseCookies)
{
if (cookie.Name.Equals(sessionCookieKey))
{
sessionCookie = cookie.Value;
}
}
return(sessionCookie);
}
public static void Enumerator_Index_Invalid()
{
CookieCollection cc = CreateCookieCollection1();
IEnumerator enumerator = cc.GetEnumerator();
Assert.Throws <InvalidOperationException>(() => enumerator.Current); // Index < 0
enumerator.MoveNext(); enumerator.MoveNext(); enumerator.MoveNext();
enumerator.MoveNext(); enumerator.MoveNext(); enumerator.MoveNext();
Assert.Throws <InvalidOperationException>(() => enumerator.Current); // Index >= count
enumerator.Reset();
Assert.Throws <InvalidOperationException>(() => enumerator.Current); // Index should be -1
}
public void Enumerator()
{
IEnumerator enumerator = col.GetEnumerator();
enumerator.MoveNext();
Cookie c = (Cookie)enumerator.Current;
Assert.AreEqual(c, col [0], "#1");
col.Add(new Cookie("name6", "value6"));
try {
enumerator.MoveNext();
Assert.Fail("#2");
} catch (InvalidOperationException) {
}
}
/// <summary>
/// Serialize cookie after aothorization
/// </summary>
/// <param name="cookies"></param>
/// <param name="address"></param>
/// <param name="stream"></param>
public static void Serialize(CookieCollection cookies, Uri address, Stream stream)
{
DataContractSerializer formatter = new DataContractSerializer(typeof(List <Cookie>));
List <Cookie> cookieList = new List <Cookie>();
for (var enumerator = cookies.GetEnumerator(); enumerator.MoveNext();)
{
var cookie = enumerator.Current as Cookie;
if (cookie == null)
{
continue;
}
cookieList.Add(cookie);
}
formatter.WriteObject(stream, cookieList);
}
public static IAsyncOperation <IDictionary <string, string> > GetCookies(string url, string token)
{
return(Task.Run <IDictionary <string, string> >(() =>
{
Uri spSiteUrl = new Uri(url, UriKind.Absolute);
var cookies = GetCookieContainer(spSiteUrl, token);
IDictionary <string, string> cookieArray = new Dictionary <string, string>();
CookieCollection collection = cookies.GetCookies(spSiteUrl);
IEnumerator enumr = collection.GetEnumerator();
while (enumr.MoveNext())
{
var Name = ((Cookie)enumr.Current).Name;
var value = ((Cookie)enumr.Current).Value;
cookieArray.Add(Name, value);
}
return cookieArray;
}).AsAsyncOperation());
}
internal static void Productivity(object o)
{
ArrayList balls = new ArrayList();
ha = ProductivityInfo.PrepareHttpAccesser(universe);
ha.Referer = referer;
string baseurl = ha.AccessUrl;
ha.Cookies = ccold;
IEnumerator i = ccnew.GetEnumerator();
while (i.MoveNext())
{
ha.Cookies.Add((Cookie)i.Current);
}
ccold = ha.Cookies;
ccnew = ha.access();
//取得第一个星球的总星球列表
pi = ProductivityInfo.AnalyzHtml();
foreach (ProductivityInfo.Ball ball in pi.Balllist)
{
ha.Referer = ha.AccessUrl;
ha.AccessUrl = baseurl + ball.AccessParm;
ha.Cookies = ccold;
IEnumerator ie = ccnew.GetEnumerator();
while (ie.MoveNext())
{
ha.Cookies.Add((Cookie)ie.Current);
}
ccnew = ha.access();
//取得每一个星球
ProductivityInfo piloop = ProductivityInfo.AnalyzHtml();
balls.Add(piloop);
MessageEventHandler(100 / pi.Balllist.Count);
}
Thread.Sleep(600);
referer = ha.AccessUrl;
ObjectEventHandler(pi);
Thread.Sleep(200);
ObjectEventHandler(balls);
Thread.Sleep(200);
}
public static void Enumerator_Version_Invalid()
{
CookieCollection cc = CreateCookieCollection1();
IEnumerator enumerator = cc.GetEnumerator();
enumerator.MoveNext();
cc.Add(new Cookie("name5", "value"));
object current = null;
var exception = Record.Exception(() => current = enumerator.Current);
// On full framework, enumerator.Current throws an exception because the collection has been modified after
// creating the enumerator.
if (exception == null)
{
Assert.NotNull(current);
}
Assert.Throws <InvalidOperationException>(() => enumerator.MoveNext()); // Enumerator out of sync
}
internal static void overview(object o)
{
ArrayList balls = new ArrayList();
ha = OverviewInfo.PrepareHttpAccesser(universe);
string baseurl = ha.AccessUrl;
ha.Cookies = ccold;
IEnumerator i = ccnew.GetEnumerator();
while (i.MoveNext())
{
ha.Cookies.Add((Cookie)i.Current);
}
ccold = ha.Cookies;
ccnew = ha.access();
//取得第一个星球的总星球列表
OverviewInfo ovf = OverviewInfo.AnalyzHtml();
foreach (OverviewInfo.Ball ball in ovf.Balllist)
{
ha.Referer = ha.AccessUrl;
ha.AccessUrl = baseurl + ball.AccessParm;
ha.Cookies = ccold;
IEnumerator ie = ccnew.GetEnumerator();
while (ie.MoveNext())
{
ha.Cookies.Add((Cookie)ie.Current);
}
ccnew = ha.access();
//取得每一个星球
OverviewInfo ovfloop = OverviewInfo.AnalyzHtml();
balls.Add(ovfloop);
}
referer = ha.AccessUrl;
ObjectEventHandler(ovf);
Thread.Sleep(200);
ObjectEventHandler(balls);
Thread.Sleep(200);
}
/// <inheritdoc />
public IEnumerator GetEnumerator()
{
return(_collection.GetEnumerator());
}
public async Task <RetObject> SendHttp(string uri, HttpMethod method = null, OrderedDictionary headers = null, CookieCollection cookies = null, string contentType = null, string body = null, string filepath = null)
{
byte[] reStream;
RetObject retObj = new RetObject();
HttpResponseMessage res = new HttpResponseMessage();
OrderedDictionary httpResponseHeaders = new OrderedDictionary();
CookieCollection responseCookies;
CookieCollection rCookies = new CookieCollection();
List <string> setCookieValue = new List <string>();
CookieContainer coo = new CookieContainer();
dynamic dom = new object();
string htmlString = String.Empty;
if (method == null)
{
method = HttpMethod.Get;
}
HttpClientHandler handle = new HttpClientHandler()
{
AutomaticDecompression = (DecompressionMethods)1 & (DecompressionMethods)2,
UseProxy = false,
AllowAutoRedirect = true,
MaxAutomaticRedirections = Int32.MaxValue,
MaxConnectionsPerServer = Int32.MaxValue,
MaxResponseHeadersLength = Int32.MaxValue,
SslProtocols = System.Security.Authentication.SslProtocols.Tls12
};
HttpClient client = new HttpClient(handle);
if (!client.DefaultRequestHeaders.Contains("User-Agent"))
{
client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36");
}
client.DefaultRequestHeaders.Add("Path", (new Uri(uri).PathAndQuery));
List <string> headersToSkip = new List <string>();
headersToSkip.Add("Accept");
headersToSkip.Add("pragma");
headersToSkip.Add("Cache-Control");
headersToSkip.Add("Date");
headersToSkip.Add("Content-Length");
headersToSkip.Add("Content-Type");
headersToSkip.Add("Expires");
headersToSkip.Add("Last-Modified");
if (headers != null)
{
headersToSkip.ForEach((i) => {
headers.Remove(i);
});
IEnumerator enume = headers.Keys.GetEnumerator();
while (enume.MoveNext())
{
string key = enume.Current.ToString();
string value = String.Join("\n", headers[key]);
if (client.DefaultRequestHeaders.Contains(key))
{
client.DefaultRequestHeaders.Remove(key);
}
try
{
client.DefaultRequestHeaders.Add(key, value);
}
catch
{
client.DefaultRequestHeaders.TryAddWithoutValidation(key, value);
}
}
}
if (cookies != null)
{
IEnumerator cnume = cookies.GetEnumerator();
while (cnume.MoveNext())
{
Cookie cook = (Cookie)cnume.Current;
coo.Add(cook);
}
handle.CookieContainer = coo;
}
bool except = false;
switch (method.ToString())
{
case "DELETE":
res = await client.SendAsync((new HttpRequestMessage(method, uri)));
if (res.Content.Headers.ContentEncoding.ToString().ToLower().Equals("gzip"))
{
reStream = res.Content.ReadAsByteArrayAsync().Result;
htmlString = Unzip(reStream);
}
else
{
htmlString = res.Content.ReadAsStringAsync().Result;
}
try
{
setCookieValue = res.Headers.GetValues("Set-Cookie").ToList();
}
catch
{ }
res.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
res.Content.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
responseCookies = handle.CookieContainer.GetCookies(new Uri(uri));
rCookies = SetCookieParser(setCookieValue, responseCookies, cookies);
if (!String.IsNullOrEmpty(htmlString))
{
dom = DOMParser(htmlString);
retObj.HtmlDocument = dom;
}
retObj.HttpResponseHeaders = httpResponseHeaders;
retObj.HttpResponseMessage = res;
break;
case "GET":
res = await client.SendAsync((new HttpRequestMessage(method, uri)));
if (res.Content.Headers.ContentEncoding.ToString().ToLower().Equals("gzip"))
{
reStream = res.Content.ReadAsByteArrayAsync().Result;
htmlString = Unzip(reStream);
}
else
{
try
{
htmlString = res.Content.ReadAsStringAsync().Result;
}
catch
{
except = true;
}
if (except)
{
var responseStream = await res.Content.ReadAsStreamAsync().ConfigureAwait(false);
using (var sr = new StreamReader(responseStream, Encoding.UTF8))
{
htmlString = await sr.ReadToEndAsync().ConfigureAwait(false);
}
}
}
try
{
setCookieValue = res.Headers.GetValues("Set-Cookie").ToList();
}
catch
{ }
res.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
res.Content.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
responseCookies = handle.CookieContainer.GetCookies(new Uri(uri));
rCookies = SetCookieParser(setCookieValue, responseCookies, cookies);
if (!String.IsNullOrEmpty(htmlString))
{
dom = DOMParser(htmlString);
retObj.HtmlDocument = dom;
}
retObj.HttpResponseHeaders = httpResponseHeaders;
retObj.HttpResponseMessage = res;
break;
case "HEAD":
res = await client.SendAsync((new HttpRequestMessage(method, uri)));
try
{
setCookieValue = res.Headers.GetValues("Set-Cookie").ToList();
}
catch
{ }
res.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
res.Content.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
responseCookies = handle.CookieContainer.GetCookies(new Uri(uri));
rCookies = SetCookieParser(setCookieValue, responseCookies, cookies);
retObj.HttpResponseHeaders = httpResponseHeaders;
retObj.HttpResponseMessage = res;
break;
case "OPTIONS":
res = await client.SendAsync((new HttpRequestMessage(method, uri)));
if (res.Content.Headers.ContentEncoding.ToString().ToLower().Equals("gzip"))
{
reStream = res.Content.ReadAsByteArrayAsync().Result;
htmlString = Unzip(reStream);
}
else
{
htmlString = res.Content.ReadAsStringAsync().Result;
}
try
{
setCookieValue = res.Headers.GetValues("Set-Cookie").ToList();
}
catch
{ }
res.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
res.Content.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
responseCookies = handle.CookieContainer.GetCookies(new Uri(uri));
rCookies = SetCookieParser(setCookieValue, responseCookies, cookies);
if (!String.IsNullOrEmpty(htmlString))
{
dom = DOMParser(htmlString);
retObj.HtmlDocument = dom;
}
retObj.HttpResponseHeaders = httpResponseHeaders;
retObj.HttpResponseMessage = res;
break;
case "POST":
if (String.IsNullOrEmpty(contentType))
{
contentType = "application/x-www-form-urlencoded";
}
if (!String.IsNullOrEmpty(body))
{
switch (contentType)
{
case @"application/x-www-form-urlencoded":
res = await client.SendAsync(
(new HttpRequestMessage(method, uri)
{
Content = (new StringContent(body, Encoding.UTF8, contentType))
})
);
break;
case @"multipart/form-data":
MultipartFormDataContent mpc = new MultipartFormDataContent("Boundary----" + DateTime.Now.Ticks.ToString("x"));
if (!String.IsNullOrEmpty(filepath))
{
if (File.Exists(filepath))
{
ByteArrayContent bac = new ByteArrayContent(File.ReadAllBytes(filepath));
bac.Headers.Add("Content-Type", MimeMapping.GetMimeMapping(filepath));
bac.Headers.ContentDisposition = ContentDispositionHeaderValue.Parse("attachment");
bac.Headers.ContentDisposition.Name = "file";
bac.Headers.ContentDisposition.FileName = new FileInfo(filepath).Name;
mpc.Add(bac, new FileInfo(filepath).Name);
}
}
if (!String.IsNullOrEmpty(body))
{
StringContent sc = new StringContent(body, Encoding.UTF8, @"application/x-www-form-urlencoded");
mpc.Add(sc);
}
res = await client.SendAsync(
(new HttpRequestMessage(method, uri)
{
Content = mpc
})
);
break;
default:
ServicePointManager.Expect100Continue = false;
res = await client.SendAsync(
(new HttpRequestMessage(method, uri)
{
Content = (new StringContent(body, Encoding.UTF8, contentType))
})
);
break;
}
if (res.Content.Headers.ContentEncoding.ToString().ToLower().Equals("gzip"))
{
reStream = res.Content.ReadAsByteArrayAsync().Result;
htmlString = Unzip(reStream);
}
else
{
try
{
htmlString = res.Content.ReadAsStringAsync().Result;
}
catch
{
except = true;
}
if (except)
{
var responseStream = await res.Content.ReadAsStreamAsync().ConfigureAwait(false);
using (var sr = new StreamReader(responseStream, Encoding.UTF8))
{
htmlString = await sr.ReadToEndAsync().ConfigureAwait(false);
}
}
}
try
{
setCookieValue = res.Headers.GetValues("Set-Cookie").ToList();
}
catch
{ }
res.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
res.Content.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
}
else
{
switch (contentType)
{
case @"application/x-www-form-urlencoded":
res = await client.SendAsync(
(new HttpRequestMessage(method, uri)
{
Content = (new StringContent(String.Empty, Encoding.UTF8, contentType))
})
);
break;
case @"multipart/form-data":
MultipartFormDataContent mpc = new MultipartFormDataContent("Boundary----" + DateTime.Now.Ticks.ToString("x"));
if (!String.IsNullOrEmpty(filepath))
{
if (File.Exists(filepath))
{
ByteArrayContent bac = new ByteArrayContent(File.ReadAllBytes(filepath));
bac.Headers.Add("Content-Type", MimeMapping.GetMimeMapping(filepath));
bac.Headers.ContentDisposition = ContentDispositionHeaderValue.Parse("attachment");
bac.Headers.ContentDisposition.Name = "file";
bac.Headers.ContentDisposition.FileName = new FileInfo(filepath).Name;
mpc.Add(bac, new FileInfo(filepath).Name);
}
}
res = await client.SendAsync(
(new HttpRequestMessage(method, uri)
{
Content = mpc
})
);
break;
default:
res = await client.SendAsync((new HttpRequestMessage(method, uri)));
break;
}
if (res.Content.Headers.ContentEncoding.ToString().ToLower().Equals("gzip"))
{
reStream = res.Content.ReadAsByteArrayAsync().Result;
htmlString = Unzip(reStream);
}
else
{
try
{
htmlString = res.Content.ReadAsStringAsync().Result;
}
catch
{
except = true;
}
if (except)
{
var responseStream = await res.Content.ReadAsStreamAsync().ConfigureAwait(false);
using (var sr = new StreamReader(responseStream, Encoding.UTF8))
{
htmlString = await sr.ReadToEndAsync().ConfigureAwait(false);
}
}
}
try
{
setCookieValue = res.Headers.GetValues("Set-Cookie").ToList();
}
catch
{ }
res.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
res.Content.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
}
responseCookies = handle.CookieContainer.GetCookies(new Uri(uri));
rCookies = SetCookieParser(setCookieValue, responseCookies, cookies);
if (!String.IsNullOrEmpty(htmlString))
{
dom = DOMParser(htmlString);
retObj.HtmlDocument = dom;
}
retObj.HttpResponseHeaders = httpResponseHeaders;
retObj.HttpResponseMessage = res;
break;
case "PUT":
if (String.IsNullOrEmpty(contentType))
{
contentType = "application/x-www-form-urlencoded";
}
if (!String.IsNullOrEmpty(body))
{
res = await client.SendAsync(
(new HttpRequestMessage(method, uri)
{
Content = (new StringContent(body, Encoding.UTF8, contentType))
})
);
if (res.Content.Headers.ContentEncoding.ToString().ToLower().Equals("gzip"))
{
reStream = res.Content.ReadAsByteArrayAsync().Result;
htmlString = Unzip(reStream);
}
else
{
htmlString = res.Content.ReadAsStringAsync().Result;
}
try
{
setCookieValue = res.Headers.GetValues("Set-Cookie").ToList();
}
catch
{ }
res.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
res.Content.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
}
else
{
res = await client.SendAsync((new HttpRequestMessage(method, uri)));
if (res.Content.Headers.ContentEncoding.ToString().ToLower().Equals("gzip"))
{
reStream = res.Content.ReadAsByteArrayAsync().Result;
htmlString = Unzip(reStream);
}
else
{
htmlString = res.Content.ReadAsStringAsync().Result;
}
try
{
setCookieValue = res.Headers.GetValues("Set-Cookie").ToList();
}
catch
{ }
res.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
res.Content.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
}
responseCookies = handle.CookieContainer.GetCookies(new Uri(uri));
rCookies = SetCookieParser(setCookieValue, responseCookies, cookies);
if (!String.IsNullOrEmpty(htmlString))
{
dom = DOMParser(htmlString);
retObj.HtmlDocument = dom;
}
retObj.HtmlDocument = dom;
retObj.HttpResponseHeaders = httpResponseHeaders;
retObj.HttpResponseMessage = res;
break;
case "TRACE":
res = await client.SendAsync((new HttpRequestMessage(method, uri)));
if (res.Content.Headers.ContentEncoding.ToString().ToLower().Equals("gzip"))
{
reStream = res.Content.ReadAsByteArrayAsync().Result;
htmlString = Unzip(reStream);
}
else
{
htmlString = res.Content.ReadAsStringAsync().Result;
}
try
{
setCookieValue = res.Headers.GetValues("Set-Cookie").ToList();
}
catch
{ }
res.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
res.Content.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
responseCookies = handle.CookieContainer.GetCookies(new Uri(uri));
rCookies = SetCookieParser(setCookieValue, responseCookies, cookies);
if (!String.IsNullOrEmpty(htmlString))
{
dom = DOMParser(htmlString);
retObj.HtmlDocument = dom;
}
retObj.HttpResponseHeaders = httpResponseHeaders;
retObj.HttpResponseMessage = res;
break;
}
if (!String.IsNullOrEmpty(htmlString))
{
retObj.ResponseText = htmlString;
}
retObj.CookieCollection = rCookies;
return(retObj);
}
/// <inheritdoc /> public IEnumerator GetEnumerator() => _collection.GetEnumerator();
/// <summary>
/// 去访问网站
/// </summary>
internal CookieCollection access()
{
this.responseText = string.Empty;
if (accessUrl == null || accessUrl.Trim().Equals(string.Empty))
{
throw new UncompleteSettingException("没有设定url");
}
//WebRequest r = WebRequest.Create("http://www.163.com");
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(accessUrl);
req.Headers = webHeader;
//host
req.Host = host;
req.UserAgent = userAgent;
req.Accept = accept;
req.Referer = referer;
req.ContentType = contentType;
//req.
if (!(connection.Equals("keep-alive") || connection.Equals("close")))
{
req.Connection = connection;
}
if (isUseCookie)
{
req.CookieContainer = this.cookies;
}
if (accessMethod == ACCESS_METHOD.GET)
{
req.Method = "GET";
}
else if (accessMethod == ACCESS_METHOD.POST)
{
//if (req.ContentType == null || req.ContentType.Equals(string.Empty))
//{
// throw new UncompleteSettingException("没有设定CONTENT_TYPE");
//}
req.Method = "POST";
byte[] bs = Encoding.GetEncoding(reqEncoding).GetBytes(urlParam);
req.ContentLength = bs.Length;
using (Stream reqStream = req.GetRequestStream())
{
reqStream.Write(bs, 0, bs.Length);
reqStream.Close();
}
}
using (WebResponse res = req.GetResponse())
{
this.responseDate = res.Headers.GetValues("Date")[0];
CookieCollection c = ((HttpWebResponse)res).Cookies;
IEnumerator i = c.GetEnumerator();
while (i.MoveNext())
{
Cookie ccc = (Cookie)i.Current;
}
Stream receiveStream = res.GetResponseStream();
Encoding encode = Encoding.GetEncoding(resEncoding);
StreamReader sr = new StreamReader(receiveStream, encode);
char[] readbuffer = new char[256];
int n = sr.Read(readbuffer, 0, 256);
while (n > 0)
{
string str = new string(readbuffer, 0, n);
responseText += str;
n = sr.Read(readbuffer, 0, 256);
}
sr.Close();
Txtout.write(responseText, "html.txt");
int r = rtime.Next(1000);
Thread.Sleep(3000 + r);
return(c);
}
}
private static async Task <RetObject> SendHttp(string uri, HttpMethod method = null, OrderedDictionary headers = null, CookieCollection cookies = null, string contentType = null, string body = null)
{
byte[] reStream;
RetObject retObj = new RetObject();
HttpResponseMessage res;
OrderedDictionary httpResponseHeaders = new OrderedDictionary();
CookieCollection responseCookies;
CookieCollection rCookies = new CookieCollection();
List <string> setCookieValue = new List <string>();
CookieContainer coo = new CookieContainer();
dynamic dom;
string htmlString = String.Empty;
if (method == null)
{
method = HttpMethod.Get;
}
HttpClientHandler handle = new HttpClientHandler()
{
AutomaticDecompression = (DecompressionMethods)1 & (DecompressionMethods)2,
UseProxy = false,
AllowAutoRedirect = true,
MaxAutomaticRedirections = 500
};
HttpClient client = new HttpClient(handle);
if (!client.DefaultRequestHeaders.Contains("User-Agent"))
{
client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36");
}
if (client.DefaultRequestHeaders.Contains("Path"))
{
client.DefaultRequestHeaders.Remove("Path");
}
client.DefaultRequestHeaders.Add("Path", (new Uri(uri).PathAndQuery));
List <string> headersToSkip = new List <string>();
headersToSkip.Add("Accept");
headersToSkip.Add("pragma");
headersToSkip.Add("Cache-Control");
headersToSkip.Add("Date");
headersToSkip.Add("Content-Length");
headersToSkip.Add("Content-Type");
headersToSkip.Add("Expires");
headersToSkip.Add("Last-Modified");
if (headers != null)
{
headersToSkip.ForEach((i) => {
headers.Remove(i);
});
IEnumerator enume = headers.Keys.GetEnumerator();
while (enume.MoveNext())
{
string key = enume.Current.ToString();
string value = String.Join("\n", headers[key]);
if (client.DefaultRequestHeaders.Contains(key))
{
client.DefaultRequestHeaders.Remove(key);
}
try
{
client.DefaultRequestHeaders.Add(key, value);
}
catch
{
client.DefaultRequestHeaders.TryAddWithoutValidation(key, value);
}
}
}
//client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("*/*"));
if (cookies != null)
{
IEnumerator cnume = cookies.GetEnumerator();
while (cnume.MoveNext())
{
Cookie cook = (Cookie)cnume.Current;
coo.Add(cook);
}
handle.CookieContainer = coo;
}
switch (method.ToString())
{
case "DELETE":
res = await client.SendAsync((new HttpRequestMessage(method, uri)));
if (res.Content.Headers.ContentEncoding.ToString().ToLower().Equals("gzip"))
{
reStream = res.Content.ReadAsByteArrayAsync().Result;
htmlString = Unzip(reStream);
}
else
{
htmlString = res.Content.ReadAsStringAsync().Result;
}
try
{
setCookieValue = res.Headers.GetValues("Set-Cookie").ToList();
}
catch
{ }
res.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
res.Content.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
responseCookies = handle.CookieContainer.GetCookies(new Uri(uri));
rCookies = SetCookieParser(setCookieValue, responseCookies, cookies);
dom = DOMParser(htmlString);
retObj.HtmlDocument = dom;
retObj.HttpResponseHeaders = httpResponseHeaders;
retObj.HttpResponseMessage = res;
break;
case "GET":
res = await client.SendAsync((new HttpRequestMessage(method, uri)));
if (res.Content.Headers.ContentEncoding.ToString().ToLower().Equals("gzip"))
{
reStream = res.Content.ReadAsByteArrayAsync().Result;
htmlString = Unzip(reStream);
}
else
{
htmlString = res.Content.ReadAsStringAsync().Result;
}
try
{
setCookieValue = res.Headers.GetValues("Set-Cookie").ToList();
}
catch
{ }
res.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
res.Content.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
responseCookies = handle.CookieContainer.GetCookies(new Uri(uri));
rCookies = SetCookieParser(setCookieValue, responseCookies, cookies);
dom = DOMParser(htmlString);
retObj.HtmlDocument = dom;
retObj.HttpResponseHeaders = httpResponseHeaders;
retObj.HttpResponseMessage = res;
break;
case "HEAD":
res = await client.SendAsync((new HttpRequestMessage(method, uri)));
try
{
setCookieValue = res.Headers.GetValues("Set-Cookie").ToList();
}
catch
{ }
res.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
res.Content.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
responseCookies = handle.CookieContainer.GetCookies(new Uri(uri));
rCookies = SetCookieParser(setCookieValue, responseCookies, cookies);
retObj.HttpResponseHeaders = httpResponseHeaders;
retObj.HttpResponseMessage = res;
break;
case "OPTIONS":
res = await client.SendAsync((new HttpRequestMessage(method, uri)));
if (res.Content.Headers.ContentEncoding.ToString().ToLower().Equals("gzip"))
{
reStream = res.Content.ReadAsByteArrayAsync().Result;
htmlString = Unzip(reStream);
}
else
{
htmlString = res.Content.ReadAsStringAsync().Result;
}
try
{
setCookieValue = res.Headers.GetValues("Set-Cookie").ToList();
}
catch
{ }
res.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
res.Content.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
responseCookies = handle.CookieContainer.GetCookies(new Uri(uri));
rCookies = SetCookieParser(setCookieValue, responseCookies, cookies);
dom = DOMParser(htmlString);
retObj.HtmlDocument = dom;
retObj.HttpResponseHeaders = httpResponseHeaders;
retObj.HttpResponseMessage = res;
break;
case "POST":
if (String.IsNullOrEmpty(contentType))
{
contentType = "application/x-www-form-urlencoded";
}
if (!String.IsNullOrEmpty(body))
{
res = await client.SendAsync(
(new HttpRequestMessage(method, uri)
{
Content = (new StringContent(body, Encoding.UTF8, contentType))
})
);
if (res.Content.Headers.ContentEncoding.ToString().ToLower().Equals("gzip"))
{
reStream = res.Content.ReadAsByteArrayAsync().Result;
htmlString = Unzip(reStream);
}
else
{
htmlString = res.Content.ReadAsStringAsync().Result;
}
try
{
setCookieValue = res.Headers.GetValues("Set-Cookie").ToList();
}
catch
{ }
res.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
res.Content.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
}
else
{
res = await client.SendAsync((new HttpRequestMessage(method, uri)));
if (res.Content.Headers.ContentEncoding.ToString().ToLower().Equals("gzip"))
{
reStream = res.Content.ReadAsByteArrayAsync().Result;
htmlString = Unzip(reStream);
}
else
{
htmlString = res.Content.ReadAsStringAsync().Result;
}
try
{
setCookieValue = res.Headers.GetValues("Set-Cookie").ToList();
}
catch
{ }
res.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
res.Content.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
}
responseCookies = handle.CookieContainer.GetCookies(new Uri(uri));
rCookies = SetCookieParser(setCookieValue, responseCookies, cookies);
dom = DOMParser(htmlString);
retObj.HtmlDocument = dom;
retObj.HttpResponseHeaders = httpResponseHeaders;
retObj.HttpResponseMessage = res;
break;
case "PUT":
if (String.IsNullOrEmpty(contentType))
{
contentType = "application/x-www-form-urlencoded";
}
if (!String.IsNullOrEmpty(body))
{
res = await client.SendAsync(
(new HttpRequestMessage(method, uri)
{
Content = (new StringContent(body, Encoding.UTF8, contentType))
})
);
if (res.Content.Headers.ContentEncoding.ToString().ToLower().Equals("gzip"))
{
reStream = res.Content.ReadAsByteArrayAsync().Result;
htmlString = Unzip(reStream);
}
else
{
htmlString = res.Content.ReadAsStringAsync().Result;
}
try
{
setCookieValue = res.Headers.GetValues("Set-Cookie").ToList();
}
catch
{ }
res.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
res.Content.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
}
else
{
res = await client.SendAsync((new HttpRequestMessage(method, uri)));
if (res.Content.Headers.ContentEncoding.ToString().ToLower().Equals("gzip"))
{
reStream = res.Content.ReadAsByteArrayAsync().Result;
htmlString = Unzip(reStream);
}
else
{
htmlString = res.Content.ReadAsStringAsync().Result;
}
try
{
setCookieValue = res.Headers.GetValues("Set-Cookie").ToList();
}
catch
{ }
res.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
res.Content.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
}
responseCookies = handle.CookieContainer.GetCookies(new Uri(uri));
rCookies = SetCookieParser(setCookieValue, responseCookies, cookies);
dom = DOMParser(htmlString);
retObj.HtmlDocument = dom;
retObj.HttpResponseHeaders = httpResponseHeaders;
retObj.HttpResponseMessage = res;
break;
case "TRACE":
res = await client.SendAsync((new HttpRequestMessage(method, uri)));
if (res.Content.Headers.ContentEncoding.ToString().ToLower().Equals("gzip"))
{
reStream = res.Content.ReadAsByteArrayAsync().Result;
htmlString = Unzip(reStream);
}
else
{
htmlString = res.Content.ReadAsStringAsync().Result;
}
try
{
setCookieValue = res.Headers.GetValues("Set-Cookie").ToList();
}
catch
{ }
res.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
res.Content.Headers.ToList().ForEach((i) =>
{
httpResponseHeaders.Add(i.Key, i.Value);
});
responseCookies = handle.CookieContainer.GetCookies(new Uri(uri));
rCookies = SetCookieParser(setCookieValue, responseCookies, cookies);
dom = DOMParser(htmlString);
retObj.HtmlDocument = dom;
retObj.HttpResponseHeaders = httpResponseHeaders;
retObj.HttpResponseMessage = res;
break;
}
if (!String.IsNullOrEmpty(htmlString))
{
retObj.ResponseText = htmlString;
}
retObj.CookieCollection = rCookies;
return(retObj);
}
