private static CookieContainer CreateCount11Container()
{
CookieContainer cc1 = new CookieContainer();
// Add(Cookie)
cc1.Add(c1);
cc1.Add(c2);
cc1.Add(c3);
cc1.Add(c4);
// Add(CookieCollection)
CookieCollection cc2 = new CookieCollection();
cc2.Add(c5);
cc2.Add(c6);
cc2.Add(c7);
cc1.Add(cc2);
// Add(Uri, Cookie)
cc1.Add(u4, c8);
cc1.Add(u4, c9);
// Add(Uri, CookieCollection)
cc2 = new CookieCollection();
cc2.Add(c10);
cc2.Add(c11);
cc1.Add(u5, cc2);
return cc1;
}
private static Cookie c5 = new Cookie("name3", "value", "path", "different-domain"); //Same name, different domain
private static CookieCollection CreateCookieCollection1()
{
CookieCollection cc = new CookieCollection();
cc.Add(c1);
cc.Add(c2);
cc.Add(c3);
cc.Add(c4);
cc.Add(c5);
return cc;
}
public static void AddCookieCollection_Success()
{
CookieContainer cc = new CookieContainer();
CookieCollection cookieCollection = new CookieCollection();
cookieCollection.Add(new Cookie("name3", "value","/",".contoso.com"));
cc.Add(cookieCollection);
Assert.Equal(1, cc.Count);
}
private static CookieCollection CreateCookieCollection2()
{
CookieCollection cc = new CookieCollection();
cc.Add(CreateCookieCollection1());
return cc;
}
internal HttpListenerWebSocketContext(
Uri requestUri,
NameValueCollection headers,
CookieCollection cookieCollection,
IPrincipal user,
bool isAuthenticated,
bool isLocal,
bool isSecureConnection,
string origin,
IEnumerable<string> secWebSocketProtocols,
string secWebSocketVersion,
string secWebSocketKey,
WebSocket webSocket)
{
Debug.Assert(requestUri != null, "requestUri shouldn't be null");
Debug.Assert(headers != null, "headers shouldn't be null");
Debug.Assert(cookieCollection != null, "cookieCollection shouldn't be null");
Debug.Assert(secWebSocketProtocols != null, "secWebSocketProtocols shouldn't be null");
Debug.Assert(webSocket != null, "webSocket shouldn't be null");
_cookieCollection = new CookieCollection();
_cookieCollection.Add(cookieCollection);
_headers = new NameValueCollection(headers);
_user = CopyPrincipal(user);
_requestUri = requestUri;
_isAuthenticated = isAuthenticated;
_isLocal = isLocal;
_isSecureConnection = isSecureConnection;
_origin = origin;
_secWebSocketProtocols = secWebSocketProtocols;
_secWebSocketVersion = secWebSocketVersion;
_secWebSocketKey = secWebSocketKey;
_webSocket = webSocket;
}
static CookieCollection ConvertCookieArraysToCookieCollection(ArrayList al, string strHost)
{
int intEachCookPartsCount;
string strCNameAndCValue;
string strPNameAndPValue;
string[] NameValuePairTemp;
Cookie cookTemp;
int firstEqual;
string firstName;
string allValue;
string strEachCook;
string[] strEachCookParts;
CookieCollection cc = new CookieCollection();
int alcount = al.Count;
for (int i = 0; i < alcount; i++)
{
strEachCook = al[i].ToString();
strEachCookParts = strEachCook.Split(';');
intEachCookPartsCount = strEachCookParts.Length;
cookTemp = new Cookie();
if (intEachCookPartsCount >= 1)
{
strCNameAndCValue = strEachCookParts[0];
if (!string.IsNullOrEmpty(strCNameAndCValue))
{
firstEqual = strCNameAndCValue.IndexOf("=");
firstName = strCNameAndCValue.Substring(0, firstEqual);
allValue = strCNameAndCValue.Substring(firstEqual + 1, strCNameAndCValue.Length - (firstEqual + 1));
cookTemp.Name = firstName;
cookTemp.Value = allValue;
}
}
for (int j = 1; j < intEachCookPartsCount; j++)
{
if (strEachCookParts[j].IndexOf("path", StringComparison.OrdinalIgnoreCase) >= 0)
{
strPNameAndPValue = strEachCookParts[j];
if (!string.IsNullOrEmpty(strPNameAndPValue))
{
NameValuePairTemp = strPNameAndPValue.Split('=');
if (NameValuePairTemp[1] != string.Empty)
{
cookTemp.Path = NameValuePairTemp[1];
}
else
{
cookTemp.Path = "/";
}
}
}
else if (strEachCookParts[j].IndexOf("domain", StringComparison.OrdinalIgnoreCase) >= 0)
{
strPNameAndPValue = strEachCookParts[j];
if (!string.IsNullOrEmpty(strPNameAndPValue))
{
NameValuePairTemp = strPNameAndPValue.Split('=');
if (!string.IsNullOrEmpty(NameValuePairTemp[1]))
{
cookTemp.Domain = NameValuePairTemp[1];
}
else
{
cookTemp.Domain = strHost;
}
}
}
}
if (string.IsNullOrEmpty(cookTemp.Path))
{
cookTemp.Path = "/";
}
if (string.IsNullOrEmpty(cookTemp.Domain))
{
cookTemp.Domain = strHost;
}
cc.Add(cookTemp);
}
return(cc);
}
/// <summary>
/// 填充头
/// </summary>
/// <param name="request"></param>
/// <param name="headers"></param>
private void fillHeaders(HttpWebRequest request, string headers, bool isPrint = false)
{
if (request == null)
{
return;
}
if (string.IsNullOrWhiteSpace(headers))
{
return;
}
string[] hsplit = headers.Split(new String[] { "\r\n" }, StringSplitOptions.RemoveEmptyEntries);
foreach (string item in hsplit)
{
if (string.IsNullOrEmpty(item.Trim()))
{
continue;
}
string[] kv = item.Split(':');
string key = kv[0].Trim();
string value = string.Join(":", kv.Skip(1)).Trim();
if (!UNCHANGEHEADS.Contains(key))
{
request.Headers.Add(key, value);
}
else
{
#region 设置http头
switch (key)
{
case "Accept":
{
request.Accept = value;
break;
}
case "Host":
{
request.Host = value;
break;
}
case "Connection":
{
if (value == "keep-alive")
{
request.KeepAlive = true;
}
else
{
request.KeepAlive = false; //just test
}
break;
}
case "Content-Type":
{
request.ContentType = value;
break;
}
case "User-Agent":
{
request.UserAgent = value;
break;
}
case "Referer":
{
request.Referer = value;
break;
}
case "Content-Length":
{
request.ContentLength = Convert.ToInt64(value);
break;
}
case "Expect":
{
request.Expect = value;
break;
}
case "If-Modified-Since":
{
request.IfModifiedSince = Convert.ToDateTime(value);
break;
}
default:
break;
}
#endregion
}
}
if (string.IsNullOrEmpty(request.Headers["Host"]))
{
request.Host = request.Address.Host;
}
CookieCollection cc = new CookieCollection();
string cookieString = request.Headers[HttpRequestHeader.Cookie];
if (!string.IsNullOrWhiteSpace(cookieString))
{
var spilit = cookieString.Split(';');
foreach (string item in spilit)
{
var kv = item.Split('=');
if (kv.Length == 2)
{
cc.Add(new Cookie(kv[0].Trim(), kv[1].Trim()));
}
}
}
trackCookies(cc);
if (!isTrackCookies)
{
request.Headers[HttpRequestHeader.Cookie] = "";
}
else
{
request.Headers[HttpRequestHeader.Cookie] = getCookieStr();
/*
* CookieContainer cookContner = new CookieContainer();
* cookContner.Add(cookieCt);
* request.CookieContainer = cookContner;
*/
}
#region 打印头
if (isPrint)
{
for (int i = 0; i < request.Headers.AllKeys.Length; i++)
{
string key = request.Headers.AllKeys[i];
System.Console.WriteLine(key + ":" + request.Headers[key]);
}
}
#endregion
}
/// <inheritdoc />
public void Add(Cookie cookie)
{
_collection.Add(cookie);
}
/// <summary>
/// Return a HTML document from a Telerik raw request.
/// Useful to copy real interaction.
/// Will also chain through post responses, result HTML document is the final response.
/// </summary>
/// <param name="iRequest"></param>
/// <param name="iCookies"></param>
/// <param name="iSpecificCOntent"></param>
/// <returns></returns>
public static string GetContentFromRequest(string iRequest, CookieCollection iCookies, ref CookieCollection oCookies, ref HttpStatusCode oStatus, string iSpecificCOntent = null, Encoding iDefaultEncoding = null)
{
List <string> listRequestArguments = iRequest.Split(new string[] { "|" }, StringSplitOptions.RemoveEmptyEntries).ToList();
if (listRequestArguments.Count <= 0)
{
return(null);
}
List <string> urlSplit = listRequestArguments[0].Split(' ').ToList();
if (urlSplit.Count != 3)
{
return(null);
}
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(urlSplit[1]);
request.Method = urlSplit[0];
//request.ProtocolVersion = HttpVersion.Version10;
var sp = request.ServicePoint;
var prop = sp.GetType().GetProperty("HttpBehaviour",
BindingFlags.Instance | BindingFlags.NonPublic);
prop.SetValue(sp, (byte)0, null);
//Telerik Arguments
string telerikCookies = "";
for (int i = 1; i < listRequestArguments.Count; i++)
{
string argument = listRequestArguments[i];
if (!argument.Contains(":"))
{
continue;
}
string key = argument.Split(':')[0].Trim();
string value = argument.Split(':')[1].Trim();
if (key == "Connection")
{
if (value.ToLower() == "keep-alive")
{
request.KeepAlive = false;
}
else
{
request.KeepAlive = false;
}
}
else if (key == "Accept")
{
request.Accept = value;
}
else if (key == "Referer")
{
request.Referer = argument.Replace(key + ":", "");
}
else if (key == "User-Agent")
{
request.UserAgent = value;
}
else if (key == "Content-Type")
{
request.ContentType = value;
}
else if (key == "Cookie")
{
telerikCookies = value;
}
else if (key == "Content-Length")
{
continue; //Recomputed, see content section
}
else if (key == "Host")
{
request.Host = value;
}
else
{
request.Headers.Add(argument);
}
}
//Uncompressed
request.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
request.ServicePoint.Expect100Continue = false;
//Cookies
request.CookieContainer = new CookieContainer();
if (telerikCookies != "")
{
string fullCookieList = "Cookie:";
List <string> telerikCookieList = telerikCookies.Split(';').ToList();
CookieCollection existingCookies = new CookieCollection();
foreach (string telerikCookie in telerikCookieList)
{
string cookieName = telerikCookie.Split('=')[0].Trim();
string cookieValue = telerikCookie.Split('=')[1].Trim();
Cookie correspondingCookie = null;
foreach (Cookie cookie in iCookies)
{
if (cookie.Name == cookieName)
{
correspondingCookie = cookie;
break;
}
}
if (null == correspondingCookie)
{
correspondingCookie = new Cookie(cookieName, cookieValue, "/", ExtractDomainFromURL(urlSplit[1]));
}
fullCookieList += correspondingCookie.Name + "=" + correspondingCookie.Value + ";";
request.CookieContainer.Add(correspondingCookie);
existingCookies.Add(correspondingCookie);
}
foreach (Cookie cookie in iCookies)
{
Cookie correspondingCookie = null;
foreach (Cookie existingCookie in existingCookies)
{
if (cookie.Name == existingCookie.Name)
{
correspondingCookie = existingCookie;
break;
}
}
if (correspondingCookie != null)
{
continue; //Cookie has been added already
}
fullCookieList += cookie.Name + "=" + cookie.Value + ";";
try
{
request.CookieContainer.Add(cookie);
existingCookies.Add(cookie);
}
catch
{
}
}
request.Headers.Add(fullCookieList);
}
else
{
string fullCookieList = "Cookie:";
foreach (Cookie cookie in iCookies)
{
fullCookieList += cookie.Name + "=" + cookie.Value + ";";
try
{
request.CookieContainer.Add(cookie);
}
catch
{
}
}
request.Headers.Add(fullCookieList);
}
//Content
if (urlSplit[0] == "POST")
{
//Content is in telerik request ?
string content = listRequestArguments[listRequestArguments.Count - 1];
if (content.Contains(":"))
{
content = "";
}
if (null != iSpecificCOntent && iSpecificCOntent != "")
{
content = iSpecificCOntent;
}
UTF8Encoding encoding = new UTF8Encoding();
byte[] bytes = encoding.GetBytes(content);
request.ContentLength = bytes.Length;
using (var stream = request.GetRequestStream())
{
stream.Write(bytes, 0, bytes.Length);
stream.Close();
}
}
//Avoid Redirect loop
if (request.Method == "POST")
{
request.AllowAutoRedirect = false;
}
//Start request
HttpWebResponse response;
try
{
response = (HttpWebResponse)request.GetResponse();
}
catch (WebException exception)
{
response = (HttpWebResponse)exception.Response;
}
if (null != response)
{
foreach (Cookie cookie in response.Cookies)
{
Cookie existingCookie = null;
foreach (Cookie otherCookie in oCookies)
{
if (cookie.Name == otherCookie.Name)
{
existingCookie = otherCookie;
break;
}
}
if (null == existingCookie)
{
oCookies.Add(cookie);
}
else
{
existingCookie.Value = cookie.Value;
}
}
oStatus = response.StatusCode;
var resultStream = response.GetResponseStream();
string result = "";
Encoding encoding = Encoding.Default;
if (null != iDefaultEncoding)
{
encoding = iDefaultEncoding;
}
using (StreamReader reader = new StreamReader(resultStream, encoding))
{
result = reader.ReadToEnd();
}
string setCookies = response.Headers[HttpResponseHeader.SetCookie];
string url = response.Headers[HttpResponseHeader.Location];
if (null != url && !url.Contains("www.") && !url.Contains("http"))
{
url = urlSplit[1].Split(':')[0] + "://" + request.Host + url;
}
response.Close();
//Adding cookies from setcookie
if (null != setCookies && setCookies != "")
{
if (setCookies.Contains(Environment.NewLine))
{
List <string> setCookieList = setCookies.Split(';').ToList();
foreach (string setCookieItem in setCookieList)
{
string cookieName = setCookieItem.Split('=')[0].Trim();
string cookieValue = "";
if (setCookieItem.Split('=').Count() > 1)
{
cookieValue = setCookieItem.Split('=')[1].Trim();
}
bool exist = false;
try
{
Cookie correspondingCookie = new Cookie(cookieName, cookieValue, "/", ExtractDomainFromURL(urlSplit[1]));
for (int i = 0; i < oCookies.Count; i++)
{
Cookie existingCookie = oCookies[i];
if (null != existingCookie && existingCookie.Name == cookieName)
{
exist = true;
break;
}
}
if (exist)
{
continue;
}
oCookies.Add(correspondingCookie);
}
catch
{
}
}
}
else
{
}
}
while (oStatus == HttpStatusCode.Found || oStatus == HttpStatusCode.Moved)
{
string requestLoop = "GET " + url + " HTTP/1.1" + iRequest.Replace(listRequestArguments[0], "");
CookieCollection cookiesLoop = new CookieCollection();
cookiesLoop.Add(iCookies);
cookiesLoop.Add(oCookies);
result = GetContentFromRequest(requestLoop, cookiesLoop, ref oCookies, ref oStatus, null, iDefaultEncoding);
}
return(result);
}
return(null);
}
internal void AddHeader(string header)
{
int colon = header.IndexOf(':');
if (colon == -1 || colon == 0)
{
context.ErrorMessage = "Bad Request";
return;
}
string name = header.Substring(0, colon).Trim();
string val = header.Substring(colon + 1).Trim();
string lower = name.ToLower(CultureInfo.InvariantCulture);
headers.SetInternal(name, val);
switch (lower)
{
case "accept-language":
user_languages = val.Split(','); // yes, only split with a ','
break;
case "accept":
accept_types = val.Split(','); // yes, only split with a ','
break;
case "content-length":
try {
//TODO: max. content_length?
content_length = Int64.Parse(val.Trim());
if (content_length < 0)
{
context.ErrorMessage = "Invalid Content-Length.";
}
cl_set = true;
} catch {
context.ErrorMessage = "Invalid Content-Length.";
}
break;
case "referer":
try {
referrer = new Uri(val);
} catch {
referrer = new Uri("http://someone.is.screwing.with.the.headers.com/");
}
break;
case "cookie":
if (cookies == null)
{
cookies = new CookieCollection();
}
string[] cookieStrings = val.Split(new char[] { ',', ';' });
Cookie current = null;
int version = 0;
foreach (string cookieString in cookieStrings)
{
string str = cookieString.Trim();
if (str.Length == 0)
{
continue;
}
if (str.StartsWith("$Version"))
{
version = Int32.Parse(Unquote(str.Substring(str.IndexOf("=") + 1)));
}
else if (str.StartsWith("$Path"))
{
if (current != null)
{
current.Path = str.Substring(str.IndexOf("=") + 1).Trim();
}
}
else if (str.StartsWith("$Domain"))
{
if (current != null)
{
current.Domain = str.Substring(str.IndexOf("=") + 1).Trim();
}
}
else if (str.StartsWith("$Port"))
{
if (current != null)
{
current.Port = str.Substring(str.IndexOf("=") + 1).Trim();
}
}
else
{
if (current != null)
{
cookies.Add(current);
}
current = new Cookie();
int idx = str.IndexOf("=");
if (idx > 0)
{
current.Name = str.Substring(0, idx).Trim();
current.Value = str.Substring(idx + 1).Trim();
}
else
{
current.Name = str.Trim();
current.Value = String.Empty;
}
current.Version = version;
}
}
if (current != null)
{
cookies.Add(current);
}
break;
}
}
// parse the Set-Cookie string (in http response header) to cookies
// Note: auto omit to parse the abnormal cookie string
// normal example for 'setCookieStr':
// MSPOK= ; expires=Thu, 30-Oct-1980 16:00:00 GMT;domain=login.live.com;path=/;HTTPOnly= ;version=1,PPAuth=Cuyf3Vp2wolkjba!TOr*0v22UMYz36ReuiwxZZBc8umHJYPlRe4qupywVFFcIpbJyvYZ5ZDLBwV4zRM1UCjXC4tUwNuKvh21iz6gQb0Tu5K7Z62!TYGfowB9VQpGA8esZ7iCRucC7d5LiP3ZAv*j4Z3MOecaJwmPHx7!wDFdAMuQUZURhHuZWJiLzHP1j8ppchB2LExnlHO6IGAdZo1f0qzSWsZ2hq*yYP6sdy*FdTTKo336Q1B0i5q8jUg1Yv6c2FoBiNxhZSzxpuU0WrNHqSytutP2k4!wNc6eSnFDeouX; domain=login.live.com;secure= ;path=/;HTTPOnly= ;version=1,PPLState=1; domain=.live.com;path=/;version=1,MSPShared=1; expires=Wed, 30-Dec-2037 16:00:00 GMT;domain=login.live.com;path=/;HTTPOnly= ;version=1,MSPPre= ;domain=login.live.com;path=/;Expires=Thu, 30-Oct-1980 16:00:00 GMT,MSPCID= ; HTTPOnly= ; domain=login.live.com;path=/;Expires=Thu, 30-Oct-1980 16:00:00 GMT,RPSTAuth=EwDoARAnAAAUWkziSC7RbDJKS1VkhugDegv7L0eAAOfCAY2+pKwbV5zUlu3XmBbgrQ8EdakmdSqK9OIKfMzAbnU8fuwwEi+FKtdGSuz/FpCYutqiHWdftd0YF21US7+1bPxuLJ0MO+wVXB8GtjLKZaA0xCXlU5u01r+DOsxSVM777DmplaUc0Q4O1+Pi9gX9cyzQLAgRKmC/QtlbVNKDA2YAAAhIwqiXOVR/DDgBocoO/n0u48RFGh79X2Q+gO4Fl5GMc9Vtpa7SUJjZCCfoaitOmcxhEjlVmR/2ppdfJx3Ykek9OFzFd+ijtn7K629yrVFt3O9q5L0lWoxfDh5/daLK7lqJGKxn1KvOew0SHlOqxuuhYRW57ezFyicxkxSI3aLxYFiqHSu9pq+TlITqiflyfcAcw4MWpvHxm9on8Y1dM2R4X3sxuwrLQBpvNsG4oIaldTYIhMEnKhmxrP6ZswxzteNqIRvMEKsxiksBzQDDK/Cnm6QYBZNsPawc6aAedZioeYwaV3Z/i3tNrAUwYTqLXve8oG6ZNXL6WLT/irKq1EMilK6Cw8lT3G13WYdk/U9a6YZPJC8LdqR0vAHYpsu/xRF39/On+xDNPE4keIThJBptweOeWQfsMDwvgrYnMBKAMjpLZwE=; domain=.live.com;path=/;HTTPOnly= ;version=1,RPSTAuthTime=1328679636; domain=login.live.com;path=/;HTTPOnly= ;version=1,MSPAuth=2OlAAMHXtDIFOtpaK1afG2n*AAxdfCnCBlJFn*gCF8gLnCa1YgXEfyVh2m9nZuF*M7npEwb4a7Erpb*!nH5G285k7AswJOrsr*gY29AVAbsiz2UscjIGHkXiKrTvIzkV2M; domain=.live.com;path=/;HTTPOnly= ;version=1,MSPProf=23ci9sti6DZRrkDXfTt1b3lHhMdheWIcTZU2zdJS9!zCloHzMKwX30MfEAcCyOjVt*5WeFSK3l2ZahtEaK7HPFMm3INMs3r!JxI8odP9PYRHivop5ryohtMYzWZzj3gVVurcEr5Bg6eJJws7rXOggo3cR4FuKLtXwz*FVX0VWuB5*aJhRkCT1GZn*L5Pxzsm9X; domain=.live.com;path=/;HTTPOnly= ;version=1,MSNPPAuth=CiGSMoUOx4gej8yQkdFBvN!gvffvAhCPeWydcrAbcg!O2lrhVb4gruWSX5NZCBPsyrtZKmHLhRLTUUIxxPA7LIhqW5TCV*YcInlG2f5hBzwzHt!PORYbg79nCkvw65LKG399gRGtJ4wvXdNlhHNldkBK1jVXD4PoqO1Xzdcpv4sj68U6!oGrNK5KgRSMXXpLJmCeehUcsRW1NmInqQXpyanjykpYOcZy0vq!6PIxkj3gMaAvm!1vO58gXM9HX9dA0GloNmCDnRv4qWDV2XKqEKp!A7jiIMWTmHup1DZ!*YCtDX3nUVQ1zAYSMjHmmbMDxRJECz!1XEwm070w16Y40TzuKAJVugo!pyF!V2OaCsLjZ9tdGxGwEQRyi0oWc*Z7M0FBn8Fz0Dh4DhCzl1NnGun9kOYjK5itrF1Wh17sT!62ipv1vI8omeu0cVRww2Kv!qM*LFgwGlPOnNHj3*VulQOuaoliN4MUUxTA4owDubYZoKAwF*yp7Mg3zq5Ds2!l9Q$$; domain=.live.com;path=/;HTTPOnly= ;version=1,MH=MSFT; domain=.live.com;path=/;version=1,MHW=; expires=Thu, 30-Oct-1980 16:00:00 GMT;domain=.live.com;path=/;version=1,MHList=; expires=Thu, 30-Oct-1980 16:00:00 GMT;domain=.live.com;path=/;version=1,NAP=V=1.9&E=bea&C=zfjCKKBD0TqjZlWGgRTp__NiK08Lme_0XFaiKPaWJ0HDuMi2uCXafQ&W=1;domain=.live.com;path=/,ANON=A=DE389D4D076BF47BCAE4DC05FFFFFFFF&E=c44&W=1;domain=.live.com;path=/,MSPVis=$9;domain=login.live.com;path=/,pres=; expires=Thu, 30-Oct-1980 16:00:00 GMT;domain=.live.com;path=/;version=1,LOpt=0; domain=login.live.com;path=/;version=1,WLSSC=EgBnAQMAAAAEgAAACoAASfCD+8dUptvK4kvFO0gS3mVG28SPT3Jo9Pz2k65r9c9KrN4ISvidiEhxXaPLCSpkfa6fxH3FbdP9UmWAa9KnzKFJu/lQNkZC3rzzMcVUMjbLUpSVVyscJHcfSXmpGGgZK4ZCxPqXaIl9EZ0xWackE4k5zWugX7GR5m/RzakyVIzWAFwA1gD9vwYA7Vazl9QKMk/UCjJPECcAAAoQoAAAFwBjcmlmYW4yMDAzQGhvdG1haWwuY29tAE8AABZjcmlmYW4yMDAzQGhvdG1haWwuY29tAAAACUNOAAYyMTM1OTIAAAZlCAQCAAB3F21AAARDAAR0aWFuAAR3YW5nBMgAAUkAAAAAAAAAAAAAAaOKNpqLi/UAANQKMk/Uf0RPAAAAAAAAAAAAAAAADgA1OC4yNDAuMjM2LjE5AAUAAAAAAAAAAAAAAAABBAABAAABAAABAAAAAAAAAAA=; domain=.live.com;secure= ;path=/;HTTPOnly= ;version=1,MSPSoftVis=@72198325083833620@:@; domain=login.live.com;path=/;version=1
// here now support parse the un-correct Set-Cookie:
// MSPRequ=/;Version=1;version<=1328770452&id=250915&co=1; path=/;version=1,MSPVis=$9; Version=1;version=1$250915;domain=login.live.com;path=/,MSPSoftVis=@72198325083833620@:@; domain=login.live.com;path=/;version=1,MSPBack=1328770312; domain=login.live.com;path=/;version=1
public CookieCollection parseSetCookie(string setCookieStr, string curDomain)
{
CookieCollection parsedCookies = new CookieCollection();
// process for expires and Expires field, for it contains ','
//refer: http://www.yaosansi.com/post/682.html
// may contains expires or Expires, so following use xpires
string commaReplaced = Regex.Replace(setCookieStr, @"xpires=\w{3},\s\d{2}-\w{3}-\d{4}", new MatchEvaluator(_processExpireField));
string[] cookieStrArr = commaReplaced.Split(',');
foreach (string cookieStr in cookieStrArr)
{
Cookie ck = new Cookie();
// recover it back
string recoveredCookieStr = Regex.Replace(cookieStr, @"xpires=\w{3}" + replacedChar + @"\s\d{2}-\w{3}-\d{4}", new MatchEvaluator(_recoverExpireField));
if (parseSingleCookie(recoveredCookieStr, ref ck))
{
if (needAddThisCookie(ck, curDomain))
{
parsedCookies.Add(ck);
}
}
}
return parsedCookies;
}
/// <summary>
/// See interface docs.
/// </summary>
/// <param name="cookie"></param>
public void SetCookie(Cookie cookie)
{
Cookies.Add(cookie);
}
/// <summary>
/// 根据response中头部的set-cookie对request中的cookie进行设置
/// </summary>
/// <param name="setCookie">The set cookie.</param>
/// <param name="defaultDomain">The default domain.</param>
/// <returns></returns>
private static CookieCollection SetCookie(HttpWebResponse response, string defaultDomain)
{
try
{
string[] setCookie = response.Headers.GetValues("Set-Cookie");
// there is bug in it,the datetime in "set-cookie" will be sepreated in two pieces.
List <string> a = new List <string>(setCookie);
for (int i = setCookie.Length - 1; i > 0; i--)
{
if (a[i].Substring(a[i].Length - 3) == "GMT")
{
a[i - 1] = a[i - 1] + ", " + a[i];
a.RemoveAt(i);
i--;
}
}
setCookie = a.ToArray <string>();
CookieCollection cookies = new CookieCollection();
foreach (string str in setCookie)
{
NameValueCollection hs = new NameValueCollection();
foreach (string i in str.Split(';'))
{
int index = i.IndexOf("=");
if (index > 0)
{
hs.Add(i.Substring(0, index).Trim(), i.Substring(index + 1).Trim());
}
else
{
switch (i)
{
case "HttpOnly":
hs.Add("HttpOnly", "True");
break;
case "Secure":
hs.Add("Secure", "True");
break;
}
}
}
Cookie ck = new Cookie();
foreach (string Key in hs.AllKeys)
{
switch (Key.ToLower().Trim())
{
case "path":
ck.Path = hs[Key];
break;
case "expires":
ck.Expires = DateTime.Parse(hs[Key]);
break;
case "domain":
ck.Domain = hs[Key];
break;
case "httpOnly":
ck.HttpOnly = true;
break;
case "secure":
ck.Secure = true;
break;
default:
ck.Name = Key;
ck.Value = hs[Key];
break;
}
}
if (ck.Domain == "")
{
ck.Domain = defaultDomain;
}
if (ck.Name != "")
{
cookies.Add(ck);
}
}
return(cookies);
}
catch
{
return(null);
}
}
private static CookieCollection ConvertCookieArraysToCookieCollection(ArrayList al, string strHost)
{
CookieCollection cc = new CookieCollection();
int alcount = al.Count;
string strEachCook;
string[] strEachCookParts;
for (int i = 0; i < alcount; i++)
{
strEachCook = al[i].ToString();
strEachCookParts = strEachCook.Split(';');
int intEachCookPartsCount = strEachCookParts.Length;
string strCNameAndCValue = string.Empty;
string strPNameAndPValue = string.Empty;
string strDNameAndDValue = string.Empty;
string[] NameValuePairTemp;
Cookie cookTemp = new Cookie();
for (int j = 0; j < intEachCookPartsCount; j++)
{
if (j == 0)
{
strCNameAndCValue = strEachCookParts[j];
if (strCNameAndCValue != string.Empty)
{
int firstEqual = strCNameAndCValue.IndexOf("=");
string firstName = strCNameAndCValue.Substring(1, firstEqual - 1);
string allValue = strCNameAndCValue.Substring(firstEqual + 1, strCNameAndCValue.Length - (firstEqual + 1));
// Debug.WriteLine("\"" + firstName + "\"");
cookTemp.Name = firstName;
cookTemp.Value = allValue;
}
continue;
}
if (strEachCookParts[j].IndexOf("path", StringComparison.OrdinalIgnoreCase) >= 0)
{
strPNameAndPValue = strEachCookParts[j];
if (strPNameAndPValue != string.Empty)
{
NameValuePairTemp = strPNameAndPValue.Split('=');
if (NameValuePairTemp[1] != string.Empty)
{
cookTemp.Path = NameValuePairTemp[1];
}
else
{
cookTemp.Path = "/";
}
}
continue;
}
if (strEachCookParts[j].IndexOf("domain", StringComparison.OrdinalIgnoreCase) >= 0)
{
strPNameAndPValue = strEachCookParts[j];
if (strPNameAndPValue != string.Empty)
{
NameValuePairTemp = strPNameAndPValue.Split('=');
if (NameValuePairTemp[1] != string.Empty)
{
cookTemp.Domain = NameValuePairTemp[1];
}
else
{
cookTemp.Domain = strHost;
}
}
continue;
}
}
if (cookTemp.Path == string.Empty)
{
cookTemp.Path = "/";
}
if (cookTemp.Domain == string.Empty)
{
cookTemp.Domain = strHost;
}
cc.Add(cookTemp);
}
return(cc);
}
private void AddUser(EprintAdminTask eprintTask, CookieCollection loginCookie)
{
//GET previous response to extract view state
string postData;
Uri getAddUser = new Uri($@"{_ePrintServerType}://{_ePrintServerIp}/cloudprintadmin/users/Edit.aspx");
HttpWebRequest getAddUserReq = (HttpWebRequest)WebRequest.Create(getAddUser);
getAddUserReq.Accept = "application/x-ms-application, image/jpeg, application/xaml+xml, image/gif, image/pjpeg, application/x-ms-xbap, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
getAddUserReq.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-US");
getAddUserReq.Headers.Add("Accept-Encoding", "gzip, deflate");
getAddUserReq.CookieContainer = new CookieContainer();
getAddUserReq.CookieContainer.Add(loginCookie);
getAddUserReq.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
HttpWebResult importResponse = HttpWebEngine.Get(getAddUserReq);
ExecutionServices.SystemTrace.LogInfo($"Requesting to add {Environment.UserName} as a {eprintTask.Operation} ");
//POST
CookieCollection adminCookie = new CookieCollection();
string setCookie = importResponse.Headers.Get("Set-Cookie");
string trimmedcookie = setCookie.Substring(12, setCookie.IndexOf(";", StringComparison.Ordinal) - 12);
adminCookie.Add(new Cookie("AdminCookie", trimmedcookie)
{
Domain = _ePrintServerIp
});
Uri addUserUrl = new Uri($@"{_ePrintServerType}://{_ePrintServerIp}/cloudprintadmin/users/Edit.aspx");
HttpWebRequest addUserReq = (HttpWebRequest)WebRequest.Create(addUserUrl);
addUserReq.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-US,en;q=0.8");
addUserReq.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
addUserReq.ContentType = "application/x-www-form-urlencoded";
addUserReq.CookieContainer = new CookieContainer();
addUserReq.CookieContainer.Add(loginCookie);
addUserReq.CookieContainer.Add(adminCookie);
addUserReq.Headers.Add("Accept-Encoding", "gzip, deflate");
addUserReq.Headers.Add(HttpRequestHeader.CacheControl, "max-age=0");
if (eprintTask.Operation.Equals(EprintAdminToolOperation.GuestUser))
{
//guest user
postData = string.Format(Properties.Resources.GuestUserPostData, HttpUtility.UrlEncode(ExtractViewState(importResponse.Response)), _credential.UserName, _userDnsName, DateTime.Today.ToString("MM-dd-yyyy"), DateTime.Today.AddMonths(1).ToString("MM-dd-yyyy"));
}
else
{
//regular user
postData = string.Format(Properties.Resources.RegularUserPostData, HttpUtility.UrlEncode(ExtractViewState(importResponse.Response)), _credential.UserName, _userDnsName, DateTime.Today.ToString("MM-dd-yyyy"), DateTime.Today.AddDays(30).ToString("MM-dd-yyyy"));
}
HttpWebResult addUserResponse = HttpWebEngine.Post(addUserReq, postData);
if (addUserResponse.StatusCode != HttpStatusCode.OK)
{
ExecutionServices.SystemTrace.LogError($"Failed to add { (object)Environment.UserName} as a { (object)eprintTask.Operation} ");
eprintTask.Status = "Failed";
}
else
{
ExecutionServices.SystemTrace.LogInfo($"Successfully added {Environment.UserName} as a {eprintTask.Operation} ");
eprintTask.Status = "Passed";
}
}
private void ePrintAddPrinter(EprintAdminTask eprintTask, CookieCollection loginCookie)
{
string assetId = eprintTask.TargetObject;
//GET previous response to extract viewstate
Uri getaddPrinterUrl = new Uri($@"{_ePrintServerType}://{_ePrintServerIp}/cloudprintadmin/printers/Insert.aspx");
HttpWebRequest getaddPrinter = (HttpWebRequest)WebRequest.Create(getaddPrinterUrl);
getaddPrinter.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-US,en;q=0.8");
getaddPrinter.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
getaddPrinter.CookieContainer = new CookieContainer();
getaddPrinter.CookieContainer.Add(loginCookie);
getaddPrinter.Headers.Add("Accept-Encoding", "gzip, deflate, sdch");
getaddPrinter.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
HttpWebResult getaddPrinterResponse = HttpWebEngine.Get(getaddPrinter);
if (getaddPrinterResponse.StatusCode != HttpStatusCode.OK)
{
ExecutionServices.SystemTrace.LogError("Unable to Login to ePrint . Please check Credentials");
//throw new PluginExecutionResult(PluginResult.Failed, "Unable to Login to ePrint . Please check Credentials");
}
CookieCollection adminCookie = new CookieCollection();
string setCookie = getaddPrinterResponse.Headers.Get("Set-Cookie");
string trimmedcookie = setCookie.Substring(12, setCookie.IndexOf(";", StringComparison.Ordinal) - 12);
adminCookie.Add(new Cookie("AdminCookie", trimmedcookie)
{
Domain = _ePrintServerIp
});
//Add the Printer: POST
Uri addPrinterUrl = new Uri($@"{_ePrintServerType}://{_ePrintServerIp}/cloudprintadmin/printers/Insert.aspx");
HttpWebRequest addPrinterReq = (HttpWebRequest)WebRequest.Create(addPrinterUrl);
HttpWebResult addPrinterResponse;
//ipv4 printer or HPAC printer
if (eprintTask.Operation.Equals(EprintAdminToolOperation.AddPrinteripv4))
{
addPrinterReq.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-US");
addPrinterReq.Accept = "text/html, application/xhtml+xml, */*";
addPrinterReq.ContentType = "application/x-www-form-urlencoded";
addPrinterReq.CookieContainer = new CookieContainer();
addPrinterReq.CookieContainer.Add(adminCookie);
addPrinterReq.CookieContainer.Add(loginCookie);
addPrinterReq.Headers.Add("Accept-Encoding", "gzip, deflate");
addPrinterReq.Headers.Add(HttpRequestHeader.Pragma, "no-cache");
string postData = string.Format(Properties.Resources.AddPrinterIPV4PostData, (HttpUtility.UrlEncode(ExtractViewState(getaddPrinterResponse.Response))), assetId, _device.GetDeviceInfo().ModelName, _device.Address);
addPrinterResponse = HttpWebEngine.Post(addPrinterReq, postData);
if (addPrinterResponse.StatusCode.Equals(HttpStatusCode.OK))
{
ExecutionServices.SystemTrace.LogInfo($"Successfully added printer { (object)assetId} IP { (object)_device.Address} ");
eprintTask.Status = "Passed";
}
else
{
ExecutionServices.SystemTrace.LogError($"Could not add printer { (object)assetId} IP { (object)_device.Address} ERROR:Status Code:{ (object)addPrinterResponse.StatusCode} ");
eprintTask.Status = "Failed";
}
}
else if (eprintTask.Operation.Equals(EprintAdminToolOperation.AddPrinterHpac))
{
addPrinterReq.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-US,en;q=0.8");
addPrinterReq.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
addPrinterReq.ContentType = "application/x-www-form-urlencoded";
addPrinterReq.CookieContainer = new CookieContainer();
addPrinterReq.CookieContainer.Add(loginCookie);
addPrinterReq.Headers.Add("Accept-Encoding", "gzip, deflate");
addPrinterReq.Headers.Add(HttpRequestHeader.CacheControl, "max-age=0");
string postData = string.Format(Properties.Resources.AddPrinterHPACPostdata, (HttpUtility.UrlEncode(ExtractViewState(getaddPrinterResponse.Response))), eprintTask.HpacInputValue.PrinterName, eprintTask.HpacInputValue.NetworkAddress, eprintTask.HpacInputValue.QueueName, eprintTask.HpacInputValue.DomainUser, eprintTask.HpacInputValue.DomainPassword);
addPrinterResponse = HttpWebEngine.Post(addPrinterReq, postData);
if (addPrinterResponse.StatusCode.Equals(HttpStatusCode.OK))
{
ExecutionServices.SystemTrace.LogInfo($"Successfully added printer using HPAC { (object)eprintTask.HpacInputValue.NetworkAddress} and Queue { (object)eprintTask.HpacInputValue.QueueName} ");
eprintTask.Status = "Passed";
}
else
{
ExecutionServices.SystemTrace.LogInfo($"Could not add printer { (object)eprintTask.HpacInputValue.PrinterName} ERROR:Status Code:{ (object)addPrinterResponse.StatusCode} ");
eprintTask.Status = "Failed";
}
}
else
{
addPrinterReq.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-US,en;q=0.8");
addPrinterReq.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
addPrinterReq.ContentType = "application/x-www-form-urlencoded";
addPrinterReq.CookieContainer = new CookieContainer();
addPrinterReq.CookieContainer.Add(adminCookie);
addPrinterReq.CookieContainer.Add(loginCookie);
addPrinterReq.Headers.Add("Accept-Encoding", "gzip, deflate");
addPrinterReq.Headers.Add(HttpRequestHeader.CacheControl, "max-age=0");
string postData = string.Format(Properties.Resources.AddPrinterSAFECOMPostdata, (HttpUtility.UrlEncode(ExtractViewState(getaddPrinterResponse.Response))), eprintTask.HpacInputValue.PrinterName, eprintTask.HpacInputValue.NetworkAddress, eprintTask.HpacInputValue.QueueName, eprintTask.HpacInputValue.DomainUser, eprintTask.HpacInputValue.DomainPassword);
addPrinterResponse = HttpWebEngine.Post(addPrinterReq, postData);
if (addPrinterResponse.StatusCode.Equals(HttpStatusCode.OK))
{
ExecutionServices.SystemTrace.LogInfo($"Successfully added printer using HPAC { (object)eprintTask.HpacInputValue.NetworkAddress} and Queue { (object)eprintTask.HpacInputValue.QueueName} ");
eprintTask.Status = "Passed";
}
else
{
ExecutionServices.SystemTrace.LogInfo($"Could not add printer {eprintTask.HpacInputValue.PrinterName} ERROR:Status Code:{addPrinterResponse.StatusCode} ");
eprintTask.Status = "Failed";
}
}
}
/// <summary>
/// Set the request headers.
/// </summary>
/// <param name="headers">The header collection.</param>
/// <param name="request">The request header.</param>
public virtual void ReadWebRequestHeaders(List <NameValue> headers, RequestResource request)
{
// Get the original headers.
OriginalHeaders = headers;
// If headers exist.
if (headers != null)
{
// Create the new header collection.
NameValueCollection headersCol = new NameValueCollection();
// For each header found.
foreach (NameValue item in headers)
{
// Assign the header collection.
headersCol.Add(item.Name, item.Value);
}
// Assign the headers.
Headers = headersCol;
HeadersFound = true;
Method = request.Method;
ProtocolVersion = request.ProtocolVersion;
Path = request.Path;
Scheme = request.Scheme;
try
{
// Get the Host
if (!String.IsNullOrEmpty(headersCol["Host"]))
{
// Get the query details.
Uri uri = new Uri("http://" + headersCol["Host"] + "/" + Path.TrimStart('/'));
Url = uri;
}
}
catch { }
// Get the Host
if (!String.IsNullOrEmpty(headersCol["Host"]))
{
// Get the content length.
Host = headersCol["Host"].Trim();
}
// Get the content length
if (!String.IsNullOrEmpty(headersCol["Content-Length"]))
{
// Get the content length.
ContentLength = Int64.Parse(headersCol["Content-Length"].Trim());
}
// Get the list of languages
if (!String.IsNullOrEmpty(headersCol["Accept-Language"]))
{
// Get the languages.
AcceptLanguages = headersCol["Accept-Language"].Split(new string[] { "," }, StringSplitOptions.None).Trim();
}
// Get the list of encoding
if (!String.IsNullOrEmpty(headersCol["Accept-Encoding"]))
{
// Get the encoding.
AcceptEncoding = headersCol["Accept-Encoding"].Split(new string[] { "," }, StringSplitOptions.None).Trim();
}
// Get the content type
if (!String.IsNullOrEmpty(headersCol["Content-Type"]))
{
// Get the content type.
ContentType = headersCol["Content-Type"].Trim();
}
// Get the connection
if (!String.IsNullOrEmpty(headersCol["Connection"]))
{
// Get keep-alive indicator.
if (headersCol["Connection"].Trim().ToLower().Contains("keep-alive"))
{
KeepAlive = true;
}
else
{
KeepAlive = false;
}
// Get the upgrade indicator if it exists.
string connUpgrade = headersCol["Connection"];
UpgradeRequest = (connUpgrade.Trim().ToLower().Contains("upgrade") ? true : false);
}
// Get the accept
if (!String.IsNullOrEmpty(headersCol["Accept"]))
{
// Get the accept.
AcceptTypes = headersCol["Accept"].Split(new string[] { "," }, StringSplitOptions.None).Trim();
}
// Get the upgrade
if (!String.IsNullOrEmpty(headersCol["Upgrade"]))
{
// Get the upgrade.
Upgrade = headersCol["Upgrade"].Trim();
}
// Get the authorization
if (!String.IsNullOrEmpty(headersCol["Authorization"]))
{
// Get the authorization.
string authorization = headersCol["Authorization"].Trim();
string[] authCredential = authorization.Split(new string[] { " " }, StringSplitOptions.None).Trim();
// If authorization data exists.
if (authCredential.Length > 0)
{
// Assign the credential.
AuthorizationType = authCredential[0].Trim();
// If authorization credential data exists.
if (authCredential.Length > 1)
{
byte[] credentialBytes = Convert.FromBase64String(authCredential[1].Trim());
string credentialString = Encoding.Default.GetString(credentialBytes);
string[] credentialData = credentialString.Split(new string[] { ":" }, StringSplitOptions.None).Trim();
// If credentails exists.
if (credentialData.Length > 1)
{
// Create the credentials
NetworkCredential credentials = new NetworkCredential(credentialData[0].Trim(), credentialData[1].Trim());
Credentials = credentials;
}
}
}
}
// Get the user agent
if (!String.IsNullOrEmpty(Headers["User-Agent"]))
{
// Get the user agent.
UserAgent = Headers["User-Agent"].Trim();
}
// Get the cookie
if (!String.IsNullOrEmpty(Headers["Cookie"]))
{
// Create a new collection.
CookieCollection CookieCollection = new CookieCollection();
// Get all cookies.
string[] cookies = Headers["Cookie"].Split(new string[] { ";" }, StringSplitOptions.None).Trim();
for (int i = 0; i < cookies.Length; i++)
{
// Make sure a cookie exists.
if (!String.IsNullOrEmpty(cookies[i].Trim()))
{
// Split the name and value header pair.
string[] nameValue = cookies[i].Split(new string[] { "=" }, StringSplitOptions.None).Trim();
// Create a new cookie and add to the collection.
Cookie cookie = new Cookie(nameValue[0].Trim(), nameValue[1].Trim());
CookieCollection.Add(cookie);
}
}
// Assign the collection.
Cookies = CookieCollection;
}
// Make sure a URI exists.
if (Url != null)
{
// Make sure a query exists before processing.
if (!String.IsNullOrEmpty(Url.Query))
{
string[] queryCol = Url.Query.TrimStart(new char[] { '?' }).Split(new string[] { "&" }, StringSplitOptions.None);
// Create a header name value collection.
NameValueCollection queries = new NameValueCollection();
// For each query
for (int i = 0; i < queryCol.Length; i++)
{
// Split the name and value header pair.
string[] nameValue = queryCol[i].Split(new string[] { "=" }, StringSplitOptions.None);
queries.Add(nameValue[0].Trim(), nameValue[1].Trim());
}
// Assign the queries.
Query = queries;
}
}
// Get the referer
if (!String.IsNullOrEmpty(Headers["Referer"]))
{
try
{
// Get the referer.
UrlReferrer = new Uri(Headers["Referer"].Trim());
}
catch { }
}
}
}
public static void Add_Cookie_Invalid()
{
CookieCollection cc = new CookieCollection();
Assert.Throws<ArgumentNullException>(() => cc.Add((Cookie)null));
}
//add a single cookie to cookies, if already exist, update its value
public void addCookieToCookies(Cookie toAdd, ref CookieCollection cookies, bool overwriteDomain)
{
bool found = false;
if (cookies.Count > 0)
{
foreach (Cookie originalCookie in cookies)
{
if (originalCookie.Name == toAdd.Name)
{
// !!! for different domain, cookie is not same,
// so should not set the cookie value here while their domains is not same
// only if it explictly need overwrite domain
if ((originalCookie.Domain == toAdd.Domain) ||
((originalCookie.Domain != toAdd.Domain) && overwriteDomain))
{
//here can not force convert CookieCollection to HttpCookieCollection,
//then use .remove to remove this cookie then add
// so no good way to copy all field value
originalCookie.Value = toAdd.Value;
originalCookie.Domain = toAdd.Domain;
originalCookie.Expires = toAdd.Expires;
originalCookie.Version = toAdd.Version;
originalCookie.Path = toAdd.Path;
//following fields seems should not change
//originalCookie.HttpOnly = toAdd.HttpOnly;
//originalCookie.Secure = toAdd.Secure;
found = true;
break;
}
}
}
}
if (!found)
{
if (toAdd.Domain != "")
{
// if add the null domain, will lead to follow req.CookieContainer.Add(cookies) failed !!!
cookies.Add(toAdd);
}
}
}
/// <summary>
/// Performs HTTP transactions for the specified page request.
/// </summary>
/// <param name="pageRequest">The page request.</param>
/// <returns>bool - Success/fail</returns>
private bool Transaction(HTTPRequest pageRequest)
{
ArrayList Blocks = new ArrayList();
byte[] Block;
byte[] readBlock;
int size;
int totalSize;
DateTime startTime = DateTime.Now;
if (pageRequest.Delay > 0)
{
Thread.Sleep(pageRequest.Delay);
}
string agent = string.IsNullOrEmpty(pageRequest.UserAgent)? _agent : pageRequest.UserAgent;
Uri pageUri = pageRequest.Uri;
try
{
// Make the Webrequest
// Create the request header
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(pageUri);
try
{
// Use the current user in case an NTLM Proxy or similar is used.
// request.Proxy = WebProxy.GetDefaultProxy();
request.Proxy.Credentials = CredentialCache.DefaultCredentials;
}
catch (Exception) {}
request.UserAgent = agent;
request.AllowAutoRedirect = false;
if (pageRequest.Cookies != string.Empty)
{
string[] cookiesArray = pageRequest.Cookies.Split(new Char[] { ';' });
foreach (string cookie in cookiesArray)
{
string[] cookieParts = cookie.Split(new Char[] { '=' });
if (cookieParts.Length >= 2)
{
if (_cookies == null)
{
_cookies = new CookieCollection();
}
_cookies.Add(new Cookie(cookieParts[0], cookieParts[1], "/", request.RequestUri.Host));
}
}
}
if (pageRequest.PostQuery == string.Empty)
{
// GET request
if (_auth != null)
{
request.Credentials = _auth.Get(pageUri.Host);
}
request.CookieContainer = new CookieContainer();
if (_cookies != null)
{
request.CookieContainer.Add(_cookies);
}
}
else
{
// POST request
request.ContentType = _postType;
request.ContentLength = pageRequest.PostQuery.Length;
request.Method = "POST";
request.CookieContainer = new CookieContainer();
if (_cookies != null)
{
request.CookieContainer.Add(_cookies);
}
// Write post message
try
{
Stream OutputStream = request.GetRequestStream();
StreamWriter WriteStream = new StreamWriter(OutputStream);
WriteStream.Write(pageRequest.PostQuery);
WriteStream.Flush();
}
catch (WebException ex)
{
_error = ex.Message;
return(false);
}
}
_response = (HttpWebResponse)request.GetResponse();
// Check for redirection
if ((_response.StatusCode == HttpStatusCode.Found) ||
(_response.StatusCode == HttpStatusCode.Redirect) ||
(_response.StatusCode == HttpStatusCode.Moved) ||
(_response.StatusCode == HttpStatusCode.MovedPermanently))
{
Uri uri = new Uri(pageUri, _response.Headers["Location"]);
HttpWebRequest redirect = (HttpWebRequest)WebRequest.Create(uri);
try
{
// Use the current user in case an NTLM Proxy or similar is used.
// request.Proxy = WebProxy.GetDefaultProxy();
redirect.Proxy.Credentials = CredentialCache.DefaultCredentials;
}
catch (Exception) {}
redirect.UserAgent = agent;
redirect.AllowAutoRedirect = false;
redirect.Referer = _response.ResponseUri.ToString();
redirect.CookieContainer = new CookieContainer();
if (_response.Headers["Set-Cookie"] != null)
{
string cookieStr = _response.Headers["Set-Cookie"];
Regex cookieParser = new Regex("(?<name>[^=]+)=(?<value>[^;]+)(;)");
Match result = cookieParser.Match(cookieStr);
if (result.Success)
{
Cookie reply = new Cookie(result.Groups["name"].ToString(), result.Groups["value"].ToString());
reply.Domain = uri.Host;
redirect.CookieContainer.Add(reply);
}
}
//redirect.ContentType = "text/html";
_response = (HttpWebResponse)redirect.GetResponse();
}
if (request.CookieContainer != null)
{
_response.Cookies = request.CookieContainer.GetCookies(request.RequestUri);
_cookies = _response.Cookies;
}
Stream ReceiveStream = _response.GetResponseStream();
Block = new byte[blockSize];
totalSize = 0;
while ((size = ReceiveStream.Read(Block, 0, blockSize)) > 0)
{
readBlock = new byte[size];
Array.Copy(Block, readBlock, size);
Blocks.Add(readBlock);
totalSize += size;
}
ReceiveStream.Close();
_response.Close();
int pos = 0;
_data = new byte[totalSize];
for (int i = 0; i < Blocks.Count; i++)
{
Block = (byte[])Blocks[i];
Block.CopyTo(_data, pos);
pos += Block.Length;
}
}
catch (WebException ex)
{
_error = ex.Message;
return(false);
}
// Collect sits statistics
if (_stats != null)
{
DateTime endTime = DateTime.Now;
TimeSpan duration = endTime - startTime;
_stats.Add(pageUri.Host, 1, _data.Length, duration);
}
return(true);
}
internal static void OnReceivedHeaders(HttpWebRequest httpWebRequest)
{
GlobalLog.Print("CookieModule.OnReceivedHeaders()");
//
// if the app doesn't want us to handle cookies then there's nothing
// to do. Note that we're leaving open the possibility that these
// settings could be changed between the request being made and the
// response received
//
try {
if (httpWebRequest.CookieContainer == null)
{
return;
}
//
// add any received cookies for this response to the container
//
HttpWebResponse response = httpWebRequest._HttpResponse as HttpWebResponse;
if (response == null)
{
return;
}
CookieCollection cookies = null;
try {
string cookieString = response.Headers.SetCookie;
GlobalLog.Print("CookieModule::OnSendingHeaders() received Set-Cookie:[" + cookieString + "]");
if ((cookieString != null) && (cookieString.Length > 0))
{
cookies = httpWebRequest.CookieContainer.CookieCutter(
response.ResponseUri,
HttpKnownHeaderNames.SetCookie,
cookieString,
false);
}
}
catch {
}
try {
string cookieString = response.Headers.SetCookie2;
GlobalLog.Print("CookieModule::OnSendingHeaders() received Set-Cookie2:[" + cookieString + "]");
if ((cookieString != null) && (cookieString.Length > 0))
{
CookieCollection cookies2 = httpWebRequest.CookieContainer.CookieCutter(
response.ResponseUri,
HttpKnownHeaderNames.SetCookie2,
cookieString,
false);
if (cookies != null && cookies.Count != 0)
{
cookies.Add(cookies2);
}
else
{
cookies = cookies2;
}
}
}
catch {
}
if (cookies != null)
{
response.Cookies = cookies;
}
}
catch {
}
}
public static void Add_CookieCollection_Success()
{
CookieContainer cc1 = new CookieContainer();
CookieCollection cc2 = new CookieCollection();
cc2.Add(new Cookie("name1", "value", "", "contoso.com"));
cc2.Add(new Cookie("name2", "value", "", "contoso.com"));
cc1.Add(cc2);
Assert.Equal(2, cc1.Count);
}
private static NameValueCollection SetFedAuthCookies(HttpResponse response, out CookieCollection cookieCollection)
{
NameValueCollection headers = new NameValueCollection();
cookieCollection = new CookieCollection();
for (int i = 0; i < CommonHeaders.Count; i++)
{
headers.Add(CommonHeaders.GetKey(i), CommonHeaders.Get(i));
}
string cookieString = response.Headers["Set-Cookie"];
if (String.IsNullOrEmpty(cookieString))
{
return(headers);
}
List <string> cookies = cookieString.Split(',').ToList();
for (int i = 0; i < cookies.Count; i++)
{
if (cookies[i].IndexOf("expires=", StringComparison.OrdinalIgnoreCase) > 0)
{
cookies[i] += cookies[i + 1];
cookies.Remove(cookies[i + 1]);
}
}
string headerCookies = String.Empty;
Func <string[], int, int, string> joinArray = (array, ps, pe) =>
{
StringBuilder sb = new StringBuilder();
for (int i = ps; i <= pe; i++)
{
sb.Append(array[i]);
}
return(sb.ToString());
};
foreach (var piece in cookies)
{
Cookie cookie = new Cookie();
foreach (var item in piece.Trim().Split(';'))
{
if (item.Trim() == "HttpOnly")
{
cookie.HttpOnly = true;
}
if (item.Contains("="))
{
//string[] kvPair = item.Split('=');
int position = item.IndexOf('=');
string key = item.Substring(0, position).Trim();
string value = item.Substring(position + 1, item.Length - (position + 1)).Trim();//kvPair.Length > 2 ? joinArray(kvPair, 1, kvPair.Length - 1) : kvPair[1].Trim();
//if (key.ToLower() == "path")
// cookie.Path = value;
if (key.ToLower() != "expires" && key.ToLower() != "path")
{
cookie.Name = key;
cookie.Value = value;
}
}
}
headerCookies += String.Format("{0}={1}; ", cookie.Name, cookie.Value);
cookieCollection.Add(cookie);
}
headers.Add("Cookie", headerCookies);
return(headers);
}
private HttpWebResponse FetchResponse(string url, HttpMethod method, string referer, string postData, string ip = "", string Ori = "", string Host = "")
{
HttpWebResponse res = null;
try
{
if (Uri.IsWellFormedUriString(url, UriKind.Absolute))
{
//don't use proxy
WebRequest.DefaultWebProxy = null;
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
req.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:15.0) Gecko/20100101 Firefox/15.0.1";
req.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
req.ContentType = "application/x-www-form-urlencoded";
req.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
if (ip != "")
{
req.Headers.Add("X-Forwarded-For", ip);
}
req.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
req.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-us,en;q=0.5");
req.Headers.Add(HttpRequestHeader.KeepAlive, "true");
req.Headers.Add("Upgrade-Insecure-Requests", "1");
if (Ori != "")
{
//req.Headers.Add("Origin", Ori);
//req.Headers.Add("X-Requested-With", "XMLHttpRequest");
req.ContentLength = 0;
}
if (Host != "")
{
req.Host = Host;
}
if (!string.IsNullOrEmpty(referer))
{
req.Referer = referer;
}
lock (_cookielocker)
{
if (setcookie)
{
req.CookieContainer = new CookieContainer();
req.CookieContainer.Add(new Uri(url), cookies);
}
else
{
req.CookieContainer = cookieContainer;
}
}
req.Method = method.ToString().ToUpper();
if (method == HttpMethod.Post)
{
//set up the post data to be sent
if (!string.IsNullOrEmpty(postData))
{
byte[] buf = postDataEncoding.GetBytes(postData);
req.ContentLength = buf.Length;
Stream poststr = req.GetRequestStream();
poststr.Write(buf, 0, buf.Length);
poststr.Close();
}
}
//get response
res = (HttpWebResponse)req.GetResponse();
//get response
res.Cookies = req.CookieContainer.GetCookies(req.RequestUri);
// Add the cookie to this instance of object, such that the cookies can be reused
// for next request
lock (_cookielocker)
{
if (setcookie)
{
cookies = res.Cookies;
cookieContainer = req.CookieContainer;
}
else
{
cookies.Add(res.Cookies);
cookieContainer.Add(cookies);
}
}
}
}
catch (Exception ex)
{
CloseResponse(res);
res = null;
error = ex.Message;
}
return(res);
}
public static void Add_Cookie_Invalid()
{
CookieCollection cc = new CookieCollection();
Assert.Throws <ArgumentNullException>(() => cc.Add((Cookie)null));
}
private static CookieCollection SetCookieParser(List <string> setCookie, CookieCollection cooks, CookieCollection initCookies)
{
List <Exception> ex = new List <Exception>();
List <Hashtable> rckevalues = new List <Hashtable>();
List <Hashtable> ckevalues = new List <Hashtable>();
List <Cookie> ckeList = new List <Cookie>();
if (initCookies != null)
{
for (int i = 0; i < initCookies.Count; i++)
{
ckeList.Add(initCookies[i]);
Hashtable h = new Hashtable();
h.Add(initCookies[i].Name, initCookies[i].Value);
ckevalues.Add(h);
}
}
try
{
List <string> rckes = new List <string>();
for (int i = 0; i < cooks.Count; i++)
{
rckes.Add(cooks[i].Name);
}
foreach (string set in setCookie)
{
Cookie cke = new Cookie();
for (int i = 0; i < set.Split(';').ToList().Count; i++)
{
List <string> v = new List <string>();
string item = set.Split(';').ToList()[i];
for (int ii = 1; ii < item.Split('=').ToList().Count; ii++)
{
v.Add(item.Split('=')[ii]);
}
string va = String.Join('='.ToString(), v);
string key = new Regex(@"^(\s*)").Replace(item.Split('=').ToList()[0], "");
string value = new Regex(@"^(\s*)").Replace(va, "");
if (i == 0)
{
cke.Name = key;
cke.Value = value;
}
else
{
switch (key.ToLower())
{
case "comment":
cke.Comment = value;
break;
case "commenturi":
cke.CommentUri = new Uri(value);
break;
case "httponly":
cke.HttpOnly = bool.Parse(value);
break;
case "discard":
cke.Discard = bool.Parse(value);
break;
case "domain":
cke.Domain = value;
break;
case "expires":
cke.Expires = DateTime.Parse(value);
break;
case "path":
cke.Path = value;
break;
case "port":
cke.Port = value;
break;
case "secure":
cke.Secure = bool.Parse(value);
break;
case "version":
cke.Version = int.Parse(value);
break;
}
}
if (!rckes.Contains(cke.Name))
{
cooks.Add(cke);
}
else
{
CookieCollection tempRCkes = new CookieCollection();
for (int ii = 0; ii < cooks.Count; ii++)
{
Cookie current = cooks[ii];
if (!current.Name.Equals(cke.Name))
{
tempRCkes.Add(current);
}
}
tempRCkes.Add(cke);
cooks = new CookieCollection();
for (int ii = 0; ii < tempRCkes.Count; ii++)
{
cooks.Add(tempRCkes[ii]);
}
rckes = new List <string>();
for (int ii = 0; ii < cooks.Count; ii++)
{
rckes.Add(cooks[ii].Name);
}
}
}
}
if (cooks != null)
{
for (int i = 0; i < cooks.Count; i++)
{
Hashtable h = new Hashtable();
h.Add(cooks[i].Name, cooks[i].Value);
rckevalues.Add(h);
}
}
if (ckevalues != null)
{
if (rckevalues.Count > 0)
{
List <string> rNames = new List <string>();
List <string> rValue = new List <string>();
for (int i = 0; i < rckevalues.Count; i++)
{
string rcken = rckevalues[i].Keys.ToString();
string rckev = rckevalues[i].Values.ToString();
rNames.Add(rcken);
rValue.Add(rckev);
}
for (int i = 0; i < ckevalues.Count; i++)
{
string ckeName = ckevalues[i].Keys.ToString();
string ckeValu = ckevalues[i].Values.ToString();
if (!rValue.Contains(ckeValu))
{
if (!rNames.Contains(ckeName))
{
cooks.Add(ckeList.Where(item => item.Name.Equals(ckeName)).FirstOrDefault());
}
}
else
{
if (!rNames.Contains(ckeName))
{
cooks.Add(ckeList.Where(item => item.Name.Equals(ckeName)).FirstOrDefault());
}
}
}
}
else
{
ckeList.ForEach(i => cooks.Add(i));
}
}
}
catch (Exception e)
{
ex.Add(e);
}
return(cooks);
}
/// <summary>
/// 会返回cookies的Get方法
/// </summary>
/// <param name="url">URL</param>
/// <param name="out_cookies">返回的cookies集合实例</param>
/// <param name="cookie">cookies集合实例</param>
/// <param name="referer">Referer</param>
/// <param name="user_agent">User-agent</param>
/// <param name="specialheaders">除前面之外的Headers</param>
/// <returns>请求返回体</returns>
public static Bitmap GetPicOutCookies(string url, out CookieCollection out_cookies, CookieCollection cookie = null,
string referer = "", string user_agent = "", WebHeaderCollection specialheaders = null)
{
Bitmap result = null;
out_cookies = new CookieCollection();
HttpWebRequest req = null;
HttpWebResponse rep = null;
try
{
req = (HttpWebRequest)WebRequest.Create(url);
if (specialheaders != null)
{
req.Headers = specialheaders;
}
if (cookie != null)
{
req.CookieContainer = new CookieContainer(cookie.Count)
{
PerDomainCapacity = cookie.Count
};
req.CookieContainer.Add(cookie);
}
if (!string.IsNullOrEmpty(referer))
{
req.Referer = referer;
}
if (!string.IsNullOrEmpty(user_agent))
{
req.UserAgent = user_agent;
}
rep = (HttpWebResponse)req.GetResponse();
result = new Bitmap(rep.GetResponseStream());
if (rep.Headers.GetValues("Set-Cookie") != null && rep.Headers.GetValues("Set-Cookie").Length > 0)
{
foreach (string i in rep.Headers.GetValues("Set-Cookie"))
{
out_cookies.Add(new Cookie(new Regex("^(?<=).*?(?==)").Match(i).Value, new Regex("(?<==).*?(?=; )").Match(i).Value)
{
Expires = DateTime.Parse(new Regex("(?<=Expires=).*?(?=;)").Match(i).Value), Domain = new Regex("(?<=Domain=).*?(?=;)").Match(i).Value, Path = new Regex("(?<=Path=).*?(?=;)").Match(i).Value
});
}
}
}
finally
{
if (rep != null)
{
rep.Close();
}
if (req != null)
{
req.Abort();
}
}
return(result);
}
static private CookieCollection ParseCookieString(string cookieString, Func <string> getCookieDomainIfItIsMissingInCookie)
{
bool secure = false;
bool httpOnly = false;
string domainFromCookie = null;
string path = null;
string expiresString = null;
Dictionary <string, string> cookiesValues = new Dictionary <string, string>();
var cookieValuePairsStrings = cookieString.Split(';');
foreach (string cookieValuePairString in cookieValuePairsStrings)
{
var pairArr = cookieValuePairString.Split('=');
int pairArrLength = pairArr.Length;
for (int i = 0; i < pairArrLength; i++)
{
pairArr[i] = pairArr[i].Trim();
}
string propertyName = pairArr[0];
if (pairArrLength == 1)
{
if (propertyName.Equals("httponly", StringComparison.OrdinalIgnoreCase))
{
httpOnly = true;
}
else if (propertyName.Equals("secure", StringComparison.OrdinalIgnoreCase))
{
secure = true;
}
else
{
throw new InvalidOperationException(string.Format("Unknown cookie property \"{0}\". All cookie is \"{1}\"", propertyName, cookieString));
}
continue;
}
string propertyValue = pairArr[1];
if (propertyName.Equals("expires", StringComparison.OrdinalIgnoreCase))
{
expiresString = propertyValue;
}
else if (propertyName.Equals("domain", StringComparison.OrdinalIgnoreCase))
{
domainFromCookie = propertyValue;
}
else if (propertyName.Equals("path", StringComparison.OrdinalIgnoreCase))
{
path = propertyValue;
}
else
{
cookiesValues.Add(propertyName, propertyValue);
}
}
DateTime expiresDateTime;
if (expiresString != null)
{
expiresDateTime = DateTime.Parse(expiresString);
}
else
{
expiresDateTime = DateTime.MinValue;
}
if (string.IsNullOrEmpty(domainFromCookie))
{
domainFromCookie = getCookieDomainIfItIsMissingInCookie();
}
CookieCollection cookieCollection = new CookieCollection();
foreach (var pair in cookiesValues)
{
Cookie cookie = new Cookie(pair.Key, pair.Value, path, domainFromCookie);
cookie.Secure = secure;
cookie.HttpOnly = httpOnly;
cookie.Expires = expiresDateTime;
cookieCollection.Add(cookie);
}
return(cookieCollection);
}
/// <summary>
/// 返回cookies的POST方法
/// </summary>
/// <param name="url">URL</param>
/// <param name="out_cookies">返回的cookies集合实例</param>
/// <param name="data">要POST发送的文本</param>
/// <param name="cookie">cookies集合实例</param>
/// <param name="contenttype">数据类型</param>
/// <param name="referer">Referer</param>
/// <param name="user_agent">User-agent</param>
/// <param name="specialheaders">除前面之外的Headers</param>
/// <returns>请求返回体</returns>
public static string PostBodyOutCookies(string url, out CookieCollection out_cookies, string data = "", CookieCollection cookie = null,
string contenttype = "application/x-www-form-urlencoded;charset=utf-8", string referer = "", string user_agent = "",
WebHeaderCollection specialheaders = null)
{
string result = "";
out_cookies = new CookieCollection();
HttpWebRequest req = null;
HttpWebResponse rep = null;
try
{
req = (HttpWebRequest)WebRequest.Create(url);
if (specialheaders != null)
{
req.Headers = specialheaders;
}
req.Method = "POST";
if (cookie != null)
{
req.CookieContainer = new CookieContainer(cookie.Count)
{
PerDomainCapacity = cookie.Count
};
req.CookieContainer.Add(cookie);
}
req.ContentType = contenttype;
byte[] bdata = Encoding.UTF8.GetBytes(data);
Stream sdata = req.GetRequestStream();
sdata.Write(bdata, 0, bdata.Length);
sdata.Close();
if (!string.IsNullOrEmpty(referer))
{
req.Referer = referer;
}
if (!string.IsNullOrEmpty(user_agent))
{
req.UserAgent = user_agent;
}
rep = (HttpWebResponse)req.GetResponse();
using (StreamReader reader = new StreamReader(rep.GetResponseStream()))
{
result = reader.ReadToEnd();
}
if (rep.Headers.GetValues("Set-Cookie") != null && rep.Headers.GetValues("Set-Cookie").Length > 0)
{
foreach (string i in rep.Headers.GetValues("Set-Cookie"))
{
out_cookies.Add(new Cookie(new Regex("^(?<=).*?(?==)").Match(i).Value, new Regex("(?<==).*?(?=; )").Match(i).Value)
{
Expires = DateTime.Parse(new Regex("(?<=Expires=).*?(?=;)").Match(i).Value), Domain = new Regex("(?<=Domain=).*?(?=;)").Match(i).Value, Path = new Regex("(?<=Path=).*?(?=;)").Match(i).Value
});
}
}
}
finally
{
if (rep != null)
{
rep.Close();
}
if (req != null)
{
req.Abort();
}
}
return(result);
}
/// <summary>
/// http请求
/// </summary>
/// <param name="url"></param>
/// <param name="method">POST,GET</param>
/// <param name="headers">http的头部,直接拷贝谷歌请求的头部即可</param>
/// <param name="content">content,每个key,value 都要UrlEncode才行</param>
/// <param name="contentEncode">content的编码</param>
/// <param name="proxyUrl">代理url</param>
/// <returns></returns>
public string http(string url, string method, string headers, string content, Encoding contentEncode, string proxyUrl)
{
/*
* HttpWebRequest request = null;
* if (url.StartsWith("https", StringComparison.OrdinalIgnoreCase))
* {
* ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult);
* request = WebRequest.Create(url) as HttpWebRequest;
* request.ProtocolVersion = HttpVersion.Version10;
* }
* else
* {
* request = WebRequest.Create(url) as HttpWebRequest;
* }*/
if (string.IsNullOrEmpty(headers) || string.IsNullOrWhiteSpace(headers))
{
headers = defaultHeaders;
}
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Method = method;
if (method.Equals("GET", StringComparison.InvariantCultureIgnoreCase))
{
request.MaximumAutomaticRedirections = 100;
request.AllowAutoRedirect = false;
}
fillHeaders(request, headers);
fillProxy(request, proxyUrl);
#region 添加Post 参数
if (contentEncode == null)
{
contentEncode = Encoding.UTF8;
}
if (!string.IsNullOrWhiteSpace(content))
{
byte[] data = contentEncode.GetBytes(content);
request.ContentLength = data.Length;
using (Stream reqStream = request.GetRequestStream())
{
reqStream.Write(data, 0, data.Length);
reqStream.Close();
}
}
#endregion
HttpWebResponse response = null;
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
try
{
sw.Start();
response = (HttpWebResponse)request.GetResponse();
sw.Stop();
AvgResponseMilliseconds = sw.ElapsedMilliseconds;
CookieCollection cc = new CookieCollection();
string cookieString = response.Headers[HttpResponseHeader.SetCookie];
if (!string.IsNullOrWhiteSpace(cookieString))
{
var segments = cookieString.Split(',');
List <string> segments2 = new List <string>();
for (int i = 0; i < segments.Count();)
{
if (segments[i].IndexOf("Expires") > 0)
{
segments[i] = string.Concat(segments[i], segments[i + 1]);
segments2.Add(segments[i]);
i += 2;
}
else
{
segments2.Add(segments[i]);
i++;
}
}
segments = segments2.ToArray();
foreach (string segment in segments)
{
var spilit = segment.Split(';');
Cookie cookie = new Cookie();
bool isHasDomain = false;
foreach (string item in spilit)
{
var kv = item.Split('=');
switch (kv[0].Trim())
{
case "Domain":
cookie.Domain = kv[1].Trim();
isHasDomain = true;
break;
case "Expires":
cookie.Expires = DateTime.Parse(kv[1].Trim());
break;
case "Path":
cookie.Path = kv[1].Trim();
break;
case "HttpOnly":
cookie.HttpOnly = true;
break;
default:
cookie.Name = kv[0].Trim();
cookie.Value = kv[1].Trim();
break;
}
}
if (!isHasDomain)
{
cookie.Domain = response.ResponseUri.Host;
}
cc.Add(cookie);
}
}
trackCookies(cc);
}
catch (Exception ex)
{
sw.Stop();
AvgResponseMilliseconds = sw.ElapsedMilliseconds;
return("");
}
string result = getResponseBody(response);
return(result);
}
/// <summary>
/// 会返回cookies的Get方法
/// </summary>
/// <param name="url">URL</param>
/// <param name="out_cookies">返回的cookies集合实例</param>
/// <param name="cookie">cookies集合实例</param>
/// <param name="referer">Referer</param>
/// <param name="user_agent">User-agent</param>
/// <param name="specialheaders">除前面之外的Headers</param>
/// <returns>请求返回体</returns>
public static string GetBodyOutCookies(string url, out CookieCollection out_cookies, CookieCollection cookie = null,
string referer = "", string user_agent = "", WebHeaderCollection specialheaders = null)
{
string result = "";
out_cookies = new CookieCollection();
HttpWebRequest req = null;
HttpWebResponse rep = null;
try
{
req = (HttpWebRequest)WebRequest.Create(url);
if (specialheaders != null)
{
req.Headers = specialheaders;
}
if (cookie != null)
{
req.CookieContainer = new CookieContainer(cookie.Count)
{
PerDomainCapacity = cookie.Count
};
req.CookieContainer.Add(cookie);
}
if (!string.IsNullOrEmpty(referer))
{
req.Referer = referer;
}
if (!string.IsNullOrEmpty(user_agent))
{
req.UserAgent = user_agent;
}
rep = (HttpWebResponse)req.GetResponse();
using (StreamReader reader = new StreamReader(rep.GetResponseStream()))
{
result = reader.ReadToEnd();
}
foreach (string i in rep.Headers.GetValues("Set-Cookie"))
{
string[] tmp = i.Split(';');
string[] tmp2 = tmp[0].Split('=');
out_cookies.Add(new Cookie(tmp2[0], tmp2[1])
{
Expires = DateTime.Parse(tmp[2].Split('=')[1])
});
}
}
finally
{
if (rep != null)
{
rep.Close();
}
if (req != null)
{
req.Abort();
}
}
return(result);
}
public bool Extract(NetworkCredential credential, out MailContactList list)
{
bool result = false;
list = new MailContactList();
try
{
CookieCollection cookies = new CookieCollection();
// Prepare login form data
HttpValueCollection loginFormValues = new HttpValueCollection();
loginFormValues["ltmpl"] = "default";
loginFormValues["ltmplcache"] = "2";
loginFormValues["continue"] = ContinueUrl;
loginFormValues["service"] = "mail";
loginFormValues["rm"] = "false";
loginFormValues["hl"] = "en";
loginFormValues["Email"] = credential.UserName;
loginFormValues["Passwd"] = credential.Password;
loginFormValues["PersistentCookie"] = "true";
loginFormValues["rmShown"] = "1";
loginFormValues["null"] = "Sign In";
// Convert to bytes
byte[] loginPostData = Encoding.UTF8.GetBytes(loginFormValues.ToString(true));
HttpWebRequest loginRequest = ( HttpWebRequest )WebRequest.Create(LoginUrl);
loginRequest.Method = "POST";
loginRequest.UserAgent = UserAgent;
loginRequest.Referer = LoginRefererUrl;
loginRequest.ContentType = "application/x-www-form-urlencoded";
loginRequest.ContentLength = loginPostData.Length;
loginRequest.AllowAutoRedirect = false;
// Create cookie container
loginRequest.CookieContainer = new CookieContainer();
// Add post data to request
Stream stream;
using (stream = loginRequest.GetRequestStream())
{
stream.Write(loginPostData, 0, loginPostData.Length);
}
HttpWebResponse loginResponse = ( HttpWebResponse )loginRequest.GetResponse();
cookies.Add(loginResponse.Cookies);
// Create request to export Google CSV page
HttpWebRequest contactsRequest = ( HttpWebRequest )WebRequest.Create(ExportUrl);
contactsRequest.Method = "GET";
contactsRequest.UserAgent = UserAgent;
contactsRequest.Referer = loginResponse.ResponseUri.ToString();
// use cookie gotten from login page
contactsRequest.CookieContainer = new CookieContainer();
foreach (Cookie cookie in cookies)
{
contactsRequest.CookieContainer.Add(cookie);
}
HttpWebResponse exportResponse = ( HttpWebResponse )contactsRequest.GetResponse();
// Read data from response stream
string csvData;
using (Stream responseStream = exportResponse.GetResponseStream())
{
using (StreamReader streamRead = new StreamReader(responseStream))
{
csvData = streamRead.ReadToEnd();
}
}
// parse google csv
string[] lines = csvData.Split('\n');
foreach (string line in lines)
{
string[] values = line.Split(',');
if (values.Length < 2)
{
continue;
}
MailContact mailContact = new MailContact();
mailContact.Email = values[1];
mailContact.Name = values[0];
if (mailContact.Email.Trim().Length > 0)
{
list.Add(mailContact);
}
}
result = true;
}
catch
{
}
return(result);
}
public static void AddCookieCollectionUri_Success()
{
Uri uri = new Uri("http://contoso.com");
String domain = "contoso.com";
CookieContainer cc1 = new CookieContainer();
CookieCollection cc2 = new CookieCollection();
cc2.Add(new Cookie("name1", "value") { Domain = domain });
cc2.Add(new Cookie("name2", "value") { Domain = domain });
cc1.Add(uri, cc2);
Assert.Equal(2, cc1.Count);
}
public HttpWebResponse CreatePostHttpResponse(string url, string content, CookieCollection newCookies)
{
if (string.IsNullOrEmpty(url))
{
throw new ArgumentNullException("url");
}
HttpWebRequest request = null;
request = WebRequest.Create(url) as HttpWebRequest;
request.Method = "POST";
request.UserAgent = DefaultUserAgent;
// if (!(HeaderParameters == null || HeaderParameters.Count == 0))
// {
// foreach (string key in HeaderParameters.Keys)
// {
// request.Headers[key] = HeaderParameters[key];
// }
// }
// if (timeout.HasValue)
// {
// request.Timeout = timeout.Value;
// }
if (Cookies != null && Cookies.Count != 0)
{
foreach (Cookie c in Cookies)
{
request.CookieContainer.Add(c);
}
}
if (newCookies != null && newCookies.Count != 0)
{
foreach (Cookie c in newCookies)
{
request.CookieContainer.Add(c);
Cookies.Add(c);
}
//request.CookieContainer.Add(cookies);
}
//如果需要POST数据
if (content != null)
{
//StringBuilder buffer = new StringBuilder();
int i = 0;
byte[] data = Encoding.ASCII.GetBytes(content.ToString());
using (Stream stream = request.GetRequestStream())
{
stream.Write(data, 0, data.Length);
}
}
HttpWebResponse Response = request.GetResponse() as HttpWebResponse;
if (Response.Cookies.Count != 0)
{
foreach (Cookie c in Response.Cookies)
{
Cookies.Add(c);
}
}
if (Response.Headers["ContentEncoding"] != null)
{
Stream receiveStream = Response.GetResponseStream();
StreamReader sr = new StreamReader(receiveStream, Encoding.GetEncoding(Response.Headers["ContentEncoding"].ToString()));
Content = sr.ReadToEnd();
}
else
{
try
{
Stream receiveStream = Response.GetResponseStream();
StreamReader sr = new StreamReader(receiveStream);
Content = sr.ReadToEnd();
}
catch
{
}
}
return(Response);
}
private static CookieCollection ParseResponse(string value)
{
var cookies = new CookieCollection();
Cookie cookie = null;
var pairs = SplitCookieHeaderValue(value);
for (var i = 0; i < pairs.Length; i++)
{
var pair = pairs[i].Trim();
if (pair.Length == 0)
continue;
if (pair.StartsWith("version", StringComparison.OrdinalIgnoreCase))
{
if (cookie != null)
cookie.Version = Int32.Parse(GetValue(pair, '=', true));
}
else if (pair.StartsWith("expires", StringComparison.OrdinalIgnoreCase))
{
var buff = new StringBuilder(GetValue(pair, '='), 32);
if (i < pairs.Length - 1)
buff.AppendFormat(", {0}", pairs[++i].Trim());
DateTime expires;
if (!DateTime.TryParseExact(
buff.ToString(),
new[] {"ddd, dd'-'MMM'-'yyyy HH':'mm':'ss 'GMT'", "r"},
new CultureInfo("en-US"),
DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal,
out expires))
expires = DateTime.Now;
if (cookie != null && cookie.Expires == DateTime.MinValue)
cookie.Expires = expires.ToLocalTime();
}
else if (pair.StartsWith("max-age", StringComparison.OrdinalIgnoreCase))
{
var max = Int32.Parse(GetValue(pair, '=', true));
var expires = DateTime.Now.AddSeconds(max);
if (cookie != null)
cookie.Expires = expires;
}
else if (pair.StartsWith("path", StringComparison.OrdinalIgnoreCase))
{
if (cookie != null)
cookie.Path = GetValue(pair, '=');
}
else if (pair.StartsWith("domain", StringComparison.OrdinalIgnoreCase))
{
if (cookie != null)
cookie.Domain = GetValue(pair, '=');
}
else if (pair.StartsWith("port", StringComparison.OrdinalIgnoreCase))
{
var port = pair.Equals("port", StringComparison.OrdinalIgnoreCase)
? "\"\""
: GetValue(pair, '=');
if (cookie != null)
cookie.Port = port;
}
else if (pair.StartsWith("comment", StringComparison.OrdinalIgnoreCase))
{
if (cookie != null)
cookie.Comment = System.Net.WebUtility.UrlDecode(GetValue(pair, '='));
}
else if (pair.StartsWith("commenturl", StringComparison.OrdinalIgnoreCase))
{
if (cookie != null)
cookie.CommentUri = GetValue(pair, '=', true).ToUri();
}
else if (pair.StartsWith("discard", StringComparison.OrdinalIgnoreCase))
{
if (cookie != null)
cookie.Discard = true;
}
else if (pair.StartsWith("secure", StringComparison.OrdinalIgnoreCase))
{
if (cookie != null)
cookie.Secure = true;
}
else if (pair.StartsWith("httponly", StringComparison.OrdinalIgnoreCase))
{
if (cookie != null)
cookie.HttpOnly = true;
}
else
{
if (cookie != null)
cookies.Add(cookie);
string name;
string val = String.Empty;
var pos = pair.IndexOf('=');
if (pos == -1)
{
name = pair;
}
else if (pos == pair.Length - 1)
{
name = pair.Substring(0, pos).TrimEnd(' ');
}
else
{
name = pair.Substring(0, pos).TrimEnd(' ');
val = pair.Substring(pos + 1).TrimStart(' ');
}
cookie = new Cookie(name, val);
}
}
if (cookie != null)
cookies.Add(cookie);
return cookies;
}
/// <summary>
/// Reads cookies from response and saves them all in user pc
/// </summary>
/// <param name="httpResponse">proxy response instance</param>
public override void RestoreCookiesFromResponse(WebResponse webResponse, bool saveAsTemporary)
{
if (!(webResponse is HttpWebResponse) || HttpContext.Current == null)
{
return;
}
// Declarations
HttpResponse userResponse = HttpContext.Current.Response;
HttpRequest userRequest = HttpContext.Current.Request;
HttpWebResponse httpWebResponse = (HttpWebResponse)webResponse;
Uri responseUrl = httpWebResponse.ResponseUri;
// if there is any cookie
if (httpWebResponse.Cookies.Count > 0)
{
CookieContainer container = new CookieContainer();
// BUGFIX: this is an temporary solution for when cookies are not specifed by back-end website
// And HttpWebResponse didn't recognize them well, so it sets extra Path value, which is not correct
bool isPathSpecified = false;
string setCookieHeader = httpWebResponse.Headers[HttpResponseHeader.SetCookie];
if (!string.IsNullOrEmpty(setCookieHeader) && setCookieHeader.ToLower().IndexOf("path=/") != -1)
{
isPathSpecified = true;
}
// if there is cookies
// restores cookies from user request
ApplyRequestCookiesToCookieContainer(container, userRequest, responseUrl);
// Response cookies
CookieCollection responseCookies = httpWebResponse.Cookies;
// BUGFIX: httpWebResponse does not parses the cookies in correct format
// Trying to parse it by Mono library functions
//responseCookies = ParseResponseCookies(httpWebResponse, responseUrl);
//---------------
// Adding response cookies,
// The new cookies will overwite the previous ones
// BUG: CookieContainer destoyes Expires value so the cookie never expires!
// BUFIX: bufix is below
// container.Add(responseCookies);
//-------------------------------
CookieCollection expiredCookies = new CookieCollection();
// BUFIX: We have to remove expired cookies!
// Because of CookieContainer bug, this method should do this for us!
// Damn this CookieContainer
foreach (Cookie cookie in responseCookies)
{
// no expired cookies
if (cookie.Expired)
{
expiredCookies.Add(cookie);
continue;
}
// good!
container.Add(cookie);
}
// Only for Micosoft .NET Framework
// Bug is fixed in .NET 4.0
if (IsRunningOnMicrosoftCLR && !IsRunningOnDotNet4)
{
// BUGFIX: CookieContainer has a bug, here is its bugfix
// To get around this bug, the domains should start with a DOT
BugFix_CookieContaierFix(container);
}
// CallMeLaNN:
// Get cookie header
// CookieCollection coll = container.GetCookies(webUri);
// Can't get all cookie header logically because it require Uri that can retrieve cookies on that domain and path only.
// Cookies in deeper subdomain or deeper path will not taken.
// We never know what subdomains or paths available here.
// So in order to get the all cookie in the entire domain, reflection is used.
// Or else we no need to use container earlier to do this.
// Container is design for easy to us to get the suitable cookies in the current domain and path + expiracy management.
// However Microsoft should add .GetAllCookies() method in case of this issue.
CookieCollection allCookies = GetAllCookies(container, responseUrl);
// SalarSoft:
// Ok, we have all cookies, we should not just get them all,
// Most of them applies to different domain
// So here i'm going to group them, then we can save them in their own domain
// Cookies are grouped by thier domains
Dictionary <string, CookieCollection> cookiesGroup = new Dictionary <string, CookieCollection>();
// applying cookies in groups
foreach (Cookie cookie in allCookies)
{
// Checking cookie expire option
// The expired cookie shouldn't be saved
// BUG: CookieContainer has destoyed Expires value so the cookie never expires!
if (cookie.Expired)
{
continue;
}
// Search within expired cookies to locate the original cookie
foreach (Cookie expired in expiredCookies)
{
// note: we do not check for value, because it can be differet
// but the import parts are tested
if (expired.Name == cookie.Name &&
expired.Domain == cookie.Domain &&
expired.Path == cookie.Path &&
expired.Port == cookie.Port)
{
// just modify expire date to be deleted next time
cookie.Expires = expired.Expires;
}
}
// BUGFIX: read the comments above
if (!isPathSpecified)
{
cookie.Path = "/";
}
// Get cookie name for current
string cookieName = GetCookieNameByDomain(cookie, responseUrl);
CookieCollection groupedCookie;
if (cookiesGroup.TryGetValue(cookieName, out groupedCookie))
{
// the groups exists before
groupedCookie.Add(cookie);
}
else
{
// new groups
groupedCookie = new CookieCollection();
groupedCookie.Add(cookie);
// add to cookie groups
cookiesGroup.Add(cookieName, groupedCookie);
}
}
// cookies calculated timeout in minute
DateTime expireDate;
if (saveAsTemporary)
{
// only 30 minutes
expireDate = DateTime.Now.AddMinutes(intExpireDateTempMinutes);
}
else
{
// To prevent cookie storage overflow, it should store cookies at most 6 days
expireDate = DateTime.Now.AddDays(intExpireDateNormalDays);
}
// Add grouped cookies to front-end user
foreach (KeyValuePair <string, CookieCollection> entry in cookiesGroup)
{
// new cookie by cookie name
HttpCookie frontendCookie = new HttpCookie(entry.Key);
// Header
string cookiesHeader = GetASProxyCookieHeader(entry.Value);
// Encode cookie header to make it safe
// CallMeLaNN: This is second layer encode
cookiesHeader = HttpUtility.UrlEncode(cookiesHeader);
// expire
if (!saveAsTemporary)
{
frontendCookie.Expires = expireDate;
}
else
{
// nothing, cookie should expire after the session (when user closed the browser)
}
// SalarSoft:
// Expire the cookie if there is no value to store
if (string.IsNullOrEmpty(cookiesHeader))
{
frontendCookie.Expires = DateTime.Now.AddYears(-5);
}
// Value
frontendCookie.Value = cookiesHeader;
// And finish...
userResponse.Cookies.Add(frontendCookie);
}
}
}
/// <summary>
/// SSO
/// </summary>
/// <param name="access_token"></param>
/// <returns>[0]=>(string)strCookies,[1]=>(string)csrf_token,[2]=>(DateTime)Expiress,[3]=>(CookieCollection)Cookies</returns>
public static object[] SSO(string access_token)
{
string parm = $"access_key={access_token}&appkey={Config.Instance.Appkey}&build={Config.Instance.Build}&gourl={UrlEncode("https://www.bilibili.com/")}&mobi_app=android&platform=android&ts={TimeStamp}";
parm += $"&sign={GetSign(parm)}";
HttpWebRequest req = null;
HttpWebResponse rep = null;
string cookies = "", csrf_token = "";
CookieCollection cookiesC = new CookieCollection();
DateTime expires = new DateTime();
try
{
req = (HttpWebRequest)WebRequest.Create("https://passport.bilibili.com/api/login/sso?" + parm);
req.AllowAutoRedirect = false;
req.UserAgent = Config.Instance.User_Agent;
rep = (HttpWebResponse)req.GetResponse();
foreach (string i in rep.Headers.GetValues("Set-Cookie"))
{
string[] tmp = i.Split(';');
string[] tmp2 = tmp[0].Split('=');
cookies += tmp[0] + "; ";
cookiesC.Add(new Cookie(tmp2[0], tmp2[1])
{
Domain = ".bilibili.com"
});
expires = DateTime.Parse(tmp[2].Split('=')[1]);
if (tmp2[0] == "bili_jct")
{
csrf_token = tmp2[1];
}
}
cookies = cookies.Substring(0, cookies.Length - 2);
}catch (WebException wex)
{
if (wex.Status == WebExceptionStatus.ProtocolError && ((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.Found)
{
foreach (string i in ((HttpWebResponse)wex.Response).Headers.GetValues("Set-Cookie"))
{
string[] tmp = i.Split(';');
string[] tmp2 = tmp[0].Split('=');
cookies += tmp[0] + "; ";
cookiesC.Add(new Cookie(tmp2[0], tmp2[1])
{
Domain = ".bilibili.com"
});
expires = DateTime.Parse(tmp[2].Split('=')[1]);
if (tmp2[0] == "bili_jct")
{
csrf_token = tmp2[1];
}
}
cookies = cookies.Substring(0, cookies.Length - 2);
}
}
finally
{
if (rep != null)
{
rep.Close();
}
if (req != null)
{
req.Abort();
}
}
return(new object[] { cookies, csrf_token, expires, cookiesC });
}
public static async Task <CookieCollection> GetCollectitonAsync()
{
var collection = new CookieCollection();
var cookiesFile = "data/cookies.txt";
if (File.Exists(cookiesFile))
{
var readAllText = File.ReadAllLines(cookiesFile).Where(s => !s.TrimStart().StartsWith("#"));
foreach (var line in readAllText)
{
var match = RegexCookiesTxt.Match(line);
if (match.Success)
{
try
{
var name = match.Groups[6].Value;
var value = match.Groups[7].Value;
if (name == "myCookie")
{
value = "username=&userPsw=";
}
collection.Add(new Cookie(name, value, match.Groups[3].Value, match.Groups[1].Value));
}
catch (Exception e)
{
Log.Warning(e.Message);
}
}
}
}
await SemaphoreSlim.WaitAsync();
if (AntiScrappingCookie == null)
{
try
{
var httpClient = new HttpClient();
httpClient.DefaultRequestHeaders.CacheControl = new CacheControlHeaderValue {
NoCache = true
};
httpClient.DefaultRequestHeaders.TryAddWithoutValidation(
"user-agent",
ScrappingConfiguration.Agent);
var requester = new HttpClientRequester(httpClient);
var config = Configuration.Default.WithRequester(requester)
.WithDefaultLoader(new LoaderOptions {
IsResourceLoadingEnabled = true
}).WithJs();
var context = BrowsingContext.New(config);
var document = await context.OpenAsync("https://www.tuenvio.cu/stores.json").WaitUntilAvailable();
var content = document.Body.TextContent;
if (!string.IsNullOrWhiteSpace(content))
{
var parametersMatch = RegexCall.Match(content);
if (parametersMatch.Success)
{
var cookieName = parametersMatch.Groups[1].Value.Trim();
var toNumbersACall = RegexA.Match(content).Groups[1].Value;
var toNumbersBCall = RegexB.Match(content).Groups[1].Value;
var toNumbersCCall = RegexC.Match(content).Groups[1].Value;
var parameters = parametersMatch.Groups[2].Value;
parameters = parameters.Replace("a", "%A%").Replace("b", "%B%").Replace("c", "%C%");
parameters = parameters.Replace("%A%", toNumbersACall).Replace("%B%", toNumbersBCall)
.Replace("%C%", toNumbersCCall);
// Review: looks like the WaitUntilAvailable method is not working properly.
var cookieValue = string.Empty;
while (string.IsNullOrWhiteSpace(cookieValue))
{
try
{
cookieValue = document.ExecuteScript($"toHex(slowAES.decrypt({parameters}))")
.ToString();
}
catch (Exception e)
{
Log.Warning(e, "Error retrieving the global cookie");
await Task.Delay(100);
}
}
AntiScrappingCookie = new Cookie(cookieName, cookieValue, "/", "www.tuenvio.cu");
}
}
}
catch (Exception e)
{
Log.Warning(e, "Error evaluating the anti-scrapping cookie.");
}
}
SemaphoreSlim.Release();
var cookie = collection.FirstOrDefault(c => c.Name == AntiScrappingCookie.Name);
if (cookie != null)
{
collection.Remove(cookie);
}
collection.Add(AntiScrappingCookie);
return(collection);
}
/// <summary>
/// 发送GET请求
/// </summary>
/// <param name="url"></param>
/// <param name="refer"></param>
/// <returns></returns>
public String HttpGet(String url, String refer)
{
String html;
try
{
//ServicePointManager.ServerCertificateValidationCallback = new System.Net.Security.RemoteCertificateValidationCallback(CheckCertificate);
HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);
request.UserAgent = _useragent;
request.Timeout = _timeout;
request.ContentType = _contenttype;
request.Accept = _accept;
request.Method = "GET";
request.Referer = refer;
request.KeepAlive = true;
request.AllowAutoRedirect = true;
request.UnsafeAuthenticatedConnectionSharing = true;
request.CookieContainer = new CookieContainer();
//据说能提高性能
//request.Proxy = null;
if (_cookiecollection != null)
{
foreach (Cookie c in _cookiecollection)
{
c.Domain = request.Host;
}
request.CookieContainer.Add(_cookiecollection);
}
foreach (KeyValuePair <String, String> hd in _headers)
{
request.Headers[hd.Key] = hd.Value;
}
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
html = GetStringFromResponse(response);
if (request.CookieContainer != null)
{
response.Cookies = request.CookieContainer.GetCookies(request.RequestUri);
}
if (response.Cookies != null)
{
_cookiecollection = response.Cookies;
}
if (response.Headers["Set-Cookie"] != null)
{
string tmpcookie = response.Headers["Set-Cookie"];
_cookiecollection.Add(ConvertCookieString(tmpcookie));
}
response.Close();
return(html);
}
catch (Exception e)
{
Trace.WriteLine("HttpGet Error: " + e.Message);
return(String.Empty);
}
}
public void MakeRequests(Object stateInfo)
{
var requests = stateInfo as TestInfo;
totalTransactions++;
int currentTransaction = totalTransactions;
Console.WriteLine("Begin Transaction # {0}", currentTransaction);
CookieCollection persistCookies = new CookieCollection();
// Start the stopwatch
var timer = new System.Diagnostics.Stopwatch();
timer.Start();
foreach (var requestData in requests.testData)
{
Uri firstUri = new Uri(requestData.url);
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(requestData.url);
if (requestData.requestType.ToLower() == "post")
{
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
StringBuilder sb = new StringBuilder();
using (StreamReader stream = new StreamReader(response.GetResponseStream())) {
string strLine;
while ((strLine = stream.ReadLine()) != null)
{
if (strLine.Length > 0)
{
sb.Append(strLine);
}
}
stream.Close();
}
response.Close();
string rawData = sb.ToString();
// Find the inputs on the page
Regex filters = new Regex(@"<input.+?\/?>");
MatchCollection matches = filters.Matches(rawData);
XmlDocument xmlDoc = new XmlDocument();
NameValueCollection formData = new NameValueCollection();
// Extract the name/value pairs of the input names and values
for (int i = 0; i < matches.Count; i++)
{
xmlDoc.LoadXml(matches[i].Value);
XmlNode node = xmlDoc.DocumentElement.Attributes["value"];
string value = node != null ? node.Value : string.Empty;
formData.Add(xmlDoc.DocumentElement.Attributes["name"].Value, value);
}
foreach (string key in requestData.postData.Keys)
{
if (formData.AllKeys.Contains(key))
{
formData[key] = requestData.postData[key];
}
}
// Post the request
HttpWebRequest requestPost = (HttpWebRequest)WebRequest.Create(requestData.url);
byte[] byteArrayPostAttendance = Encoding.UTF8.GetBytes(Helpers.GenerateTargetRequestData(formData));
requestPost.ContentType = "application/x-www-form-urlencoded";
requestPost.Method = "POST";
requestPost.AllowAutoRedirect = false;
requestPost.CookieContainer = new CookieContainer();
requestPost.ContentLength = byteArrayPostAttendance.Length;
using (Stream postDataStreamNew = requestPost.GetRequestStream()) {
postDataStreamNew.Write(byteArrayPostAttendance, 0, byteArrayPostAttendance.Length);
}
HttpWebResponse requestPostResponse = (HttpWebResponse)requestPost.GetResponse();
persistCookies = requestPostResponse.Cookies;
if (requestPostResponse.StatusCode == HttpStatusCode.Found)
{
requestPostResponse.Close();
string target = string.Empty;
if (Uri.IsWellFormedUriString(requestPostResponse.Headers["Location"], UriKind.Absolute))
{
target = requestPostResponse.Headers["Location"];
}
else
{
target = firstUri.Scheme + "://" + firstUri.Authority + requestPostResponse.Headers["Location"];
}
HttpWebRequest redirectRequest = (HttpWebRequest)WebRequest.Create(target);
CookieContainer cookieContainer = new CookieContainer();
foreach (Cookie cookie in persistCookies)
{
cookieContainer.Add(cookie);
}
redirectRequest.CookieContainer = cookieContainer;
redirectRequest.AllowAutoRedirect = false;
requestPostResponse = (HttpWebResponse)redirectRequest.GetResponse();
persistCookies.Add(requestPostResponse.Cookies);
}
requestPostResponse.Close();
}
else
{
CookieContainer cookieContainer = new CookieContainer();
foreach (Cookie cookie in persistCookies)
{
cookieContainer.Add(cookie);
}
request.CookieContainer = cookieContainer;
request.GetResponse();
}
}
// Stop the stopwatch, write the result to the console
timer.Stop();
Console.WriteLine("End Transaction # {0} of {1} - {2} took {3}ms", currentTransaction, totalTransactions, requests.testData[requests.testData.Count - 1].url, timer.ElapsedMilliseconds);
}
private void GoSerf()
{
MyProxy p = null;
try
{
while (true)
{
// выбираем случайный прокси
lock (proxies)
if (proxies.Count == 0) { manager.StopJob("Готово!"); return; }
else
{
int rand = r.Next(proxies.Count);
p = proxies[rand];
proxies.Remove(p);
}
// получаем aid
CookieCollection cook = new CookieCollection();
cook.Add("tbver", version, ".alexa.com");
string res = VkRequest.Request("http://alexa.com/toolbar?utm_source=top-nav&utm_medium=www&utm_campaign=toolbar", proxy: p, cookies: cook).Content;
long random = r.Next(111111111,999999999) * 9999 + 1;
res = VkRequest.Request("http://log.alexa.com/prefs/?aid=&plugin=alxg-3.2&random=" + random +"&name=extensions.alexa.toolbarLocaleChange&value=", proxy: p, cookies: cook).Content;
res = VkRequest.Request("http://data.alexa.com/data?cli=10&stc", proxy: p, cookies: cook).Content;
string aid = cook["aid"];
string lv = cook["lv"];
string temp = string.Format("oeu{0}r0.{1}", (100000000 + r.Next(900000000)) * 1000, (100000000 + r.Next(900000000)) * 100000);
cook.Add("optimizelyEndUserId", temp, ".alexa.com");
int num = 176000000 + r.Next(999999);
temp = string.Format(@"%7B%22{0}%22%3A%22gc%22%2C%22{1}%22%3A%22campaign%22%2C%22{2}%22%3A%22false%22%7D", 176000000 + r.Next(999999), num, num + 1);
cook.Add("optimizelySegments", temp, ".alexa.com");
cook.Add("optimizelyBuckets", "%7B%7D", ".alexa.com");
cook.Add("__asc", Path.GetRandomFileName().Replace(".", "") + Path.GetRandomFileName().Replace(".", ""), ".alexa.com");
cook.Add("__auc", Path.GetRandomFileName().Replace(".", "") + Path.GetRandomFileName().Replace(".", ""), ".alexa.com");
num = r.Next(100000000) + 100000000;
temp = string.Format("{0}.{1}.{2}.{3}.{4}.{5}", num, r.Next(200000000), r.Next(200000000), r.Next(200000000), r.Next(200000000), r.Next(200000000));
cook.Add("__utma", temp, ".alexa.com");
temp = string.Format("{0}.{1}.{2}.{3}", num, r.Next(200000000), r.Next(200000000), r.Next(200000000));
cook.Add("__utmb", temp, ".alexa.com");
cook.Add("__utmc", num.ToString(), ".alexa.com");
temp = string.Format("{0}.{1}.{2}.{3}", num, r.Next(200000000), r.Next(200000000), r.Next(200000000));
cook.Add("__utmz", temp, ".alexa.com");
cook.Add("utmcsr", "top-nav|utmccn=toolbar|utmcmd=www", ".alexa.com");
random = r.Next(111111111, 999999999) * 9999 + 1;
res = VkRequest.Request("http://log.alexa.com/prefs/?aid=" + aid + "&plugin=alxg-3.2&random=" + random + "&name=extensions.alexa.post-defined.platform_version&value=3.2", proxy: p, cookies: cook).Content;
random = r.Next(111111111, 999999999) * 9999 + 1;
res = VkRequest.Request("http://log.alexa.com/prefs/?aid=" + aid + "&plugin=alxg-3.2&random=" + random + "&name=extensions.alexa.session&value="+aid, proxy: p, cookies: cook).Content;
random = r.Next(111111111, 999999999) * 9999 + 1;
res = VkRequest.Request("http://log.alexa.com/prefs/?aid=" + aid + "&plugin=alxg-3.2&random=" + random + "&name=extensions.alexa.giurls.giResponseTimeOut&value=3000", proxy: p, cookies: cook).Content;
random = r.Next(111111111, 999999999) * 9999 + 1;
res = VkRequest.Request("http://log.alexa.com/prefs/?aid=" + aid + "&plugin=alxg-3.2&random=" + random + "&name=extensions.alexa.giurls.giUrls&value=%5B%22http(s)%3F%3A%5C%5C%2F%5C%5C%2F(www(%7C%5B0-9%5D)%7Cencrypted)%5C%5C.(%7Cl%5C%5C.)google%5C%5C..*%5C%5C%2F%22%5D",
proxy: p, cookies: cook).Content;
random = r.Next(111111111, 999999999) * 9999 + 1;
res = VkRequest.Request("http://log.alexa.com/prefs/?aid=" + aid + "&plugin=alxg-3.2&random=" + random + "&name=extensions.alexa.post-defined.base_toolbar&value=https%3A%2F%2Fs3.amazonaws.com%2Fcom.alexa.toolbar%2Falx%2Fg", proxy: p, cookies: cook).Content;
random = r.Next(111111111, 999999999) * 9999 + 1;
res = VkRequest.Request("http://log.alexa.com/prefs/?aid=" + aid + "&plugin=alxg-3.2&random=" + random + "&dad=0&search_provider=0&error_handling=0&toolbar_locale=default", proxy: p, cookies: cook).Content;
random = r.Next(111111111, 999999999) * 9999 + 1;
res = VkRequest.Request("http://log.alexa.com/prefs/?aid=" + aid + "&plugin=alxg-3.2&random=" + random + "&name=extensions.alexa.toolbarXMLVersion&value=alxg-3.2", proxy: p, cookies: cook).Content;
Thread.Sleep(r.Next(end - beg));
string post = string.Format("amzn_id={0}=&browser=chrome&plugin=&gender=1&age=18-24&income=0-30kðnicity=&education=&has_children=&install_location=&zip=&Submit=+++Submit+++&version=6.5", aid);
res = VkRequest.Request("http://www.alexa.com/php/demographics/index.php", post, proxy: p, cookies: cook).Headers["Set-Cookie"].ToString();
int currentCountRequest = 0;
int count = countRequestProxy + r.Next(countRequestEnd - countRequestProxy);
//Uri.EscapeUriString("");
while ((currentCountRequest < count) && (aid != null))
{
string uri = urlStat[r.Next(urlStat.Count)];
string get = string.Format("http://data.alexa.com/data/{0}?cli=10&ver=alxg-3.2&dat=ns&cdt=rq=0&wid=4&t=0&ss=1366x768&bw=1366&winid=1&cttl=3941&m=GET&s=200&ref={1}&url={2}", aid, Uri.EscapeUriString(uri), Uri.EscapeUriString(getPageUri(uri)));
var result = VkRequest.Request(get, proxy: p, cookies: cook);
//string page = VkRequest.Request(get, proxy: p, cookies: cook).Content ;
if (result.Headers.HTTPResponseCode == 200)
{
request++;
currentCountRequest++;
}
else
{
if (result.Headers.HTTPResponseCode == 400 || result.Headers.HTTPResponseCode == 403)
{
log(string.Format("Невалидный прокси {0} удален.", p.ToString()));
lock (proxies)
{
proxies.Remove(p);
currentCountRequest = countRequestProxy;
}
}
else
{
log(string.Format("Ошибка при запросе с прокси {0}", p.ToString()));
p.ErrorsCount++;
if (p.ErrorsCount >= MyProxy.MaxErrors)
{
currentCountRequest = countRequestProxy;
lock (proxies)
proxies.Remove(p);
}
}
}
Thread.Sleep(beg + r.Next(end - beg));
//Wait(beg + r.Next(end - beg));
}
goodProxy++;
log(string.Format("Прокси {0} работу закончил, удален", p));
lock (proxies)
proxies.Remove(p);
}
}
catch (Exception e)
{
if (p != null)
{
log(string.Format("Ошибка: {0}; Прокси = {1}", e.InnerMessages(), p));
p.ErrorsCount++;
if (p.ErrorsCount >= MyProxy.MaxErrors)
lock (proxies)
proxies.Remove(p);
}
else log(string.Format("Ошибка: {0}", e.InnerMessages()));
}
}
/// <summary>
/// 获取所有Cookie 通过Set-Cookie
/// </summary>
/// <param name="setCookie"></param>
/// <returns></returns>
public static CookieCollection GetCookiesByHeader(string setCookie)
{
CookieCollection cookieCollection = new CookieCollection();
//拆分Cookie
//var listStr = RegexSplitCookie.Split(setCookie);
setCookie += ",T";//配合RegexSplitCookie2 加入后缀
MatchCollection listStr = RegexSplitCookie2.Matches(setCookie);
//循环遍历
foreach (Match item in listStr)
{
//根据; 拆分Cookie 内容
string[] cookieItem = item.Value.Split(';');
Cookie cookie = new Cookie();
for (int index = 0; index < cookieItem.Length; index++)
{
string info = cookieItem[index];
//第一个 默认 Cookie Name
//判断键值对
if (info.Contains("="))
{
int indexK = info.IndexOf('=');
string name = info.Substring(0, indexK).Trim();
string val = info.Substring(indexK + 1);
if (index == 0)
{
cookie.Name = name;
cookie.Value = val;
continue;
}
if (name.Equals("Domain", StringComparison.OrdinalIgnoreCase))
{
cookie.Domain = val;
}
else if (name.Equals("Expires", StringComparison.OrdinalIgnoreCase))
{
DateTime.TryParse(val, out var expires);
cookie.Expires = expires;
}
else if (name.Equals("Path", StringComparison.OrdinalIgnoreCase))
{
cookie.Path = val;
}
else if (name.Equals("Version", StringComparison.OrdinalIgnoreCase))
{
cookie.Version = Convert.ToInt32(val);
}
}
else
{
if (info.Trim().Equals("HttpOnly", StringComparison.OrdinalIgnoreCase))
{
cookie.HttpOnly = true;
}
}
}
cookieCollection.Add(cookie);
}
return(cookieCollection);
}
private static CookieCollection ParseRequest(string value)
{
var cookies = new CookieCollection();
Cookie cookie = null;
var ver = 0;
var pairs = SplitCookieHeaderValue(value);
foreach (var t in pairs)
{
var pair = t.Trim();
if (pair.Length == 0)
continue;
if (pair.StartsWith("$version", StringComparison.OrdinalIgnoreCase))
{
ver = int.Parse(GetValue(pair, '=', true));
}
else if (pair.StartsWith("$path", StringComparison.OrdinalIgnoreCase))
{
if (cookie != null)
cookie.Path = GetValue(pair, '=');
}
else if (pair.StartsWith("$domain", StringComparison.OrdinalIgnoreCase))
{
if (cookie != null)
cookie.Domain = GetValue(pair, '=');
}
else if (pair.StartsWith("$port", StringComparison.OrdinalIgnoreCase))
{
var port = pair.Equals("$port", StringComparison.OrdinalIgnoreCase)
? "\"\""
: GetValue(pair, '=');
if (cookie != null)
cookie.Port = port;
}
else
{
if (cookie != null)
cookies.Add(cookie);
string name;
var val = String.Empty;
var pos = pair.IndexOf('=');
if (pos == -1)
{
name = pair;
}
else if (pos == pair.Length - 1)
{
name = pair.Substring(0, pos).TrimEnd(' ');
}
else
{
name = pair.Substring(0, pos).TrimEnd(' ');
val = pair.Substring(pos + 1).TrimStart(' ');
}
cookie = new Cookie(name, val);
if (ver != 0)
cookie.Version = ver;
}
}
if (cookie != null)
cookies.Add(cookie);
return cookies;
}