static void OnBrowserRequestLogged(Browser req, HttpRequestLog log)
{
Console.WriteLine(" -> " + log.Method + " request to " + log.Url);
Console.WriteLine(" <- Response status code: " + log.ResponseCode);
}
void HandleRequestLogged(Browser b, HttpRequestLog req)
{
this.LastRequest = req;
}
internal bool DoRequest(Uri uri, string method, NameValueCollection userVariables, string postData, string contentType, string encodingType, int timeoutMilliseconds)
{
/* IMPORTANT INFORMATION:
* HttpWebRequest has a bug where if a 302 redirect is encountered (such as from a Response.Redirect), any cookies
* generated during the request are ignored and discarded during the internal redirect process. The headers are in
* fact returned, but the normal process where the cookie headers are turned into Cookie objects in the cookie
* container is skipped, thus breaking the login processes of half the sites on the internet.
*
* The workaround is as follows:
* 1. Turn off AllowAutoRedirect so we can intercept the redirect and do things manually
* 2. Read the Set-Cookie headers from the response and manually insert them into the cookie container
* 3. Get the Location header and redirect to the location specified in the "Location" response header
*
* Worth noting that even if this bug has been solved in .Net 4 (I haven't checked) we should still use manual
* redirection so that we can properly log responses.
*
* OBSOLETE ISSUE: (Bug has been resolved in the .Net 4 framework, which this library is now targeted at)
* //CookieContainer also has a horrible bug relating to the specified cookie domain. Basically, if it contains
* //a cookie where the "domain" token is specified as ".domain.xxx" and you attempt to request http://domain.ext,
* //the cookies are not retrievable via that Uri, as you would expect them to be. CookieContainer is incorrectly
* //assuming that the leading dot is a prerequisite specifying that a subdomain is required as opposed to the
* //correct behaviour which would be to take it to mean that the domain and all subdomains are valid for the cookie.
* //http://channel9.msdn.com/forums/TechOff/260235-Bug-in-CookieContainer-where-do-I-report/?CommentID=397315
* //The workaround is as follows:
* //When retrieving the response, iterate through the Set-Cookie header and any cookie that explicitly sets
* //the domain token with a leading dot should also set the cookie without the leading dot.
*/
bool handle301Or302Redirect;
int maxRedirects = 10;
string html;
string postBody = "";
do
{
Debug.WriteLine(uri.ToString());
if (maxRedirects-- == 0)
{
Log("Too many 302 redirects", LogMessageType.Error);
return(false);
}
handle301Or302Redirect = false;
IHttpWebRequest req = null;
try
{
req = PrepareRequestObject(uri, method, contentType, timeoutMilliseconds);
}
catch (NotSupportedException)
{
// Happens when the URL cannot be parsed (example: 'javascript:')
return(false);
}
foreach (var header in _extraHeaders)
{
if (header.StartsWith("host:", StringComparison.OrdinalIgnoreCase))
{
req.Host = header.Split(':')[1];
}
else
{
req.Headers.Add(header);
}
}
if (!string.IsNullOrEmpty(encodingType))
{
req.Headers.Add(HttpRequestHeader.ContentEncoding, encodingType);
}
if (_includeFormValues != null)
{
if (userVariables == null)
{
userVariables = _includeFormValues;
}
else
{
userVariables.Add(_includeFormValues);
}
}
if (userVariables != null)
{
if (method == "POST")
{
postBody = StringUtil.MakeQueryString(userVariables);
byte[] data = Encoding.GetEncoding(28591).GetBytes(postBody);
req.ContentLength = data.Length;
using (Stream stream = req.GetRequestStream())
{
stream.Write(data, 0, data.Length);
}
}
else
{
uri = new Uri(
uri.Scheme + "://" + uri.Host + ":" + uri.Port + uri.AbsolutePath
+ ((userVariables.Count > 0) ? "?" + StringUtil.MakeQueryString(userVariables) : "")
);
req = PrepareRequestObject(uri, method, contentType, timeoutMilliseconds);
}
}
else if (postData != null)
{
if (method == "GET")
{
throw new InvalidOperationException("Cannot call DoRequest with method GET and non-null postData");
}
postBody = postData;
byte[] data = Encoding.GetEncoding(28591).GetBytes(postData);
req.ContentLength = data.Length;
using (Stream stream = req.GetRequestStream())
{
stream.Write(data, 0, data.Length);
}
}
if (contentType != null)
{
req.ContentType = contentType;
}
_lastRequestLog = new HttpRequestLog
{
Method = method,
PostData = userVariables,
PostBody = postBody,
RequestHeaders = req.Headers,
Url = uri
};
try
{
using (IHttpWebResponse response = req.GetResponse())
{
Encoding responseEncoding = Encoding.UTF8; //default
string charSet = response.CharacterSet;
if (!String.IsNullOrEmpty(charSet))
{
try
{
responseEncoding = Encoding.GetEncoding(charSet);
}
catch (ArgumentException)
{
responseEncoding = Encoding.UTF8; // try using utf8
}
}
//ensure the stream is disposed
using (Stream rs = response.GetResponseStream())
{
using (StreamReader reader = new StreamReader(rs, responseEncoding))
{
html = reader.ReadToEnd();
}
}
_doc = null;
_includeFormValues = null;
_lastRequestLog.Text = html;
_lastRequestLog.ResponseHeaders = response.Headers;
_lastRequestLog.StatusCode = (int)response.StatusCode;
if (method == "GET" && uri.Query.Length > 0 && uri.Query != "?")
{
_lastRequestLog.QueryStringData = HttpUtility.ParseQueryString(uri.Query);
}
if ((int)response.StatusCode == 302 || (int)response.StatusCode == 301)
{
uri = new Uri(uri, response.Headers["Location"]);
handle301Or302Redirect = true;
Debug.WriteLine("Redirecting to: " + uri);
method = "GET";
postData = null;
userVariables = null;
}
}
}
catch (WebException ex)
{
_lastRequestLog.StatusCode = (int)ex.Status.GetTypeCode();
if (ex.Response != null)
{
_lastRequestLog.ResponseHeaders = ex.Response.Headers;
//ensure the stream is disposed
using (Stream rs = ex.Response.GetResponseStream())
{
using (StreamReader reader = new StreamReader(rs))
{
html = reader.ReadToEnd();
}
}
_lastRequestLog.Text = html;
}
LastWebException = ex;
switch (ex.Status)
{
case WebExceptionStatus.Timeout:
Log("A timeout occurred while trying to load the web page", LogMessageType.Error);
break;
case WebExceptionStatus.ReceiveFailure:
Log("The response was cut short prematurely", LogMessageType.Error);
break;
default:
Log("An exception was thrown while trying to load the page: " + ex.Message, LogMessageType.Error);
break;
}
return(false);
}
finally
{
LogRequestData();
}
} while (handle301Or302Redirect);
this.RemoveChildBrowsers(); //Any frames contained in the previous state should be removed. They will be recreated if we ever navigate back
this.AddNavigationState(new NavigationState()
{
Html = html, Url = uri, ContentType = contentType
});
return(true);
}
internal bool DoRequest(Uri uri, string method, NameValueCollection userVariables, string postData, string contentType, string encodingType, int timeoutMilliseconds)
{
string html;
string referer = null;
if (uri.IsFile)
{
StreamReader reader = new StreamReader(uri.AbsolutePath);
html = reader.ReadToEnd();
reader.Close();
}
else
{
bool handle3xxRedirect = false;
int maxRedirects = 5; // Per RFC2068, Section 10.3
string postBody = string.Empty;
do
{
Debug.WriteLine(uri.ToString());
if (maxRedirects-- == 0)
{
Log("Too many 3xx redirects", LogMessageType.Error);
return(false);
}
handle3xxRedirect = false;
IHttpWebRequest req = null;
try
{
req = PrepareRequestObject(uri, method, contentType, timeoutMilliseconds);
}
catch (NotSupportedException)
{
// Happens when the URL cannot be parsed (example: 'javascript:')
return(false);
}
foreach (var header in _extraHeaders)
{
if (header.StartsWith("host:", StringComparison.OrdinalIgnoreCase))
{
req.Host = header.Split(':')[1];
}
else
{
req.Headers.Add(header);
}
}
if (!string.IsNullOrEmpty(encodingType))
{
req.Headers.Add(HttpRequestHeader.ContentEncoding, encodingType);
}
// Remove all expired basic authentication tokens
List <BasicAuthenticationToken> expired =
_basicAuthenticationTokens.Values.Where(t => DateTime.Now > t.Expiration).ToList();
foreach (var expiredToken in expired)
{
_basicAuthenticationTokens.Remove(expiredToken.Domain);
}
// If an authentication token exists for the domain, add the authorization header.
foreach (var token in _basicAuthenticationTokens.Values)
{
if (req.Host.Contains(token.Domain))
{
// Extend the expiration.
token.UpdateExpiration();
// Add the authentication header.
req.Headers.Add(string.Format(
"Authorization: Basic {0}",
token.Token));
}
}
if (_includeFormValues != null)
{
if (userVariables == null)
{
userVariables = _includeFormValues;
}
else
{
userVariables.Add(_includeFormValues);
}
}
if (userVariables != null)
{
if (method == "POST")
{
postBody = StringUtil.MakeQueryString(userVariables);
byte[] data = Encoding.GetEncoding(28591).GetBytes(postBody);
req.ContentLength = data.Length;
using (Stream stream = req.GetRequestStream())
{
stream.Write(data, 0, data.Length);
}
}
else
{
uri = new Uri(
uri.Scheme + "://" + uri.Host + ":" + uri.Port + uri.AbsolutePath
+ ((userVariables.Count > 0) ? "?" + StringUtil.MakeQueryString(userVariables) : "")
);
req = PrepareRequestObject(uri, method, contentType, timeoutMilliseconds);
}
}
else if (postData != null)
{
if (method == "GET")
{
throw new InvalidOperationException("Cannot call DoRequest with method GET and non-null postData");
}
postBody = postData;
byte[] data = Encoding.GetEncoding(28591).GetBytes(postData);
req.ContentLength = data.Length;
using (Stream stream = req.GetRequestStream())
{
stream.Write(data, 0, data.Length);
}
}
referer = req.Referer;
if (contentType != null)
{
req.ContentType = contentType;
}
_lastRequestLog = new HttpRequestLog
{
Method = method,
PostData = userVariables,
PostBody = postBody,
RequestHeaders = req.Headers,
Url = uri
};
try
{
using (IHttpWebResponse response = req.GetResponse())
{
Encoding responseEncoding = Encoding.UTF8; //default
string charSet = response.CharacterSet;
if (!String.IsNullOrEmpty(charSet))
{
try
{
responseEncoding = Encoding.GetEncoding(charSet);
}
catch (ArgumentException)
{
responseEncoding = Encoding.UTF8; // try using utf8
}
}
//ensure the stream is disposed
using (Stream rs = response.GetResponseStream())
{
using (StreamReader reader = new StreamReader(rs, responseEncoding))
{
html = reader.ReadToEnd();
}
}
_doc = null;
_includeFormValues = null;
_lastRequestLog.Text = html;
_lastRequestLog.ResponseHeaders = response.Headers;
_lastRequestLog.ResponseCode = (int)response.StatusCode;
if (method == "GET" && uri.Query.Length > 0 && uri.Query != "?")
{
_lastRequestLog.QueryStringData = HttpUtility.ParseQueryString(uri.Query);
}
if (AutoRedirect == true &&
(((int)response.StatusCode == 300 || // Not entirely supported. If provided, the server's preference from the Location header is honored.
(int)response.StatusCode == 301 ||
(int)response.StatusCode == 302 ||
(int)response.StatusCode == 303 ||
// 304 - Unsupported, conditional Get requests are not supported (mostly because SimpleBrowser does not cache content)
// 305 - Unsupported, possible security threat
// 306 - No longer used, per RFC2616, Section 10.3.7
(int)response.StatusCode == 307 ||
(int)response.StatusCode == 308) &&
response.Headers.AllKeys.Contains("Location")))
{
uri = new Uri(uri, response.Headers["Location"]);
handle3xxRedirect = true;
Debug.WriteLine("Redirecting to: " + uri);
method = "GET";
postData = null;
userVariables = null;
}
if (response.Headers.AllKeys.Contains("Set-Cookie"))
{
var cookies = SetCookieHeaderParser.GetAllCookiesFromHeader(uri.Host, response.Headers["Set-Cookie"]);
Cookies.Add(cookies);
}
}
}
catch (WebException ex)
{
if (ex.Response != null)
{
_lastRequestLog.ResponseHeaders = ex.Response.Headers;
//ensure the stream is disposed
using (Stream rs = ex.Response.GetResponseStream())
{
using (StreamReader reader = new StreamReader(rs))
{
html = reader.ReadToEnd();
}
}
_lastRequestLog.Text = html;
}
LastWebException = ex;
switch (ex.Status)
{
case WebExceptionStatus.Timeout:
Log("A timeout occurred while trying to load the web page", LogMessageType.Error);
break;
case WebExceptionStatus.ReceiveFailure:
Log("The response was cut short prematurely", LogMessageType.Error);
break;
default:
Log("An exception was thrown while trying to load the page: " + ex.Message, LogMessageType.Error);
break;
}
return(false);
}
finally
{
LogRequestData();
}
}while (handle3xxRedirect);
}
this._navigationAttributes = null;
this.RemoveChildBrowsers(); //Any frames contained in the previous state should be removed. They will be recreated if we ever navigate back
this.AddNavigationState(
new NavigationState()
{
Html = html,
Url = uri,
ContentType = contentType,
Referer = string.IsNullOrEmpty(referer) ? null : new Uri(Uri.UnescapeDataString(referer))
});
return(true);
}
private bool DoRequest(Uri uri, string method, NameValueCollection userVariables, string postData, string contentType, int timeoutMilliseconds)
{
/* IMPORTANT INFORMATION:
* HttpWebRequest has a bug where if a 302 redirect is encountered (such as from a Response.Redirect), any cookies
* generated during the request are ignored and discarded during the internal redirect process. The headers are in
* fact returned, but the normal process where the cookie headers are turned into Cookie objects in the cookie
* container is skipped, thus breaking the login processes of half the sites on the internet.
*
* The workaround is as follows:
* 1. Turn off AllowAutoRedirect so we can intercept the redirect and do things manually
* 2. Read the Set-Cookie headers from the response and manually insert them into the cookie container
* 3. Get the Location header and redirect to the location specified in the "Location" response header
*
* Worth noting that even if this bug has been solved in .Net 4 (I haven't checked) we should still use manual
* redirection so that we can properly log responses.
*
* OBSOLETE ISSUE: (Bug has been resolved in the .Net 4 framework, which this library is now targeted at)
* //CookieContainer also has a horrible bug relating to the specified cookie domain. Basically, if it contains
* //a cookie where the "domain" token is specified as ".domain.xxx" and you attempt to request http://domain.ext,
* //the cookies are not retrievable via that Uri, as you would expect them to be. CookieContainer is incorrectly
* //assuming that the leading dot is a prerequisite specifying that a subdomain is required as opposed to the
* //correct behaviour which would be to take it to mean that the domain and all subdomains are valid for the cookie.
* //http://channel9.msdn.com/forums/TechOff/260235-Bug-in-CookieContainer-where-do-I-report/?CommentID=397315
* //The workaround is as follows:
* //When retrieving the response, iterate through the Set-Cookie header and any cookie that explicitly sets
* //the domain token with a leading dot should also set the cookie without the leading dot.
*/
bool handle301Or302Redirect;
int maxRedirects = 10;
string html;
do
{
Debug.WriteLine(uri.ToString());
if(maxRedirects-- == 0)
{
Log("Too many 302 redirects", LogMessageType.Error);
return false;
}
handle301Or302Redirect = false;
HttpWebRequest req = PrepareRequestObject(uri, method, timeoutMilliseconds);
foreach(var header in _extraHeaders)
req.Headers.Add(header);
if(_includeFormValues != null)
{
if(userVariables == null)
userVariables = _includeFormValues;
else
userVariables.Add(_includeFormValues);
}
if(userVariables != null)
{
if(method == "POST")
{
byte[] data = Encoding.ASCII.GetBytes(StringUtil.MakeQueryString(userVariables));
req.ContentLength = data.Length;
Stream stream = req.GetRequestStream();
stream.Write(data, 0, data.Length);
stream.Close();
}
else
{
uri = new Uri(uri.Scheme + "://" + uri.Host + uri.AbsolutePath + "?" + StringUtil.MakeQueryString(userVariables));
req = PrepareRequestObject(uri, method, timeoutMilliseconds);
}
}
else if(postData != null)
{
if(method == "GET")
throw new InvalidOperationException("Cannot call DoRequest with method GET and non-null postData");
byte[] data = Encoding.ASCII.GetBytes(postData);
req.ContentLength = data.Length;
Stream stream = req.GetRequestStream();
stream.Write(data, 0, data.Length);
stream.Close();
}
if(contentType != null)
req.ContentType = contentType;
try
{
using(HttpWebResponse response = (HttpWebResponse)req.GetResponse())
{
StreamReader reader = new StreamReader(response.GetResponseStream());
html = reader.ReadToEnd();
ResponseText = html;
reader.Close();
string oldHTML = html;
//html = StripAndRebuildHtml(html);
CurrentHtml = html;
ContentType = response.ContentType;
_doc = null;
_includeFormValues = null;
_lastRequestLog = new HttpRequestLog
{
Text = oldHTML,
ParsedHtml = XDocument.ToString(),
Method = method,
PostData = userVariables,
RequestHeaders = req.Headers,
ResponseHeaders = response.Headers,
StatusCode = (int)response.StatusCode,
Url = uri
};
if(method == "GET" && uri.Query.Length > 0 && uri.Query != "?")
_lastRequestLog.QueryStringData = HttpUtility.ParseQueryString(uri.Query);
LogRequestData();
if((int)response.StatusCode == 302 || (int)response.StatusCode == 301)
{
//url = AdjustUrl(url, response.Headers["Location"]);
uri = new Uri(uri, response.Headers["Location"]);
handle301Or302Redirect = true;
Url = uri;
Debug.WriteLine("Redirecting to: " + Url);
method = "GET";
postData = null;
userVariables = null;
}
}
}
catch(WebException ex)
{
LastWebException = ex;
switch(ex.Status)
{
case WebExceptionStatus.Timeout:
Log("A timeout occurred while trying to load the web page", LogMessageType.Error);
break;
case WebExceptionStatus.ReceiveFailure:
Log("The response was cut short prematurely", LogMessageType.Error);
break;
default:
Log("An exception was thrown while trying to load the page: " + ex.Message, LogMessageType.Error);
break;
}
return false;
}
} while(handle301Or302Redirect);
Url = uri;
_referrerUrl = uri;
CurrentHtml = html;
return true;
}
private bool DoRequest(Uri uri, string method, NameValueCollection userVariables, string postData, string contentType, int timeoutMilliseconds)
{
/* IMPORTANT INFORMATION:
* HttpWebRequest has a bug where if a 302 redirect is encountered (such as from a Response.Redirect), any cookies
* generated during the request are ignored and discarded during the internal redirect process. The headers are in
* fact returned, but the normal process where the cookie headers are turned into Cookie objects in the cookie
* container is skipped, thus breaking the login processes of half the sites on the internet.
*
* The workaround is as follows:
* 1. Turn off AllowAutoRedirect so we can intercept the redirect and do things manually
* 2. Read the Set-Cookie headers from the response and manually insert them into the cookie container
* 3. Get the Location header and redirect to the location specified in the "Location" response header
*
* Worth noting that even if this bug has been solved in .Net 4 (I haven't checked) we should still use manual
* redirection so that we can properly log responses.
*
* OBSOLETE ISSUE: (Bug has been resolved in the .Net 4 framework, which this library is now targeted at)
* //CookieContainer also has a horrible bug relating to the specified cookie domain. Basically, if it contains
* //a cookie where the "domain" token is specified as ".domain.xxx" and you attempt to request http://domain.ext,
* //the cookies are not retrievable via that Uri, as you would expect them to be. CookieContainer is incorrectly
* //assuming that the leading dot is a prerequisite specifying that a subdomain is required as opposed to the
* //correct behaviour which would be to take it to mean that the domain and all subdomains are valid for the cookie.
* //http://channel9.msdn.com/forums/TechOff/260235-Bug-in-CookieContainer-where-do-I-report/?CommentID=397315
* //The workaround is as follows:
* //When retrieving the response, iterate through the Set-Cookie header and any cookie that explicitly sets
* //the domain token with a leading dot should also set the cookie without the leading dot.
*/
bool handle301Or302Redirect;
int maxRedirects = 10;
string html;
do
{
Debug.WriteLine(uri.ToString());
if (maxRedirects-- == 0)
{
Log("Too many 302 redirects", LogMessageType.Error);
return(false);
}
handle301Or302Redirect = false;
HttpWebRequest req = PrepareRequestObject(uri, method, timeoutMilliseconds);
foreach (var header in _extraHeaders)
{
req.Headers.Add(header);
}
if (_includeFormValues != null)
{
if (userVariables == null)
{
userVariables = _includeFormValues;
}
else
{
userVariables.Add(_includeFormValues);
}
}
if (userVariables != null)
{
if (method == "POST")
{
byte[] data = Encoding.ASCII.GetBytes(StringUtil.MakeQueryString(userVariables));
req.ContentLength = data.Length;
Stream stream = req.GetRequestStream();
stream.Write(data, 0, data.Length);
stream.Close();
}
else
{
uri = new Uri(uri.Scheme + "://" + uri.Host + uri.AbsolutePath + "?" + StringUtil.MakeQueryString(userVariables));
req = PrepareRequestObject(uri, method, timeoutMilliseconds);
}
}
else if (postData != null)
{
if (method == "GET")
{
throw new InvalidOperationException("Cannot call DoRequest with method GET and non-null postData");
}
byte[] data = Encoding.ASCII.GetBytes(postData);
req.ContentLength = data.Length;
Stream stream = req.GetRequestStream();
stream.Write(data, 0, data.Length);
stream.Close();
}
if (contentType != null)
{
req.ContentType = contentType;
}
try
{
using (HttpWebResponse response = (HttpWebResponse)req.GetResponse())
{
StreamReader reader = new StreamReader(response.GetResponseStream());
html = reader.ReadToEnd();
ResponseText = html;
reader.Close();
string oldHTML = html;
//html = StripAndRebuildHtml(html);
CurrentHtml = html;
ContentType = response.ContentType;
_doc = null;
_includeFormValues = null;
_lastRequestLog = new HttpRequestLog
{
Text = oldHTML,
ParsedHtml = XDocument.ToString(),
Method = method,
PostData = userVariables,
RequestHeaders = req.Headers,
ResponseHeaders = response.Headers,
StatusCode = (int)response.StatusCode,
Url = uri
};
if (method == "GET" && uri.Query.Length > 0 && uri.Query != "?")
{
_lastRequestLog.QueryStringData = HttpUtility.ParseQueryString(uri.Query);
}
LogRequestData();
if ((int)response.StatusCode == 302 || (int)response.StatusCode == 301)
{
//url = AdjustUrl(url, response.Headers["Location"]);
uri = new Uri(uri, response.Headers["Location"]);
handle301Or302Redirect = true;
Url = uri;
Debug.WriteLine("Redirecting to: " + Url);
method = "GET";
postData = null;
userVariables = null;
}
}
}
catch (WebException ex)
{
LastWebException = ex;
switch (ex.Status)
{
case WebExceptionStatus.Timeout:
Log("A timeout occurred while trying to load the web page", LogMessageType.Error);
break;
case WebExceptionStatus.ReceiveFailure:
Log("The response was cut short prematurely", LogMessageType.Error);
break;
default:
Log("An exception was thrown while trying to load the page: " + ex.Message, LogMessageType.Error);
break;
}
return(false);
}
} while(handle301Or302Redirect);
Url = uri;
_referrerUrl = uri;
CurrentHtml = html;
return(true);
}