static void OnBrowserRequestLogged(Browser req, HttpRequestLog log) { Console.WriteLine(" -> " + log.Method + " request to " + log.Url); Console.WriteLine(" <- Response status code: " + log.ResponseCode); }
void HandleRequestLogged(Browser b, HttpRequestLog req) { this.LastRequest = req; }
internal bool DoRequest(Uri uri, string method, NameValueCollection userVariables, string postData, string contentType, string encodingType, int timeoutMilliseconds) { /* IMPORTANT INFORMATION: * HttpWebRequest has a bug where if a 302 redirect is encountered (such as from a Response.Redirect), any cookies * generated during the request are ignored and discarded during the internal redirect process. The headers are in * fact returned, but the normal process where the cookie headers are turned into Cookie objects in the cookie * container is skipped, thus breaking the login processes of half the sites on the internet. * * The workaround is as follows: * 1. Turn off AllowAutoRedirect so we can intercept the redirect and do things manually * 2. Read the Set-Cookie headers from the response and manually insert them into the cookie container * 3. Get the Location header and redirect to the location specified in the "Location" response header * * Worth noting that even if this bug has been solved in .Net 4 (I haven't checked) we should still use manual * redirection so that we can properly log responses. * * OBSOLETE ISSUE: (Bug has been resolved in the .Net 4 framework, which this library is now targeted at) * //CookieContainer also has a horrible bug relating to the specified cookie domain. Basically, if it contains * //a cookie where the "domain" token is specified as ".domain.xxx" and you attempt to request http://domain.ext, * //the cookies are not retrievable via that Uri, as you would expect them to be. CookieContainer is incorrectly * //assuming that the leading dot is a prerequisite specifying that a subdomain is required as opposed to the * //correct behaviour which would be to take it to mean that the domain and all subdomains are valid for the cookie. * //http://channel9.msdn.com/forums/TechOff/260235-Bug-in-CookieContainer-where-do-I-report/?CommentID=397315 * //The workaround is as follows: * //When retrieving the response, iterate through the Set-Cookie header and any cookie that explicitly sets * //the domain token with a leading dot should also set the cookie without the leading dot. */ bool handle301Or302Redirect; int maxRedirects = 10; string html; string postBody = ""; do { Debug.WriteLine(uri.ToString()); if (maxRedirects-- == 0) { Log("Too many 302 redirects", LogMessageType.Error); return(false); } handle301Or302Redirect = false; IHttpWebRequest req = null; try { req = PrepareRequestObject(uri, method, contentType, timeoutMilliseconds); } catch (NotSupportedException) { // Happens when the URL cannot be parsed (example: 'javascript:') return(false); } foreach (var header in _extraHeaders) { if (header.StartsWith("host:", StringComparison.OrdinalIgnoreCase)) { req.Host = header.Split(':')[1]; } else { req.Headers.Add(header); } } if (!string.IsNullOrEmpty(encodingType)) { req.Headers.Add(HttpRequestHeader.ContentEncoding, encodingType); } if (_includeFormValues != null) { if (userVariables == null) { userVariables = _includeFormValues; } else { userVariables.Add(_includeFormValues); } } if (userVariables != null) { if (method == "POST") { postBody = StringUtil.MakeQueryString(userVariables); byte[] data = Encoding.GetEncoding(28591).GetBytes(postBody); req.ContentLength = data.Length; using (Stream stream = req.GetRequestStream()) { stream.Write(data, 0, data.Length); } } else { uri = new Uri( uri.Scheme + "://" + uri.Host + ":" + uri.Port + uri.AbsolutePath + ((userVariables.Count > 0) ? "?" + StringUtil.MakeQueryString(userVariables) : "") ); req = PrepareRequestObject(uri, method, contentType, timeoutMilliseconds); } } else if (postData != null) { if (method == "GET") { throw new InvalidOperationException("Cannot call DoRequest with method GET and non-null postData"); } postBody = postData; byte[] data = Encoding.GetEncoding(28591).GetBytes(postData); req.ContentLength = data.Length; using (Stream stream = req.GetRequestStream()) { stream.Write(data, 0, data.Length); } } if (contentType != null) { req.ContentType = contentType; } _lastRequestLog = new HttpRequestLog { Method = method, PostData = userVariables, PostBody = postBody, RequestHeaders = req.Headers, Url = uri }; try { using (IHttpWebResponse response = req.GetResponse()) { Encoding responseEncoding = Encoding.UTF8; //default string charSet = response.CharacterSet; if (!String.IsNullOrEmpty(charSet)) { try { responseEncoding = Encoding.GetEncoding(charSet); } catch (ArgumentException) { responseEncoding = Encoding.UTF8; // try using utf8 } } //ensure the stream is disposed using (Stream rs = response.GetResponseStream()) { using (StreamReader reader = new StreamReader(rs, responseEncoding)) { html = reader.ReadToEnd(); } } _doc = null; _includeFormValues = null; _lastRequestLog.Text = html; _lastRequestLog.ResponseHeaders = response.Headers; _lastRequestLog.StatusCode = (int)response.StatusCode; if (method == "GET" && uri.Query.Length > 0 && uri.Query != "?") { _lastRequestLog.QueryStringData = HttpUtility.ParseQueryString(uri.Query); } if ((int)response.StatusCode == 302 || (int)response.StatusCode == 301) { uri = new Uri(uri, response.Headers["Location"]); handle301Or302Redirect = true; Debug.WriteLine("Redirecting to: " + uri); method = "GET"; postData = null; userVariables = null; } } } catch (WebException ex) { _lastRequestLog.StatusCode = (int)ex.Status.GetTypeCode(); if (ex.Response != null) { _lastRequestLog.ResponseHeaders = ex.Response.Headers; //ensure the stream is disposed using (Stream rs = ex.Response.GetResponseStream()) { using (StreamReader reader = new StreamReader(rs)) { html = reader.ReadToEnd(); } } _lastRequestLog.Text = html; } LastWebException = ex; switch (ex.Status) { case WebExceptionStatus.Timeout: Log("A timeout occurred while trying to load the web page", LogMessageType.Error); break; case WebExceptionStatus.ReceiveFailure: Log("The response was cut short prematurely", LogMessageType.Error); break; default: Log("An exception was thrown while trying to load the page: " + ex.Message, LogMessageType.Error); break; } return(false); } finally { LogRequestData(); } } while (handle301Or302Redirect); this.RemoveChildBrowsers(); //Any frames contained in the previous state should be removed. They will be recreated if we ever navigate back this.AddNavigationState(new NavigationState() { Html = html, Url = uri, ContentType = contentType }); return(true); }
internal bool DoRequest(Uri uri, string method, NameValueCollection userVariables, string postData, string contentType, string encodingType, int timeoutMilliseconds) { string html; string referer = null; if (uri.IsFile) { StreamReader reader = new StreamReader(uri.AbsolutePath); html = reader.ReadToEnd(); reader.Close(); } else { bool handle3xxRedirect = false; int maxRedirects = 5; // Per RFC2068, Section 10.3 string postBody = string.Empty; do { Debug.WriteLine(uri.ToString()); if (maxRedirects-- == 0) { Log("Too many 3xx redirects", LogMessageType.Error); return(false); } handle3xxRedirect = false; IHttpWebRequest req = null; try { req = PrepareRequestObject(uri, method, contentType, timeoutMilliseconds); } catch (NotSupportedException) { // Happens when the URL cannot be parsed (example: 'javascript:') return(false); } foreach (var header in _extraHeaders) { if (header.StartsWith("host:", StringComparison.OrdinalIgnoreCase)) { req.Host = header.Split(':')[1]; } else { req.Headers.Add(header); } } if (!string.IsNullOrEmpty(encodingType)) { req.Headers.Add(HttpRequestHeader.ContentEncoding, encodingType); } // Remove all expired basic authentication tokens List <BasicAuthenticationToken> expired = _basicAuthenticationTokens.Values.Where(t => DateTime.Now > t.Expiration).ToList(); foreach (var expiredToken in expired) { _basicAuthenticationTokens.Remove(expiredToken.Domain); } // If an authentication token exists for the domain, add the authorization header. foreach (var token in _basicAuthenticationTokens.Values) { if (req.Host.Contains(token.Domain)) { // Extend the expiration. token.UpdateExpiration(); // Add the authentication header. req.Headers.Add(string.Format( "Authorization: Basic {0}", token.Token)); } } if (_includeFormValues != null) { if (userVariables == null) { userVariables = _includeFormValues; } else { userVariables.Add(_includeFormValues); } } if (userVariables != null) { if (method == "POST") { postBody = StringUtil.MakeQueryString(userVariables); byte[] data = Encoding.GetEncoding(28591).GetBytes(postBody); req.ContentLength = data.Length; using (Stream stream = req.GetRequestStream()) { stream.Write(data, 0, data.Length); } } else { uri = new Uri( uri.Scheme + "://" + uri.Host + ":" + uri.Port + uri.AbsolutePath + ((userVariables.Count > 0) ? "?" + StringUtil.MakeQueryString(userVariables) : "") ); req = PrepareRequestObject(uri, method, contentType, timeoutMilliseconds); } } else if (postData != null) { if (method == "GET") { throw new InvalidOperationException("Cannot call DoRequest with method GET and non-null postData"); } postBody = postData; byte[] data = Encoding.GetEncoding(28591).GetBytes(postData); req.ContentLength = data.Length; using (Stream stream = req.GetRequestStream()) { stream.Write(data, 0, data.Length); } } referer = req.Referer; if (contentType != null) { req.ContentType = contentType; } _lastRequestLog = new HttpRequestLog { Method = method, PostData = userVariables, PostBody = postBody, RequestHeaders = req.Headers, Url = uri }; try { using (IHttpWebResponse response = req.GetResponse()) { Encoding responseEncoding = Encoding.UTF8; //default string charSet = response.CharacterSet; if (!String.IsNullOrEmpty(charSet)) { try { responseEncoding = Encoding.GetEncoding(charSet); } catch (ArgumentException) { responseEncoding = Encoding.UTF8; // try using utf8 } } //ensure the stream is disposed using (Stream rs = response.GetResponseStream()) { using (StreamReader reader = new StreamReader(rs, responseEncoding)) { html = reader.ReadToEnd(); } } _doc = null; _includeFormValues = null; _lastRequestLog.Text = html; _lastRequestLog.ResponseHeaders = response.Headers; _lastRequestLog.ResponseCode = (int)response.StatusCode; if (method == "GET" && uri.Query.Length > 0 && uri.Query != "?") { _lastRequestLog.QueryStringData = HttpUtility.ParseQueryString(uri.Query); } if (AutoRedirect == true && (((int)response.StatusCode == 300 || // Not entirely supported. If provided, the server's preference from the Location header is honored. (int)response.StatusCode == 301 || (int)response.StatusCode == 302 || (int)response.StatusCode == 303 || // 304 - Unsupported, conditional Get requests are not supported (mostly because SimpleBrowser does not cache content) // 305 - Unsupported, possible security threat // 306 - No longer used, per RFC2616, Section 10.3.7 (int)response.StatusCode == 307 || (int)response.StatusCode == 308) && response.Headers.AllKeys.Contains("Location"))) { uri = new Uri(uri, response.Headers["Location"]); handle3xxRedirect = true; Debug.WriteLine("Redirecting to: " + uri); method = "GET"; postData = null; userVariables = null; } if (response.Headers.AllKeys.Contains("Set-Cookie")) { var cookies = SetCookieHeaderParser.GetAllCookiesFromHeader(uri.Host, response.Headers["Set-Cookie"]); Cookies.Add(cookies); } } } catch (WebException ex) { if (ex.Response != null) { _lastRequestLog.ResponseHeaders = ex.Response.Headers; //ensure the stream is disposed using (Stream rs = ex.Response.GetResponseStream()) { using (StreamReader reader = new StreamReader(rs)) { html = reader.ReadToEnd(); } } _lastRequestLog.Text = html; } LastWebException = ex; switch (ex.Status) { case WebExceptionStatus.Timeout: Log("A timeout occurred while trying to load the web page", LogMessageType.Error); break; case WebExceptionStatus.ReceiveFailure: Log("The response was cut short prematurely", LogMessageType.Error); break; default: Log("An exception was thrown while trying to load the page: " + ex.Message, LogMessageType.Error); break; } return(false); } finally { LogRequestData(); } }while (handle3xxRedirect); } this._navigationAttributes = null; this.RemoveChildBrowsers(); //Any frames contained in the previous state should be removed. They will be recreated if we ever navigate back this.AddNavigationState( new NavigationState() { Html = html, Url = uri, ContentType = contentType, Referer = string.IsNullOrEmpty(referer) ? null : new Uri(Uri.UnescapeDataString(referer)) }); return(true); }
private bool DoRequest(Uri uri, string method, NameValueCollection userVariables, string postData, string contentType, int timeoutMilliseconds) { /* IMPORTANT INFORMATION: * HttpWebRequest has a bug where if a 302 redirect is encountered (such as from a Response.Redirect), any cookies * generated during the request are ignored and discarded during the internal redirect process. The headers are in * fact returned, but the normal process where the cookie headers are turned into Cookie objects in the cookie * container is skipped, thus breaking the login processes of half the sites on the internet. * * The workaround is as follows: * 1. Turn off AllowAutoRedirect so we can intercept the redirect and do things manually * 2. Read the Set-Cookie headers from the response and manually insert them into the cookie container * 3. Get the Location header and redirect to the location specified in the "Location" response header * * Worth noting that even if this bug has been solved in .Net 4 (I haven't checked) we should still use manual * redirection so that we can properly log responses. * * OBSOLETE ISSUE: (Bug has been resolved in the .Net 4 framework, which this library is now targeted at) * //CookieContainer also has a horrible bug relating to the specified cookie domain. Basically, if it contains * //a cookie where the "domain" token is specified as ".domain.xxx" and you attempt to request http://domain.ext, * //the cookies are not retrievable via that Uri, as you would expect them to be. CookieContainer is incorrectly * //assuming that the leading dot is a prerequisite specifying that a subdomain is required as opposed to the * //correct behaviour which would be to take it to mean that the domain and all subdomains are valid for the cookie. * //http://channel9.msdn.com/forums/TechOff/260235-Bug-in-CookieContainer-where-do-I-report/?CommentID=397315 * //The workaround is as follows: * //When retrieving the response, iterate through the Set-Cookie header and any cookie that explicitly sets * //the domain token with a leading dot should also set the cookie without the leading dot. */ bool handle301Or302Redirect; int maxRedirects = 10; string html; do { Debug.WriteLine(uri.ToString()); if(maxRedirects-- == 0) { Log("Too many 302 redirects", LogMessageType.Error); return false; } handle301Or302Redirect = false; HttpWebRequest req = PrepareRequestObject(uri, method, timeoutMilliseconds); foreach(var header in _extraHeaders) req.Headers.Add(header); if(_includeFormValues != null) { if(userVariables == null) userVariables = _includeFormValues; else userVariables.Add(_includeFormValues); } if(userVariables != null) { if(method == "POST") { byte[] data = Encoding.ASCII.GetBytes(StringUtil.MakeQueryString(userVariables)); req.ContentLength = data.Length; Stream stream = req.GetRequestStream(); stream.Write(data, 0, data.Length); stream.Close(); } else { uri = new Uri(uri.Scheme + "://" + uri.Host + uri.AbsolutePath + "?" + StringUtil.MakeQueryString(userVariables)); req = PrepareRequestObject(uri, method, timeoutMilliseconds); } } else if(postData != null) { if(method == "GET") throw new InvalidOperationException("Cannot call DoRequest with method GET and non-null postData"); byte[] data = Encoding.ASCII.GetBytes(postData); req.ContentLength = data.Length; Stream stream = req.GetRequestStream(); stream.Write(data, 0, data.Length); stream.Close(); } if(contentType != null) req.ContentType = contentType; try { using(HttpWebResponse response = (HttpWebResponse)req.GetResponse()) { StreamReader reader = new StreamReader(response.GetResponseStream()); html = reader.ReadToEnd(); ResponseText = html; reader.Close(); string oldHTML = html; //html = StripAndRebuildHtml(html); CurrentHtml = html; ContentType = response.ContentType; _doc = null; _includeFormValues = null; _lastRequestLog = new HttpRequestLog { Text = oldHTML, ParsedHtml = XDocument.ToString(), Method = method, PostData = userVariables, RequestHeaders = req.Headers, ResponseHeaders = response.Headers, StatusCode = (int)response.StatusCode, Url = uri }; if(method == "GET" && uri.Query.Length > 0 && uri.Query != "?") _lastRequestLog.QueryStringData = HttpUtility.ParseQueryString(uri.Query); LogRequestData(); if((int)response.StatusCode == 302 || (int)response.StatusCode == 301) { //url = AdjustUrl(url, response.Headers["Location"]); uri = new Uri(uri, response.Headers["Location"]); handle301Or302Redirect = true; Url = uri; Debug.WriteLine("Redirecting to: " + Url); method = "GET"; postData = null; userVariables = null; } } } catch(WebException ex) { LastWebException = ex; switch(ex.Status) { case WebExceptionStatus.Timeout: Log("A timeout occurred while trying to load the web page", LogMessageType.Error); break; case WebExceptionStatus.ReceiveFailure: Log("The response was cut short prematurely", LogMessageType.Error); break; default: Log("An exception was thrown while trying to load the page: " + ex.Message, LogMessageType.Error); break; } return false; } } while(handle301Or302Redirect); Url = uri; _referrerUrl = uri; CurrentHtml = html; return true; }
private bool DoRequest(Uri uri, string method, NameValueCollection userVariables, string postData, string contentType, int timeoutMilliseconds) { /* IMPORTANT INFORMATION: * HttpWebRequest has a bug where if a 302 redirect is encountered (such as from a Response.Redirect), any cookies * generated during the request are ignored and discarded during the internal redirect process. The headers are in * fact returned, but the normal process where the cookie headers are turned into Cookie objects in the cookie * container is skipped, thus breaking the login processes of half the sites on the internet. * * The workaround is as follows: * 1. Turn off AllowAutoRedirect so we can intercept the redirect and do things manually * 2. Read the Set-Cookie headers from the response and manually insert them into the cookie container * 3. Get the Location header and redirect to the location specified in the "Location" response header * * Worth noting that even if this bug has been solved in .Net 4 (I haven't checked) we should still use manual * redirection so that we can properly log responses. * * OBSOLETE ISSUE: (Bug has been resolved in the .Net 4 framework, which this library is now targeted at) * //CookieContainer also has a horrible bug relating to the specified cookie domain. Basically, if it contains * //a cookie where the "domain" token is specified as ".domain.xxx" and you attempt to request http://domain.ext, * //the cookies are not retrievable via that Uri, as you would expect them to be. CookieContainer is incorrectly * //assuming that the leading dot is a prerequisite specifying that a subdomain is required as opposed to the * //correct behaviour which would be to take it to mean that the domain and all subdomains are valid for the cookie. * //http://channel9.msdn.com/forums/TechOff/260235-Bug-in-CookieContainer-where-do-I-report/?CommentID=397315 * //The workaround is as follows: * //When retrieving the response, iterate through the Set-Cookie header and any cookie that explicitly sets * //the domain token with a leading dot should also set the cookie without the leading dot. */ bool handle301Or302Redirect; int maxRedirects = 10; string html; do { Debug.WriteLine(uri.ToString()); if (maxRedirects-- == 0) { Log("Too many 302 redirects", LogMessageType.Error); return(false); } handle301Or302Redirect = false; HttpWebRequest req = PrepareRequestObject(uri, method, timeoutMilliseconds); foreach (var header in _extraHeaders) { req.Headers.Add(header); } if (_includeFormValues != null) { if (userVariables == null) { userVariables = _includeFormValues; } else { userVariables.Add(_includeFormValues); } } if (userVariables != null) { if (method == "POST") { byte[] data = Encoding.ASCII.GetBytes(StringUtil.MakeQueryString(userVariables)); req.ContentLength = data.Length; Stream stream = req.GetRequestStream(); stream.Write(data, 0, data.Length); stream.Close(); } else { uri = new Uri(uri.Scheme + "://" + uri.Host + uri.AbsolutePath + "?" + StringUtil.MakeQueryString(userVariables)); req = PrepareRequestObject(uri, method, timeoutMilliseconds); } } else if (postData != null) { if (method == "GET") { throw new InvalidOperationException("Cannot call DoRequest with method GET and non-null postData"); } byte[] data = Encoding.ASCII.GetBytes(postData); req.ContentLength = data.Length; Stream stream = req.GetRequestStream(); stream.Write(data, 0, data.Length); stream.Close(); } if (contentType != null) { req.ContentType = contentType; } try { using (HttpWebResponse response = (HttpWebResponse)req.GetResponse()) { StreamReader reader = new StreamReader(response.GetResponseStream()); html = reader.ReadToEnd(); ResponseText = html; reader.Close(); string oldHTML = html; //html = StripAndRebuildHtml(html); CurrentHtml = html; ContentType = response.ContentType; _doc = null; _includeFormValues = null; _lastRequestLog = new HttpRequestLog { Text = oldHTML, ParsedHtml = XDocument.ToString(), Method = method, PostData = userVariables, RequestHeaders = req.Headers, ResponseHeaders = response.Headers, StatusCode = (int)response.StatusCode, Url = uri }; if (method == "GET" && uri.Query.Length > 0 && uri.Query != "?") { _lastRequestLog.QueryStringData = HttpUtility.ParseQueryString(uri.Query); } LogRequestData(); if ((int)response.StatusCode == 302 || (int)response.StatusCode == 301) { //url = AdjustUrl(url, response.Headers["Location"]); uri = new Uri(uri, response.Headers["Location"]); handle301Or302Redirect = true; Url = uri; Debug.WriteLine("Redirecting to: " + Url); method = "GET"; postData = null; userVariables = null; } } } catch (WebException ex) { LastWebException = ex; switch (ex.Status) { case WebExceptionStatus.Timeout: Log("A timeout occurred while trying to load the web page", LogMessageType.Error); break; case WebExceptionStatus.ReceiveFailure: Log("The response was cut short prematurely", LogMessageType.Error); break; default: Log("An exception was thrown while trying to load the page: " + ex.Message, LogMessageType.Error); break; } return(false); } } while(handle301Or302Redirect); Url = uri; _referrerUrl = uri; CurrentHtml = html; return(true); }