void SetupData()
{
var bytes = _encoder.Encode(Data, ContentType);
if (bytes.Length > 0)
{
httpWebRequest.ContentLength = bytes.Length;
}
var requestStream = httpWebRequest.GetRequestStream();
requestStream.Write(bytes, 0, bytes.Length);
requestStream.Close();
}
/// <summary>
/// Initiate an http POST request.
/// </summary>
///
/// <param name="url">
/// URL of the document.
/// </param>
/// <param name="postData">
/// The information describing the post data to be sent this request.
/// </param>
///
/// <returns>
/// The data returned by the POST request.
/// </returns>
public string Post(string url, IEnumerable <KeyValuePair <string, string> > postData)
{
var encoding = new ASCIIEncoding();
byte[] data = encoding.GetBytes(PostDataString);
IHttpWebRequest request = WebRequestFactory.Create(new Uri(Url));
ApplyOptions(request);
request.Method = HttpWebRequestMethod.POST;
request.ContentType = "application/x-www-form-urlencoded";
request.ContentLength = data.Length;
var newStream = request.GetRequestStream();
newStream.Write(data, 0, data.Length);
newStream.Close();
using (var response = GetResponseStreamReader(request))
{
Html = response.ReadToEnd();
}
return(Html);
}
private static void InsertSoapEnvelopeIntoWebRequest(XmlDocument soapEnvelopeXml, IHttpWebRequest webRequest)
{
using (Stream stream = webRequest.GetRequestStream())
{
soapEnvelopeXml.Save(stream);
}
}
string MakeRequest(string relativeUrl, HttpMethod method, string data)
{
IHttpWebRequest request = createRequest(relativeUrl, method);
if (method == HttpMethod.Post || method == HttpMethod.Put)
{
var dataBytes = Encoding.UTF8.GetBytes(data);
try {
using (var writeStream = request.GetRequestStream()) {
writeStream.Write(dataBytes, 0, dataBytes.Length);
}
} catch (WebException) {
throw new Error.ApiConnectionError(Constants.Messages.ApiConnectionError, null, null);
}
}
return(createResponse(request));
}
private HttpWebHelperResult WriteAndGetReponse(IHttpWebRequest httpWebRequest, Stream requestBody)
{
Stream requestStream = null;
Exception exception = null;
IHttpWebResponse httpWebResponse = null;
HttpWebHelperResult result;
try
{
requestStream = httpWebRequest.GetRequestStream();
}
catch (WebException ex)
{
var webException = new WebExceptionWrapper(ex);
httpWebResponse = webException.GetResponse();
exception = webException;
}
finally
{
if (exception == null)
{
// we got the stream, so copy to the stream
result = CopyRequestStream(httpWebRequest, requestBody, requestStream);
}
else
{
// there was an error
if (httpWebResponse == null)
{
result = new HttpWebHelperResult(httpWebRequest, null, exception, null, false, true, null, null);
}
else
{
var args = new ResponseReceivedEventArgs(httpWebResponse, exception, null);
OnResponseReceived(args);
result = ReadResponseStream(httpWebRequest, httpWebResponse, exception, args.ResponseSaveStream);
}
}
}
return(result);
}
private HttpWebHelperResult WriteAndGetReponse(IHttpWebRequest httpWebRequest, Stream requestBody)
{
Stream requestStream = null;
Exception exception = null;
IHttpWebResponse httpWebResponse = null;
HttpWebHelperResult result;
try
{
requestStream = httpWebRequest.GetRequestStream();
}
catch (WebException ex)
{
var webException = new WebExceptionWrapper(ex);
httpWebResponse = webException.GetResponse();
exception = webException;
}
finally
{
if (exception == null)
{
// we got the stream, so copy to the stream
result = CopyRequestStream(httpWebRequest, requestBody, requestStream);
}
else
{
// there was an error
if (httpWebResponse == null)
{
result = new HttpWebHelperResult(httpWebRequest, null, exception, null, false, true, null, null);
}
else
{
var args = new ResponseReceivedEventArgs(httpWebResponse, exception, null);
OnResponseReceived(args);
result = ReadResponseStream(httpWebRequest, httpWebResponse, exception, args.ResponseSaveStream);
}
}
}
return result;
}
internal bool DoRequest(Uri uri, string method, NameValueCollection userVariables, string postData, string contentType, string encodingType, int timeoutMilliseconds)
{
/* IMPORTANT INFORMATION:
* HttpWebRequest has a bug where if a 302 redirect is encountered (such as from a Response.Redirect), any cookies
* generated during the request are ignored and discarded during the internal redirect process. The headers are in
* fact returned, but the normal process where the cookie headers are turned into Cookie objects in the cookie
* container is skipped, thus breaking the login processes of half the sites on the internet.
*
* The workaround is as follows:
* 1. Turn off AllowAutoRedirect so we can intercept the redirect and do things manually
* 2. Read the Set-Cookie headers from the response and manually insert them into the cookie container
* 3. Get the Location header and redirect to the location specified in the "Location" response header
*
* Worth noting that even if this bug has been solved in .Net 4 (I haven't checked) we should still use manual
* redirection so that we can properly log responses.
*
* OBSOLETE ISSUE: (Bug has been resolved in the .Net 4 framework, which this library is now targeted at)
* //CookieContainer also has a horrible bug relating to the specified cookie domain. Basically, if it contains
* //a cookie where the "domain" token is specified as ".domain.xxx" and you attempt to request http://domain.ext,
* //the cookies are not retrievable via that Uri, as you would expect them to be. CookieContainer is incorrectly
* //assuming that the leading dot is a prerequisite specifying that a subdomain is required as opposed to the
* //correct behaviour which would be to take it to mean that the domain and all subdomains are valid for the cookie.
* //http://channel9.msdn.com/forums/TechOff/260235-Bug-in-CookieContainer-where-do-I-report/?CommentID=397315
* //The workaround is as follows:
* //When retrieving the response, iterate through the Set-Cookie header and any cookie that explicitly sets
* //the domain token with a leading dot should also set the cookie without the leading dot.
*/
bool handle301Or302Redirect;
int maxRedirects = 10;
string html;
string responseContentType;
string postBody = "";
do
{
Debug.WriteLine(uri.ToString());
if (maxRedirects-- == 0)
{
Log("Too many 302 redirects", LogMessageType.Error);
return(false);
}
handle301Or302Redirect = false;
IHttpWebRequest req = null;
try
{
req = PrepareRequestObject(uri, method, contentType, timeoutMilliseconds);
}
catch (NotSupportedException e)
{
// Happens when the URL cannot be parsed (example: 'javascript:')
return(false);
}
foreach (var header in _extraHeaders)
{
req.Headers.Add(header);
}
if (encodingType != null)
{
req.Headers.Add(HttpRequestHeader.ContentEncoding, encodingType);
}
if (_includeFormValues != null)
{
if (userVariables == null)
{
userVariables = _includeFormValues;
}
else
{
userVariables.Add(_includeFormValues);
}
}
if (userVariables != null)
{
if (method == "POST")
{
postBody = StringUtil.MakeQueryString(userVariables);
byte[] data = Encoding.GetEncoding(28591).GetBytes(postBody);
req.ContentLength = data.Length;
Stream stream = req.GetRequestStream();
stream.Write(data, 0, data.Length);
stream.Close();
}
else
{
uri = new Uri(
uri.Scheme + "://" + uri.Host + ":" + uri.Port + uri.AbsolutePath
+ ((userVariables.Count > 0) ? "?" + StringUtil.MakeQueryString(userVariables) : "")
);
req = PrepareRequestObject(uri, method, contentType, timeoutMilliseconds);
}
}
else if (postData != null)
{
if (method == "GET")
{
throw new InvalidOperationException("Cannot call DoRequest with method GET and non-null postData");
}
postBody = postData;
byte[] data = Encoding.GetEncoding(28591).GetBytes(postData);
req.ContentLength = data.Length;
Stream stream = req.GetRequestStream();
stream.Write(data, 0, data.Length);
stream.Close();
}
if (contentType != null)
{
req.ContentType = contentType;
}
_lastRequestLog = new HttpRequestLog
{
Method = method,
PostData = userVariables,
PostBody = postBody,
RequestHeaders = req.Headers,
Url = uri
};
try
{
using (IHttpWebResponse response = req.GetResponse())
{
Encoding responseEncoding = Encoding.UTF8; //default
string charSet = response.CharacterSet;
if (!String.IsNullOrEmpty(charSet))
{
try
{
responseEncoding = Encoding.GetEncoding(charSet);
}
catch (ArgumentException)
{
responseEncoding = Encoding.UTF8; // try using utf8
}
}
StreamReader reader = new StreamReader(response.GetResponseStream(), responseEncoding);
html = reader.ReadToEnd();
responseContentType = response.ContentType;
reader.Close();
_doc = null;
_includeFormValues = null;
_lastRequestLog.Text = html;
_lastRequestLog.ParsedHtml = html;
_lastRequestLog.ResponseHeaders = response.Headers;
_lastRequestLog.StatusCode = (int)response.StatusCode;
if (method == "GET" && uri.Query.Length > 0 && uri.Query != "?")
{
_lastRequestLog.QueryStringData = HttpUtility.ParseQueryString(uri.Query);
}
if ((int)response.StatusCode == 302 || (int)response.StatusCode == 301)
{
uri = new Uri(uri, response.Headers["Location"]);
handle301Or302Redirect = true;
Debug.WriteLine("Redirecting to: " + uri);
method = "GET";
postData = null;
userVariables = null;
}
}
}
catch (WebException ex)
{
_lastRequestLog.StatusCode = (int)ex.Status.GetTypeCode();
if (ex.Response != null)
{
_lastRequestLog.ResponseHeaders = ex.Response.Headers;
StreamReader reader = new StreamReader(ex.Response.GetResponseStream());
html = reader.ReadToEnd();
_lastRequestLog.Text = html;
}
LastWebException = ex;
switch (ex.Status)
{
case WebExceptionStatus.Timeout:
Log("A timeout occurred while trying to load the web page", LogMessageType.Error);
break;
case WebExceptionStatus.ReceiveFailure:
Log("The response was cut short prematurely", LogMessageType.Error);
break;
default:
Log("An exception was thrown while trying to load the page: " + ex.Message, LogMessageType.Error);
break;
}
return(false);
}
finally
{
LogRequestData();
}
} while (handle301Or302Redirect);
this.RemoveChildBrowsers(); //Any frames contained in the previous state should be removed. They will be recreated if we ever navigate back
this.AddNavigationState(new NavigationState()
{
Html = html, Url = uri, ContentType = contentType
});
return(true);
}
private static void SetFormValuesForQuery(IHttpWebRequest webRequest, string safeQueryString)
{
if (webRequest == null) throw new ArgumentNullException("webRequest");
webRequest.ContentType = "application/x-www-form-urlencoded";
var stream = webRequest.GetRequestStream();
var bodyBytes = new ASCIIEncoding().GetBytes(safeQueryString);
stream.Write(bodyBytes, 0, bodyBytes.Length);
stream.Flush();
stream.Close();
}
internal bool DoRequest(Uri uri, string method, NameValueCollection userVariables, string postData, string contentType, string encodingType, int timeoutMilliseconds)
{
string html;
string referer = null;
if (uri.IsFile)
{
StreamReader reader = new StreamReader(uri.AbsolutePath);
html = reader.ReadToEnd();
reader.Close();
}
else
{
bool handle3xxRedirect = false;
int maxRedirects = 5; // Per RFC2068, Section 10.3
string postBody = string.Empty;
do
{
Debug.WriteLine(uri.ToString());
if (maxRedirects-- == 0)
{
Log("Too many 3xx redirects", LogMessageType.Error);
return(false);
}
handle3xxRedirect = false;
IHttpWebRequest req = null;
try
{
req = PrepareRequestObject(uri, method, contentType, timeoutMilliseconds);
}
catch (NotSupportedException)
{
// Happens when the URL cannot be parsed (example: 'javascript:')
return(false);
}
foreach (var header in _extraHeaders)
{
if (header.StartsWith("host:", StringComparison.OrdinalIgnoreCase))
{
req.Host = header.Split(':')[1];
}
else
{
req.Headers.Add(header);
}
}
if (!string.IsNullOrEmpty(encodingType))
{
req.Headers.Add(HttpRequestHeader.ContentEncoding, encodingType);
}
// Remove all expired basic authentication tokens
List <BasicAuthenticationToken> expired =
_basicAuthenticationTokens.Values.Where(t => DateTime.Now > t.Expiration).ToList();
foreach (var expiredToken in expired)
{
_basicAuthenticationTokens.Remove(expiredToken.Domain);
}
// If an authentication token exists for the domain, add the authorization header.
foreach (var token in _basicAuthenticationTokens.Values)
{
if (req.Host.Contains(token.Domain))
{
// Extend the expiration.
token.UpdateExpiration();
// Add the authentication header.
req.Headers.Add(string.Format(
"Authorization: Basic {0}",
token.Token));
}
}
if (_includeFormValues != null)
{
if (userVariables == null)
{
userVariables = _includeFormValues;
}
else
{
userVariables.Add(_includeFormValues);
}
}
if (userVariables != null)
{
if (method == "POST")
{
postBody = StringUtil.MakeQueryString(userVariables);
byte[] data = Encoding.GetEncoding(28591).GetBytes(postBody);
req.ContentLength = data.Length;
using (Stream stream = req.GetRequestStream())
{
stream.Write(data, 0, data.Length);
}
}
else
{
uri = new Uri(
uri.Scheme + "://" + uri.Host + ":" + uri.Port + uri.AbsolutePath
+ ((userVariables.Count > 0) ? "?" + StringUtil.MakeQueryString(userVariables) : "")
);
req = PrepareRequestObject(uri, method, contentType, timeoutMilliseconds);
}
}
else if (postData != null)
{
if (method == "GET")
{
throw new InvalidOperationException("Cannot call DoRequest with method GET and non-null postData");
}
postBody = postData;
byte[] data = Encoding.GetEncoding(28591).GetBytes(postData);
req.ContentLength = data.Length;
using (Stream stream = req.GetRequestStream())
{
stream.Write(data, 0, data.Length);
}
}
referer = req.Referer;
if (contentType != null)
{
req.ContentType = contentType;
}
_lastRequestLog = new HttpRequestLog
{
Method = method,
PostData = userVariables,
PostBody = postBody,
RequestHeaders = req.Headers,
Url = uri
};
try
{
using (IHttpWebResponse response = req.GetResponse())
{
Encoding responseEncoding = Encoding.UTF8; //default
string charSet = response.CharacterSet;
if (!String.IsNullOrEmpty(charSet))
{
try
{
responseEncoding = Encoding.GetEncoding(charSet);
}
catch (ArgumentException)
{
responseEncoding = Encoding.UTF8; // try using utf8
}
}
//ensure the stream is disposed
using (Stream rs = response.GetResponseStream())
{
using (StreamReader reader = new StreamReader(rs, responseEncoding))
{
html = reader.ReadToEnd();
}
}
_doc = null;
_includeFormValues = null;
_lastRequestLog.Text = html;
_lastRequestLog.ResponseHeaders = response.Headers;
_lastRequestLog.ResponseCode = (int)response.StatusCode;
if (method == "GET" && uri.Query.Length > 0 && uri.Query != "?")
{
_lastRequestLog.QueryStringData = HttpUtility.ParseQueryString(uri.Query);
}
if (AutoRedirect == true &&
(((int)response.StatusCode == 300 || // Not entirely supported. If provided, the server's preference from the Location header is honored.
(int)response.StatusCode == 301 ||
(int)response.StatusCode == 302 ||
(int)response.StatusCode == 303 ||
// 304 - Unsupported, conditional Get requests are not supported (mostly because SimpleBrowser does not cache content)
// 305 - Unsupported, possible security threat
// 306 - No longer used, per RFC2616, Section 10.3.7
(int)response.StatusCode == 307 ||
(int)response.StatusCode == 308) &&
response.Headers.AllKeys.Contains("Location")))
{
uri = new Uri(uri, response.Headers["Location"]);
handle3xxRedirect = true;
Debug.WriteLine("Redirecting to: " + uri);
method = "GET";
postData = null;
userVariables = null;
}
if (response.Headers.AllKeys.Contains("Set-Cookie"))
{
var cookies = SetCookieHeaderParser.GetAllCookiesFromHeader(uri.Host, response.Headers["Set-Cookie"]);
Cookies.Add(cookies);
}
}
}
catch (WebException ex)
{
if (ex.Response != null)
{
_lastRequestLog.ResponseHeaders = ex.Response.Headers;
//ensure the stream is disposed
using (Stream rs = ex.Response.GetResponseStream())
{
using (StreamReader reader = new StreamReader(rs))
{
html = reader.ReadToEnd();
}
}
_lastRequestLog.Text = html;
}
LastWebException = ex;
switch (ex.Status)
{
case WebExceptionStatus.Timeout:
Log("A timeout occurred while trying to load the web page", LogMessageType.Error);
break;
case WebExceptionStatus.ReceiveFailure:
Log("The response was cut short prematurely", LogMessageType.Error);
break;
default:
Log("An exception was thrown while trying to load the page: " + ex.Message, LogMessageType.Error);
break;
}
return(false);
}
finally
{
LogRequestData();
}
}while (handle3xxRedirect);
}
this._navigationAttributes = null;
this.RemoveChildBrowsers(); //Any frames contained in the previous state should be removed. They will be recreated if we ever navigate back
this.AddNavigationState(
new NavigationState()
{
Html = html,
Url = uri,
ContentType = contentType,
Referer = string.IsNullOrEmpty(referer) ? null : new Uri(Uri.UnescapeDataString(referer))
});
return(true);
}