/// <include file='IWebClient.xml' path='/IWebClient/SendRequest_WebRequest/*'/>
public WebResponse SendRequest( WebRequest webRequest )
{
var httpWebRequest = (HttpWebRequest)System.Net.WebRequest.Create( webRequest.Destination );
httpWebRequest.Method = webRequest.Type.ToString().ToUpperInvariant();
// We shall handle redirects by hand so that we may capture cookies and properly
// handle login forms.
//
// Automating Web Login With HttpWebRequest
// https://www.stevefenton.co.uk/Content/Blog/Date/201210/Blog/Automating-Web-Login-With-HttpWebRequest/
#if !NETSTANDARD1_5
httpWebRequest.AllowAutoRedirect = false;
#endif
// Default headers.
httpWebRequest.Accept = "*/*";
httpWebRequest.Headers[HttpRequestHeader.UserAgent] = UserAgent;
// Set and/or override any provided headers.
foreach ( var headerName in webRequest.Headers.AllKeys ) {
ConfigureHeader( httpWebRequest, headerName, webRequest.Headers[headerName] );
}
httpWebRequest.CookieContainer = new CookieContainer();
httpWebRequest.CookieContainer.Add( webRequest.Destination, cookieJar.GetCookies( webRequest.Destination ) );
if ( webRequest.Type == WebRequestType.Post ) {
var postRequest = (PostWebRequest)webRequest;
var requestDataBytes = Encoding.UTF8.GetBytes( postRequest.RequestData );
Stream requestStream = null;
httpWebRequest.Headers[HttpRequestHeader.ContentLength] = requestDataBytes.Length.ToString();
httpWebRequest.ContentType = postRequest.ContentType;
#if !NETSTANDARD1_5
httpWebRequest.ServicePoint.Expect100Continue = false;
#endif
try {
requestStream = httpWebRequest.GetRequestStreamAsync().Result;
requestStream.Write( requestDataBytes, 0, requestDataBytes.Length );
}
finally {
if ( requestStream != null ) {
requestStream.Dispose();
}
}
}
OnSendingRequest( new SendingRequestEventArgs( webRequest ) );
WebResponse response;
HttpWebResponse webResponse = null;
try {
webResponse = ( HttpWebResponse )httpWebRequest.GetResponseAsync().Result;
OnProcessingResponse( new ProcessingResponseEventArgs( webResponse ) );
if ( httpWebRequest.HaveResponse ) {
// Process cookies that the .NET client 'forgot' to include,
// see http://stackoverflow.com/questions/15103513/httpwebresponse-cookies-empty-despite-set-cookie-header-no-redirect
// for more details;
// an example cookie which is not parsed is this one:
//
// Set-Cookie:ADCDownloadAuth=[long token];Version=1;Comment=;Domain=apple.com;Path=/;Max-Age=108000;HttpOnly;Expires=Tue, 03 May 2016 13:30:57 GMT
// Handle cookies that are offered
CookieCollection cookies = new CookieCollection();
cookies.Add(webResponse.Cookies);
if (webResponse.Headers.AllKeys.Contains("Set-Cookie"))
{
cookies.Parse(webResponse.Headers["Set-Cookie"], httpWebRequest.Headers[HttpRequestHeader.Host]);
}
foreach ( Cookie responseCookie in cookies ) {
var cookieFound = false;
foreach ( Cookie existingCookie in cookieJar.GetCookies( webRequest.Destination ) ) {
if ( responseCookie.Name.Equals( existingCookie.Name ) ) {
existingCookie.Value = responseCookie.Value;
cookieFound = true;
}
}
if ( !cookieFound ) {
var args = new AddingCookieEventArgs( responseCookie );
OnAddingCookie( args );
if ( !args.Cancel ) {
cookieJar.Add( new Uri(responseCookie.Domain), responseCookie );
}
}
}
if ( redirectionStatusCodes.Contains( webResponse.StatusCode ) ) {
// We have a redirected response, so get the new location.
var location = webResponse.Headers[CommonHeaders.Location];
// Locations should always be absolute, per the RFC (http://tools.ietf.org/html/rfc2616#section-14.30), but
// that won't always be the case.
Uri redirectUri;
if ( Uri.IsWellFormedUriString( location, UriKind.Absolute ) ) {
redirectUri = new Uri( location );
}
else {
redirectUri = new Uri( webRequest.Destination, new Uri( location, UriKind.Relative ) );
}
if ( webRequest.AutoRedirect ) {
// We are auto redirecting, so make a recursive call to perform the redirect by hand.
response = SendRequest( new GetWebRequest( redirectUri ) );
}
else {
// We are not auto redirecting, so send the caller a redirect response.
response = new RedirectedWebResponse( webResponse.ResponseUri, webRequest, redirectUri );
}
webResponse.Dispose();
}
else {
// We have a non-redirected response.
response = WebResponseFactory.CreateResponse( webResponse );
if ( response.ResponseType == WebResponseType.Html ) {
// We have an HTML response, so check for an old school Meta refresh tag
var metaRefreshUrl = GetMetaRefreshUrl( ( ( HtmlWebResponse )response ).Html );
if ( !string.IsNullOrWhiteSpace( metaRefreshUrl ) ) {
// The page has a Meta refresh tag, so build the redirect Url
var redirectUri = new Uri( response.ResponseUrl, metaRefreshUrl );
if ( webRequest.AutoRedirect ) {
response.Dispose();
// We are auto redirecting, so make a recursive call to perform the redirect
#if !NETSTANDARD1_5
response = SendRequest( new GetWebRequest( redirectUri, httpWebRequest.AllowAutoRedirect ) );
#else
response = SendRequest( new GetWebRequest( redirectUri ) );
#endif
}
else {
var responseUrl = response.ResponseUrl;
response.Dispose();
// We are not auto redirecting, so send the caller a redirect response
response = new RedirectedWebResponse( responseUrl, webRequest, redirectUri );
}
}
}
}
}
else {
response = new ExceptionWebResponse( webRequest.Destination, new WebException( NScrapeResources.NoResponse ) );
webResponse.Dispose();
}
}
catch ( WebException ex ) {
response = new ExceptionWebResponse( webRequest.Destination, ex );
if ( webResponse != null ) {
webResponse.Dispose();
}
}
return response;
}
