/// <include file='IWebClient.xml' path='/IWebClient/SendRequest_WebRequest/*'/> public WebResponse SendRequest( WebRequest webRequest ) { var httpWebRequest = (HttpWebRequest)System.Net.WebRequest.Create( webRequest.Destination ); httpWebRequest.Method = webRequest.Type.ToString().ToUpperInvariant(); // We shall handle redirects by hand so that we may capture cookies and properly // handle login forms. // // Automating Web Login With HttpWebRequest // https://www.stevefenton.co.uk/Content/Blog/Date/201210/Blog/Automating-Web-Login-With-HttpWebRequest/ #if !NETSTANDARD1_5 httpWebRequest.AllowAutoRedirect = false; #endif // Default headers. httpWebRequest.Accept = "*/*"; httpWebRequest.Headers[HttpRequestHeader.UserAgent] = UserAgent; // Set and/or override any provided headers. foreach ( var headerName in webRequest.Headers.AllKeys ) { ConfigureHeader( httpWebRequest, headerName, webRequest.Headers[headerName] ); } httpWebRequest.CookieContainer = new CookieContainer(); httpWebRequest.CookieContainer.Add( webRequest.Destination, cookieJar.GetCookies( webRequest.Destination ) ); if ( webRequest.Type == WebRequestType.Post ) { var postRequest = (PostWebRequest)webRequest; var requestDataBytes = Encoding.UTF8.GetBytes( postRequest.RequestData ); Stream requestStream = null; httpWebRequest.Headers[HttpRequestHeader.ContentLength] = requestDataBytes.Length.ToString(); httpWebRequest.ContentType = postRequest.ContentType; #if !NETSTANDARD1_5 httpWebRequest.ServicePoint.Expect100Continue = false; #endif try { requestStream = httpWebRequest.GetRequestStreamAsync().Result; requestStream.Write( requestDataBytes, 0, requestDataBytes.Length ); } finally { if ( requestStream != null ) { requestStream.Dispose(); } } } OnSendingRequest( new SendingRequestEventArgs( webRequest ) ); WebResponse response; HttpWebResponse webResponse = null; try { webResponse = ( HttpWebResponse )httpWebRequest.GetResponseAsync().Result; OnProcessingResponse( new ProcessingResponseEventArgs( webResponse ) ); if ( httpWebRequest.HaveResponse ) { // Process cookies that the .NET client 'forgot' to include, // see http://stackoverflow.com/questions/15103513/httpwebresponse-cookies-empty-despite-set-cookie-header-no-redirect // for more details; // an example cookie which is not parsed is this one: // // Set-Cookie:ADCDownloadAuth=[long token];Version=1;Comment=;Domain=apple.com;Path=/;Max-Age=108000;HttpOnly;Expires=Tue, 03 May 2016 13:30:57 GMT // Handle cookies that are offered CookieCollection cookies = new CookieCollection(); cookies.Add(webResponse.Cookies); if (webResponse.Headers.AllKeys.Contains("Set-Cookie")) { cookies.Parse(webResponse.Headers["Set-Cookie"], httpWebRequest.Headers[HttpRequestHeader.Host]); } foreach ( Cookie responseCookie in cookies ) { var cookieFound = false; foreach ( Cookie existingCookie in cookieJar.GetCookies( webRequest.Destination ) ) { if ( responseCookie.Name.Equals( existingCookie.Name ) ) { existingCookie.Value = responseCookie.Value; cookieFound = true; } } if ( !cookieFound ) { var args = new AddingCookieEventArgs( responseCookie ); OnAddingCookie( args ); if ( !args.Cancel ) { cookieJar.Add( new Uri(responseCookie.Domain), responseCookie ); } } } if ( redirectionStatusCodes.Contains( webResponse.StatusCode ) ) { // We have a redirected response, so get the new location. var location = webResponse.Headers[CommonHeaders.Location]; // Locations should always be absolute, per the RFC (http://tools.ietf.org/html/rfc2616#section-14.30), but // that won't always be the case. Uri redirectUri; if ( Uri.IsWellFormedUriString( location, UriKind.Absolute ) ) { redirectUri = new Uri( location ); } else { redirectUri = new Uri( webRequest.Destination, new Uri( location, UriKind.Relative ) ); } if ( webRequest.AutoRedirect ) { // We are auto redirecting, so make a recursive call to perform the redirect by hand. response = SendRequest( new GetWebRequest( redirectUri ) ); } else { // We are not auto redirecting, so send the caller a redirect response. response = new RedirectedWebResponse( webResponse.ResponseUri, webRequest, redirectUri ); } webResponse.Dispose(); } else { // We have a non-redirected response. response = WebResponseFactory.CreateResponse( webResponse ); if ( response.ResponseType == WebResponseType.Html ) { // We have an HTML response, so check for an old school Meta refresh tag var metaRefreshUrl = GetMetaRefreshUrl( ( ( HtmlWebResponse )response ).Html ); if ( !string.IsNullOrWhiteSpace( metaRefreshUrl ) ) { // The page has a Meta refresh tag, so build the redirect Url var redirectUri = new Uri( response.ResponseUrl, metaRefreshUrl ); if ( webRequest.AutoRedirect ) { response.Dispose(); // We are auto redirecting, so make a recursive call to perform the redirect #if !NETSTANDARD1_5 response = SendRequest( new GetWebRequest( redirectUri, httpWebRequest.AllowAutoRedirect ) ); #else response = SendRequest( new GetWebRequest( redirectUri ) ); #endif } else { var responseUrl = response.ResponseUrl; response.Dispose(); // We are not auto redirecting, so send the caller a redirect response response = new RedirectedWebResponse( responseUrl, webRequest, redirectUri ); } } } } } else { response = new ExceptionWebResponse( webRequest.Destination, new WebException( NScrapeResources.NoResponse ) ); webResponse.Dispose(); } } catch ( WebException ex ) { response = new ExceptionWebResponse( webRequest.Destination, ex ); if ( webResponse != null ) { webResponse.Dispose(); } } return response; }