Example #1
0
        /// <include file='IWebClient.xml' path='/IWebClient/SendRequest_WebRequest/*'/>
        public WebResponse SendRequest( WebRequest webRequest )
        {
            var httpWebRequest = (HttpWebRequest)System.Net.WebRequest.Create( webRequest.Destination );

            httpWebRequest.Method = webRequest.Type.ToString().ToUpperInvariant();

            // We shall handle redirects by hand so that we may capture cookies and properly
            // handle login forms.
            //
            // Automating Web Login With HttpWebRequest
            // https://www.stevefenton.co.uk/Content/Blog/Date/201210/Blog/Automating-Web-Login-With-HttpWebRequest/
            #if !NETSTANDARD1_5
            httpWebRequest.AllowAutoRedirect = false;
            #endif

            // Default headers.
            httpWebRequest.Accept = "*/*";
            httpWebRequest.Headers[HttpRequestHeader.UserAgent] = UserAgent;

            // Set and/or override any provided headers.
            foreach ( var headerName in webRequest.Headers.AllKeys ) {
                ConfigureHeader( httpWebRequest, headerName, webRequest.Headers[headerName] );
            }

            httpWebRequest.CookieContainer = new CookieContainer();
            httpWebRequest.CookieContainer.Add( webRequest.Destination, cookieJar.GetCookies( webRequest.Destination ) );

            if ( webRequest.Type == WebRequestType.Post ) {
                var postRequest = (PostWebRequest)webRequest;

                var requestDataBytes = Encoding.UTF8.GetBytes( postRequest.RequestData );
                Stream requestStream = null;

                httpWebRequest.Headers[HttpRequestHeader.ContentLength] = requestDataBytes.Length.ToString();
                httpWebRequest.ContentType = postRequest.ContentType;
            #if !NETSTANDARD1_5
                httpWebRequest.ServicePoint.Expect100Continue = false;
            #endif

                try {
                    requestStream = httpWebRequest.GetRequestStreamAsync().Result;
                    requestStream.Write( requestDataBytes, 0, requestDataBytes.Length );
                }
                finally {
                    if ( requestStream != null ) {
                        requestStream.Dispose();
                    }
                }
            }

            OnSendingRequest( new SendingRequestEventArgs( webRequest ) );

            WebResponse response;
            HttpWebResponse webResponse = null;

            try {
                webResponse = ( HttpWebResponse )httpWebRequest.GetResponseAsync().Result;

                OnProcessingResponse( new ProcessingResponseEventArgs( webResponse ) );

                if ( httpWebRequest.HaveResponse ) {
                    // Process cookies that the .NET client 'forgot' to include,
                    // see http://stackoverflow.com/questions/15103513/httpwebresponse-cookies-empty-despite-set-cookie-header-no-redirect
                    // for more details;
                    // an example cookie which is not parsed is this one:
                    //
                    // Set-Cookie:ADCDownloadAuth=[long token];Version=1;Comment=;Domain=apple.com;Path=/;Max-Age=108000;HttpOnly;Expires=Tue, 03 May 2016 13:30:57 GMT

                    // Handle cookies that are offered
                    CookieCollection cookies = new CookieCollection();
                    cookies.Add(webResponse.Cookies);

                    if (webResponse.Headers.AllKeys.Contains("Set-Cookie"))
                    {
                        cookies.Parse(webResponse.Headers["Set-Cookie"], httpWebRequest.Headers[HttpRequestHeader.Host]);
                    }

                    foreach ( Cookie responseCookie in cookies ) {
                        var cookieFound = false;

                        foreach ( Cookie existingCookie in cookieJar.GetCookies( webRequest.Destination ) ) {
                            if ( responseCookie.Name.Equals( existingCookie.Name ) ) {
                                existingCookie.Value = responseCookie.Value;
                                cookieFound = true;
                            }
                        }

                        if ( !cookieFound ) {
                            var args = new AddingCookieEventArgs( responseCookie );

                            OnAddingCookie( args );

                            if ( !args.Cancel ) {
                                cookieJar.Add( new Uri(responseCookie.Domain), responseCookie );
                            }
                        }
                    }

                    if ( redirectionStatusCodes.Contains( webResponse.StatusCode ) ) {
                        // We have a redirected response, so get the new location.
                        var location = webResponse.Headers[CommonHeaders.Location];

                        // Locations should always be absolute, per the RFC (http://tools.ietf.org/html/rfc2616#section-14.30), but
                        // that won't always be the case.
                        Uri redirectUri;
                        if ( Uri.IsWellFormedUriString( location, UriKind.Absolute ) ) {
                            redirectUri = new Uri( location );
                        }
                        else {
                            redirectUri = new Uri( webRequest.Destination, new Uri( location, UriKind.Relative ) );
                        }

                        if ( webRequest.AutoRedirect ) {
                            // We are auto redirecting, so make a recursive call to perform the redirect by hand.
                            response = SendRequest( new GetWebRequest( redirectUri ) );
                        }
                        else {
                            // We are not auto redirecting, so send the caller a redirect response.
                            response = new RedirectedWebResponse( webResponse.ResponseUri, webRequest, redirectUri );
                        }

                        webResponse.Dispose();
                    }
                    else {
                        // We have a non-redirected response.
                        response = WebResponseFactory.CreateResponse( webResponse );

                        if ( response.ResponseType == WebResponseType.Html ) {
                            // We have an HTML response, so check for an old school Meta refresh tag
                            var metaRefreshUrl = GetMetaRefreshUrl( ( ( HtmlWebResponse )response ).Html );

                            if ( !string.IsNullOrWhiteSpace( metaRefreshUrl ) ) {
                                // The page has a Meta refresh tag, so build the redirect Url
                                var redirectUri = new Uri( response.ResponseUrl, metaRefreshUrl );

                                if ( webRequest.AutoRedirect ) {
                                    response.Dispose();

                                    // We are auto redirecting, so make a recursive call to perform the redirect
            #if !NETSTANDARD1_5
                                    response = SendRequest( new GetWebRequest( redirectUri, httpWebRequest.AllowAutoRedirect ) );
            #else
                                    response = SendRequest( new GetWebRequest( redirectUri ) );
            #endif
                                }
                                else {
                                    var responseUrl = response.ResponseUrl;

                                    response.Dispose();

                                    // We are not auto redirecting, so send the caller a redirect response
                                    response = new RedirectedWebResponse( responseUrl, webRequest, redirectUri );
                                }
                            }
                        }
                    }
                }
                else {
                    response = new ExceptionWebResponse( webRequest.Destination, new WebException( NScrapeResources.NoResponse ) );

                    webResponse.Dispose();
                }
            }
            catch ( WebException ex ) {
                response = new ExceptionWebResponse( webRequest.Destination, ex );

                if ( webResponse != null ) {
                    webResponse.Dispose();
                }
            }

            return response;
        }
Example #2
0
        /// <include file='IWebClient.xml' path='/IWebClient/SendRequest_WebRequest/*'/>
        public WebResponse SendRequest(WebRequest webRequest)
        {
            var httpWebRequest = (HttpWebRequest)System.Net.WebRequest.Create(webRequest.Destination);

            httpWebRequest.Method = webRequest.Type.ToString().ToUpperInvariant();

            // We shall handle redirects by hand so that we may capture cookies and properly
            // handle login forms.
            //
            // Automating Web Login With HttpWebRequest
            // https://www.stevefenton.co.uk/Content/Blog/Date/201210/Blog/Automating-Web-Login-With-HttpWebRequest/
            httpWebRequest.AllowAutoRedirect = false;

            // Default headers.
            httpWebRequest.Accept    = "*/*";
            httpWebRequest.UserAgent = UserAgent;

            // Set and/or override any provided headers.
            foreach (var headerName in webRequest.Headers.AllKeys)
            {
                ConfigureHeader(httpWebRequest, headerName, webRequest.Headers[headerName]);
            }

            httpWebRequest.CookieContainer = new CookieContainer();
            httpWebRequest.CookieContainer.Add(cookieJar.GetCookies(webRequest.Destination));

            if (webRequest.Type == WebRequestType.Post)
            {
                var postRequest = (PostWebRequest)webRequest;

                var    requestDataBytes = Encoding.UTF8.GetBytes(postRequest.RequestData);
                Stream requestStream    = null;

                httpWebRequest.ContentLength = requestDataBytes.Length;
                httpWebRequest.ContentType   = postRequest.ContentType;
                httpWebRequest.ServicePoint.Expect100Continue = false;

                try {
                    requestStream = httpWebRequest.GetRequestStream();
                    requestStream.Write(requestDataBytes, 0, requestDataBytes.Length);
                }
                finally {
                    requestStream?.Close();
                }
            }

            OnSendingRequest(new SendingRequestEventArgs(webRequest));

            WebResponse     response;
            HttpWebResponse webResponse = null;

            try {
                webResponse = ( HttpWebResponse )httpWebRequest.GetResponse();

                OnProcessingResponse(new ProcessingResponseEventArgs(webResponse));

                if (httpWebRequest.HaveResponse)
                {
                    var responseCookies = new CookieCollection {
                        webResponse.Cookies
                    };

                    // Some cookies in the Set-Cookie header can be omitted from the response's CookieCollection. For example:
                    //	Set-Cookie:ADCDownloadAuth=[long token];Version=1;Comment=;Domain=apple.com;Path=/;Max-Age=108000;HttpOnly;Expires=Tue, 03 May 2016 13:30:57 GMT
                    //
                    // See also:
                    // http://stackoverflow.com/questions/15103513/httpwebresponse-cookies-empty-despite-set-cookie-header-no-redirect
                    //
                    // To catch these, we parse the header manually and add any cookie that is missing.
                    if (webResponse.Headers.AllKeys.Contains(CommonHeaders.SetCookie))
                    {
                        var responseCookieList = responseCookies.OfType <Cookie>().ToList();

                        var cookies = NScrapeUtility.ParseSetCookieHeader(webResponse.Headers[CommonHeaders.SetCookie], httpWebRequest.Host);

                        foreach (var cookie in cookies)
                        {
                            if (responseCookieList.All(c => c.Name != cookie.Name))
                            {
                                responseCookies.Add(cookie);
                            }
                        }
                    }

                    // Handle cookies that are offered
                    foreach (Cookie responseCookie in responseCookies)
                    {
                        var cookieFound = false;

                        foreach (Cookie existingCookie in cookieJar.GetCookies(webRequest.Destination))
                        {
                            if (responseCookie.Name.Equals(existingCookie.Name))
                            {
                                existingCookie.Value = responseCookie.Value;
                                cookieFound          = true;
                            }
                        }

                        if (!cookieFound)
                        {
                            var args = new AddingCookieEventArgs(responseCookie);

                            OnAddingCookie(args);

                            if (!args.Cancel)
                            {
                                cookieJar.Add(responseCookie);
                            }
                        }
                    }

                    if (redirectionStatusCodes.Contains(webResponse.StatusCode))
                    {
                        // We have a redirected response, so get the new location.
                        var location = webResponse.Headers[CommonHeaders.Location];

                        // Locations should always be absolute, per the RFC (http://tools.ietf.org/html/rfc2616#section-14.30), but
                        // that won't always be the case.
                        Uri redirectUri;
                        if (Uri.IsWellFormedUriString(location, UriKind.Absolute))
                        {
                            redirectUri = new Uri(location);
                        }
                        else
                        {
                            redirectUri = new Uri(webRequest.Destination, new Uri(location, UriKind.Relative));
                        }

                        if (webRequest.AutoRedirect)
                        {
                            // We are auto redirecting, so make a recursive call to perform the redirect by hand.
                            response = SendRequest(new GetWebRequest(redirectUri));
                        }
                        else
                        {
                            // We are not auto redirecting, so send the caller a redirect response.
                            response = new RedirectedWebResponse(webResponse.ResponseUri, webRequest, redirectUri);
                        }

                        webResponse.Dispose();
                    }
                    else
                    {
                        // We have a non-redirected response.
                        response = WebResponseFactory.CreateResponse(webResponse);

                        if (response.ResponseType == WebResponseType.Html)
                        {
                            // We have an HTML response, so check for an old school Meta refresh tag
                            var metaRefreshUrl = GetMetaRefreshUrl((( HtmlWebResponse )response).Html);

                            if (!string.IsNullOrWhiteSpace(metaRefreshUrl))
                            {
                                // The page has a Meta refresh tag, so build the redirect Url
                                var redirectUri = new Uri(response.ResponseUrl, metaRefreshUrl);

                                if (webRequest.AutoRedirect)
                                {
                                    response.Dispose();

                                    // We are auto redirecting, so make a recursive call to perform the redirect
                                    response = SendRequest(new GetWebRequest(redirectUri, httpWebRequest.AllowAutoRedirect));
                                }
                                else
                                {
                                    var responseUrl = response.ResponseUrl;

                                    response.Dispose();

                                    // We are not auto redirecting, so send the caller a redirect response
                                    response = new RedirectedWebResponse(responseUrl, webRequest, redirectUri);
                                }
                            }
                        }
                    }
                }
                else
                {
                    response = new ExceptionWebResponse(webRequest.Destination, new WebException(NScrapeResources.NoResponse));

                    webResponse.Dispose();
                }
            }
            catch (WebException ex) {
                response = new ExceptionWebResponse(webRequest.Destination, ex);

                webResponse?.Dispose();
            }

            return(response);
        }
Example #3
0
        /// <summary>
        /// Sends a GET or POST request.
        /// </summary>
        /// <param name="webRequest">The request to send.</param>
        /// <remarks>
        /// Sends a GET or POST request. 
        /// </remarks>
        /// <returns>The response from the server.</returns>
        /// <example>
        /// <code>
        ///	var webClient = new WebClient();
        ///	
        ///	var request = new PostWebRequest() {
        ///		Destination = new Uri( "http://www.foo.com" ),
        ///		RequestData = "step=confirmation&amp;rt=L&amp;rp=%2Flogin%2Fhome&amp;p=0&amp;inputEmailHandle=foo&amp;inputPassword=bar"
        ///	};
        ///
        ///	var response = webClient.SendRequest( request );
        /// </code>
        /// </example>
        /// <seealso cref="WebRequest"/>
        /// <seealso cref="WebResponse"/>
        public WebResponse SendRequest( WebRequest webRequest )
        {
            var httpWebRequest = (HttpWebRequest)System.Net.WebRequest.Create( webRequest.Destination );

            httpWebRequest.Method = webRequest.Type.ToString().ToUpperInvariant();

            // We shall handle redirects by hand so that we may capture cookies and properly
            // handle login forms.
            //
            // Automating Web Login With HttpWebRequest
            // https://www.stevefenton.co.uk/Content/Blog/Date/201210/Blog/Automating-Web-Login-With-HttpWebRequest/
            httpWebRequest.AllowAutoRedirect = false;

            // Default headers.
            httpWebRequest.Accept = "*/*";
            httpWebRequest.UserAgent = UserAgent;

            // Set and/or override any provided headers.
            foreach ( var headerName in webRequest.Headers.AllKeys ) {
                ConfigureHeader( httpWebRequest, headerName, webRequest.Headers[headerName] );
            }

            httpWebRequest.CookieContainer = new CookieContainer();
            httpWebRequest.CookieContainer.Add( cookieJar.GetCookies( webRequest.Destination ) );

            if ( webRequest.Type == WebRequestType.Post ) {
                var postRequest = (PostWebRequest)webRequest;

                var requestDataBytes = Encoding.UTF8.GetBytes( postRequest.RequestData );
                Stream requestStream = null;

                httpWebRequest.ContentLength = requestDataBytes.Length;
                httpWebRequest.ContentType = postRequest.ContentType;
                httpWebRequest.ServicePoint.Expect100Continue = false;

                try {
                    requestStream = httpWebRequest.GetRequestStream();
                    requestStream.Write( requestDataBytes, 0, requestDataBytes.Length );
                }
                finally {
                    if ( requestStream != null ) {
                        requestStream.Close();
                    }
                }
            }

            OnSendingRequest( new SendingRequestEventArgs( webRequest ) );

            WebResponse response;

            try {
                var webResponse = ( HttpWebResponse )httpWebRequest.GetResponse();

                if ( httpWebRequest.HaveResponse ) {
                    // Handle cookies that are offered
                    foreach ( Cookie responseCookie in webResponse.Cookies ) {
                        var cookieFound = false;

                        foreach ( Cookie existingCookie in cookieJar.GetCookies( webRequest.Destination ) ) {
                            if ( responseCookie.Name.Equals( existingCookie.Name ) ) {
                                existingCookie.Value = responseCookie.Value;
                                cookieFound = true;
                            }
                        }

                        if ( !cookieFound ) {
                            var args = new AddingCookieEventArgs( responseCookie );

                            OnAddingCookie( args );

                            if ( !args.Cancel ) {
                                cookieJar.Add( responseCookie );
                            }
                        }
                    }

                    if ( redirectionStatusCodes.Contains( webResponse.StatusCode ) ) {
                        // We have a redirected response.

                        // Get the new location.
                        Uri redirectUri;
                        var location = webResponse.Headers[CommonHeaders.Location];

                        // Locations should always be absolute, per the RFC (http://tools.ietf.org/html/rfc2616#section-14.30), but
                        // that won't always be the case.
                        if ( Uri.IsWellFormedUriString( location, UriKind.Absolute ) ) {
                            redirectUri = new Uri( location );
                        }
                        else {
                            redirectUri = new Uri( webRequest.Destination, new Uri( location, UriKind.Relative ) );
                        }

                        if ( webRequest.AutoRedirect ) {
                            // We are auto redirecting, so make a recursive call to perform the redirect by hand.
                            response = SendRequest( new GetWebRequest( redirectUri ) );
                        }
                        else {
                            // We are not auto redirecting, so send the caller a redirect response.
                            response = new RedirectedWebResponse( webResponse.ResponseUri, webRequest, redirectUri );
                        }
                    }
                    else {
                        // We have a regular response.

                        var contentType = webResponse.Headers[CommonHeaders.ContentType];

                        if ( contentType.StartsWith( "image/", StringComparison.OrdinalIgnoreCase ) ) {
                            var image = new Bitmap( 0, 0 );

                            var s = webResponse.GetResponseStream();

                            if ( s != null ) {
                                image = new Bitmap( s );
                            }

                            response = new ImageWebResponse( true, webResponse.ResponseUri, image );
                        }
                        else {
                            // If a character set is not specified, RFC2616 section 3.7.1 says to use ISO-8859-1, per the page below.
                            // The page says this is more or less useless, but I did find that Chrome and Firefox behaved this way
                            // for javascript files that I tested.
                            // http://www.w3.org/TR/html4/charset.html#h-5.2.2
                            var characterSet = ( !string.IsNullOrEmpty( webResponse.CharacterSet ) ? webResponse.CharacterSet : "iso-8859-1" );
                            var encoding = Encoding.GetEncoding( characterSet );

                            if ( contentType.StartsWith( "text/html", StringComparison.OrdinalIgnoreCase ) ) {
                                var html = ReadResponseText( webResponse, encoding );

                                var haveRefresh = false;

                                // Check for an old school Meta refresh tag
                                var match = RegexCache.Instance.Regex( RegexLibrary.ParseMetaRefresh, RegexLibrary.ParseMetaRefreshOptions ).Match( html );

                                if ( match.Success ) {
                                    // If the Meta refresh is not within a NOSCRIPT block, we'll use it
                                    if ( match.Groups[RegexLibrary.ParseMetaRefreshStartNoScriptGroup].Value.Length == 0 && match.Groups[RegexLibrary.ParseMetaRefreshEndNoScriptGroup].Value.Length == 0 ) {
                                        haveRefresh = true;
                                    }
                                }

                                if ( haveRefresh ) {
                                    // The page has a Meta refresh tag, so build the redirect Url
                                    var redirectUri = new Uri( webResponse.ResponseUri, match.Groups[RegexLibrary.ParseMetaRefreshUrlGroup].Value );

                                    if ( webRequest.AutoRedirect ) {
                                        // We are auto redirecting, so make a recursive call to perform the redirect
                                        response = SendRequest( new GetWebRequest( redirectUri, httpWebRequest.AllowAutoRedirect ) );
                                    }
                                    else {
                                        // We are not auto redirecting, so send the caller a redirect response
                                        response = new RedirectedWebResponse( webResponse.ResponseUri, webRequest, redirectUri );
                                    }
                                }
                                else {
                                    // Just a regular Html page
                                    response = new HtmlWebResponse( true, webResponse.ResponseUri, html, encoding );
                                }
                            }
                            else if ( contentType.StartsWith( "text/xml", StringComparison.OrdinalIgnoreCase)
                                || contentType.StartsWith("application/xml", StringComparison.OrdinalIgnoreCase)) {
                                var xml = ReadResponseText( webResponse, encoding );

                                response = new XmlWebResponse( true, webResponse.ResponseUri, xml, encoding );
                            }
                            else if ( contentType.StartsWith( "text/plain", StringComparison.OrdinalIgnoreCase ) ) {
                                var text = ReadResponseText( webResponse, encoding );

                                response = new PlainTextWebResponse( true, webResponse.ResponseUri, text, encoding );
                            }
                            // Javascript might be specified in a few ways
                            // http://stackoverflow.com/a/1998417
                            else if ( contentType.Contains( "javascript" ) ) {
                                var text = ReadResponseText( webResponse, encoding );

                                response = new JavaScriptWebResponse( true, webResponse.ResponseUri, text, encoding );
                            }
                            else if ( contentType.StartsWith( "application/json", StringComparison.OrdinalIgnoreCase ) ) {
                                var text = ReadResponseText( webResponse, encoding );
                                response = new JsonWebResponse(true, webResponse.ResponseUri, text, encoding);
                            }
                            else {
                                response = new UnsupportedWebResponse( contentType, webResponse.ResponseUri );
                            }
                        }
                    }

                    webResponse.Close(); // also closes stream opened via GetResponseStream()
                }
                else {
                    response = new ExceptionWebResponse( new WebException( NScrapeResources.NoResponse ), webRequest.Destination );
                }
            }
            catch ( WebException ex ) {
                response = new ExceptionWebResponse( ex, webRequest.Destination );
            }

            return response;
        }
Example #4
0
        /// <include file='IWebClient.xml' path='/IWebClient/SendRequest_WebRequest/*'/>
        public WebResponse SendRequest(WebRequest webRequest)
        {
            var httpWebRequest = (HttpWebRequest)System.Net.WebRequest.Create(webRequest.Destination);

            httpWebRequest.Method = webRequest.Type.ToString().ToUpperInvariant();

            // We shall handle redirects by hand so that we may capture cookies and properly
            // handle login forms.
            //
            // Automating Web Login With HttpWebRequest
            // https://www.stevefenton.co.uk/Content/Blog/Date/201210/Blog/Automating-Web-Login-With-HttpWebRequest/
            httpWebRequest.AllowAutoRedirect = false;

            // Default headers.
            httpWebRequest.Accept    = "*/*";
            httpWebRequest.UserAgent = UserAgent;

            // Set and/or override any provided headers.
            foreach (var headerName in webRequest.Headers.AllKeys)
            {
                ConfigureHeader(httpWebRequest, headerName, webRequest.Headers[headerName]);
            }

            httpWebRequest.CookieContainer = new CookieContainer();
            httpWebRequest.CookieContainer.Add(webRequest.Destination, cookieJar.GetCookies(webRequest.Destination));

            if (webRequest.Type == WebRequestType.Post)
            {
                var postRequest = (PostWebRequest)webRequest;

                var    requestDataBytes = Encoding.UTF8.GetBytes(postRequest.RequestData);
                Stream requestStream    = null;

                httpWebRequest.ContentLength = requestDataBytes.Length;
                httpWebRequest.ContentType   = postRequest.ContentType;

                try {
                    requestStream = httpWebRequest.GetRequestStreamAsync().Result;
                    requestStream.Write(requestDataBytes, 0, requestDataBytes.Length);
                }
                finally {
                    requestStream?.Dispose();
                }
            }

            OnSendingRequest(new SendingRequestEventArgs(webRequest));

            WebResponse     response;
            HttpWebResponse webResponse = null;

            try {
                try {
                    webResponse = ( HttpWebResponse )httpWebRequest.GetResponseAsync().Result;
                }
                catch (AggregateException ex) {
                    // While the line above executes without exception under the .Net Framework, under
                    // .Net Core, it will throw an exception for non-successful (non-200) status codes.
                    // However, the response we need is buried within the exception, so pull it out and
                    // continue.
                    //
                    // See thread on the following page, notably the comment from davidsh on Sep 6, 2017:
                    //
                    // HttpWebRequest in .NET Core 2.0 throwing 301 Moved Permanently #23422
                    // https://github.com/dotnet/corefx/issues/23422
                    if (ex.InnerExceptions.Count == 1)
                    {
                        if (ex.InnerExceptions[0] is WebException webException)
                        {
                            if (webException.Response is HttpWebResponse httpWebResponse)
                            {
                                webResponse = httpWebResponse;
                            }
                        }
                    }

                    if (webResponse == null)
                    {
                        // The exception was not as expected so we can't process it.
                        throw;
                    }
                }

                OnProcessingResponse(new ProcessingResponseEventArgs(webResponse));

                if (httpWebRequest.HaveResponse)
                {
                    var responseCookies = new CookieCollection {
                        webResponse.Cookies
                    };

                    // Some cookies in the Set-Cookie header can be omitted from the response's CookieCollection. For example:
                    //	Set-Cookie:ADCDownloadAuth=[long token];Version=1;Comment=;Domain=apple.com;Path=/;Max-Age=108000;HttpOnly;Expires=Tue, 03 May 2016 13:30:57 GMT
                    //
                    // See also:
                    // http://stackoverflow.com/questions/15103513/httpwebresponse-cookies-empty-despite-set-cookie-header-no-redirect
                    //
                    // To catch these, we parse the header manually and add any cookie that is missing.
                    if (webResponse.Headers.AllKeys.Contains(CommonHeaders.SetCookie))
                    {
                        var responseCookieList = responseCookies.OfType <Cookie>().ToList();

                        var host    = httpWebRequest.Host;
                        var cookies = NScrapeUtility.ParseSetCookieHeader(webResponse.Headers[CommonHeaders.SetCookie], host);

                        foreach (var cookie in cookies)
                        {
                            if (responseCookieList.All(c => c.Name != cookie.Name))
                            {
                                responseCookies.Add(cookie);
                            }
                        }
                    }

                    // Handle cookies that are offered
                    foreach (Cookie responseCookie in responseCookies)
                    {
                        var cookieFound = false;

                        foreach (Cookie existingCookie in cookieJar.GetCookies(webRequest.Destination))
                        {
                            if (responseCookie.Name.Equals(existingCookie.Name))
                            {
                                existingCookie.Value = responseCookie.Value;
                                cookieFound          = true;
                            }
                        }

                        if (!cookieFound)
                        {
                            var args = new AddingCookieEventArgs(responseCookie);

                            OnAddingCookie(args);

                            if (!args.Cancel)
                            {
                                // .NET Core seems to enforce the fact that the cookie domain _must_ start with a dot,
                                // so let's make sure that's the case.
                                if (!string.IsNullOrEmpty(responseCookie.Domain) && !responseCookie.Domain.StartsWith("."))
                                {
                                    responseCookie.Domain = "." + responseCookie.Domain;
                                }

                                string url = responseCookie.Secure ? "https://" : "http://";
                                url += responseCookie.Domain.Substring(1);

                                var uri = new Uri(url);
                                cookieJar.Add(uri, responseCookie);
                            }
                        }
                    }

                    if (redirectionStatusCodes.Contains(webResponse.StatusCode))
                    {
                        // We have a redirected response, so get the new location.
                        var location = webResponse.Headers[CommonHeaders.Location];

                        // Locations should always be absolute, per the RFC (http://tools.ietf.org/html/rfc2616#section-14.30), but
                        // that won't always be the case.
                        Uri redirectUri;
                        if (Uri.IsWellFormedUriString(location, UriKind.Absolute))
                        {
                            redirectUri = new Uri(location);
                        }
                        else
                        {
                            redirectUri = new Uri(webRequest.Destination, new Uri(location, UriKind.Relative));
                        }

                        if (webRequest.AutoRedirect)
                        {
                            // Dispose webResponse before auto redirect, otherwise the connection will not be closed until all the auto redirect finished.
                            // http://www.wadewegner.com/2007/08/systemnetwebexception-when-issuing-more-than-two-concurrent-webrequests/
                            webResponse.Dispose();

                            // We are auto redirecting, so make a recursive call to perform the redirect by hand.
                            response = SendRequest(new GetWebRequest(redirectUri));
                        }
                        else
                        {
                            var responseUri = webResponse.ResponseUri;
                            webResponse.Dispose();

                            // We are not auto redirecting, so send the caller a redirect response.
                            response = new RedirectedWebResponse(responseUri, webRequest, redirectUri);
                        }
                    }
                    else
                    {
                        // We have a non-redirected response.
                        response = WebResponseFactory.CreateResponse(webResponse);

                        if (response.ResponseType == WebResponseType.Html)
                        {
                            // We have an HTML response, so check for an old school Meta refresh tag
                            var metaRefreshUrl = GetMetaRefreshUrl((( HtmlWebResponse )response).Html);

                            if (!string.IsNullOrWhiteSpace(metaRefreshUrl))
                            {
                                // The page has a Meta refresh tag, so build the redirect URL
                                var redirectUri = new Uri(response.ResponseUrl, metaRefreshUrl);

                                if (webRequest.AutoRedirect)
                                {
                                    // Dispose webResponse before auto redirect, otherwise the connection will not be closed until all the auto redirect finished.
                                    // http://www.wadewegner.com/2007/08/systemnetwebexception-when-issuing-more-than-two-concurrent-webrequests/
                                    response.Dispose();

                                    // We are auto redirecting, so make a recursive call to perform the redirect
                                    response = SendRequest(new GetWebRequest(redirectUri, httpWebRequest.AllowAutoRedirect));
                                }
                                else
                                {
                                    var responseUrl = response.ResponseUrl;

                                    response.Dispose();

                                    // We are not auto redirecting, so send the caller a redirect response
                                    response = new RedirectedWebResponse(responseUrl, webRequest, redirectUri);
                                }
                            }
                        }
                    }
                }
                else
                {
                    response = new ExceptionWebResponse(webRequest.Destination, new WebException(Properties.Resources.NoResponse));

                    webResponse.Dispose();
                }
            }
            catch (WebException ex) {
                response = new ExceptionWebResponse(webRequest.Destination, ex);

                webResponse?.Dispose();
            }

            return(response);
        }