/// <include file='IWebClient.xml' path='/IWebClient/SendRequest_WebRequest/*'/>
public WebResponse SendRequest( WebRequest webRequest )
{
var httpWebRequest = (HttpWebRequest)System.Net.WebRequest.Create( webRequest.Destination );
httpWebRequest.Method = webRequest.Type.ToString().ToUpperInvariant();
// We shall handle redirects by hand so that we may capture cookies and properly
// handle login forms.
//
// Automating Web Login With HttpWebRequest
// https://www.stevefenton.co.uk/Content/Blog/Date/201210/Blog/Automating-Web-Login-With-HttpWebRequest/
#if !NETSTANDARD1_5
httpWebRequest.AllowAutoRedirect = false;
#endif
// Default headers.
httpWebRequest.Accept = "*/*";
httpWebRequest.Headers[HttpRequestHeader.UserAgent] = UserAgent;
// Set and/or override any provided headers.
foreach ( var headerName in webRequest.Headers.AllKeys ) {
ConfigureHeader( httpWebRequest, headerName, webRequest.Headers[headerName] );
}
httpWebRequest.CookieContainer = new CookieContainer();
httpWebRequest.CookieContainer.Add( webRequest.Destination, cookieJar.GetCookies( webRequest.Destination ) );
if ( webRequest.Type == WebRequestType.Post ) {
var postRequest = (PostWebRequest)webRequest;
var requestDataBytes = Encoding.UTF8.GetBytes( postRequest.RequestData );
Stream requestStream = null;
httpWebRequest.Headers[HttpRequestHeader.ContentLength] = requestDataBytes.Length.ToString();
httpWebRequest.ContentType = postRequest.ContentType;
#if !NETSTANDARD1_5
httpWebRequest.ServicePoint.Expect100Continue = false;
#endif
try {
requestStream = httpWebRequest.GetRequestStreamAsync().Result;
requestStream.Write( requestDataBytes, 0, requestDataBytes.Length );
}
finally {
if ( requestStream != null ) {
requestStream.Dispose();
}
}
}
OnSendingRequest( new SendingRequestEventArgs( webRequest ) );
WebResponse response;
HttpWebResponse webResponse = null;
try {
webResponse = ( HttpWebResponse )httpWebRequest.GetResponseAsync().Result;
OnProcessingResponse( new ProcessingResponseEventArgs( webResponse ) );
if ( httpWebRequest.HaveResponse ) {
// Process cookies that the .NET client 'forgot' to include,
// see http://stackoverflow.com/questions/15103513/httpwebresponse-cookies-empty-despite-set-cookie-header-no-redirect
// for more details;
// an example cookie which is not parsed is this one:
//
// Set-Cookie:ADCDownloadAuth=[long token];Version=1;Comment=;Domain=apple.com;Path=/;Max-Age=108000;HttpOnly;Expires=Tue, 03 May 2016 13:30:57 GMT
// Handle cookies that are offered
CookieCollection cookies = new CookieCollection();
cookies.Add(webResponse.Cookies);
if (webResponse.Headers.AllKeys.Contains("Set-Cookie"))
{
cookies.Parse(webResponse.Headers["Set-Cookie"], httpWebRequest.Headers[HttpRequestHeader.Host]);
}
foreach ( Cookie responseCookie in cookies ) {
var cookieFound = false;
foreach ( Cookie existingCookie in cookieJar.GetCookies( webRequest.Destination ) ) {
if ( responseCookie.Name.Equals( existingCookie.Name ) ) {
existingCookie.Value = responseCookie.Value;
cookieFound = true;
}
}
if ( !cookieFound ) {
var args = new AddingCookieEventArgs( responseCookie );
OnAddingCookie( args );
if ( !args.Cancel ) {
cookieJar.Add( new Uri(responseCookie.Domain), responseCookie );
}
}
}
if ( redirectionStatusCodes.Contains( webResponse.StatusCode ) ) {
// We have a redirected response, so get the new location.
var location = webResponse.Headers[CommonHeaders.Location];
// Locations should always be absolute, per the RFC (http://tools.ietf.org/html/rfc2616#section-14.30), but
// that won't always be the case.
Uri redirectUri;
if ( Uri.IsWellFormedUriString( location, UriKind.Absolute ) ) {
redirectUri = new Uri( location );
}
else {
redirectUri = new Uri( webRequest.Destination, new Uri( location, UriKind.Relative ) );
}
if ( webRequest.AutoRedirect ) {
// We are auto redirecting, so make a recursive call to perform the redirect by hand.
response = SendRequest( new GetWebRequest( redirectUri ) );
}
else {
// We are not auto redirecting, so send the caller a redirect response.
response = new RedirectedWebResponse( webResponse.ResponseUri, webRequest, redirectUri );
}
webResponse.Dispose();
}
else {
// We have a non-redirected response.
response = WebResponseFactory.CreateResponse( webResponse );
if ( response.ResponseType == WebResponseType.Html ) {
// We have an HTML response, so check for an old school Meta refresh tag
var metaRefreshUrl = GetMetaRefreshUrl( ( ( HtmlWebResponse )response ).Html );
if ( !string.IsNullOrWhiteSpace( metaRefreshUrl ) ) {
// The page has a Meta refresh tag, so build the redirect Url
var redirectUri = new Uri( response.ResponseUrl, metaRefreshUrl );
if ( webRequest.AutoRedirect ) {
response.Dispose();
// We are auto redirecting, so make a recursive call to perform the redirect
#if !NETSTANDARD1_5
response = SendRequest( new GetWebRequest( redirectUri, httpWebRequest.AllowAutoRedirect ) );
#else
response = SendRequest( new GetWebRequest( redirectUri ) );
#endif
}
else {
var responseUrl = response.ResponseUrl;
response.Dispose();
// We are not auto redirecting, so send the caller a redirect response
response = new RedirectedWebResponse( responseUrl, webRequest, redirectUri );
}
}
}
}
}
else {
response = new ExceptionWebResponse( webRequest.Destination, new WebException( NScrapeResources.NoResponse ) );
webResponse.Dispose();
}
}
catch ( WebException ex ) {
response = new ExceptionWebResponse( webRequest.Destination, ex );
if ( webResponse != null ) {
webResponse.Dispose();
}
}
return response;
}
/// <summary>
/// Raises the <see cref="AddingCookie"/> event.
/// </summary>
/// <remarks>
/// Called when a cookie is added to the <see cref="CookieJar"/>.
/// </remarks>
/// <param name="args">An <see cref="AddingCookieEventArgs"/> that contains the event data.</param>
/// <seealso cref="AddingCookie"/>
/// <seealso cref="AddingCookieEventArgs"/>
protected virtual void OnAddingCookie( AddingCookieEventArgs args )
{
if ( AddingCookie != null ) {
AddingCookie( this, args );
}
}
/// <summary>
/// Sends a GET or POST request.
/// </summary>
/// <param name="webRequest">The request to send.</param>
/// <remarks>
/// Sends a GET or POST request.
/// </remarks>
/// <returns>The response from the server.</returns>
/// <example>
/// <code>
/// var webClient = new WebClient();
///
/// var request = new PostWebRequest() {
/// Destination = new Uri( "http://www.foo.com" ),
/// RequestData = "step=confirmation&rt=L&rp=%2Flogin%2Fhome&p=0&inputEmailHandle=foo&inputPassword=bar"
/// };
///
/// var response = webClient.SendRequest( request );
/// </code>
/// </example>
/// <seealso cref="WebRequest"/>
/// <seealso cref="WebResponse"/>
public WebResponse SendRequest( WebRequest webRequest )
{
var httpWebRequest = (HttpWebRequest)System.Net.WebRequest.Create( webRequest.Destination );
httpWebRequest.Method = webRequest.Type.ToString().ToUpperInvariant();
// We shall handle redirects by hand so that we may capture cookies and properly
// handle login forms.
//
// Automating Web Login With HttpWebRequest
// https://www.stevefenton.co.uk/Content/Blog/Date/201210/Blog/Automating-Web-Login-With-HttpWebRequest/
httpWebRequest.AllowAutoRedirect = false;
// Default headers.
httpWebRequest.Accept = "*/*";
httpWebRequest.UserAgent = UserAgent;
// Set and/or override any provided headers.
foreach ( var headerName in webRequest.Headers.AllKeys ) {
ConfigureHeader( httpWebRequest, headerName, webRequest.Headers[headerName] );
}
httpWebRequest.CookieContainer = new CookieContainer();
httpWebRequest.CookieContainer.Add( cookieJar.GetCookies( webRequest.Destination ) );
if ( webRequest.Type == WebRequestType.Post ) {
var postRequest = (PostWebRequest)webRequest;
var requestDataBytes = Encoding.UTF8.GetBytes( postRequest.RequestData );
Stream requestStream = null;
httpWebRequest.ContentLength = requestDataBytes.Length;
httpWebRequest.ContentType = postRequest.ContentType;
httpWebRequest.ServicePoint.Expect100Continue = false;
try {
requestStream = httpWebRequest.GetRequestStream();
requestStream.Write( requestDataBytes, 0, requestDataBytes.Length );
}
finally {
if ( requestStream != null ) {
requestStream.Close();
}
}
}
OnSendingRequest( new SendingRequestEventArgs( webRequest ) );
WebResponse response;
try {
var webResponse = ( HttpWebResponse )httpWebRequest.GetResponse();
if ( httpWebRequest.HaveResponse ) {
// Handle cookies that are offered
foreach ( Cookie responseCookie in webResponse.Cookies ) {
var cookieFound = false;
foreach ( Cookie existingCookie in cookieJar.GetCookies( webRequest.Destination ) ) {
if ( responseCookie.Name.Equals( existingCookie.Name ) ) {
existingCookie.Value = responseCookie.Value;
cookieFound = true;
}
}
if ( !cookieFound ) {
var args = new AddingCookieEventArgs( responseCookie );
OnAddingCookie( args );
if ( !args.Cancel ) {
cookieJar.Add( responseCookie );
}
}
}
if ( redirectionStatusCodes.Contains( webResponse.StatusCode ) ) {
// We have a redirected response.
// Get the new location.
Uri redirectUri;
var location = webResponse.Headers[CommonHeaders.Location];
// Locations should always be absolute, per the RFC (http://tools.ietf.org/html/rfc2616#section-14.30), but
// that won't always be the case.
if ( Uri.IsWellFormedUriString( location, UriKind.Absolute ) ) {
redirectUri = new Uri( location );
}
else {
redirectUri = new Uri( webRequest.Destination, new Uri( location, UriKind.Relative ) );
}
if ( webRequest.AutoRedirect ) {
// We are auto redirecting, so make a recursive call to perform the redirect by hand.
response = SendRequest( new GetWebRequest( redirectUri ) );
}
else {
// We are not auto redirecting, so send the caller a redirect response.
response = new RedirectedWebResponse( webResponse.ResponseUri, webRequest, redirectUri );
}
}
else {
// We have a regular response.
var contentType = webResponse.Headers[CommonHeaders.ContentType];
if ( contentType.StartsWith( "image/", StringComparison.OrdinalIgnoreCase ) ) {
var image = new Bitmap( 0, 0 );
var s = webResponse.GetResponseStream();
if ( s != null ) {
image = new Bitmap( s );
}
response = new ImageWebResponse( true, webResponse.ResponseUri, image );
}
else {
// If a character set is not specified, RFC2616 section 3.7.1 says to use ISO-8859-1, per the page below.
// The page says this is more or less useless, but I did find that Chrome and Firefox behaved this way
// for javascript files that I tested.
// http://www.w3.org/TR/html4/charset.html#h-5.2.2
var characterSet = ( !string.IsNullOrEmpty( webResponse.CharacterSet ) ? webResponse.CharacterSet : "iso-8859-1" );
var encoding = Encoding.GetEncoding( characterSet );
if ( contentType.StartsWith( "text/html", StringComparison.OrdinalIgnoreCase ) ) {
var html = ReadResponseText( webResponse, encoding );
var haveRefresh = false;
// Check for an old school Meta refresh tag
var match = RegexCache.Instance.Regex( RegexLibrary.ParseMetaRefresh, RegexLibrary.ParseMetaRefreshOptions ).Match( html );
if ( match.Success ) {
// If the Meta refresh is not within a NOSCRIPT block, we'll use it
if ( match.Groups[RegexLibrary.ParseMetaRefreshStartNoScriptGroup].Value.Length == 0 && match.Groups[RegexLibrary.ParseMetaRefreshEndNoScriptGroup].Value.Length == 0 ) {
haveRefresh = true;
}
}
if ( haveRefresh ) {
// The page has a Meta refresh tag, so build the redirect Url
var redirectUri = new Uri( webResponse.ResponseUri, match.Groups[RegexLibrary.ParseMetaRefreshUrlGroup].Value );
if ( webRequest.AutoRedirect ) {
// We are auto redirecting, so make a recursive call to perform the redirect
response = SendRequest( new GetWebRequest( redirectUri, httpWebRequest.AllowAutoRedirect ) );
}
else {
// We are not auto redirecting, so send the caller a redirect response
response = new RedirectedWebResponse( webResponse.ResponseUri, webRequest, redirectUri );
}
}
else {
// Just a regular Html page
response = new HtmlWebResponse( true, webResponse.ResponseUri, html, encoding );
}
}
else if ( contentType.StartsWith( "text/xml", StringComparison.OrdinalIgnoreCase)
|| contentType.StartsWith("application/xml", StringComparison.OrdinalIgnoreCase)) {
var xml = ReadResponseText( webResponse, encoding );
response = new XmlWebResponse( true, webResponse.ResponseUri, xml, encoding );
}
else if ( contentType.StartsWith( "text/plain", StringComparison.OrdinalIgnoreCase ) ) {
var text = ReadResponseText( webResponse, encoding );
response = new PlainTextWebResponse( true, webResponse.ResponseUri, text, encoding );
}
// Javascript might be specified in a few ways
// http://stackoverflow.com/a/1998417
else if ( contentType.Contains( "javascript" ) ) {
var text = ReadResponseText( webResponse, encoding );
response = new JavaScriptWebResponse( true, webResponse.ResponseUri, text, encoding );
}
else if ( contentType.StartsWith( "application/json", StringComparison.OrdinalIgnoreCase ) ) {
var text = ReadResponseText( webResponse, encoding );
response = new JsonWebResponse(true, webResponse.ResponseUri, text, encoding);
}
else {
response = new UnsupportedWebResponse( contentType, webResponse.ResponseUri );
}
}
}
webResponse.Close(); // also closes stream opened via GetResponseStream()
}
else {
response = new ExceptionWebResponse( new WebException( NScrapeResources.NoResponse ), webRequest.Destination );
}
}
catch ( WebException ex ) {
response = new ExceptionWebResponse( ex, webRequest.Destination );
}
return response;
}
/// <include file='IWebClient.xml' path='/IWebClient/SendRequest_WebRequest/*'/>
public WebResponse SendRequest(WebRequest webRequest)
{
var httpWebRequest = (HttpWebRequest)System.Net.WebRequest.Create(webRequest.Destination);
httpWebRequest.Method = webRequest.Type.ToString().ToUpperInvariant();
// We shall handle redirects by hand so that we may capture cookies and properly
// handle login forms.
//
// Automating Web Login With HttpWebRequest
// https://www.stevefenton.co.uk/Content/Blog/Date/201210/Blog/Automating-Web-Login-With-HttpWebRequest/
httpWebRequest.AllowAutoRedirect = false;
// Default headers.
httpWebRequest.Accept = "*/*";
httpWebRequest.UserAgent = UserAgent;
// Set and/or override any provided headers.
foreach (var headerName in webRequest.Headers.AllKeys)
{
ConfigureHeader(httpWebRequest, headerName, webRequest.Headers[headerName]);
}
httpWebRequest.CookieContainer = new CookieContainer();
httpWebRequest.CookieContainer.Add(cookieJar.GetCookies(webRequest.Destination));
if (webRequest.Type == WebRequestType.Post)
{
var postRequest = (PostWebRequest)webRequest;
var requestDataBytes = Encoding.UTF8.GetBytes(postRequest.RequestData);
Stream requestStream = null;
httpWebRequest.ContentLength = requestDataBytes.Length;
httpWebRequest.ContentType = postRequest.ContentType;
httpWebRequest.ServicePoint.Expect100Continue = false;
try {
requestStream = httpWebRequest.GetRequestStream();
requestStream.Write(requestDataBytes, 0, requestDataBytes.Length);
}
finally {
requestStream?.Close();
}
}
OnSendingRequest(new SendingRequestEventArgs(webRequest));
WebResponse response;
HttpWebResponse webResponse = null;
try {
webResponse = ( HttpWebResponse )httpWebRequest.GetResponse();
OnProcessingResponse(new ProcessingResponseEventArgs(webResponse));
if (httpWebRequest.HaveResponse)
{
var responseCookies = new CookieCollection {
webResponse.Cookies
};
// Some cookies in the Set-Cookie header can be omitted from the response's CookieCollection. For example:
// Set-Cookie:ADCDownloadAuth=[long token];Version=1;Comment=;Domain=apple.com;Path=/;Max-Age=108000;HttpOnly;Expires=Tue, 03 May 2016 13:30:57 GMT
//
// See also:
// http://stackoverflow.com/questions/15103513/httpwebresponse-cookies-empty-despite-set-cookie-header-no-redirect
//
// To catch these, we parse the header manually and add any cookie that is missing.
if (webResponse.Headers.AllKeys.Contains(CommonHeaders.SetCookie))
{
var responseCookieList = responseCookies.OfType <Cookie>().ToList();
var cookies = NScrapeUtility.ParseSetCookieHeader(webResponse.Headers[CommonHeaders.SetCookie], httpWebRequest.Host);
foreach (var cookie in cookies)
{
if (responseCookieList.All(c => c.Name != cookie.Name))
{
responseCookies.Add(cookie);
}
}
}
// Handle cookies that are offered
foreach (Cookie responseCookie in responseCookies)
{
var cookieFound = false;
foreach (Cookie existingCookie in cookieJar.GetCookies(webRequest.Destination))
{
if (responseCookie.Name.Equals(existingCookie.Name))
{
existingCookie.Value = responseCookie.Value;
cookieFound = true;
}
}
if (!cookieFound)
{
var args = new AddingCookieEventArgs(responseCookie);
OnAddingCookie(args);
if (!args.Cancel)
{
cookieJar.Add(responseCookie);
}
}
}
if (redirectionStatusCodes.Contains(webResponse.StatusCode))
{
// We have a redirected response, so get the new location.
var location = webResponse.Headers[CommonHeaders.Location];
// Locations should always be absolute, per the RFC (http://tools.ietf.org/html/rfc2616#section-14.30), but
// that won't always be the case.
Uri redirectUri;
if (Uri.IsWellFormedUriString(location, UriKind.Absolute))
{
redirectUri = new Uri(location);
}
else
{
redirectUri = new Uri(webRequest.Destination, new Uri(location, UriKind.Relative));
}
if (webRequest.AutoRedirect)
{
// We are auto redirecting, so make a recursive call to perform the redirect by hand.
response = SendRequest(new GetWebRequest(redirectUri));
}
else
{
// We are not auto redirecting, so send the caller a redirect response.
response = new RedirectedWebResponse(webResponse.ResponseUri, webRequest, redirectUri);
}
webResponse.Dispose();
}
else
{
// We have a non-redirected response.
response = WebResponseFactory.CreateResponse(webResponse);
if (response.ResponseType == WebResponseType.Html)
{
// We have an HTML response, so check for an old school Meta refresh tag
var metaRefreshUrl = GetMetaRefreshUrl((( HtmlWebResponse )response).Html);
if (!string.IsNullOrWhiteSpace(metaRefreshUrl))
{
// The page has a Meta refresh tag, so build the redirect Url
var redirectUri = new Uri(response.ResponseUrl, metaRefreshUrl);
if (webRequest.AutoRedirect)
{
response.Dispose();
// We are auto redirecting, so make a recursive call to perform the redirect
response = SendRequest(new GetWebRequest(redirectUri, httpWebRequest.AllowAutoRedirect));
}
else
{
var responseUrl = response.ResponseUrl;
response.Dispose();
// We are not auto redirecting, so send the caller a redirect response
response = new RedirectedWebResponse(responseUrl, webRequest, redirectUri);
}
}
}
}
}
else
{
response = new ExceptionWebResponse(webRequest.Destination, new WebException(NScrapeResources.NoResponse));
webResponse.Dispose();
}
}
catch (WebException ex) {
response = new ExceptionWebResponse(webRequest.Destination, ex);
webResponse?.Dispose();
}
return(response);
}
/// <summary>
/// Raises the <see cref="AddingCookie"/> event.
/// </summary>
/// <remarks>
/// Called when a cookie is added to the <see cref="CookieJar"/>.
/// </remarks>
/// <param name="args">An <see cref="AddingCookieEventArgs"/> that contains the event data.</param>
/// <seealso cref="AddingCookie"/>
/// <seealso cref="AddingCookieEventArgs"/>
protected virtual void OnAddingCookie(AddingCookieEventArgs args)
{
AddingCookie?.Invoke(this, args);
}
/// <include file='IWebClient.xml' path='/IWebClient/SendRequest_WebRequest/*'/>
public WebResponse SendRequest(WebRequest webRequest)
{
var httpWebRequest = (HttpWebRequest)System.Net.WebRequest.Create(webRequest.Destination);
httpWebRequest.Method = webRequest.Type.ToString().ToUpperInvariant();
// We shall handle redirects by hand so that we may capture cookies and properly
// handle login forms.
//
// Automating Web Login With HttpWebRequest
// https://www.stevefenton.co.uk/Content/Blog/Date/201210/Blog/Automating-Web-Login-With-HttpWebRequest/
httpWebRequest.AllowAutoRedirect = false;
// Default headers.
httpWebRequest.Accept = "*/*";
httpWebRequest.UserAgent = UserAgent;
// Set and/or override any provided headers.
foreach (var headerName in webRequest.Headers.AllKeys)
{
ConfigureHeader(httpWebRequest, headerName, webRequest.Headers[headerName]);
}
httpWebRequest.CookieContainer = new CookieContainer();
httpWebRequest.CookieContainer.Add(webRequest.Destination, cookieJar.GetCookies(webRequest.Destination));
if (webRequest.Type == WebRequestType.Post)
{
var postRequest = (PostWebRequest)webRequest;
var requestDataBytes = Encoding.UTF8.GetBytes(postRequest.RequestData);
Stream requestStream = null;
httpWebRequest.ContentLength = requestDataBytes.Length;
httpWebRequest.ContentType = postRequest.ContentType;
try {
requestStream = httpWebRequest.GetRequestStreamAsync().Result;
requestStream.Write(requestDataBytes, 0, requestDataBytes.Length);
}
finally {
requestStream?.Dispose();
}
}
OnSendingRequest(new SendingRequestEventArgs(webRequest));
WebResponse response;
HttpWebResponse webResponse = null;
try {
try {
webResponse = ( HttpWebResponse )httpWebRequest.GetResponseAsync().Result;
}
catch (AggregateException ex) {
// While the line above executes without exception under the .Net Framework, under
// .Net Core, it will throw an exception for non-successful (non-200) status codes.
// However, the response we need is buried within the exception, so pull it out and
// continue.
//
// See thread on the following page, notably the comment from davidsh on Sep 6, 2017:
//
// HttpWebRequest in .NET Core 2.0 throwing 301 Moved Permanently #23422
// https://github.com/dotnet/corefx/issues/23422
if (ex.InnerExceptions.Count == 1)
{
if (ex.InnerExceptions[0] is WebException webException)
{
if (webException.Response is HttpWebResponse httpWebResponse)
{
webResponse = httpWebResponse;
}
}
}
if (webResponse == null)
{
// The exception was not as expected so we can't process it.
throw;
}
}
OnProcessingResponse(new ProcessingResponseEventArgs(webResponse));
if (httpWebRequest.HaveResponse)
{
var responseCookies = new CookieCollection {
webResponse.Cookies
};
// Some cookies in the Set-Cookie header can be omitted from the response's CookieCollection. For example:
// Set-Cookie:ADCDownloadAuth=[long token];Version=1;Comment=;Domain=apple.com;Path=/;Max-Age=108000;HttpOnly;Expires=Tue, 03 May 2016 13:30:57 GMT
//
// See also:
// http://stackoverflow.com/questions/15103513/httpwebresponse-cookies-empty-despite-set-cookie-header-no-redirect
//
// To catch these, we parse the header manually and add any cookie that is missing.
if (webResponse.Headers.AllKeys.Contains(CommonHeaders.SetCookie))
{
var responseCookieList = responseCookies.OfType <Cookie>().ToList();
var host = httpWebRequest.Host;
var cookies = NScrapeUtility.ParseSetCookieHeader(webResponse.Headers[CommonHeaders.SetCookie], host);
foreach (var cookie in cookies)
{
if (responseCookieList.All(c => c.Name != cookie.Name))
{
responseCookies.Add(cookie);
}
}
}
// Handle cookies that are offered
foreach (Cookie responseCookie in responseCookies)
{
var cookieFound = false;
foreach (Cookie existingCookie in cookieJar.GetCookies(webRequest.Destination))
{
if (responseCookie.Name.Equals(existingCookie.Name))
{
existingCookie.Value = responseCookie.Value;
cookieFound = true;
}
}
if (!cookieFound)
{
var args = new AddingCookieEventArgs(responseCookie);
OnAddingCookie(args);
if (!args.Cancel)
{
// .NET Core seems to enforce the fact that the cookie domain _must_ start with a dot,
// so let's make sure that's the case.
if (!string.IsNullOrEmpty(responseCookie.Domain) && !responseCookie.Domain.StartsWith("."))
{
responseCookie.Domain = "." + responseCookie.Domain;
}
string url = responseCookie.Secure ? "https://" : "http://";
url += responseCookie.Domain.Substring(1);
var uri = new Uri(url);
cookieJar.Add(uri, responseCookie);
}
}
}
if (redirectionStatusCodes.Contains(webResponse.StatusCode))
{
// We have a redirected response, so get the new location.
var location = webResponse.Headers[CommonHeaders.Location];
// Locations should always be absolute, per the RFC (http://tools.ietf.org/html/rfc2616#section-14.30), but
// that won't always be the case.
Uri redirectUri;
if (Uri.IsWellFormedUriString(location, UriKind.Absolute))
{
redirectUri = new Uri(location);
}
else
{
redirectUri = new Uri(webRequest.Destination, new Uri(location, UriKind.Relative));
}
if (webRequest.AutoRedirect)
{
// Dispose webResponse before auto redirect, otherwise the connection will not be closed until all the auto redirect finished.
// http://www.wadewegner.com/2007/08/systemnetwebexception-when-issuing-more-than-two-concurrent-webrequests/
webResponse.Dispose();
// We are auto redirecting, so make a recursive call to perform the redirect by hand.
response = SendRequest(new GetWebRequest(redirectUri));
}
else
{
var responseUri = webResponse.ResponseUri;
webResponse.Dispose();
// We are not auto redirecting, so send the caller a redirect response.
response = new RedirectedWebResponse(responseUri, webRequest, redirectUri);
}
}
else
{
// We have a non-redirected response.
response = WebResponseFactory.CreateResponse(webResponse);
if (response.ResponseType == WebResponseType.Html)
{
// We have an HTML response, so check for an old school Meta refresh tag
var metaRefreshUrl = GetMetaRefreshUrl((( HtmlWebResponse )response).Html);
if (!string.IsNullOrWhiteSpace(metaRefreshUrl))
{
// The page has a Meta refresh tag, so build the redirect URL
var redirectUri = new Uri(response.ResponseUrl, metaRefreshUrl);
if (webRequest.AutoRedirect)
{
// Dispose webResponse before auto redirect, otherwise the connection will not be closed until all the auto redirect finished.
// http://www.wadewegner.com/2007/08/systemnetwebexception-when-issuing-more-than-two-concurrent-webrequests/
response.Dispose();
// We are auto redirecting, so make a recursive call to perform the redirect
response = SendRequest(new GetWebRequest(redirectUri, httpWebRequest.AllowAutoRedirect));
}
else
{
var responseUrl = response.ResponseUrl;
response.Dispose();
// We are not auto redirecting, so send the caller a redirect response
response = new RedirectedWebResponse(responseUrl, webRequest, redirectUri);
}
}
}
}
}
else
{
response = new ExceptionWebResponse(webRequest.Destination, new WebException(Properties.Resources.NoResponse));
webResponse.Dispose();
}
}
catch (WebException ex) {
response = new ExceptionWebResponse(webRequest.Destination, ex);
webResponse?.Dispose();
}
return(response);
}
