protected async Task <string> RequestDataAsync(string url, Dictionary <string, string> headers = null, IEnumerable <string> cookieHosts = null) { var requestRegistration = new CancellationTokenRegistration(); try { int redirects = 0; ResponseDetails responseDetails; do { HttpWebRequest request = WebRequestFactory.CreateGetRequest(url, string.Empty, headers, false); cookieHosts = cookieHosts ?? new List <string>(); foreach (string cookieHost in cookieHosts) { CookieService.GetUriCookie(request.CookieContainer, new Uri(cookieHost)); } requestRegistration = Ct.Register(() => request.Abort()); responseDetails = await WebRequestFactory.ReadRequestToEnd2Async(request); if (responseDetails.HttpStatusCode == HttpStatusCode.Found) { url = request.RequestUri.GetLeftPart(UriPartial.Authority) + responseDetails.RedirectUrl; } } while (responseDetails.HttpStatusCode == HttpStatusCode.Found && redirects++ < 5); if (responseDetails.HttpStatusCode == HttpStatusCode.Found) { throw new WebException("Too many automatic redirections were attempted.", WebExceptionStatus.ProtocolError); } return(responseDetails.Response); } finally { requestRegistration.Dispose(); } }
protected async Task <string> RequestDataAsync(string url, Dictionary <string, string> headers = null, IEnumerable <string> cookieHosts = null) { var requestRegistration = new CancellationTokenRegistration(); try { int redirects = 0; ResponseDetails responseDetails; do { HttpWebRequest request = WebRequestFactory.CreateGetRequest(url, string.Empty, headers, false); cookieHosts = cookieHosts ?? new List <string>(); foreach (string cookieHost in cookieHosts) { CookieService.GetUriCookie(request.CookieContainer, new Uri(cookieHost)); } requestRegistration = Ct.Register(() => request.Abort()); responseDetails = await WebRequestFactory.ReadRequestToEnd2Async(request); url = responseDetails.RedirectUrl ?? url; if (responseDetails.HttpStatusCode == HttpStatusCode.Found) { if (url.Contains("privacy/consent")) { var ex = new Exception("Acceptance of privacy consent needed!"); ShellService.ShowError(new TumblrPrivacyConsentException(ex), Resources.ConfirmationTumblrPrivacyConsentNeeded); throw ex; } if (!url.StartsWith("http", StringComparison.InvariantCultureIgnoreCase)) { url = request.RequestUri.GetLeftPart(UriPartial.Authority) + url; } } if (responseDetails.HttpStatusCode == HttpStatusCode.Moved) { Uri uri = new Uri(url); if (!uri.Authority.Contains(".tumblr.")) { Blog.Url = uri.GetLeftPart(UriPartial.Authority); } } } while ((responseDetails.HttpStatusCode == HttpStatusCode.Found || responseDetails.HttpStatusCode == HttpStatusCode.Moved) && redirects++ < 5); if (responseDetails.HttpStatusCode == HttpStatusCode.Found) { throw new WebException("Too many automatic redirections were attempted.", WebExceptionStatus.ProtocolError); } return(responseDetails.Response); } catch (Exception e) { Logger.Error("AbstractCrawler.RequestDataAsync: {0}", e); throw; } finally { requestRegistration.Dispose(); } }