protected async Task <string> RequestApiDataAsync(string url, string bearerToken, Dictionary <string, string> headers = null, IEnumerable <string> cookieHosts = null) { var requestRegistration = new CancellationTokenRegistration(); try { HttpWebRequest request = WebRequestFactory.CreateGetRequest(url, string.Empty, headers); cookieHosts = cookieHosts ?? new List <string>(); foreach (string cookieHost in cookieHosts) { CookieService.GetUriCookie(request.CookieContainer, new Uri(cookieHost)); } request.PreAuthenticate = true; request.Headers.Add("Authorization", "Bearer " + bearerToken); request.Accept = "application/json"; requestRegistration = Ct.Register(() => request.Abort()); return(await WebRequestFactory.ReadRequestToEndAsync(request)); } finally { requestRegistration.Dispose(); } }
protected string RetrieveOriginalImageUrl(string url, int width, int height) { if (width > height) { (width, height) = (height, width); } if (ShellService.Settings.ImageSize != "best" || !url.Contains("/s1280x1920/") || (width <= 1280 && height <= 1920)) { return(url); } url = url.Replace("/s1280x1920/", (width <= 2048 && height <= 3072) ? "/s2048x3072/" : "/s99999x99999/"); string pageContent = ""; try { HttpWebRequest request = WebRequestFactory.CreateGetRequest(url, "", new Dictionary <string, string>() { { "Accept-Language", "en-US" }, { "Accept-Encoding", "gzip, deflate" } }, false); request.Accept = "text/html, application/xhtml+xml, */*"; request.UserAgent = ShellService.Settings.UserAgent; request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip; pageContent = WebRequestFactory.ReadRequestToEndAsync(request).GetAwaiter().GetResult(); } catch (Exception e) { Logger.Error("TumblrBlogCrawler:RetrieveRawImageUrl:Exception {0}", e); return(url); } pageContent = extractJsonFromPage.Match(pageContent).Groups[1].Value; ImageResponse imgRsp = ConvertJsonToClass <ImageResponse>(pageContent); Image img = imgRsp.Images.FirstOrDefault(x => x.HasOriginalDimensions = true); return(string.IsNullOrEmpty(img.MediaKey) ? url : img.Url); }
protected async Task <string> RequestDataAsync(string url, Dictionary <string, string> headers = null, IEnumerable <string> cookieHosts = null) { var requestRegistration = new CancellationTokenRegistration(); try { int redirects = 0; ResponseDetails responseDetails; do { HttpWebRequest request = WebRequestFactory.CreateGetRequest(url, string.Empty, headers, false); cookieHosts = cookieHosts ?? new List <string>(); foreach (string cookieHost in cookieHosts) { CookieService.GetUriCookie(request.CookieContainer, new Uri(cookieHost)); } requestRegistration = Ct.Register(() => request.Abort()); responseDetails = await WebRequestFactory.ReadRequestToEnd2Async(request); if (responseDetails.HttpStatusCode == HttpStatusCode.Found) { url = request.RequestUri.GetLeftPart(UriPartial.Authority) + responseDetails.RedirectUrl; } } while (responseDetails.HttpStatusCode == HttpStatusCode.Found && redirects++ < 5); if (responseDetails.HttpStatusCode == HttpStatusCode.Found) { throw new WebException("Too many automatic redirections were attempted.", WebExceptionStatus.ProtocolError); } return(responseDetails.Response); } finally { requestRegistration.Dispose(); } }
protected async Task <string> RequestDataAsync(string url, Dictionary <string, string> headers = null, IEnumerable <string> cookieHosts = null) { var requestRegistration = new CancellationTokenRegistration(); try { int redirects = 0; ResponseDetails responseDetails; do { HttpWebRequest request = WebRequestFactory.CreateGetRequest(url, string.Empty, headers, false); cookieHosts = cookieHosts ?? new List <string>(); foreach (string cookieHost in cookieHosts) { CookieService.GetUriCookie(request.CookieContainer, new Uri(cookieHost)); } requestRegistration = Ct.Register(() => request.Abort()); responseDetails = await WebRequestFactory.ReadRequestToEnd2Async(request); url = responseDetails.RedirectUrl ?? url; if (responseDetails.HttpStatusCode == HttpStatusCode.Found) { if (url.Contains("privacy/consent")) { var ex = new Exception("Acceptance of privacy consent needed!"); ShellService.ShowError(new TumblrPrivacyConsentException(ex), Resources.ConfirmationTumblrPrivacyConsentNeeded); throw ex; } if (!url.StartsWith("http", StringComparison.InvariantCultureIgnoreCase)) { url = request.RequestUri.GetLeftPart(UriPartial.Authority) + url; } } if (responseDetails.HttpStatusCode == HttpStatusCode.Moved) { Uri uri = new Uri(url); if (!uri.Authority.Contains(".tumblr.")) { Blog.Url = uri.GetLeftPart(UriPartial.Authority); } } } while ((responseDetails.HttpStatusCode == HttpStatusCode.Found || responseDetails.HttpStatusCode == HttpStatusCode.Moved) && redirects++ < 5); if (responseDetails.HttpStatusCode == HttpStatusCode.Found) { throw new WebException("Too many automatic redirections were attempted.", WebExceptionStatus.ProtocolError); } return(responseDetails.Response); } catch (Exception e) { Logger.Error("AbstractCrawler.RequestDataAsync: {0}", e); throw; } finally { requestRegistration.Dispose(); } }
protected string RetrieveOriginalImageUrl(string url, int width, int height) { if (width > height) { (width, height) = (height, width); } if (ShellService.Settings.ImageSize != "best" || !url.Contains("/s1280x1920/") || (width <= 1280 && height <= 1920)) { return(url); } url = url.Replace("/s1280x1920/", (width <= 2048 && height <= 3072) ? "/s2048x3072/" : "/s99999x99999/"); string pageContent = ""; int errCnt = 0; Exception lastError = null; do { try { HttpWebRequest request = WebRequestFactory.CreateGetRequest(url, "", new Dictionary <string, string>() { { "Accept-Language", "en-US" }, { "Accept-Encoding", "gzip, deflate" } }, false); request.Accept = "text/html, application/xhtml+xml, */*"; request.UserAgent = ShellService.Settings.UserAgent; request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip; pageContent = WebRequestFactory.ReadRequestToEndAsync(request).GetAwaiter().GetResult(); errCnt = 9; } catch (WebException we) { if (we.Response != null && ((HttpWebResponse)we.Response).StatusCode == HttpStatusCode.NotFound) { return(url); } } catch (Exception e) { errCnt++; Logger.Error("AbstractTumblrCrawler:RetrieveOriginalImageUrl: {0}", e); lastError = e; if (errCnt < 3) { Thread.Sleep(errCnt * 10000); } } } while (errCnt < 3); if (errCnt == 3) { ShellService.ShowError(lastError, Resources.PostNotParsable, Blog.Name); throw new NullReferenceException("RetrieveOriginalImageUrl download", lastError); } try { var extracted = extractJsonFromPage.Match(pageContent).Groups[1].Value; extracted = new Regex("/.*/").Replace(extracted, "\"\""); ImageResponse imgRsp = DeserializeImageResponse(extracted); int maxWidth = imgRsp.Images.Max(x => x.Width); Image img = imgRsp.Images.FirstOrDefault(x => x.Width == maxWidth); return(string.IsNullOrEmpty(img?.MediaKey) ? url : img.Url); } catch (Exception ex) { Logger.Error("AbstractTumblrCrawler:RetrieveOriginalImageUrl: {0}", ex); ShellService.ShowError(ex, Resources.PostNotParsable, Blog.Name); throw new NullReferenceException("RetrieveOriginalImageUrl parsing", ex); } }