public void ResureRedirectIsFullyQualified_makes_redicts_fully_qualified() { var res = new WebClientByteResult() { RedirectingTo = "list?p=1" }; var req = new WebRequest() { Url = "http://my.domain.com/page.php" }; // Not fully qualified requiring redirect ServerUtil.ResureRedirectIsFullyQualified(req, res); Assert.AreEqual(res.RedirectingTo, "http://my.domain.com/list?p=1"); // Fully qualified not needing modified res.RedirectingTo = "http://a.domain/page.htm"; ServerUtil.ResureRedirectIsFullyQualified(req, res); Assert.AreEqual(res.RedirectingTo, "http://a.domain/page.htm"); // Relative requiring redirect req.Url = "http://my.domain.com/dir/page.php"; res.RedirectingTo = "a/dir/page.html"; ServerUtil.ResureRedirectIsFullyQualified(req, res); Assert.AreEqual(res.RedirectingTo, "http://my.domain.com/dir/a/dir/page.html"); }
private async Task DoFollowIfRedirect(WebClientByteResult incomingResponse, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null, bool accumulateCookies = false) { if (incomingResponse.IsRedirect) { var redirRequestCookies = ""; if (accumulateCookies) { redirRequestCookies = ResolveCookies((CookieHeader != "" ? CookieHeader + " " : "") + (overrideCookies != null ? overrideCookies : "")); } else { redirRequestCookies = (overrideCookies != null ? overrideCookies : ""); } // Do redirect var redirectedResponse = await webclient.GetBytes(new WebRequest() { Url = overrideRedirectUrl ?? incomingResponse.RedirectingTo, Referer = referrer, Cookies = redirRequestCookies, Encoding = Encoding }); Mapper.Map(redirectedResponse, incomingResponse); } }
protected async Task FollowIfRedirect(WebClientStringResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null, bool accumulateCookies = false) { var byteResult = new WebClientByteResult(); // Map to byte Mapper.Map(response, byteResult); await FollowIfRedirect(byteResult, referrer, overrideRedirectUrl, overrideCookies, accumulateCookies); // Map to string Mapper.Map(byteResult, response); }
protected async Task FollowIfRedirect(WebClientByteResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null) { // Follow up to 5 redirects for (int i = 0; i < 5; i++) { if (!response.IsRedirect) { break; } await DoFollowIfRedirect(response, referrer, overrideRedirectUrl, overrideCookies); } }
public async Task <CQ> getDOM(string url) { Console.WriteLine(url); Dictionary <String, String> headers = new Dictionary <string, string>(); headers.Add("Content-Type", "text/html;charset=ISO-8859-1"); WebClientByteResult results = await RequestStringWithCookiesAndRetry(url, null, null, headers); byte[] htmlResponseBytes = results.Content; CQ dom = Encoding.GetEncoding("ISO-8859-1").GetString(results.Content); return(dom); }
private async Task DoFollowIfRedirect(WebClientByteResult incomingResponse, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null) { if (incomingResponse.IsRedirect) { // Do redirect var redirectedResponse = await webclient.GetBytes(new WebRequest() { Url = overrideRedirectUrl ?? incomingResponse.RedirectingTo, Referer = referrer, Cookies = overrideCookies ?? CookieHeader }); Mapper.Map(redirectedResponse, incomingResponse); } }
protected bool CheckIfLoginIsNeeded(WebClientByteResult Result, IHtmlDocument document) { if (Result.IsRedirect) { return(true); } if (Definition.Login.Test.Selector != null) { var selection = document.QuerySelectorAll(Definition.Login.Test.Selector); if (selection.Length == 0) { return(true); } } return(false); }
protected async Task FollowIfRedirect(WebClientByteResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null, bool accumulateCookies = false) { // Follow up to 5 redirects for (int i = 0; i < 5; i++) { if (!response.IsRedirect) break; await DoFollowIfRedirect(response, referrer, overrideRedirectUrl, overrideCookies, accumulateCookies); if (accumulateCookies) { CookieHeader = ResolveCookies((CookieHeader != null && CookieHeader != "" ? CookieHeader + " " : "") + (overrideCookies != null && overrideCookies != "" ? overrideCookies + " " : "") + response.Cookies); overrideCookies = response.Cookies = CookieHeader; } if (overrideCookies != null && response.Cookies == null) { response.Cookies = overrideCookies; } } }
protected async new Task <WebClientByteResult> RequestStringWithCookies(string url, string cookieOverride = null, string referer = null, Dictionary <string, string> headers = null) { var request = new Utils.Clients.WebRequest() { Url = url, Type = RequestType.GET, Cookies = CookieHeader, Referer = referer, Headers = headers }; if (cookieOverride != null) { request.Cookies = cookieOverride; } WebClientByteResult result = await webclient.GetBytes(request); UpdateCookieHeader(result.Cookies, cookieOverride); return(result); }