public void ResureRedirectIsFullyQualified_makes_redicts_fully_qualified()
{
var res = new WebClientByteResult()
{
RedirectingTo = "list?p=1"
};
var req = new WebRequest()
{
Url = "http://my.domain.com/page.php"
};
// Not fully qualified requiring redirect
ServerUtil.ResureRedirectIsFullyQualified(req, res);
Assert.AreEqual(res.RedirectingTo, "http://my.domain.com/list?p=1");
// Fully qualified not needing modified
res.RedirectingTo = "http://a.domain/page.htm";
ServerUtil.ResureRedirectIsFullyQualified(req, res);
Assert.AreEqual(res.RedirectingTo, "http://a.domain/page.htm");
// Relative requiring redirect
req.Url = "http://my.domain.com/dir/page.php";
res.RedirectingTo = "a/dir/page.html";
ServerUtil.ResureRedirectIsFullyQualified(req, res);
Assert.AreEqual(res.RedirectingTo, "http://my.domain.com/dir/a/dir/page.html");
}
private async Task DoFollowIfRedirect(WebClientByteResult incomingResponse, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null, bool accumulateCookies = false)
{
if (incomingResponse.IsRedirect)
{
var redirRequestCookies = "";
if (accumulateCookies)
{
redirRequestCookies = ResolveCookies((CookieHeader != "" ? CookieHeader + " " : "") + (overrideCookies != null ? overrideCookies : ""));
}
else
{
redirRequestCookies = (overrideCookies != null ? overrideCookies : "");
}
// Do redirect
var redirectedResponse = await webclient.GetBytes(new WebRequest()
{
Url = overrideRedirectUrl ?? incomingResponse.RedirectingTo,
Referer = referrer,
Cookies = redirRequestCookies,
Encoding = Encoding
});
Mapper.Map(redirectedResponse, incomingResponse);
}
}
protected async Task FollowIfRedirect(WebClientStringResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null, bool accumulateCookies = false)
{
var byteResult = new WebClientByteResult();
// Map to byte
Mapper.Map(response, byteResult);
await FollowIfRedirect(byteResult, referrer, overrideRedirectUrl, overrideCookies, accumulateCookies);
// Map to string
Mapper.Map(byteResult, response);
}
protected async Task FollowIfRedirect(WebClientByteResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null)
{
// Follow up to 5 redirects
for (int i = 0; i < 5; i++)
{
if (!response.IsRedirect)
{
break;
}
await DoFollowIfRedirect(response, referrer, overrideRedirectUrl, overrideCookies);
}
}
public async Task <CQ> getDOM(string url)
{
Console.WriteLine(url);
Dictionary <String, String> headers = new Dictionary <string, string>();
headers.Add("Content-Type", "text/html;charset=ISO-8859-1");
WebClientByteResult results = await RequestStringWithCookiesAndRetry(url, null, null, headers);
byte[] htmlResponseBytes = results.Content;
CQ dom = Encoding.GetEncoding("ISO-8859-1").GetString(results.Content);
return(dom);
}
private async Task DoFollowIfRedirect(WebClientByteResult incomingResponse, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null)
{
if (incomingResponse.IsRedirect)
{
// Do redirect
var redirectedResponse = await webclient.GetBytes(new WebRequest()
{
Url = overrideRedirectUrl ?? incomingResponse.RedirectingTo,
Referer = referrer,
Cookies = overrideCookies ?? CookieHeader
});
Mapper.Map(redirectedResponse, incomingResponse);
}
}
protected bool CheckIfLoginIsNeeded(WebClientByteResult Result, IHtmlDocument document)
{
if (Result.IsRedirect)
{
return(true);
}
if (Definition.Login.Test.Selector != null)
{
var selection = document.QuerySelectorAll(Definition.Login.Test.Selector);
if (selection.Length == 0)
{
return(true);
}
}
return(false);
}
protected async Task FollowIfRedirect(WebClientByteResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null, bool accumulateCookies = false)
{
// Follow up to 5 redirects
for (int i = 0; i < 5; i++)
{
if (!response.IsRedirect)
break;
await DoFollowIfRedirect(response, referrer, overrideRedirectUrl, overrideCookies, accumulateCookies);
if (accumulateCookies)
{
CookieHeader = ResolveCookies((CookieHeader != null && CookieHeader != "" ? CookieHeader + " " : "") + (overrideCookies != null && overrideCookies != "" ? overrideCookies + " " : "") + response.Cookies);
overrideCookies = response.Cookies = CookieHeader;
}
if (overrideCookies != null && response.Cookies == null)
{
response.Cookies = overrideCookies;
}
}
}
protected async new Task <WebClientByteResult> RequestStringWithCookies(string url, string cookieOverride = null, string referer = null, Dictionary <string, string> headers = null)
{
var request = new Utils.Clients.WebRequest()
{
Url = url,
Type = RequestType.GET,
Cookies = CookieHeader,
Referer = referer,
Headers = headers
};
if (cookieOverride != null)
{
request.Cookies = cookieOverride;
}
WebClientByteResult result = await webclient.GetBytes(request);
UpdateCookieHeader(result.Cookies, cookieOverride);
return(result);
}
