public async Task <string> GetCookieFromRootDirectives()
{
var realisedcookie = "";
var pctx = ConstructRequestContext(realisedcookie);
var n = await _httpmanager.Get("https://www.whoscored.com/", pctx);
foreach (var sc in n.Headers.Where(x => x.Key == "Set-Cookie"))
{
foreach (var scv in sc.Value)
{
var v = scv.Split(';')[0];
realisedcookie = $"{realisedcookie}; {v}";
}
}
return(realisedcookie.Substring(2));
}
public async Task <IHarvestRequestResult> MakeRequest(string url, IHttpRequestContext ctx, bool isretry = false)
{
var requestTimer = new Stopwatch();
requestTimer.Start();
HtmlDocument doc = null;
try
{
_logger.LogDebug(string.Format("Making web request: {0}", url));
InitialiseCertificates();
using (var resp = await _httpManager.Get(url, ctx))
{
doc = new HtmlDocument();
doc.LoadHtml((await resp.Content.ReadAsStringAsync()).Trim());
if (doc.DocumentNode.InnerText.Contains("Request unsuccessful"))
{
_logger.LogError(string.Format("Incapsula request recieved!"));
throw new Exception("Request did not fail but reurned an Incapsula request!");
}
_logger.LogDebug(string.Format("Web request response successfully recieved & serialised to cache: {0}bytes", doc.DocumentNode.OuterLength));
}
LastRequestFailed = false;
}
catch (Exception ex)
{
LastRequestFailed = true;
_logger.LogDebug(string.Format("Web request failed as follows: {0}", ex.Message));
_logger.LogDebug(string.Format("Web request was cancelled & failed to complete: {0}", url));
throw ex;
}
await ApplyRequestThrottle(requestTimer);
return(new HarvestRequestResult()
{
InnerHtml = doc.DocumentNode.InnerHtml,
InnerText = doc.DocumentNode.InnerText
});
}