public async Task <string> GetCookieFromRootDirectives() { var realisedcookie = ""; var pctx = ConstructRequestContext(realisedcookie); var n = await _httpmanager.Get("https://www.whoscored.com/", pctx); foreach (var sc in n.Headers.Where(x => x.Key == "Set-Cookie")) { foreach (var scv in sc.Value) { var v = scv.Split(';')[0]; realisedcookie = $"{realisedcookie}; {v}"; } } return(realisedcookie.Substring(2)); }
public async Task <IHarvestRequestResult> MakeRequest(string url, IHttpRequestContext ctx, bool isretry = false) { var requestTimer = new Stopwatch(); requestTimer.Start(); HtmlDocument doc = null; try { _logger.LogDebug(string.Format("Making web request: {0}", url)); InitialiseCertificates(); using (var resp = await _httpManager.Get(url, ctx)) { doc = new HtmlDocument(); doc.LoadHtml((await resp.Content.ReadAsStringAsync()).Trim()); if (doc.DocumentNode.InnerText.Contains("Request unsuccessful")) { _logger.LogError(string.Format("Incapsula request recieved!")); throw new Exception("Request did not fail but reurned an Incapsula request!"); } _logger.LogDebug(string.Format("Web request response successfully recieved & serialised to cache: {0}bytes", doc.DocumentNode.OuterLength)); } LastRequestFailed = false; } catch (Exception ex) { LastRequestFailed = true; _logger.LogDebug(string.Format("Web request failed as follows: {0}", ex.Message)); _logger.LogDebug(string.Format("Web request was cancelled & failed to complete: {0}", url)); throw ex; } await ApplyRequestThrottle(requestTimer); return(new HarvestRequestResult() { InnerHtml = doc.DocumentNode.InnerHtml, InnerText = doc.DocumentNode.InnerText }); }