public override HttpResponse PostResponse(HttpResponse response) { if (!IsCloudflareProtected(response)) { _logger.Debug("CF Protection not detected, returning original response"); return(response); } var flaresolverrResponse = _httpClient.Execute(GenerateFlareSolverrRequest(response.Request)); FlareSolverrResponse result = null; if (flaresolverrResponse.StatusCode != HttpStatusCode.OK && flaresolverrResponse.StatusCode != HttpStatusCode.InternalServerError) { throw new FlareSolverrException("HTTP StatusCode not 200 or 500. Status is :" + response.StatusCode); } result = JsonConvert.DeserializeObject <FlareSolverrResponse>(flaresolverrResponse.Content); var newRequest = response.Request; //Cache the user-agent so we can inject it in next request to avoid re-solve _cache.Set(response.Request.Url.Host, result.Solution.UserAgent); newRequest.Headers.UserAgent = result.Solution.UserAgent; InjectCookies(newRequest, result); //Request again with User-Agent and Cookies from Flaresolvrr var finalResponse = _httpClient.Execute(newRequest); return(finalResponse); }
private void InjectCookies(HttpRequestMessage request, FlareSolverrResponse flareSolverrResponse) { var rCookies = flareSolverrResponse.Solution.Cookies; if (!rCookies.Any()) { return; } var rCookiesList = rCookies.Select(x => x.Name).ToList(); if (HttpClientHandler.UseCookies) { var oldCookies = HttpClientHandler.CookieContainer.GetCookies(request.RequestUri); foreach (Cookie oldCookie in oldCookies) { if (rCookiesList.Contains(oldCookie.Name)) { oldCookie.Expired = true; } } foreach (var rCookie in rCookies) { HttpClientHandler.CookieContainer.Add(request.RequestUri, rCookie.ToCookieObj()); } } else { foreach (var rCookie in rCookies) { request.Headers.Add(HttpHeaders.Cookie, rCookie.ToHeaderValue()); } } }
private void InjectCookies(HttpRequestMessage request, FlareSolverrResponse flareSolverrResponse) { var flareCookies = flareSolverrResponse.Solution.Cookies; if (!flareCookies.Any()) { return; } if (HttpClientHandler.UseCookies) { var currentCookies = HttpClientHandler.CookieContainer.GetCookies(request.RequestUri); // remove previous FlareSolverr cookies var expiredCount = 0; foreach (var flareCookie in flareCookies) { var cookie = currentCookies[flareCookie.Name]; if (cookie == null) { continue; } cookie.Expired = true; expiredCount += 1; } // there is a max number of cookies, we have to make space (we assume the first var cookieExcess = currentCookies.Count + flareCookies.Length - expiredCount - HttpClientHandler.CookieContainer.PerDomainCapacity; foreach (Cookie cookie in currentCookies) { if (cookieExcess == 0) { break; } if (cookie.Expired) { continue; } cookie.Expired = true; cookieExcess -= 1; } // add FlareSolverr cookies foreach (var rCookie in flareCookies) { HttpClientHandler.CookieContainer.Add(request.RequestUri, rCookie.ToCookieObj()); } } else { foreach (var rCookie in flareCookies) { request.Headers.Add(HttpHeaders.Cookie, rCookie.ToHeaderValue()); } } }
private void InjectSetCookieHeader(HttpResponseMessage response, FlareSolverrResponse flareSolverrResponse) { var rCookies = flareSolverrResponse.Solution.Cookies; if (!rCookies.Any()) { return; } // inject set-cookie headers in the response foreach (var rCookie in rCookies) { response.Headers.Add(HttpHeaders.SetCookie, rCookie.ToHeaderValue()); } }
private void InjectCookies(HttpRequest request, FlareSolverrResponse flareSolverrResponse) { var rCookies = flareSolverrResponse.Solution.Cookies; if (!rCookies.Any()) { return; } var rCookiesList = rCookies.Select(x => x.Name).ToList(); foreach (var rCookie in rCookies) { request.Cookies.Remove(rCookie.Name); request.Cookies.Add(rCookie.Name, rCookie.Value); } }
public async Task <FlareSolverrResponse> Solve(HttpRequestMessage request) { FlareSolverrResponse result = null; await Locker.LockAsync(async() => { HttpResponseMessage response; try { _httpClient = new HttpClient(); response = await _httpClient.PostAsync(_flareSolverrUri, GenerateFlareSolverrRequest(request)); } catch (HttpRequestException e) { throw new FlareSolverrException("Error connecting to FlareSolverr server: " + e); } catch (Exception e) { throw new FlareSolverrException(e.ToString()); } finally { _httpClient.Dispose(); } var resContent = await response.Content.ReadAsStringAsync(); try { result = JsonConvert.DeserializeObject <FlareSolverrResponse>(resContent); } catch (Exception) { throw new FlareSolverrException("Error parsing response, check FlareSolverr version. Response: " + resContent); } if (response.StatusCode != HttpStatusCode.OK) { throw new FlareSolverrException(result.Message); } }); return(result); }
public async Task <FlareSolverrResponse> Solve(HttpRequestMessage request) { FlareSolverrResponse result = null; await Locker.LockAsync(async() => { HttpResponseMessage response; try { _httpClient = new HttpClient(); response = await _httpClient.PostAsync(_flareSolverrUri, GenerateFlareSolverrRequest(request)); } catch (HttpRequestException e) { throw new FlareSolverrException("Error connecting to FlareSolverr server: " + e); } catch (Exception e) { throw new FlareSolverrException("Exception: " + e); } finally { _httpClient.Dispose(); } // Don't try parsing if FlareSolverr hasn't returned 200 or 500 if (response.StatusCode != HttpStatusCode.OK && response.StatusCode != HttpStatusCode.InternalServerError) { throw new FlareSolverrException("HTTP StatusCode not 200 or 500. Status is :" + response.StatusCode); } var resContent = await response.Content.ReadAsStringAsync(); try { result = JsonConvert.DeserializeObject <FlareSolverrResponse>(resContent); } catch (Exception) { throw new FlareSolverrException("Error parsing response, check FlareSolverr. Response: " + resContent); } try { Enum.TryParse(result.Status, true, out FlareSolverrStatusCode returnStatusCode); if (returnStatusCode.Equals(FlareSolverrStatusCode.ok)) { return(result); } if (returnStatusCode.Equals(FlareSolverrStatusCode.warning)) { throw new FlareSolverrException( "FlareSolverr was able to process the request, but a captcha was detected. Message: " + result.Message); } if (returnStatusCode.Equals(FlareSolverrStatusCode.error)) { throw new FlareSolverrException( "FlareSolverr was unable to process the request, please check FlareSolverr logs. Message: " + result.Message); } throw new FlareSolverrException("Unable to map FlareSolverr returned status code, received code: " + result.Status + ". Message: " + result.Message); } catch (ArgumentException) { throw new FlareSolverrException("Error parsing status code, check FlareSolverr log. Status: " + result.Status + ". Message: " + result.Message); } }); return(result); }