public override HttpResponse PostResponse(HttpResponse response)
{
if (!IsCloudflareProtected(response))
{
_logger.Debug("CF Protection not detected, returning original response");
return(response);
}
var flaresolverrResponse = _httpClient.Execute(GenerateFlareSolverrRequest(response.Request));
FlareSolverrResponse result = null;
if (flaresolverrResponse.StatusCode != HttpStatusCode.OK && flaresolverrResponse.StatusCode != HttpStatusCode.InternalServerError)
{
throw new FlareSolverrException("HTTP StatusCode not 200 or 500. Status is :" + response.StatusCode);
}
result = JsonConvert.DeserializeObject <FlareSolverrResponse>(flaresolverrResponse.Content);
var newRequest = response.Request;
//Cache the user-agent so we can inject it in next request to avoid re-solve
_cache.Set(response.Request.Url.Host, result.Solution.UserAgent);
newRequest.Headers.UserAgent = result.Solution.UserAgent;
InjectCookies(newRequest, result);
//Request again with User-Agent and Cookies from Flaresolvrr
var finalResponse = _httpClient.Execute(newRequest);
return(finalResponse);
}
private void InjectCookies(HttpRequestMessage request, FlareSolverrResponse flareSolverrResponse)
{
var rCookies = flareSolverrResponse.Solution.Cookies;
if (!rCookies.Any())
{
return;
}
var rCookiesList = rCookies.Select(x => x.Name).ToList();
if (HttpClientHandler.UseCookies)
{
var oldCookies = HttpClientHandler.CookieContainer.GetCookies(request.RequestUri);
foreach (Cookie oldCookie in oldCookies)
{
if (rCookiesList.Contains(oldCookie.Name))
{
oldCookie.Expired = true;
}
}
foreach (var rCookie in rCookies)
{
HttpClientHandler.CookieContainer.Add(request.RequestUri, rCookie.ToCookieObj());
}
}
else
{
foreach (var rCookie in rCookies)
{
request.Headers.Add(HttpHeaders.Cookie, rCookie.ToHeaderValue());
}
}
}
private void InjectCookies(HttpRequestMessage request, FlareSolverrResponse flareSolverrResponse)
{
var flareCookies = flareSolverrResponse.Solution.Cookies;
if (!flareCookies.Any())
{
return;
}
if (HttpClientHandler.UseCookies)
{
var currentCookies = HttpClientHandler.CookieContainer.GetCookies(request.RequestUri);
// remove previous FlareSolverr cookies
var expiredCount = 0;
foreach (var flareCookie in flareCookies)
{
var cookie = currentCookies[flareCookie.Name];
if (cookie == null)
{
continue;
}
cookie.Expired = true;
expiredCount += 1;
}
// there is a max number of cookies, we have to make space (we assume the first
var cookieExcess = currentCookies.Count + flareCookies.Length
- expiredCount - HttpClientHandler.CookieContainer.PerDomainCapacity;
foreach (Cookie cookie in currentCookies)
{
if (cookieExcess == 0)
{
break;
}
if (cookie.Expired)
{
continue;
}
cookie.Expired = true;
cookieExcess -= 1;
}
// add FlareSolverr cookies
foreach (var rCookie in flareCookies)
{
HttpClientHandler.CookieContainer.Add(request.RequestUri, rCookie.ToCookieObj());
}
}
else
{
foreach (var rCookie in flareCookies)
{
request.Headers.Add(HttpHeaders.Cookie, rCookie.ToHeaderValue());
}
}
}
private void InjectSetCookieHeader(HttpResponseMessage response, FlareSolverrResponse flareSolverrResponse)
{
var rCookies = flareSolverrResponse.Solution.Cookies;
if (!rCookies.Any())
{
return;
}
// inject set-cookie headers in the response
foreach (var rCookie in rCookies)
{
response.Headers.Add(HttpHeaders.SetCookie, rCookie.ToHeaderValue());
}
}
private void InjectCookies(HttpRequest request, FlareSolverrResponse flareSolverrResponse)
{
var rCookies = flareSolverrResponse.Solution.Cookies;
if (!rCookies.Any())
{
return;
}
var rCookiesList = rCookies.Select(x => x.Name).ToList();
foreach (var rCookie in rCookies)
{
request.Cookies.Remove(rCookie.Name);
request.Cookies.Add(rCookie.Name, rCookie.Value);
}
}
public async Task <FlareSolverrResponse> Solve(HttpRequestMessage request)
{
FlareSolverrResponse result = null;
await Locker.LockAsync(async() =>
{
HttpResponseMessage response;
try
{
_httpClient = new HttpClient();
response = await _httpClient.PostAsync(_flareSolverrUri, GenerateFlareSolverrRequest(request));
}
catch (HttpRequestException e)
{
throw new FlareSolverrException("Error connecting to FlareSolverr server: " + e);
}
catch (Exception e)
{
throw new FlareSolverrException(e.ToString());
}
finally
{
_httpClient.Dispose();
}
var resContent = await response.Content.ReadAsStringAsync();
try
{
result = JsonConvert.DeserializeObject <FlareSolverrResponse>(resContent);
}
catch (Exception)
{
throw new FlareSolverrException("Error parsing response, check FlareSolverr version. Response: " + resContent);
}
if (response.StatusCode != HttpStatusCode.OK)
{
throw new FlareSolverrException(result.Message);
}
});
return(result);
}
public async Task <FlareSolverrResponse> Solve(HttpRequestMessage request)
{
FlareSolverrResponse result = null;
await Locker.LockAsync(async() =>
{
HttpResponseMessage response;
try
{
_httpClient = new HttpClient();
response = await _httpClient.PostAsync(_flareSolverrUri, GenerateFlareSolverrRequest(request));
}
catch (HttpRequestException e)
{
throw new FlareSolverrException("Error connecting to FlareSolverr server: " + e);
}
catch (Exception e)
{
throw new FlareSolverrException("Exception: " + e);
}
finally
{
_httpClient.Dispose();
}
// Don't try parsing if FlareSolverr hasn't returned 200 or 500
if (response.StatusCode != HttpStatusCode.OK && response.StatusCode != HttpStatusCode.InternalServerError)
{
throw new FlareSolverrException("HTTP StatusCode not 200 or 500. Status is :" + response.StatusCode);
}
var resContent = await response.Content.ReadAsStringAsync();
try
{
result = JsonConvert.DeserializeObject <FlareSolverrResponse>(resContent);
}
catch (Exception)
{
throw new FlareSolverrException("Error parsing response, check FlareSolverr. Response: " + resContent);
}
try
{
Enum.TryParse(result.Status, true, out FlareSolverrStatusCode returnStatusCode);
if (returnStatusCode.Equals(FlareSolverrStatusCode.ok))
{
return(result);
}
if (returnStatusCode.Equals(FlareSolverrStatusCode.warning))
{
throw new FlareSolverrException(
"FlareSolverr was able to process the request, but a captcha was detected. Message: "
+ result.Message);
}
if (returnStatusCode.Equals(FlareSolverrStatusCode.error))
{
throw new FlareSolverrException(
"FlareSolverr was unable to process the request, please check FlareSolverr logs. Message: "
+ result.Message);
}
throw new FlareSolverrException("Unable to map FlareSolverr returned status code, received code: "
+ result.Status + ". Message: " + result.Message);
}
catch (ArgumentException)
{
throw new FlareSolverrException("Error parsing status code, check FlareSolverr log. Status: "
+ result.Status + ". Message: " + result.Message);
}
});
return(result);
}
