public async Task SolveOkProxy()
{
var uri = new Uri("https://www.google.com/");
var flareSolverr = new FlareSolverr(Settings.FlareSolverrApiUrl)
{
ProxyUrl = Settings.ProxyUrl
};
var request = new HttpRequestMessage(HttpMethod.Get, uri);
var flareSolverrResponse = await flareSolverr.Solve(request);
Assert.AreEqual("ok", flareSolverrResponse.Status);
Assert.AreEqual("", flareSolverrResponse.Message);
Assert.IsTrue(flareSolverrResponse.StartTimestamp > 0);
Assert.IsTrue(flareSolverrResponse.EndTimestamp > flareSolverrResponse.StartTimestamp);
Assert.IsTrue(flareSolverrResponse.Version.Contains("2."));
Assert.AreEqual("https://www.google.com/", flareSolverrResponse.Solution.Url);
Assert.IsTrue(flareSolverrResponse.Solution.Response.Contains("<title>Google</title>"));
Assert.IsTrue(flareSolverrResponse.Solution.Cookies.Any());
Assert.IsTrue(flareSolverrResponse.Solution.UserAgent.Contains("Firefox/"));
var firstCookie = flareSolverrResponse.Solution.Cookies.First();
Assert.IsTrue(!string.IsNullOrWhiteSpace(firstCookie.Name));
Assert.IsTrue(!string.IsNullOrWhiteSpace(firstCookie.Value));
}
public async Task SolveTestSessions()
{
// create new session
var flareSolverr = new FlareSolverr(Settings.FlareSolverrApiUrl);
var flareSolverrResponse = await flareSolverr.CreateSession();
Assert.AreEqual("ok", flareSolverrResponse.Status);
Assert.AreEqual("Session created successfully.", flareSolverrResponse.Message);
Assert.IsTrue(flareSolverrResponse.StartTimestamp > 0);
Assert.IsTrue(flareSolverrResponse.EndTimestamp > flareSolverrResponse.StartTimestamp);
Assert.IsTrue(flareSolverrResponse.Version.Contains("2."));
Assert.IsTrue(flareSolverrResponse.Session.Length > 0);
// request with session
var sessionId = flareSolverrResponse.Session;
var uri = new Uri("https://www.google.com/");
var request = new HttpRequestMessage(HttpMethod.Get, uri);
flareSolverrResponse = await flareSolverr.Solve(request, sessionId);
Assert.AreEqual("ok", flareSolverrResponse.Status);
Assert.AreEqual("200", flareSolverrResponse.Solution.Status);
// list sessions
flareSolverrResponse = await flareSolverr.ListSessions();
Assert.AreEqual("ok", flareSolverrResponse.Status);
Assert.IsTrue(flareSolverrResponse.Sessions.Contains(sessionId));
// destroy session
flareSolverrResponse = await flareSolverr.DestroySession(sessionId);
Assert.AreEqual("ok", flareSolverrResponse.Status);
}
/// <summary>
/// Sends an HTTP request to the inner handler to send to the server as an asynchronous operation.
/// </summary>
/// <param name="request">The HTTP request message to send to the server.</param>
/// <param name="cancellationToken">A cancellation token to cancel operation.</param>
/// <returns>The task object representing the asynchronous operation.</returns>
protected override async Task <HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
// Init FlareSolverr
if (_flareSolverr == null && !string.IsNullOrWhiteSpace(_flareSolverrApiUrl))
{
_flareSolverr = new FlareSolverr(_flareSolverrApiUrl)
{
MaxTimeout = MaxTimeout,
ProxyUrl = ProxyUrl
};
}
// Set the User-Agent if required
SetUserAgentHeader(request);
// Perform the original user request
var response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false);
// Detect if there is a challenge in the response
if (ChallengeDetector.IsClearanceRequired(response))
{
if (_flareSolverr == null)
{
throw new FlareSolverrException("Challenge detected but FlareSolverr is not configured");
}
// Resolve the challenge using FlareSolverr API
var flareSolverrResponse = await _flareSolverr.Solve(request);
// Save the FlareSolverr User-Agent for the following requests
var flareSolverUserAgent = flareSolverrResponse.Solution.UserAgent;
if (flareSolverUserAgent != null && !flareSolverUserAgent.Equals(request.Headers.UserAgent.ToString()))
{
_userAgent = flareSolverUserAgent;
// Set the User-Agent if required
SetUserAgentHeader(request);
}
// Change the cookies in the original request with the cookies provided by FlareSolverr
InjectCookies(request, flareSolverrResponse);
response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false);
// Detect if there is a challenge in the response
if (ChallengeDetector.IsClearanceRequired(response))
{
throw new FlareSolverrException("The cookies provided by FlareSolverr are not valid");
}
// Add the "Set-Cookie" header in the response with the cookies provided by FlareSolverr
InjectSetCookieHeader(response, flareSolverrResponse);
}
return(response);
}
public async Task SolveOkUserAgent()
{
const string userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36";
var uri = new Uri("https://www.google.com/");
var flareSolverr = new FlareSolverr(Settings.FlareSolverrApiUrl);
var request = new HttpRequestMessage(HttpMethod.Get, uri);
request.Headers.Add(HttpHeaders.UserAgent, userAgent);
var flareSolverrResponse = await flareSolverr.Solve(request);
Assert.AreEqual("ok", flareSolverrResponse.Status);
Assert.AreEqual(userAgent, flareSolverrResponse.Solution.UserAgent);
}
public async Task SolveOkUserAgent()
{
const string userAgent = "Mozilla/5.0 (X11; Linux i686; rv:77.0) Gecko/20100101 Firefox/77.0";
var uri = new Uri("https://www.google.com/");
var flareSolverr = new FlareSolverr(Settings.FlareSolverrApiUrl);
var request = new HttpRequestMessage(HttpMethod.Get, uri);
request.Headers.Add(HttpHeaders.UserAgent, userAgent);
var flareSolverrResponse = await flareSolverr.Solve(request);
Assert.AreEqual("ok", flareSolverrResponse.Status);
Assert.AreEqual(userAgent, flareSolverrResponse.Solution.UserAgent);
}
/// <summary>
/// Creates a new instance of the <see cref="ClearanceHandler"/>.
/// </summary>
/// <param name="flareSolverrApiUrl">FlareSolverr API URL. If null or empty it will detect the challenges, but
/// they will not be solved. Example: "http://localhost:8191/"</param>
public ClearanceHandler(string flareSolverrApiUrl)
: base(new HttpClientHandler())
{
_client = new HttpClient(new HttpClientHandler
{
AllowAutoRedirect = false,
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
CookieContainer = new CookieContainer()
});
if (!string.IsNullOrWhiteSpace(flareSolverrApiUrl))
{
_flareSolverr = new FlareSolverr(flareSolverrApiUrl)
{
MaxTimeout = MaxTimeout
};
}
}
public async Task SolveErrorConfig()
{
var uri = new Uri("https://www.google.com/");
var flareSolverr = new FlareSolverr("http://localhost:44445");
var request = new HttpRequestMessage(HttpMethod.Get, uri);
try
{
await flareSolverr.Solve(request);
Assert.Fail("Exception not thrown");
}
catch (FlareSolverrException e)
{
Assert.IsTrue(e.Message.Contains("Error connecting to FlareSolverr server"));
}
catch (Exception e)
{
Assert.Fail("Unexpected exception: " + e);
}
}
public async Task SolveError()
{
var uri = new Uri("https://www.google.bad1/");
var flareSolverr = new FlareSolverr(Settings.FlareSolverrApiUrl);
var request = new HttpRequestMessage(HttpMethod.Get, uri);
try
{
await flareSolverr.Solve(request);
Assert.Fail("Exception not thrown");
}
catch (FlareSolverrException e)
{
Assert.AreEqual("FlareSolverr was unable to process the request, please check FlareSolverr logs. Message: Unable to process browser request. Error: Error: net::ERR_NAME_NOT_RESOLVED at https://www.google.bad1/", e.Message);
}
catch (Exception e)
{
Assert.Fail("Unexpected exception: " + e);
}
}
public async Task SolveError()
{
var uri = new Uri("https://www.google.bad1/");
var flareSolverr = new FlareSolverr(Settings.FlareSolverrApiUrl);
var request = new HttpRequestMessage(HttpMethod.Get, uri);
try
{
await flareSolverr.Solve(request);
Assert.Fail("Exception not thrown");
}
catch (FlareSolverrException e)
{
Assert.AreEqual("NS_ERROR_UNKNOWN_HOST at https://www.google.bad1/", e.Message);
}
catch (Exception e)
{
Assert.Fail("Unexpected exception: " + e);
}
}
public async Task SolveTestSessions()
{
var flareSolverr = new FlareSolverr(Settings.FlareSolverrApiUrl);
var flareSolverrResponse = await flareSolverr.CreateSession();
Assert.AreEqual("ok", flareSolverrResponse.Status);
Assert.AreEqual("Session created successfully.", flareSolverrResponse.Message);
Assert.IsTrue(flareSolverrResponse.StartTimestamp > 0);
Assert.IsTrue(flareSolverrResponse.EndTimestamp > flareSolverrResponse.StartTimestamp);
Assert.IsTrue(flareSolverrResponse.Version.Contains("2."));
var sessionID = flareSolverrResponse.Session;
flareSolverrResponse = await flareSolverr.ListSessions();
Assert.AreEqual("ok", flareSolverrResponse.Status);
Assert.IsTrue(flareSolverrResponse.Sessions.Contains(sessionID));
flareSolverrResponse = await flareSolverr.DestroySession(sessionID);
Assert.AreEqual("ok", flareSolverrResponse.Status);
}
public async Task SolveErrorTimeout()
{
var uri = new Uri("https://www.google.com/");
var flareSolverr = new FlareSolverr(Settings.FlareSolverrApiUrl)
{
MaxTimeout = 100
};
var request = new HttpRequestMessage(HttpMethod.Get, uri);
try
{
await flareSolverr.Solve(request);
Assert.Fail("Exception not thrown");
}
catch (FlareSolverrException e)
{
Assert.IsTrue(e.Message.Contains("FlareSolverr was unable to process the request, please check FlareSolverr logs. Message: Error: Unable to process browser request. Error: Maximum timeout reached. maxTimeout=100 (ms)"));
}
catch (Exception e)
{
Assert.Fail("Unexpected exception: " + e);
}
}
public async Task SolveErrorProxy()
{
var uri = new Uri("https://www.google.com/");
var flareSolverr = new FlareSolverr(Settings.FlareSolverrApiUrl)
{
ProxyUrl = "http://localhost:44445"
};
var request = new HttpRequestMessage(HttpMethod.Get, uri);
try
{
await flareSolverr.Solve(request);
Assert.Fail("Exception not thrown");
}
catch (FlareSolverrException e)
{
Assert.IsTrue(e.Message.Contains("FlareSolverr was unable to process the request, please check FlareSolverr logs. Message: Error: Unable to process browser request. Error: NS_ERROR_PROXY_CONNECTION_REFUSED at https://www.google.com/"));
}
catch (Exception e)
{
Assert.Fail("Unexpected exception: " + e);
}
}
