/// <inheritdoc />
public override void Process(BotData data)
{
base.Process(data);
// If the clearance info is already set and we're not getting it fresh each time, skip
if (data.UseProxies)
{
if (data.Proxy.Clearance != "" && !data.GlobalSettings.Proxies.AlwaysGetClearance)
{
data.Log(new LogEntry("Skipping CF Bypass because there is already a valid cookie", Colors.White));
data.Cookies["cf_clearance"] = data.Proxy.Clearance;
return;
}
}
var localUrl = ReplaceValues(url, data);
var uri = new Uri(localUrl);
var timeout = data.GlobalSettings.General.RequestTimeout * 1000;
var request = new HttpRequest();
request.IgnoreProtocolErrors = true;
request.ConnectTimeout = timeout;
request.ReadWriteTimeout = timeout;
request.Cookies = new CookieStorage();
foreach (var cookie in data.Cookies)
{
request.Cookies.Add(new Cookie(cookie.Key, cookie.Value, "/", uri.Host));
}
if (data.UseProxies)
{
switch (data.Proxy.Type)
{
case Extreme.Net.ProxyType.Http:
request.Proxy = HttpProxyClient.Parse(data.Proxy.Proxy);
break;
case Extreme.Net.ProxyType.Socks4:
request.Proxy = Socks4ProxyClient.Parse(data.Proxy.Proxy);
break;
case Extreme.Net.ProxyType.Socks4a:
request.Proxy = Socks4AProxyClient.Parse(data.Proxy.Proxy);
break;
case Extreme.Net.ProxyType.Socks5:
request.Proxy = Socks5ProxyClient.Parse(data.Proxy.Proxy);
break;
case Extreme.Net.ProxyType.Chain:
throw new Exception("The Chain Proxy Type is not supported in Leaf.xNet (used for CF Bypass).");
}
request.Proxy.ReadWriteTimeout = timeout;
request.Proxy.ConnectTimeout = timeout;
request.Proxy.Username = data.Proxy.Username;
request.Proxy.Password = data.Proxy.Password;
}
request.UserAgent = ReplaceValues(userAgent, data);
var twoCapToken = data.GlobalSettings.Captchas.TwoCapToken;
if (twoCapToken != "")
{
request.CaptchaSolver = new TwoCaptchaSolver()
{
ApiKey = data.GlobalSettings.Captchas.TwoCapToken
}
}
;
var response = request.GetThroughCloudflare(new Uri(localUrl));
var responseString = response.ToString();
// Save the cookies
var ck = response.Cookies.GetCookies(localUrl);
var clearance = "";
var cfduid = "";
try
{
clearance = ck["cf_clearance"].Value;
cfduid = ck["__cfduid"].Value;
}
catch { }
if (data.UseProxies)
{
data.Proxy.Clearance = clearance;
data.Proxy.Cfduid = cfduid;
}
if (clearance != "")
{
data.Log(new LogEntry("Got Cloudflare clearance!", Colors.GreenYellow));
data.Log(new LogEntry(clearance + Environment.NewLine + cfduid + Environment.NewLine, Colors.White));
}
// Get code
data.ResponseCode = ((int)response.StatusCode).ToString();
if (PrintResponseInfo)
{
data.Log(new LogEntry($"Response code: {data.ResponseCode}", Colors.Cyan));
}
// Get headers
if (PrintResponseInfo)
{
data.Log(new LogEntry("Received headers:", Colors.DeepPink));
}
var receivedHeaders = response.EnumerateHeaders();
data.ResponseHeaders.Clear();
while (receivedHeaders.MoveNext())
{
var header = receivedHeaders.Current;
data.ResponseHeaders.Add(header.Key, header.Value);
if (PrintResponseInfo)
{
data.Log(new LogEntry($"{header.Key}: {header.Value}", Colors.LightPink));
}
}
if (!response.ContainsHeader(HttpHeader.ContentLength))
{
if (data.ResponseHeaders.ContainsKey("Content-Encoding") && data.ResponseHeaders["Content-Encoding"].Contains("gzip"))
{
data.ResponseHeaders["Content-Length"] = GZip.Zip(responseString).Length.ToString();
}
else
{
data.ResponseHeaders["Content-Length"] = responseString.Length.ToString();
}
if (PrintResponseInfo)
{
data.Log(new LogEntry($"Content-Length: {data.ResponseHeaders["Content-Length"]}", Colors.LightPink));
}
}
// Get cookies
if (PrintResponseInfo)
{
data.Log(new LogEntry("Received cookies:", Colors.Goldenrod));
}
foreach (Cookie cookie in response.Cookies.GetCookies(localUrl))
{
if (data.Cookies.ContainsKey(cookie.Name))
{
data.Cookies[cookie.Name] = cookie.Value;
}
else
{
data.Cookies.Add(cookie.Name, cookie.Value);
}
if (PrintResponseInfo)
{
data.Log(new LogEntry($"{cookie.Name}: {cookie.Value}", Colors.LightGoldenrodYellow));
}
}
data.ResponseSource = responseString;
if (PrintResponseInfo)
{
data.Log(new LogEntry("Response Source:", Colors.Green));
data.Log(new LogEntry(data.ResponseSource, Colors.GreenYellow));
}
if (ErrorOn302 && data.ResponseCode.Contains("302"))
{
data.Status = BotStatus.ERROR;
}
}
}
/// <inheritdoc />
public override void Process(BotData data)
{
base.Process(data);
// If the clearance info is already set and we're not getting it fresh each time, skip
if (data.UseProxies)
{
if (data.Proxy.Clearance != "" && !data.GlobalSettings.Proxies.AlwaysGetClearance)
{
data.Log(new LogEntry("Skipping CF Bypass because there is already a valid cookie", Colors.White));
data.Cookies["cf_clearance"] = data.Proxy.Clearance;
data.Cookies["__cfduid"] = data.Proxy.Cfduid;
return;
}
}
var localUrl = ReplaceValues(url, data);
var uri = new Uri(localUrl);
var timeout = data.GlobalSettings.General.RequestTimeout * 1000;
// Initialize the captcha provider
// TODO: Add more providers by implementing the ICaptchaProvider interface on the missing ones
ICaptchaProvider provider = null;
switch (data.GlobalSettings.Captchas.CurrentService)
{
case CaptchaServices.ServiceType.AntiCaptcha:
provider = new AntiCaptchaProvider(data.GlobalSettings.Captchas.AntiCapToken);
break;
case CaptchaServices.ServiceType.TwoCaptcha:
provider = new TwoCaptchaProvider(data.GlobalSettings.Captchas.TwoCapToken);
break;
}
// Initialize the Cloudflare Solver
CloudflareSolver cf = new CloudflareSolver(provider, ReplaceValues(UserAgent, data))
{
ClearanceDelay = 3000,
MaxCaptchaTries = 1,
MaxTries = 3
};
// Create the cookie container
CookieContainer cookies = new CookieContainer();
foreach (var cookie in data.Cookies)
{
cookies.Add(new Cookie(cookie.Key, cookie.Value, "/", uri.Host));
}
// Initialize the http handler
HttpClientHandler handler = new HttpClientHandler
{
AllowAutoRedirect = true,
CookieContainer = cookies,
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
};
// Assign the proxy to the inner handler if necessary
if (data.UseProxies)
{
if (data.Proxy.Type != Extreme.Net.ProxyType.Http)
{
throw new Exception($"The proxy type {data.Proxy.Type} is not supported by this block yet");
}
handler.Proxy = new WebProxy(data.Proxy.Proxy, false);
handler.UseProxy = true;
if (!string.IsNullOrEmpty(data.Proxy.Username))
{
handler.DefaultProxyCredentials = new NetworkCredential(data.Proxy.Username, data.Proxy.Password);
}
}
// Initialize the http client
HttpClient http = new HttpClient(handler);
http.Timeout = TimeSpan.FromMinutes(timeout);
http.DefaultRequestHeaders.Add("User-Agent", ReplaceValues(UserAgent, data));
var result = cf.Solve(http, handler, uri, ReplaceValues(UserAgent, data)).Result;
if (result.Success)
{
data.Log(new LogEntry($"[Success] Protection bypassed: {result.DetectResult.Protection}", Colors.GreenYellow));
}
else if (result.DetectResult.Protection == CloudflareProtection.Unknown)
{
data.Log(new LogEntry($"Unknown protection, skipping the bypass!", Colors.Tomato));
}
else
{
throw new Exception($"CF Bypass Failed: {result.FailReason}");
}
// Now that we got the cookies, proceed with the normal request
HttpResponseMessage response = null;
try
{
response = http.GetAsync(uri).Result;
}
catch (Exception ex)
{
throw new Exception(ex.Message);
}
finally
{
handler.Dispose();
http.Dispose();
}
var responseString = response.Content.ReadAsStringAsync().Result;
// Save the cloudflare cookies
var clearance = "";
var cfduid = "";
foreach (Cookie cookie in cookies.GetCookies(uri))
{
switch (cookie.Name)
{
case "cf_clearance":
clearance = cookie.Value;
break;
case "__cfduid":
cfduid = cookie.Value;
break;
}
}
// Save the cookies in the proxy
if (data.UseProxies)
{
data.Proxy.Clearance = clearance;
data.Proxy.Cfduid = cfduid;
}
if (clearance != "")
{
data.Log(new LogEntry("Got Cloudflare clearance!", Colors.GreenYellow));
data.Log(new LogEntry(clearance + Environment.NewLine + cfduid + Environment.NewLine, Colors.White));
}
// Get code
data.ResponseCode = ((int)response.StatusCode).ToString();
if (PrintResponseInfo)
{
data.Log(new LogEntry($"Response code: {data.ResponseCode}", Colors.Cyan));
}
// Get headers
if (PrintResponseInfo)
{
data.Log(new LogEntry("Received headers:", Colors.DeepPink));
}
data.ResponseHeaders.Clear();
foreach (var header in response.Headers)
{
var h = new KeyValuePair <string, string>(header.Key, header.Value.First());
data.ResponseHeaders.Add(h.Key, h.Value);
if (PrintResponseInfo)
{
data.Log(new LogEntry($"{h.Key}: {h.Value}", Colors.LightPink));
}
}
// Add the Content-Length header if it was not sent by the server
if (!data.ResponseHeaders.ContainsKey("Content-Length"))
{
if (data.ResponseHeaders.ContainsKey("Content-Encoding") && data.ResponseHeaders["Content-Encoding"].Contains("gzip"))
{
data.ResponseHeaders["Content-Length"] = GZip.Zip(responseString).Length.ToString();
}
else
{
data.ResponseHeaders["Content-Length"] = responseString.Length.ToString();
}
if (PrintResponseInfo)
{
data.Log(new LogEntry($"Content-Length: {data.ResponseHeaders["Content-Length"]}", Colors.LightPink));
}
}
// Get cookies
if (PrintResponseInfo)
{
data.Log(new LogEntry("Received cookies:", Colors.Goldenrod));
}
foreach (Cookie cookie in cookies.GetCookies(uri))
{
data.Cookies[cookie.Name] = cookie.Value;
if (PrintResponseInfo)
{
data.Log(new LogEntry($"{cookie.Name}: {cookie.Value}", Colors.LightGoldenrodYellow));
}
}
// Print source
data.ResponseSource = responseString;
if (PrintResponseInfo)
{
data.Log(new LogEntry("Response Source:", Colors.Green));
data.Log(new LogEntry(data.ResponseSource, Colors.GreenYellow));
}
// Error on 302 status
if (ErrorOn302 && response.StatusCode == HttpStatusCode.Redirect)
{
data.Status = BotStatus.ERROR;
}
}
/// <inheritdoc />
public override void Process(BotData data)
{
base.Process(data);
#region Request
// Set base URL
var localUrl = ReplaceValues(url, data);
var cType = ReplaceValues(contentType, data);
var oldJar = data.Cookies;
// Create request
HttpRequest request = new HttpRequest();
// Setup options
var timeout = data.GlobalSettings.General.RequestTimeout * 1000;
request.IgnoreProtocolErrors = true;
request.AllowAutoRedirect = autoRedirect;
request.EnableEncodingContent = acceptEncoding;
request.ReadWriteTimeout = timeout;
request.ConnectTimeout = timeout;
request.KeepAlive = true;
request.MaximumAutomaticRedirections = data.ConfigSettings.MaxRedirects;
// Check if it has GET parameters
if (ParseQuery && localUrl.Contains('?') && localUrl.Contains('='))
{
// Remove the query from the base URL
localUrl = ReplaceValues(url.Split('?')[0], data);
data.Log(new LogEntry($"Calling Base URL: {localUrl}", Colors.MediumTurquoise));
// Parse the GET parameters
var getParams = ReplaceValues(url.Split('?')[1], data);
var paramList = getParams.Split('&');
// Build the query, first replace variables in them and encode the parameters
foreach (var par in paramList)
{
var split = par.Split('=');
// Encode them if needed
if (split[0].Contains('%'))
{
split[0] = Uri.EscapeDataString(split[0]);
}
if (split[1].Contains('%'))
{
split[1] = Uri.EscapeDataString(split[1]);
}
// Add them to the query
request.AddUrlParam(split[0], split[1]);
data.Log(new LogEntry($"Added Query Parameter: {split[0]} = {split[1]}", Colors.MediumTurquoise));
}
}
else
{
data.Log(new LogEntry($"Calling URL: {localUrl}", Colors.MediumTurquoise));
}
// Set up the Content and Content-Type
HttpContent content = null;
switch (requestType)
{
case RequestType.Standard:
var pData = ReplaceValues(Regex.Replace(postData, @"(?<!\\)\\n", Environment.NewLine).Replace(@"\\n", @"\n"), data);
if (CanContainBody(method))
{
if (encodeContent)
{
// Very dirty but it works
var nonce = data.Random.Next(1000000, 9999999);
pData = pData.Replace("&", $"{nonce}&{nonce}").Replace("=", $"{nonce}={nonce}");
pData = string.Join("", BlockFunction.SplitInChunks(pData, 2080)
.Select(s => Uri.EscapeDataString(s)))
.Replace($"{nonce}%26{nonce}", "&").Replace($"{nonce}%3D{nonce}", "=");
}
content = new StringContent(pData);
content.ContentType = cType;
data.Log(new LogEntry($"Post Data: {pData}", Colors.MediumTurquoise));
}
break;
case RequestType.Multipart:
var bdry = multipartBoundary != "" ? ReplaceValues(multipartBoundary, data) : GenerateMultipartBoundary();
content = new Extreme.Net.MultipartContent(bdry);
var mContent = content as Extreme.Net.MultipartContent;
data.Log(new LogEntry($"Content-Type: multipart/form-data; boundary={bdry}", Colors.MediumTurquoise));
data.Log(new LogEntry("Multipart Data:", Colors.MediumTurquoise));
data.Log(new LogEntry(bdry, Colors.MediumTurquoise));
foreach (var c in MultipartContents)
{
var rValue = ReplaceValues(c.Value, data);
var rName = ReplaceValues(c.Name, data);
var rContentType = ReplaceValues(c.ContentType, data);
if (c.Type == MultipartContentType.String)
{
mContent.Add(new StringContent(rValue), rName);
data.Log(new LogEntry($"Content-Disposition: form-data; name=\"{rName}\"{Environment.NewLine}{Environment.NewLine}{rValue}", Colors.MediumTurquoise));
}
else if (c.Type == MultipartContentType.File)
{
mContent.Add(new FileContent(rValue), rName, rValue, rContentType);
data.Log(new LogEntry($"Content-Disposition: form-data; name=\"{rName}\"; filename=\"{rValue}\"{Environment.NewLine}Content-Type: {rContentType}{Environment.NewLine}{Environment.NewLine}[FILE CONTENT OMITTED]", Colors.MediumTurquoise));
}
data.Log(new LogEntry(bdry, Colors.MediumTurquoise));
}
break;
default:
break;
}
// Set proxy
if (data.UseProxies)
{
request.Proxy = data.Proxy.GetClient();
try
{
request.Proxy.ReadWriteTimeout = timeout;
request.Proxy.ConnectTimeout = timeout;
request.Proxy.Username = data.Proxy.Username;
request.Proxy.Password = data.Proxy.Password;
}
catch { }
}
// Set headers
data.Log(new LogEntry("Sent Headers:", Colors.DarkTurquoise));
// var fixedNames = Enum.GetNames(typeof(HttpHeader)).Select(n => n.ToLower());
foreach (var header in CustomHeaders)
{
try
{
var key = ReplaceValues(header.Key, data);
var replacedKey = key.Replace("-", "").ToLower(); // Used to compare with the HttpHeader enum
var val = ReplaceValues(header.Value, data);
if (replacedKey == "contenttype" && content != null)
{
continue;
} // Disregard additional Content-Type headers
if (replacedKey == "acceptencoding" && acceptEncoding)
{
continue;
} // Disregard additional Accept-Encoding headers
// else if (fixedNames.Contains(replacedKey)) request.AddHeader((HttpHeader)Enum.Parse(typeof(HttpHeader), replacedKey, true), val);
else
{
request.AddHeader(key, val);
}
data.Log(new LogEntry(key + ": " + val, Colors.MediumTurquoise));
}
catch { }
}
// Add the authorization header on a Basic Auth request
if (requestType == RequestType.BasicAuth)
{
var usr = ReplaceValues(authUser, data);
var pwd = ReplaceValues(authPass, data);
var auth = "Basic " + BlockFunction.Base64Encode(usr + ":" + pwd);
request.AddHeader("Authorization", auth);
data.Log(new LogEntry($"Authorization: {auth}", Colors.MediumTurquoise));
}
// Add the content-type header
if (CanContainBody(method) && content != null && requestType == RequestType.Standard)
{
data.Log(new LogEntry($"Content-Type: {cType}", Colors.MediumTurquoise));
}
// Add new user-defined custom cookies to the bot's cookie jar
request.Cookies = new CookieDictionary();
foreach (var cookie in CustomCookies)
{
data.Cookies[ReplaceValues(cookie.Key, data)] = ReplaceValues(cookie.Value, data);
}
// Set cookies from the bot's cookie jar to the request's CookieDictionary
data.Log(new LogEntry("Sent Cookies:", Colors.MediumTurquoise));
foreach (var cookie in data.Cookies)
{
request.Cookies.Add(cookie.Key, cookie.Value);
data.Log(new LogEntry($"{cookie.Key}: {cookie.Value}", Colors.MediumTurquoise));
}
data.LogNewLine();
#endregion
#region Response
// Create the response
HttpResponse response = null;
try
{
// Get response
response = request.Raw(method, localUrl, content);
var responseString = "";
// Get address
data.Address = response.Address.ToString();
data.Log(new LogEntry("Address: " + data.Address, Colors.Cyan));
// Get code
data.ResponseCode = ((int)response.StatusCode).ToString();
data.Log(new LogEntry($"Response code: {data.ResponseCode} ({response.StatusCode})", Colors.Cyan));
// Get headers
data.Log(new LogEntry("Received headers:", Colors.DeepPink));
var headerList = new List <KeyValuePair <string, string> >();
var receivedHeaders = response.EnumerateHeaders();
data.ResponseHeaders.Clear();
while (receivedHeaders.MoveNext())
{
var header = receivedHeaders.Current;
data.ResponseHeaders.Add(header.Key, header.Value);
data.Log(new LogEntry($"{header.Key}: {header.Value}", Colors.LightPink));
}
if (!response.ContainsHeader(HttpHeader.ContentLength) && ResponseType != ResponseType.File)
{
responseString = response.ToString(); // Read the stream
if (data.ResponseHeaders.ContainsKey("Content-Encoding") && data.ResponseHeaders["Content-Encoding"].Contains("gzip"))
{
data.ResponseHeaders["Content-Length"] = GZip.Zip(responseString).Length.ToString();
}
else
{
data.ResponseHeaders["Content-Length"] = responseString.Length.ToString();
}
data.Log(new LogEntry($"Content-Length: {data.ResponseHeaders["Content-Length"]}", Colors.LightPink));
}
// Get cookies
data.Log(new LogEntry("Received cookies:", Colors.Goldenrod));
data.Cookies = response.Cookies;
foreach (var cookie in response.Cookies)
{
// If the cookie was already present before, don't log it
if (oldJar.ContainsKey(cookie.Key) && oldJar[cookie.Key] == cookie.Value)
{
continue;
}
data.Log(new LogEntry($"{cookie.Key}: {cookie.Value}", Colors.LightGoldenrodYellow));
}
// Save the response content
switch (responseType)
{
case ResponseType.String:
data.Log(new LogEntry("Response Source:", Colors.Green));
if (readResponseSource)
{
if (responseString == "")
{
responseString = response.ToString(); // Read the stream if you didn't already read it
}
data.ResponseSource = responseString;
data.Log(new LogEntry(data.ResponseSource, Colors.GreenYellow));
}
else
{
data.ResponseSource = "";
data.Log(new LogEntry("[SKIPPED]", Colors.GreenYellow));
}
break;
case ResponseType.File:
if (SaveAsScreenshot)
{
SaveScreenshot(response.ToMemoryStream(), data); // Read the stream
data.Log(new LogEntry("File saved as screenshot", Colors.Green));
}
else
{
var filePath = ReplaceValues(downloadPath, data);
var dirName = Path.GetDirectoryName(filePath);
if (dirName != "")
{
dirName += Path.DirectorySeparatorChar.ToString();
}
var fileName = Path.GetFileNameWithoutExtension(filePath);
var fileExtension = Path.GetExtension(filePath);
var sanitizedPath = $"{dirName}{MakeValidFileName(fileName)}{fileExtension}";
using (var stream = File.Create(sanitizedPath)) { response.ToMemoryStream().CopyTo(stream); } // Read the stream
data.Log(new LogEntry("File saved as " + sanitizedPath, Colors.Green));
}
break;
default:
break;
}
}
catch (Exception ex)
{
data.Log(new LogEntry(ex.Message, Colors.White));
if (ex.GetType() == typeof(HttpException))
{
data.ResponseCode = ((HttpException)ex).HttpStatusCode.ToString();
data.Log(new LogEntry("Status code: " + data.ResponseCode, Colors.Cyan));
}
if (!data.ConfigSettings.IgnoreResponseErrors)
{
throw;
}
}
#endregion
}