/// <summary>
/// Opens a browser of the given type (if not already open) and with the given settings.
/// </summary>
/// <param name="data">The BotData where the settings are stored.</param>
public static void OpenBrowser(BotData data)
{
if (!data.BrowserOpen)
{
data.Log(new LogEntry("Opening browser...", Colors.White));
switch (data.GlobalSettings.Selenium.Browser)
{
case BrowserType.Chrome:
try
{
ChromeOptions chromeop = new ChromeOptions();
ChromeDriverService chromeservice = ChromeDriverService.CreateDefaultService();
chromeservice.SuppressInitialDiagnosticInformation = true;
chromeservice.HideCommandPromptWindow = true;
chromeservice.EnableVerboseLogging = false;
chromeop.AddArgument("--log-level=3");
chromeop.BinaryLocation = data.GlobalSettings.Selenium.ChromeBinaryLocation;
if (data.GlobalSettings.Selenium.Headless || data.ConfigSettings.ForceHeadless)
{
chromeop.AddArgument("--headless");
}
else if (data.GlobalSettings.Selenium.ChromeExtensions.Count > 0) // This should only be done when not headless
{
chromeop.AddExtensions(data.GlobalSettings.Selenium.ChromeExtensions
.Where(ext => ext.EndsWith(".crx"))
.Select(ext => Directory.GetCurrentDirectory() + "\\ChromeExtensions\\" + ext));
}
if (data.ConfigSettings.DisableNotifications)
{
chromeop.AddArgument("--disable-notifications");
}
if (data.ConfigSettings.CustomCMDArgs != "")
{
chromeop.AddArgument(data.ConfigSettings.CustomCMDArgs);
}
if (data.ConfigSettings.RandomUA)
{
chromeop.AddArgument("--user-agent=" + BlockFunction.RandomUserAgent(data.rand));
}
else if (data.ConfigSettings.CustomUserAgent != "")
{
chromeop.AddArgument("--user-agent=" + data.ConfigSettings.CustomUserAgent);
}
if (data.UseProxies)
{
chromeop.AddArgument("--proxy-server=" + data.Proxy.Type.ToString().ToLower() + "://" + data.Proxy.Proxy);
}
data.Driver = new ChromeDriver(chromeservice, chromeop);
}
catch (Exception ex) { data.Log(new LogEntry(ex.ToString(), Colors.White)); return; }
break;
case BrowserType.Firefox:
try
{
FirefoxOptions fireop = new FirefoxOptions();
FirefoxDriverService fireservice = FirefoxDriverService.CreateDefaultService();
FirefoxProfile fireprofile = new FirefoxProfile();
fireservice.SuppressInitialDiagnosticInformation = true;
fireservice.HideCommandPromptWindow = true;
fireop.AddArgument("--log-level=3");
fireop.BrowserExecutableLocation = data.GlobalSettings.Selenium.FirefoxBinaryLocation;
if (data.GlobalSettings.Selenium.Headless || data.ConfigSettings.ForceHeadless)
{
fireop.AddArgument("--headless");
}
if (data.ConfigSettings.DisableNotifications)
{
fireprofile.SetPreference("dom.webnotifications.enabled", false);
}
if (data.ConfigSettings.CustomCMDArgs != "")
{
fireop.AddArgument(data.ConfigSettings.CustomCMDArgs);
}
if (data.ConfigSettings.RandomUA)
{
fireprofile.SetPreference("general.useragent.override", BlockFunction.RandomUserAgent(data.rand));
}
else if (data.ConfigSettings.CustomUserAgent != "")
{
fireprofile.SetPreference("general.useragent.override", data.ConfigSettings.CustomUserAgent);
}
if (data.UseProxies)
{
fireprofile.SetPreference("network.proxy.type", 1);
if (data.Proxy.Type == Extreme.Net.ProxyType.Http)
{
fireprofile.SetPreference("network.proxy.http", data.Proxy.Host);
fireprofile.SetPreference("network.proxy.http_port", int.Parse(data.Proxy.Port));
fireprofile.SetPreference("network.proxy.ssl", data.Proxy.Host);
fireprofile.SetPreference("network.proxy.ssl_port", int.Parse(data.Proxy.Port));
}
else
{
fireprofile.SetPreference("network.proxy.socks", data.Proxy.Host);
fireprofile.SetPreference("network.proxy.socks_port", int.Parse(data.Proxy.Port));
if (data.Proxy.Type == Extreme.Net.ProxyType.Socks4)
{
fireprofile.SetPreference("network.proxy.socks_version", 4);
}
else if (data.Proxy.Type == Extreme.Net.ProxyType.Socks5)
{
fireprofile.SetPreference("network.proxy.socks_version", 5);
}
}
}
fireop.Profile = fireprofile;
data.Driver = new FirefoxDriver(fireservice, fireop, new TimeSpan(0, 1, 0));
}
catch (Exception ex) { data.Log(new LogEntry(ex.ToString(), Colors.White)); return; }
break;
}
data.Driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(data.GlobalSettings.Selenium.PageLoadTimeout);
data.Log(new LogEntry("Opened!", Colors.White));
data.BrowserOpen = true;
}
else
{
try
{
UpdateSeleniumData(data);
}
catch { }
}
}
/// <inheritdoc />
public override void Process(BotData data)
{
base.Process(data);
#region Request
// Set base URL
var localUrl = ReplaceValues(url, data);
var cType = ReplaceValues(contentType, data);
// Create request
HttpRequest request = new HttpRequest();
// Setup options
var timeout = data.GlobalSettings.General.RequestTimeout * 1000;
request.IgnoreProtocolErrors = true;
request.AllowAutoRedirect = autoRedirect;
request.EnableEncodingContent = true;
request.ReadWriteTimeout = timeout;
request.ConnectTimeout = timeout;
request.KeepAlive = true;
request.MaximumAutomaticRedirections = data.ConfigSettings.MaxRedirects;
// Check if it has GET parameters
if (ParseQuery && localUrl.Contains('?') && localUrl.Contains('='))
{
// Remove the query from the base URL
localUrl = ReplaceValues(url.Split('?')[0], data);
data.Log(new LogEntry($"Calling Base URL: {localUrl}", Colors.MediumTurquoise));
// Parse the GET parameters
var getParams = ReplaceValues(url.Split('?')[1], data);
var paramList = getParams.Split('&');
// Build the query, first replace variables in them and encode the parameters
foreach (var par in paramList)
{
var split = par.Split('=');
// Encode them if needed
if (split[0].Contains('%'))
{
split[0] = Uri.EscapeDataString(split[0]);
}
if (split[1].Contains('%'))
{
split[1] = Uri.EscapeDataString(split[1]);
}
// Add them to the query
request.AddUrlParam(split[0], split[1]);
data.Log(new LogEntry($"Added Query Parameter: {split[0]} = {split[1]}", Colors.MediumTurquoise));
}
}
else
{
data.Log(new LogEntry($"Calling URL: {localUrl}", Colors.MediumTurquoise));
}
// Set up the Content and Content-Type
HttpContent content = null;
switch (requestType)
{
case RequestType.Standard:
var pData = string.Join(Environment.NewLine, postData
.Split(new string[] { "\\n" }, StringSplitOptions.None)
.Select(p => ReplaceValues(p, data)));
if (pData != "")
{
if (encodeContent)
{
// Very dirty but it works
var nonce = data.rand.Next(1000000, 9999999);
pData = pData.Replace("&", $"{nonce}&{nonce}").Replace("=", $"{nonce}={nonce}");
pData = System.Uri.EscapeDataString(pData).Replace($"{nonce}%26{nonce}", "&").Replace($"{nonce}%3D{nonce}", "=");
}
content = new StringContent(pData);
content.ContentType = cType;
data.Log(new LogEntry(string.Format("Post Data: {0}", pData), Colors.MediumTurquoise));
}
break;
case RequestType.Multipart:
if (multipartBoundary != "")
{
content = new Extreme.Net.MultipartContent(multipartBoundary);
}
else
{
content = new Extreme.Net.MultipartContent(GenerateMultipartBoundary());
}
var mContent = content as Extreme.Net.MultipartContent;
foreach (var c in MultipartContents)
{
if (c.Type == MultipartContentType.String)
{
mContent.Add(new StringContent(ReplaceValues(c.Value, data)), ReplaceValues(c.Name, data));
}
else if (c.Type == MultipartContentType.File)
{
mContent.Add(new FileContent(ReplaceValues(c.Value, data)), ReplaceValues(c.Name, data));
}
}
break;
default:
break;
}
// Set proxy
if (data.UseProxies)
{
request.Proxy = data.Proxy.GetClient();
try
{
request.Proxy.ReadWriteTimeout = timeout;
request.Proxy.ConnectTimeout = timeout;
request.Proxy.Username = data.Proxy.Username;
request.Proxy.Password = data.Proxy.Password;
}
catch { }
}
// Set headers
data.Log(new LogEntry("Sent Headers:", Colors.DarkTurquoise));
var fixedNames = Enum.GetNames(typeof(HttpHeader)).Select(n => n.ToLower());
foreach (var header in CustomHeaders)
{
try
{
var key = ReplaceValues(header.Key, data);
var replacedKey = key.Replace("-", "").ToLower(); // Used to compare with the HttpHeader enum
var val = ReplaceValues(header.Value, data);
if (replacedKey == "contenttype") // If it's somewhat needed to define Content-Type in e.g. a GET request
{
content = new StringContent("");
content.ContentType = val;
}
else if (fixedNames.Contains(replacedKey))
{
request.AddHeader((HttpHeader)Enum.Parse(typeof(HttpHeader), replacedKey, true), val);
}
else
{
request.AddHeader(key, val);
}
data.Log(new LogEntry(key + ": " + val, Colors.MediumTurquoise));
}
catch { }
}
// Add the authorization header on a Basic Auth request
if (requestType == RequestType.BasicAuth)
{
var usr = ReplaceValues(authUser, data);
var pwd = ReplaceValues(authPass, data);
var auth = "Basic " + BlockFunction.Base64Encode(usr + ":" + pwd);
request.AddHeader("Authorization", auth);
data.Log(new LogEntry("Authorization: " + auth, Colors.MediumTurquoise));
}
// Add the content-type header
if ((method == HttpMethod.POST || method == HttpMethod.PUT || method == HttpMethod.DELETE) && cType != "")
{
data.Log(new LogEntry("Content-Type: " + cType, Colors.MediumTurquoise));
}
// Add new user-defined custom cookies to the bot's cookie jar
request.Cookies = new CookieDictionary();
foreach (var cookie in CustomCookies)
{
data.Cookies[ReplaceValues(cookie.Key, data)] = ReplaceValues(cookie.Value, data);
}
// Set cookies from the bot's cookie jar to the request's CookieDictionary
data.Log(new LogEntry("Sent Cookies:", Colors.MediumTurquoise));
foreach (var cookie in data.Cookies)
{
request.Cookies.Add(cookie.Key, cookie.Value);
data.Log(new LogEntry(cookie.Key + " : " + cookie.Value, Colors.MediumTurquoise));
}
data.LogNewLine();
#endregion
#region Response
// Create the response
HttpResponse response = null;
try
{
// Get response
response = request.Raw(method, localUrl, content);
// Get address
data.Address = response.Address.ToString();
data.Log(new LogEntry("Address: " + data.Address, Colors.Cyan));
// Get code
data.ResponseCode = ((int)response.StatusCode).ToString();
data.Log(new LogEntry("Response code: " + data.ResponseCode, Colors.Cyan));
// Get headers
data.Log(new LogEntry("Received headers:", Colors.DeepPink));
var headerList = new List <KeyValuePair <string, string> >();
var receivedHeaders = response.EnumerateHeaders();
data.ResponseHeaders.Clear();
while (receivedHeaders.MoveNext())
{
var header = receivedHeaders.Current;
data.ResponseHeaders.Add(header.Key, header.Value);
data.Log(new LogEntry(header.Key + ": " + header.Value, Colors.LightPink));
}
// Get cookies
data.Log(new LogEntry("Received cookies:", Colors.Goldenrod));
data.Cookies = response.Cookies;
foreach (var cookie in response.Cookies)
{
data.Log(new LogEntry(cookie.Key + ": " + cookie.Value, Colors.LightGoldenrodYellow));
}
// Save the response content
switch (responseType)
{
case ResponseType.String:
data.Log(new LogEntry("Response Source:", Colors.Green));
if (readResponseSource)
{
data.ResponseSource = response.ToString();
data.Log(new LogEntry(data.ResponseSource, Colors.GreenYellow));
}
else
{
data.ResponseSource = "";
data.Log(new LogEntry("[SKIPPED]", Colors.GreenYellow));
}
break;
case ResponseType.File:
var file = ReplaceValues(downloadPath, data);
using (var stream = File.Create(file)) { response.ToMemoryStream().CopyTo(stream); }
data.Log(new LogEntry("File saved as " + file, Colors.Green));
break;
default:
break;
}
}
catch (Exception ex)
{
data.Log(new LogEntry(ex.Message, Colors.White));
if (ex.GetType() == typeof(HttpException))
{
data.ResponseCode = ((HttpException)ex).HttpStatusCode.ToString();
data.Log(new LogEntry("Status code: " + data.ResponseCode, Colors.Cyan));
}
if (!data.ConfigSettings.IgnoreResponseErrors)
{
throw;
}
}
#endregion
}
/// <inheritdoc />
public override void Process(BotData data)
{
base.Process(data);
#region Request
// Set base URL
var localUrl = ReplaceValues(url, data);
var cType = ReplaceValues(contentType, data);
var oldJar = data.Cookies;
// Create request
HttpRequest request = new HttpRequest();
// Setup options
var timeout = data.GlobalSettings.General.RequestTimeout * 1000;
request.IgnoreProtocolErrors = true;
request.AllowAutoRedirect = autoRedirect;
request.EnableEncodingContent = acceptEncoding;
request.ReadWriteTimeout = timeout;
request.ConnectTimeout = timeout;
request.KeepAlive = true;
request.MaximumAutomaticRedirections = data.ConfigSettings.MaxRedirects;
// Check if it has GET parameters
if (ParseQuery && localUrl.Contains('?') && localUrl.Contains('='))
{
// Remove the query from the base URL
localUrl = ReplaceValues(url.Split('?')[0], data);
data.Log(new LogEntry($"Calling Base URL: {localUrl}", Colors.MediumTurquoise));
// Parse the GET parameters
var getParams = ReplaceValues(url.Split('?')[1], data);
var paramList = getParams.Split('&');
// Build the query, first replace variables in them and encode the parameters
foreach (var par in paramList)
{
var split = par.Split('=');
// Encode them if needed
if (split[0].Contains('%'))
{
split[0] = Uri.EscapeDataString(split[0]);
}
if (split[1].Contains('%'))
{
split[1] = Uri.EscapeDataString(split[1]);
}
// Add them to the query
request.AddUrlParam(split[0], split[1]);
data.Log(new LogEntry($"Added Query Parameter: {split[0]} = {split[1]}", Colors.MediumTurquoise));
}
}
else
{
data.Log(new LogEntry($"Calling URL: {localUrl}", Colors.MediumTurquoise));
}
// Set up the Content and Content-Type
HttpContent content = null;
switch (requestType)
{
case RequestType.Standard:
var pData = ReplaceValues(Regex.Replace(postData, @"(?<!\\)\\n", Environment.NewLine).Replace(@"\\n", @"\n"), data);
if (CanContainBody(method))
{
if (encodeContent)
{
// Very dirty but it works
var nonce = data.Random.Next(1000000, 9999999);
pData = pData.Replace("&", $"{nonce}&{nonce}").Replace("=", $"{nonce}={nonce}");
pData = string.Join("", BlockFunction.SplitInChunks(pData, 2080)
.Select(s => Uri.EscapeDataString(s)))
.Replace($"{nonce}%26{nonce}", "&").Replace($"{nonce}%3D{nonce}", "=");
}
content = new StringContent(pData);
content.ContentType = cType;
data.Log(new LogEntry($"Post Data: {pData}", Colors.MediumTurquoise));
}
break;
case RequestType.Multipart:
var bdry = multipartBoundary != "" ? ReplaceValues(multipartBoundary, data) : GenerateMultipartBoundary();
content = new Extreme.Net.MultipartContent(bdry);
var mContent = content as Extreme.Net.MultipartContent;
data.Log(new LogEntry($"Content-Type: multipart/form-data; boundary={bdry}", Colors.MediumTurquoise));
data.Log(new LogEntry("Multipart Data:", Colors.MediumTurquoise));
data.Log(new LogEntry(bdry, Colors.MediumTurquoise));
foreach (var c in MultipartContents)
{
var rValue = ReplaceValues(c.Value, data);
var rName = ReplaceValues(c.Name, data);
var rContentType = ReplaceValues(c.ContentType, data);
if (c.Type == MultipartContentType.String)
{
mContent.Add(new StringContent(rValue), rName);
data.Log(new LogEntry($"Content-Disposition: form-data; name=\"{rName}\"{Environment.NewLine}{Environment.NewLine}{rValue}", Colors.MediumTurquoise));
}
else if (c.Type == MultipartContentType.File)
{
mContent.Add(new FileContent(rValue), rName, rValue, rContentType);
data.Log(new LogEntry($"Content-Disposition: form-data; name=\"{rName}\"; filename=\"{rValue}\"{Environment.NewLine}Content-Type: {rContentType}{Environment.NewLine}{Environment.NewLine}[FILE CONTENT OMITTED]", Colors.MediumTurquoise));
}
data.Log(new LogEntry(bdry, Colors.MediumTurquoise));
}
break;
default:
break;
}
// Set proxy
if (data.UseProxies)
{
request.Proxy = data.Proxy.GetClient();
try
{
request.Proxy.ReadWriteTimeout = timeout;
request.Proxy.ConnectTimeout = timeout;
request.Proxy.Username = data.Proxy.Username;
request.Proxy.Password = data.Proxy.Password;
}
catch { }
}
// Set headers
data.Log(new LogEntry("Sent Headers:", Colors.DarkTurquoise));
// var fixedNames = Enum.GetNames(typeof(HttpHeader)).Select(n => n.ToLower());
foreach (var header in CustomHeaders)
{
try
{
var key = ReplaceValues(header.Key, data);
var replacedKey = key.Replace("-", "").ToLower(); // Used to compare with the HttpHeader enum
var val = ReplaceValues(header.Value, data);
if (replacedKey == "contenttype" && content != null)
{
continue;
} // Disregard additional Content-Type headers
if (replacedKey == "acceptencoding" && acceptEncoding)
{
continue;
} // Disregard additional Accept-Encoding headers
// else if (fixedNames.Contains(replacedKey)) request.AddHeader((HttpHeader)Enum.Parse(typeof(HttpHeader), replacedKey, true), val);
else
{
request.AddHeader(key, val);
}
data.Log(new LogEntry(key + ": " + val, Colors.MediumTurquoise));
}
catch { }
}
// Add the authorization header on a Basic Auth request
if (requestType == RequestType.BasicAuth)
{
var usr = ReplaceValues(authUser, data);
var pwd = ReplaceValues(authPass, data);
var auth = "Basic " + BlockFunction.Base64Encode(usr + ":" + pwd);
request.AddHeader("Authorization", auth);
data.Log(new LogEntry($"Authorization: {auth}", Colors.MediumTurquoise));
}
// Add the content-type header
if (CanContainBody(method) && content != null && requestType == RequestType.Standard)
{
data.Log(new LogEntry($"Content-Type: {cType}", Colors.MediumTurquoise));
}
// Add new user-defined custom cookies to the bot's cookie jar
request.Cookies = new CookieDictionary();
foreach (var cookie in CustomCookies)
{
data.Cookies[ReplaceValues(cookie.Key, data)] = ReplaceValues(cookie.Value, data);
}
// Set cookies from the bot's cookie jar to the request's CookieDictionary
data.Log(new LogEntry("Sent Cookies:", Colors.MediumTurquoise));
foreach (var cookie in data.Cookies)
{
request.Cookies.Add(cookie.Key, cookie.Value);
data.Log(new LogEntry($"{cookie.Key}: {cookie.Value}", Colors.MediumTurquoise));
}
data.LogNewLine();
#endregion
#region Response
// Create the response
HttpResponse response = null;
try
{
// Get response
response = request.Raw(method, localUrl, content);
var responseString = "";
// Get address
data.Address = response.Address.ToString();
data.Log(new LogEntry("Address: " + data.Address, Colors.Cyan));
// Get code
data.ResponseCode = ((int)response.StatusCode).ToString();
data.Log(new LogEntry($"Response code: {data.ResponseCode} ({response.StatusCode})", Colors.Cyan));
// Get headers
data.Log(new LogEntry("Received headers:", Colors.DeepPink));
var headerList = new List <KeyValuePair <string, string> >();
var receivedHeaders = response.EnumerateHeaders();
data.ResponseHeaders.Clear();
while (receivedHeaders.MoveNext())
{
var header = receivedHeaders.Current;
data.ResponseHeaders.Add(header.Key, header.Value);
data.Log(new LogEntry($"{header.Key}: {header.Value}", Colors.LightPink));
}
if (!response.ContainsHeader(HttpHeader.ContentLength) && ResponseType != ResponseType.File)
{
responseString = response.ToString(); // Read the stream
if (data.ResponseHeaders.ContainsKey("Content-Encoding") && data.ResponseHeaders["Content-Encoding"].Contains("gzip"))
{
data.ResponseHeaders["Content-Length"] = GZip.Zip(responseString).Length.ToString();
}
else
{
data.ResponseHeaders["Content-Length"] = responseString.Length.ToString();
}
data.Log(new LogEntry($"Content-Length: {data.ResponseHeaders["Content-Length"]}", Colors.LightPink));
}
// Get cookies
data.Log(new LogEntry("Received cookies:", Colors.Goldenrod));
data.Cookies = response.Cookies;
foreach (var cookie in response.Cookies)
{
// If the cookie was already present before, don't log it
if (oldJar.ContainsKey(cookie.Key) && oldJar[cookie.Key] == cookie.Value)
{
continue;
}
data.Log(new LogEntry($"{cookie.Key}: {cookie.Value}", Colors.LightGoldenrodYellow));
}
// Save the response content
switch (responseType)
{
case ResponseType.String:
data.Log(new LogEntry("Response Source:", Colors.Green));
if (readResponseSource)
{
if (responseString == "")
{
responseString = response.ToString(); // Read the stream if you didn't already read it
}
data.ResponseSource = responseString;
data.Log(new LogEntry(data.ResponseSource, Colors.GreenYellow));
}
else
{
data.ResponseSource = "";
data.Log(new LogEntry("[SKIPPED]", Colors.GreenYellow));
}
break;
case ResponseType.File:
if (SaveAsScreenshot)
{
SaveScreenshot(response.ToMemoryStream(), data); // Read the stream
data.Log(new LogEntry("File saved as screenshot", Colors.Green));
}
else
{
var filePath = ReplaceValues(downloadPath, data);
var dirName = Path.GetDirectoryName(filePath);
if (dirName != "")
{
dirName += Path.DirectorySeparatorChar.ToString();
}
var fileName = Path.GetFileNameWithoutExtension(filePath);
var fileExtension = Path.GetExtension(filePath);
var sanitizedPath = $"{dirName}{MakeValidFileName(fileName)}{fileExtension}";
using (var stream = File.Create(sanitizedPath)) { response.ToMemoryStream().CopyTo(stream); } // Read the stream
data.Log(new LogEntry("File saved as " + sanitizedPath, Colors.Green));
}
break;
default:
break;
}
}
catch (Exception ex)
{
data.Log(new LogEntry(ex.Message, Colors.White));
if (ex.GetType() == typeof(HttpException))
{
data.ResponseCode = ((HttpException)ex).HttpStatusCode.ToString();
data.Log(new LogEntry("Status code: " + data.ResponseCode, Colors.Cyan));
}
if (!data.ConfigSettings.IgnoreResponseErrors)
{
throw;
}
}
#endregion
}
