/// <inheritdoc />
public override void Process(BotData data)
{
base.Process(data);
#region Request
// Set base URL
var localUrl = ReplaceValues(url, data);
var cType = ReplaceValues(contentType, data);
var oldJar = data.Cookies;
// Create request
HttpRequest request = new HttpRequest();
// Setup options
var timeout = data.GlobalSettings.General.RequestTimeout * 1000;
request.IgnoreProtocolErrors = true;
request.AllowAutoRedirect = autoRedirect;
request.EnableEncodingContent = acceptEncoding;
request.ReadWriteTimeout = timeout;
request.ConnectTimeout = timeout;
request.KeepAlive = true;
request.MaximumAutomaticRedirections = data.ConfigSettings.MaxRedirects;
// Check if it has GET parameters
if (ParseQuery && localUrl.Contains('?') && localUrl.Contains('='))
{
// Remove the query from the base URL
localUrl = ReplaceValues(url.Split('?')[0], data);
data.Log(new LogEntry($"Calling Base URL: {localUrl}", Colors.MediumTurquoise));
// Parse the GET parameters
var getParams = ReplaceValues(url.Split('?')[1], data);
var paramList = getParams.Split('&');
// Build the query, first replace variables in them and encode the parameters
foreach (var par in paramList)
{
var split = par.Split('=');
// Encode them if needed
if (split[0].Contains('%'))
{
split[0] = Uri.EscapeDataString(split[0]);
}
if (split[1].Contains('%'))
{
split[1] = Uri.EscapeDataString(split[1]);
}
// Add them to the query
request.AddUrlParam(split[0], split[1]);
data.Log(new LogEntry($"Added Query Parameter: {split[0]} = {split[1]}", Colors.MediumTurquoise));
}
}
else
{
data.Log(new LogEntry($"Calling URL: {localUrl}", Colors.MediumTurquoise));
}
// Set up the Content and Content-Type
HttpContent content = null;
switch (requestType)
{
case RequestType.Standard:
var pData = ReplaceValues(Regex.Replace(postData, @"(?<!\\)\\n", Environment.NewLine).Replace(@"\\n", @"\n"), data);
if (CanContainBody(method))
{
if (encodeContent)
{
// Very dirty but it works
var nonce = data.Random.Next(1000000, 9999999);
pData = pData.Replace("&", $"{nonce}&{nonce}").Replace("=", $"{nonce}={nonce}");
pData = string.Join("", BlockFunction.SplitInChunks(pData, 2080)
.Select(s => Uri.EscapeDataString(s)))
.Replace($"{nonce}%26{nonce}", "&").Replace($"{nonce}%3D{nonce}", "=");
}
content = new StringContent(pData);
content.ContentType = cType;
data.Log(new LogEntry($"Post Data: {pData}", Colors.MediumTurquoise));
}
break;
case RequestType.Multipart:
var bdry = multipartBoundary != "" ? ReplaceValues(multipartBoundary, data) : GenerateMultipartBoundary();
content = new Extreme.Net.MultipartContent(bdry);
var mContent = content as Extreme.Net.MultipartContent;
data.Log(new LogEntry($"Content-Type: multipart/form-data; boundary={bdry}", Colors.MediumTurquoise));
data.Log(new LogEntry("Multipart Data:", Colors.MediumTurquoise));
data.Log(new LogEntry(bdry, Colors.MediumTurquoise));
foreach (var c in MultipartContents)
{
var rValue = ReplaceValues(c.Value, data);
var rName = ReplaceValues(c.Name, data);
var rContentType = ReplaceValues(c.ContentType, data);
if (c.Type == MultipartContentType.String)
{
mContent.Add(new StringContent(rValue), rName);
data.Log(new LogEntry($"Content-Disposition: form-data; name=\"{rName}\"{Environment.NewLine}{Environment.NewLine}{rValue}", Colors.MediumTurquoise));
}
else if (c.Type == MultipartContentType.File)
{
mContent.Add(new FileContent(rValue), rName, rValue, rContentType);
data.Log(new LogEntry($"Content-Disposition: form-data; name=\"{rName}\"; filename=\"{rValue}\"{Environment.NewLine}Content-Type: {rContentType}{Environment.NewLine}{Environment.NewLine}[FILE CONTENT OMITTED]", Colors.MediumTurquoise));
}
data.Log(new LogEntry(bdry, Colors.MediumTurquoise));
}
break;
default:
break;
}
// Set proxy
if (data.UseProxies)
{
request.Proxy = data.Proxy.GetClient();
try
{
request.Proxy.ReadWriteTimeout = timeout;
request.Proxy.ConnectTimeout = timeout;
request.Proxy.Username = data.Proxy.Username;
request.Proxy.Password = data.Proxy.Password;
}
catch { }
}
// Set headers
data.Log(new LogEntry("Sent Headers:", Colors.DarkTurquoise));
// var fixedNames = Enum.GetNames(typeof(HttpHeader)).Select(n => n.ToLower());
foreach (var header in CustomHeaders)
{
try
{
var key = ReplaceValues(header.Key, data);
var replacedKey = key.Replace("-", "").ToLower(); // Used to compare with the HttpHeader enum
var val = ReplaceValues(header.Value, data);
if (replacedKey == "contenttype" && content != null)
{
continue;
} // Disregard additional Content-Type headers
if (replacedKey == "acceptencoding" && acceptEncoding)
{
continue;
} // Disregard additional Accept-Encoding headers
// else if (fixedNames.Contains(replacedKey)) request.AddHeader((HttpHeader)Enum.Parse(typeof(HttpHeader), replacedKey, true), val);
else
{
request.AddHeader(key, val);
}
data.Log(new LogEntry(key + ": " + val, Colors.MediumTurquoise));
}
catch { }
}
// Add the authorization header on a Basic Auth request
if (requestType == RequestType.BasicAuth)
{
var usr = ReplaceValues(authUser, data);
var pwd = ReplaceValues(authPass, data);
var auth = "Basic " + BlockFunction.Base64Encode(usr + ":" + pwd);
request.AddHeader("Authorization", auth);
data.Log(new LogEntry($"Authorization: {auth}", Colors.MediumTurquoise));
}
// Add the content-type header
if (CanContainBody(method) && content != null && requestType == RequestType.Standard)
{
data.Log(new LogEntry($"Content-Type: {cType}", Colors.MediumTurquoise));
}
// Add new user-defined custom cookies to the bot's cookie jar
request.Cookies = new CookieDictionary();
foreach (var cookie in CustomCookies)
{
data.Cookies[ReplaceValues(cookie.Key, data)] = ReplaceValues(cookie.Value, data);
}
// Set cookies from the bot's cookie jar to the request's CookieDictionary
data.Log(new LogEntry("Sent Cookies:", Colors.MediumTurquoise));
foreach (var cookie in data.Cookies)
{
request.Cookies.Add(cookie.Key, cookie.Value);
data.Log(new LogEntry($"{cookie.Key}: {cookie.Value}", Colors.MediumTurquoise));
}
data.LogNewLine();
#endregion
#region Response
// Create the response
HttpResponse response = null;
try
{
// Get response
response = request.Raw(method, localUrl, content);
var responseString = "";
// Get address
data.Address = response.Address.ToString();
data.Log(new LogEntry("Address: " + data.Address, Colors.Cyan));
// Get code
data.ResponseCode = ((int)response.StatusCode).ToString();
data.Log(new LogEntry($"Response code: {data.ResponseCode} ({response.StatusCode})", Colors.Cyan));
// Get headers
data.Log(new LogEntry("Received headers:", Colors.DeepPink));
var headerList = new List <KeyValuePair <string, string> >();
var receivedHeaders = response.EnumerateHeaders();
data.ResponseHeaders.Clear();
while (receivedHeaders.MoveNext())
{
var header = receivedHeaders.Current;
data.ResponseHeaders.Add(header.Key, header.Value);
data.Log(new LogEntry($"{header.Key}: {header.Value}", Colors.LightPink));
}
if (!response.ContainsHeader(HttpHeader.ContentLength) && ResponseType != ResponseType.File)
{
responseString = response.ToString(); // Read the stream
if (data.ResponseHeaders.ContainsKey("Content-Encoding") && data.ResponseHeaders["Content-Encoding"].Contains("gzip"))
{
data.ResponseHeaders["Content-Length"] = GZip.Zip(responseString).Length.ToString();
}
else
{
data.ResponseHeaders["Content-Length"] = responseString.Length.ToString();
}
data.Log(new LogEntry($"Content-Length: {data.ResponseHeaders["Content-Length"]}", Colors.LightPink));
}
// Get cookies
data.Log(new LogEntry("Received cookies:", Colors.Goldenrod));
data.Cookies = response.Cookies;
foreach (var cookie in response.Cookies)
{
// If the cookie was already present before, don't log it
if (oldJar.ContainsKey(cookie.Key) && oldJar[cookie.Key] == cookie.Value)
{
continue;
}
data.Log(new LogEntry($"{cookie.Key}: {cookie.Value}", Colors.LightGoldenrodYellow));
}
// Save the response content
switch (responseType)
{
case ResponseType.String:
data.Log(new LogEntry("Response Source:", Colors.Green));
if (readResponseSource)
{
if (responseString == "")
{
responseString = response.ToString(); // Read the stream if you didn't already read it
}
data.ResponseSource = responseString;
data.Log(new LogEntry(data.ResponseSource, Colors.GreenYellow));
}
else
{
data.ResponseSource = "";
data.Log(new LogEntry("[SKIPPED]", Colors.GreenYellow));
}
break;
case ResponseType.File:
if (SaveAsScreenshot)
{
SaveScreenshot(response.ToMemoryStream(), data); // Read the stream
data.Log(new LogEntry("File saved as screenshot", Colors.Green));
}
else
{
var filePath = ReplaceValues(downloadPath, data);
var dirName = Path.GetDirectoryName(filePath);
if (dirName != "")
{
dirName += Path.DirectorySeparatorChar.ToString();
}
var fileName = Path.GetFileNameWithoutExtension(filePath);
var fileExtension = Path.GetExtension(filePath);
var sanitizedPath = $"{dirName}{MakeValidFileName(fileName)}{fileExtension}";
using (var stream = File.Create(sanitizedPath)) { response.ToMemoryStream().CopyTo(stream); } // Read the stream
data.Log(new LogEntry("File saved as " + sanitizedPath, Colors.Green));
}
break;
default:
break;
}
}
catch (Exception ex)
{
data.Log(new LogEntry(ex.Message, Colors.White));
if (ex.GetType() == typeof(HttpException))
{
data.ResponseCode = ((HttpException)ex).HttpStatusCode.ToString();
data.Log(new LogEntry("Status code: " + data.ResponseCode, Colors.Cyan));
}
if (!data.ConfigSettings.IgnoreResponseErrors)
{
throw;
}
}
#endregion
}
/// <inheritdoc />
public override void Process(BotData data)
{
base.Process(data);
#region Request
// Set base URL
var localUrl = ReplaceValues(url, data);
var cType = ReplaceValues(contentType, data);
// Create request
HttpRequest request = new HttpRequest();
// Setup options
var timeout = data.GlobalSettings.General.RequestTimeout * 1000;
request.IgnoreProtocolErrors = true;
request.AllowAutoRedirect = autoRedirect;
request.EnableEncodingContent = true;
request.ReadWriteTimeout = timeout;
request.ConnectTimeout = timeout;
request.KeepAlive = true;
request.MaximumAutomaticRedirections = data.ConfigSettings.MaxRedirects;
// Check if it has GET parameters
if (ParseQuery && localUrl.Contains('?') && localUrl.Contains('='))
{
// Remove the query from the base URL
localUrl = ReplaceValues(url.Split('?')[0], data);
data.Log(new LogEntry($"Calling Base URL: {localUrl}", Colors.MediumTurquoise));
// Parse the GET parameters
var getParams = ReplaceValues(url.Split('?')[1], data);
var paramList = getParams.Split('&');
// Build the query, first replace variables in them and encode the parameters
foreach (var par in paramList)
{
var split = par.Split('=');
// Encode them if needed
if (split[0].Contains('%'))
{
split[0] = Uri.EscapeDataString(split[0]);
}
if (split[1].Contains('%'))
{
split[1] = Uri.EscapeDataString(split[1]);
}
// Add them to the query
request.AddUrlParam(split[0], split[1]);
data.Log(new LogEntry($"Added Query Parameter: {split[0]} = {split[1]}", Colors.MediumTurquoise));
}
}
else
{
data.Log(new LogEntry($"Calling URL: {localUrl}", Colors.MediumTurquoise));
}
// Set up the Content and Content-Type
HttpContent content = null;
switch (requestType)
{
case RequestType.Standard:
var pData = string.Join(Environment.NewLine, postData
.Split(new string[] { "\\n" }, StringSplitOptions.None)
.Select(p => ReplaceValues(p, data)));
if (pData != "")
{
if (encodeContent)
{
// Very dirty but it works
var nonce = data.rand.Next(1000000, 9999999);
pData = pData.Replace("&", $"{nonce}&{nonce}").Replace("=", $"{nonce}={nonce}");
pData = System.Uri.EscapeDataString(pData).Replace($"{nonce}%26{nonce}", "&").Replace($"{nonce}%3D{nonce}", "=");
}
content = new StringContent(pData);
content.ContentType = cType;
data.Log(new LogEntry(string.Format("Post Data: {0}", pData), Colors.MediumTurquoise));
}
break;
case RequestType.Multipart:
if (multipartBoundary != "")
{
content = new Extreme.Net.MultipartContent(multipartBoundary);
}
else
{
content = new Extreme.Net.MultipartContent(GenerateMultipartBoundary());
}
var mContent = content as Extreme.Net.MultipartContent;
foreach (var c in MultipartContents)
{
if (c.Type == MultipartContentType.String)
{
mContent.Add(new StringContent(ReplaceValues(c.Value, data)), ReplaceValues(c.Name, data));
}
else if (c.Type == MultipartContentType.File)
{
mContent.Add(new FileContent(ReplaceValues(c.Value, data)), ReplaceValues(c.Name, data));
}
}
break;
default:
break;
}
// Set proxy
if (data.UseProxies)
{
request.Proxy = data.Proxy.GetClient();
try
{
request.Proxy.ReadWriteTimeout = timeout;
request.Proxy.ConnectTimeout = timeout;
request.Proxy.Username = data.Proxy.Username;
request.Proxy.Password = data.Proxy.Password;
}
catch { }
}
// Set headers
data.Log(new LogEntry("Sent Headers:", Colors.DarkTurquoise));
var fixedNames = Enum.GetNames(typeof(HttpHeader)).Select(n => n.ToLower());
foreach (var header in CustomHeaders)
{
try
{
var key = ReplaceValues(header.Key, data);
var replacedKey = key.Replace("-", "").ToLower(); // Used to compare with the HttpHeader enum
var val = ReplaceValues(header.Value, data);
if (replacedKey == "contenttype") // If it's somewhat needed to define Content-Type in e.g. a GET request
{
content = new StringContent("");
content.ContentType = val;
}
else if (fixedNames.Contains(replacedKey))
{
request.AddHeader((HttpHeader)Enum.Parse(typeof(HttpHeader), replacedKey, true), val);
}
else
{
request.AddHeader(key, val);
}
data.Log(new LogEntry(key + ": " + val, Colors.MediumTurquoise));
}
catch { }
}
// Add the authorization header on a Basic Auth request
if (requestType == RequestType.BasicAuth)
{
var usr = ReplaceValues(authUser, data);
var pwd = ReplaceValues(authPass, data);
var auth = "Basic " + BlockFunction.Base64Encode(usr + ":" + pwd);
request.AddHeader("Authorization", auth);
data.Log(new LogEntry("Authorization: " + auth, Colors.MediumTurquoise));
}
// Add the content-type header
if ((method == HttpMethod.POST || method == HttpMethod.PUT || method == HttpMethod.DELETE) && cType != "")
{
data.Log(new LogEntry("Content-Type: " + cType, Colors.MediumTurquoise));
}
// Add new user-defined custom cookies to the bot's cookie jar
request.Cookies = new CookieDictionary();
foreach (var cookie in CustomCookies)
{
data.Cookies[ReplaceValues(cookie.Key, data)] = ReplaceValues(cookie.Value, data);
}
// Set cookies from the bot's cookie jar to the request's CookieDictionary
data.Log(new LogEntry("Sent Cookies:", Colors.MediumTurquoise));
foreach (var cookie in data.Cookies)
{
request.Cookies.Add(cookie.Key, cookie.Value);
data.Log(new LogEntry(cookie.Key + " : " + cookie.Value, Colors.MediumTurquoise));
}
data.LogNewLine();
#endregion
#region Response
// Create the response
HttpResponse response = null;
try
{
// Get response
response = request.Raw(method, localUrl, content);
// Get address
data.Address = response.Address.ToString();
data.Log(new LogEntry("Address: " + data.Address, Colors.Cyan));
// Get code
data.ResponseCode = ((int)response.StatusCode).ToString();
data.Log(new LogEntry("Response code: " + data.ResponseCode, Colors.Cyan));
// Get headers
data.Log(new LogEntry("Received headers:", Colors.DeepPink));
var headerList = new List <KeyValuePair <string, string> >();
var receivedHeaders = response.EnumerateHeaders();
data.ResponseHeaders.Clear();
while (receivedHeaders.MoveNext())
{
var header = receivedHeaders.Current;
data.ResponseHeaders.Add(header.Key, header.Value);
data.Log(new LogEntry(header.Key + ": " + header.Value, Colors.LightPink));
}
// Get cookies
data.Log(new LogEntry("Received cookies:", Colors.Goldenrod));
data.Cookies = response.Cookies;
foreach (var cookie in response.Cookies)
{
data.Log(new LogEntry(cookie.Key + ": " + cookie.Value, Colors.LightGoldenrodYellow));
}
// Save the response content
switch (responseType)
{
case ResponseType.String:
data.Log(new LogEntry("Response Source:", Colors.Green));
if (readResponseSource)
{
data.ResponseSource = response.ToString();
data.Log(new LogEntry(data.ResponseSource, Colors.GreenYellow));
}
else
{
data.ResponseSource = "";
data.Log(new LogEntry("[SKIPPED]", Colors.GreenYellow));
}
break;
case ResponseType.File:
var file = ReplaceValues(downloadPath, data);
using (var stream = File.Create(file)) { response.ToMemoryStream().CopyTo(stream); }
data.Log(new LogEntry("File saved as " + file, Colors.Green));
break;
default:
break;
}
}
catch (Exception ex)
{
data.Log(new LogEntry(ex.Message, Colors.White));
if (ex.GetType() == typeof(HttpException))
{
data.ResponseCode = ((HttpException)ex).HttpStatusCode.ToString();
data.Log(new LogEntry("Status code: " + data.ResponseCode, Colors.Cyan));
}
if (!data.ConfigSettings.IgnoreResponseErrors)
{
throw;
}
}
#endregion
}
/// <inheritdoc />
public override void Process(BotData data)
{
base.Process(data);
// Setup
var request = new Request();
request.Setup(data.GlobalSettings, securityProtocol, AutoRedirect, data.ConfigSettings.MaxRedirects, AcceptEncoding);
var localUrl = ReplaceValues(Url, data);
data.Log(new LogEntry($"Calling URL: {localUrl}", Colors.MediumTurquoise));
// Set content
switch (RequestType)
{
case RequestType.Standard:
request.SetStandardContent(ReplaceValues(PostData, data), ReplaceValues(ContentType, data), Method, EncodeContent, GetLogBuffer(data));
break;
case RequestType.BasicAuth:
request.SetBasicAuth(ReplaceValues(AuthUser, data), ReplaceValues(AuthPass, data));
break;
case RequestType.Multipart:
var contents = MultipartContents.Select(m =>
new MultipartContent()
{
Name = ReplaceValues(m.Name, data),
Value = ReplaceValues(m.Value, data),
ContentType = ReplaceValues(m.Value, data),
Type = m.Type
});
request.SetMultipartContent(contents, ReplaceValues(MultipartBoundary, data), GetLogBuffer(data));
break;
case RequestType.Raw:
request.SetRawContent(ReplaceValues(RawData, data), ReplaceValues(ContentType, data), Method, GetLogBuffer(data));
break;
}
// Set proxy
if (data.UseProxies)
{
request.SetProxy(data.Proxy);
}
// Set headers
data.Log(new LogEntry("Sent Headers:", Colors.DarkTurquoise));
var headers = CustomHeaders.Select(h =>
new KeyValuePair <string, string>(ReplaceValues(h.Key, data), ReplaceValues(h.Value, data))
).ToDictionary(h => h.Key, h => h.Value);
request.SetHeaders(headers, AcceptEncoding, GetLogBuffer(data));
// Set cookies
data.Log(new LogEntry("Sent Cookies:", Colors.MediumTurquoise));
foreach (var cookie in CustomCookies) // Add new user-defined custom cookies to the bot's cookie jar
{
data.Cookies[ReplaceValues(cookie.Key, data)] = ReplaceValues(cookie.Value, data);
}
request.SetCookies(data.Cookies, GetLogBuffer(data));
// End the request part
data.LogNewLine();
// Perform the request
try
{
(data.Address, data.ResponseCode, data.ResponseHeaders, data.Cookies) = request.Perform(localUrl, Method, GetLogBuffer(data));
}
catch (Exception ex)
{
if (data.ConfigSettings.IgnoreResponseErrors)
{
data.Log(new LogEntry(ex.Message, Colors.Tomato));
data.ResponseSource = ex.Message;
return;
}
throw;
}
// Save the response content
switch (ResponseType)
{
case ResponseType.String:
data.ResponseSource = request.SaveString(ReadResponseSource, data.ResponseHeaders, GetLogBuffer(data));
break;
case ResponseType.File:
if (SaveAsScreenshot)
{
Files.SaveScreenshot(request.GetResponseStream(), data); // Read the stream
data.Log(new LogEntry("File saved as screenshot", Colors.Green));
}
else
{
request.SaveFile(ReplaceValues(DownloadPath, data), GetLogBuffer(data));
}
break;
case ResponseType.Base64String:
var base64 = Convert.ToBase64String(request.GetResponseStream().ToArray());
InsertVariable(data, false, base64, OutputVariable);
break;
default:
break;
}
}