public AttackResult Check(AttackRequest request)
{
var result = new AttackResult();
foreach (var fua in _fileUploads)
{
var webRequest = new WebPageRequest();
webRequest.Address = request.URL + fua.FilePath;
webRequest.Method = fua.HTTPVerb;
webRequest.RequestBody = fua.Body;
webRequest.Log = true;
WebPageLoader.Load(webRequest);
int responseCode = 0;
int.TryParse(webRequest.Response.Code, out responseCode);
if (fua.SuccessResponseHTTPCode.Contains(responseCode))
{
result.Success = true;
result.Results.Enqueue("CVE-2017-12615 success: " + webRequest.Address);
}
}
return(result);
}
public static List <string> Gather_FindSubdomains(string domain)
{
List <string> subdomains = new List <string>();
WebPageRequest request = new WebPageRequest();
request.Address = "https://findsubdomains.com/subdomains-of/" + domain;
WebPageLoader.Load(request);
HtmlDocument htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(request.Response.Body);
var divNodes = htmlDoc.DocumentNode.SelectNodes("//div[contains(@class,'js-domain-name')]");
if (divNodes != null)
{
foreach (HtmlNode div in divNodes)
{
subdomains.Add(Regex.Replace(div.InnerText, @"\n|\s+", ""));
}
}
return(subdomains);
}
private void _btnWebArchiveTest_Click(object sender, EventArgs e)
{
string address = _txtDomain.Text;
StringBuilder sb = new StringBuilder();
try
{
sb.Append("Starting WebArchive Test: " + address + Environment.NewLine);
WebPageRequest request = new WebPageRequest();
request.Address = address;
// WebPageLoader.Load(request);
CheckForFileType(request.Address, sb, "html");
}
catch (Exception ex)
{
string inner = "";
if (ex.InnerException != null)
{
inner = ex.InnerException.Message;
}
sb.Append("!!!!!Exception: " + ex.Message + " Inner: " + inner);
}
LogTest(sb.ToString());
}
private static List <string> Gather_ThreatCrowd(string domain)
{
List <string> subdomains = new List <string>();
WebPageRequest request = new WebPageRequest();
request.Address = "https://www.threatcrowd.org/searchApi/v2/domain/report/?domain=" + domain;
WebPageLoader.Load(request);
if (String.IsNullOrEmpty(request.Response.Body))
{
return(subdomains);
}
dynamic d = JObject.Parse(request.Response.Body);
if (String.IsNullOrEmpty(request.Response.Body))
{
return(subdomains);
}
if (d.subdomains != null)
{
foreach (string subdomain in d.subdomains)
{
subdomains.Add(subdomain);
}
}
return(subdomains);
}
public static ScannerResult Check(ScannerRequest request)
{
if (_knownAttackFiles.Count == 0)
{
lock (_syncObject)
{
if (_knownAttackFiles.Count == 0)
{
Initialize();
}
}
}
ScannerResult result = new ScannerResult();
List <string> returnList = new List <string>();
string testedFile = request.URL.Trim('/') + "/lkfkjsalkalkln3nfioaoisf0090cvlklkvkllkalk";
WebPageRequest webRequest = new WebPageRequest(testedFile);
//WebPageLoader.Load(webRequest);
//if (webRequest.Response.Code.Equals("200"))
//{
// return result;
//}
foreach (AttackFile attack in _knownAttackFiles)
{
testedFile = request.URL.Trim('/') + "/" + attack.File;
webRequest = new WebPageRequest(testedFile);
WebPageLoader.Load(webRequest);
if (webRequest.Response.Code.Equals("200"))
{
bool anyFingerPrint = false;
foreach (string fp in attack.FingerPrint)
{
if (webRequest.Response.Body.Contains(fp))
{
anyFingerPrint = true;
break;
}
}
if (anyFingerPrint)
{
result.Success = true;
string attackString = attack.Attacks.FirstOrDefault();
if (!String.IsNullOrEmpty(attackString))
{
testedFile = testedFile + attackString;
}
returnList.Add(testedFile);
}
}
}
result.Results.AddRange(returnList);
return(result);
}
public AttackResult Check(AttackRequest request)
{
var sResult = new AttackResult();
WebPageRequest webRequest = new WebPageRequest(request.URL.Trim('/'));
webRequest.Log = true;
webRequest.LogDir = request.LogDir;
foreach (var attack in _headers)
{
webRequest.Headers = attack.AttackHeaderCollection;
WebPageLoader.Load(webRequest);
foreach (var headerFP in attack.FingerPrintHeaders.AllKeys)
{
var header = webRequest.Response.Headers.Get(headerFP);
if (header != null)
{
if (attack.FingerPrintHeaders[headerFP] == webRequest.Response.Headers[headerFP])
{
sResult.Success = true;
sResult.Results.Enqueue("URL: " + request.URL + " ::: Header=" + headerFP);
}
}
}
}
return(sResult);
}
private void FindFile(string url, VulnerableFile vuln, AttackResult result)
{
string testedFile = url.Trim('/') + "/" + vuln.File;
WebPageRequest webRequest = new WebPageRequest(testedFile);
WebPageLoader.Load(webRequest);
if (webRequest.Response.Code.Equals("200"))
{
bool anyFingerPrint = false;
foreach (string fp in vuln.FingerPrint)
{
if (webRequest.Response.Body.Contains(fp))
{
anyFingerPrint = true;
break;
}
}
if (anyFingerPrint)
{
result.Success = true;
string attackString = vuln.Attacks.FirstOrDefault();
if (!String.IsNullOrEmpty(attackString))
{
testedFile = testedFile + attackString;
}
result.Results.Enqueue(testedFile);
}
}
}
public static ScannerResult Check(ScannerRequest request)
{
if (_fileNames.Count == 0)
{
lock (_syncObject)
{
if (_fileNames.Count == 0)
{
Initialize();
}
}
}
ScannerResult result = new ScannerResult();
foreach (string fileName in _fileNames)
{
string testedFile = request.URL.Trim('/') + "/" + fileName;
WebPageRequest webRequest = new WebPageRequest(testedFile);
WebPageLoader.Load(webRequest);
if (Check_Contents(webRequest.Response.Body))
{
result.Success = true;
result.Results.Add(testedFile);
return(result);
}
}
return(result);
}
public static ScannerResult Check(ScannerRequest request)
{
if (_payloads.Count == 0)
{
lock (_syncObject)
{
if (_payloads.Count == 0)
{
Initialize();
}
}
}
ScannerResult result = new ScannerResult();
foreach (string payload in _payloads)
{
string testUrl = request.URL.Trim('/') + "/" + payload;
WebPageRequest webRequest = new WebPageRequest(testUrl);
WebPageLoader.Load(webRequest);
if (webRequest.Response.Headers.AllKeys.Contains("headername"))
{
result.Success = true;
result.Results.Add(testUrl);
return(result);
}
}
return(result);
}
public AttackResult Check(AttackRequest request)
{
var result = new AttackResult();
WebPageRequest webRequest = new WebPageRequest(request.URL);
WebPageLoader.Load(webRequest);
List <string> links = new List <string>();
if (!String.IsNullOrEmpty(webRequest.Response.Body))
{
links = LinkFinder.Parse(webRequest.Response.Body, request.URL);
}
//Do javascript files too, expand library to check load/ajax/etc
foreach (var link in links)
{
var testLink = link.Remove(0, 5);
foreach (var fp in _fingerprints)
{
if (testLink.Contains(fp))
{
result.Success = true;
result.Results.Enqueue("Possible SSRF: " + link);
}
}
}
return(result);
}
private void _btnFindSubdomains_Click(object sender, EventArgs e)
{
//This site also offers an API 4 request/min and there is also limited info on a page
//try the api then try the page
List <string> subdomains = new List <string>();
WebPageRequest request = new WebPageRequest();
request.Address = "https://www.virustotal.com/vtapi/v2/domain/report?apikey=aff8183060bfe0fcfa977c9f5273b2d76eecdd2cec2b83a7c22d3fc105866894&domain=yahoo.com";
WebPageLoader.Load(request);
dynamic d = JObject.Parse(request.Response.Body);
if (d.subdomains != null)
{
foreach (string subdomain in d.subdomains)
{
subdomains.Add(subdomain);
}
}
if (subdomains.Count == 0)
{
request = new WebPageRequest();
request.Address = "https://www.virustotal.com/en/domain/yahoo.com/information";
request.CookieJar.Add(new System.Net.Cookie()
{
Domain = "www.virustotal.com",
HttpOnly = false,
Path = "/",
Name = "VT_PREFERRED_LANGUAGE",
Value = "en"
});
WebPageLoader.Load(request);
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(request.Response.Body);
var divNode = htmlDoc.DocumentNode.SelectSingleNode("//div[@id='observed-subdomains']");
if (divNode != null)
{
var aNodes = divNode.Descendants("a");
foreach (HtmlNode a in aNodes)
{
string found = Regex.Replace(a.InnerText, @"\n|\s+", "");
if (found.Contains("yahoo.com"))
{
subdomains.Add(found);
}
}
}
int x = 0;
}
}
private void CheckCRLF(string url, string payload, AttackResult result)
{
string testUrl = url.Trim('/') + "/" + payload;
WebPageRequest webRequest = new WebPageRequest(testUrl);
WebPageLoader.Load(webRequest);
if (webRequest.Response.Headers.AllKeys.Contains("headername"))
{
result.Success = true;
result.Results.Enqueue("Tested URL: " + testUrl);
}
}
private void FindLeak(string url, Leak leak, AttackResult result)
{
foreach (var fileName in leak.FileNames)
{
string testedFile = url + "/" + fileName;
WebPageRequest webRequest = new WebPageRequest(testedFile);
WebPageLoader.Load(webRequest);
if (Check_Contents(webRequest.Response.Body, leak.FingerPrints))
{
result.Success = true;
result.Results.Enqueue("Tested File: " + testedFile);
break;
}
}
}
public static bool Check(string domain)
{
WebPageRequest request = new WebPageRequest(domain.Trim('/') + "/my.policy");
WebPageLoader.Load(request);
if (request.Response.Code.Equals("200"))
{
WebPageRequest compareRequest = new WebPageRequest(domain.Trim('/') + "/asfasdfasdf");
WebPageLoader.Load(compareRequest);
if (request.Response.Body != compareRequest.Response.Body)
{
return(true);
}
}
return(false);
}
private void _btnSubdomainTakeoverTest_Click(object sender, EventArgs e)
{
string address = _txtDomain.Text;
StringBuilder sb = new StringBuilder();
try
{
sb.Append("Starting Subdomain Takeover Test : " + address + Environment.NewLine);
WebPageRequest request = new WebPageRequest();
request.Address = address;
WebPageLoader.Load(request);
if (request.Response.Body.Equals(String.Empty) && request.Response.TimeOut == false)
{
sb.Append("\tNo body found." + Environment.NewLine);
}
else
{
ScannerRequest sRequest = new ScannerRequest();
sRequest.Body = request.Response.Body;
sRequest.URL = address;
sRequest.Domain = DomainUtility.GetDomainFromUrl(address);
sb.Append("\tBody found." + Environment.NewLine);
if (request.Response.TimeOut == false)
{
CheckEngine(sRequest, sb);
}
else
{
sb.Append("\tTimed out" + Environment.NewLine);
// CheckBigIPService(request, sb);
}
}
}
catch (Exception ex)
{
string inner = "";
if (ex.InnerException != null)
{
inner = ex.InnerException.Message;
}
sb.Append("!!!!!Exception: " + ex.Message + " Inner: " + inner);
}
LogTest(sb.ToString());
}
public static ScannerResult Check(ScannerRequest request)
{
if (_knownAttackHeaders.Count == 0)
{
lock (_syncObject)
{
if (_knownAttackHeaders.Count == 0)
{
Initialize();
}
}
}
ScannerResult result = new ScannerResult();
List <string> returnList = new List <string>();
string testedFile = request.URL.Trim('/');
WebPageRequest webRequest = new WebPageRequest(testedFile);
webRequest = new WebPageRequest(request.URL.Trim('/'));
webRequest.Log = true;
webRequest.LogDir = request.LogDir;
foreach (AttackHeader attack in _knownAttackHeaders)
{
webRequest.Headers = attack.AttackHeaderCollection;
WebPageLoader.Load(webRequest);
foreach (var headerFP in attack.FingerPrintHeaders.AllKeys)
{
var header = webRequest.Response.Headers.Get(headerFP);
if (header != null)
{
if (attack.FingerPrintHeaders[headerFP] == webRequest.Response.Headers[headerFP])
{
returnList.Add(headerFP);
}
}
}
}
result.Results.AddRange(returnList);
return(result);
}
public static ScannerResult BucketCheck(ScannerRequest request)
{
if (_bucketURLRegex.Count == 0)
{
lock (_syncObject)
{
if (_bucketURLRegex.Count == 0)
{
Initilize();
}
}
}
ScannerResult result = new ScannerResult();
List <string> referencedBuckets = new List <string>();
List <string> bustedBuckets = new List <string>();
foreach (string search in _bucketURLRegex)
{
MatchCollection collection = Regex.Matches(request.Body, search);
referencedBuckets.AddRange(collection.Cast <Match>().Select(match => match.Value).ToList());
}
referencedBuckets = referencedBuckets.Distinct().ToList();
//todo: make better regex so we dont have to do this
referencedBuckets.RemoveAll(x => x.Trim('/') == @"http://s3.amazonaws.com");
referencedBuckets.RemoveAll(x => x.Trim('/') == @"https://s3.amazonaws.com");
foreach (string bucket in referencedBuckets)
{
WebPageRequest webRequest = new WebPageRequest(bucket);
WebPageLoader.Load(webRequest);
if (webRequest.Response.Body.Contains("<Code>NoSuchBucket</Code>"))
{
result.Success = true;
bustedBuckets.Add(bucket);
}
}
result.Results.AddRange(bustedBuckets);
return(result);
}
private static bool CheckURL(string startingDomain, string url)
{
string foundURL = DomainUtility.StripProtocol(url.Split('?')[0]);
if (CheckIfSocialMediaSite(startingDomain, foundURL))
{
//if (userNames.Count == 0)
//{
// if (!foundUrls.ContainsKey(foundURL))
// {
WebPageRequest request = new WebPageRequest(DomainUtility.EnsureHTTPS(foundURL).ToLower());
WebPageLoader.Load(request);
if (!request.Response.Code.Equals("200"))
{
return(true);
}
// else if (!returnOnlyNone200)
// foundUrls.Add(foundURL, url);
// }
//}
//else
//{
// foreach (string userName in userNames)
// {
// if (foundURL.ToLower().Contains(userName.ToLower()))
// {
// if (!foundUrls.ContainsKey(foundURL))
// {
// Request request = new Request(DomainUtility.EnsureHTTPS(foundURL));
// RequestUtility.GetWebText(request);
// if (!request.Response.Code.Equals("200"))
// foundUrls.Add(foundURL, url);
// else if (!returnOnlyNone200)
// foundUrls.Add(foundURL, url);
// }
// }
// }
//}
}
return(false);
}
public AttackResult Check(AttackRequest request)
{
var result = new AttackResult();
foreach (var appendedRedirect in _redirectAppends)
{
string testURL = request.URL.Trim('/') + appendedRedirect;
var webRequest = new WebPageRequest(testURL);
webRequest.FollowRedirects = true;
WebPageLoader.Load(webRequest);
if (webRequest.Response.Body.Contains("<title>Google</title>"))
{
result.Success = true;
result.Results.Enqueue("Open Redirect: " + testURL);
}
}
return(result);
}
private void CheckBigIPService(WebPageRequest request, StringBuilder sb, StringBuilder linkBuilder = null)
{
//to be checked only if it base directroy / times out
//https://twitter.com/_ayoubfathi_/status/1039070515690844160
bool bigIP = BigIP.Check(request.Address);
if (bigIP)
{
sb.Append("\tBig IP Service Found! " + request.Address + "/my.service" + " Email sent." + Environment.NewLine);
SendEmail("Big IP Service Found", request.Address + " appears to have a Big IP service running at " + request.Address + "/my.service");
if (linkBuilder != null)
{
linkBuilder.Append(request.Address + Environment.NewLine);
}
}
else
{
sb.Append("\tNo engine found." + Environment.NewLine);
}
}
private static void LogRequest(HttpWebRequest request, WebPageRequest webRequest)
{
List <string> nontransferableHeaders = new List <string>()
{
"Content-Type", "Referer"
};
StringBuilder sb = new StringBuilder();
sb.Append(request.Method);
sb.Append(" ");
sb.Append(request.Address);
sb.Append(Environment.NewLine);
sb.Append("Content-Type: ");
sb.Append(request.ContentType);
sb.Append(Environment.NewLine);
sb.Append("Referer: ");
sb.Append(request.Referer);
sb.Append(Environment.NewLine);
for (int i = 0; i < request.Headers.Count; ++i)
{
string header = request.Headers.GetKey(i);
if (nontransferableHeaders.Contains(header))
{
continue;
}
foreach (string value in request.Headers.GetValues(i))
{
sb.Append(header);
sb.Append(": ");
sb.Append(value);
sb.Append(Environment.NewLine);
}
}
sb.Append("Content-Length:");
sb.Append(request.ContentLength);
sb.Append(Environment.NewLine);
TextFileLogger.Log(webRequest.LogDir, "HTTPRequests-" + DateTime.Now.ToString("yyyy-MM-dd") + ".txt", DateTime.Now + Environment.NewLine + sb.ToString());
}
public AttackResult Check(AttackRequest request)
{
var result = new AttackResult();
WebPageRequest webRequest = new WebPageRequest(request.URL);
webRequest.FollowRedirects = false;
WebPageLoader.Load(webRequest);
if (webRequest.Response.Code == "401" && request.URL.StartsWith("http:"))
{
result.Success = true;
result.Results.Enqueue("401 over HTTP found at " + request.URL);
}
if ((webRequest.Response.Code == "302" || webRequest.Response.Code == "301") && webRequest.Response.Body.Length != 0)
{
result.Success = true;
result.Results.Enqueue("302/301 response with body " + request.URL);//possibly add a post check here as well
}
return(result);
}
public static List <string> Gather_SecurityTrails(string domain, string apiKey)
{
//https://api.securitytrails.com/v1/ping?apikey=your_api_key
//
//https://docs.securitytrails.com/docs/how-to-use-the-dsl
List <string> subdomains = new List <string>();
WebPageRequest request = new WebPageRequest();
request.Address = "https://api.securitytrails.com/v1/domain/" + domain + "/subdomains";
request.Headers.Add("APIKEY", apiKey);
WebPageLoader.Load(request);
if (String.IsNullOrEmpty(request.Response.Body))
{
return(subdomains);
}
if (String.IsNullOrEmpty(request.Response.Body))
{
return(subdomains);
}
dynamic d = JObject.Parse(request.Response.Body);
if (d.subdomains != null)
{
foreach (string subdomain in d.subdomains)
{
subdomains.Add(subdomain + "." + domain);
}
}
return(subdomains);
}
//Eventually get a dynamic version loading
//https://stackoverflow.com/questions/24288726/scraping-webpage-generated-by-javascript-with-c-sharp
//https://www.seleniumhq.org/docs/03_webdriver.jsp
public static void Load(WebPageRequest webRequest)
{
if (webRequest.Address.StartsWith("http"))
{
try
{
MakeRequest(webRequest);//webRequest.Address, webRequest.Response, webRequest.CookieJar, webRequest.Headers, webRequest.Method);
}
catch (Exception ex)
{
if (ex.Message == "Trust Issue")
{
// webRequest.Address = DomainUtility.EnsureHTTPS(webRequest.Address);
// MakeRequest(webRequest);
}
else
{
throw new Exception("HTTP not available.");
}
}
if (!String.IsNullOrEmpty(webRequest.Response.Body))
{
HtmlDocument document = new HtmlDocument();
document.LoadHtml(webRequest.Response.Body);
var scriptNodes = document.DocumentNode.SelectNodes("//script[contains(@src,'/js')]");
if (scriptNodes != null)
{
foreach (var scriptNode in scriptNodes)
{
try
{
string script = scriptNode.Attributes["src"].Value;
string scriptSrc = ResolveRelativePaths(script, webRequest.Address);
if (!webRequest.Response.Scripts.ContainsKey(scriptSrc))
{
//WebPageResponse scriptResponse = new WebPageResponse();
WebPageRequest scriptRequest = new WebPageRequest(webRequest);//scriptSrc
scriptRequest.Address = scriptSrc;
scriptRequest.RequestBody = "";
MakeRequest(scriptRequest);
webRequest.Response.Scripts.Add(scriptSrc, scriptRequest.Response.Body);
}
}
catch { }
}
}
}
}
else
{
// throw new Exception("You should never get here!!!!!!!!!!!!!!!!!!!");
string vanillaAddress = webRequest.Address;
webRequest.Address = "http://" + vanillaAddress;
MakeRequest(webRequest);
if (String.IsNullOrEmpty(webRequest.Response.Body))
{
webRequest.Address = "https://" + vanillaAddress;
MakeRequest(webRequest);
}
if (!String.IsNullOrEmpty(webRequest.Response.Body))
{
HtmlDocument document = new HtmlDocument();
document.LoadHtml(webRequest.Response.Body);
var scriptNodes = document.DocumentNode.SelectNodes("//script[contains(@src,'/js')]");
if (scriptNodes != null)
{
foreach (var scriptNode in scriptNodes)
{
try
{
string script = scriptNode.Attributes["src"].Value;
string scriptSrc = ResolveRelativePaths(script, webRequest.Address);
if (!webRequest.Response.Scripts.ContainsKey(scriptSrc))
{
//WebPageResponse scriptResponse = new WebPageResponse();
WebPageRequest scriptRequest = new WebPageRequest(webRequest);//scriptSrc
scriptRequest.Address = scriptSrc;
scriptRequest.RequestBody = "";
MakeRequest(scriptRequest);
webRequest.Response.Scripts.Add(scriptSrc, scriptRequest.Response.Body);
}
}
catch { }
}
}
}
}
}
private void Test(string url)
{
StringBuilder sb = new StringBuilder();
try
{
List <string> schemas = new List <string>();
schemas.Add("http");
schemas.Add("https");
foreach (var schema in schemas)
{
if (schema.Equals("http"))
{
url = DomainUtility.EnsureHTTP(url);
}
else
{
url = DomainUtility.EnsureHTTPS(url);
}
sb.Append(Environment.NewLine);
sb.Append(Environment.NewLine);
sb.Append(DateTime.Now.ToString());
sb.Append(" Checking: " + url + Environment.NewLine);
sb.Append("------------------");
List <IAttack> attacks = new List <IAttack>();
//Future: Auto detect dlls
attacks.Add(new Clark.Attack.ContentScanner.Processor());
attacks.Add(new Clark.Attack.SocialMedia.Processor());
attacks.Add(new Clark.Attack.InformationLeak.Processor());
attacks.Add(new Clark.Attack.CRLF.Processor());
attacks.Add(new Clark.Attack.VulnerableFiles.Processor());
attacks.Add(new Clark.Attack.HTTPHeader.Processor());
attacks.Add(new Clark.Attack.CSP.Processor());
attacks.Add(new Clark.Attack.HTTPResponse.Processor());
attacks.Add(new Clark.Attack.FileUpload.Processor());
attacks.Add(new Clark.Attack.Redirect.Processor());
WebPageRequest request = new WebPageRequest();
request.Address = url;
WebPageLoader.Load(request);
var sRequest = new AttackRequest();
sRequest.Body = request.Response.Body;
sRequest.URL = url;
sRequest.Domain = DomainUtility.GetDomainFromUrl(url);
sRequest.LogDir = Settings.LogDir;
_countdown = new CountdownEvent(attacks.Count);
List <Thread> lstThreads = new List <Thread>();
foreach (var attack in attacks)
{
Thread th = new Thread(() => { sb.Append(Environment.NewLine + ExecuteAttack(attack, sRequest)); });
lstThreads.Add(th);
}
foreach (Thread th in lstThreads)
{
th.Start();
}
_countdown.Wait();
}
}
catch (Exception ex)
{
string inner = "";
if (ex.InnerException != null)
{
inner = ex.InnerException.Message;
}
sb.Append("!!!!!Exception: " + ex.Message + " Inner: " + inner + " Stack: " + ex.StackTrace);
LogError("!!!!!Exception: " + ex.Message + " Inner: " + inner + " Stack: " + ex.StackTrace);
}
Log(sb.ToString());
}
public static List <string> SearchFileType(CrawlRequest request, bool filterKnownAttackFiles)//, CrawlerContext context)
{
if (_knownAttackFiles.Count == 0)
{
Initilize();
}
List <string> foundFiles = new List <string>();
string resumeKey = "";
bool continueLoop = true;
do
{
WebPageRequest webRequest = new WebPageRequest();
// webRequest.Address = "https://web.archive.org/cdx/search?url=" + request.Address + "&matchType=domain&collapse=urlkey&output=text&fl=original&filter=urlkey:.*"+request.FileType+"&limit=10&page=1";
webRequest.Address = "https://web.archive.org/cdx/search?url=" + request.Address + "/&matchType=host" +
"&collapse=urlkey" +
"&output=text" +
"&fl=original" +
@"&filter=original:.*\." + request.FileType + "$" +
"&filter=statuscode:200" +
"&limit=" + request.Limit +
"&showResumeKey=" + request.FindAll.ToString().ToLower() +
"&resumeKey=" + resumeKey;
WebPageLoader.Load(webRequest);
if (!String.IsNullOrEmpty(webRequest.Response.Body))
{
// return webRequest.Response.Body;
List <string> foundStrings = webRequest.Response.Body.Split(new string[] { "\n" }, StringSplitOptions.RemoveEmptyEntries).ToList();
if (foundStrings.Count <= request.Limit)
{
foundFiles.AddRange(foundStrings);
continueLoop = false;
}
else
{
foundFiles.AddRange(foundStrings.Take(request.Limit));
resumeKey = foundStrings.LastOrDefault();
if (resumeKey == null)
{
continueLoop = false;
}
}
}
else
{
continueLoop = false;
}
} while (request.FindAll && continueLoop);
if (filterKnownAttackFiles && foundFiles.Count != 0)
{
List <string> dangerzone = new List <string>();
foreach (string url in foundFiles)
{
string file = url.Split('/').LastOrDefault();
if (file == null)
{
continue;
}
foreach (AttackFile attack in _knownAttackFiles)
{
if (file.Equals(attack.File, StringComparison.InvariantCultureIgnoreCase))
{
foreach (string attackString in attack.Attacks)
{
dangerzone.Add(url + attackString);
}
}
}
}
foundFiles = dangerzone;
}
return(foundFiles);
}
public static List <string> Gather_NetCraft(string domain)
{
//https://searchdns.netcraft.com/?restriction=site+contains&host=yahoo.com&lookup=wait..&position=limited
//site ends with doesn't work
//https://searchdns.netcraft.com/?host=yahoo.com&last=es.yahoo.com&from=21&restriction=site%20contains&position=limited
WebPageRequest exampleRequest = new WebPageRequest();
exampleRequest.Address = "https://searchdns.netcraft.com/?restriction=site+contains&host=*.example.com&lookup=wait..&position=limited";
WebPageLoader.Load(exampleRequest);
List <string> subdomains = new List <string>();
string nextLink = "";
Cookie requestCookie = new Cookie();
Cookie responseCookie = new Cookie();
do
{
WebPageRequest request = new WebPageRequest();
if (String.IsNullOrEmpty(nextLink))
{
request.Address = "https://searchdns.netcraft.com/?restriction=site+contains&host=*." + domain + "&lookup=wait..&position=limited";
if (exampleRequest.Response.Headers.AllKeys.Contains("Set-Cookie"))
{
var setCookie = exampleRequest.Response.Headers["Set-Cookie"];
List <string> cookieList = setCookie.Substring(0, setCookie.IndexOf(';')).Split('=').ToList();
string value = Crypto.SHA1HashStringForUTF8String(HttpUtility.UrlDecode(cookieList[1]));
requestCookie = new Cookie(cookieList[0], cookieList[1], "/", "searchdns.netcraft.com");
responseCookie = new Cookie("netcraft_js_verification_response", value, "/", "searchdns.netcraft.com");
}
else
{
requestCookie = new Cookie("netcraft_js_verification_challenge", HttpUtility.UrlDecode("djF8UUhuTnh1YjZMZzB4ZlZTcjJLOU9JVVVOdnVpZFFLMDZ5TGN5NDluaFJYRy9LK2FVQVFLR0tT%0AZEJvdFE5RnpIcHBsTlljTy9ENjMwTQpzRzZDaFpGVHF3PT0KfDE1Mzc4NTEzNzY%3D%0A%7Cc005d4074568d002e7caa065ff3e501d8fd729fd"), "/", "searchdns.netcraft.com");
responseCookie = new Cookie("netcraft_js_verification_response", "cd86bb04f8659807f97e212b358dbe82e56f092d", "/", "searchdns.netcraft.com");
}
request.CookieJar.Add(requestCookie);
request.CookieJar.Add(responseCookie);
}
else
{
request.Address = "https://searchdns.netcraft.com" + nextLink;
request.CookieJar.Add(requestCookie);
request.CookieJar.Add(responseCookie);
}
WebPageLoader.Load(request);
MatchCollection urls = Regex.Matches(request.Response.Body, "<a href=\"http://toolbar.netcraft.com/site_report\\?url=(.*)\">");
foreach (Match url in urls)
{
Match found = Regex.Match(url.Value, "url=?.+" + domain);
if (!String.IsNullOrEmpty(found.Value))
{
string foundDomain = found.Value.Replace("url=http://", "").Replace("url=https://", "");
if (foundDomain.EndsWith(domain))
{
subdomains.Add(foundDomain);
}
}
}
Match nextPage = Regex.Match(request.Response.Body, "<a href=\"(.*?)\"><b>Next page</b></a>", RegexOptions.IgnoreCase);
if (!String.IsNullOrEmpty(nextPage.Value))
{
string val = nextPage.Value;
int firstIndex = val.IndexOf('"');
int lasIndex = val.LastIndexOf('"');
nextLink = val.Substring(firstIndex + 1, (lasIndex - 1 - firstIndex));
}
else
{
nextLink = "";
}
} while (!String.IsNullOrEmpty(nextLink));
return(subdomains.Distinct().ToList());
}
public AttackResult Check(AttackRequest request)
{
var result = new AttackResult();
WebPageRequest webRequest = new WebPageRequest(request.URL);
WebPageLoader.Load(webRequest);
List <string> csp = new List <string>();
if (webRequest.Response.Headers.AllKeys.Contains("Content-Security-Policy"))
{
csp = webRequest.Response.Headers["Content-Security-Policy"].Split(' ').ToList();
Uri uriResult;
foreach (string url in csp)
{
string testUrl = url;
if (!testUrl.Contains('.'))
{
continue;
}
if (testUrl.StartsWith("wss"))
{
continue;
}
if (testUrl.StartsWith("*."))
{
testUrl = testUrl.Replace("*.", "");
}
// if (!testUrl.StartsWith("http"))
// testUrl = DomainUtility.EnsureHTTPS(testUrl);
bool validURL = Uri.TryCreate(testUrl, UriKind.Absolute, out uriResult) &&
(uriResult.Scheme == Uri.UriSchemeHttp || uriResult.Scheme == Uri.UriSchemeHttps);
if (validURL)
{
if (Ignore.Contains(uriResult.Host.ToString().Trim('/')))
{
continue;
}
WebPageRequest testRequest = new WebPageRequest(uriResult.ToString());
WebPageLoader.Load(testRequest);
if (!testRequest.Response.Code.Equals("200"))
{
if (!testRequest.Response.Code.Equals("403"))
{
result.Success = true;
result.Results.Enqueue("CSP URL: " + uriResult.ToString());
}
}
}
}
}
return(result);
}
private static List <string> Gather_VirusTotal(string domain, string apiKey)
{
//This site also offers an API 4 request/min and there is also limited info on a page
//try the api then try the page
List <string> subdomains = new List <string>();
WebPageRequest request = new WebPageRequest();
request.Address = "https://www.virustotal.com/vtapi/v2/domain/report?apikey=" + apiKey + "&domain=" + domain;
WebPageLoader.Load(request);
if (String.IsNullOrEmpty(request.Response.Body))
{
return(subdomains);
}
if (!String.IsNullOrEmpty(request.Response.Body))
{
dynamic d = JObject.Parse(request.Response.Body);
if (d.subdomains != null)
{
foreach (string subdomain in d.subdomains)
{
subdomains.Add(subdomain);
}
}
}
if (subdomains.Count == 0)
{
request = new WebPageRequest();
request.Address = "https://www.virustotal.com/en/domain/" + domain + "/information";
request.CookieJar.Add(new System.Net.Cookie()
{
Domain = "www.virustotal.com",
HttpOnly = false,
Path = "/",
Name = "VT_PREFERRED_LANGUAGE",
Value = "en"
});
WebPageLoader.Load(request);
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(request.Response.Body);
var divNode = htmlDoc.DocumentNode.SelectSingleNode("//div[@id='observed-subdomains']");
if (divNode != null)
{
var aNodes = divNode.Descendants("a");
foreach (HtmlNode a in aNodes)
{
string found = Regex.Replace(a.InnerText, @"\n|\s+", "");
if (found.Contains("yahoo.com"))
{
subdomains.Add(found);
}
}
}
}
return(subdomains);
}
private void TestDomain(string address, bool signalEnd)
{
StringBuilder sb = new StringBuilder();
StringBuilder linkBuilder = new StringBuilder();
try
{
List <string> schemas = new List <string>();
schemas.Add("http");
schemas.Add("https");
foreach (var schema in schemas)
{
if (schema.Equals("http"))
{
address = DomainUtility.EnsureHTTP(address);
}
else
{
address = DomainUtility.EnsureHTTPS(address);
}
sb.Append("Checking: " + address + Environment.NewLine);
WebPageRequest request = new WebPageRequest();
request.Address = address;
WebPageLoader.Load(request);
ScannerRequest sRequest = new ScannerRequest();
sRequest.Body = request.Response.Body;
sRequest.URL = address;
sRequest.Domain = DomainUtility.GetDomainFromUrl(address);
ScannerResult result = new ScannerResult();
ScannerContext scannerContext = new ScannerContext();
if (request.Response.Body.Equals(String.Empty) && request.Response.TimeOut == false)
{
sb.Append("\tNo body found." + Environment.NewLine);
}
else
{
if (request.Response.TimeOut == false)
{
result = CheckEngine(sRequest, sb, linkBuilder);
if (result.Success)
{
scannerContext.FoundVulnerabilities.Add(new Vulnerability()
{
URL = "", Results = result.Results
});
}
result = CheckBuckets(sRequest, sb, linkBuilder);
if (result.Success)
{
scannerContext.FoundVulnerabilities.Add(new Vulnerability()
{
URL = "", Results = result.Results
});
}
result = CheckSocialMedia(sRequest, sb, linkBuilder);
if (result.Success)
{
scannerContext.FoundVulnerabilities.Add(new Vulnerability()
{
URL = "", Results = result.Results
});
}
result = CheckServices(sRequest, sb, linkBuilder);
if (result.Success)
{
scannerContext.FoundVulnerabilities.Add(new Vulnerability()
{
URL = "", Results = result.Results
});
}
result = CheckDefaultpages(sRequest, sb, linkBuilder);
if (result.Success)
{
scannerContext.FoundVulnerabilities.Add(new Vulnerability()
{
URL = "", Results = result.Results
});
}
result = CheckIndexOf(sRequest, sb, linkBuilder);
if (result.Success)
{
scannerContext.FoundVulnerabilities.Add(new Vulnerability()
{
URL = "", Results = result.Results
});
}
foreach (var script in request.Response.Scripts)
{
ScannerRequest scriptRequest = new ScannerRequest();
scriptRequest.Body = script.Value;
scriptRequest.URL = script.Key;
result = CheckBuckets(scriptRequest, sb, linkBuilder);
if (result.Success)
{
scannerContext.FoundVulnerabilities.Add(new Vulnerability()
{
URL = "", Results = result.Results
});
}
}
}
else
{
// CheckBigIPService(request, sb);
}
}
//CheckForFileType(request.Address, sb, "swf", linkBuilder);
//result = CheckForFileType(request.Address, sb, "php", linkBuilder);
//if (result.Success) { scannerContext.FoundVulnerabilities.Add(new Vulnerability() { URL = "", Results = result.Results }); }
//result = CheckForFileType(request.Address, sb, "xml", linkBuilder);
//if (result.Success) { scannerContext.FoundVulnerabilities.Add(new Vulnerability() { URL = "", Results = result.Results }); }
//result = CheckForFileType(request.Address, sb, "conf", linkBuilder);
//if (result.Success) { scannerContext.FoundVulnerabilities.Add(new Vulnerability() { URL = "", Results = result.Results }); }
//result = CheckForFileType(request.Address, sb, "env", linkBuilder);
//if (result.Success) { scannerContext.FoundVulnerabilities.Add(new Vulnerability() { URL = "", Results = result.Results }); }
result = CheckPHPInfo(sRequest, sb, linkBuilder);
if (result.Success)
{
scannerContext.FoundVulnerabilities.Add(new Vulnerability()
{
URL = "", Results = result.Results
});
}
result = CheckKnownAttackFiles(sRequest, sb, linkBuilder);
if (result.Success)
{
scannerContext.FoundVulnerabilities.Add(new Vulnerability()
{
URL = "", Results = result.Results
});
}
result = CheckCRLF(sRequest, sb, linkBuilder);
if (result.Success)
{
scannerContext.FoundVulnerabilities.Add(new Vulnerability()
{
URL = "", Results = result.Results
});
}
result = CheckCSP(sRequest, sb, linkBuilder);
if (result.Success)
{
scannerContext.FoundVulnerabilities.Add(new Vulnerability()
{
URL = "", Results = result.Results
});
}
result = CheckHeaders(sRequest, sb);
if (result.Success)
{
scannerContext.FoundVulnerabilities.Add(new Vulnerability()
{
URL = "", Results = result.Results
});
}
}
}
catch (Exception ex)
{
string inner = "";
if (ex.InnerException != null)
{
inner = ex.InnerException.Message;
}
sb.Append("!!!!!Exception: " + ex.Message + " Inner: " + inner + " Stack: " + ex.StackTrace);
}
Log(sb.ToString());
LogLinks(linkBuilder.ToString());
if (signalEnd)
{
_countdown.Signal();
}
}
