public static List <Request> GetLinkClicks(Request Req, Response Res, CookieStore Cookies)
{
List <Request> LinkClicks = new List <Request>();
List <string> Links = GetLinks(Req, Res);
foreach (string Link in Links)
{
try
{
Request LinkReq = new Request(Link);
LinkReq.SetCookie(Cookies);
LinkClicks.Add(LinkReq);
}
catch { }
}
return(LinkClicks);
}
public static List <Request> GetRedirects(Request Req, Response Res, CookieStore Cookies)
{
List <Request> Redirects = new List <Request>();
List <string> RedirectUrls = GetRedirectUrls(Req, Res);
foreach (string RedirectUrl in RedirectUrls)
{
try
{
Request RedirectReq = new Request(RedirectUrl);
RedirectReq.SetCookie(Cookies);
Redirects.Add(RedirectReq);
}
catch { }
}
return(Redirects);
}
public Request GetRedirect(Response Res)
{
if (Res.Code == 301 || Res.Code == 302 || Res.Code == 303 || Res.Code == 307)
{
if (Res.Headers.Has("Location"))
{
string NewUrl = Res.Headers.Get("Location");
Request NewReq = new Request(this.FullURL);
if (NewUrl.StartsWith("/"))
{
NewReq.URL = NewUrl;
}
else if (NewUrl.StartsWith("http://") || NewUrl.StartsWith("https://"))
{
NewReq.FullURL = NewUrl;
}
else if (NewUrl.StartsWith(".."))
{
int DirBackNumber = 0;
List<string> UpdatedNewUrlPathParts = NewReq.UrlPathParts;
List<string> NewUrlPathParts = new List<string>(NewUrl.Split('/'));
if (!NewReq.Url.EndsWith("/") && UpdatedNewUrlPathParts.Count > 0) UpdatedNewUrlPathParts.RemoveAt(UpdatedNewUrlPathParts.Count - 1);
foreach (string NewUrlPathPart in NewUrlPathParts)
{
if (NewUrlPathPart.Equals(".."))
DirBackNumber++;
else
break;
}
while (UpdatedNewUrlPathParts.Count > 0 && DirBackNumber > 0)
{
UpdatedNewUrlPathParts.RemoveAt(UpdatedNewUrlPathParts.Count - 1);
if (NewUrlPathParts.Count > 0) NewUrlPathParts.RemoveAt(0);
DirBackNumber--;
}
UpdatedNewUrlPathParts.AddRange(NewUrlPathParts);
NewReq.UrlPathParts = UpdatedNewUrlPathParts;
if (NewUrl.EndsWith("/"))
{
if (!NewReq.Url.EndsWith("/")) NewReq.Url = NewReq.Url + "/";
}
else
{
if (NewReq.Url.EndsWith("/") && NewReq.Url.Length > 1) NewReq.Url = NewReq.Url.TrimEnd(new char[] { '/' });
}
}
else
{
List<string> NewUrlPathParts = NewReq.UrlPathParts;
if (!NewReq.Url.EndsWith("/") && NewUrlPathParts.Count > 0) NewUrlPathParts.RemoveAt(NewUrlPathParts.Count - 1);
NewReq.UrlPathParts = NewUrlPathParts;
if (!NewReq.Url.EndsWith("/")) NewReq.Url = NewReq.Url + "/";
NewReq.Url = NewReq.Url + NewUrl;
}
//this check is needed since sometimes the redirect can go to a different domain
if (this.Host == NewReq.Host)
{
NewReq.CookieString = this.CookieString;
NewReq.SetCookie(Res.SetCookies);
}
return NewReq;
}
}
return null;
}
public static List <Request> GetFormSubmissions(Request Req, Response Res, CookieStore Cookies)
{
List <Request> FormSubmissions = new List <Request>();
List <HtmlNode> FormNodes = Res.Html.GetForms();
foreach (HtmlNode FormNode in FormNodes)
{
Request SubReq = Req.GetClone();
SubReq.Method = "GET";
SubReq.BodyString = "";
foreach (HtmlAttribute Attr in FormNode.Attributes)
{
if (Attr.Name.Equals("method"))
{
SubReq.Method = Attr.Value.ToUpper();
}
else if (Attr.Name.Equals("action"))
{
if (Attr.Value.StartsWith("javascript:"))
{
continue;
}
string ActionUrl = NormalizeUrl(Req, Tools.HtmlDecode(Attr.Value.Trim()));
if (ActionUrl.Length > 0)
{
SubReq.FullUrl = ActionUrl;
}
}
}
if (SubReq.Method == "GET")
{
SubReq.Query.RemoveAll();
}
else
{
SubReq.Headers.Set("Content-Type", "application/x-www-form-urlencoded");
}
foreach (HtmlNode InputNode in FormNode.ChildNodes)
{
string Name = "";
string Value = "";
foreach (HtmlAttribute Attr in InputNode.Attributes)
{
switch (Attr.Name)
{
case ("name"):
Name = Attr.Value;
break;
case ("type"):
if (Attr.Value.Equals("submit"))
{
Name = "";
}
break;
case ("value"):
Value = Attr.Value;
break;
}
}
if (Value.Length == 0)
{
Value = Tools.GetRandomString(2, 5);
}
if (Name.Length > 0)
{
if (SubReq.Method.Equals("GET"))
{
SubReq.Query.Add(Name, Value);
}
else
{
SubReq.Body.Add(Name, Value);
}
}
}
SubReq.SetCookie(Cookies);
FormSubmissions.Add(SubReq);
}
return(FormSubmissions);
}
void Crawl(Request Req, int Depth, bool Scraped)
{
if (Stopped)
{
return;
}
if (Depth > MaxDepth)
{
return;
}
if (WasCrawled(Req))
{
return;
}
if (!CanCrawl(Req))
{
return;
}
lock (PageSignatures)
{
PageSignatures.Add(GetPageSignature(Req));
}
Req.Source = RequestSource.Probe;
Req.SetCookie(Cookies);
if (UserAgent.Length > 0)
{
Req.Headers.Set("User-Agent", UserAgent);
}
if (SpecialHeader[0] != null)
{
Req.Headers.Set(SpecialHeader[0], SpecialHeader[1]);
}
if (Stopped)
{
return;
}
Response Res = Req.Send();
if (Stopped)
{
return;
}
Cookies.Add(Req, Res);
bool Is404File = IsA404(Req, Res);
if (!Res.IsHtml)
{
return;
}
if (Depth + 1 > MaxDepth)
{
return;
}
List <Request> Redirects = GetRedirects(Req, Res);
foreach (Request Redirect in Redirects)
{
AddToCrawlQueue(Redirect, Depth + 1, true);
}
List <Request> LinkClicks = GetLinkClicks(Req, Res);
foreach (Request LinkClick in LinkClicks)
{
AddToCrawlQueue(LinkClick, Depth + 1, true);
}
List <Request> FormSubmissions = GetFormSubmissions(Req, Res);
foreach (Request FormSubmission in FormSubmissions)
{
AddToCrawlQueue(FormSubmission, Depth + 1, true);
}
Request DirCheck = Req.GetClone();
DirCheck.Method = "GET";
DirCheck.Body.RemoveAll();
DirCheck.Url = DirCheck.UrlDir;
if (!Req.Url.EndsWith("/"))
{
AddToCrawlQueue(DirCheck, Depth + 1, false);
}
if (PerformDirAndFileGuessing && !Is404File)
{
foreach (string File in FileNamesToCheck)
{
Request FileCheck = DirCheck.GetClone();
FileCheck.Url = FileCheck.Url + File;
AddToCrawlQueue(FileCheck, Depth + 1, false);
}
foreach (string Dir in DirNamesToCheck)
{
Request DirectoryCheck = DirCheck.GetClone();
DirectoryCheck.Url = DirectoryCheck.Url + Dir + "/";
AddToCrawlQueue(DirectoryCheck, Depth + 1, false);
}
}
if (Stopped)
{
return;
}
if (Scraped || !Is404File)
{
lock (CrawledRequests)
{
CrawledRequests.Enqueue(Req);
}
IronUpdater.AddToSiteMap(Req);
}
}
void Crawl(Request Req, int Depth, bool Scraped)
{
if (Stopped) return;
if (Depth > MaxDepth) return;
if (WasCrawled(Req)) return;
if (!CanCrawl(Req)) return;
lock (PageSignatures)
{
PageSignatures.Add(GetPageSignature(Req));
}
Req.Source = RequestSource.Probe;
Req.SetCookie(Cookies);
if (UserAgent.Length > 0) Req.Headers.Set("User-Agent", UserAgent);
Response Res = Req.Send();
Cookies.Add(Req, Res);
bool Is404File = IsA404(Req, Res);
if (!Res.IsHtml)
{
try
{
Res.ProcessHtml();
}
catch
{
return;
}
}
if (Depth + 1 > MaxDepth) return;
List<Request> LinkClicks = GetLinkClicks(Req, Res);
foreach (Request LinkClick in LinkClicks)
{
AddToCrawlQueue(LinkClick, Depth + 1, true);
}
List<Request> FormSubmissions = GetFormSubmissionRequests(Req, Res);
foreach (Request FormSubmission in FormSubmissions)
{
AddToCrawlQueue(FormSubmission, Depth + 1, true);
}
Request DirCheck = Req.GetClone();
DirCheck.Method = "GET";
DirCheck.Body.RemoveAll();
DirCheck.Url = DirCheck.UrlDir;
if (!Req.Url.EndsWith("/"))
{
AddToCrawlQueue(DirCheck, Depth + 1, false);
}
if (PerformDirAndFileGuessing && !Is404File)
{
foreach (string File in FileNamesToCheck)
{
Request FileCheck = DirCheck.GetClone();
FileCheck.Url = FileCheck.Url + File;
AddToCrawlQueue(FileCheck, Depth + 1, false);
}
foreach (string Dir in DirNamesToCheck)
{
Request DirectoryCheck = DirCheck.GetClone();
DirectoryCheck.Url = DirectoryCheck.Url + Dir + "/";
AddToCrawlQueue(DirectoryCheck, Depth + 1, false);
}
}
if (Scraped || !Is404File)
{
lock (CrawledRequests)
{
CrawledRequests.Enqueue(Req);
}
IronUpdater.AddToSiteMap(Req);
}
}
List<Request> GetLinkClicks(Request Req, Response Res)
{
List<Request> LinkClicks = new List<Request>();
List<string> Links = GetLinks(Req, Res);
foreach (string Link in Links)
{
try
{
Request LinkReq = new Request(Link);
LinkReq.SetCookie(Cookies);
LinkClicks.Add(LinkReq);
}
catch { }
}
return LinkClicks;
}
public static List<Request> GetRedirects(Request Req, Response Res, CookieStore Cookies)
{
List<Request> Redirects = new List<Request>();
List<string> RedirectUrls = GetRedirectUrls(Req, Res);
foreach (string RedirectUrl in RedirectUrls)
{
try
{
Request RedirectReq = new Request(RedirectUrl);
RedirectReq.SetCookie(Cookies);
Redirects.Add(RedirectReq);
}
catch { }
}
return Redirects;
}
public Request GetRedirect(Response Res)
{
if (Res.IsRedirect)
{
if (Res.Headers.Has("Location"))
{
string NewUrl = Res.Headers.Get("Location");
Request NewReq = new Request(this.FullURL);
if (NewUrl.StartsWith("/"))
{
NewReq.URL = NewUrl;
}
else if (NewUrl.StartsWith("http://") || NewUrl.StartsWith("https://"))
{
NewReq.FullURL = NewUrl;
}
else if (NewUrl.StartsWith(".."))
{
int DirBackNumber = 0;
List<string> UpdatedNewUrlPathParts = NewReq.UrlPathParts;
List<string> NewUrlPathParts = new List<string>(NewUrl.Split('/'));
if (!NewReq.Url.EndsWith("/") && UpdatedNewUrlPathParts.Count > 0) UpdatedNewUrlPathParts.RemoveAt(UpdatedNewUrlPathParts.Count - 1);
foreach (string NewUrlPathPart in NewUrlPathParts)
{
if (NewUrlPathPart.Equals(".."))
DirBackNumber++;
else
break;
}
while (UpdatedNewUrlPathParts.Count > 0 && DirBackNumber > 0)
{
UpdatedNewUrlPathParts.RemoveAt(UpdatedNewUrlPathParts.Count - 1);
if (NewUrlPathParts.Count > 0) NewUrlPathParts.RemoveAt(0);
DirBackNumber--;
}
UpdatedNewUrlPathParts.AddRange(NewUrlPathParts);
NewReq.UrlPathParts = UpdatedNewUrlPathParts;
if (NewUrl.EndsWith("/"))
{
if (!NewReq.Url.EndsWith("/")) NewReq.Url = NewReq.Url + "/";
}
else
{
if (NewReq.Url.EndsWith("/") && NewReq.Url.Length > 1) NewReq.Url = NewReq.Url.TrimEnd(new char[] { '/' });
}
}
else
{
List<string> NewUrlPathParts = NewReq.UrlPathParts;
if (!NewReq.Url.EndsWith("/") && NewUrlPathParts.Count > 0) NewUrlPathParts.RemoveAt(NewUrlPathParts.Count - 1);
NewReq.UrlPathParts = NewUrlPathParts;
if (!NewReq.Url.EndsWith("/")) NewReq.Url = NewReq.Url + "/";
NewReq.Url = NewReq.Url + NewUrl;
}
//this check is needed since sometimes the redirect can go to a different domain
if (this.Host == NewReq.Host)
{
NewReq.CookieString = this.CookieString;
NewReq.SetCookie(Res.SetCookies);
}
//Update the Source of the redirect request with the value of the current request
NewReq.Source = this.Source;
if (this.Source.Equals("Scan"))
NewReq.ScanID = this.ScanID;
//Update general headers associated with the current request in to the header of the redirect request
foreach (string HeaderName in this.Headers.GetNames())
{
if (!(HeaderName.Equals("Host", StringComparison.OrdinalIgnoreCase) || HeaderName.Equals("Content-Length", StringComparison.OrdinalIgnoreCase) || HeaderName.Equals("Content-Type", StringComparison.OrdinalIgnoreCase) || HeaderName.Equals("Cookie", StringComparison.OrdinalIgnoreCase)))
{
foreach (string HeaderValue in this.Headers.GetAll(HeaderName))
{
NewReq.Headers.Add(HeaderName, HeaderValue);
}
}
}
return NewReq;
}
}
return null;
}
public virtual Request Update(Request Request, Response Response)
{
Request.SetCookie(Response);
return Request;
}
public static void ReadFromStore(Request Req)
{
Req.SetCookie(StaticCookieStore);
}
public static Request GetFormSubmission(Request Req, HtmlNode FormNode, CookieStore Cookies, bool LoginFormOnly, bool FillEmptyFields)
{
//Login request signatures:
//form must have one password type input field
//three or more parameters must be present in the request query/body
Request SubReq = Req.GetClone();
SubReq.Method = "GET";
SubReq.BodyString = "";
foreach (HtmlAttribute Attr in FormNode.Attributes)
{
if (Attr.Name.Equals("method"))
{
SubReq.Method = Attr.Value.ToUpper();
}
else if (Attr.Name.Equals("action"))
{
if (Attr.Value.StartsWith("javascript:"))
{
continue;
}
string ActionUrl = NormalizeUrl(Req, Tools.HtmlDecode(Attr.Value.Trim()));
if (ActionUrl.Length > 0)
{
SubReq.FullUrl = ActionUrl;
}
}
}
if (SubReq.Method == "GET")
{
SubReq.Query.RemoveAll();
}
else
{
SubReq.Headers.Set("Content-Type", "application/x-www-form-urlencoded");
}
bool PasswordFieldPresent = false;
foreach (HtmlNode InputNode in FormNode.ChildNodes)
{
string Name = "";
string Value = "";
foreach (HtmlAttribute Attr in InputNode.Attributes)
{
switch (Attr.Name)
{
case ("name"):
Name = Attr.Value;
break;
case ("type"):
if (Attr.Value.Equals("password", StringComparison.OrdinalIgnoreCase))
{
PasswordFieldPresent = true;
}
break;
case ("value"):
Value = Attr.Value;
break;
}
}
if (FillEmptyFields && Value.Length == 0)
{
Value = Tools.GetRandomString(2, 5);
}
if (Name.Length > 0)
{
if (SubReq.Method.Equals("GET"))
{
SubReq.Query.Add(Name, Value);
}
else
{
SubReq.Body.Add(Name, Value);
}
}
}
SubReq.SetCookie(Cookies);
if (LoginFormOnly)
{
if (PasswordFieldPresent)
{
if ((SubReq.Method.Equals("GET", StringComparison.OrdinalIgnoreCase) && SubReq.Query.Count >= 3) || (SubReq.Method.Equals("POST", StringComparison.OrdinalIgnoreCase) && SubReq.Body.Count >= 3))
{
return(SubReq);
}
}
}
else
{
return(SubReq);
}
return(null);
}
public Request GetRedirect(Response Res)
{
if (Res.Code == 301 || Res.Code == 302 || Res.Code == 303 || Res.Code == 307)
{
if (Res.Headers.Has("Location"))
{
string NewUrl = Res.Headers.Get("Location");
Request NewReq = new Request(this.FullURL);
if (NewUrl.StartsWith("/"))
{
NewReq.URL = NewUrl;
}
else if (NewUrl.StartsWith("http://") || NewUrl.StartsWith("https://"))
{
NewReq.FullURL = NewUrl;
}
else if (NewUrl.StartsWith(".."))
{
int DirBackNumber = 0;
List <string> UpdatedNewUrlPathParts = NewReq.UrlPathParts;
List <string> NewUrlPathParts = new List <string>(NewUrl.Split('/'));
if (!NewReq.Url.EndsWith("/") && UpdatedNewUrlPathParts.Count > 0)
{
UpdatedNewUrlPathParts.RemoveAt(UpdatedNewUrlPathParts.Count - 1);
}
foreach (string NewUrlPathPart in NewUrlPathParts)
{
if (NewUrlPathPart.Equals(".."))
{
DirBackNumber++;
}
else
{
break;
}
}
while (UpdatedNewUrlPathParts.Count > 0 && DirBackNumber > 0)
{
UpdatedNewUrlPathParts.RemoveAt(UpdatedNewUrlPathParts.Count - 1);
if (NewUrlPathParts.Count > 0)
{
NewUrlPathParts.RemoveAt(0);
}
DirBackNumber--;
}
UpdatedNewUrlPathParts.AddRange(NewUrlPathParts);
NewReq.UrlPathParts = UpdatedNewUrlPathParts;
if (NewUrl.EndsWith("/"))
{
if (!NewReq.Url.EndsWith("/"))
{
NewReq.Url = NewReq.Url + "/";
}
}
else
{
if (NewReq.Url.EndsWith("/") && NewReq.Url.Length > 1)
{
NewReq.Url = NewReq.Url.TrimEnd(new char[] { '/' });
}
}
}
else
{
List <string> NewUrlPathParts = NewReq.UrlPathParts;
if (!NewReq.Url.EndsWith("/") && NewUrlPathParts.Count > 0)
{
NewUrlPathParts.RemoveAt(NewUrlPathParts.Count - 1);
}
NewReq.UrlPathParts = NewUrlPathParts;
if (!NewReq.Url.EndsWith("/"))
{
NewReq.Url = NewReq.Url + "/";
}
NewReq.Url = NewReq.Url + NewUrl;
}
//this check is needed since sometimes the redirect can go to a different domain
if (this.Host == NewReq.Host)
{
NewReq.CookieString = this.CookieString;
NewReq.SetCookie(Res.SetCookies);
}
return(NewReq);
}
}
return(null);
}
public static void ReadFromStore(Request Req)
{
Req.SetCookie(StaticCookieStore);
}
virtual public Request Update(Request Request, Response Response)
{
Request.SetCookie(Response);
return(Request);
}
public Request GetRedirect(Response Res)
{
if (Res.Code == 301 || Res.Code == 302 || Res.Code == 303 || Res.Code == 307)
{
if (Res.Headers.Has("Location"))
{
string NewUrl = Res.Headers.Get("Location");
Request NewReq = new Request(this.FullURL);
if (NewUrl.StartsWith("/"))
{
NewReq.URL = NewUrl;
return NewReq;
}
if (NewUrl.StartsWith("http://") || NewUrl.StartsWith("https://"))
{
NewReq.FullURL = NewUrl;
if (this.Host == NewReq.Host)
{
NewReq.CookieString = this.CookieString;
NewReq.SetCookie(Res.SetCookies);
}
return NewReq;
}
}
}
return null;
}
