public static List <Request> GetLinkClicks(Request Req, Response Res, CookieStore Cookies) { List <Request> LinkClicks = new List <Request>(); List <string> Links = GetLinks(Req, Res); foreach (string Link in Links) { try { Request LinkReq = new Request(Link); LinkReq.SetCookie(Cookies); LinkClicks.Add(LinkReq); } catch { } } return(LinkClicks); }
public static List <Request> GetRedirects(Request Req, Response Res, CookieStore Cookies) { List <Request> Redirects = new List <Request>(); List <string> RedirectUrls = GetRedirectUrls(Req, Res); foreach (string RedirectUrl in RedirectUrls) { try { Request RedirectReq = new Request(RedirectUrl); RedirectReq.SetCookie(Cookies); Redirects.Add(RedirectReq); } catch { } } return(Redirects); }
public Request GetRedirect(Response Res) { if (Res.Code == 301 || Res.Code == 302 || Res.Code == 303 || Res.Code == 307) { if (Res.Headers.Has("Location")) { string NewUrl = Res.Headers.Get("Location"); Request NewReq = new Request(this.FullURL); if (NewUrl.StartsWith("/")) { NewReq.URL = NewUrl; } else if (NewUrl.StartsWith("http://") || NewUrl.StartsWith("https://")) { NewReq.FullURL = NewUrl; } else if (NewUrl.StartsWith("..")) { int DirBackNumber = 0; List<string> UpdatedNewUrlPathParts = NewReq.UrlPathParts; List<string> NewUrlPathParts = new List<string>(NewUrl.Split('/')); if (!NewReq.Url.EndsWith("/") && UpdatedNewUrlPathParts.Count > 0) UpdatedNewUrlPathParts.RemoveAt(UpdatedNewUrlPathParts.Count - 1); foreach (string NewUrlPathPart in NewUrlPathParts) { if (NewUrlPathPart.Equals("..")) DirBackNumber++; else break; } while (UpdatedNewUrlPathParts.Count > 0 && DirBackNumber > 0) { UpdatedNewUrlPathParts.RemoveAt(UpdatedNewUrlPathParts.Count - 1); if (NewUrlPathParts.Count > 0) NewUrlPathParts.RemoveAt(0); DirBackNumber--; } UpdatedNewUrlPathParts.AddRange(NewUrlPathParts); NewReq.UrlPathParts = UpdatedNewUrlPathParts; if (NewUrl.EndsWith("/")) { if (!NewReq.Url.EndsWith("/")) NewReq.Url = NewReq.Url + "/"; } else { if (NewReq.Url.EndsWith("/") && NewReq.Url.Length > 1) NewReq.Url = NewReq.Url.TrimEnd(new char[] { '/' }); } } else { List<string> NewUrlPathParts = NewReq.UrlPathParts; if (!NewReq.Url.EndsWith("/") && NewUrlPathParts.Count > 0) NewUrlPathParts.RemoveAt(NewUrlPathParts.Count - 1); NewReq.UrlPathParts = NewUrlPathParts; if (!NewReq.Url.EndsWith("/")) NewReq.Url = NewReq.Url + "/"; NewReq.Url = NewReq.Url + NewUrl; } //this check is needed since sometimes the redirect can go to a different domain if (this.Host == NewReq.Host) { NewReq.CookieString = this.CookieString; NewReq.SetCookie(Res.SetCookies); } return NewReq; } } return null; }
public static List <Request> GetFormSubmissions(Request Req, Response Res, CookieStore Cookies) { List <Request> FormSubmissions = new List <Request>(); List <HtmlNode> FormNodes = Res.Html.GetForms(); foreach (HtmlNode FormNode in FormNodes) { Request SubReq = Req.GetClone(); SubReq.Method = "GET"; SubReq.BodyString = ""; foreach (HtmlAttribute Attr in FormNode.Attributes) { if (Attr.Name.Equals("method")) { SubReq.Method = Attr.Value.ToUpper(); } else if (Attr.Name.Equals("action")) { if (Attr.Value.StartsWith("javascript:")) { continue; } string ActionUrl = NormalizeUrl(Req, Tools.HtmlDecode(Attr.Value.Trim())); if (ActionUrl.Length > 0) { SubReq.FullUrl = ActionUrl; } } } if (SubReq.Method == "GET") { SubReq.Query.RemoveAll(); } else { SubReq.Headers.Set("Content-Type", "application/x-www-form-urlencoded"); } foreach (HtmlNode InputNode in FormNode.ChildNodes) { string Name = ""; string Value = ""; foreach (HtmlAttribute Attr in InputNode.Attributes) { switch (Attr.Name) { case ("name"): Name = Attr.Value; break; case ("type"): if (Attr.Value.Equals("submit")) { Name = ""; } break; case ("value"): Value = Attr.Value; break; } } if (Value.Length == 0) { Value = Tools.GetRandomString(2, 5); } if (Name.Length > 0) { if (SubReq.Method.Equals("GET")) { SubReq.Query.Add(Name, Value); } else { SubReq.Body.Add(Name, Value); } } } SubReq.SetCookie(Cookies); FormSubmissions.Add(SubReq); } return(FormSubmissions); }
void Crawl(Request Req, int Depth, bool Scraped) { if (Stopped) { return; } if (Depth > MaxDepth) { return; } if (WasCrawled(Req)) { return; } if (!CanCrawl(Req)) { return; } lock (PageSignatures) { PageSignatures.Add(GetPageSignature(Req)); } Req.Source = RequestSource.Probe; Req.SetCookie(Cookies); if (UserAgent.Length > 0) { Req.Headers.Set("User-Agent", UserAgent); } if (SpecialHeader[0] != null) { Req.Headers.Set(SpecialHeader[0], SpecialHeader[1]); } if (Stopped) { return; } Response Res = Req.Send(); if (Stopped) { return; } Cookies.Add(Req, Res); bool Is404File = IsA404(Req, Res); if (!Res.IsHtml) { return; } if (Depth + 1 > MaxDepth) { return; } List <Request> Redirects = GetRedirects(Req, Res); foreach (Request Redirect in Redirects) { AddToCrawlQueue(Redirect, Depth + 1, true); } List <Request> LinkClicks = GetLinkClicks(Req, Res); foreach (Request LinkClick in LinkClicks) { AddToCrawlQueue(LinkClick, Depth + 1, true); } List <Request> FormSubmissions = GetFormSubmissions(Req, Res); foreach (Request FormSubmission in FormSubmissions) { AddToCrawlQueue(FormSubmission, Depth + 1, true); } Request DirCheck = Req.GetClone(); DirCheck.Method = "GET"; DirCheck.Body.RemoveAll(); DirCheck.Url = DirCheck.UrlDir; if (!Req.Url.EndsWith("/")) { AddToCrawlQueue(DirCheck, Depth + 1, false); } if (PerformDirAndFileGuessing && !Is404File) { foreach (string File in FileNamesToCheck) { Request FileCheck = DirCheck.GetClone(); FileCheck.Url = FileCheck.Url + File; AddToCrawlQueue(FileCheck, Depth + 1, false); } foreach (string Dir in DirNamesToCheck) { Request DirectoryCheck = DirCheck.GetClone(); DirectoryCheck.Url = DirectoryCheck.Url + Dir + "/"; AddToCrawlQueue(DirectoryCheck, Depth + 1, false); } } if (Stopped) { return; } if (Scraped || !Is404File) { lock (CrawledRequests) { CrawledRequests.Enqueue(Req); } IronUpdater.AddToSiteMap(Req); } }
void Crawl(Request Req, int Depth, bool Scraped) { if (Stopped) return; if (Depth > MaxDepth) return; if (WasCrawled(Req)) return; if (!CanCrawl(Req)) return; lock (PageSignatures) { PageSignatures.Add(GetPageSignature(Req)); } Req.Source = RequestSource.Probe; Req.SetCookie(Cookies); if (UserAgent.Length > 0) Req.Headers.Set("User-Agent", UserAgent); Response Res = Req.Send(); Cookies.Add(Req, Res); bool Is404File = IsA404(Req, Res); if (!Res.IsHtml) { try { Res.ProcessHtml(); } catch { return; } } if (Depth + 1 > MaxDepth) return; List<Request> LinkClicks = GetLinkClicks(Req, Res); foreach (Request LinkClick in LinkClicks) { AddToCrawlQueue(LinkClick, Depth + 1, true); } List<Request> FormSubmissions = GetFormSubmissionRequests(Req, Res); foreach (Request FormSubmission in FormSubmissions) { AddToCrawlQueue(FormSubmission, Depth + 1, true); } Request DirCheck = Req.GetClone(); DirCheck.Method = "GET"; DirCheck.Body.RemoveAll(); DirCheck.Url = DirCheck.UrlDir; if (!Req.Url.EndsWith("/")) { AddToCrawlQueue(DirCheck, Depth + 1, false); } if (PerformDirAndFileGuessing && !Is404File) { foreach (string File in FileNamesToCheck) { Request FileCheck = DirCheck.GetClone(); FileCheck.Url = FileCheck.Url + File; AddToCrawlQueue(FileCheck, Depth + 1, false); } foreach (string Dir in DirNamesToCheck) { Request DirectoryCheck = DirCheck.GetClone(); DirectoryCheck.Url = DirectoryCheck.Url + Dir + "/"; AddToCrawlQueue(DirectoryCheck, Depth + 1, false); } } if (Scraped || !Is404File) { lock (CrawledRequests) { CrawledRequests.Enqueue(Req); } IronUpdater.AddToSiteMap(Req); } }
List<Request> GetLinkClicks(Request Req, Response Res) { List<Request> LinkClicks = new List<Request>(); List<string> Links = GetLinks(Req, Res); foreach (string Link in Links) { try { Request LinkReq = new Request(Link); LinkReq.SetCookie(Cookies); LinkClicks.Add(LinkReq); } catch { } } return LinkClicks; }
public static List<Request> GetRedirects(Request Req, Response Res, CookieStore Cookies) { List<Request> Redirects = new List<Request>(); List<string> RedirectUrls = GetRedirectUrls(Req, Res); foreach (string RedirectUrl in RedirectUrls) { try { Request RedirectReq = new Request(RedirectUrl); RedirectReq.SetCookie(Cookies); Redirects.Add(RedirectReq); } catch { } } return Redirects; }
public Request GetRedirect(Response Res) { if (Res.IsRedirect) { if (Res.Headers.Has("Location")) { string NewUrl = Res.Headers.Get("Location"); Request NewReq = new Request(this.FullURL); if (NewUrl.StartsWith("/")) { NewReq.URL = NewUrl; } else if (NewUrl.StartsWith("http://") || NewUrl.StartsWith("https://")) { NewReq.FullURL = NewUrl; } else if (NewUrl.StartsWith("..")) { int DirBackNumber = 0; List<string> UpdatedNewUrlPathParts = NewReq.UrlPathParts; List<string> NewUrlPathParts = new List<string>(NewUrl.Split('/')); if (!NewReq.Url.EndsWith("/") && UpdatedNewUrlPathParts.Count > 0) UpdatedNewUrlPathParts.RemoveAt(UpdatedNewUrlPathParts.Count - 1); foreach (string NewUrlPathPart in NewUrlPathParts) { if (NewUrlPathPart.Equals("..")) DirBackNumber++; else break; } while (UpdatedNewUrlPathParts.Count > 0 && DirBackNumber > 0) { UpdatedNewUrlPathParts.RemoveAt(UpdatedNewUrlPathParts.Count - 1); if (NewUrlPathParts.Count > 0) NewUrlPathParts.RemoveAt(0); DirBackNumber--; } UpdatedNewUrlPathParts.AddRange(NewUrlPathParts); NewReq.UrlPathParts = UpdatedNewUrlPathParts; if (NewUrl.EndsWith("/")) { if (!NewReq.Url.EndsWith("/")) NewReq.Url = NewReq.Url + "/"; } else { if (NewReq.Url.EndsWith("/") && NewReq.Url.Length > 1) NewReq.Url = NewReq.Url.TrimEnd(new char[] { '/' }); } } else { List<string> NewUrlPathParts = NewReq.UrlPathParts; if (!NewReq.Url.EndsWith("/") && NewUrlPathParts.Count > 0) NewUrlPathParts.RemoveAt(NewUrlPathParts.Count - 1); NewReq.UrlPathParts = NewUrlPathParts; if (!NewReq.Url.EndsWith("/")) NewReq.Url = NewReq.Url + "/"; NewReq.Url = NewReq.Url + NewUrl; } //this check is needed since sometimes the redirect can go to a different domain if (this.Host == NewReq.Host) { NewReq.CookieString = this.CookieString; NewReq.SetCookie(Res.SetCookies); } //Update the Source of the redirect request with the value of the current request NewReq.Source = this.Source; if (this.Source.Equals("Scan")) NewReq.ScanID = this.ScanID; //Update general headers associated with the current request in to the header of the redirect request foreach (string HeaderName in this.Headers.GetNames()) { if (!(HeaderName.Equals("Host", StringComparison.OrdinalIgnoreCase) || HeaderName.Equals("Content-Length", StringComparison.OrdinalIgnoreCase) || HeaderName.Equals("Content-Type", StringComparison.OrdinalIgnoreCase) || HeaderName.Equals("Cookie", StringComparison.OrdinalIgnoreCase))) { foreach (string HeaderValue in this.Headers.GetAll(HeaderName)) { NewReq.Headers.Add(HeaderName, HeaderValue); } } } return NewReq; } } return null; }
public virtual Request Update(Request Request, Response Response) { Request.SetCookie(Response); return Request; }
public static void ReadFromStore(Request Req) { Req.SetCookie(StaticCookieStore); }
public static Request GetFormSubmission(Request Req, HtmlNode FormNode, CookieStore Cookies, bool LoginFormOnly, bool FillEmptyFields) { //Login request signatures: //form must have one password type input field //three or more parameters must be present in the request query/body Request SubReq = Req.GetClone(); SubReq.Method = "GET"; SubReq.BodyString = ""; foreach (HtmlAttribute Attr in FormNode.Attributes) { if (Attr.Name.Equals("method")) { SubReq.Method = Attr.Value.ToUpper(); } else if (Attr.Name.Equals("action")) { if (Attr.Value.StartsWith("javascript:")) { continue; } string ActionUrl = NormalizeUrl(Req, Tools.HtmlDecode(Attr.Value.Trim())); if (ActionUrl.Length > 0) { SubReq.FullUrl = ActionUrl; } } } if (SubReq.Method == "GET") { SubReq.Query.RemoveAll(); } else { SubReq.Headers.Set("Content-Type", "application/x-www-form-urlencoded"); } bool PasswordFieldPresent = false; foreach (HtmlNode InputNode in FormNode.ChildNodes) { string Name = ""; string Value = ""; foreach (HtmlAttribute Attr in InputNode.Attributes) { switch (Attr.Name) { case ("name"): Name = Attr.Value; break; case ("type"): if (Attr.Value.Equals("password", StringComparison.OrdinalIgnoreCase)) { PasswordFieldPresent = true; } break; case ("value"): Value = Attr.Value; break; } } if (FillEmptyFields && Value.Length == 0) { Value = Tools.GetRandomString(2, 5); } if (Name.Length > 0) { if (SubReq.Method.Equals("GET")) { SubReq.Query.Add(Name, Value); } else { SubReq.Body.Add(Name, Value); } } } SubReq.SetCookie(Cookies); if (LoginFormOnly) { if (PasswordFieldPresent) { if ((SubReq.Method.Equals("GET", StringComparison.OrdinalIgnoreCase) && SubReq.Query.Count >= 3) || (SubReq.Method.Equals("POST", StringComparison.OrdinalIgnoreCase) && SubReq.Body.Count >= 3)) { return(SubReq); } } } else { return(SubReq); } return(null); }
public Request GetRedirect(Response Res) { if (Res.Code == 301 || Res.Code == 302 || Res.Code == 303 || Res.Code == 307) { if (Res.Headers.Has("Location")) { string NewUrl = Res.Headers.Get("Location"); Request NewReq = new Request(this.FullURL); if (NewUrl.StartsWith("/")) { NewReq.URL = NewUrl; } else if (NewUrl.StartsWith("http://") || NewUrl.StartsWith("https://")) { NewReq.FullURL = NewUrl; } else if (NewUrl.StartsWith("..")) { int DirBackNumber = 0; List <string> UpdatedNewUrlPathParts = NewReq.UrlPathParts; List <string> NewUrlPathParts = new List <string>(NewUrl.Split('/')); if (!NewReq.Url.EndsWith("/") && UpdatedNewUrlPathParts.Count > 0) { UpdatedNewUrlPathParts.RemoveAt(UpdatedNewUrlPathParts.Count - 1); } foreach (string NewUrlPathPart in NewUrlPathParts) { if (NewUrlPathPart.Equals("..")) { DirBackNumber++; } else { break; } } while (UpdatedNewUrlPathParts.Count > 0 && DirBackNumber > 0) { UpdatedNewUrlPathParts.RemoveAt(UpdatedNewUrlPathParts.Count - 1); if (NewUrlPathParts.Count > 0) { NewUrlPathParts.RemoveAt(0); } DirBackNumber--; } UpdatedNewUrlPathParts.AddRange(NewUrlPathParts); NewReq.UrlPathParts = UpdatedNewUrlPathParts; if (NewUrl.EndsWith("/")) { if (!NewReq.Url.EndsWith("/")) { NewReq.Url = NewReq.Url + "/"; } } else { if (NewReq.Url.EndsWith("/") && NewReq.Url.Length > 1) { NewReq.Url = NewReq.Url.TrimEnd(new char[] { '/' }); } } } else { List <string> NewUrlPathParts = NewReq.UrlPathParts; if (!NewReq.Url.EndsWith("/") && NewUrlPathParts.Count > 0) { NewUrlPathParts.RemoveAt(NewUrlPathParts.Count - 1); } NewReq.UrlPathParts = NewUrlPathParts; if (!NewReq.Url.EndsWith("/")) { NewReq.Url = NewReq.Url + "/"; } NewReq.Url = NewReq.Url + NewUrl; } //this check is needed since sometimes the redirect can go to a different domain if (this.Host == NewReq.Host) { NewReq.CookieString = this.CookieString; NewReq.SetCookie(Res.SetCookies); } return(NewReq); } } return(null); }
virtual public Request Update(Request Request, Response Response) { Request.SetCookie(Response); return(Request); }
public Request GetRedirect(Response Res) { if (Res.Code == 301 || Res.Code == 302 || Res.Code == 303 || Res.Code == 307) { if (Res.Headers.Has("Location")) { string NewUrl = Res.Headers.Get("Location"); Request NewReq = new Request(this.FullURL); if (NewUrl.StartsWith("/")) { NewReq.URL = NewUrl; return NewReq; } if (NewUrl.StartsWith("http://") || NewUrl.StartsWith("https://")) { NewReq.FullURL = NewUrl; if (this.Host == NewReq.Host) { NewReq.CookieString = this.CookieString; NewReq.SetCookie(Res.SetCookies); } return NewReq; } } } return null; }