public Trigger(string RequestTrigger, Request Req, string ResponseTrigger, Response Res) { this.RequestTrigger = RequestTrigger; this.Request = Req.GetClone(); this.ResponseTrigger = ResponseTrigger; this.Response = Res.GetClone(); }
public static void AddProbeString(string ProbeString, Request InjectedRequest) { Request ClonedReq = InjectedRequest.GetClone(); lock (ProbeStrings) { ProbeStrings.Add(ProbeString, ClonedReq); } }
public Trigger(string RequestTrigger, string RequestTriggerDescription, Request Req, string ResponseTrigger, string ResponseTriggerDescription, Response Res) { this.RequestTrigger = RequestTrigger; this.RequestTriggerDescription = RequestTriggerDescription; this.Request = Req.GetClone(); this.ResponseTrigger = ResponseTrigger; this.RawResponseTriggerDescription = ResponseTriggerDescription; this.Response = Res.GetClone(); }
string NormalizeUrl(Request Req, string RawLink) { if (RawLink.IndexOf('#') > -1) { RawLink = RawLink.Substring(0, RawLink.IndexOf('#')); } if (RawLink.StartsWith("http://") || RawLink.StartsWith("https://")) { return RawLink; } else if (RawLink.StartsWith("//")) { if (Req.SSL) RawLink = "https:" + RawLink; else RawLink = "http:" + RawLink; } else if (RawLink.StartsWith("/")) { Request TempReq = Req.GetClone(); TempReq.Url = RawLink; return TempReq.FullUrl; } else if (RawLink.StartsWith("javascript:") || RawLink.StartsWith("file:")) { //ignore } else { List<string> UrlPathParts = Req.UrlPathParts; if (UrlPathParts.Count > 0) { if (!Req.Url.EndsWith("/")) UrlPathParts.RemoveAt(UrlPathParts.Count - 1); } if (RawLink.StartsWith("../")) { string[] RawUrlParts = RawLink.Split(new char[] { '/' }); List<string> TreatedRawUrlParts = new List<string>(RawUrlParts); foreach (string Part in RawUrlParts) { if (Part.Equals("..") && (UrlPathParts.Count > 0)) { UrlPathParts.RemoveAt(UrlPathParts.Count - 1); TreatedRawUrlParts.RemoveAt(0); } else { break; } } StringBuilder TreatedRawUrlBuilder = new StringBuilder("/"); foreach (string RawPart in TreatedRawUrlParts) { TreatedRawUrlBuilder.Append(RawPart); } string TreatedRawUrl = TreatedRawUrlBuilder.ToString(); if (!RawLink.EndsWith("/")) { TreatedRawUrl = TreatedRawUrl.TrimEnd(new char[] { '/' }); } Request TempReq = Req.GetClone(); Request NormaliserRequest = new Request(TempReq.FullUrl); NormaliserRequest.Url = "/"; NormaliserRequest.UrlPathParts = UrlPathParts; TempReq.Url = NormaliserRequest.Url; TempReq.Url = TempReq.Url.TrimEnd(new char[] { '/' }) + TreatedRawUrl; return TempReq.FullUrl; } else if (RawLink.Length > 0) { Request TempReq = Req.GetClone(); Request NormaliserRequest = new Request(TempReq.FullUrl); NormaliserRequest.Url = "/"; NormaliserRequest.UrlPathParts = UrlPathParts; TempReq.Url = NormaliserRequest.Url; if (TempReq.Url.EndsWith("/")) { TempReq.Url = TempReq.Url + RawLink; } else { TempReq.Url = TempReq.Url + "/" + RawLink; } return TempReq.FullURL; } } return ""; }
bool IsA404(Request Req, Response Res) { Response NotFoundResponse; if (NotFoundSignatures.ContainsKey(Req.SSL.ToString() + Req.Host + Req.File)) NotFoundResponse = NotFoundSignatures[Req.SSL.ToString() + Req.Host + Req.File]; else { Request NotFoundGetter = Req.GetClone(); NotFoundGetter.Method = "GET"; NotFoundGetter.Body.RemoveAll(); if(Req.File.Length > 0) NotFoundGetter.Url = NotFoundGetter.UrlDir + "should_not_xist_" + Tools.GetRandomString(10,15) + "." + Req.File; else NotFoundGetter.Url = NotFoundGetter.UrlDir + "should_not_xist_" + Tools.GetRandomString(10, 15); NotFoundResponse = NotFoundGetter.Send(); NotFoundResponse.Flags.Add("Url", NotFoundGetter.Url); lock (NotFoundSignatures) { if (!NotFoundSignatures.ContainsKey(Req.SSL.ToString() + Req.Host + Req.File)) NotFoundSignatures.Add(Req.SSL.ToString() + Req.Host + Req.File, NotFoundResponse); } } if(Res.Code == 200 && NotFoundResponse.Code != 200) return false; if(Res.Code == 404) return true; if(Res.Code == 403) { if(NotFoundResponse.Code == 403) return true; else return false; } if (Res.Code > 400) { if (NotFoundResponse.Code == Res.Code) return true; else return false; } string NotFoundGetterUrl = NotFoundResponse.Flags["Url"].ToString(); if (Res.Code == 301 || Res.Code == 302 || Res.Code == 303 || Res.Code == 307) { string RedirectedUrl = Res.Headers.Get("Location"); if (NotFoundResponse.Code == 301 || NotFoundResponse.Code == 302 || NotFoundResponse.Code == 303 || NotFoundResponse.Code == 307) { string NotFoundRedirectedUrl = NotFoundResponse.Headers.Get("Location"); if (RedirectedUrl.ToLower().Equals(NotFoundRedirectedUrl.ToLower())) return true; else if (Regex.IsMatch(RedirectedUrl, @".*not\Wfound.*", RegexOptions.IgnoreCase)) return true; else if (NotFoundRedirectedUrl.Replace(NotFoundGetterUrl,"").Equals(RedirectedUrl.Replace(Req.Url, ""))) return true; else { Request RedirectedLocationReq; if (RedirectedUrl.StartsWith("http://") || RedirectedUrl.StartsWith("https://")) { RedirectedLocationReq = new Request(RedirectedUrl); } else if (RedirectedUrl.StartsWith("/")) { RedirectedLocationReq = Req.GetClone(); RedirectedLocationReq.Url = RedirectedUrl; } else { return true; } Request NotFoundRedirectedLocationReq; if (NotFoundRedirectedUrl.StartsWith("http://") || NotFoundRedirectedUrl.StartsWith("https://")) { NotFoundRedirectedLocationReq = new Request(NotFoundRedirectedUrl); } else if (NotFoundRedirectedUrl.StartsWith("/")) { NotFoundRedirectedLocationReq = Req.GetClone(); NotFoundRedirectedLocationReq.Url = NotFoundRedirectedUrl; } else { return false; } if (RedirectedLocationReq.Url.Equals(NotFoundRedirectedLocationReq.Url)) return true; } } else return false; } return false; }
public Trigger(string RequestTrigger, string RequestTriggerDescription, Request Req) { this.RequestTrigger = RequestTrigger; this.RequestTriggerDescription = RequestTriggerDescription; this.Request = Req.GetClone(); }
public virtual bool Is(Request Request) { try { Request Req = Request.GetClone(); string XmlString = ToXmlFromRequest(Req); if (!Tools.IsXml(XmlString)) return false; Request FinalReq = ToRequestFromXml(Req, XmlString); if (Req.ToString().Equals(FinalReq.ToString())) return true; } catch { return false; } return false; }
public static List <Request> GetFormSubmissions(Request Req, Response Res, CookieStore Cookies) { List <Request> FormSubmissions = new List <Request>(); List <HtmlNode> FormNodes = Res.Html.GetForms(); foreach (HtmlNode FormNode in FormNodes) { Request SubReq = Req.GetClone(); SubReq.Method = "GET"; SubReq.BodyString = ""; foreach (HtmlAttribute Attr in FormNode.Attributes) { if (Attr.Name.Equals("method")) { SubReq.Method = Attr.Value.ToUpper(); } else if (Attr.Name.Equals("action")) { if (Attr.Value.StartsWith("javascript:")) { continue; } string ActionUrl = NormalizeUrl(Req, Tools.HtmlDecode(Attr.Value.Trim())); if (ActionUrl.Length > 0) { SubReq.FullUrl = ActionUrl; } } } if (SubReq.Method == "GET") { SubReq.Query.RemoveAll(); } else { SubReq.Headers.Set("Content-Type", "application/x-www-form-urlencoded"); } foreach (HtmlNode InputNode in FormNode.ChildNodes) { string Name = ""; string Value = ""; foreach (HtmlAttribute Attr in InputNode.Attributes) { switch (Attr.Name) { case ("name"): Name = Attr.Value; break; case ("type"): if (Attr.Value.Equals("submit")) { Name = ""; } break; case ("value"): Value = Attr.Value; break; } } if (Value.Length == 0) { Value = Tools.GetRandomString(2, 5); } if (Name.Length > 0) { if (SubReq.Method.Equals("GET")) { SubReq.Query.Add(Name, Value); } else { SubReq.Body.Add(Name, Value); } } } SubReq.SetCookie(Cookies); FormSubmissions.Add(SubReq); } return(FormSubmissions); }
void Crawl(Request Req, int Depth, bool Scraped) { if (Stopped) { return; } if (Depth > MaxDepth) { return; } if (WasCrawled(Req)) { return; } if (!CanCrawl(Req)) { return; } lock (PageSignatures) { PageSignatures.Add(GetPageSignature(Req)); } Req.Source = RequestSource.Probe; Req.SetCookie(Cookies); if (UserAgent.Length > 0) { Req.Headers.Set("User-Agent", UserAgent); } Response Res = Req.Send(); Cookies.Add(Req, Res); bool Is404File = IsA404(Req, Res); if (!Res.IsHtml) { return; } if (Depth + 1 > MaxDepth) { return; } List <Request> Redirects = GetRedirects(Req, Res); foreach (Request Redirect in Redirects) { AddToCrawlQueue(Redirect, Depth + 1, true); } List <Request> LinkClicks = GetLinkClicks(Req, Res); foreach (Request LinkClick in LinkClicks) { AddToCrawlQueue(LinkClick, Depth + 1, true); } List <Request> FormSubmissions = GetFormSubmissions(Req, Res); foreach (Request FormSubmission in FormSubmissions) { AddToCrawlQueue(FormSubmission, Depth + 1, true); } Request DirCheck = Req.GetClone(); DirCheck.Method = "GET"; DirCheck.Body.RemoveAll(); DirCheck.Url = DirCheck.UrlDir; if (!Req.Url.EndsWith("/")) { AddToCrawlQueue(DirCheck, Depth + 1, false); } if (PerformDirAndFileGuessing && !Is404File) { foreach (string File in FileNamesToCheck) { Request FileCheck = DirCheck.GetClone(); FileCheck.Url = FileCheck.Url + File; AddToCrawlQueue(FileCheck, Depth + 1, false); } foreach (string Dir in DirNamesToCheck) { Request DirectoryCheck = DirCheck.GetClone(); DirectoryCheck.Url = DirectoryCheck.Url + Dir + "/"; AddToCrawlQueue(DirectoryCheck, Depth + 1, false); } } if (Scraped || !Is404File) { lock (CrawledRequests) { CrawledRequests.Enqueue(Req); } IronUpdater.AddToSiteMap(Req); } }
public static List<Request> GetFormSubmissions(Request Req, Response Res, CookieStore Cookies) { List<Request> FormSubmissions = new List<Request>(); List<HtmlNode> FormNodes = Res.Html.GetForms(); foreach (HtmlNode FormNode in FormNodes) { Request SubReq = Req.GetClone(); SubReq.Method = "GET"; SubReq.BodyString = ""; foreach (HtmlAttribute Attr in FormNode.Attributes) { if (Attr.Name.Equals("method")) { SubReq.Method = Attr.Value.ToUpper(); } else if(Attr.Name.Equals("action")) { if (Attr.Value.StartsWith("javascript:")) continue; string ActionUrl = NormalizeUrl(Req, Tools.HtmlDecode(Attr.Value.Trim())); if (ActionUrl.Length > 0) { SubReq.FullUrl = ActionUrl; } } } if (SubReq.Method == "GET") { SubReq.Query.RemoveAll(); } else { SubReq.Headers.Set("Content-Type", "application/x-www-form-urlencoded"); } foreach (HtmlNode InputNode in FormNode.ChildNodes) { string Name = ""; string Value = ""; foreach (HtmlAttribute Attr in InputNode.Attributes) { switch(Attr.Name) { case("name"): Name = Attr.Value; break; case("type"): if(Attr.Value.Equals("submit")) Name = ""; break; case("value"): Value = Attr.Value; break; } } if (Value.Length == 0) { Value = Tools.GetRandomString(2,5); } if (Name.Length > 0) { if (SubReq.Method.Equals("GET")) SubReq.Query.Add(Name, Value); else SubReq.Body.Add(Name, Value); } } SubReq.SetCookie(Cookies); FormSubmissions.Add(SubReq); } return FormSubmissions; }
internal static void AddShellRequest(Request Request) { if (Request != null) { try { Request ClonedRequest = Request.GetClone(true); if (ClonedRequest != null) { lock (ShellRequestQ) { ShellRequestQ.Enqueue(ClonedRequest); } } else { Tools.Trace("IronUpdater", "Null Shell Request"); } } catch (Exception Exp) { IronException.Report("Error adding Shell Request for updating", Exp.Message, Exp.StackTrace); } } }
internal static string NormalizeUrl(Request Req, string RawLink) { if (RawLink.IndexOf('#') > -1) { RawLink = RawLink.Substring(0, RawLink.IndexOf('#')); } if (RawLink.StartsWith("http://") || RawLink.StartsWith("https://")) { return(RawLink); } else if (RawLink.StartsWith("//")) { if (Req.SSL) { RawLink = string.Format("https:{0}", RawLink); } else { RawLink = string.Format("http:{0}", RawLink); } } else if (RawLink.StartsWith("/")) { Request TempReq = Req.GetClone(); TempReq.Url = RawLink; return(TempReq.FullUrl); } else if (RawLink.StartsWith("javascript:") || RawLink.StartsWith("file:")) { //ignore } else { List <string> UrlPathParts = Req.UrlPathParts; if (UrlPathParts.Count > 0) { if (!Req.Url.EndsWith("/")) { UrlPathParts.RemoveAt(UrlPathParts.Count - 1); } } if (RawLink.StartsWith("../")) { string[] RawUrlParts = RawLink.Split(new char[] { '/' }); List <string> TreatedRawUrlParts = new List <string>(RawUrlParts); foreach (string Part in RawUrlParts) { if (Part.Equals("..") && (UrlPathParts.Count > 0)) { UrlPathParts.RemoveAt(UrlPathParts.Count - 1); TreatedRawUrlParts.RemoveAt(0); } else { break; } } UrlPathParts.AddRange(TreatedRawUrlParts); StringBuilder FinalUrlBuilder = new StringBuilder(); foreach (string UrlPart in UrlPathParts) { FinalUrlBuilder.Append("/"); FinalUrlBuilder.Append(UrlPart); } Request TempReq = Req.GetClone(); TempReq.Url = FinalUrlBuilder.ToString(); return(TempReq.FullUrl); } else if (RawLink.Length > 0) { return(string.Format("{0}{1}{2}", Req.BaseUrl.TrimEnd('/'), Req.UrlDir, RawLink)); } } return(""); }
public static Request GetFormSubmission(Request Req, HtmlNode FormNode, CookieStore Cookies, bool LoginFormOnly, bool FillEmptyFields) { //Login request signatures: //form must have one password type input field //three or more parameters must be present in the request query/body Request SubReq = Req.GetClone(); SubReq.Method = "GET"; SubReq.BodyString = ""; foreach (HtmlAttribute Attr in FormNode.Attributes) { if (Attr.Name.Equals("method")) { SubReq.Method = Attr.Value.ToUpper(); } else if (Attr.Name.Equals("action")) { if (Attr.Value.StartsWith("javascript:")) { continue; } string ActionUrl = NormalizeUrl(Req, Tools.HtmlDecode(Attr.Value.Trim())); if (ActionUrl.Length > 0) { SubReq.FullUrl = ActionUrl; } } } if (SubReq.Method == "GET") { SubReq.Query.RemoveAll(); } else { SubReq.Headers.Set("Content-Type", "application/x-www-form-urlencoded"); } bool PasswordFieldPresent = false; foreach (HtmlNode InputNode in FormNode.ChildNodes) { string Name = ""; string Value = ""; foreach (HtmlAttribute Attr in InputNode.Attributes) { switch (Attr.Name) { case ("name"): Name = Attr.Value; break; case ("type"): if (Attr.Value.Equals("password", StringComparison.OrdinalIgnoreCase)) { PasswordFieldPresent = true; } break; case ("value"): Value = Attr.Value; break; } } if (FillEmptyFields && Value.Length == 0) { Value = Tools.GetRandomString(2, 5); } if (Name.Length > 0) { if (SubReq.Method.Equals("GET")) { SubReq.Query.Add(Name, Value); } else { SubReq.Body.Add(Name, Value); } } } SubReq.SetCookie(Cookies); if (LoginFormOnly) { if (PasswordFieldPresent) { if ((SubReq.Method.Equals("GET", StringComparison.OrdinalIgnoreCase) && SubReq.Query.Count >= 3) || (SubReq.Method.Equals("POST", StringComparison.OrdinalIgnoreCase) && SubReq.Body.Count >= 3)) { return(SubReq); } } } else { return(SubReq); } return(null); }
public void StoreRequest(Request Request) { ManualTesting.StoreRequest(Request.GetClone()); }
bool IsA404(Request Req, Response Res) { Response NotFoundResponse = null; lock (NotFoundSignatures) { if (NotFoundSignatures.ContainsKey(string.Format("{0}{1}{2}{3}", Req.SSL, Req.Host, Req.UrlDir, Req.File))) { NotFoundResponse = NotFoundSignatures[string.Format("{0}{1}{2}{3}", Req.SSL, Req.Host, Req.UrlDir, Req.File)]; } } if(NotFoundResponse == null) { Request NotFoundGetter = Req.GetClone(); NotFoundGetter.Method = "GET"; NotFoundGetter.Body.RemoveAll(); if (Req.File.Length > 0) NotFoundGetter.Url = string.Format("{0}should_not_xist_{1}.{2}", NotFoundGetter.UrlDir, Tools.GetRandomString(10, 15), Req.File); else NotFoundGetter.Url = string.Format("{0}should_not_xist_{1}", NotFoundGetter.UrlDir, Tools.GetRandomString(10, 15)); NotFoundResponse = NotFoundGetter.Send(); if (Stopped) return true; NotFoundResponse.BodyString = ""; List<string> HeaderNames = NotFoundResponse.Headers.GetNames(); foreach (string HeaderName in HeaderNames) { if (!HeaderName.Equals("Location")) { NotFoundResponse.Headers.Remove(HeaderName); } } NotFoundResponse.Flags.Add("Url", NotFoundGetter.Url); lock (NotFoundSignatures) { if (!NotFoundSignatures.ContainsKey(string.Format("{0}{1}{2}{3}", Req.SSL, Req.Host, Req.UrlDir, Req.File))) NotFoundSignatures.Add(string.Format("{0}{1}{2}{3}", Req.SSL, Req.Host, Req.UrlDir, Req.File), NotFoundResponse); } } if(Res.Code == 200 && NotFoundResponse.Code != 200) return false; if(Res.Code == 404) return true; if (Res.Code > 400) { if (NotFoundResponse.Code == Res.Code) return true; else return false; } string NotFoundGetterUrl = NotFoundResponse.Flags["Url"].ToString(); if (Res.Code == 301 || Res.Code == 302 || Res.Code == 303 || Res.Code == 307) { string RedirectedUrl = Res.Headers.Get("Location"); if (NotFoundResponse.Code == 301 || NotFoundResponse.Code == 302 || NotFoundResponse.Code == 303 || NotFoundResponse.Code == 307) { string NotFoundRedirectedUrl = NotFoundResponse.Headers.Get("Location"); if (RedirectedUrl.ToLower().Equals(NotFoundRedirectedUrl.ToLower())) return true; else if (Regex.IsMatch(RedirectedUrl, @".*not\Wfound.*", RegexOptions.IgnoreCase)) return true; else if (NotFoundRedirectedUrl.Replace(NotFoundGetterUrl,"").Equals(RedirectedUrl.Replace(Req.Url, ""))) return true; else { Request RedirectedLocationReq; if (RedirectedUrl.StartsWith("http://") || RedirectedUrl.StartsWith("https://")) { RedirectedLocationReq = new Request(RedirectedUrl); } else if (RedirectedUrl.StartsWith("/")) { RedirectedLocationReq = Req.GetClone(); RedirectedLocationReq.Url = RedirectedUrl; } else { return true; } Request NotFoundRedirectedLocationReq; if (NotFoundRedirectedUrl.StartsWith("http://") || NotFoundRedirectedUrl.StartsWith("https://")) { NotFoundRedirectedLocationReq = new Request(NotFoundRedirectedUrl); } else if (NotFoundRedirectedUrl.StartsWith("/")) { NotFoundRedirectedLocationReq = Req.GetClone(); NotFoundRedirectedLocationReq.Url = NotFoundRedirectedUrl; } else { return false; } if (RedirectedLocationReq.Url.Equals(NotFoundRedirectedLocationReq.Url)) return true; } } else return false; } return false; }
Request GetTestRequest(Request Req) { Request TestRequest = Req.GetClone(); switch (TestType) { case(AddParameterTest): return AddParameter(TestRequest); case(EditParameterTest): return EditParameter(TestRequest); case(DeleteParameterTest): return DeleteParameter(TestRequest); } return TestRequest; }
public Request InjectInRequest(Request Req, int InjectionPoint, string Payload) { string CurrentRequestHash = Tools.MD5(Req.ToString()); string XML = ""; if(this.RequestHash.Equals(CurrentRequestHash)) { XML = this.RequestXml; } else { XML = this.ToXmlFromRequest(Req); this.RequestXml = XML; this.RequestHash = CurrentRequestHash; } string InjectedXml = InjectInXml(this.RequestXml, InjectionPoint, Payload); return this.ToRequestFromXml(Req.GetClone(true), InjectedXml); }
public Trigger(string RequestTrigger, Request Req) { this.RequestTrigger = RequestTrigger; this.Request = Req.GetClone(); }
bool IsA404(Request Req, Response Res) { Response NotFoundResponse = null; lock (NotFoundSignatures) { if (NotFoundSignatures.ContainsKey(Req.SSL.ToString() + Req.Host + Req.UrlDir + Req.File)) { NotFoundResponse = NotFoundSignatures[Req.SSL.ToString() + Req.Host + Req.UrlDir + Req.File]; } } if (NotFoundResponse == null) { Request NotFoundGetter = Req.GetClone(); NotFoundGetter.Method = "GET"; NotFoundGetter.Body.RemoveAll(); if (Req.File.Length > 0) { NotFoundGetter.Url = NotFoundGetter.UrlDir + "should_not_xist_" + Tools.GetRandomString(10, 15) + "." + Req.File; } else { NotFoundGetter.Url = NotFoundGetter.UrlDir + "should_not_xist_" + Tools.GetRandomString(10, 15); } NotFoundResponse = NotFoundGetter.Send(); NotFoundResponse.BodyString = ""; List <string> HeaderNames = NotFoundResponse.Headers.GetNames(); foreach (string HeaderName in HeaderNames) { if (!HeaderName.Equals("Location")) { NotFoundResponse.Headers.Remove(HeaderName); } } NotFoundResponse.Flags.Add("Url", NotFoundGetter.Url); lock (NotFoundSignatures) { if (!NotFoundSignatures.ContainsKey(Req.SSL.ToString() + Req.Host + Req.UrlDir + Req.File)) { NotFoundSignatures.Add(Req.SSL.ToString() + Req.Host + Req.UrlDir + Req.File, NotFoundResponse); } } } if (Res.Code == 200 && NotFoundResponse.Code != 200) { return(false); } if (Res.Code == 404) { return(true); } if (Res.Code > 400) { if (NotFoundResponse.Code == Res.Code) { return(true); } else { return(false); } } string NotFoundGetterUrl = NotFoundResponse.Flags["Url"].ToString(); if (Res.Code == 301 || Res.Code == 302 || Res.Code == 303 || Res.Code == 307) { string RedirectedUrl = Res.Headers.Get("Location"); if (NotFoundResponse.Code == 301 || NotFoundResponse.Code == 302 || NotFoundResponse.Code == 303 || NotFoundResponse.Code == 307) { string NotFoundRedirectedUrl = NotFoundResponse.Headers.Get("Location"); if (RedirectedUrl.ToLower().Equals(NotFoundRedirectedUrl.ToLower())) { return(true); } else if (Regex.IsMatch(RedirectedUrl, @".*not\Wfound.*", RegexOptions.IgnoreCase)) { return(true); } else if (NotFoundRedirectedUrl.Replace(NotFoundGetterUrl, "").Equals(RedirectedUrl.Replace(Req.Url, ""))) { return(true); } else { Request RedirectedLocationReq; if (RedirectedUrl.StartsWith("http://") || RedirectedUrl.StartsWith("https://")) { RedirectedLocationReq = new Request(RedirectedUrl); } else if (RedirectedUrl.StartsWith("/")) { RedirectedLocationReq = Req.GetClone(); RedirectedLocationReq.Url = RedirectedUrl; } else { return(true); } Request NotFoundRedirectedLocationReq; if (NotFoundRedirectedUrl.StartsWith("http://") || NotFoundRedirectedUrl.StartsWith("https://")) { NotFoundRedirectedLocationReq = new Request(NotFoundRedirectedUrl); } else if (NotFoundRedirectedUrl.StartsWith("/")) { NotFoundRedirectedLocationReq = Req.GetClone(); NotFoundRedirectedLocationReq.Url = NotFoundRedirectedUrl; } else { return(false); } if (RedirectedLocationReq.Url.Equals(NotFoundRedirectedLocationReq.Url)) { return(true); } } } else { return(false); } } return(false); }
static string NormalizeUrl(Request Req, string RawLink) { if (RawLink.IndexOf('#') > -1) { RawLink = RawLink.Substring(0, RawLink.IndexOf('#')); } if (RawLink.StartsWith("http://") || RawLink.StartsWith("https://")) { return(RawLink); } else if (RawLink.StartsWith("//")) { if (Req.SSL) { RawLink = "https:" + RawLink; } else { RawLink = "http:" + RawLink; } } else if (RawLink.StartsWith("/")) { Request TempReq = Req.GetClone(); TempReq.Url = RawLink; return(TempReq.FullUrl); } else if (RawLink.StartsWith("javascript:") || RawLink.StartsWith("file:")) { //ignore } else { List <string> UrlPathParts = Req.UrlPathParts; if (UrlPathParts.Count > 0) { if (!Req.Url.EndsWith("/")) { UrlPathParts.RemoveAt(UrlPathParts.Count - 1); } } if (RawLink.StartsWith("../")) { string[] RawUrlParts = RawLink.Split(new char[] { '/' }); List <string> TreatedRawUrlParts = new List <string>(RawUrlParts); foreach (string Part in RawUrlParts) { if (Part.Equals("..") && (UrlPathParts.Count > 0)) { UrlPathParts.RemoveAt(UrlPathParts.Count - 1); TreatedRawUrlParts.RemoveAt(0); } else { break; } } StringBuilder TreatedRawUrlBuilder = new StringBuilder("/"); foreach (string RawPart in TreatedRawUrlParts) { TreatedRawUrlBuilder.Append(RawPart); } string TreatedRawUrl = TreatedRawUrlBuilder.ToString(); if (!RawLink.EndsWith("/")) { TreatedRawUrl = TreatedRawUrl.TrimEnd(new char[] { '/' }); } Request TempReq = Req.GetClone(); Request NormaliserRequest = new Request(TempReq.FullUrl); NormaliserRequest.Url = "/"; NormaliserRequest.UrlPathParts = UrlPathParts; TempReq.Url = NormaliserRequest.Url; TempReq.Url = TempReq.Url.TrimEnd(new char[] { '/' }) + TreatedRawUrl; return(TempReq.FullUrl); } else if (RawLink.Length > 0) { Request TempReq = Req.GetClone(); Request NormaliserRequest = new Request(TempReq.FullUrl); NormaliserRequest.Url = "/"; NormaliserRequest.UrlPathParts = UrlPathParts; TempReq.Url = NormaliserRequest.Url; if (TempReq.Url.EndsWith("/")) { TempReq.Url = TempReq.Url + RawLink; } else { TempReq.Url = TempReq.Url + "/" + RawLink; } return(TempReq.FullURL); } } return(""); }
void Crawl(Request Req, int Depth, bool Scraped) { if (Stopped) return; if (Depth > MaxDepth) return; if (WasCrawled(Req)) return; if (!CanCrawl(Req)) return; lock (PageSignatures) { PageSignatures.Add(GetPageSignature(Req)); } Req.Source = RequestSource.Probe; Req.SetCookie(Cookies); if (UserAgent.Length > 0) Req.Headers.Set("User-Agent", UserAgent); Response Res = Req.Send(); Cookies.Add(Req, Res); bool Is404File = IsA404(Req, Res); if (!Res.IsHtml) { try { Res.ProcessHtml(); } catch { return; } } if (Depth + 1 > MaxDepth) return; List<Request> LinkClicks = GetLinkClicks(Req, Res); foreach (Request LinkClick in LinkClicks) { AddToCrawlQueue(LinkClick, Depth + 1, true); } List<Request> FormSubmissions = GetFormSubmissionRequests(Req, Res); foreach (Request FormSubmission in FormSubmissions) { AddToCrawlQueue(FormSubmission, Depth + 1, true); } Request DirCheck = Req.GetClone(); DirCheck.Method = "GET"; DirCheck.Body.RemoveAll(); DirCheck.Url = DirCheck.UrlDir; if (!Req.Url.EndsWith("/")) { AddToCrawlQueue(DirCheck, Depth + 1, false); } if (PerformDirAndFileGuessing && !Is404File) { foreach (string File in FileNamesToCheck) { Request FileCheck = DirCheck.GetClone(); FileCheck.Url = FileCheck.Url + File; AddToCrawlQueue(FileCheck, Depth + 1, false); } foreach (string Dir in DirNamesToCheck) { Request DirectoryCheck = DirCheck.GetClone(); DirectoryCheck.Url = DirectoryCheck.Url + Dir + "/"; AddToCrawlQueue(DirectoryCheck, Depth + 1, false); } } if (Scraped || !Is404File) { lock (CrawledRequests) { CrawledRequests.Enqueue(Req); } IronUpdater.AddToSiteMap(Req); } }
public Scanner(Request Request) { this.OriginalRequest = Request.GetClone(); }
List<Request> GetFormSubmissionRequests(Request Req, Response Res) { List<Request> FormSubmissions = new List<Request>(); List<HtmlNode> FormNodes = Res.Html.GetForms(); foreach (HtmlNode FormNode in FormNodes) { Request SubReq = Req.GetClone(); SubReq.Method = "GET"; SubReq.Body.RemoveAll(); foreach (HtmlAttribute Attr in FormNode.Attributes) { if (Attr.Name.Equals("method")) { SubReq.Method = Attr.Value.ToUpper(); } if (SubReq.Method.Equals("GET")) SubReq.Query.RemoveAll(); if (Attr.Name.Equals("action")) { if (Attr.Value.StartsWith("javascript:")) continue; string ActionUrl = NormalizeUrl(Req, Attr.Value.Trim()); if (ActionUrl.Length > 0) { SubReq.FullUrl = ActionUrl; } } } foreach (HtmlNode InputNode in FormNode.ChildNodes) { string Name = ""; string Value = ""; foreach (HtmlAttribute Attr in InputNode.Attributes) { switch(Attr.Name) { case("name"): Name = Attr.Value; break; case("type"): if(Attr.Value.Equals("submit")) Name = ""; break; case("value"): Value = Attr.Value; break; } } if (Value.Length == 0) { Value = Tools.GetRandomString(2,5); } if (Name.Length > 0) { if (SubReq.Method.Equals("GET")) SubReq.Query.Add(Name, Value); else SubReq.Body.Add(Name, Value); } } FormSubmissions.Add(SubReq); } return FormSubmissions; }