public Trigger(string RequestTrigger, Request Req, string ResponseTrigger, Response Res)
{
this.RequestTrigger = RequestTrigger;
this.Request = Req.GetClone();
this.ResponseTrigger = ResponseTrigger;
this.Response = Res.GetClone();
}
public static void AddProbeString(string ProbeString, Request InjectedRequest)
{
Request ClonedReq = InjectedRequest.GetClone();
lock (ProbeStrings)
{
ProbeStrings.Add(ProbeString, ClonedReq);
}
}
public Trigger(string RequestTrigger, string RequestTriggerDescription, Request Req, string ResponseTrigger, string ResponseTriggerDescription, Response Res)
{
this.RequestTrigger = RequestTrigger;
this.RequestTriggerDescription = RequestTriggerDescription;
this.Request = Req.GetClone();
this.ResponseTrigger = ResponseTrigger;
this.RawResponseTriggerDescription = ResponseTriggerDescription;
this.Response = Res.GetClone();
}
string NormalizeUrl(Request Req, string RawLink)
{
if (RawLink.IndexOf('#') > -1)
{
RawLink = RawLink.Substring(0, RawLink.IndexOf('#'));
}
if (RawLink.StartsWith("http://") || RawLink.StartsWith("https://"))
{
return RawLink;
}
else if (RawLink.StartsWith("//"))
{
if (Req.SSL)
RawLink = "https:" + RawLink;
else
RawLink = "http:" + RawLink;
}
else if (RawLink.StartsWith("/"))
{
Request TempReq = Req.GetClone();
TempReq.Url = RawLink;
return TempReq.FullUrl;
}
else if (RawLink.StartsWith("javascript:") || RawLink.StartsWith("file:"))
{
//ignore
}
else
{
List<string> UrlPathParts = Req.UrlPathParts;
if (UrlPathParts.Count > 0)
{
if (!Req.Url.EndsWith("/")) UrlPathParts.RemoveAt(UrlPathParts.Count - 1);
}
if (RawLink.StartsWith("../"))
{
string[] RawUrlParts = RawLink.Split(new char[] { '/' });
List<string> TreatedRawUrlParts = new List<string>(RawUrlParts);
foreach (string Part in RawUrlParts)
{
if (Part.Equals("..") && (UrlPathParts.Count > 0))
{
UrlPathParts.RemoveAt(UrlPathParts.Count - 1);
TreatedRawUrlParts.RemoveAt(0);
}
else
{
break;
}
}
StringBuilder TreatedRawUrlBuilder = new StringBuilder("/");
foreach (string RawPart in TreatedRawUrlParts)
{
TreatedRawUrlBuilder.Append(RawPart);
}
string TreatedRawUrl = TreatedRawUrlBuilder.ToString();
if (!RawLink.EndsWith("/"))
{
TreatedRawUrl = TreatedRawUrl.TrimEnd(new char[] { '/' });
}
Request TempReq = Req.GetClone();
Request NormaliserRequest = new Request(TempReq.FullUrl);
NormaliserRequest.Url = "/";
NormaliserRequest.UrlPathParts = UrlPathParts;
TempReq.Url = NormaliserRequest.Url;
TempReq.Url = TempReq.Url.TrimEnd(new char[] { '/' }) + TreatedRawUrl;
return TempReq.FullUrl;
}
else if (RawLink.Length > 0)
{
Request TempReq = Req.GetClone();
Request NormaliserRequest = new Request(TempReq.FullUrl);
NormaliserRequest.Url = "/";
NormaliserRequest.UrlPathParts = UrlPathParts;
TempReq.Url = NormaliserRequest.Url;
if (TempReq.Url.EndsWith("/"))
{
TempReq.Url = TempReq.Url + RawLink;
}
else
{
TempReq.Url = TempReq.Url + "/" + RawLink;
}
return TempReq.FullURL;
}
}
return "";
}
bool IsA404(Request Req, Response Res)
{
Response NotFoundResponse;
if (NotFoundSignatures.ContainsKey(Req.SSL.ToString() + Req.Host + Req.File))
NotFoundResponse = NotFoundSignatures[Req.SSL.ToString() + Req.Host + Req.File];
else
{
Request NotFoundGetter = Req.GetClone();
NotFoundGetter.Method = "GET";
NotFoundGetter.Body.RemoveAll();
if(Req.File.Length > 0)
NotFoundGetter.Url = NotFoundGetter.UrlDir + "should_not_xist_" + Tools.GetRandomString(10,15) + "." + Req.File;
else
NotFoundGetter.Url = NotFoundGetter.UrlDir + "should_not_xist_" + Tools.GetRandomString(10, 15);
NotFoundResponse = NotFoundGetter.Send();
NotFoundResponse.Flags.Add("Url", NotFoundGetter.Url);
lock (NotFoundSignatures)
{
if (!NotFoundSignatures.ContainsKey(Req.SSL.ToString() + Req.Host + Req.File))
NotFoundSignatures.Add(Req.SSL.ToString() + Req.Host + Req.File, NotFoundResponse);
}
}
if(Res.Code == 200 && NotFoundResponse.Code != 200) return false;
if(Res.Code == 404) return true;
if(Res.Code == 403)
{
if(NotFoundResponse.Code == 403)
return true;
else
return false;
}
if (Res.Code > 400)
{
if (NotFoundResponse.Code == Res.Code)
return true;
else
return false;
}
string NotFoundGetterUrl = NotFoundResponse.Flags["Url"].ToString();
if (Res.Code == 301 || Res.Code == 302 || Res.Code == 303 || Res.Code == 307)
{
string RedirectedUrl = Res.Headers.Get("Location");
if (NotFoundResponse.Code == 301 || NotFoundResponse.Code == 302 || NotFoundResponse.Code == 303 || NotFoundResponse.Code == 307)
{
string NotFoundRedirectedUrl = NotFoundResponse.Headers.Get("Location");
if (RedirectedUrl.ToLower().Equals(NotFoundRedirectedUrl.ToLower()))
return true;
else if (Regex.IsMatch(RedirectedUrl, @".*not\Wfound.*", RegexOptions.IgnoreCase))
return true;
else if (NotFoundRedirectedUrl.Replace(NotFoundGetterUrl,"").Equals(RedirectedUrl.Replace(Req.Url, "")))
return true;
else
{
Request RedirectedLocationReq;
if (RedirectedUrl.StartsWith("http://") || RedirectedUrl.StartsWith("https://"))
{
RedirectedLocationReq = new Request(RedirectedUrl);
}
else if (RedirectedUrl.StartsWith("/"))
{
RedirectedLocationReq = Req.GetClone();
RedirectedLocationReq.Url = RedirectedUrl;
}
else
{
return true;
}
Request NotFoundRedirectedLocationReq;
if (NotFoundRedirectedUrl.StartsWith("http://") || NotFoundRedirectedUrl.StartsWith("https://"))
{
NotFoundRedirectedLocationReq = new Request(NotFoundRedirectedUrl);
}
else if (NotFoundRedirectedUrl.StartsWith("/"))
{
NotFoundRedirectedLocationReq = Req.GetClone();
NotFoundRedirectedLocationReq.Url = NotFoundRedirectedUrl;
}
else
{
return false;
}
if (RedirectedLocationReq.Url.Equals(NotFoundRedirectedLocationReq.Url)) return true;
}
}
else
return false;
}
return false;
}
public Trigger(string RequestTrigger, string RequestTriggerDescription, Request Req)
{
this.RequestTrigger = RequestTrigger;
this.RequestTriggerDescription = RequestTriggerDescription;
this.Request = Req.GetClone();
}
public virtual bool Is(Request Request)
{
try
{
Request Req = Request.GetClone();
string XmlString = ToXmlFromRequest(Req);
if (!Tools.IsXml(XmlString)) return false;
Request FinalReq = ToRequestFromXml(Req, XmlString);
if (Req.ToString().Equals(FinalReq.ToString())) return true;
}
catch { return false; }
return false;
}
public static List <Request> GetFormSubmissions(Request Req, Response Res, CookieStore Cookies)
{
List <Request> FormSubmissions = new List <Request>();
List <HtmlNode> FormNodes = Res.Html.GetForms();
foreach (HtmlNode FormNode in FormNodes)
{
Request SubReq = Req.GetClone();
SubReq.Method = "GET";
SubReq.BodyString = "";
foreach (HtmlAttribute Attr in FormNode.Attributes)
{
if (Attr.Name.Equals("method"))
{
SubReq.Method = Attr.Value.ToUpper();
}
else if (Attr.Name.Equals("action"))
{
if (Attr.Value.StartsWith("javascript:"))
{
continue;
}
string ActionUrl = NormalizeUrl(Req, Tools.HtmlDecode(Attr.Value.Trim()));
if (ActionUrl.Length > 0)
{
SubReq.FullUrl = ActionUrl;
}
}
}
if (SubReq.Method == "GET")
{
SubReq.Query.RemoveAll();
}
else
{
SubReq.Headers.Set("Content-Type", "application/x-www-form-urlencoded");
}
foreach (HtmlNode InputNode in FormNode.ChildNodes)
{
string Name = "";
string Value = "";
foreach (HtmlAttribute Attr in InputNode.Attributes)
{
switch (Attr.Name)
{
case ("name"):
Name = Attr.Value;
break;
case ("type"):
if (Attr.Value.Equals("submit"))
{
Name = "";
}
break;
case ("value"):
Value = Attr.Value;
break;
}
}
if (Value.Length == 0)
{
Value = Tools.GetRandomString(2, 5);
}
if (Name.Length > 0)
{
if (SubReq.Method.Equals("GET"))
{
SubReq.Query.Add(Name, Value);
}
else
{
SubReq.Body.Add(Name, Value);
}
}
}
SubReq.SetCookie(Cookies);
FormSubmissions.Add(SubReq);
}
return(FormSubmissions);
}
void Crawl(Request Req, int Depth, bool Scraped)
{
if (Stopped)
{
return;
}
if (Depth > MaxDepth)
{
return;
}
if (WasCrawled(Req))
{
return;
}
if (!CanCrawl(Req))
{
return;
}
lock (PageSignatures)
{
PageSignatures.Add(GetPageSignature(Req));
}
Req.Source = RequestSource.Probe;
Req.SetCookie(Cookies);
if (UserAgent.Length > 0)
{
Req.Headers.Set("User-Agent", UserAgent);
}
Response Res = Req.Send();
Cookies.Add(Req, Res);
bool Is404File = IsA404(Req, Res);
if (!Res.IsHtml)
{
return;
}
if (Depth + 1 > MaxDepth)
{
return;
}
List <Request> Redirects = GetRedirects(Req, Res);
foreach (Request Redirect in Redirects)
{
AddToCrawlQueue(Redirect, Depth + 1, true);
}
List <Request> LinkClicks = GetLinkClicks(Req, Res);
foreach (Request LinkClick in LinkClicks)
{
AddToCrawlQueue(LinkClick, Depth + 1, true);
}
List <Request> FormSubmissions = GetFormSubmissions(Req, Res);
foreach (Request FormSubmission in FormSubmissions)
{
AddToCrawlQueue(FormSubmission, Depth + 1, true);
}
Request DirCheck = Req.GetClone();
DirCheck.Method = "GET";
DirCheck.Body.RemoveAll();
DirCheck.Url = DirCheck.UrlDir;
if (!Req.Url.EndsWith("/"))
{
AddToCrawlQueue(DirCheck, Depth + 1, false);
}
if (PerformDirAndFileGuessing && !Is404File)
{
foreach (string File in FileNamesToCheck)
{
Request FileCheck = DirCheck.GetClone();
FileCheck.Url = FileCheck.Url + File;
AddToCrawlQueue(FileCheck, Depth + 1, false);
}
foreach (string Dir in DirNamesToCheck)
{
Request DirectoryCheck = DirCheck.GetClone();
DirectoryCheck.Url = DirectoryCheck.Url + Dir + "/";
AddToCrawlQueue(DirectoryCheck, Depth + 1, false);
}
}
if (Scraped || !Is404File)
{
lock (CrawledRequests)
{
CrawledRequests.Enqueue(Req);
}
IronUpdater.AddToSiteMap(Req);
}
}
public static List<Request> GetFormSubmissions(Request Req, Response Res, CookieStore Cookies)
{
List<Request> FormSubmissions = new List<Request>();
List<HtmlNode> FormNodes = Res.Html.GetForms();
foreach (HtmlNode FormNode in FormNodes)
{
Request SubReq = Req.GetClone();
SubReq.Method = "GET";
SubReq.BodyString = "";
foreach (HtmlAttribute Attr in FormNode.Attributes)
{
if (Attr.Name.Equals("method"))
{
SubReq.Method = Attr.Value.ToUpper();
}
else if(Attr.Name.Equals("action"))
{
if (Attr.Value.StartsWith("javascript:")) continue;
string ActionUrl = NormalizeUrl(Req, Tools.HtmlDecode(Attr.Value.Trim()));
if (ActionUrl.Length > 0)
{
SubReq.FullUrl = ActionUrl;
}
}
}
if (SubReq.Method == "GET")
{
SubReq.Query.RemoveAll();
}
else
{
SubReq.Headers.Set("Content-Type", "application/x-www-form-urlencoded");
}
foreach (HtmlNode InputNode in FormNode.ChildNodes)
{
string Name = "";
string Value = "";
foreach (HtmlAttribute Attr in InputNode.Attributes)
{
switch(Attr.Name)
{
case("name"):
Name = Attr.Value;
break;
case("type"):
if(Attr.Value.Equals("submit")) Name = "";
break;
case("value"):
Value = Attr.Value;
break;
}
}
if (Value.Length == 0)
{
Value = Tools.GetRandomString(2,5);
}
if (Name.Length > 0)
{
if (SubReq.Method.Equals("GET"))
SubReq.Query.Add(Name, Value);
else
SubReq.Body.Add(Name, Value);
}
}
SubReq.SetCookie(Cookies);
FormSubmissions.Add(SubReq);
}
return FormSubmissions;
}
internal static void AddShellRequest(Request Request)
{
if (Request != null)
{
try
{
Request ClonedRequest = Request.GetClone(true);
if (ClonedRequest != null)
{
lock (ShellRequestQ)
{
ShellRequestQ.Enqueue(ClonedRequest);
}
}
else
{
Tools.Trace("IronUpdater", "Null Shell Request");
}
}
catch (Exception Exp)
{
IronException.Report("Error adding Shell Request for updating", Exp.Message, Exp.StackTrace);
}
}
}
internal static string NormalizeUrl(Request Req, string RawLink)
{
if (RawLink.IndexOf('#') > -1)
{
RawLink = RawLink.Substring(0, RawLink.IndexOf('#'));
}
if (RawLink.StartsWith("http://") || RawLink.StartsWith("https://"))
{
return(RawLink);
}
else if (RawLink.StartsWith("//"))
{
if (Req.SSL)
{
RawLink = string.Format("https:{0}", RawLink);
}
else
{
RawLink = string.Format("http:{0}", RawLink);
}
}
else if (RawLink.StartsWith("/"))
{
Request TempReq = Req.GetClone();
TempReq.Url = RawLink;
return(TempReq.FullUrl);
}
else if (RawLink.StartsWith("javascript:") || RawLink.StartsWith("file:"))
{
//ignore
}
else
{
List <string> UrlPathParts = Req.UrlPathParts;
if (UrlPathParts.Count > 0)
{
if (!Req.Url.EndsWith("/"))
{
UrlPathParts.RemoveAt(UrlPathParts.Count - 1);
}
}
if (RawLink.StartsWith("../"))
{
string[] RawUrlParts = RawLink.Split(new char[] { '/' });
List <string> TreatedRawUrlParts = new List <string>(RawUrlParts);
foreach (string Part in RawUrlParts)
{
if (Part.Equals("..") && (UrlPathParts.Count > 0))
{
UrlPathParts.RemoveAt(UrlPathParts.Count - 1);
TreatedRawUrlParts.RemoveAt(0);
}
else
{
break;
}
}
UrlPathParts.AddRange(TreatedRawUrlParts);
StringBuilder FinalUrlBuilder = new StringBuilder();
foreach (string UrlPart in UrlPathParts)
{
FinalUrlBuilder.Append("/"); FinalUrlBuilder.Append(UrlPart);
}
Request TempReq = Req.GetClone();
TempReq.Url = FinalUrlBuilder.ToString();
return(TempReq.FullUrl);
}
else if (RawLink.Length > 0)
{
return(string.Format("{0}{1}{2}", Req.BaseUrl.TrimEnd('/'), Req.UrlDir, RawLink));
}
}
return("");
}
public static Request GetFormSubmission(Request Req, HtmlNode FormNode, CookieStore Cookies, bool LoginFormOnly, bool FillEmptyFields)
{
//Login request signatures:
//form must have one password type input field
//three or more parameters must be present in the request query/body
Request SubReq = Req.GetClone();
SubReq.Method = "GET";
SubReq.BodyString = "";
foreach (HtmlAttribute Attr in FormNode.Attributes)
{
if (Attr.Name.Equals("method"))
{
SubReq.Method = Attr.Value.ToUpper();
}
else if (Attr.Name.Equals("action"))
{
if (Attr.Value.StartsWith("javascript:"))
{
continue;
}
string ActionUrl = NormalizeUrl(Req, Tools.HtmlDecode(Attr.Value.Trim()));
if (ActionUrl.Length > 0)
{
SubReq.FullUrl = ActionUrl;
}
}
}
if (SubReq.Method == "GET")
{
SubReq.Query.RemoveAll();
}
else
{
SubReq.Headers.Set("Content-Type", "application/x-www-form-urlencoded");
}
bool PasswordFieldPresent = false;
foreach (HtmlNode InputNode in FormNode.ChildNodes)
{
string Name = "";
string Value = "";
foreach (HtmlAttribute Attr in InputNode.Attributes)
{
switch (Attr.Name)
{
case ("name"):
Name = Attr.Value;
break;
case ("type"):
if (Attr.Value.Equals("password", StringComparison.OrdinalIgnoreCase))
{
PasswordFieldPresent = true;
}
break;
case ("value"):
Value = Attr.Value;
break;
}
}
if (FillEmptyFields && Value.Length == 0)
{
Value = Tools.GetRandomString(2, 5);
}
if (Name.Length > 0)
{
if (SubReq.Method.Equals("GET"))
{
SubReq.Query.Add(Name, Value);
}
else
{
SubReq.Body.Add(Name, Value);
}
}
}
SubReq.SetCookie(Cookies);
if (LoginFormOnly)
{
if (PasswordFieldPresent)
{
if ((SubReq.Method.Equals("GET", StringComparison.OrdinalIgnoreCase) && SubReq.Query.Count >= 3) || (SubReq.Method.Equals("POST", StringComparison.OrdinalIgnoreCase) && SubReq.Body.Count >= 3))
{
return(SubReq);
}
}
}
else
{
return(SubReq);
}
return(null);
}
public void StoreRequest(Request Request)
{
ManualTesting.StoreRequest(Request.GetClone());
}
bool IsA404(Request Req, Response Res)
{
Response NotFoundResponse = null;
lock (NotFoundSignatures)
{
if (NotFoundSignatures.ContainsKey(string.Format("{0}{1}{2}{3}", Req.SSL, Req.Host, Req.UrlDir, Req.File)))
{
NotFoundResponse = NotFoundSignatures[string.Format("{0}{1}{2}{3}", Req.SSL, Req.Host, Req.UrlDir, Req.File)];
}
}
if(NotFoundResponse == null)
{
Request NotFoundGetter = Req.GetClone();
NotFoundGetter.Method = "GET";
NotFoundGetter.Body.RemoveAll();
if (Req.File.Length > 0)
NotFoundGetter.Url = string.Format("{0}should_not_xist_{1}.{2}", NotFoundGetter.UrlDir, Tools.GetRandomString(10, 15), Req.File);
else
NotFoundGetter.Url = string.Format("{0}should_not_xist_{1}", NotFoundGetter.UrlDir, Tools.GetRandomString(10, 15));
NotFoundResponse = NotFoundGetter.Send();
if (Stopped) return true;
NotFoundResponse.BodyString = "";
List<string> HeaderNames = NotFoundResponse.Headers.GetNames();
foreach (string HeaderName in HeaderNames)
{
if (!HeaderName.Equals("Location"))
{
NotFoundResponse.Headers.Remove(HeaderName);
}
}
NotFoundResponse.Flags.Add("Url", NotFoundGetter.Url);
lock (NotFoundSignatures)
{
if (!NotFoundSignatures.ContainsKey(string.Format("{0}{1}{2}{3}", Req.SSL, Req.Host, Req.UrlDir, Req.File)))
NotFoundSignatures.Add(string.Format("{0}{1}{2}{3}", Req.SSL, Req.Host, Req.UrlDir, Req.File), NotFoundResponse);
}
}
if(Res.Code == 200 && NotFoundResponse.Code != 200) return false;
if(Res.Code == 404) return true;
if (Res.Code > 400)
{
if (NotFoundResponse.Code == Res.Code)
return true;
else
return false;
}
string NotFoundGetterUrl = NotFoundResponse.Flags["Url"].ToString();
if (Res.Code == 301 || Res.Code == 302 || Res.Code == 303 || Res.Code == 307)
{
string RedirectedUrl = Res.Headers.Get("Location");
if (NotFoundResponse.Code == 301 || NotFoundResponse.Code == 302 || NotFoundResponse.Code == 303 || NotFoundResponse.Code == 307)
{
string NotFoundRedirectedUrl = NotFoundResponse.Headers.Get("Location");
if (RedirectedUrl.ToLower().Equals(NotFoundRedirectedUrl.ToLower()))
return true;
else if (Regex.IsMatch(RedirectedUrl, @".*not\Wfound.*", RegexOptions.IgnoreCase))
return true;
else if (NotFoundRedirectedUrl.Replace(NotFoundGetterUrl,"").Equals(RedirectedUrl.Replace(Req.Url, "")))
return true;
else
{
Request RedirectedLocationReq;
if (RedirectedUrl.StartsWith("http://") || RedirectedUrl.StartsWith("https://"))
{
RedirectedLocationReq = new Request(RedirectedUrl);
}
else if (RedirectedUrl.StartsWith("/"))
{
RedirectedLocationReq = Req.GetClone();
RedirectedLocationReq.Url = RedirectedUrl;
}
else
{
return true;
}
Request NotFoundRedirectedLocationReq;
if (NotFoundRedirectedUrl.StartsWith("http://") || NotFoundRedirectedUrl.StartsWith("https://"))
{
NotFoundRedirectedLocationReq = new Request(NotFoundRedirectedUrl);
}
else if (NotFoundRedirectedUrl.StartsWith("/"))
{
NotFoundRedirectedLocationReq = Req.GetClone();
NotFoundRedirectedLocationReq.Url = NotFoundRedirectedUrl;
}
else
{
return false;
}
if (RedirectedLocationReq.Url.Equals(NotFoundRedirectedLocationReq.Url)) return true;
}
}
else
return false;
}
return false;
}
Request GetTestRequest(Request Req)
{
Request TestRequest = Req.GetClone();
switch (TestType)
{
case(AddParameterTest):
return AddParameter(TestRequest);
case(EditParameterTest):
return EditParameter(TestRequest);
case(DeleteParameterTest):
return DeleteParameter(TestRequest);
}
return TestRequest;
}
public Request InjectInRequest(Request Req, int InjectionPoint, string Payload)
{
string CurrentRequestHash = Tools.MD5(Req.ToString());
string XML = "";
if(this.RequestHash.Equals(CurrentRequestHash))
{
XML = this.RequestXml;
}
else
{
XML = this.ToXmlFromRequest(Req);
this.RequestXml = XML;
this.RequestHash = CurrentRequestHash;
}
string InjectedXml = InjectInXml(this.RequestXml, InjectionPoint, Payload);
return this.ToRequestFromXml(Req.GetClone(true), InjectedXml);
}
public Trigger(string RequestTrigger, Request Req)
{
this.RequestTrigger = RequestTrigger;
this.Request = Req.GetClone();
}
bool IsA404(Request Req, Response Res)
{
Response NotFoundResponse = null;
lock (NotFoundSignatures)
{
if (NotFoundSignatures.ContainsKey(Req.SSL.ToString() + Req.Host + Req.UrlDir + Req.File))
{
NotFoundResponse = NotFoundSignatures[Req.SSL.ToString() + Req.Host + Req.UrlDir + Req.File];
}
}
if (NotFoundResponse == null)
{
Request NotFoundGetter = Req.GetClone();
NotFoundGetter.Method = "GET";
NotFoundGetter.Body.RemoveAll();
if (Req.File.Length > 0)
{
NotFoundGetter.Url = NotFoundGetter.UrlDir + "should_not_xist_" + Tools.GetRandomString(10, 15) + "." + Req.File;
}
else
{
NotFoundGetter.Url = NotFoundGetter.UrlDir + "should_not_xist_" + Tools.GetRandomString(10, 15);
}
NotFoundResponse = NotFoundGetter.Send();
NotFoundResponse.BodyString = "";
List <string> HeaderNames = NotFoundResponse.Headers.GetNames();
foreach (string HeaderName in HeaderNames)
{
if (!HeaderName.Equals("Location"))
{
NotFoundResponse.Headers.Remove(HeaderName);
}
}
NotFoundResponse.Flags.Add("Url", NotFoundGetter.Url);
lock (NotFoundSignatures)
{
if (!NotFoundSignatures.ContainsKey(Req.SSL.ToString() + Req.Host + Req.UrlDir + Req.File))
{
NotFoundSignatures.Add(Req.SSL.ToString() + Req.Host + Req.UrlDir + Req.File, NotFoundResponse);
}
}
}
if (Res.Code == 200 && NotFoundResponse.Code != 200)
{
return(false);
}
if (Res.Code == 404)
{
return(true);
}
if (Res.Code > 400)
{
if (NotFoundResponse.Code == Res.Code)
{
return(true);
}
else
{
return(false);
}
}
string NotFoundGetterUrl = NotFoundResponse.Flags["Url"].ToString();
if (Res.Code == 301 || Res.Code == 302 || Res.Code == 303 || Res.Code == 307)
{
string RedirectedUrl = Res.Headers.Get("Location");
if (NotFoundResponse.Code == 301 || NotFoundResponse.Code == 302 || NotFoundResponse.Code == 303 || NotFoundResponse.Code == 307)
{
string NotFoundRedirectedUrl = NotFoundResponse.Headers.Get("Location");
if (RedirectedUrl.ToLower().Equals(NotFoundRedirectedUrl.ToLower()))
{
return(true);
}
else if (Regex.IsMatch(RedirectedUrl, @".*not\Wfound.*", RegexOptions.IgnoreCase))
{
return(true);
}
else if (NotFoundRedirectedUrl.Replace(NotFoundGetterUrl, "").Equals(RedirectedUrl.Replace(Req.Url, "")))
{
return(true);
}
else
{
Request RedirectedLocationReq;
if (RedirectedUrl.StartsWith("http://") || RedirectedUrl.StartsWith("https://"))
{
RedirectedLocationReq = new Request(RedirectedUrl);
}
else if (RedirectedUrl.StartsWith("/"))
{
RedirectedLocationReq = Req.GetClone();
RedirectedLocationReq.Url = RedirectedUrl;
}
else
{
return(true);
}
Request NotFoundRedirectedLocationReq;
if (NotFoundRedirectedUrl.StartsWith("http://") || NotFoundRedirectedUrl.StartsWith("https://"))
{
NotFoundRedirectedLocationReq = new Request(NotFoundRedirectedUrl);
}
else if (NotFoundRedirectedUrl.StartsWith("/"))
{
NotFoundRedirectedLocationReq = Req.GetClone();
NotFoundRedirectedLocationReq.Url = NotFoundRedirectedUrl;
}
else
{
return(false);
}
if (RedirectedLocationReq.Url.Equals(NotFoundRedirectedLocationReq.Url))
{
return(true);
}
}
}
else
{
return(false);
}
}
return(false);
}
public Trigger(string RequestTrigger, string RequestTriggerDescription, Request Req)
{
this.RequestTrigger = RequestTrigger;
this.RequestTriggerDescription = RequestTriggerDescription;
this.Request = Req.GetClone();
}
static string NormalizeUrl(Request Req, string RawLink)
{
if (RawLink.IndexOf('#') > -1)
{
RawLink = RawLink.Substring(0, RawLink.IndexOf('#'));
}
if (RawLink.StartsWith("http://") || RawLink.StartsWith("https://"))
{
return(RawLink);
}
else if (RawLink.StartsWith("//"))
{
if (Req.SSL)
{
RawLink = "https:" + RawLink;
}
else
{
RawLink = "http:" + RawLink;
}
}
else if (RawLink.StartsWith("/"))
{
Request TempReq = Req.GetClone();
TempReq.Url = RawLink;
return(TempReq.FullUrl);
}
else if (RawLink.StartsWith("javascript:") || RawLink.StartsWith("file:"))
{
//ignore
}
else
{
List <string> UrlPathParts = Req.UrlPathParts;
if (UrlPathParts.Count > 0)
{
if (!Req.Url.EndsWith("/"))
{
UrlPathParts.RemoveAt(UrlPathParts.Count - 1);
}
}
if (RawLink.StartsWith("../"))
{
string[] RawUrlParts = RawLink.Split(new char[] { '/' });
List <string> TreatedRawUrlParts = new List <string>(RawUrlParts);
foreach (string Part in RawUrlParts)
{
if (Part.Equals("..") && (UrlPathParts.Count > 0))
{
UrlPathParts.RemoveAt(UrlPathParts.Count - 1);
TreatedRawUrlParts.RemoveAt(0);
}
else
{
break;
}
}
StringBuilder TreatedRawUrlBuilder = new StringBuilder("/");
foreach (string RawPart in TreatedRawUrlParts)
{
TreatedRawUrlBuilder.Append(RawPart);
}
string TreatedRawUrl = TreatedRawUrlBuilder.ToString();
if (!RawLink.EndsWith("/"))
{
TreatedRawUrl = TreatedRawUrl.TrimEnd(new char[] { '/' });
}
Request TempReq = Req.GetClone();
Request NormaliserRequest = new Request(TempReq.FullUrl);
NormaliserRequest.Url = "/";
NormaliserRequest.UrlPathParts = UrlPathParts;
TempReq.Url = NormaliserRequest.Url;
TempReq.Url = TempReq.Url.TrimEnd(new char[] { '/' }) + TreatedRawUrl;
return(TempReq.FullUrl);
}
else if (RawLink.Length > 0)
{
Request TempReq = Req.GetClone();
Request NormaliserRequest = new Request(TempReq.FullUrl);
NormaliserRequest.Url = "/";
NormaliserRequest.UrlPathParts = UrlPathParts;
TempReq.Url = NormaliserRequest.Url;
if (TempReq.Url.EndsWith("/"))
{
TempReq.Url = TempReq.Url + RawLink;
}
else
{
TempReq.Url = TempReq.Url + "/" + RawLink;
}
return(TempReq.FullURL);
}
}
return("");
}
public void StoreRequest(Request Request)
{
ManualTesting.StoreRequest(Request.GetClone());
}
public Trigger(string RequestTrigger, Request Req)
{
this.RequestTrigger = RequestTrigger;
this.Request = Req.GetClone();
}
void Crawl(Request Req, int Depth, bool Scraped)
{
if (Stopped) return;
if (Depth > MaxDepth) return;
if (WasCrawled(Req)) return;
if (!CanCrawl(Req)) return;
lock (PageSignatures)
{
PageSignatures.Add(GetPageSignature(Req));
}
Req.Source = RequestSource.Probe;
Req.SetCookie(Cookies);
if (UserAgent.Length > 0) Req.Headers.Set("User-Agent", UserAgent);
Response Res = Req.Send();
Cookies.Add(Req, Res);
bool Is404File = IsA404(Req, Res);
if (!Res.IsHtml)
{
try
{
Res.ProcessHtml();
}
catch
{
return;
}
}
if (Depth + 1 > MaxDepth) return;
List<Request> LinkClicks = GetLinkClicks(Req, Res);
foreach (Request LinkClick in LinkClicks)
{
AddToCrawlQueue(LinkClick, Depth + 1, true);
}
List<Request> FormSubmissions = GetFormSubmissionRequests(Req, Res);
foreach (Request FormSubmission in FormSubmissions)
{
AddToCrawlQueue(FormSubmission, Depth + 1, true);
}
Request DirCheck = Req.GetClone();
DirCheck.Method = "GET";
DirCheck.Body.RemoveAll();
DirCheck.Url = DirCheck.UrlDir;
if (!Req.Url.EndsWith("/"))
{
AddToCrawlQueue(DirCheck, Depth + 1, false);
}
if (PerformDirAndFileGuessing && !Is404File)
{
foreach (string File in FileNamesToCheck)
{
Request FileCheck = DirCheck.GetClone();
FileCheck.Url = FileCheck.Url + File;
AddToCrawlQueue(FileCheck, Depth + 1, false);
}
foreach (string Dir in DirNamesToCheck)
{
Request DirectoryCheck = DirCheck.GetClone();
DirectoryCheck.Url = DirectoryCheck.Url + Dir + "/";
AddToCrawlQueue(DirectoryCheck, Depth + 1, false);
}
}
if (Scraped || !Is404File)
{
lock (CrawledRequests)
{
CrawledRequests.Enqueue(Req);
}
IronUpdater.AddToSiteMap(Req);
}
}
public Scanner(Request Request)
{
this.OriginalRequest = Request.GetClone();
}
List<Request> GetFormSubmissionRequests(Request Req, Response Res)
{
List<Request> FormSubmissions = new List<Request>();
List<HtmlNode> FormNodes = Res.Html.GetForms();
foreach (HtmlNode FormNode in FormNodes)
{
Request SubReq = Req.GetClone();
SubReq.Method = "GET";
SubReq.Body.RemoveAll();
foreach (HtmlAttribute Attr in FormNode.Attributes)
{
if (Attr.Name.Equals("method"))
{
SubReq.Method = Attr.Value.ToUpper();
}
if (SubReq.Method.Equals("GET")) SubReq.Query.RemoveAll();
if (Attr.Name.Equals("action"))
{
if (Attr.Value.StartsWith("javascript:")) continue;
string ActionUrl = NormalizeUrl(Req, Attr.Value.Trim());
if (ActionUrl.Length > 0)
{
SubReq.FullUrl = ActionUrl;
}
}
}
foreach (HtmlNode InputNode in FormNode.ChildNodes)
{
string Name = "";
string Value = "";
foreach (HtmlAttribute Attr in InputNode.Attributes)
{
switch(Attr.Name)
{
case("name"):
Name = Attr.Value;
break;
case("type"):
if(Attr.Value.Equals("submit")) Name = "";
break;
case("value"):
Value = Attr.Value;
break;
}
}
if (Value.Length == 0)
{
Value = Tools.GetRandomString(2,5);
}
if (Name.Length > 0)
{
if (SubReq.Method.Equals("GET"))
SubReq.Query.Add(Name, Value);
else
SubReq.Body.Add(Name, Value);
}
}
FormSubmissions.Add(SubReq);
}
return FormSubmissions;
}
