public static void ScrapeUrls(Request Req, Response Res) { List <string> LocalScrapedUrls = new List <string>(); if (Res.IsHtml) { foreach (string Comment in Res.Html.Comments) { LocalScrapedUrls.AddRange(ScrapeUrls(Req, Comment)); } foreach (string Script in Res.Html.GetJavaScript()) { LocalScrapedUrls.AddRange(ScrapeUrls(Req, Script)); } } else if (Res.IsJson) { FormatParameters JsonParams = FormatPlugin.GetJsonParameters(Res); for (int i = 0; i < JsonParams.Count; i++) { LocalScrapedUrls.AddRange(ScrapeUrls(Req, JsonParams.GetValue(i))); } } else if (Res.IsJavaScript) { LocalScrapedUrls.AddRange(ScrapeUrls(Req, Res.BodyString)); } if (Res.IsRedirect) { try { HTML ResHtml = new HTML(Res.BodyString); foreach (string Link in ResHtml.Links) { string FullUrl = Req.RelativeUrlToAbsoluteUrl(Link); if (!FullUrl.Equals(Req.FullUrl)) { if (!LocalScrapedUrls.Contains(FullUrl)) { LocalScrapedUrls.Add(FullUrl); } } } } catch { } } lock (ScrapedUrls) { ScrapedUrls.AddRange(LocalScrapedUrls); } }