public static void ScrapeUrls(Request Req, Response Res)
{
List <string> LocalScrapedUrls = new List <string>();
if (Res.IsHtml)
{
foreach (string Comment in Res.Html.Comments)
{
LocalScrapedUrls.AddRange(ScrapeUrls(Req, Comment));
}
foreach (string Script in Res.Html.GetJavaScript())
{
LocalScrapedUrls.AddRange(ScrapeUrls(Req, Script));
}
}
else if (Res.IsJson)
{
FormatParameters JsonParams = FormatPlugin.GetJsonParameters(Res);
for (int i = 0; i < JsonParams.Count; i++)
{
LocalScrapedUrls.AddRange(ScrapeUrls(Req, JsonParams.GetValue(i)));
}
}
else if (Res.IsJavaScript)
{
LocalScrapedUrls.AddRange(ScrapeUrls(Req, Res.BodyString));
}
if (Res.IsRedirect)
{
try
{
HTML ResHtml = new HTML(Res.BodyString);
foreach (string Link in ResHtml.Links)
{
string FullUrl = Req.RelativeUrlToAbsoluteUrl(Link);
if (!FullUrl.Equals(Req.FullUrl))
{
if (!LocalScrapedUrls.Contains(FullUrl))
{
LocalScrapedUrls.Add(FullUrl);
}
}
}
}
catch { }
}
lock (ScrapedUrls)
{
ScrapedUrls.AddRange(LocalScrapedUrls);
}
}