public List <string> GetPages(string url)
{
url = url.ToLower();
if (_urls.Contains(url))
{
return(_urls);
}
_urls.Add(url);
try
{
var html = _client.DownloadString(url);
var anchors = _document.GetAnchors(html);
foreach (var anchor in anchors)
{
var childUrl = anchor.GetHref();
if (_website.IsMailToUrl(childUrl))
{
continue;
}
if (_website.IsExternalUrl(childUrl))
{
continue;
}
if (!_website.IsHtmlUrl(childUrl))
{
continue;
}
if (_website.IsSiteRelativeUrl(childUrl))
{
childUrl = _website.GetAbsoluteFromSiteRelativeUrl(childUrl);
}
if (_website.IsPageRelativeUrl(childUrl))
{
childUrl = _website.GetAbsoluteFromPageRelativeUrl(url, childUrl);
}
GetPages(childUrl);
}
}
catch (WebException ex)
{
_log.LogFail(url, ex);
}
return(_urls);
}