private void processUrl(Uri uri)
{
Models.PageAssets page;
try
{
_site_info.URIs.Add(uri);
page = _wbh.processPage(uri);
if (_settings.TakeScreenShots)
{
page.BitmapFilename = _wbh.takeScreenshot();
}
_site_info.Pages.Add(page);
// this next block should be refactored out into its own method.
foreach (string link in page.All_links)
{
string localurl = link;
if (link.Contains("#"))
{
localurl = link.Remove(link.IndexOf("#"));
}
localurl = Uri.EscapeUriString(localurl).Trim();
Uri localUri;
if (Uri.TryCreate(uri, localurl, out localUri))
{
if (ShouldFollowURL(localUri, uri) &&
(_settings.MaxDepthToSpider == 0 || _currentDepth <= _settings.MaxDepthToSpider))
{
processUrl(localUri);
_currentDepth++;
}
}
}
}
catch (Exception ex)
{
Exception uhoh = new Exception("Exception while processing " + uri.ToString() + "\n " + ex.Message, ex);
throw uhoh;
}
}
private Models.PageAssets ProcessUrl(string url)
{
try
{
Uri uri = new Uri(url);
Models.PageAssets page;
page = _wbh.processPage(uri);
_site_info.Pages.Add(page);
if (_settings.TakeScreenShots)
{
page.BitmapFilename = _wbh.takeScreenshot();
}
return(page);
}
catch
{
throw;
}
}