public void ShouldBeAbleToPrintVisualSiteMap()
{
var printer = new SiteMapPrinter();
PageNode root = new PageNode("/");
PageNode about = new PageNode("/about/");
root.AddChild(about);
PageNode blog = new PageNode("/blog/");
root.AddChild(blog);
PageNode blogOne = new PageNode("/blog/one/");
blog.AddChild(blogOne);
PageNode blogOneB = new PageNode("/blog/one/b/");
blogOne.AddChild(blogOneB);
var siteMap = printer.Format(root);
var expectedSiteMap = new StringBuilder();
expectedSiteMap.AppendLine("/");
expectedSiteMap.AppendLine(" /about/");
expectedSiteMap.AppendLine(" /blog/");
expectedSiteMap.AppendLine(" /blog/one/");
expectedSiteMap.AppendLine(" /blog/one/b/");
siteMap.ShouldBe(expectedSiteMap.ToString());
}
public override Node VisitPage([NotNull] PageContext context)
{
var page = new PageNode(context.Start, context.LABEL().GetText());
foreach (SectionContext section in context.section())
{
page.AddChild(Visit(section));
}
return(page);
}
private async Task ScrapeUrl(string rootDomain, PageNode currentNode, BlockingCollection <PageNode> urlsToProcessQueue, ConcurrentDictionary <string, bool> urlsVisited)
{
Console.WriteLine("Scraping: " + currentNode.Url);
var childUrls = await _urlScraper.ScrapeUrls(rootDomain, currentNode.Url);
foreach (var childUrl in childUrls)
{
if (!urlsVisited.ContainsKey(childUrl))
{
var childNode = new PageNode(childUrl);
currentNode.AddChild(childNode);
if (urlsVisited.TryAdd(childUrl, true))
{
urlsToProcessQueue.Add(childNode);
}
}
}
}