public void ShouldBeAbleToPrintVisualSiteMap() { var printer = new SiteMapPrinter(); PageNode root = new PageNode("/"); PageNode about = new PageNode("/about/"); root.AddChild(about); PageNode blog = new PageNode("/blog/"); root.AddChild(blog); PageNode blogOne = new PageNode("/blog/one/"); blog.AddChild(blogOne); PageNode blogOneB = new PageNode("/blog/one/b/"); blogOne.AddChild(blogOneB); var siteMap = printer.Format(root); var expectedSiteMap = new StringBuilder(); expectedSiteMap.AppendLine("/"); expectedSiteMap.AppendLine(" /about/"); expectedSiteMap.AppendLine(" /blog/"); expectedSiteMap.AppendLine(" /blog/one/"); expectedSiteMap.AppendLine(" /blog/one/b/"); siteMap.ShouldBe(expectedSiteMap.ToString()); }
public override Node VisitPage([NotNull] PageContext context) { var page = new PageNode(context.Start, context.LABEL().GetText()); foreach (SectionContext section in context.section()) { page.AddChild(Visit(section)); } return(page); }
private async Task ScrapeUrl(string rootDomain, PageNode currentNode, BlockingCollection <PageNode> urlsToProcessQueue, ConcurrentDictionary <string, bool> urlsVisited) { Console.WriteLine("Scraping: " + currentNode.Url); var childUrls = await _urlScraper.ScrapeUrls(rootDomain, currentNode.Url); foreach (var childUrl in childUrls) { if (!urlsVisited.ContainsKey(childUrl)) { var childNode = new PageNode(childUrl); currentNode.AddChild(childNode); if (urlsVisited.TryAdd(childUrl, true)) { urlsToProcessQueue.Add(childNode); } } } }