Ejemplo n.º 1
0
        public void ShouldBeAbleToPrintVisualSiteMap()
        {
            var printer = new SiteMapPrinter();

            PageNode root  = new PageNode("/");
            PageNode about = new PageNode("/about/");

            root.AddChild(about);
            PageNode blog = new PageNode("/blog/");

            root.AddChild(blog);
            PageNode blogOne = new PageNode("/blog/one/");

            blog.AddChild(blogOne);
            PageNode blogOneB = new PageNode("/blog/one/b/");

            blogOne.AddChild(blogOneB);

            var siteMap = printer.Format(root);

            var expectedSiteMap = new StringBuilder();

            expectedSiteMap.AppendLine("/");
            expectedSiteMap.AppendLine("  /about/");
            expectedSiteMap.AppendLine("  /blog/");
            expectedSiteMap.AppendLine("    /blog/one/");
            expectedSiteMap.AppendLine("      /blog/one/b/");

            siteMap.ShouldBe(expectedSiteMap.ToString());
        }
Ejemplo n.º 2
0
        public override Node VisitPage([NotNull] PageContext context)
        {
            var page = new PageNode(context.Start, context.LABEL().GetText());

            foreach (SectionContext section in context.section())
            {
                page.AddChild(Visit(section));
            }

            return(page);
        }
Ejemplo n.º 3
0
        private async Task ScrapeUrl(string rootDomain, PageNode currentNode, BlockingCollection <PageNode> urlsToProcessQueue, ConcurrentDictionary <string, bool> urlsVisited)
        {
            Console.WriteLine("Scraping: " + currentNode.Url);
            var childUrls = await _urlScraper.ScrapeUrls(rootDomain, currentNode.Url);

            foreach (var childUrl in childUrls)
            {
                if (!urlsVisited.ContainsKey(childUrl))
                {
                    var childNode = new PageNode(childUrl);
                    currentNode.AddChild(childNode);
                    if (urlsVisited.TryAdd(childUrl, true))
                    {
                        urlsToProcessQueue.Add(childNode);
                    }
                }
            }
        }