Пример #1
0
        public void Start()
        {
            var validator      = new InputValidator();
            var websiteCrawler = new WebsiteCrawler();
            var sitemapCrawler = new SitemapCrawler();

            var differencePrinter = new LinksDifferencePrinter();
            var responcePrinter   = new ResponsePrinter();

            var performanceEvaluationGetter = new PerformanceEvaluationGetter();

            bool inputResult = false;
            Uri  WebsiteUrl  = null;;

            while (inputResult == false)
            {
                Console.WriteLine(@"Enter the website url (e.g. https://www.example.com/):");
                string websiteLink      = Console.ReadLine();
                var    validationResult = validator.Validate(websiteLink, new UrlValidator(), new RedirectionValidator());

                if (!validationResult)
                {
                    inputResult = false;
                }
                else
                {
                    WebsiteUrl  = new Uri(websiteLink);
                    inputResult = true;
                }
            }

            Console.WriteLine("Crawling website. It will take some time...");
            var websiteLinks = websiteCrawler.Crawl(WebsiteUrl, new PageDownloader(), new PageParser());

            Console.WriteLine("Crawling sitemap. It will take some time...");
            var sitemapLinks = sitemapCrawler.Crawl(WebsiteUrl, new SitemapLinkReceiver(), new PageDownloader(), new SitemapParser());

            differencePrinter.PrintDifference(sitemapLinks, websiteLinks);

            Console.WriteLine("Response time processing. It will take some time...");
            var combinedLinks = sitemapLinks.Union(websiteLinks).ToList();

            responcePrinter.PrintTable(performanceEvaluationGetter.PrepareLinks(combinedLinks, new PerformanceEvaluator()));

            Console.WriteLine("Enter to exit.");
            Console.ReadLine();
        }
Пример #2
0
        public void Crawl_ShouldSupportCommonSitemap()
        {
            //Arrange
            var expected = new List <Uri>()
            {
                new Uri("https://www.example1.com/"), new Uri("https://www.example2.com/")
            };

            sitemapParserMock.Setup(s => s.GetLinks(It.IsAny <string>(), It.IsAny <Uri>())).Returns(expected);
            var sitemapCrawler = new SitemapCrawler();

            //Act
            List <Uri> result = sitemapCrawler.Crawl(It.IsAny <Uri>(), linkReceiverMock.Object, pageDownloaderMock.Object, sitemapParserMock.Object);

            //Assert
            Assert.Equal(expected, result);
        }