예제 #1
0
        public void startCrawlingFtdTest()
        {
            Page page = new Page();

            page.urlStr = "http://www.ftd.de";
            CrawlerController crawlerController = new CrawlerController();
            ICrawlerJobView   crawlerJobView    = crawlerController.getCrawlerJobView();

            //
            int peekProcessInformationCount = 0;
            int updateEventCounter          = 0;
            int imageEventCounter           = 0;

            int    trackedProcessInformationUpdateCounter      = 0;
            int    trackedProcessInformationImageUpdateCounter = 0;
            double trackedProcessInformationPercentagePeek     = 0.0000;

            //
            CrawlerProcessInformation trackedProcessInformation = null;

            crawlerJobView.updateProcessInformationEvent += new CrawlerJobViewUpdateEventProcessInformation(delegate(CrawlerProcessInformation crawlerProcessInformation)
            {
                List <CrawlerProcessInformation> crawlerProcessInformationList = crawlerJobView.getActiveCrawlerProcessInformationList();
                peekProcessInformationCount = Math.Max(peekProcessInformationCount, crawlerProcessInformationList.Count);
                updateEventCounter++;

                //
                if (trackedProcessInformation == null && crawlerProcessInformation != null)
                {
                    trackedProcessInformation = crawlerProcessInformation;
                    trackedProcessInformation.crawlerProcessInformationUpdateEvent += new CrawlerProcessInformationUpdateEvent(delegate(CrawlerProcessInformation information)
                    {
                        trackedProcessInformationUpdateCounter++;
                        trackedProcessInformationPercentagePeek = Math.Max(trackedProcessInformationPercentagePeek, information.progressPercentage);
                    });
                    trackedProcessInformation.crawlerProcessInformationNewImageEvent += new CrawlerProcessInformationNewImageEvent(delegate(CrawlerImage image)
                    {
                        trackedProcessInformationImageUpdateCounter++;
                    });
                }
            });
            crawlerJobView.updateImageEvent += new CrawlerJobViewUpdateEventImage(delegate(CrawlerImage crawlerImage)
            {
                imageEventCounter++;
            });

            //
            crawlerController.startCrawling(page);
            crawlerController.getFinishedEvent().WaitOne(System.TimeSpan.FromSeconds(5), false);

            //
            Assert.IsTrue(peekProcessInformationCount > 0);
            Assert.IsTrue(updateEventCounter > 10);

            Assert.IsTrue(trackedProcessInformationUpdateCounter > 10);
            Assert.IsTrue(trackedProcessInformationPercentagePeek > 0.3);
        }
예제 #2
0
        public void startCrawlingTest()
        {
            //
            Page page = new Page();

            page.urlStr = "http://www.google.de";
            CrawlerController crawlerController = new CrawlerController();
            ICrawlerJobView   crawlerJobView    = crawlerController.getCrawlerJobView();

            //
            Assert.IsNotNull(crawlerJobView);

            //
            int peekProcessInformationCount = 0;
            int updateEventCounter          = 0;
            int imageEventCounter           = 0;

            int    trackedProcessInformationUpdateCounter      = 0;
            int    trackedProcessInformationImageUpdateCounter = 0;
            double trackedProcessInformationPercentagePeek     = 0.0000;
            //
            CrawlerProcessInformation trackedProcessInformation = null;

            crawlerJobView.updateProcessInformationEvent += new CrawlerJobViewUpdateEventProcessInformation(delegate(CrawlerProcessInformation crawlerProcessInformation)
            {
                List <CrawlerProcessInformation> crawlerProcessInformationList = crawlerJobView.getActiveCrawlerProcessInformationList();
                peekProcessInformationCount = Math.Max(peekProcessInformationCount, crawlerProcessInformationList.Count);
                updateEventCounter++;

                //
                if (trackedProcessInformation == null && crawlerProcessInformation != null)
                {
                    trackedProcessInformation = crawlerProcessInformation;
                    trackedProcessInformation.crawlerProcessInformationUpdateEvent += new CrawlerProcessInformationUpdateEvent(delegate(CrawlerProcessInformation information) {
                        trackedProcessInformationUpdateCounter++;
                        trackedProcessInformationPercentagePeek = Math.Max(trackedProcessInformationPercentagePeek, information.progressPercentage);
                    });
                    trackedProcessInformation.crawlerProcessInformationNewImageEvent += new CrawlerProcessInformationNewImageEvent(delegate(CrawlerImage image){
                        trackedProcessInformationImageUpdateCounter++;
                    });
                }
            });
            crawlerJobView.updateImageEvent += new CrawlerJobViewUpdateEventImage(delegate(CrawlerImage crawlerImage){
                imageEventCounter++;
            });

            //
            int crawlerJobCount     = 100;
            int crawlerJobPartCount = 10;

            crawlerController.setInitialJobFactory(new InitialJobFactoryMock(crawlerJobCount, crawlerJobPartCount));
            crawlerController.setMaximumCrawlerJobThreadCount(20);
            crawlerController.startCrawling(page);
            crawlerController.getFinishedEvent().WaitOne(Timeout.Infinite, false);

            //
            Assert.IsTrue(peekProcessInformationCount > 1);
            Assert.IsTrue(updateEventCounter > crawlerJobCount * 10);
            Assert.AreEqual(crawlerJobCount * crawlerJobPartCount, imageEventCounter);

            Assert.IsTrue(trackedProcessInformationUpdateCounter > 10);
            Assert.IsTrue(trackedProcessInformationPercentagePeek > 0.3);
            Assert.AreEqual(crawlerJobPartCount, trackedProcessInformationImageUpdateCounter);
        }