Beispiel #1
0
        public void startCrawlingFtdTest()
        {
            Page page = new Page();

            page.urlStr = "http://www.ftd.de";
            CrawlerController crawlerController = new CrawlerController();
            ICrawlerJobView   crawlerJobView    = crawlerController.getCrawlerJobView();

            //
            int peekProcessInformationCount = 0;
            int updateEventCounter          = 0;
            int imageEventCounter           = 0;

            int    trackedProcessInformationUpdateCounter      = 0;
            int    trackedProcessInformationImageUpdateCounter = 0;
            double trackedProcessInformationPercentagePeek     = 0.0000;

            //
            CrawlerProcessInformation trackedProcessInformation = null;

            crawlerJobView.updateProcessInformationEvent += new CrawlerJobViewUpdateEventProcessInformation(delegate(CrawlerProcessInformation crawlerProcessInformation)
            {
                List <CrawlerProcessInformation> crawlerProcessInformationList = crawlerJobView.getActiveCrawlerProcessInformationList();
                peekProcessInformationCount = Math.Max(peekProcessInformationCount, crawlerProcessInformationList.Count);
                updateEventCounter++;

                //
                if (trackedProcessInformation == null && crawlerProcessInformation != null)
                {
                    trackedProcessInformation = crawlerProcessInformation;
                    trackedProcessInformation.crawlerProcessInformationUpdateEvent += new CrawlerProcessInformationUpdateEvent(delegate(CrawlerProcessInformation information)
                    {
                        trackedProcessInformationUpdateCounter++;
                        trackedProcessInformationPercentagePeek = Math.Max(trackedProcessInformationPercentagePeek, information.progressPercentage);
                    });
                    trackedProcessInformation.crawlerProcessInformationNewImageEvent += new CrawlerProcessInformationNewImageEvent(delegate(CrawlerImage image)
                    {
                        trackedProcessInformationImageUpdateCounter++;
                    });
                }
            });
            crawlerJobView.updateImageEvent += new CrawlerJobViewUpdateEventImage(delegate(CrawlerImage crawlerImage)
            {
                imageEventCounter++;
            });

            //
            crawlerController.startCrawling(page);
            crawlerController.getFinishedEvent().WaitOne(System.TimeSpan.FromSeconds(5), false);

            //
            Assert.IsTrue(peekProcessInformationCount > 0);
            Assert.IsTrue(updateEventCounter > 10);

            Assert.IsTrue(trackedProcessInformationUpdateCounter > 10);
            Assert.IsTrue(trackedProcessInformationPercentagePeek > 0.3);
        }
Beispiel #2
0
                public IButtonStateAction clickPlay()
                {
                    //
                    if (this.form.crawlerController != null && this.form.crawlerController.getCrawlerJobView().isCrawlingProcessActive())
                    {
                        this.form.crawlerController.stopCrawling();
                    }

                    //
                    this.form.clearFlowLayoutPanelProcessStatus();
                    this.form.crawlerController = new CrawlerController();
                    ICrawlerJobView crawlerJobView = this.form.crawlerController.getCrawlerJobView();

                    this.form.crawlerController.setCrawlerImageFilter(this.form.generateCrawlerImageFilterFromFilterPanel());
                    this.form.synchronizeComboBoxThreadCountToCrawlerController();

                    //
                    this.form.imageListView.Clear();
                    this.form.imageList.Images.Clear();
                    lock (this.form.collectingCrawlerImageList)
                    {
                        this.form.collectingCrawlerImageList.Clear();
                        this.form.shownCrawlerImageList.Clear();
                    }

                    //
                    this.form.synchronizeImageLoggerFolder();

                    //
                    this.form.crawlerController.crawlerControllerFinishedWorkingEvent += new CrawlerControllerFinishedWorkingEvent(delegate()
                    {
                        try
                        {
                            if (this.form.crawlerButtonStateController.isStopped())
                            {
                                this.form.Invoke(new MethodInvoker(delegate()
                                {
                                    this.form.imageList.Images.Clear();
                                    this.form.imageListView.Clear();
                                }));
                            }

                            this.form.Invoke(new MethodInvoker(delegate()
                            {
                                this.form.crawlerButtonStateController.currentButtonStateAction = new ButtonStateActionStopped(this);
                            }));
                        }
                        catch (Exception e) { }
                    });

                    //
                    if (crawlerJobView != null)
                    {
                        crawlerJobView.crawlerJobViewUpdateEventAddedNewProcessInformation += new CrawlerJobViewUpdateEventAddedNewProcessInformation(delegate(CrawlerProcessInformation crawlerProcessInformation)
                        {
                            try
                            {
                                if (crawlerJobView.isCrawlingProcessActive())
                                {
                                    this.form.Invoke(new MethodInvoker(delegate()
                                    {
                                        Panel panel = new ProcessInformationPanel(crawlerProcessInformation);
                                        this.form.panelCrawlerStatus.Controls.Add(panel);
                                    }));
                                }
                            }
                            catch (Exception e) { }
                        });
                        crawlerJobView.updateProcessInformationEvent += new CrawlerJobViewUpdateEventProcessInformation(delegate(CrawlerProcessInformation crawlerProcessInformation)
                        {
                            try
                            {
                                if (crawlerJobView.isCrawlingProcessActive())
                                {
                                    //
                                    int undonePagesCount = crawlerJobView.getPageBacklogUndonePagesCount();
                                    int allPagesCount    = crawlerJobView.getPageBacklogAllPagesCount();
                                    int runningJobsCount = Math.Max(0, Math.Min(allPagesCount - undonePagesCount, crawlerJobView.getCurrentlyRunningWorkerJobCount()));
                                    int progressValue    = ((allPagesCount - (undonePagesCount + runningJobsCount)) * 100) / Math.Max(1, allPagesCount);
                                    String backlogStatus = "Remaining pages: " + (undonePagesCount + runningJobsCount) + "/" + allPagesCount;

                                    if (!this.form.labelBacklogStatus.Text.Equals(backlogStatus))
                                    {
                                        //
                                        this.form.Invoke(new MethodInvoker(delegate()
                                        {
                                            //
                                            this.form.labelBacklogStatus.Text      = backlogStatus;
                                            this.form.progressBarPageBacklog.Value = progressValue;
                                        }));
                                    }
                                }
                                else
                                {
                                    this.form.Invoke(new MethodInvoker(delegate()
                                    {
                                        this.form.resetPageBacklogStatus();
                                        this.form.clearFlowLayoutPanelProcessStatus();
                                    }));
                                }
                            }
                            catch (Exception e) { }
                        });
                        crawlerJobView.updateImageEvent += new CrawlerJobViewUpdateEventImage(this.form.retrieveImageEvent);
                    }

                    //
                    if (this.form.checkBoxIncludeExternalPages.Checked)
                    {
                        this.form.crawlerController.addPage(new Page(this.form.comboBoxUrl.Text));
                        this.form.crawlerController.startCrawling(null);
                    }
                    else
                    {
                        this.form.crawlerController.startCrawling(new Page(this.form.comboBoxUrl.Text));
                    }

                    //
                    return(new ButtonStateActionPlaying(this));
                }
Beispiel #3
0
        public void startCrawlingTest()
        {
            //
            Page page = new Page();

            page.urlStr = "http://www.google.de";
            CrawlerController crawlerController = new CrawlerController();
            ICrawlerJobView   crawlerJobView    = crawlerController.getCrawlerJobView();

            //
            Assert.IsNotNull(crawlerJobView);

            //
            int peekProcessInformationCount = 0;
            int updateEventCounter          = 0;
            int imageEventCounter           = 0;

            int    trackedProcessInformationUpdateCounter      = 0;
            int    trackedProcessInformationImageUpdateCounter = 0;
            double trackedProcessInformationPercentagePeek     = 0.0000;
            //
            CrawlerProcessInformation trackedProcessInformation = null;

            crawlerJobView.updateProcessInformationEvent += new CrawlerJobViewUpdateEventProcessInformation(delegate(CrawlerProcessInformation crawlerProcessInformation)
            {
                List <CrawlerProcessInformation> crawlerProcessInformationList = crawlerJobView.getActiveCrawlerProcessInformationList();
                peekProcessInformationCount = Math.Max(peekProcessInformationCount, crawlerProcessInformationList.Count);
                updateEventCounter++;

                //
                if (trackedProcessInformation == null && crawlerProcessInformation != null)
                {
                    trackedProcessInformation = crawlerProcessInformation;
                    trackedProcessInformation.crawlerProcessInformationUpdateEvent += new CrawlerProcessInformationUpdateEvent(delegate(CrawlerProcessInformation information) {
                        trackedProcessInformationUpdateCounter++;
                        trackedProcessInformationPercentagePeek = Math.Max(trackedProcessInformationPercentagePeek, information.progressPercentage);
                    });
                    trackedProcessInformation.crawlerProcessInformationNewImageEvent += new CrawlerProcessInformationNewImageEvent(delegate(CrawlerImage image){
                        trackedProcessInformationImageUpdateCounter++;
                    });
                }
            });
            crawlerJobView.updateImageEvent += new CrawlerJobViewUpdateEventImage(delegate(CrawlerImage crawlerImage){
                imageEventCounter++;
            });

            //
            int crawlerJobCount     = 100;
            int crawlerJobPartCount = 10;

            crawlerController.setInitialJobFactory(new InitialJobFactoryMock(crawlerJobCount, crawlerJobPartCount));
            crawlerController.setMaximumCrawlerJobThreadCount(20);
            crawlerController.startCrawling(page);
            crawlerController.getFinishedEvent().WaitOne(Timeout.Infinite, false);

            //
            Assert.IsTrue(peekProcessInformationCount > 1);
            Assert.IsTrue(updateEventCounter > crawlerJobCount * 10);
            Assert.AreEqual(crawlerJobCount * crawlerJobPartCount, imageEventCounter);

            Assert.IsTrue(trackedProcessInformationUpdateCounter > 10);
            Assert.IsTrue(trackedProcessInformationPercentagePeek > 0.3);
            Assert.AreEqual(crawlerJobPartCount, trackedProcessInformationImageUpdateCounter);
        }