C# (CSharp) ICrawlDecisionMaker.ShouldDownloadPageContent Examples

Example #1

File: CrawlerLight.cs Project: jaskiier/WebSiteSpeedTest

        protected virtual CrawlDecision ShouldDownloadPageContent(CrawledPage crawledPage)
        {
            var decision = _crawlDecisionMaker.ShouldDownloadPageContent(crawledPage, _crawlContext);

            SignalCrawlStopIfNeeded(decision);
            return(decision);
        }

Example #2

File: WebCrawler.cs Project: RocChing/Crawler

        protected virtual CrawlDecision ShouldDownloadPageContent(CrawledPage crawledPage)
        {
            CrawlDecision decision = _crawlDecisionMaker.ShouldDownloadPageContent(crawledPage, _crawlContext);

            if (decision.Allow)
            {
                decision = (_shouldDownloadPageContentDecisionMaker != null) ? _shouldDownloadPageContentDecisionMaker.Invoke(crawledPage, _crawlContext) : CrawlDecision.AllowCrawl();
            }

            //SignalCrawlStopIfNeeded(decision);
            return(decision);
        }

Example #3

File: WebCrawler.cs Project: dagstuan/abot

        protected virtual CrawlDecision ShouldDownloadPageContentWrapper(CrawledPage crawledPage)
        {
            CrawlDecision decision = _crawlDecisionMaker.ShouldDownloadPageContent(crawledPage, _crawlContext);

            if (decision.Allow)
            {
                decision = (_shouldDownloadPageContentDecisionMaker != null) ? _shouldDownloadPageContentDecisionMaker.Invoke(crawledPage, _crawlContext) : new CrawlDecision {
                    Allow = true
                }
            }
            ;

            return(decision);
        }

Example #4

        protected virtual CrawlDecision ShouldDownloadPageContent(CrawledPage crawledPage)
        {
            var decision = _crawlDecisionMaker.ShouldDownloadPageContent(crawledPage, _crawlContext);

            if (decision.Allow)
            {
                decision = (ShouldDownloadPageContentDecisionMaker != null) ? ShouldDownloadPageContentDecisionMaker.Invoke(crawledPage, _crawlContext) : new CrawlDecision {
                    Allow = true
                }
            }
            ;

            SignalCrawlStopIfNeeded(decision);
            return(decision);
        }