protected virtual bool ShouldCrawlPageLinks(CrawledPage crawledPage) { CrawlDecision shouldCrawlPageLinksDecision = _crawlDecisionMaker.ShouldCrawlPageLinks(crawledPage, _crawlContext); //if (shouldCrawlPageLinksDecision.Allow) // shouldCrawlPageLinksDecision = (_shouldCrawlPageLinksDecisionMaker != null) ? _shouldCrawlPageLinksDecisionMaker.Invoke(crawledPage, _crawlContext) : new CrawlDecision { Allow = true }; //if (!shouldCrawlPageLinksDecision.Allow) //{ // _logger.DebugFormat("Links on page [{0}] not crawled, [{1}]", crawledPage.Uri.AbsoluteUri, shouldCrawlPageLinksDecision.Reason); // FirePageLinksCrawlDisallowedEventAsync(crawledPage, shouldCrawlPageLinksDecision.Reason); // FirePageLinksCrawlDisallowedEvent(crawledPage, shouldCrawlPageLinksDecision.Reason); //} SignalCrawlStopIfNeeded(shouldCrawlPageLinksDecision); return(shouldCrawlPageLinksDecision.Allow); }
protected virtual bool ShouldCrawlPageLinks(CrawledPage crawledPage) { CrawlDecision shouldCrawlPageLinksDecision = _crawlDecisionMaker.ShouldCrawlPageLinks(crawledPage, _crawlContext); if (shouldCrawlPageLinksDecision.Allow) { shouldCrawlPageLinksDecision = (_shouldCrawlPageLinksDecisionMaker != null) ? _shouldCrawlPageLinksDecisionMaker.Invoke(crawledPage, _crawlContext) : CrawlDecision.AllowCrawl(); } if (!shouldCrawlPageLinksDecision.Allow) { _logger.LogDebug("Links on page [{0}] not crawled, [{1}]", crawledPage.Uri.AbsoluteUri, shouldCrawlPageLinksDecision.Reason); FirePageLinksCrawlDisallowedEventAsync(crawledPage, shouldCrawlPageLinksDecision.Reason); //FirePageLinksCrawlDisallowedEvent(crawledPage, shouldCrawlPageLinksDecision.Reason); } return(shouldCrawlPageLinksDecision.Allow); }
protected virtual bool ShouldCrawlPageLinks(CrawledPage crawledPage) { var shouldCrawlPageLinksDecision = _crawlDecisionMaker.ShouldCrawlPageLinks(crawledPage, _crawlContext); if (shouldCrawlPageLinksDecision.Allow) { shouldCrawlPageLinksDecision = (ShouldCrawlPageLinksDecisionMaker != null) ? ShouldCrawlPageLinksDecisionMaker.Invoke(crawledPage, _crawlContext) : new CrawlDecision { Allow = true } } ; if (!shouldCrawlPageLinksDecision.Allow) { Log.Debug("Links on page [{0}] not crawled, [{1}]", crawledPage.Uri.AbsoluteUri, shouldCrawlPageLinksDecision.Reason); FirePageLinksCrawlDisallowedEvent(crawledPage, shouldCrawlPageLinksDecision.Reason); } SignalCrawlStopIfNeeded(shouldCrawlPageLinksDecision); return(shouldCrawlPageLinksDecision.Allow); }