protected virtual bool ShouldCrawlPageLinks(CrawledPage crawledPage)
{
CrawlDecision shouldCrawlPageLinksDecision = _crawlDecisionMaker.ShouldCrawlPageLinks(crawledPage, _crawlContext);
//if (shouldCrawlPageLinksDecision.Allow)
// shouldCrawlPageLinksDecision = (_shouldCrawlPageLinksDecisionMaker != null) ? _shouldCrawlPageLinksDecisionMaker.Invoke(crawledPage, _crawlContext) : new CrawlDecision { Allow = true };
//if (!shouldCrawlPageLinksDecision.Allow)
//{
// _logger.DebugFormat("Links on page [{0}] not crawled, [{1}]", crawledPage.Uri.AbsoluteUri, shouldCrawlPageLinksDecision.Reason);
// FirePageLinksCrawlDisallowedEventAsync(crawledPage, shouldCrawlPageLinksDecision.Reason);
// FirePageLinksCrawlDisallowedEvent(crawledPage, shouldCrawlPageLinksDecision.Reason);
//}
SignalCrawlStopIfNeeded(shouldCrawlPageLinksDecision);
return(shouldCrawlPageLinksDecision.Allow);
}
protected virtual bool ShouldCrawlPageLinks(CrawledPage crawledPage)
{
CrawlDecision shouldCrawlPageLinksDecision = _crawlDecisionMaker.ShouldCrawlPageLinks(crawledPage, _crawlContext);
if (shouldCrawlPageLinksDecision.Allow)
{
shouldCrawlPageLinksDecision = (_shouldCrawlPageLinksDecisionMaker != null) ? _shouldCrawlPageLinksDecisionMaker.Invoke(crawledPage, _crawlContext) : CrawlDecision.AllowCrawl();
}
if (!shouldCrawlPageLinksDecision.Allow)
{
_logger.LogDebug("Links on page [{0}] not crawled, [{1}]", crawledPage.Uri.AbsoluteUri, shouldCrawlPageLinksDecision.Reason);
FirePageLinksCrawlDisallowedEventAsync(crawledPage, shouldCrawlPageLinksDecision.Reason);
//FirePageLinksCrawlDisallowedEvent(crawledPage, shouldCrawlPageLinksDecision.Reason);
}
return(shouldCrawlPageLinksDecision.Allow);
}
protected virtual bool ShouldCrawlPageLinks(CrawledPage crawledPage)
{
var shouldCrawlPageLinksDecision = _crawlDecisionMaker.ShouldCrawlPageLinks(crawledPage, _crawlContext);
if (shouldCrawlPageLinksDecision.Allow)
{
shouldCrawlPageLinksDecision = (ShouldCrawlPageLinksDecisionMaker != null) ? ShouldCrawlPageLinksDecisionMaker.Invoke(crawledPage, _crawlContext) : new CrawlDecision {
Allow = true
}
}
;
if (!shouldCrawlPageLinksDecision.Allow)
{
Log.Debug("Links on page [{0}] not crawled, [{1}]", crawledPage.Uri.AbsoluteUri, shouldCrawlPageLinksDecision.Reason);
FirePageLinksCrawlDisallowedEvent(crawledPage, shouldCrawlPageLinksDecision.Reason);
}
SignalCrawlStopIfNeeded(shouldCrawlPageLinksDecision);
return(shouldCrawlPageLinksDecision.Allow);
}
