/// <summary>
/// Processes the hyper links.
/// </summary>
/// <param name = "crawlRequest">The crawl request.</param>
/// <param name = "arachnodeDAO">The arachnode DAO.</param>
public override void ProcessHyperLinks(CrawlRequest <TArachnodeDAO> crawlRequest, IArachnodeDAO arachnodeDAO)
{
if (ApplicationSettings.AssignHyperLinkDiscoveries)
{
_discoveryManager.AssignHyperLinkDiscoveries(crawlRequest, arachnodeDAO);
}
foreach (Discovery <TArachnodeDAO> hyperLinkDiscovery in crawlRequest.Discoveries.HyperLinks.Values)
{
if (!hyperLinkDiscovery.IsDisallowed)
{
if (hyperLinkDiscovery.DiscoveryState == DiscoveryState.Undiscovered)
{
if (crawlRequest.CurrentDepth < crawlRequest.MaximumDepth)
{
if (!_discoveryManager.IsCrawlRestricted(crawlRequest, hyperLinkDiscovery.Uri.AbsoluteUri))
{
_cache.AddCrawlRequestToBeCrawled(new CrawlRequest <TArachnodeDAO>(crawlRequest, hyperLinkDiscovery, crawlRequest.CurrentDepth + 1, crawlRequest.MaximumDepth, crawlRequest.RestrictCrawlTo, crawlRequest.RestrictDiscoveriesTo, crawlRequest.Priority + hyperLinkDiscovery.PriorityBoost, crawlRequest.RenderTypeForChildren, crawlRequest.RenderTypeForChildren), false, false, arachnodeDAO);
}
}
if (ApplicationSettings.InsertHyperLinks && hyperLinkDiscovery.IsStorable)
{
arachnodeDAO.InsertHyperLink(crawlRequest.Discovery.Uri.AbsoluteUri, hyperLinkDiscovery.Uri.AbsoluteUri, ApplicationSettings.ClassifyAbsoluteUris);
}
_consoleManager.OutputHyperLinkDiscovered(crawlRequest.Crawl.CrawlInfo.ThreadNumber, crawlRequest, hyperLinkDiscovery);
}
else
{
if (ApplicationSettings.InsertHyperLinkDiscoveries && hyperLinkDiscovery.IsStorable)
{
arachnodeDAO.InsertHyperLinkDiscovery(crawlRequest.Discovery.Uri.AbsoluteUri, hyperLinkDiscovery.Uri.AbsoluteUri);
}
_consoleManager.OutputCacheHit(crawlRequest.Crawl.CrawlInfo, crawlRequest, hyperLinkDiscovery);
}
}
else
{
if (ApplicationSettings.InsertDisallowedAbsoluteUris)
{
if (hyperLinkDiscovery.DiscoveryState == DiscoveryState.Undiscovered)
{
arachnodeDAO.InsertDisallowedAbsoluteUri(crawlRequest.DataType.ContentTypeID, (int)crawlRequest.DataType.DiscoveryType, crawlRequest.Discovery.Uri.AbsoluteUri, hyperLinkDiscovery.Uri.AbsoluteUri, hyperLinkDiscovery.IsDisallowedReason, ApplicationSettings.ClassifyAbsoluteUris);
}
else
{
if (ApplicationSettings.InsertDisallowedAbsoluteUriDiscoveries)
{
arachnodeDAO.InsertDisallowedAbsoluteUriDiscovery(crawlRequest.Discovery.Uri.AbsoluteUri, hyperLinkDiscovery.Uri.AbsoluteUri);
}
}
}
_consoleManager.OutputIsDisallowedReason(crawlRequest.Crawl.CrawlInfo, crawlRequest, hyperLinkDiscovery);
}
}
Counters.GetInstance().HyperLinksDiscovered(crawlRequest.Discoveries.HyperLinks.Count);
}