public void AddCrawlRule(ACrawlRule <TArachnodeDAO> crawlRule, CrawlRuleType crawlRuleType, bool isEnabled, int order)
{
//TODO: AssemblyName should be Namespace...
crawlRule.AssemblyName = crawlRule.GetType().Namespace;
crawlRule.CrawlRuleType = crawlRuleType;
crawlRule.IsEnabled = isEnabled;
crawlRule.Order = order;
crawlRule.TypeName = crawlRule.GetType().FullName;
CrawlRules.Add(crawlRule.TypeName, crawlRule);
}
/// <summary>
/// Determines whether the specified crawl request is disallowed.
/// </summary>
/// <param name = "discovery">The discovery.</param>
/// <param name = "crawlRuleType">Type of the rule.</param>
/// <param name = "arachnodeDAO">The arachnode DAO.</param>
/// <returns>
/// <c>true</c> if the specified crawl request is disallowed; otherwise, <c>false</c>.
/// </returns>
public override bool IsDisallowed(Discovery <TArachnodeDAO> discovery, CrawlRuleType crawlRuleType, IArachnodeDAO arachnodeDAO)
{
switch (crawlRuleType)
{
case CrawlRuleType.PreRequest:
foreach (List <ACrawlRule <TArachnodeDAO> > crawlRules in _preRequestCrawlRules.Values)
{
foreach (ACrawlRule <TArachnodeDAO> crawlRule in crawlRules)
{
try
{
if (crawlRule.IsEnabled && crawlRule.IsDisallowed(discovery, arachnodeDAO))
{
discovery.IsDisallowed = true;
return(true);
}
}
catch (Exception exception)
{
arachnodeDAO.InsertException(discovery.Uri.AbsoluteUri, discovery.Uri.AbsoluteUri, exception, false);
return(true);
}
}
}
break;
case CrawlRuleType.PreGet:
foreach (List <ACrawlRule <TArachnodeDAO> > crawlRules in _preGetCrawlRules.Values)
{
foreach (ACrawlRule <TArachnodeDAO> crawlRule in crawlRules)
{
try
{
if (crawlRule.IsEnabled && crawlRule.IsDisallowed(discovery, arachnodeDAO))
{
discovery.IsDisallowed = true;
return(true);
}
}
catch (Exception exception)
{
arachnodeDAO.InsertException(discovery.Uri.AbsoluteUri, discovery.Uri.AbsoluteUri, exception, false);
return(true);
}
}
}
break;
case CrawlRuleType.PostRequest:
foreach (List <ACrawlRule <TArachnodeDAO> > crawlRules in _postRequestCrawlRules.Values)
{
foreach (ACrawlRule <TArachnodeDAO> crawlRule in crawlRules)
{
try
{
if (crawlRule.IsEnabled && crawlRule.IsDisallowed(discovery, arachnodeDAO))
{
discovery.IsDisallowed = true;
return(true);
}
}
catch (Exception exception)
{
arachnodeDAO.InsertException(discovery.Uri.AbsoluteUri, discovery.Uri.AbsoluteUri, exception, false);
return(true);
}
}
}
break;
}
return(false);
}
/// <summary> /// Determines whether the specified crawl request is disallowed. /// </summary> /// <param name = "discovery">The discovery.</param> /// <param name = "crawlRuleType">Type of the rule.</param> /// <param name = "arachnodeDAO">The arachnode DAO.</param> /// <returns> /// <c>true</c> if the specified crawl request is disallowed; otherwise, <c>false</c>. /// </returns> public abstract bool IsDisallowed(Discovery <TArachnodeDAO> discovery, CrawlRuleType crawlRuleType, IArachnodeDAO arachnodeDAO);
/// <summary> /// Determines whether the specified crawl request is disallowed. /// </summary> /// <param name = "crawlRequest">The crawl request.</param> /// <param name = "crawlRuleType">Type of the rule.</param> /// <param name = "arachnodeDAO">The arachnode DAO.</param> /// <returns> /// <c>true</c> if the specified crawl request is disallowed; otherwise, <c>false</c>. /// </returns> public abstract bool IsDisallowed(CrawlRequest <TArachnodeDAO> crawlRequest, CrawlRuleType crawlRuleType, IArachnodeDAO arachnodeDAO);
