/// <summary>
/// Whether the user agent is "allowed" to crawl the root url
/// NOTE: "userAgentString" NOT EQUAL with User Agent from Crawl Config
/// </summary>
/// <param name="userAgentString"></param>
/// <returns>True if find such user agent in content</returns>
public bool IsUserAgentAllowed(string userAgentString)
{
return(!string.IsNullOrEmpty(userAgentString) &&
Robots.Allowed(RootUri.AbsoluteUri, userAgentString));
}
/// <summary>
/// Whether the spider is "allowed" to crawl the param link
/// </summary>
public bool IsUrlAllowed(string url, string userAgentString)
{
return(!RootUri.IsBaseOf(new Uri(url)) ||
Robots.Allowed(url, userAgentString));
}