public void TestRobotsCache()
{
CollectorManager cm = new CollectorManager();
RobotsCache cache = new RobotsCache(cm, "http://www.yahoo.com");
Assert.IsTrue(cache.RobotsExclusion.Count > 0, "Robots not loaded");
}
protected virtual void LoadAndCheckRobotsRule()
{
RobotsCache cache = new RobotsCache(this.CM, this.DomainSchemeAndServer);
RobotsExclusion = cache.RobotsExclusion;
cache = null;
foreach (string robotExclusion in this.RobotsExclusion)
{
Uri robotUri = new Uri(new Uri(this.DomainSchemeAndServer), robotExclusion);
if (this.Link.StartsWith(robotUri.ToString(), StringComparison.InvariantCultureIgnoreCase))
{
LinkExcludedInRobots = true;
break;
}
}
}
