public void AnyAllowedInBlankFile()
{
var robotsFile = new RobotsExclusionTools.RobotsFile(new Uri("http://www.example.org/"));
Assert.AreEqual(0, robotsFile.SiteAccessEntries.Count());
Assert.IsTrue(robotsFile.IsAllowedAccess(new Uri("http://www.example.org/"), "GoogleBot"));
Assert.IsTrue(robotsFile.IsAllowedAccess(new Uri("http://www.example.org/any"), "Webcrawler/1.0"));
Assert.IsTrue(robotsFile.IsAllowedAccess(new Uri("http://www.example.org/path"), "Unhipbot/1.0"));
Assert.IsTrue(robotsFile.IsAllowedAccess(new Uri("http://www.example.org/works"), "SuperSpecialCrawler/1.0"));
}
public static void AssertAccess(this RobotsExclusionTools.RobotsFile robotsFile, bool isAllowed, string userAgent, string url)
{
Assert.AreEqual(
isAllowed,
robotsFile.IsAllowedAccess(
new Uri(url),
userAgent
)
);
}
