public void CrawlDelay_UserAgentCrawlDelayNotSpecified_ReturnsZero()
{
Assert.AreEqual(0, RobotsContent.GetCrawlDelay("userAgentCrawlDelayNotSpecified"));
}
public void CrawlDelay_UserAgentCrawlDelayEmpty_ReturnsZero()
{
Assert.AreEqual(0, RobotsContent.GetCrawlDelay("userAgentCrawlDelayEmpty"));
}
public void Explicit_relative_disallowed_file_Test()
{
bool actual = RobotsContent.Allowed("/disallowedfile.txt");
Assert.AreEqual(false, actual);
}
public void Explicit_allowed_folder_Test()
{
bool actual = RobotsContent.Allowed(BASE_URL + "/allowedfolder2");
Assert.AreEqual(true, actual);
}
public void Explicit_disallowed_complete_folder_Test_different_folder()
{
bool actual = RobotsContent.Allowed(BASE_URL + "/disallowedcompletefolderwithmoretext");
Assert.AreEqual(true, actual);
}
public void Explicit_disallowed_complete_folder_Test()
{
bool actual = RobotsContent.Allowed(BASE_URL + "/disallowedcompletefolder");
Assert.AreEqual(false, actual);
}
public void Implicit_allowed_file_Test()
{
bool actual = RobotsContent.Allowed(BASE_URL + "/allowedfile.xml");
Assert.AreEqual(true, actual);
}
