/** Crawl Delay ***********************************************************/
public int GetCrawlDelay(string Url)
{
int Delay = 0;
Robots robot = this.FetchRobot(Url: Url);
if (robot != null)
{
long CrawlDelayTime = robot.CrawlDelay(this.UserAgent());
if (CrawlDelayTime == 0)
{
CrawlDelayTime = robot.CrawlDelay("*");
}
if (CrawlDelayTime > 0)
{
Delay = ( int )(CrawlDelayTime / 1000);
}
DebugMsg(string.Format("ROBOTS CrawlDelayTime: {0}", CrawlDelayTime));
DebugMsg(string.Format("ROBOTS Delay: {0}", Delay));
}
return(Delay);
}
public void CrawlDelayRuleWithoutUserAgent()
{
string s = "Crawl-delay: 1";
Robots r = Robots.Load(s);
Assert.AreNotEqual(1000, r.CrawlDelay("Google"));
Assert.AreEqual(0, r.CrawlDelay("Google"));
}
public void CrawlDelayInvalidRuleZero()
{
string s = @"User-agent: *" + this.newLine + "Crawl-delay: foo";
Robots r = new Robots(s);
Assert.AreEqual(0, r.CrawlDelay("Google"));
}
public void CrawlDelayNoRuleForRobotZero()
{
string s = @"User-agent: Slurp" + this.newLine + "Crawl-delay: 2";
Robots r = new Robots(s);
Assert.AreEqual(0, r.CrawlDelay("Google"));
}
public void CrawlDelayNoCrawlDelayRuleZero()
{
string s = @"User-agent: *" + this.newLine + "Disallow: /dir/";
Robots r = new Robots(s);
Assert.AreEqual(0, r.CrawlDelay("*"));
}
public void CrawlDelay_InvalidRule_Zero()
{
string s = @"User-agent: *" + nl + "Crawl-delay: foo";
Robots r = new Robots(s);
Assert.Equal(0, r.CrawlDelay("Google"));
}
public void CrawlDelay_NoRuleForRobot_Zero()
{
string s = @"User-agent: Slurp" + nl + "Crawl-delay: 2";
Robots r = new Robots(s);
Assert.Equal(0, r.CrawlDelay("Google"));
}
public void CrawlDelay_NoCrawlDelayRule_Zero()
{
string s = @"User-agent: *" + nl + "Disallow: /dir/";
Robots r = new Robots(s);
Assert.Equal(0, r.CrawlDelay("*"));
}
public void CrawlDelayEmptyUserAgentThrowsArgumentException(
[Values("", " ")] string userAgent // white space considered empty
)
{
Robots r = new Robots(String.Empty);
Assert.Throws <ArgumentException>(() => r.CrawlDelay(userAgent));
}
public void CrawlDelayValidRule(
[Values(2000, 2000, 500, 500)] long expected,
[Values("Google", "google", "Slurp", "slurp")] string userAgent)
{
string s = @"User-agent: Google" + this.newLine + "Crawl-delay: 2" + this.newLine +
"User-agent: Slurp" + this.newLine + "Crawl-delay: 0.5";
Robots r = new Robots(s);
Assert.AreEqual(expected, r.CrawlDelay(userAgent));
}
public void CrawlDelay_ValidRule(
long expected,
string userAgent)
{
string s = @"User-agent: Google" + nl + "Crawl-delay: 2" + nl +
"User-agent: Slurp" + nl + "Crawl-delay: 0.5";
Robots r = new Robots(s);
Assert.Equal(expected, r.CrawlDelay(userAgent));
}
public void CrawlDelayNoRulesZero()
{
Robots r = new Robots(String.Empty);
Assert.AreEqual(0, r.CrawlDelay("*"));
}
