/** Crawl Delay ***********************************************************/ public int GetCrawlDelay(string Url) { int Delay = 0; Robots robot = this.FetchRobot(Url: Url); if (robot != null) { long CrawlDelayTime = robot.CrawlDelay(this.UserAgent()); if (CrawlDelayTime == 0) { CrawlDelayTime = robot.CrawlDelay("*"); } if (CrawlDelayTime > 0) { Delay = ( int )(CrawlDelayTime / 1000); } DebugMsg(string.Format("ROBOTS CrawlDelayTime: {0}", CrawlDelayTime)); DebugMsg(string.Format("ROBOTS Delay: {0}", Delay)); } return(Delay); }
public void CrawlDelayRuleWithoutUserAgent() { string s = "Crawl-delay: 1"; Robots r = Robots.Load(s); Assert.AreNotEqual(1000, r.CrawlDelay("Google")); Assert.AreEqual(0, r.CrawlDelay("Google")); }
public void CrawlDelayInvalidRuleZero() { string s = @"User-agent: *" + this.newLine + "Crawl-delay: foo"; Robots r = new Robots(s); Assert.AreEqual(0, r.CrawlDelay("Google")); }
public void CrawlDelayNoRuleForRobotZero() { string s = @"User-agent: Slurp" + this.newLine + "Crawl-delay: 2"; Robots r = new Robots(s); Assert.AreEqual(0, r.CrawlDelay("Google")); }
public void CrawlDelayNoCrawlDelayRuleZero() { string s = @"User-agent: *" + this.newLine + "Disallow: /dir/"; Robots r = new Robots(s); Assert.AreEqual(0, r.CrawlDelay("*")); }
public void CrawlDelay_InvalidRule_Zero() { string s = @"User-agent: *" + nl + "Crawl-delay: foo"; Robots r = new Robots(s); Assert.Equal(0, r.CrawlDelay("Google")); }
public void CrawlDelay_NoRuleForRobot_Zero() { string s = @"User-agent: Slurp" + nl + "Crawl-delay: 2"; Robots r = new Robots(s); Assert.Equal(0, r.CrawlDelay("Google")); }
public void CrawlDelay_NoCrawlDelayRule_Zero() { string s = @"User-agent: *" + nl + "Disallow: /dir/"; Robots r = new Robots(s); Assert.Equal(0, r.CrawlDelay("*")); }
public void CrawlDelayEmptyUserAgentThrowsArgumentException( [Values("", " ")] string userAgent // white space considered empty ) { Robots r = new Robots(String.Empty); Assert.Throws <ArgumentException>(() => r.CrawlDelay(userAgent)); }
public void CrawlDelayValidRule( [Values(2000, 2000, 500, 500)] long expected, [Values("Google", "google", "Slurp", "slurp")] string userAgent) { string s = @"User-agent: Google" + this.newLine + "Crawl-delay: 2" + this.newLine + "User-agent: Slurp" + this.newLine + "Crawl-delay: 0.5"; Robots r = new Robots(s); Assert.AreEqual(expected, r.CrawlDelay(userAgent)); }
public void CrawlDelay_ValidRule( long expected, string userAgent) { string s = @"User-agent: Google" + nl + "Crawl-delay: 2" + nl + "User-agent: Slurp" + nl + "Crawl-delay: 0.5"; Robots r = new Robots(s); Assert.Equal(expected, r.CrawlDelay(userAgent)); }
public void CrawlDelayNoRulesZero() { Robots r = new Robots(String.Empty); Assert.AreEqual(0, r.CrawlDelay("*")); }