public async Task <bool> VerifyUrlIsBlocked(Uri uri) { var robots = await RobotsFetcher.GetFile(uri); if (robots == null) { return(false); } return(robots.IsDisallowed(uri, FetchoConfiguration.Current.UserAgent)); }
public void R_UnusualRobotsRule() { const string txtfile = "User-agent: *\n\nDisallow: /news/0\n"; Uri uri = new Uri("https://www.example.com/news/world-asia-40360168"); Uri uri2 = new Uri("https://www.example.com/040360168"); var buffer = System.Text.Encoding.UTF8.GetBytes(txtfile); var robots = new RobotsFile(RobotsFetcher.MakeRobotsUri(uri), buffer); Console.WriteLine(robots.ToString()); Assert.IsTrue(!robots.IsDisallowed(uri)); Assert.IsTrue(!robots.IsDisallowed(uri2)); }