public async Task <bool> VerifyUrlIsBlocked(Uri uri)
{
var robots = await RobotsFetcher.GetFile(uri);
if (robots == null)
{
return(false);
}
return(robots.IsDisallowed(uri, FetchoConfiguration.Current.UserAgent));
}
public void R_UnusualRobotsRule()
{
const string txtfile = "User-agent: *\n\nDisallow: /news/0\n";
Uri uri = new Uri("https://www.example.com/news/world-asia-40360168");
Uri uri2 = new Uri("https://www.example.com/040360168");
var buffer = System.Text.Encoding.UTF8.GetBytes(txtfile);
var robots = new RobotsFile(RobotsFetcher.MakeRobotsUri(uri), buffer);
Console.WriteLine(robots.ToString());
Assert.IsTrue(!robots.IsDisallowed(uri));
Assert.IsTrue(!robots.IsDisallowed(uri2));
}
