protected RobotsExclusionTools.RobotsFile GetRobotsFile(string name)
{
var robots = LoadResource("RobotsFile/" + name);
var robotsFile = new RobotsFileParser().FromString(robots, new Uri("http://www.example.org"));
return(robotsFile);
}
protected static RobotsExclusionTools.RobotsFile GetSampleRobotsFile(string domainName)
{
var robots = LoadResource($"RobotsFile/Samples/{domainName}.txt");
var robotsFile = new RobotsFileParser().FromString(robots, new Uri($"http://{domainName}"));
return(robotsFile);
}
