private RobotsFile FromTokens(IEnumerable <Token> tokens, Uri baseUri)
{
var validationResult = PatternValidator.Validate(tokens);
if (validationResult.IsValid)
{
return(new RobotsFile(baseUri)
{
SiteAccessEntries = TokenParser.GetSiteAccessEntries(tokens),
SitemapEntries = TokenParser.GetSitemapUrlEntries(tokens)
});
}
return(RobotsFile.AllowAllRobots(baseUri));
}
